Coverage for python/lsst/pipe/tasks/assembleCoadd.py : 68%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of pipe_tasks.
2#
3# LSST Data Management System
4# This product includes software developed by the
5# LSST Project (http://www.lsst.org/).
6# See COPYRIGHT file at the top of the source tree.
7#
8# This program is free software: you can redistribute it and/or modify
9# it under the terms of the GNU General Public License as published by
10# the Free Software Foundation, either version 3 of the License, or
11# (at your option) any later version.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the LSST License Statement and
19# the GNU General Public License along with this program. If not,
20# see <https://www.lsstcorp.org/LegalNotices/>.
21#
22import os
23import copy
24import numpy
25import warnings
26import lsst.pex.config as pexConfig
27import lsst.pex.exceptions as pexExceptions
28import lsst.geom as geom
29import lsst.afw.geom as afwGeom
30import lsst.afw.image as afwImage
31import lsst.afw.math as afwMath
32import lsst.afw.table as afwTable
33import lsst.afw.detection as afwDet
34import lsst.coadd.utils as coaddUtils
35import lsst.pipe.base as pipeBase
36import lsst.meas.algorithms as measAlg
37import lsst.log as log
38import lsstDebug
39import lsst.utils as utils
40from lsst.skymap import BaseSkyMap
41from .coaddBase import CoaddBaseTask, SelectDataIdContainer, makeSkyInfo, makeCoaddSuffix, reorderAndPadList
42from .interpImage import InterpImageTask
43from .scaleZeroPoint import ScaleZeroPointTask
44from .coaddHelpers import groupPatchExposures, getGroupDataRef
45from .scaleVariance import ScaleVarianceTask
46from .maskStreaks import MaskStreaksTask
47from .healSparseMapping import HealSparseInputMapTask
48from lsst.meas.algorithms import SourceDetectionTask
49from lsst.daf.butler import DeferredDatasetHandle
51__all__ = ["AssembleCoaddTask", "AssembleCoaddConnections", "AssembleCoaddConfig",
52 "SafeClipAssembleCoaddTask", "SafeClipAssembleCoaddConfig",
53 "CompareWarpAssembleCoaddTask", "CompareWarpAssembleCoaddConfig"]
56class AssembleCoaddConnections(pipeBase.PipelineTaskConnections,
57 dimensions=("tract", "patch", "band", "skymap"),
58 defaultTemplates={"inputCoaddName": "deep",
59 "outputCoaddName": "deep",
60 "warpType": "direct",
61 "warpTypeSuffix": ""}):
63 inputWarps = pipeBase.connectionTypes.Input(
64 doc=("Input list of warps to be assemebled i.e. stacked."
65 "WarpType (e.g. direct, psfMatched) is controlled by the warpType config parameter"),
66 name="{inputCoaddName}Coadd_{warpType}Warp",
67 storageClass="ExposureF",
68 dimensions=("tract", "patch", "skymap", "visit", "instrument"),
69 deferLoad=True,
70 multiple=True
71 )
72 skyMap = pipeBase.connectionTypes.Input(
73 doc="Input definition of geometry/bbox and projection/wcs for coadded exposures",
74 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
75 storageClass="SkyMap",
76 dimensions=("skymap", ),
77 )
78 selectedVisits = pipeBase.connectionTypes.Input(
79 doc="Selected visits to be coadded.",
80 name="{outputCoaddName}Visits",
81 storageClass="StructuredDataDict",
82 dimensions=("instrument", "tract", "patch", "skymap", "band")
83 )
84 brightObjectMask = pipeBase.connectionTypes.PrerequisiteInput(
85 doc=("Input Bright Object Mask mask produced with external catalogs to be applied to the mask plane"
86 " BRIGHT_OBJECT."),
87 name="brightObjectMask",
88 storageClass="ObjectMaskCatalog",
89 dimensions=("tract", "patch", "skymap", "band"),
90 )
91 coaddExposure = pipeBase.connectionTypes.Output(
92 doc="Output coadded exposure, produced by stacking input warps",
93 name="{outputCoaddName}Coadd{warpTypeSuffix}",
94 storageClass="ExposureF",
95 dimensions=("tract", "patch", "skymap", "band"),
96 )
97 nImage = pipeBase.connectionTypes.Output(
98 doc="Output image of number of input images per pixel",
99 name="{outputCoaddName}Coadd_nImage",
100 storageClass="ImageU",
101 dimensions=("tract", "patch", "skymap", "band"),
102 )
103 inputMap = pipeBase.connectionTypes.Output(
104 doc="Output healsparse map of input images",
105 name="{outputCoaddName}Coadd_inputMap",
106 storageClass="HealSparseMap",
107 dimensions=("tract", "patch", "skymap", "band"),
108 )
110 def __init__(self, *, config=None):
111 super().__init__(config=config)
113 # Override the connection's name template with config to replicate Gen2 behavior
114 # This duplicates some of the logic in the base class, due to wanting Gen2 and
115 # Gen3 configs to stay in sync. This should be removed when gen2 is deprecated
116 templateValues = {name: getattr(config.connections, name) for name in self.defaultTemplates}
117 templateValues['warpType'] = config.warpType
118 templateValues['warpTypeSuffix'] = makeCoaddSuffix(config.warpType)
119 self._nameOverrides = {name: getattr(config.connections, name).format(**templateValues)
120 for name in self.allConnections}
121 self._typeNameToVarName = {v: k for k, v in self._nameOverrides.items()}
122 # End code to remove after deprecation
124 if not config.doMaskBrightObjects:
125 self.prerequisiteInputs.remove("brightObjectMask")
127 if not config.doSelectVisits:
128 self.inputs.remove("selectedVisits")
130 if not config.doNImage:
131 self.outputs.remove("nImage")
133 if not self.config.doInputMap:
134 self.outputs.remove("inputMap")
137class AssembleCoaddConfig(CoaddBaseTask.ConfigClass, pipeBase.PipelineTaskConfig,
138 pipelineConnections=AssembleCoaddConnections):
139 """Configuration parameters for the `AssembleCoaddTask`.
141 Notes
142 -----
143 The `doMaskBrightObjects` and `brightObjectMaskName` configuration options
144 only set the bitplane config.brightObjectMaskName. To make this useful you
145 *must* also configure the flags.pixel algorithm, for example by adding
147 .. code-block:: none
149 config.measurement.plugins["base_PixelFlags"].masksFpCenter.append("BRIGHT_OBJECT")
150 config.measurement.plugins["base_PixelFlags"].masksFpAnywhere.append("BRIGHT_OBJECT")
152 to your measureCoaddSources.py and forcedPhotCoadd.py config overrides.
153 """
154 warpType = pexConfig.Field(
155 doc="Warp name: one of 'direct' or 'psfMatched'",
156 dtype=str,
157 default="direct",
158 )
159 subregionSize = pexConfig.ListField(
160 dtype=int,
161 doc="Width, height of stack subregion size; "
162 "make small enough that a full stack of images will fit into memory at once.",
163 length=2,
164 default=(2000, 2000),
165 )
166 statistic = pexConfig.Field(
167 dtype=str,
168 doc="Main stacking statistic for aggregating over the epochs.",
169 default="MEANCLIP",
170 )
171 doSigmaClip = pexConfig.Field(
172 dtype=bool,
173 doc="Perform sigma clipped outlier rejection with MEANCLIP statistic? (DEPRECATED)",
174 default=False,
175 )
176 sigmaClip = pexConfig.Field(
177 dtype=float,
178 doc="Sigma for outlier rejection; ignored if non-clipping statistic selected.",
179 default=3.0,
180 )
181 clipIter = pexConfig.Field(
182 dtype=int,
183 doc="Number of iterations of outlier rejection; ignored if non-clipping statistic selected.",
184 default=2,
185 )
186 calcErrorFromInputVariance = pexConfig.Field(
187 dtype=bool,
188 doc="Calculate coadd variance from input variance by stacking statistic."
189 "Passed to StatisticsControl.setCalcErrorFromInputVariance()",
190 default=True,
191 )
192 scaleZeroPoint = pexConfig.ConfigurableField(
193 target=ScaleZeroPointTask,
194 doc="Task to adjust the photometric zero point of the coadd temp exposures",
195 )
196 doInterp = pexConfig.Field(
197 doc="Interpolate over NaN pixels? Also extrapolate, if necessary, but the results are ugly.",
198 dtype=bool,
199 default=True,
200 )
201 interpImage = pexConfig.ConfigurableField(
202 target=InterpImageTask,
203 doc="Task to interpolate (and extrapolate) over NaN pixels",
204 )
205 doWrite = pexConfig.Field(
206 doc="Persist coadd?",
207 dtype=bool,
208 default=True,
209 )
210 doNImage = pexConfig.Field(
211 doc="Create image of number of contributing exposures for each pixel",
212 dtype=bool,
213 default=False,
214 )
215 doUsePsfMatchedPolygons = pexConfig.Field(
216 doc="Use ValidPolygons from shrunk Psf-Matched Calexps? Should be set to True by CompareWarp only.",
217 dtype=bool,
218 default=False,
219 )
220 maskPropagationThresholds = pexConfig.DictField(
221 keytype=str,
222 itemtype=float,
223 doc=("Threshold (in fractional weight) of rejection at which we propagate a mask plane to "
224 "the coadd; that is, we set the mask bit on the coadd if the fraction the rejected frames "
225 "would have contributed exceeds this value."),
226 default={"SAT": 0.1},
227 )
228 removeMaskPlanes = pexConfig.ListField(dtype=str, default=["NOT_DEBLENDED"],
229 doc="Mask planes to remove before coadding")
230 doMaskBrightObjects = pexConfig.Field(dtype=bool, default=False,
231 doc="Set mask and flag bits for bright objects?")
232 brightObjectMaskName = pexConfig.Field(dtype=str, default="BRIGHT_OBJECT",
233 doc="Name of mask bit used for bright objects")
234 coaddPsf = pexConfig.ConfigField(
235 doc="Configuration for CoaddPsf",
236 dtype=measAlg.CoaddPsfConfig,
237 )
238 doAttachTransmissionCurve = pexConfig.Field(
239 dtype=bool, default=False, optional=False,
240 doc=("Attach a piecewise TransmissionCurve for the coadd? "
241 "(requires all input Exposures to have TransmissionCurves).")
242 )
243 hasFakes = pexConfig.Field(
244 dtype=bool,
245 default=False,
246 doc="Should be set to True if fake sources have been inserted into the input data."
247 )
248 doSelectVisits = pexConfig.Field(
249 doc="Coadd only visits selected by a SelectVisitsTask",
250 dtype=bool,
251 default=False,
252 )
253 doInputMap = pexConfig.Field(
254 doc="Create a bitwise map of coadd inputs",
255 dtype=bool,
256 default=False,
257 )
258 inputMapper = pexConfig.ConfigurableField(
259 doc="Input map creation subtask.",
260 target=HealSparseInputMapTask,
261 )
263 def setDefaults(self):
264 super().setDefaults()
265 self.badMaskPlanes = ["NO_DATA", "BAD", "SAT", "EDGE"]
267 def validate(self):
268 super().validate()
269 if self.doPsfMatch:
270 # Backwards compatibility.
271 # Configs do not have loggers
272 log.warn("Config doPsfMatch deprecated. Setting warpType='psfMatched'")
273 self.warpType = 'psfMatched'
274 if self.doSigmaClip and self.statistic != "MEANCLIP":
275 log.warn('doSigmaClip deprecated. To replicate behavior, setting statistic to "MEANCLIP"')
276 self.statistic = "MEANCLIP"
277 if self.doInterp and self.statistic not in ['MEAN', 'MEDIAN', 'MEANCLIP', 'VARIANCE', 'VARIANCECLIP']:
278 raise ValueError("Must set doInterp=False for statistic=%s, which does not "
279 "compute and set a non-zero coadd variance estimate." % (self.statistic))
281 unstackableStats = ['NOTHING', 'ERROR', 'ORMASK']
282 if not hasattr(afwMath.Property, self.statistic) or self.statistic in unstackableStats:
283 stackableStats = [str(k) for k in afwMath.Property.__members__.keys()
284 if str(k) not in unstackableStats]
285 raise ValueError("statistic %s is not allowed. Please choose one of %s."
286 % (self.statistic, stackableStats))
289class AssembleCoaddTask(CoaddBaseTask, pipeBase.PipelineTask):
290 """Assemble a coadded image from a set of warps (coadded temporary exposures).
292 We want to assemble a coadded image from a set of Warps (also called
293 coadded temporary exposures or ``coaddTempExps``).
294 Each input Warp covers a patch on the sky and corresponds to a single
295 run/visit/exposure of the covered patch. We provide the task with a list
296 of Warps (``selectDataList``) from which it selects Warps that cover the
297 specified patch (pointed at by ``dataRef``).
298 Each Warp that goes into a coadd will typically have an independent
299 photometric zero-point. Therefore, we must scale each Warp to set it to
300 a common photometric zeropoint. WarpType may be one of 'direct' or
301 'psfMatched', and the boolean configs `config.makeDirect` and
302 `config.makePsfMatched` set which of the warp types will be coadded.
303 The coadd is computed as a mean with optional outlier rejection.
304 Criteria for outlier rejection are set in `AssembleCoaddConfig`.
305 Finally, Warps can have bad 'NaN' pixels which received no input from the
306 source calExps. We interpolate over these bad (NaN) pixels.
308 `AssembleCoaddTask` uses several sub-tasks. These are
310 - `ScaleZeroPointTask`
311 - create and use an ``imageScaler`` object to scale the photometric zeropoint for each Warp
312 - `InterpImageTask`
313 - interpolate across bad pixels (NaN) in the final coadd
315 You can retarget these subtasks if you wish.
317 Notes
318 -----
319 The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a
320 flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``; see
321 `baseDebug` for more about ``debug.py`` files. `AssembleCoaddTask` has
322 no debug variables of its own. Some of the subtasks may support debug
323 variables. See the documentation for the subtasks for further information.
325 Examples
326 --------
327 `AssembleCoaddTask` assembles a set of warped images into a coadded image.
328 The `AssembleCoaddTask` can be invoked by running ``assembleCoadd.py``
329 with the flag '--legacyCoadd'. Usage of assembleCoadd.py expects two
330 inputs: a data reference to the tract patch and filter to be coadded, and
331 a list of Warps to attempt to coadd. These are specified using ``--id`` and
332 ``--selectId``, respectively:
334 .. code-block:: none
336 --id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]
337 --selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]
339 Only the Warps that cover the specified tract and patch will be coadded.
340 A list of the available optional arguments can be obtained by calling
341 ``assembleCoadd.py`` with the ``--help`` command line argument:
343 .. code-block:: none
345 assembleCoadd.py --help
347 To demonstrate usage of the `AssembleCoaddTask` in the larger context of
348 multi-band processing, we will generate the HSC-I & -R band coadds from
349 HSC engineering test data provided in the ``ci_hsc`` package. To begin,
350 assuming that the lsst stack has been already set up, we must set up the
351 obs_subaru and ``ci_hsc`` packages. This defines the environment variable
352 ``$CI_HSC_DIR`` and points at the location of the package. The raw HSC
353 data live in the ``$CI_HSC_DIR/raw directory``. To begin assembling the
354 coadds, we must first
356 - processCcd
357 - process the individual ccds in $CI_HSC_RAW to produce calibrated exposures
358 - makeSkyMap
359 - create a skymap that covers the area of the sky present in the raw exposures
360 - makeCoaddTempExp
361 - warp the individual calibrated exposures to the tangent plane of the coadd
363 We can perform all of these steps by running
365 .. code-block:: none
367 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
369 This will produce warped exposures for each visit. To coadd the warped
370 data, we call assembleCoadd.py as follows:
372 .. code-block:: none
374 assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \
375 --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \
376 --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \
377 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \
378 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \
379 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \
380 --selectId visit=903988 ccd=24
382 that will process the HSC-I band data. The results are written in
383 ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``.
385 You may also choose to run:
387 .. code-block:: none
389 scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346
390 assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R \
391 --selectId visit=903334 ccd=16 --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 \
392 --selectId visit=903334 ccd=100 --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 \
393 --selectId visit=903338 ccd=18 --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 \
394 --selectId visit=903342 ccd=10 --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 \
395 --selectId visit=903344 ccd=5 --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 \
396 --selectId visit=903346 ccd=6 --selectId visit=903346 ccd=12
398 to generate the coadd for the HSC-R band if you are interested in
399 following multiBand Coadd processing as discussed in `pipeTasks_multiBand`
400 (but note that normally, one would use the `SafeClipAssembleCoaddTask`
401 rather than `AssembleCoaddTask` to make the coadd.
402 """
403 ConfigClass = AssembleCoaddConfig
404 _DefaultName = "assembleCoadd"
406 def __init__(self, *args, **kwargs):
407 # TODO: DM-17415 better way to handle previously allowed passed args e.g.`AssembleCoaddTask(config)`
408 if args: 408 ↛ 409line 408 didn't jump to line 409, because the condition on line 408 was never true
409 argNames = ["config", "name", "parentTask", "log"]
410 kwargs.update({k: v for k, v in zip(argNames, args)})
411 warnings.warn("AssembleCoadd received positional args, and casting them as kwargs: %s. "
412 "PipelineTask will not take positional args" % argNames, FutureWarning)
414 super().__init__(**kwargs)
415 self.makeSubtask("interpImage")
416 self.makeSubtask("scaleZeroPoint")
418 if self.config.doMaskBrightObjects: 418 ↛ 419line 418 didn't jump to line 419, because the condition on line 418 was never true
419 mask = afwImage.Mask()
420 try:
421 self.brightObjectBitmask = 1 << mask.addMaskPlane(self.config.brightObjectMaskName)
422 except pexExceptions.LsstCppException:
423 raise RuntimeError("Unable to define mask plane for bright objects; planes used are %s" %
424 mask.getMaskPlaneDict().keys())
425 del mask
427 if self.config.doInputMap: 427 ↛ 428line 427 didn't jump to line 428, because the condition on line 427 was never true
428 self.makeSubtask("inputMapper")
430 self.warpType = self.config.warpType
432 @utils.inheritDoc(pipeBase.PipelineTask)
433 def runQuantum(self, butlerQC, inputRefs, outputRefs):
434 # Docstring to be formatted with info from PipelineTask.runQuantum
435 """
436 Notes
437 -----
438 Assemble a coadd from a set of Warps.
440 PipelineTask (Gen3) entry point to Coadd a set of Warps.
441 Analogous to `runDataRef`, it prepares all the data products to be
442 passed to `run`, and processes the results before returning a struct
443 of results to be written out. AssembleCoadd cannot fit all Warps in memory.
444 Therefore, its inputs are accessed subregion by subregion
445 by the Gen3 `DeferredDatasetHandle` that is analagous to the Gen2
446 `lsst.daf.persistence.ButlerDataRef`. Any updates to this method should
447 correspond to an update in `runDataRef` while both entry points
448 are used.
449 """
450 inputData = butlerQC.get(inputRefs)
452 # Construct skyInfo expected by run
453 # Do not remove skyMap from inputData in case makeSupplementaryDataGen3 needs it
454 skyMap = inputData["skyMap"]
455 outputDataId = butlerQC.quantum.dataId
457 inputData['skyInfo'] = makeSkyInfo(skyMap,
458 tractId=outputDataId['tract'],
459 patchId=outputDataId['patch'])
461 if self.config.doSelectVisits:
462 warpRefList = self.filterWarps(inputData['inputWarps'], inputData['selectedVisits'])
463 else:
464 warpRefList = inputData['inputWarps']
466 # Perform same middle steps as `runDataRef` does
467 inputs = self.prepareInputs(warpRefList)
468 self.log.info("Found %d %s", len(inputs.tempExpRefList),
469 self.getTempExpDatasetName(self.warpType))
470 if len(inputs.tempExpRefList) == 0:
471 raise pipeBase.NoWorkFound("No coadd temporary exposures found")
473 supplementaryData = self.makeSupplementaryDataGen3(butlerQC, inputRefs, outputRefs)
474 retStruct = self.run(inputData['skyInfo'], inputs.tempExpRefList, inputs.imageScalerList,
475 inputs.weightList, supplementaryData=supplementaryData)
477 inputData.setdefault('brightObjectMask', None)
478 self.processResults(retStruct.coaddExposure, inputData['brightObjectMask'], outputDataId)
480 if self.config.doWrite:
481 butlerQC.put(retStruct, outputRefs)
482 return retStruct
484 @pipeBase.timeMethod
485 def runDataRef(self, dataRef, selectDataList=None, warpRefList=None):
486 """Assemble a coadd from a set of Warps.
488 Pipebase.CmdlineTask entry point to Coadd a set of Warps.
489 Compute weights to be applied to each Warp and
490 find scalings to match the photometric zeropoint to a reference Warp.
491 Assemble the Warps using `run`. Interpolate over NaNs and
492 optionally write the coadd to disk. Return the coadded exposure.
494 Parameters
495 ----------
496 dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
497 Data reference defining the patch for coaddition and the
498 reference Warp (if ``config.autoReference=False``).
499 Used to access the following data products:
500 - ``self.config.coaddName + "Coadd_skyMap"``
501 - ``self.config.coaddName + "Coadd_ + <warpType> + "Warp"`` (optionally)
502 - ``self.config.coaddName + "Coadd"``
503 selectDataList : `list`
504 List of data references to Calexps. Data to be coadded will be
505 selected from this list based on overlap with the patch defined
506 by dataRef, grouped by visit, and converted to a list of data
507 references to warps.
508 warpRefList : `list`
509 List of data references to Warps to be coadded.
510 Note: `warpRefList` is just the new name for `tempExpRefList`.
512 Returns
513 -------
514 retStruct : `lsst.pipe.base.Struct`
515 Result struct with components:
517 - ``coaddExposure``: coadded exposure (``Exposure``).
518 - ``nImage``: exposure count image (``Image``).
519 """
520 if selectDataList and warpRefList: 520 ↛ 521line 520 didn't jump to line 521, because the condition on line 520 was never true
521 raise RuntimeError("runDataRef received both a selectDataList and warpRefList, "
522 "and which to use is ambiguous. Please pass only one.")
524 skyInfo = self.getSkyInfo(dataRef)
525 if warpRefList is None:
526 calExpRefList = self.selectExposures(dataRef, skyInfo, selectDataList=selectDataList)
527 if len(calExpRefList) == 0: 527 ↛ 528line 527 didn't jump to line 528, because the condition on line 527 was never true
528 self.log.warn("No exposures to coadd")
529 return
530 self.log.info("Coadding %d exposures", len(calExpRefList))
532 warpRefList = self.getTempExpRefList(dataRef, calExpRefList)
534 inputData = self.prepareInputs(warpRefList)
535 self.log.info("Found %d %s", len(inputData.tempExpRefList),
536 self.getTempExpDatasetName(self.warpType))
537 if len(inputData.tempExpRefList) == 0: 537 ↛ 538line 537 didn't jump to line 538, because the condition on line 537 was never true
538 self.log.warn("No coadd temporary exposures found")
539 return
541 supplementaryData = self.makeSupplementaryData(dataRef, warpRefList=inputData.tempExpRefList)
543 retStruct = self.run(skyInfo, inputData.tempExpRefList, inputData.imageScalerList,
544 inputData.weightList, supplementaryData=supplementaryData)
546 brightObjects = self.readBrightObjectMasks(dataRef) if self.config.doMaskBrightObjects else None
547 self.processResults(retStruct.coaddExposure, brightObjectMasks=brightObjects, dataId=dataRef.dataId)
549 if self.config.doWrite:
550 if self.getCoaddDatasetName(self.warpType) == "deepCoadd" and self.config.hasFakes: 550 ↛ 551line 550 didn't jump to line 551, because the condition on line 550 was never true
551 coaddDatasetName = "fakes_" + self.getCoaddDatasetName(self.warpType)
552 else:
553 coaddDatasetName = self.getCoaddDatasetName(self.warpType)
554 self.log.info("Persisting %s" % coaddDatasetName)
555 dataRef.put(retStruct.coaddExposure, coaddDatasetName)
556 if self.config.doNImage and retStruct.nImage is not None: 556 ↛ 557line 556 didn't jump to line 557, because the condition on line 556 was never true
557 dataRef.put(retStruct.nImage, self.getCoaddDatasetName(self.warpType) + '_nImage')
559 return retStruct
561 def processResults(self, coaddExposure, brightObjectMasks=None, dataId=None):
562 """Interpolate over missing data and mask bright stars.
564 Parameters
565 ----------
566 coaddExposure : `lsst.afw.image.Exposure`
567 The coadded exposure to process.
568 dataRef : `lsst.daf.persistence.ButlerDataRef`
569 Butler data reference for supplementary data.
570 """
571 if self.config.doInterp: 571 ↛ 578line 571 didn't jump to line 578, because the condition on line 571 was never false
572 self.interpImage.run(coaddExposure.getMaskedImage(), planeName="NO_DATA")
573 # The variance must be positive; work around for DM-3201.
574 varArray = coaddExposure.variance.array
575 with numpy.errstate(invalid="ignore"):
576 varArray[:] = numpy.where(varArray > 0, varArray, numpy.inf)
578 if self.config.doMaskBrightObjects: 578 ↛ 579line 578 didn't jump to line 579, because the condition on line 578 was never true
579 self.setBrightObjectMasks(coaddExposure, brightObjectMasks, dataId)
581 def makeSupplementaryData(self, dataRef, selectDataList=None, warpRefList=None):
582 """Make additional inputs to run() specific to subclasses (Gen2)
584 Duplicates interface of `runDataRef` method
585 Available to be implemented by subclasses only if they need the
586 coadd dataRef for performing preliminary processing before
587 assembling the coadd.
589 Parameters
590 ----------
591 dataRef : `lsst.daf.persistence.ButlerDataRef`
592 Butler data reference for supplementary data.
593 selectDataList : `list` (optional)
594 Optional List of data references to Calexps.
595 warpRefList : `list` (optional)
596 Optional List of data references to Warps.
597 """
598 return pipeBase.Struct()
600 def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs):
601 """Make additional inputs to run() specific to subclasses (Gen3)
603 Duplicates interface of `runQuantum` method.
604 Available to be implemented by subclasses only if they need the
605 coadd dataRef for performing preliminary processing before
606 assembling the coadd.
608 Parameters
609 ----------
610 butlerQC : `lsst.pipe.base.ButlerQuantumContext`
611 Gen3 Butler object for fetching additional data products before
612 running the Task specialized for quantum being processed
613 inputRefs : `lsst.pipe.base.InputQuantizedConnection`
614 Attributes are the names of the connections describing input dataset types.
615 Values are DatasetRefs that task consumes for corresponding dataset type.
616 DataIds are guaranteed to match data objects in ``inputData``.
617 outputRefs : `lsst.pipe.base.OutputQuantizedConnection`
618 Attributes are the names of the connections describing output dataset types.
619 Values are DatasetRefs that task is to produce
620 for corresponding dataset type.
621 """
622 return pipeBase.Struct()
624 def getTempExpRefList(self, patchRef, calExpRefList):
625 """Generate list data references corresponding to warped exposures
626 that lie within the patch to be coadded.
628 Parameters
629 ----------
630 patchRef : `dataRef`
631 Data reference for patch.
632 calExpRefList : `list`
633 List of data references for input calexps.
635 Returns
636 -------
637 tempExpRefList : `list`
638 List of Warp/CoaddTempExp data references.
639 """
640 butler = patchRef.getButler()
641 groupData = groupPatchExposures(patchRef, calExpRefList, self.getCoaddDatasetName(self.warpType),
642 self.getTempExpDatasetName(self.warpType))
643 tempExpRefList = [getGroupDataRef(butler, self.getTempExpDatasetName(self.warpType),
644 g, groupData.keys) for
645 g in groupData.groups.keys()]
646 return tempExpRefList
648 def prepareInputs(self, refList):
649 """Prepare the input warps for coaddition by measuring the weight for
650 each warp and the scaling for the photometric zero point.
652 Each Warp has its own photometric zeropoint and background variance.
653 Before coadding these Warps together, compute a scale factor to
654 normalize the photometric zeropoint and compute the weight for each Warp.
656 Parameters
657 ----------
658 refList : `list`
659 List of data references to tempExp
661 Returns
662 -------
663 result : `lsst.pipe.base.Struct`
664 Result struct with components:
666 - ``tempExprefList``: `list` of data references to tempExp.
667 - ``weightList``: `list` of weightings.
668 - ``imageScalerList``: `list` of image scalers.
669 """
670 statsCtrl = afwMath.StatisticsControl()
671 statsCtrl.setNumSigmaClip(self.config.sigmaClip)
672 statsCtrl.setNumIter(self.config.clipIter)
673 statsCtrl.setAndMask(self.getBadPixelMask())
674 statsCtrl.setNanSafe(True)
675 # compute tempExpRefList: a list of tempExpRef that actually exist
676 # and weightList: a list of the weight of the associated coadd tempExp
677 # and imageScalerList: a list of scale factors for the associated coadd tempExp
678 tempExpRefList = []
679 weightList = []
680 imageScalerList = []
681 tempExpName = self.getTempExpDatasetName(self.warpType)
682 for tempExpRef in refList:
683 # Gen3's DeferredDatasetHandles are guaranteed to exist and
684 # therefore have no datasetExists() method
685 if not isinstance(tempExpRef, DeferredDatasetHandle): 685 ↛ 690line 685 didn't jump to line 690, because the condition on line 685 was never false
686 if not tempExpRef.datasetExists(tempExpName):
687 self.log.warn("Could not find %s %s; skipping it", tempExpName, tempExpRef.dataId)
688 continue
690 tempExp = tempExpRef.get(datasetType=tempExpName, immediate=True)
691 # Ignore any input warp that is empty of data
692 if numpy.isnan(tempExp.image.array).all(): 692 ↛ 693line 692 didn't jump to line 693, because the condition on line 692 was never true
693 continue
694 maskedImage = tempExp.getMaskedImage()
695 imageScaler = self.scaleZeroPoint.computeImageScaler(
696 exposure=tempExp,
697 dataRef=tempExpRef,
698 )
699 try:
700 imageScaler.scaleMaskedImage(maskedImage)
701 except Exception as e:
702 self.log.warn("Scaling failed for %s (skipping it): %s", tempExpRef.dataId, e)
703 continue
704 statObj = afwMath.makeStatistics(maskedImage.getVariance(), maskedImage.getMask(),
705 afwMath.MEANCLIP, statsCtrl)
706 meanVar, meanVarErr = statObj.getResult(afwMath.MEANCLIP)
707 weight = 1.0 / float(meanVar)
708 if not numpy.isfinite(weight): 708 ↛ 709line 708 didn't jump to line 709, because the condition on line 708 was never true
709 self.log.warn("Non-finite weight for %s: skipping", tempExpRef.dataId)
710 continue
711 self.log.info("Weight of %s %s = %0.3f", tempExpName, tempExpRef.dataId, weight)
713 del maskedImage
714 del tempExp
716 tempExpRefList.append(tempExpRef)
717 weightList.append(weight)
718 imageScalerList.append(imageScaler)
720 return pipeBase.Struct(tempExpRefList=tempExpRefList, weightList=weightList,
721 imageScalerList=imageScalerList)
723 def prepareStats(self, mask=None):
724 """Prepare the statistics for coadding images.
726 Parameters
727 ----------
728 mask : `int`, optional
729 Bit mask value to exclude from coaddition.
731 Returns
732 -------
733 stats : `lsst.pipe.base.Struct`
734 Statistics structure with the following fields:
736 - ``statsCtrl``: Statistics control object for coadd
737 (`lsst.afw.math.StatisticsControl`)
738 - ``statsFlags``: Statistic for coadd (`lsst.afw.math.Property`)
739 """
740 if mask is None:
741 mask = self.getBadPixelMask()
742 statsCtrl = afwMath.StatisticsControl()
743 statsCtrl.setNumSigmaClip(self.config.sigmaClip)
744 statsCtrl.setNumIter(self.config.clipIter)
745 statsCtrl.setAndMask(mask)
746 statsCtrl.setNanSafe(True)
747 statsCtrl.setWeighted(True)
748 statsCtrl.setCalcErrorFromInputVariance(self.config.calcErrorFromInputVariance)
749 for plane, threshold in self.config.maskPropagationThresholds.items():
750 bit = afwImage.Mask.getMaskPlane(plane)
751 statsCtrl.setMaskPropagationThreshold(bit, threshold)
752 statsFlags = afwMath.stringToStatisticsProperty(self.config.statistic)
753 return pipeBase.Struct(ctrl=statsCtrl, flags=statsFlags)
755 @pipeBase.timeMethod
756 def run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
757 altMaskList=None, mask=None, supplementaryData=None):
758 """Assemble a coadd from input warps
760 Assemble the coadd using the provided list of coaddTempExps. Since
761 the full coadd covers a patch (a large area), the assembly is
762 performed over small areas on the image at a time in order to
763 conserve memory usage. Iterate over subregions within the outer
764 bbox of the patch using `assembleSubregion` to stack the corresponding
765 subregions from the coaddTempExps with the statistic specified.
766 Set the edge bits the coadd mask based on the weight map.
768 Parameters
769 ----------
770 skyInfo : `lsst.pipe.base.Struct`
771 Struct with geometric information about the patch.
772 tempExpRefList : `list`
773 List of data references to Warps (previously called CoaddTempExps).
774 imageScalerList : `list`
775 List of image scalers.
776 weightList : `list`
777 List of weights
778 altMaskList : `list`, optional
779 List of alternate masks to use rather than those stored with
780 tempExp.
781 mask : `int`, optional
782 Bit mask value to exclude from coaddition.
783 supplementaryData : lsst.pipe.base.Struct, optional
784 Struct with additional data products needed to assemble coadd.
785 Only used by subclasses that implement `makeSupplementaryData`
786 and override `run`.
788 Returns
789 -------
790 result : `lsst.pipe.base.Struct`
791 Result struct with components:
793 - ``coaddExposure``: coadded exposure (``lsst.afw.image.Exposure``).
794 - ``nImage``: exposure count image (``lsst.afw.image.Image``), if requested.
795 - ``inputMap``: bit-wise map of inputs, if requested.
796 - ``warpRefList``: input list of refs to the warps (
797 ``lsst.daf.butler.DeferredDatasetHandle`` or
798 ``lsst.daf.persistence.ButlerDataRef``)
799 (unmodified)
800 - ``imageScalerList``: input list of image scalers (unmodified)
801 - ``weightList``: input list of weights (unmodified)
802 """
803 tempExpName = self.getTempExpDatasetName(self.warpType)
804 self.log.info("Assembling %s %s", len(tempExpRefList), tempExpName)
805 stats = self.prepareStats(mask=mask)
807 if altMaskList is None:
808 altMaskList = [None]*len(tempExpRefList)
810 coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
811 coaddExposure.setPhotoCalib(self.scaleZeroPoint.getPhotoCalib())
812 coaddExposure.getInfo().setCoaddInputs(self.inputRecorder.makeCoaddInputs())
813 self.assembleMetadata(coaddExposure, tempExpRefList, weightList)
814 coaddMaskedImage = coaddExposure.getMaskedImage()
815 subregionSizeArr = self.config.subregionSize
816 subregionSize = geom.Extent2I(subregionSizeArr[0], subregionSizeArr[1])
817 # if nImage is requested, create a zero one which can be passed to assembleSubregion
818 if self.config.doNImage: 818 ↛ 819line 818 didn't jump to line 819, because the condition on line 818 was never true
819 nImage = afwImage.ImageU(skyInfo.bbox)
820 else:
821 nImage = None
822 # If inputMap is requested, create the initial version that can be masked in
823 # assembleSubregion.
824 if self.config.doInputMap: 824 ↛ 825line 824 didn't jump to line 825, because the condition on line 824 was never true
825 self.inputMapper.build_ccd_input_map(skyInfo.bbox,
826 skyInfo.wcs,
827 coaddExposure.getInfo().getCoaddInputs().ccds)
829 for subBBox in self._subBBoxIter(skyInfo.bbox, subregionSize):
830 try:
831 self.assembleSubregion(coaddExposure, subBBox, tempExpRefList, imageScalerList,
832 weightList, altMaskList, stats.flags, stats.ctrl,
833 nImage=nImage)
834 except Exception as e:
835 self.log.fatal("Cannot compute coadd %s: %s", subBBox, e)
837 # If inputMap is requested, we must finalize the map after the accumulation.
838 if self.config.doInputMap: 838 ↛ 839line 838 didn't jump to line 839, because the condition on line 838 was never true
839 self.inputMapper.finalize_ccd_input_map_mask()
840 inputMap = self.inputMapper.ccd_input_map
841 else:
842 inputMap = None
844 self.setInexactPsf(coaddMaskedImage.getMask())
845 # Despite the name, the following doesn't really deal with "EDGE" pixels: it identifies
846 # pixels that didn't receive any unmasked inputs (as occurs around the edge of the field).
847 coaddUtils.setCoaddEdgeBits(coaddMaskedImage.getMask(), coaddMaskedImage.getVariance())
848 return pipeBase.Struct(coaddExposure=coaddExposure, nImage=nImage,
849 warpRefList=tempExpRefList, imageScalerList=imageScalerList,
850 weightList=weightList, inputMap=inputMap)
852 def assembleMetadata(self, coaddExposure, tempExpRefList, weightList):
853 """Set the metadata for the coadd.
855 This basic implementation sets the filter from the first input.
857 Parameters
858 ----------
859 coaddExposure : `lsst.afw.image.Exposure`
860 The target exposure for the coadd.
861 tempExpRefList : `list`
862 List of data references to tempExp.
863 weightList : `list`
864 List of weights.
865 """
866 assert len(tempExpRefList) == len(weightList), "Length mismatch"
867 tempExpName = self.getTempExpDatasetName(self.warpType)
868 # We load a single pixel of each coaddTempExp, because we just want to get at the metadata
869 # (and we need more than just the PropertySet that contains the header), which is not possible
870 # with the current butler (see #2777).
871 bbox = geom.Box2I(coaddExposure.getBBox().getMin(), geom.Extent2I(1, 1))
873 if isinstance(tempExpRefList[0], DeferredDatasetHandle): 873 ↛ 875line 873 didn't jump to line 875, because the condition on line 873 was never true
874 # Gen 3 API
875 tempExpList = [tempExpRef.get(parameters={'bbox': bbox}) for tempExpRef in tempExpRefList]
876 else:
877 # Gen 2 API. Delete this when Gen 2 retired
878 tempExpList = [tempExpRef.get(tempExpName + "_sub", bbox=bbox, immediate=True)
879 for tempExpRef in tempExpRefList]
880 numCcds = sum(len(tempExp.getInfo().getCoaddInputs().ccds) for tempExp in tempExpList)
882 # Set the coadd FilterLabel to the band of the first input exposure:
883 # Coadds are calibrated, so the physical label is now meaningless.
884 coaddExposure.setFilterLabel(afwImage.FilterLabel(tempExpList[0].getFilterLabel().bandLabel))
885 coaddInputs = coaddExposure.getInfo().getCoaddInputs()
886 coaddInputs.ccds.reserve(numCcds)
887 coaddInputs.visits.reserve(len(tempExpList))
889 for tempExp, weight in zip(tempExpList, weightList):
890 self.inputRecorder.addVisitToCoadd(coaddInputs, tempExp, weight)
892 if self.config.doUsePsfMatchedPolygons:
893 self.shrinkValidPolygons(coaddInputs)
895 coaddInputs.visits.sort()
896 if self.warpType == "psfMatched":
897 # The modelPsf BBox for a psfMatchedWarp/coaddTempExp was dynamically defined by
898 # ModelPsfMatchTask as the square box bounding its spatially-variable, pre-matched WarpedPsf.
899 # Likewise, set the PSF of a PSF-Matched Coadd to the modelPsf
900 # having the maximum width (sufficient because square)
901 modelPsfList = [tempExp.getPsf() for tempExp in tempExpList]
902 modelPsfWidthList = [modelPsf.computeBBox().getWidth() for modelPsf in modelPsfList]
903 psf = modelPsfList[modelPsfWidthList.index(max(modelPsfWidthList))]
904 else:
905 psf = measAlg.CoaddPsf(coaddInputs.ccds, coaddExposure.getWcs(),
906 self.config.coaddPsf.makeControl())
907 coaddExposure.setPsf(psf)
908 apCorrMap = measAlg.makeCoaddApCorrMap(coaddInputs.ccds, coaddExposure.getBBox(afwImage.PARENT),
909 coaddExposure.getWcs())
910 coaddExposure.getInfo().setApCorrMap(apCorrMap)
911 if self.config.doAttachTransmissionCurve:
912 transmissionCurve = measAlg.makeCoaddTransmissionCurve(coaddExposure.getWcs(), coaddInputs.ccds)
913 coaddExposure.getInfo().setTransmissionCurve(transmissionCurve)
915 def assembleSubregion(self, coaddExposure, bbox, tempExpRefList, imageScalerList, weightList,
916 altMaskList, statsFlags, statsCtrl, nImage=None):
917 """Assemble the coadd for a sub-region.
919 For each coaddTempExp, check for (and swap in) an alternative mask
920 if one is passed. Remove mask planes listed in
921 `config.removeMaskPlanes`. Finally, stack the actual exposures using
922 `lsst.afw.math.statisticsStack` with the statistic specified by
923 statsFlags. Typically, the statsFlag will be one of lsst.afw.math.MEAN for
924 a mean-stack or `lsst.afw.math.MEANCLIP` for outlier rejection using
925 an N-sigma clipped mean where N and iterations are specified by
926 statsCtrl. Assign the stacked subregion back to the coadd.
928 Parameters
929 ----------
930 coaddExposure : `lsst.afw.image.Exposure`
931 The target exposure for the coadd.
932 bbox : `lsst.geom.Box`
933 Sub-region to coadd.
934 tempExpRefList : `list`
935 List of data reference to tempExp.
936 imageScalerList : `list`
937 List of image scalers.
938 weightList : `list`
939 List of weights.
940 altMaskList : `list`
941 List of alternate masks to use rather than those stored with
942 tempExp, or None. Each element is dict with keys = mask plane
943 name to which to add the spans.
944 statsFlags : `lsst.afw.math.Property`
945 Property object for statistic for coadd.
946 statsCtrl : `lsst.afw.math.StatisticsControl`
947 Statistics control object for coadd.
948 nImage : `lsst.afw.image.ImageU`, optional
949 Keeps track of exposure count for each pixel.
950 """
951 self.log.debug("Computing coadd over %s", bbox)
952 tempExpName = self.getTempExpDatasetName(self.warpType)
953 coaddExposure.mask.addMaskPlane("REJECTED")
954 coaddExposure.mask.addMaskPlane("CLIPPED")
955 coaddExposure.mask.addMaskPlane("SENSOR_EDGE")
956 maskMap = self.setRejectedMaskMapping(statsCtrl)
957 clipped = afwImage.Mask.getPlaneBitMask("CLIPPED")
958 maskedImageList = []
959 if nImage is not None: 959 ↛ 960line 959 didn't jump to line 960, because the condition on line 959 was never true
960 subNImage = afwImage.ImageU(bbox.getWidth(), bbox.getHeight())
961 for tempExpRef, imageScaler, altMask in zip(tempExpRefList, imageScalerList, altMaskList):
963 if isinstance(tempExpRef, DeferredDatasetHandle): 963 ↛ 965line 963 didn't jump to line 965, because the condition on line 963 was never true
964 # Gen 3 API
965 exposure = tempExpRef.get(parameters={'bbox': bbox})
966 else:
967 # Gen 2 API. Delete this when Gen 2 retired
968 exposure = tempExpRef.get(tempExpName + "_sub", bbox=bbox)
970 maskedImage = exposure.getMaskedImage()
971 mask = maskedImage.getMask()
972 if altMask is not None:
973 self.applyAltMaskPlanes(mask, altMask)
974 imageScaler.scaleMaskedImage(maskedImage)
976 # Add 1 for each pixel which is not excluded by the exclude mask.
977 # In legacyCoadd, pixels may also be excluded by afwMath.statisticsStack.
978 if nImage is not None: 978 ↛ 979line 978 didn't jump to line 979, because the condition on line 978 was never true
979 subNImage.getArray()[maskedImage.getMask().getArray() & statsCtrl.getAndMask() == 0] += 1
980 if self.config.removeMaskPlanes: 980 ↛ 982line 980 didn't jump to line 982, because the condition on line 980 was never false
981 self.removeMaskPlanes(maskedImage)
982 maskedImageList.append(maskedImage)
984 if self.config.doInputMap: 984 ↛ 985line 984 didn't jump to line 985, because the condition on line 984 was never true
985 visit = exposure.getInfo().getCoaddInputs().visits[0].getId()
986 self.inputMapper.mask_warp_bbox(bbox, visit, mask, statsCtrl.getAndMask())
988 with self.timer("stack"):
989 coaddSubregion = afwMath.statisticsStack(maskedImageList, statsFlags, statsCtrl, weightList,
990 clipped, # also set output to CLIPPED if sigma-clipped
991 maskMap)
992 coaddExposure.maskedImage.assign(coaddSubregion, bbox)
993 if nImage is not None: 993 ↛ 994line 993 didn't jump to line 994, because the condition on line 993 was never true
994 nImage.assign(subNImage, bbox)
996 def removeMaskPlanes(self, maskedImage):
997 """Unset the mask of an image for mask planes specified in the config.
999 Parameters
1000 ----------
1001 maskedImage : `lsst.afw.image.MaskedImage`
1002 The masked image to be modified.
1003 """
1004 mask = maskedImage.getMask()
1005 for maskPlane in self.config.removeMaskPlanes:
1006 try:
1007 mask &= ~mask.getPlaneBitMask(maskPlane)
1008 except pexExceptions.InvalidParameterError:
1009 self.log.debug("Unable to remove mask plane %s: no mask plane with that name was found.",
1010 maskPlane)
1012 @staticmethod
1013 def setRejectedMaskMapping(statsCtrl):
1014 """Map certain mask planes of the warps to new planes for the coadd.
1016 If a pixel is rejected due to a mask value other than EDGE, NO_DATA,
1017 or CLIPPED, set it to REJECTED on the coadd.
1018 If a pixel is rejected due to EDGE, set the coadd pixel to SENSOR_EDGE.
1019 If a pixel is rejected due to CLIPPED, set the coadd pixel to CLIPPED.
1021 Parameters
1022 ----------
1023 statsCtrl : `lsst.afw.math.StatisticsControl`
1024 Statistics control object for coadd
1026 Returns
1027 -------
1028 maskMap : `list` of `tuple` of `int`
1029 A list of mappings of mask planes of the warped exposures to
1030 mask planes of the coadd.
1031 """
1032 edge = afwImage.Mask.getPlaneBitMask("EDGE")
1033 noData = afwImage.Mask.getPlaneBitMask("NO_DATA")
1034 clipped = afwImage.Mask.getPlaneBitMask("CLIPPED")
1035 toReject = statsCtrl.getAndMask() & (~noData) & (~edge) & (~clipped)
1036 maskMap = [(toReject, afwImage.Mask.getPlaneBitMask("REJECTED")),
1037 (edge, afwImage.Mask.getPlaneBitMask("SENSOR_EDGE")),
1038 (clipped, clipped)]
1039 return maskMap
1041 def applyAltMaskPlanes(self, mask, altMaskSpans):
1042 """Apply in place alt mask formatted as SpanSets to a mask.
1044 Parameters
1045 ----------
1046 mask : `lsst.afw.image.Mask`
1047 Original mask.
1048 altMaskSpans : `dict`
1049 SpanSet lists to apply. Each element contains the new mask
1050 plane name (e.g. "CLIPPED and/or "NO_DATA") as the key,
1051 and list of SpanSets to apply to the mask.
1053 Returns
1054 -------
1055 mask : `lsst.afw.image.Mask`
1056 Updated mask.
1057 """
1058 if self.config.doUsePsfMatchedPolygons:
1059 if ("NO_DATA" in altMaskSpans) and ("NO_DATA" in self.config.badMaskPlanes): 1059 ↛ 1067line 1059 didn't jump to line 1067, because the condition on line 1059 was never false
1060 # Clear away any other masks outside the validPolygons. These pixels are no longer
1061 # contributing to inexact PSFs, and will still be rejected because of NO_DATA
1062 # self.config.doUsePsfMatchedPolygons should be True only in CompareWarpAssemble
1063 # This mask-clearing step must only occur *before* applying the new masks below
1064 for spanSet in altMaskSpans['NO_DATA']:
1065 spanSet.clippedTo(mask.getBBox()).clearMask(mask, self.getBadPixelMask())
1067 for plane, spanSetList in altMaskSpans.items():
1068 maskClipValue = mask.addMaskPlane(plane)
1069 for spanSet in spanSetList:
1070 spanSet.clippedTo(mask.getBBox()).setMask(mask, 2**maskClipValue)
1071 return mask
1073 def shrinkValidPolygons(self, coaddInputs):
1074 """Shrink coaddInputs' ccds' ValidPolygons in place.
1076 Either modify each ccd's validPolygon in place, or if CoaddInputs
1077 does not have a validPolygon, create one from its bbox.
1079 Parameters
1080 ----------
1081 coaddInputs : `lsst.afw.image.coaddInputs`
1082 Original mask.
1084 """
1085 for ccd in coaddInputs.ccds:
1086 polyOrig = ccd.getValidPolygon()
1087 validPolyBBox = polyOrig.getBBox() if polyOrig else ccd.getBBox()
1088 validPolyBBox.grow(-self.config.matchingKernelSize//2)
1089 if polyOrig: 1089 ↛ 1090line 1089 didn't jump to line 1090, because the condition on line 1089 was never true
1090 validPolygon = polyOrig.intersectionSingle(validPolyBBox)
1091 else:
1092 validPolygon = afwGeom.polygon.Polygon(geom.Box2D(validPolyBBox))
1093 ccd.setValidPolygon(validPolygon)
1095 def readBrightObjectMasks(self, dataRef):
1096 """Retrieve the bright object masks.
1098 Returns None on failure.
1100 Parameters
1101 ----------
1102 dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
1103 A Butler dataRef.
1105 Returns
1106 -------
1107 result : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
1108 Bright object mask from the Butler object, or None if it cannot
1109 be retrieved.
1110 """
1111 try:
1112 return dataRef.get(datasetType="brightObjectMask", immediate=True)
1113 except Exception as e:
1114 self.log.warn("Unable to read brightObjectMask for %s: %s", dataRef.dataId, e)
1115 return None
1117 def setBrightObjectMasks(self, exposure, brightObjectMasks, dataId=None):
1118 """Set the bright object masks.
1120 Parameters
1121 ----------
1122 exposure : `lsst.afw.image.Exposure`
1123 Exposure under consideration.
1124 dataId : `lsst.daf.persistence.dataId`
1125 Data identifier dict for patch.
1126 brightObjectMasks : `lsst.afw.table`
1127 Table of bright objects to mask.
1128 """
1130 if brightObjectMasks is None:
1131 self.log.warn("Unable to apply bright object mask: none supplied")
1132 return
1133 self.log.info("Applying %d bright object masks to %s", len(brightObjectMasks), dataId)
1134 mask = exposure.getMaskedImage().getMask()
1135 wcs = exposure.getWcs()
1136 plateScale = wcs.getPixelScale().asArcseconds()
1138 for rec in brightObjectMasks:
1139 center = geom.PointI(wcs.skyToPixel(rec.getCoord()))
1140 if rec["type"] == "box":
1141 assert rec["angle"] == 0.0, ("Angle != 0 for mask object %s" % rec["id"])
1142 width = rec["width"].asArcseconds()/plateScale # convert to pixels
1143 height = rec["height"].asArcseconds()/plateScale # convert to pixels
1145 halfSize = geom.ExtentI(0.5*width, 0.5*height)
1146 bbox = geom.Box2I(center - halfSize, center + halfSize)
1148 bbox = geom.BoxI(geom.PointI(int(center[0] - 0.5*width), int(center[1] - 0.5*height)),
1149 geom.PointI(int(center[0] + 0.5*width), int(center[1] + 0.5*height)))
1150 spans = afwGeom.SpanSet(bbox)
1151 elif rec["type"] == "circle":
1152 radius = int(rec["radius"].asArcseconds()/plateScale) # convert to pixels
1153 spans = afwGeom.SpanSet.fromShape(radius, offset=center)
1154 else:
1155 self.log.warn("Unexpected region type %s at %s" % rec["type"], center)
1156 continue
1157 spans.clippedTo(mask.getBBox()).setMask(mask, self.brightObjectBitmask)
1159 def setInexactPsf(self, mask):
1160 """Set INEXACT_PSF mask plane.
1162 If any of the input images isn't represented in the coadd (due to
1163 clipped pixels or chip gaps), the `CoaddPsf` will be inexact. Flag
1164 these pixels.
1166 Parameters
1167 ----------
1168 mask : `lsst.afw.image.Mask`
1169 Coadded exposure's mask, modified in-place.
1170 """
1171 mask.addMaskPlane("INEXACT_PSF")
1172 inexactPsf = mask.getPlaneBitMask("INEXACT_PSF")
1173 sensorEdge = mask.getPlaneBitMask("SENSOR_EDGE") # chip edges (so PSF is discontinuous)
1174 clipped = mask.getPlaneBitMask("CLIPPED") # pixels clipped from coadd
1175 rejected = mask.getPlaneBitMask("REJECTED") # pixels rejected from coadd due to masks
1176 array = mask.getArray()
1177 selected = array & (sensorEdge | clipped | rejected) > 0
1178 array[selected] |= inexactPsf
1180 @classmethod
1181 def _makeArgumentParser(cls):
1182 """Create an argument parser.
1183 """
1184 parser = pipeBase.ArgumentParser(name=cls._DefaultName)
1185 parser.add_id_argument("--id", cls.ConfigClass().coaddName + "Coadd_"
1186 + cls.ConfigClass().warpType + "Warp",
1187 help="data ID, e.g. --id tract=12345 patch=1,2",
1188 ContainerClass=AssembleCoaddDataIdContainer)
1189 parser.add_id_argument("--selectId", "calexp", help="data ID, e.g. --selectId visit=6789 ccd=0..9",
1190 ContainerClass=SelectDataIdContainer)
1191 return parser
1193 @staticmethod
1194 def _subBBoxIter(bbox, subregionSize):
1195 """Iterate over subregions of a bbox.
1197 Parameters
1198 ----------
1199 bbox : `lsst.geom.Box2I`
1200 Bounding box over which to iterate.
1201 subregionSize: `lsst.geom.Extent2I`
1202 Size of sub-bboxes.
1204 Yields
1205 ------
1206 subBBox : `lsst.geom.Box2I`
1207 Next sub-bounding box of size ``subregionSize`` or smaller; each ``subBBox``
1208 is contained within ``bbox``, so it may be smaller than ``subregionSize`` at
1209 the edges of ``bbox``, but it will never be empty.
1210 """
1211 if bbox.isEmpty(): 1211 ↛ 1212line 1211 didn't jump to line 1212, because the condition on line 1211 was never true
1212 raise RuntimeError("bbox %s is empty" % (bbox,))
1213 if subregionSize[0] < 1 or subregionSize[1] < 1: 1213 ↛ 1214line 1213 didn't jump to line 1214, because the condition on line 1213 was never true
1214 raise RuntimeError("subregionSize %s must be nonzero" % (subregionSize,))
1216 for rowShift in range(0, bbox.getHeight(), subregionSize[1]):
1217 for colShift in range(0, bbox.getWidth(), subregionSize[0]):
1218 subBBox = geom.Box2I(bbox.getMin() + geom.Extent2I(colShift, rowShift), subregionSize)
1219 subBBox.clip(bbox)
1220 if subBBox.isEmpty(): 1220 ↛ 1221line 1220 didn't jump to line 1221, because the condition on line 1220 was never true
1221 raise RuntimeError("Bug: empty bbox! bbox=%s, subregionSize=%s, "
1222 "colShift=%s, rowShift=%s" %
1223 (bbox, subregionSize, colShift, rowShift))
1224 yield subBBox
1226 def filterWarps(self, inputs, goodVisits):
1227 """Return list of only inputRefs with visitId in goodVisits ordered by goodVisit
1229 Parameters
1230 ----------
1231 inputs : list
1232 List of `lsst.pipe.base.connections.DeferredDatasetRef` with dataId containing visit
1233 goodVisit : `dict`
1234 Dictionary with good visitIds as the keys. Value ignored.
1236 Returns:
1237 --------
1238 filteredInputs : `list`
1239 Filtered and sorted list of `lsst.pipe.base.connections.DeferredDatasetRef`
1240 """
1241 inputWarpDict = {inputRef.ref.dataId['visit']: inputRef for inputRef in inputs}
1242 filteredInputs = []
1243 for visit in goodVisits.keys():
1244 filteredInputs.append(inputWarpDict[visit])
1245 return filteredInputs
1248class AssembleCoaddDataIdContainer(pipeBase.DataIdContainer):
1249 """A version of `lsst.pipe.base.DataIdContainer` specialized for assembleCoadd.
1250 """
1252 def makeDataRefList(self, namespace):
1253 """Make self.refList from self.idList.
1255 Parameters
1256 ----------
1257 namespace
1258 Results of parsing command-line (with ``butler`` and ``log`` elements).
1259 """
1260 datasetType = namespace.config.coaddName + "Coadd"
1261 keysCoadd = namespace.butler.getKeys(datasetType=datasetType, level=self.level)
1263 for dataId in self.idList:
1264 # tract and patch are required
1265 for key in keysCoadd:
1266 if key not in dataId:
1267 raise RuntimeError("--id must include " + key)
1269 dataRef = namespace.butler.dataRef(
1270 datasetType=datasetType,
1271 dataId=dataId,
1272 )
1273 self.refList.append(dataRef)
1276def countMaskFromFootprint(mask, footprint, bitmask, ignoreMask):
1277 """Function to count the number of pixels with a specific mask in a
1278 footprint.
1280 Find the intersection of mask & footprint. Count all pixels in the mask
1281 that are in the intersection that have bitmask set but do not have
1282 ignoreMask set. Return the count.
1284 Parameters
1285 ----------
1286 mask : `lsst.afw.image.Mask`
1287 Mask to define intersection region by.
1288 footprint : `lsst.afw.detection.Footprint`
1289 Footprint to define the intersection region by.
1290 bitmask
1291 Specific mask that we wish to count the number of occurances of.
1292 ignoreMask
1293 Pixels to not consider.
1295 Returns
1296 -------
1297 result : `int`
1298 Count of number of pixels in footprint with specified mask.
1299 """
1300 bbox = footprint.getBBox()
1301 bbox.clip(mask.getBBox(afwImage.PARENT))
1302 fp = afwImage.Mask(bbox)
1303 subMask = mask.Factory(mask, bbox, afwImage.PARENT)
1304 footprint.spans.setMask(fp, bitmask)
1305 return numpy.logical_and((subMask.getArray() & fp.getArray()) > 0,
1306 (subMask.getArray() & ignoreMask) == 0).sum()
1309class SafeClipAssembleCoaddConfig(AssembleCoaddConfig, pipelineConnections=AssembleCoaddConnections):
1310 """Configuration parameters for the SafeClipAssembleCoaddTask.
1311 """
1312 clipDetection = pexConfig.ConfigurableField(
1313 target=SourceDetectionTask,
1314 doc="Detect sources on difference between unclipped and clipped coadd")
1315 minClipFootOverlap = pexConfig.Field(
1316 doc="Minimum fractional overlap of clipped footprint with visit DETECTED to be clipped",
1317 dtype=float,
1318 default=0.6
1319 )
1320 minClipFootOverlapSingle = pexConfig.Field(
1321 doc="Minimum fractional overlap of clipped footprint with visit DETECTED to be "
1322 "clipped when only one visit overlaps",
1323 dtype=float,
1324 default=0.5
1325 )
1326 minClipFootOverlapDouble = pexConfig.Field(
1327 doc="Minimum fractional overlap of clipped footprints with visit DETECTED to be "
1328 "clipped when two visits overlap",
1329 dtype=float,
1330 default=0.45
1331 )
1332 maxClipFootOverlapDouble = pexConfig.Field(
1333 doc="Maximum fractional overlap of clipped footprints with visit DETECTED when "
1334 "considering two visits",
1335 dtype=float,
1336 default=0.15
1337 )
1338 minBigOverlap = pexConfig.Field(
1339 doc="Minimum number of pixels in footprint to use DETECTED mask from the single visits "
1340 "when labeling clipped footprints",
1341 dtype=int,
1342 default=100
1343 )
1345 def setDefaults(self):
1346 """Set default values for clipDetection.
1348 Notes
1349 -----
1350 The numeric values for these configuration parameters were
1351 empirically determined, future work may further refine them.
1352 """
1353 AssembleCoaddConfig.setDefaults(self)
1354 self.clipDetection.doTempLocalBackground = False
1355 self.clipDetection.reEstimateBackground = False
1356 self.clipDetection.returnOriginalFootprints = False
1357 self.clipDetection.thresholdPolarity = "both"
1358 self.clipDetection.thresholdValue = 2
1359 self.clipDetection.nSigmaToGrow = 2
1360 self.clipDetection.minPixels = 4
1361 self.clipDetection.isotropicGrow = True
1362 self.clipDetection.thresholdType = "pixel_stdev"
1363 self.sigmaClip = 1.5
1364 self.clipIter = 3
1365 self.statistic = "MEAN"
1367 def validate(self):
1368 if self.doSigmaClip:
1369 log.warn("Additional Sigma-clipping not allowed in Safe-clipped Coadds. "
1370 "Ignoring doSigmaClip.")
1371 self.doSigmaClip = False
1372 if self.statistic != "MEAN":
1373 raise ValueError("Only MEAN statistic allowed for final stacking in SafeClipAssembleCoadd "
1374 "(%s chosen). Please set statistic to MEAN."
1375 % (self.statistic))
1376 AssembleCoaddTask.ConfigClass.validate(self)
1379class SafeClipAssembleCoaddTask(AssembleCoaddTask):
1380 """Assemble a coadded image from a set of coadded temporary exposures,
1381 being careful to clip & flag areas with potential artifacts.
1383 In ``AssembleCoaddTask``, we compute the coadd as an clipped mean (i.e.,
1384 we clip outliers). The problem with doing this is that when computing the
1385 coadd PSF at a given location, individual visit PSFs from visits with
1386 outlier pixels contribute to the coadd PSF and cannot be treated correctly.
1387 In this task, we correct for this behavior by creating a new
1388 ``badMaskPlane`` 'CLIPPED'. We populate this plane on the input
1389 coaddTempExps and the final coadd where
1391 i. difference imaging suggests that there is an outlier and
1392 ii. this outlier appears on only one or two images.
1394 Such regions will not contribute to the final coadd. Furthermore, any
1395 routine to determine the coadd PSF can now be cognizant of clipped regions.
1396 Note that the algorithm implemented by this task is preliminary and works
1397 correctly for HSC data. Parameter modifications and or considerable
1398 redesigning of the algorithm is likley required for other surveys.
1400 ``SafeClipAssembleCoaddTask`` uses a ``SourceDetectionTask``
1401 "clipDetection" subtask and also sub-classes ``AssembleCoaddTask``.
1402 You can retarget the ``SourceDetectionTask`` "clipDetection" subtask
1403 if you wish.
1405 Notes
1406 -----
1407 The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a
1408 flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``;
1409 see `baseDebug` for more about ``debug.py`` files.
1410 `SafeClipAssembleCoaddTask` has no debug variables of its own.
1411 The ``SourceDetectionTask`` "clipDetection" subtasks may support debug
1412 variables. See the documetation for `SourceDetectionTask` "clipDetection"
1413 for further information.
1415 Examples
1416 --------
1417 `SafeClipAssembleCoaddTask` assembles a set of warped ``coaddTempExp``
1418 images into a coadded image. The `SafeClipAssembleCoaddTask` is invoked by
1419 running assembleCoadd.py *without* the flag '--legacyCoadd'.
1421 Usage of ``assembleCoadd.py`` expects a data reference to the tract patch
1422 and filter to be coadded (specified using
1423 '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]')
1424 along with a list of coaddTempExps to attempt to coadd (specified using
1425 '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]').
1426 Only the coaddTempExps that cover the specified tract and patch will be
1427 coadded. A list of the available optional arguments can be obtained by
1428 calling assembleCoadd.py with the --help command line argument:
1430 .. code-block:: none
1432 assembleCoadd.py --help
1434 To demonstrate usage of the `SafeClipAssembleCoaddTask` in the larger
1435 context of multi-band processing, we will generate the HSC-I & -R band
1436 coadds from HSC engineering test data provided in the ci_hsc package.
1437 To begin, assuming that the lsst stack has been already set up, we must
1438 set up the obs_subaru and ci_hsc packages. This defines the environment
1439 variable $CI_HSC_DIR and points at the location of the package. The raw
1440 HSC data live in the ``$CI_HSC_DIR/raw`` directory. To begin assembling
1441 the coadds, we must first
1443 - ``processCcd``
1444 process the individual ccds in $CI_HSC_RAW to produce calibrated exposures
1445 - ``makeSkyMap``
1446 create a skymap that covers the area of the sky present in the raw exposures
1447 - ``makeCoaddTempExp``
1448 warp the individual calibrated exposures to the tangent plane of the coadd</DD>
1450 We can perform all of these steps by running
1452 .. code-block:: none
1454 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
1456 This will produce warped coaddTempExps for each visit. To coadd the
1457 warped data, we call ``assembleCoadd.py`` as follows:
1459 .. code-block:: none
1461 assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \
1462 --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \
1463 --selectId visit=903986 ccd=100--selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \
1464 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \
1465 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \
1466 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \
1467 --selectId visit=903988 ccd=24
1469 This will process the HSC-I band data. The results are written in
1470 ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``.
1472 You may also choose to run:
1474 .. code-block:: none
1476 scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346 nnn
1477 assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R --selectId visit=903334 ccd=16 \
1478 --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 --selectId visit=903334 ccd=100 \
1479 --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 --selectId visit=903338 ccd=18 \
1480 --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 --selectId visit=903342 ccd=10 \
1481 --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 --selectId visit=903344 ccd=5 \
1482 --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 --selectId visit=903346 ccd=6 \
1483 --selectId visit=903346 ccd=12
1485 to generate the coadd for the HSC-R band if you are interested in following
1486 multiBand Coadd processing as discussed in ``pipeTasks_multiBand``.
1487 """
1488 ConfigClass = SafeClipAssembleCoaddConfig
1489 _DefaultName = "safeClipAssembleCoadd"
1491 def __init__(self, *args, **kwargs):
1492 AssembleCoaddTask.__init__(self, *args, **kwargs)
1493 schema = afwTable.SourceTable.makeMinimalSchema()
1494 self.makeSubtask("clipDetection", schema=schema)
1496 @utils.inheritDoc(AssembleCoaddTask)
1497 @pipeBase.timeMethod
1498 def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, *args, **kwargs):
1499 """Assemble the coadd for a region.
1501 Compute the difference of coadds created with and without outlier
1502 rejection to identify coadd pixels that have outlier values in some
1503 individual visits.
1504 Detect clipped regions on the difference image and mark these regions
1505 on the one or two individual coaddTempExps where they occur if there
1506 is significant overlap between the clipped region and a source. This
1507 leaves us with a set of footprints from the difference image that have
1508 been identified as having occured on just one or two individual visits.
1509 However, these footprints were generated from a difference image. It
1510 is conceivable for a large diffuse source to have become broken up
1511 into multiple footprints acrosss the coadd difference in this process.
1512 Determine the clipped region from all overlapping footprints from the
1513 detected sources in each visit - these are big footprints.
1514 Combine the small and big clipped footprints and mark them on a new
1515 bad mask plane.
1516 Generate the coadd using `AssembleCoaddTask.run` without outlier
1517 removal. Clipped footprints will no longer make it into the coadd
1518 because they are marked in the new bad mask plane.
1520 Notes
1521 -----
1522 args and kwargs are passed but ignored in order to match the call
1523 signature expected by the parent task.
1524 """
1525 exp = self.buildDifferenceImage(skyInfo, tempExpRefList, imageScalerList, weightList)
1526 mask = exp.getMaskedImage().getMask()
1527 mask.addMaskPlane("CLIPPED")
1529 result = self.detectClip(exp, tempExpRefList)
1531 self.log.info('Found %d clipped objects', len(result.clipFootprints))
1533 maskClipValue = mask.getPlaneBitMask("CLIPPED")
1534 maskDetValue = mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE")
1535 # Append big footprints from individual Warps to result.clipSpans
1536 bigFootprints = self.detectClipBig(result.clipSpans, result.clipFootprints, result.clipIndices,
1537 result.detectionFootprints, maskClipValue, maskDetValue,
1538 exp.getBBox())
1539 # Create mask of the current clipped footprints
1540 maskClip = mask.Factory(mask.getBBox(afwImage.PARENT))
1541 afwDet.setMaskFromFootprintList(maskClip, result.clipFootprints, maskClipValue)
1543 maskClipBig = maskClip.Factory(mask.getBBox(afwImage.PARENT))
1544 afwDet.setMaskFromFootprintList(maskClipBig, bigFootprints, maskClipValue)
1545 maskClip |= maskClipBig
1547 # Assemble coadd from base class, but ignoring CLIPPED pixels
1548 badMaskPlanes = self.config.badMaskPlanes[:]
1549 badMaskPlanes.append("CLIPPED")
1550 badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes)
1551 return AssembleCoaddTask.run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
1552 result.clipSpans, mask=badPixelMask)
1554 def buildDifferenceImage(self, skyInfo, tempExpRefList, imageScalerList, weightList):
1555 """Return an exposure that contains the difference between unclipped
1556 and clipped coadds.
1558 Generate a difference image between clipped and unclipped coadds.
1559 Compute the difference image by subtracting an outlier-clipped coadd
1560 from an outlier-unclipped coadd. Return the difference image.
1562 Parameters
1563 ----------
1564 skyInfo : `lsst.pipe.base.Struct`
1565 Patch geometry information, from getSkyInfo
1566 tempExpRefList : `list`
1567 List of data reference to tempExp
1568 imageScalerList : `list`
1569 List of image scalers
1570 weightList : `list`
1571 List of weights
1573 Returns
1574 -------
1575 exp : `lsst.afw.image.Exposure`
1576 Difference image of unclipped and clipped coadd wrapped in an Exposure
1577 """
1578 config = AssembleCoaddConfig()
1579 # getattr necessary because subtasks do not survive Config.toDict()
1580 # exclude connections because the class of self.config.connections is not
1581 # the same as AssembleCoaddConfig.connections, and the connections are not
1582 # needed to run this task anyway.
1583 configIntersection = {k: getattr(self.config, k)
1584 for k, v in self.config.toDict().items()
1585 if (k in config.keys() and k != "connections")}
1586 configIntersection['doInputMap'] = False
1587 configIntersection['doNImage'] = False
1588 config.update(**configIntersection)
1590 # statistic MEAN copied from self.config.statistic, but for clarity explicitly assign
1591 config.statistic = 'MEAN'
1592 task = AssembleCoaddTask(config=config)
1593 coaddMean = task.run(skyInfo, tempExpRefList, imageScalerList, weightList).coaddExposure
1595 config.statistic = 'MEANCLIP'
1596 task = AssembleCoaddTask(config=config)
1597 coaddClip = task.run(skyInfo, tempExpRefList, imageScalerList, weightList).coaddExposure
1599 coaddDiff = coaddMean.getMaskedImage().Factory(coaddMean.getMaskedImage())
1600 coaddDiff -= coaddClip.getMaskedImage()
1601 exp = afwImage.ExposureF(coaddDiff)
1602 exp.setPsf(coaddMean.getPsf())
1603 return exp
1605 def detectClip(self, exp, tempExpRefList):
1606 """Detect clipped regions on an exposure and set the mask on the
1607 individual tempExp masks.
1609 Detect footprints in the difference image after smoothing the
1610 difference image with a Gaussian kernal. Identify footprints that
1611 overlap with one or two input ``coaddTempExps`` by comparing the
1612 computed overlap fraction to thresholds set in the config. A different
1613 threshold is applied depending on the number of overlapping visits
1614 (restricted to one or two). If the overlap exceeds the thresholds,
1615 the footprint is considered "CLIPPED" and is marked as such on the
1616 coaddTempExp. Return a struct with the clipped footprints, the indices
1617 of the ``coaddTempExps`` that end up overlapping with the clipped
1618 footprints, and a list of new masks for the ``coaddTempExps``.
1620 Parameters
1621 ----------
1622 exp : `lsst.afw.image.Exposure`
1623 Exposure to run detection on.
1624 tempExpRefList : `list`
1625 List of data reference to tempExp.
1627 Returns
1628 -------
1629 result : `lsst.pipe.base.Struct`
1630 Result struct with components:
1632 - ``clipFootprints``: list of clipped footprints.
1633 - ``clipIndices``: indices for each ``clippedFootprint`` in
1634 ``tempExpRefList``.
1635 - ``clipSpans``: List of dictionaries containing spanSet lists
1636 to clip. Each element contains the new maskplane name
1637 ("CLIPPED") as the key and list of ``SpanSets`` as the value.
1638 - ``detectionFootprints``: List of DETECTED/DETECTED_NEGATIVE plane
1639 compressed into footprints.
1640 """
1641 mask = exp.getMaskedImage().getMask()
1642 maskDetValue = mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE")
1643 fpSet = self.clipDetection.detectFootprints(exp, doSmooth=True, clearMask=True)
1644 # Merge positive and negative together footprints together
1645 fpSet.positive.merge(fpSet.negative)
1646 footprints = fpSet.positive
1647 self.log.info('Found %d potential clipped objects', len(footprints.getFootprints()))
1648 ignoreMask = self.getBadPixelMask()
1650 clipFootprints = []
1651 clipIndices = []
1652 artifactSpanSets = [{'CLIPPED': list()} for _ in tempExpRefList]
1654 # for use by detectClipBig
1655 visitDetectionFootprints = []
1657 dims = [len(tempExpRefList), len(footprints.getFootprints())]
1658 overlapDetArr = numpy.zeros(dims, dtype=numpy.uint16)
1659 ignoreArr = numpy.zeros(dims, dtype=numpy.uint16)
1661 # Loop over masks once and extract/store only relevant overlap metrics and detection footprints
1662 for i, warpRef in enumerate(tempExpRefList):
1663 tmpExpMask = warpRef.get(datasetType=self.getTempExpDatasetName(self.warpType),
1664 immediate=True).getMaskedImage().getMask()
1665 maskVisitDet = tmpExpMask.Factory(tmpExpMask, tmpExpMask.getBBox(afwImage.PARENT),
1666 afwImage.PARENT, True)
1667 maskVisitDet &= maskDetValue
1668 visitFootprints = afwDet.FootprintSet(maskVisitDet, afwDet.Threshold(1))
1669 visitDetectionFootprints.append(visitFootprints)
1671 for j, footprint in enumerate(footprints.getFootprints()):
1672 ignoreArr[i, j] = countMaskFromFootprint(tmpExpMask, footprint, ignoreMask, 0x0)
1673 overlapDetArr[i, j] = countMaskFromFootprint(tmpExpMask, footprint, maskDetValue, ignoreMask)
1675 # build a list of clipped spans for each visit
1676 for j, footprint in enumerate(footprints.getFootprints()):
1677 nPixel = footprint.getArea()
1678 overlap = [] # hold the overlap with each visit
1679 indexList = [] # index of visit in global list
1680 for i in range(len(tempExpRefList)):
1681 ignore = ignoreArr[i, j]
1682 overlapDet = overlapDetArr[i, j]
1683 totPixel = nPixel - ignore
1685 # If we have more bad pixels than detection skip
1686 if ignore > overlapDet or totPixel <= 0.5*nPixel or overlapDet == 0: 1686 ↛ 1688line 1686 didn't jump to line 1688, because the condition on line 1686 was never false
1687 continue
1688 overlap.append(overlapDet/float(totPixel))
1689 indexList.append(i)
1691 overlap = numpy.array(overlap)
1692 if not len(overlap): 1692 ↛ 1695line 1692 didn't jump to line 1695, because the condition on line 1692 was never false
1693 continue
1695 keep = False # Should this footprint be marked as clipped?
1696 keepIndex = [] # Which tempExps does the clipped footprint belong to
1698 # If footprint only has one overlap use a lower threshold
1699 if len(overlap) == 1:
1700 if overlap[0] > self.config.minClipFootOverlapSingle:
1701 keep = True
1702 keepIndex = [0]
1703 else:
1704 # This is the general case where only visit should be clipped
1705 clipIndex = numpy.where(overlap > self.config.minClipFootOverlap)[0]
1706 if len(clipIndex) == 1:
1707 keep = True
1708 keepIndex = [clipIndex[0]]
1710 # Test if there are clipped objects that overlap two different visits
1711 clipIndex = numpy.where(overlap > self.config.minClipFootOverlapDouble)[0]
1712 if len(clipIndex) == 2 and len(overlap) > 3:
1713 clipIndexComp = numpy.where(overlap <= self.config.minClipFootOverlapDouble)[0]
1714 if numpy.max(overlap[clipIndexComp]) <= self.config.maxClipFootOverlapDouble:
1715 keep = True
1716 keepIndex = clipIndex
1718 if not keep:
1719 continue
1721 for index in keepIndex:
1722 globalIndex = indexList[index]
1723 artifactSpanSets[globalIndex]['CLIPPED'].append(footprint.spans)
1725 clipIndices.append(numpy.array(indexList)[keepIndex])
1726 clipFootprints.append(footprint)
1728 return pipeBase.Struct(clipFootprints=clipFootprints, clipIndices=clipIndices,
1729 clipSpans=artifactSpanSets, detectionFootprints=visitDetectionFootprints)
1731 def detectClipBig(self, clipList, clipFootprints, clipIndices, detectionFootprints,
1732 maskClipValue, maskDetValue, coaddBBox):
1733 """Return individual warp footprints for large artifacts and append
1734 them to ``clipList`` in place.
1736 Identify big footprints composed of many sources in the coadd
1737 difference that may have originated in a large diffuse source in the
1738 coadd. We do this by indentifying all clipped footprints that overlap
1739 significantly with each source in all the coaddTempExps.
1741 Parameters
1742 ----------
1743 clipList : `list`
1744 List of alt mask SpanSets with clipping information. Modified.
1745 clipFootprints : `list`
1746 List of clipped footprints.
1747 clipIndices : `list`
1748 List of which entries in tempExpClipList each footprint belongs to.
1749 maskClipValue
1750 Mask value of clipped pixels.
1751 maskDetValue
1752 Mask value of detected pixels.
1753 coaddBBox : `lsst.geom.Box`
1754 BBox of the coadd and warps.
1756 Returns
1757 -------
1758 bigFootprintsCoadd : `list`
1759 List of big footprints
1760 """
1761 bigFootprintsCoadd = []
1762 ignoreMask = self.getBadPixelMask()
1763 for index, (clippedSpans, visitFootprints) in enumerate(zip(clipList, detectionFootprints)):
1764 maskVisitDet = afwImage.MaskX(coaddBBox, 0x0)
1765 for footprint in visitFootprints.getFootprints(): 1765 ↛ 1766line 1765 didn't jump to line 1766, because the loop on line 1765 never started
1766 footprint.spans.setMask(maskVisitDet, maskDetValue)
1768 # build a mask of clipped footprints that are in this visit
1769 clippedFootprintsVisit = []
1770 for foot, clipIndex in zip(clipFootprints, clipIndices): 1770 ↛ 1771line 1770 didn't jump to line 1771, because the loop on line 1770 never started
1771 if index not in clipIndex:
1772 continue
1773 clippedFootprintsVisit.append(foot)
1774 maskVisitClip = maskVisitDet.Factory(maskVisitDet.getBBox(afwImage.PARENT))
1775 afwDet.setMaskFromFootprintList(maskVisitClip, clippedFootprintsVisit, maskClipValue)
1777 bigFootprintsVisit = []
1778 for foot in visitFootprints.getFootprints(): 1778 ↛ 1779line 1778 didn't jump to line 1779, because the loop on line 1778 never started
1779 if foot.getArea() < self.config.minBigOverlap:
1780 continue
1781 nCount = countMaskFromFootprint(maskVisitClip, foot, maskClipValue, ignoreMask)
1782 if nCount > self.config.minBigOverlap:
1783 bigFootprintsVisit.append(foot)
1784 bigFootprintsCoadd.append(foot)
1786 for footprint in bigFootprintsVisit: 1786 ↛ 1787line 1786 didn't jump to line 1787, because the loop on line 1786 never started
1787 clippedSpans["CLIPPED"].append(footprint.spans)
1789 return bigFootprintsCoadd
1792class CompareWarpAssembleCoaddConnections(AssembleCoaddConnections):
1793 psfMatchedWarps = pipeBase.connectionTypes.Input(
1794 doc=("PSF-Matched Warps are required by CompareWarp regardless of the coadd type requested. "
1795 "Only PSF-Matched Warps make sense for image subtraction. "
1796 "Therefore, they must be an additional declared input."),
1797 name="{inputCoaddName}Coadd_psfMatchedWarp",
1798 storageClass="ExposureF",
1799 dimensions=("tract", "patch", "skymap", "visit"),
1800 deferLoad=True,
1801 multiple=True
1802 )
1803 templateCoadd = pipeBase.connectionTypes.Output(
1804 doc=("Model of the static sky, used to find temporal artifacts. Typically a PSF-Matched, "
1805 "sigma-clipped coadd. Written if and only if assembleStaticSkyModel.doWrite=True"),
1806 name="{outputCoaddName}CoaddPsfMatched",
1807 storageClass="ExposureF",
1808 dimensions=("tract", "patch", "skymap", "band"),
1809 )
1811 def __init__(self, *, config=None):
1812 super().__init__(config=config)
1813 if not config.assembleStaticSkyModel.doWrite:
1814 self.outputs.remove("templateCoadd")
1815 config.validate()
1818class CompareWarpAssembleCoaddConfig(AssembleCoaddConfig,
1819 pipelineConnections=CompareWarpAssembleCoaddConnections):
1820 assembleStaticSkyModel = pexConfig.ConfigurableField(
1821 target=AssembleCoaddTask,
1822 doc="Task to assemble an artifact-free, PSF-matched Coadd to serve as a"
1823 " naive/first-iteration model of the static sky.",
1824 )
1825 detect = pexConfig.ConfigurableField(
1826 target=SourceDetectionTask,
1827 doc="Detect outlier sources on difference between each psfMatched warp and static sky model"
1828 )
1829 detectTemplate = pexConfig.ConfigurableField(
1830 target=SourceDetectionTask,
1831 doc="Detect sources on static sky model. Only used if doPreserveContainedBySource is True"
1832 )
1833 maskStreaks = pexConfig.ConfigurableField(
1834 target=MaskStreaksTask,
1835 doc="Detect streaks on difference between each psfMatched warp and static sky model. Only used if "
1836 "doFilterMorphological is True. Adds a mask plane to an exposure, with the mask plane name set by"
1837 "streakMaskName"
1838 )
1839 streakMaskName = pexConfig.Field(
1840 dtype=str,
1841 default="STREAK",
1842 doc="Name of mask bit used for streaks"
1843 )
1844 maxNumEpochs = pexConfig.Field(
1845 doc="Charactistic maximum local number of epochs/visits in which an artifact candidate can appear "
1846 "and still be masked. The effective maxNumEpochs is a broken linear function of local "
1847 "number of epochs (N): min(maxFractionEpochsLow*N, maxNumEpochs + maxFractionEpochsHigh*N). "
1848 "For each footprint detected on the image difference between the psfMatched warp and static sky "
1849 "model, if a significant fraction of pixels (defined by spatialThreshold) are residuals in more "
1850 "than the computed effective maxNumEpochs, the artifact candidate is deemed persistant rather "
1851 "than transient and not masked.",
1852 dtype=int,
1853 default=2
1854 )
1855 maxFractionEpochsLow = pexConfig.RangeField(
1856 doc="Fraction of local number of epochs (N) to use as effective maxNumEpochs for low N. "
1857 "Effective maxNumEpochs = "
1858 "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)",
1859 dtype=float,
1860 default=0.4,
1861 min=0., max=1.,
1862 )
1863 maxFractionEpochsHigh = pexConfig.RangeField(
1864 doc="Fraction of local number of epochs (N) to use as effective maxNumEpochs for high N. "
1865 "Effective maxNumEpochs = "
1866 "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)",
1867 dtype=float,
1868 default=0.03,
1869 min=0., max=1.,
1870 )
1871 spatialThreshold = pexConfig.RangeField(
1872 doc="Unitless fraction of pixels defining how much of the outlier region has to meet the "
1873 "temporal criteria. If 0, clip all. If 1, clip none.",
1874 dtype=float,
1875 default=0.5,
1876 min=0., max=1.,
1877 inclusiveMin=True, inclusiveMax=True
1878 )
1879 doScaleWarpVariance = pexConfig.Field(
1880 doc="Rescale Warp variance plane using empirical noise?",
1881 dtype=bool,
1882 default=True,
1883 )
1884 scaleWarpVariance = pexConfig.ConfigurableField(
1885 target=ScaleVarianceTask,
1886 doc="Rescale variance on warps",
1887 )
1888 doPreserveContainedBySource = pexConfig.Field(
1889 doc="Rescue artifacts from clipping that completely lie within a footprint detected"
1890 "on the PsfMatched Template Coadd. Replicates a behavior of SafeClip.",
1891 dtype=bool,
1892 default=True,
1893 )
1894 doPrefilterArtifacts = pexConfig.Field(
1895 doc="Ignore artifact candidates that are mostly covered by the bad pixel mask, "
1896 "because they will be excluded anyway. This prevents them from contributing "
1897 "to the outlier epoch count image and potentially being labeled as persistant."
1898 "'Mostly' is defined by the config 'prefilterArtifactsRatio'.",
1899 dtype=bool,
1900 default=True
1901 )
1902 prefilterArtifactsMaskPlanes = pexConfig.ListField(
1903 doc="Prefilter artifact candidates that are mostly covered by these bad mask planes.",
1904 dtype=str,
1905 default=('NO_DATA', 'BAD', 'SAT', 'SUSPECT'),
1906 )
1907 prefilterArtifactsRatio = pexConfig.Field(
1908 doc="Prefilter artifact candidates with less than this fraction overlapping good pixels",
1909 dtype=float,
1910 default=0.05
1911 )
1912 doFilterMorphological = pexConfig.Field(
1913 doc="Filter artifact candidates based on morphological criteria, i.g. those that appear to "
1914 "be streaks.",
1915 dtype=bool,
1916 default=False
1917 )
1919 def setDefaults(self):
1920 AssembleCoaddConfig.setDefaults(self)
1921 self.statistic = 'MEAN'
1922 self.doUsePsfMatchedPolygons = True
1924 # Real EDGE removed by psfMatched NO_DATA border half the width of the matching kernel
1925 # CompareWarp applies psfMatched EDGE pixels to directWarps before assembling
1926 if "EDGE" in self.badMaskPlanes: 1926 ↛ 1928line 1926 didn't jump to line 1928, because the condition on line 1926 was never false
1927 self.badMaskPlanes.remove('EDGE')
1928 self.removeMaskPlanes.append('EDGE')
1929 self.assembleStaticSkyModel.badMaskPlanes = ["NO_DATA", ]
1930 self.assembleStaticSkyModel.warpType = 'psfMatched'
1931 self.assembleStaticSkyModel.connections.warpType = 'psfMatched'
1932 self.assembleStaticSkyModel.statistic = 'MEANCLIP'
1933 self.assembleStaticSkyModel.sigmaClip = 2.5
1934 self.assembleStaticSkyModel.clipIter = 3
1935 self.assembleStaticSkyModel.calcErrorFromInputVariance = False
1936 self.assembleStaticSkyModel.doWrite = False
1937 self.detect.doTempLocalBackground = False
1938 self.detect.reEstimateBackground = False
1939 self.detect.returnOriginalFootprints = False
1940 self.detect.thresholdPolarity = "both"
1941 self.detect.thresholdValue = 5
1942 self.detect.minPixels = 4
1943 self.detect.isotropicGrow = True
1944 self.detect.thresholdType = "pixel_stdev"
1945 self.detect.nSigmaToGrow = 0.4
1946 # The default nSigmaToGrow for SourceDetectionTask is already 2.4,
1947 # Explicitly restating because ratio with detect.nSigmaToGrow matters
1948 self.detectTemplate.nSigmaToGrow = 2.4
1949 self.detectTemplate.doTempLocalBackground = False
1950 self.detectTemplate.reEstimateBackground = False
1951 self.detectTemplate.returnOriginalFootprints = False
1953 def validate(self):
1954 super().validate()
1955 if self.assembleStaticSkyModel.doNImage:
1956 raise ValueError("No dataset type exists for a PSF-Matched Template N Image."
1957 "Please set assembleStaticSkyModel.doNImage=False")
1959 if self.assembleStaticSkyModel.doWrite and (self.warpType == self.assembleStaticSkyModel.warpType):
1960 raise ValueError("warpType (%s) == assembleStaticSkyModel.warpType (%s) and will compete for "
1961 "the same dataset name. Please set assembleStaticSkyModel.doWrite to False "
1962 "or warpType to 'direct'. assembleStaticSkyModel.warpType should ways be "
1963 "'PsfMatched'" % (self.warpType, self.assembleStaticSkyModel.warpType))
1966class CompareWarpAssembleCoaddTask(AssembleCoaddTask):
1967 """Assemble a compareWarp coadded image from a set of warps
1968 by masking artifacts detected by comparing PSF-matched warps.
1970 In ``AssembleCoaddTask``, we compute the coadd as an clipped mean (i.e.,
1971 we clip outliers). The problem with doing this is that when computing the
1972 coadd PSF at a given location, individual visit PSFs from visits with
1973 outlier pixels contribute to the coadd PSF and cannot be treated correctly.
1974 In this task, we correct for this behavior by creating a new badMaskPlane
1975 'CLIPPED' which marks pixels in the individual warps suspected to contain
1976 an artifact. We populate this plane on the input warps by comparing
1977 PSF-matched warps with a PSF-matched median coadd which serves as a
1978 model of the static sky. Any group of pixels that deviates from the
1979 PSF-matched template coadd by more than config.detect.threshold sigma,
1980 is an artifact candidate. The candidates are then filtered to remove
1981 variable sources and sources that are difficult to subtract such as
1982 bright stars. This filter is configured using the config parameters
1983 ``temporalThreshold`` and ``spatialThreshold``. The temporalThreshold is
1984 the maximum fraction of epochs that the deviation can appear in and still
1985 be considered an artifact. The spatialThreshold is the maximum fraction of
1986 pixels in the footprint of the deviation that appear in other epochs
1987 (where other epochs is defined by the temporalThreshold). If the deviant
1988 region meets this criteria of having a significant percentage of pixels
1989 that deviate in only a few epochs, these pixels have the 'CLIPPED' bit
1990 set in the mask. These regions will not contribute to the final coadd.
1991 Furthermore, any routine to determine the coadd PSF can now be cognizant
1992 of clipped regions. Note that the algorithm implemented by this task is
1993 preliminary and works correctly for HSC data. Parameter modifications and
1994 or considerable redesigning of the algorithm is likley required for other
1995 surveys.
1997 ``CompareWarpAssembleCoaddTask`` sub-classes
1998 ``AssembleCoaddTask`` and instantiates ``AssembleCoaddTask``
1999 as a subtask to generate the TemplateCoadd (the model of the static sky).
2001 Notes
2002 -----
2003 The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a
2004 flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``; see
2005 ``baseDebug`` for more about ``debug.py`` files.
2007 This task supports the following debug variables:
2009 - ``saveCountIm``
2010 If True then save the Epoch Count Image as a fits file in the `figPath`
2011 - ``figPath``
2012 Path to save the debug fits images and figures
2014 For example, put something like:
2016 .. code-block:: python
2018 import lsstDebug
2019 def DebugInfo(name):
2020 di = lsstDebug.getInfo(name)
2021 if name == "lsst.pipe.tasks.assembleCoadd":
2022 di.saveCountIm = True
2023 di.figPath = "/desired/path/to/debugging/output/images"
2024 return di
2025 lsstDebug.Info = DebugInfo
2027 into your ``debug.py`` file and run ``assemebleCoadd.py`` with the
2028 ``--debug`` flag. Some subtasks may have their own debug variables;
2029 see individual Task documentation.
2031 Examples
2032 --------
2033 ``CompareWarpAssembleCoaddTask`` assembles a set of warped images into a
2034 coadded image. The ``CompareWarpAssembleCoaddTask`` is invoked by running
2035 ``assembleCoadd.py`` with the flag ``--compareWarpCoadd``.
2036 Usage of ``assembleCoadd.py`` expects a data reference to the tract patch
2037 and filter to be coadded (specified using
2038 '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]')
2039 along with a list of coaddTempExps to attempt to coadd (specified using
2040 '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]').
2041 Only the warps that cover the specified tract and patch will be coadded.
2042 A list of the available optional arguments can be obtained by calling
2043 ``assembleCoadd.py`` with the ``--help`` command line argument:
2045 .. code-block:: none
2047 assembleCoadd.py --help
2049 To demonstrate usage of the ``CompareWarpAssembleCoaddTask`` in the larger
2050 context of multi-band processing, we will generate the HSC-I & -R band
2051 oadds from HSC engineering test data provided in the ``ci_hsc`` package.
2052 To begin, assuming that the lsst stack has been already set up, we must
2053 set up the ``obs_subaru`` and ``ci_hsc`` packages.
2054 This defines the environment variable ``$CI_HSC_DIR`` and points at the
2055 location of the package. The raw HSC data live in the ``$CI_HSC_DIR/raw``
2056 directory. To begin assembling the coadds, we must first
2058 - processCcd
2059 process the individual ccds in $CI_HSC_RAW to produce calibrated exposures
2060 - makeSkyMap
2061 create a skymap that covers the area of the sky present in the raw exposures
2062 - makeCoaddTempExp
2063 warp the individual calibrated exposures to the tangent plane of the coadd
2065 We can perform all of these steps by running
2067 .. code-block:: none
2069 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
2071 This will produce warped ``coaddTempExps`` for each visit. To coadd the
2072 warped data, we call ``assembleCoadd.py`` as follows:
2074 .. code-block:: none
2076 assembleCoadd.py --compareWarpCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \
2077 --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \
2078 --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \
2079 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \
2080 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \
2081 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \
2082 --selectId visit=903988 ccd=24
2084 This will process the HSC-I band data. The results are written in
2085 ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``.
2086 """
2087 ConfigClass = CompareWarpAssembleCoaddConfig
2088 _DefaultName = "compareWarpAssembleCoadd"
2090 def __init__(self, *args, **kwargs):
2091 AssembleCoaddTask.__init__(self, *args, **kwargs)
2092 self.makeSubtask("assembleStaticSkyModel")
2093 detectionSchema = afwTable.SourceTable.makeMinimalSchema()
2094 self.makeSubtask("detect", schema=detectionSchema)
2095 if self.config.doPreserveContainedBySource: 2095 ↛ 2097line 2095 didn't jump to line 2097, because the condition on line 2095 was never false
2096 self.makeSubtask("detectTemplate", schema=afwTable.SourceTable.makeMinimalSchema())
2097 if self.config.doScaleWarpVariance: 2097 ↛ 2099line 2097 didn't jump to line 2099, because the condition on line 2097 was never false
2098 self.makeSubtask("scaleWarpVariance")
2099 if self.config.doFilterMorphological: 2099 ↛ 2100line 2099 didn't jump to line 2100, because the condition on line 2099 was never true
2100 self.makeSubtask("maskStreaks")
2102 @utils.inheritDoc(AssembleCoaddTask)
2103 def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs):
2104 """
2105 Generate a templateCoadd to use as a naive model of static sky to
2106 subtract from PSF-Matched warps.
2108 Returns
2109 -------
2110 result : `lsst.pipe.base.Struct`
2111 Result struct with components:
2113 - ``templateCoadd`` : coadded exposure (``lsst.afw.image.Exposure``)
2114 - ``nImage`` : N Image (``lsst.afw.image.Image``)
2115 """
2116 # Ensure that psfMatchedWarps are used as input warps for template generation
2117 staticSkyModelInputRefs = copy.deepcopy(inputRefs)
2118 staticSkyModelInputRefs.inputWarps = inputRefs.psfMatchedWarps
2120 # Because subtasks don't have connections we have to make one.
2121 # The main task's `templateCoadd` is the subtask's `coaddExposure`
2122 staticSkyModelOutputRefs = copy.deepcopy(outputRefs)
2123 if self.config.assembleStaticSkyModel.doWrite:
2124 staticSkyModelOutputRefs.coaddExposure = staticSkyModelOutputRefs.templateCoadd
2125 # Remove template coadd from both subtask's and main tasks outputs,
2126 # because it is handled by the subtask as `coaddExposure`
2127 del outputRefs.templateCoadd
2128 del staticSkyModelOutputRefs.templateCoadd
2130 # A PSF-Matched nImage does not exist as a dataset type
2131 if 'nImage' in staticSkyModelOutputRefs.keys():
2132 del staticSkyModelOutputRefs.nImage
2134 templateCoadd = self.assembleStaticSkyModel.runQuantum(butlerQC, staticSkyModelInputRefs,
2135 staticSkyModelOutputRefs)
2136 if templateCoadd is None:
2137 raise RuntimeError(self._noTemplateMessage(self.assembleStaticSkyModel.warpType))
2139 return pipeBase.Struct(templateCoadd=templateCoadd.coaddExposure,
2140 nImage=templateCoadd.nImage,
2141 warpRefList=templateCoadd.warpRefList,
2142 imageScalerList=templateCoadd.imageScalerList,
2143 weightList=templateCoadd.weightList)
2145 @utils.inheritDoc(AssembleCoaddTask)
2146 def makeSupplementaryData(self, dataRef, selectDataList=None, warpRefList=None):
2147 """
2148 Generate a templateCoadd to use as a naive model of static sky to
2149 subtract from PSF-Matched warps.
2151 Returns
2152 -------
2153 result : `lsst.pipe.base.Struct`
2154 Result struct with components:
2156 - ``templateCoadd``: coadded exposure (``lsst.afw.image.Exposure``)
2157 - ``nImage``: N Image (``lsst.afw.image.Image``)
2158 """
2159 templateCoadd = self.assembleStaticSkyModel.runDataRef(dataRef, selectDataList, warpRefList)
2160 if templateCoadd is None: 2160 ↛ 2161line 2160 didn't jump to line 2161, because the condition on line 2160 was never true
2161 raise RuntimeError(self._noTemplateMessage(self.assembleStaticSkyModel.warpType))
2163 return pipeBase.Struct(templateCoadd=templateCoadd.coaddExposure,
2164 nImage=templateCoadd.nImage,
2165 warpRefList=templateCoadd.warpRefList,
2166 imageScalerList=templateCoadd.imageScalerList,
2167 weightList=templateCoadd.weightList)
2169 def _noTemplateMessage(self, warpType):
2170 warpName = (warpType[0].upper() + warpType[1:])
2171 message = """No %(warpName)s warps were found to build the template coadd which is
2172 required to run CompareWarpAssembleCoaddTask. To continue assembling this type of coadd,
2173 first either rerun makeCoaddTempExp with config.make%(warpName)s=True or
2174 coaddDriver with config.makeCoadTempExp.make%(warpName)s=True, before assembleCoadd.
2176 Alternatively, to use another algorithm with existing warps, retarget the CoaddDriverConfig to
2177 another algorithm like:
2179 from lsst.pipe.tasks.assembleCoadd import SafeClipAssembleCoaddTask
2180 config.assemble.retarget(SafeClipAssembleCoaddTask)
2181 """ % {"warpName": warpName}
2182 return message
2184 @utils.inheritDoc(AssembleCoaddTask)
2185 @pipeBase.timeMethod
2186 def run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
2187 supplementaryData, *args, **kwargs):
2188 """Assemble the coadd.
2190 Find artifacts and apply them to the warps' masks creating a list of
2191 alternative masks with a new "CLIPPED" plane and updated "NO_DATA"
2192 plane. Then pass these alternative masks to the base class's `run`
2193 method.
2195 The input parameters ``supplementaryData`` is a `lsst.pipe.base.Struct`
2196 that must contain a ``templateCoadd`` that serves as the
2197 model of the static sky.
2198 """
2200 # Check and match the order of the supplementaryData
2201 # (PSF-matched) inputs to the order of the direct inputs,
2202 # so that the artifact mask is applied to the right warp
2203 dataIds = [ref.dataId for ref in tempExpRefList]
2204 psfMatchedDataIds = [ref.dataId for ref in supplementaryData.warpRefList]
2206 if dataIds != psfMatchedDataIds:
2207 self.log.info("Reordering and or/padding PSF-matched visit input list")
2208 supplementaryData.warpRefList = reorderAndPadList(supplementaryData.warpRefList,
2209 psfMatchedDataIds, dataIds)
2210 supplementaryData.imageScalerList = reorderAndPadList(supplementaryData.imageScalerList,
2211 psfMatchedDataIds, dataIds)
2213 # Use PSF-Matched Warps (and corresponding scalers) and coadd to find artifacts
2214 spanSetMaskList = self.findArtifacts(supplementaryData.templateCoadd,
2215 supplementaryData.warpRefList,
2216 supplementaryData.imageScalerList)
2218 badMaskPlanes = self.config.badMaskPlanes[:]
2219 badMaskPlanes.append("CLIPPED")
2220 badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes)
2222 result = AssembleCoaddTask.run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
2223 spanSetMaskList, mask=badPixelMask)
2225 # Propagate PSF-matched EDGE pixels to coadd SENSOR_EDGE and INEXACT_PSF
2226 # Psf-Matching moves the real edge inwards
2227 self.applyAltEdgeMask(result.coaddExposure.maskedImage.mask, spanSetMaskList)
2228 return result
2230 def applyAltEdgeMask(self, mask, altMaskList):
2231 """Propagate alt EDGE mask to SENSOR_EDGE AND INEXACT_PSF planes.
2233 Parameters
2234 ----------
2235 mask : `lsst.afw.image.Mask`
2236 Original mask.
2237 altMaskList : `list`
2238 List of Dicts containing ``spanSet`` lists.
2239 Each element contains the new mask plane name (e.g. "CLIPPED
2240 and/or "NO_DATA") as the key, and list of ``SpanSets`` to apply to
2241 the mask.
2242 """
2243 maskValue = mask.getPlaneBitMask(["SENSOR_EDGE", "INEXACT_PSF"])
2244 for visitMask in altMaskList:
2245 if "EDGE" in visitMask: 2245 ↛ 2244line 2245 didn't jump to line 2244, because the condition on line 2245 was never false
2246 for spanSet in visitMask['EDGE']:
2247 spanSet.clippedTo(mask.getBBox()).setMask(mask, maskValue)
2249 def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList):
2250 """Find artifacts.
2252 Loop through warps twice. The first loop builds a map with the count
2253 of how many epochs each pixel deviates from the templateCoadd by more
2254 than ``config.chiThreshold`` sigma. The second loop takes each
2255 difference image and filters the artifacts detected in each using
2256 count map to filter out variable sources and sources that are
2257 difficult to subtract cleanly.
2259 Parameters
2260 ----------
2261 templateCoadd : `lsst.afw.image.Exposure`
2262 Exposure to serve as model of static sky.
2263 tempExpRefList : `list`
2264 List of data references to warps.
2265 imageScalerList : `list`
2266 List of image scalers.
2268 Returns
2269 -------
2270 altMasks : `list`
2271 List of dicts containing information about CLIPPED
2272 (i.e., artifacts), NO_DATA, and EDGE pixels.
2273 """
2275 self.log.debug("Generating Count Image, and mask lists.")
2276 coaddBBox = templateCoadd.getBBox()
2277 slateIm = afwImage.ImageU(coaddBBox)
2278 epochCountImage = afwImage.ImageU(coaddBBox)
2279 nImage = afwImage.ImageU(coaddBBox)
2280 spanSetArtifactList = []
2281 spanSetNoDataMaskList = []
2282 spanSetEdgeList = []
2283 spanSetBadMorphoList = []
2284 badPixelMask = self.getBadPixelMask()
2286 # mask of the warp diffs should = that of only the warp
2287 templateCoadd.mask.clearAllMaskPlanes()
2289 if self.config.doPreserveContainedBySource: 2289 ↛ 2292line 2289 didn't jump to line 2292, because the condition on line 2289 was never false
2290 templateFootprints = self.detectTemplate.detectFootprints(templateCoadd)
2291 else:
2292 templateFootprints = None
2294 for warpRef, imageScaler in zip(tempExpRefList, imageScalerList):
2295 warpDiffExp = self._readAndComputeWarpDiff(warpRef, imageScaler, templateCoadd)
2296 if warpDiffExp is not None:
2297 # This nImage only approximates the final nImage because it uses the PSF-matched mask
2298 nImage.array += (numpy.isfinite(warpDiffExp.image.array)
2299 * ((warpDiffExp.mask.array & badPixelMask) == 0)).astype(numpy.uint16)
2300 fpSet = self.detect.detectFootprints(warpDiffExp, doSmooth=False, clearMask=True)
2301 fpSet.positive.merge(fpSet.negative)
2302 footprints = fpSet.positive
2303 slateIm.set(0)
2304 spanSetList = [footprint.spans for footprint in footprints.getFootprints()]
2306 # Remove artifacts due to defects before they contribute to the epochCountImage
2307 if self.config.doPrefilterArtifacts: 2307 ↛ 2311line 2307 didn't jump to line 2311, because the condition on line 2307 was never false
2308 spanSetList = self.prefilterArtifacts(spanSetList, warpDiffExp)
2310 # Clear mask before adding prefiltered spanSets
2311 self.detect.clearMask(warpDiffExp.mask)
2312 for spans in spanSetList:
2313 spans.setImage(slateIm, 1, doClip=True)
2314 spans.setMask(warpDiffExp.mask, warpDiffExp.mask.getPlaneBitMask("DETECTED"))
2315 epochCountImage += slateIm
2317 if self.config.doFilterMorphological: 2317 ↛ 2318line 2317 didn't jump to line 2318, because the condition on line 2317 was never true
2318 maskName = self.config.streakMaskName
2319 _ = self.maskStreaks.run(warpDiffExp)
2320 streakMask = warpDiffExp.mask
2321 spanSetStreak = afwGeom.SpanSet.fromMask(streakMask,
2322 streakMask.getPlaneBitMask(maskName)).split()
2324 # PSF-Matched warps have less available area (~the matching kernel) because the calexps
2325 # undergo a second convolution. Pixels with data in the direct warp
2326 # but not in the PSF-matched warp will not have their artifacts detected.
2327 # NaNs from the PSF-matched warp therefore must be masked in the direct warp
2328 nans = numpy.where(numpy.isnan(warpDiffExp.maskedImage.image.array), 1, 0)
2329 nansMask = afwImage.makeMaskFromArray(nans.astype(afwImage.MaskPixel))
2330 nansMask.setXY0(warpDiffExp.getXY0())
2331 edgeMask = warpDiffExp.mask
2332 spanSetEdgeMask = afwGeom.SpanSet.fromMask(edgeMask,
2333 edgeMask.getPlaneBitMask("EDGE")).split()
2334 else:
2335 # If the directWarp has <1% coverage, the psfMatchedWarp can have 0% and not exist
2336 # In this case, mask the whole epoch
2337 nansMask = afwImage.MaskX(coaddBBox, 1)
2338 spanSetList = []
2339 spanSetEdgeMask = []
2340 spanSetStreak = []
2342 spanSetNoDataMask = afwGeom.SpanSet.fromMask(nansMask).split()
2344 spanSetNoDataMaskList.append(spanSetNoDataMask)
2345 spanSetArtifactList.append(spanSetList)
2346 spanSetEdgeList.append(spanSetEdgeMask)
2347 if self.config.doFilterMorphological: 2347 ↛ 2348line 2347 didn't jump to line 2348, because the condition on line 2347 was never true
2348 spanSetBadMorphoList.append(spanSetStreak)
2350 if lsstDebug.Info(__name__).saveCountIm: 2350 ↛ 2351line 2350 didn't jump to line 2351, because the condition on line 2350 was never true
2351 path = self._dataRef2DebugPath("epochCountIm", tempExpRefList[0], coaddLevel=True)
2352 epochCountImage.writeFits(path)
2354 for i, spanSetList in enumerate(spanSetArtifactList):
2355 if spanSetList:
2356 filteredSpanSetList = self.filterArtifacts(spanSetList, epochCountImage, nImage,
2357 templateFootprints)
2358 spanSetArtifactList[i] = filteredSpanSetList
2359 if self.config.doFilterMorphological: 2359 ↛ 2360line 2359 didn't jump to line 2360, because the condition on line 2359 was never true
2360 spanSetArtifactList[i] += spanSetBadMorphoList[i]
2362 altMasks = []
2363 for artifacts, noData, edge in zip(spanSetArtifactList, spanSetNoDataMaskList, spanSetEdgeList):
2364 altMasks.append({'CLIPPED': artifacts,
2365 'NO_DATA': noData,
2366 'EDGE': edge})
2367 return altMasks
2369 def prefilterArtifacts(self, spanSetList, exp):
2370 """Remove artifact candidates covered by bad mask plane.
2372 Any future editing of the candidate list that does not depend on
2373 temporal information should go in this method.
2375 Parameters
2376 ----------
2377 spanSetList : `list`
2378 List of SpanSets representing artifact candidates.
2379 exp : `lsst.afw.image.Exposure`
2380 Exposure containing mask planes used to prefilter.
2382 Returns
2383 -------
2384 returnSpanSetList : `list`
2385 List of SpanSets with artifacts.
2386 """
2387 badPixelMask = exp.mask.getPlaneBitMask(self.config.prefilterArtifactsMaskPlanes)
2388 goodArr = (exp.mask.array & badPixelMask) == 0
2389 returnSpanSetList = []
2390 bbox = exp.getBBox()
2391 x0, y0 = exp.getXY0()
2392 for i, span in enumerate(spanSetList):
2393 y, x = span.clippedTo(bbox).indices()
2394 yIndexLocal = numpy.array(y) - y0
2395 xIndexLocal = numpy.array(x) - x0
2396 goodRatio = numpy.count_nonzero(goodArr[yIndexLocal, xIndexLocal])/span.getArea()
2397 if goodRatio > self.config.prefilterArtifactsRatio: 2397 ↛ 2392line 2397 didn't jump to line 2392, because the condition on line 2397 was never false
2398 returnSpanSetList.append(span)
2399 return returnSpanSetList
2401 def filterArtifacts(self, spanSetList, epochCountImage, nImage, footprintsToExclude=None):
2402 """Filter artifact candidates.
2404 Parameters
2405 ----------
2406 spanSetList : `list`
2407 List of SpanSets representing artifact candidates.
2408 epochCountImage : `lsst.afw.image.Image`
2409 Image of accumulated number of warpDiff detections.
2410 nImage : `lsst.afw.image.Image`
2411 Image of the accumulated number of total epochs contributing.
2413 Returns
2414 -------
2415 maskSpanSetList : `list`
2416 List of SpanSets with artifacts.
2417 """
2419 maskSpanSetList = []
2420 x0, y0 = epochCountImage.getXY0()
2421 for i, span in enumerate(spanSetList):
2422 y, x = span.indices()
2423 yIdxLocal = [y1 - y0 for y1 in y]
2424 xIdxLocal = [x1 - x0 for x1 in x]
2425 outlierN = epochCountImage.array[yIdxLocal, xIdxLocal]
2426 totalN = nImage.array[yIdxLocal, xIdxLocal]
2428 # effectiveMaxNumEpochs is broken line (fraction of N) with characteristic config.maxNumEpochs
2429 effMaxNumEpochsHighN = (self.config.maxNumEpochs
2430 + self.config.maxFractionEpochsHigh*numpy.mean(totalN))
2431 effMaxNumEpochsLowN = self.config.maxFractionEpochsLow * numpy.mean(totalN)
2432 effectiveMaxNumEpochs = int(min(effMaxNumEpochsLowN, effMaxNumEpochsHighN))
2433 nPixelsBelowThreshold = numpy.count_nonzero((outlierN > 0)
2434 & (outlierN <= effectiveMaxNumEpochs))
2435 percentBelowThreshold = nPixelsBelowThreshold / len(outlierN)
2436 if percentBelowThreshold > self.config.spatialThreshold:
2437 maskSpanSetList.append(span)
2439 if self.config.doPreserveContainedBySource and footprintsToExclude is not None: 2439 ↛ 2452line 2439 didn't jump to line 2452, because the condition on line 2439 was never false
2440 # If a candidate is contained by a footprint on the template coadd, do not clip
2441 filteredMaskSpanSetList = []
2442 for span in maskSpanSetList:
2443 doKeep = True
2444 for footprint in footprintsToExclude.positive.getFootprints():
2445 if footprint.spans.contains(span):
2446 doKeep = False
2447 break
2448 if doKeep:
2449 filteredMaskSpanSetList.append(span)
2450 maskSpanSetList = filteredMaskSpanSetList
2452 return maskSpanSetList
2454 def _readAndComputeWarpDiff(self, warpRef, imageScaler, templateCoadd):
2455 """Fetch a warp from the butler and return a warpDiff.
2457 Parameters
2458 ----------
2459 warpRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
2460 Butler dataRef for the warp.
2461 imageScaler : `lsst.pipe.tasks.scaleZeroPoint.ImageScaler`
2462 An image scaler object.
2463 templateCoadd : `lsst.afw.image.Exposure`
2464 Exposure to be substracted from the scaled warp.
2466 Returns
2467 -------
2468 warp : `lsst.afw.image.Exposure`
2469 Exposure of the image difference between the warp and template.
2470 """
2472 # If the PSF-Matched warp did not exist for this direct warp
2473 # None is holding its place to maintain order in Gen 3
2474 if warpRef is None:
2475 return None
2476 # Warp comparison must use PSF-Matched Warps regardless of requested coadd warp type
2477 warpName = self.getTempExpDatasetName('psfMatched')
2478 if not isinstance(warpRef, DeferredDatasetHandle): 2478 ↛ 2482line 2478 didn't jump to line 2482, because the condition on line 2478 was never false
2479 if not warpRef.datasetExists(warpName): 2479 ↛ 2480line 2479 didn't jump to line 2480, because the condition on line 2479 was never true
2480 self.log.warn("Could not find %s %s; skipping it", warpName, warpRef.dataId)
2481 return None
2482 warp = warpRef.get(datasetType=warpName, immediate=True)
2483 # direct image scaler OK for PSF-matched Warp
2484 imageScaler.scaleMaskedImage(warp.getMaskedImage())
2485 mi = warp.getMaskedImage()
2486 if self.config.doScaleWarpVariance: 2486 ↛ 2491line 2486 didn't jump to line 2491, because the condition on line 2486 was never false
2487 try:
2488 self.scaleWarpVariance.run(mi)
2489 except Exception as exc:
2490 self.log.warn("Unable to rescale variance of warp (%s); leaving it as-is" % (exc,))
2491 mi -= templateCoadd.getMaskedImage()
2492 return warp
2494 def _dataRef2DebugPath(self, prefix, warpRef, coaddLevel=False):
2495 """Return a path to which to write debugging output.
2497 Creates a hyphen-delimited string of dataId values for simple filenames.
2499 Parameters
2500 ----------
2501 prefix : `str`
2502 Prefix for filename.
2503 warpRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
2504 Butler dataRef to make the path from.
2505 coaddLevel : `bool`, optional.
2506 If True, include only coadd-level keys (e.g., 'tract', 'patch',
2507 'filter', but no 'visit').
2509 Returns
2510 -------
2511 result : `str`
2512 Path for debugging output.
2513 """
2514 if coaddLevel:
2515 keys = warpRef.getButler().getKeys(self.getCoaddDatasetName(self.warpType))
2516 else:
2517 keys = warpRef.dataId.keys()
2518 keyList = sorted(keys, reverse=True)
2519 directory = lsstDebug.Info(__name__).figPath if lsstDebug.Info(__name__).figPath else "."
2520 filename = "%s-%s.fits" % (prefix, '-'.join([str(warpRef.dataId[k]) for k in keyList]))
2521 return os.path.join(directory, filename)