Coverage for python/lsst/pipe/tasks/assembleCoadd.py : 14%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of pipe_tasks.
2#
3# LSST Data Management System
4# This product includes software developed by the
5# LSST Project (http://www.lsst.org/).
6# See COPYRIGHT file at the top of the source tree.
7#
8# This program is free software: you can redistribute it and/or modify
9# it under the terms of the GNU General Public License as published by
10# the Free Software Foundation, either version 3 of the License, or
11# (at your option) any later version.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the LSST License Statement and
19# the GNU General Public License along with this program. If not,
20# see <https://www.lsstcorp.org/LegalNotices/>.
21#
22import os
23import copy
24import numpy
25import warnings
26import lsst.pex.config as pexConfig
27import lsst.pex.exceptions as pexExceptions
28import lsst.geom as geom
29import lsst.afw.geom as afwGeom
30import lsst.afw.image as afwImage
31import lsst.afw.math as afwMath
32import lsst.afw.table as afwTable
33import lsst.afw.detection as afwDet
34import lsst.coadd.utils as coaddUtils
35import lsst.pipe.base as pipeBase
36import lsst.meas.algorithms as measAlg
37import lsst.log as log
38import lsstDebug
39import lsst.utils as utils
40from lsst.skymap import BaseSkyMap
41from .coaddBase import CoaddBaseTask, SelectDataIdContainer, makeSkyInfo, makeCoaddSuffix, reorderAndPadList
42from .interpImage import InterpImageTask
43from .scaleZeroPoint import ScaleZeroPointTask
44from .coaddHelpers import groupPatchExposures, getGroupDataRef
45from .scaleVariance import ScaleVarianceTask
46from .maskStreaks import MaskStreaksTask
47from lsst.meas.algorithms import SourceDetectionTask
48from lsst.daf.butler import DeferredDatasetHandle
50__all__ = ["AssembleCoaddTask", "AssembleCoaddConnections", "AssembleCoaddConfig",
51 "SafeClipAssembleCoaddTask", "SafeClipAssembleCoaddConfig",
52 "CompareWarpAssembleCoaddTask", "CompareWarpAssembleCoaddConfig"]
55class AssembleCoaddConnections(pipeBase.PipelineTaskConnections,
56 dimensions=("tract", "patch", "band", "skymap"),
57 defaultTemplates={"inputCoaddName": "deep",
58 "outputCoaddName": "deep",
59 "warpType": "direct",
60 "warpTypeSuffix": "",
61 "fakesType": ""}):
62 inputWarps = pipeBase.connectionTypes.Input(
63 doc=("Input list of warps to be assemebled i.e. stacked."
64 "WarpType (e.g. direct, psfMatched) is controlled by the warpType config parameter"),
65 name="{inputCoaddName}Coadd_{warpType}Warp",
66 storageClass="ExposureF",
67 dimensions=("tract", "patch", "skymap", "visit", "instrument"),
68 deferLoad=True,
69 multiple=True
70 )
71 skyMap = pipeBase.connectionTypes.Input(
72 doc="Input definition of geometry/bbox and projection/wcs for coadded exposures",
73 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
74 storageClass="SkyMap",
75 dimensions=("skymap", ),
76 )
77 selectedVisits = pipeBase.connectionTypes.Input(
78 doc="Selected visits to be coadded.",
79 name="{outputCoaddName}VisitsDict",
80 storageClass="StructuredDataDict",
81 dimensions=("instrument", "tract", "patch", "skymap", "band")
82 )
83 brightObjectMask = pipeBase.connectionTypes.PrerequisiteInput(
84 doc=("Input Bright Object Mask mask produced with external catalogs to be applied to the mask plane"
85 " BRIGHT_OBJECT."),
86 name="brightObjectMask",
87 storageClass="ObjectMaskCatalog",
88 dimensions=("tract", "patch", "skymap", "band"),
89 )
90 coaddExposure = pipeBase.connectionTypes.Output(
91 doc="Output coadded exposure, produced by stacking input warps",
92 name="{fakesType}{outputCoaddName}Coadd{warpTypeSuffix}",
93 storageClass="ExposureF",
94 dimensions=("tract", "patch", "skymap", "band"),
95 )
96 nImage = pipeBase.connectionTypes.Output(
97 doc="Output image of number of input images per pixel",
98 name="{outputCoaddName}Coadd_nImage",
99 storageClass="ImageU",
100 dimensions=("tract", "patch", "skymap", "band"),
101 )
103 def __init__(self, *, config=None):
104 super().__init__(config=config)
106 # Override the connection's name template with config to replicate Gen2 behavior
107 # This duplicates some of the logic in the base class, due to wanting Gen2 and
108 # Gen3 configs to stay in sync. This should be removed when gen2 is deprecated
109 templateValues = {name: getattr(config.connections, name) for name in self.defaultTemplates}
110 templateValues['warpType'] = config.warpType
111 templateValues['warpTypeSuffix'] = makeCoaddSuffix(config.warpType)
112 if config.hasFakes:
113 templateValues['fakesType'] = "_fakes"
114 self._nameOverrides = {name: getattr(config.connections, name).format(**templateValues)
115 for name in self.allConnections}
116 self._typeNameToVarName = {v: k for k, v in self._nameOverrides.items()}
117 # End code to remove after deprecation
119 if not config.doMaskBrightObjects:
120 self.prerequisiteInputs.remove("brightObjectMask")
122 if not config.doSelectVisits:
123 self.inputs.remove("selectedVisits")
125 if not config.doNImage:
126 self.outputs.remove("nImage")
129class AssembleCoaddConfig(CoaddBaseTask.ConfigClass, pipeBase.PipelineTaskConfig,
130 pipelineConnections=AssembleCoaddConnections):
131 """Configuration parameters for the `AssembleCoaddTask`.
133 Notes
134 -----
135 The `doMaskBrightObjects` and `brightObjectMaskName` configuration options
136 only set the bitplane config.brightObjectMaskName. To make this useful you
137 *must* also configure the flags.pixel algorithm, for example by adding
139 .. code-block:: none
141 config.measurement.plugins["base_PixelFlags"].masksFpCenter.append("BRIGHT_OBJECT")
142 config.measurement.plugins["base_PixelFlags"].masksFpAnywhere.append("BRIGHT_OBJECT")
144 to your measureCoaddSources.py and forcedPhotCoadd.py config overrides.
145 """
146 warpType = pexConfig.Field(
147 doc="Warp name: one of 'direct' or 'psfMatched'",
148 dtype=str,
149 default="direct",
150 )
151 subregionSize = pexConfig.ListField(
152 dtype=int,
153 doc="Width, height of stack subregion size; "
154 "make small enough that a full stack of images will fit into memory at once.",
155 length=2,
156 default=(2000, 2000),
157 )
158 statistic = pexConfig.Field(
159 dtype=str,
160 doc="Main stacking statistic for aggregating over the epochs.",
161 default="MEANCLIP",
162 )
163 doSigmaClip = pexConfig.Field(
164 dtype=bool,
165 doc="Perform sigma clipped outlier rejection with MEANCLIP statistic? (DEPRECATED)",
166 default=False,
167 )
168 sigmaClip = pexConfig.Field(
169 dtype=float,
170 doc="Sigma for outlier rejection; ignored if non-clipping statistic selected.",
171 default=3.0,
172 )
173 clipIter = pexConfig.Field(
174 dtype=int,
175 doc="Number of iterations of outlier rejection; ignored if non-clipping statistic selected.",
176 default=2,
177 )
178 calcErrorFromInputVariance = pexConfig.Field(
179 dtype=bool,
180 doc="Calculate coadd variance from input variance by stacking statistic."
181 "Passed to StatisticsControl.setCalcErrorFromInputVariance()",
182 default=True,
183 )
184 scaleZeroPoint = pexConfig.ConfigurableField(
185 target=ScaleZeroPointTask,
186 doc="Task to adjust the photometric zero point of the coadd temp exposures",
187 )
188 doInterp = pexConfig.Field(
189 doc="Interpolate over NaN pixels? Also extrapolate, if necessary, but the results are ugly.",
190 dtype=bool,
191 default=True,
192 )
193 interpImage = pexConfig.ConfigurableField(
194 target=InterpImageTask,
195 doc="Task to interpolate (and extrapolate) over NaN pixels",
196 )
197 doWrite = pexConfig.Field(
198 doc="Persist coadd?",
199 dtype=bool,
200 default=True,
201 )
202 doNImage = pexConfig.Field(
203 doc="Create image of number of contributing exposures for each pixel",
204 dtype=bool,
205 default=False,
206 )
207 doUsePsfMatchedPolygons = pexConfig.Field(
208 doc="Use ValidPolygons from shrunk Psf-Matched Calexps? Should be set to True by CompareWarp only.",
209 dtype=bool,
210 default=False,
211 )
212 maskPropagationThresholds = pexConfig.DictField(
213 keytype=str,
214 itemtype=float,
215 doc=("Threshold (in fractional weight) of rejection at which we propagate a mask plane to "
216 "the coadd; that is, we set the mask bit on the coadd if the fraction the rejected frames "
217 "would have contributed exceeds this value."),
218 default={"SAT": 0.1},
219 )
220 removeMaskPlanes = pexConfig.ListField(dtype=str, default=["NOT_DEBLENDED"],
221 doc="Mask planes to remove before coadding")
222 doMaskBrightObjects = pexConfig.Field(dtype=bool, default=False,
223 doc="Set mask and flag bits for bright objects?")
224 brightObjectMaskName = pexConfig.Field(dtype=str, default="BRIGHT_OBJECT",
225 doc="Name of mask bit used for bright objects")
226 coaddPsf = pexConfig.ConfigField(
227 doc="Configuration for CoaddPsf",
228 dtype=measAlg.CoaddPsfConfig,
229 )
230 doAttachTransmissionCurve = pexConfig.Field(
231 dtype=bool, default=False, optional=False,
232 doc=("Attach a piecewise TransmissionCurve for the coadd? "
233 "(requires all input Exposures to have TransmissionCurves).")
234 )
235 hasFakes = pexConfig.Field(
236 dtype=bool,
237 default=False,
238 doc="Should be set to True if fake sources have been inserted into the input data."
239 )
240 doSelectVisits = pexConfig.Field(
241 doc="Coadd only visits selected by a SelectVisitsTask",
242 dtype=bool,
243 default=False,
244 )
246 def setDefaults(self):
247 super().setDefaults()
248 self.badMaskPlanes = ["NO_DATA", "BAD", "SAT", "EDGE"]
250 def validate(self):
251 super().validate()
252 if self.doPsfMatch:
253 # Backwards compatibility.
254 # Configs do not have loggers
255 log.warn("Config doPsfMatch deprecated. Setting warpType='psfMatched'")
256 self.warpType = 'psfMatched'
257 if self.doSigmaClip and self.statistic != "MEANCLIP":
258 log.warn('doSigmaClip deprecated. To replicate behavior, setting statistic to "MEANCLIP"')
259 self.statistic = "MEANCLIP"
260 if self.doInterp and self.statistic not in ['MEAN', 'MEDIAN', 'MEANCLIP', 'VARIANCE', 'VARIANCECLIP']:
261 raise ValueError("Must set doInterp=False for statistic=%s, which does not "
262 "compute and set a non-zero coadd variance estimate." % (self.statistic))
264 unstackableStats = ['NOTHING', 'ERROR', 'ORMASK']
265 if not hasattr(afwMath.Property, self.statistic) or self.statistic in unstackableStats:
266 stackableStats = [str(k) for k in afwMath.Property.__members__.keys()
267 if str(k) not in unstackableStats]
268 raise ValueError("statistic %s is not allowed. Please choose one of %s."
269 % (self.statistic, stackableStats))
272class AssembleCoaddTask(CoaddBaseTask, pipeBase.PipelineTask):
273 """Assemble a coadded image from a set of warps (coadded temporary exposures).
275 We want to assemble a coadded image from a set of Warps (also called
276 coadded temporary exposures or ``coaddTempExps``).
277 Each input Warp covers a patch on the sky and corresponds to a single
278 run/visit/exposure of the covered patch. We provide the task with a list
279 of Warps (``selectDataList``) from which it selects Warps that cover the
280 specified patch (pointed at by ``dataRef``).
281 Each Warp that goes into a coadd will typically have an independent
282 photometric zero-point. Therefore, we must scale each Warp to set it to
283 a common photometric zeropoint. WarpType may be one of 'direct' or
284 'psfMatched', and the boolean configs `config.makeDirect` and
285 `config.makePsfMatched` set which of the warp types will be coadded.
286 The coadd is computed as a mean with optional outlier rejection.
287 Criteria for outlier rejection are set in `AssembleCoaddConfig`.
288 Finally, Warps can have bad 'NaN' pixels which received no input from the
289 source calExps. We interpolate over these bad (NaN) pixels.
291 `AssembleCoaddTask` uses several sub-tasks. These are
293 - `ScaleZeroPointTask`
294 - create and use an ``imageScaler`` object to scale the photometric zeropoint for each Warp
295 - `InterpImageTask`
296 - interpolate across bad pixels (NaN) in the final coadd
298 You can retarget these subtasks if you wish.
300 Notes
301 -----
302 The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a
303 flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``; see
304 `baseDebug` for more about ``debug.py`` files. `AssembleCoaddTask` has
305 no debug variables of its own. Some of the subtasks may support debug
306 variables. See the documentation for the subtasks for further information.
308 Examples
309 --------
310 `AssembleCoaddTask` assembles a set of warped images into a coadded image.
311 The `AssembleCoaddTask` can be invoked by running ``assembleCoadd.py``
312 with the flag '--legacyCoadd'. Usage of assembleCoadd.py expects two
313 inputs: a data reference to the tract patch and filter to be coadded, and
314 a list of Warps to attempt to coadd. These are specified using ``--id`` and
315 ``--selectId``, respectively:
317 .. code-block:: none
319 --id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]
320 --selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]
322 Only the Warps that cover the specified tract and patch will be coadded.
323 A list of the available optional arguments can be obtained by calling
324 ``assembleCoadd.py`` with the ``--help`` command line argument:
326 .. code-block:: none
328 assembleCoadd.py --help
330 To demonstrate usage of the `AssembleCoaddTask` in the larger context of
331 multi-band processing, we will generate the HSC-I & -R band coadds from
332 HSC engineering test data provided in the ``ci_hsc`` package. To begin,
333 assuming that the lsst stack has been already set up, we must set up the
334 obs_subaru and ``ci_hsc`` packages. This defines the environment variable
335 ``$CI_HSC_DIR`` and points at the location of the package. The raw HSC
336 data live in the ``$CI_HSC_DIR/raw directory``. To begin assembling the
337 coadds, we must first
339 - processCcd
340 - process the individual ccds in $CI_HSC_RAW to produce calibrated exposures
341 - makeSkyMap
342 - create a skymap that covers the area of the sky present in the raw exposures
343 - makeCoaddTempExp
344 - warp the individual calibrated exposures to the tangent plane of the coadd
346 We can perform all of these steps by running
348 .. code-block:: none
350 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
352 This will produce warped exposures for each visit. To coadd the warped
353 data, we call assembleCoadd.py as follows:
355 .. code-block:: none
357 assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \
358 --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \
359 --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \
360 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \
361 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \
362 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \
363 --selectId visit=903988 ccd=24
365 that will process the HSC-I band data. The results are written in
366 ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``.
368 You may also choose to run:
370 .. code-block:: none
372 scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346
373 assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R \
374 --selectId visit=903334 ccd=16 --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 \
375 --selectId visit=903334 ccd=100 --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 \
376 --selectId visit=903338 ccd=18 --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 \
377 --selectId visit=903342 ccd=10 --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 \
378 --selectId visit=903344 ccd=5 --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 \
379 --selectId visit=903346 ccd=6 --selectId visit=903346 ccd=12
381 to generate the coadd for the HSC-R band if you are interested in
382 following multiBand Coadd processing as discussed in `pipeTasks_multiBand`
383 (but note that normally, one would use the `SafeClipAssembleCoaddTask`
384 rather than `AssembleCoaddTask` to make the coadd.
385 """
386 ConfigClass = AssembleCoaddConfig
387 _DefaultName = "assembleCoadd"
389 def __init__(self, *args, **kwargs):
390 # TODO: DM-17415 better way to handle previously allowed passed args e.g.`AssembleCoaddTask(config)`
391 if args:
392 argNames = ["config", "name", "parentTask", "log"]
393 kwargs.update({k: v for k, v in zip(argNames, args)})
394 warnings.warn("AssembleCoadd received positional args, and casting them as kwargs: %s. "
395 "PipelineTask will not take positional args" % argNames, FutureWarning)
397 super().__init__(**kwargs)
398 self.makeSubtask("interpImage")
399 self.makeSubtask("scaleZeroPoint")
401 if self.config.doMaskBrightObjects:
402 mask = afwImage.Mask()
403 try:
404 self.brightObjectBitmask = 1 << mask.addMaskPlane(self.config.brightObjectMaskName)
405 except pexExceptions.LsstCppException:
406 raise RuntimeError("Unable to define mask plane for bright objects; planes used are %s" %
407 mask.getMaskPlaneDict().keys())
408 del mask
410 self.warpType = self.config.warpType
412 @utils.inheritDoc(pipeBase.PipelineTask)
413 def runQuantum(self, butlerQC, inputRefs, outputRefs):
414 # Docstring to be formatted with info from PipelineTask.runQuantum
415 """
416 Notes
417 -----
418 Assemble a coadd from a set of Warps.
420 PipelineTask (Gen3) entry point to Coadd a set of Warps.
421 Analogous to `runDataRef`, it prepares all the data products to be
422 passed to `run`, and processes the results before returning a struct
423 of results to be written out. AssembleCoadd cannot fit all Warps in memory.
424 Therefore, its inputs are accessed subregion by subregion
425 by the Gen3 `DeferredDatasetHandle` that is analagous to the Gen2
426 `lsst.daf.persistence.ButlerDataRef`. Any updates to this method should
427 correspond to an update in `runDataRef` while both entry points
428 are used.
429 """
430 inputData = butlerQC.get(inputRefs)
432 # Construct skyInfo expected by run
433 # Do not remove skyMap from inputData in case makeSupplementaryDataGen3 needs it
434 skyMap = inputData["skyMap"]
435 outputDataId = butlerQC.quantum.dataId
437 inputData['skyInfo'] = makeSkyInfo(skyMap,
438 tractId=outputDataId['tract'],
439 patchId=outputDataId['patch'])
441 if self.config.doSelectVisits:
442 warpRefList = self.filterWarps(inputData['inputWarps'], inputData['selectedVisits'])
443 else:
444 warpRefList = inputData['inputWarps']
446 # Perform same middle steps as `runDataRef` does
447 inputs = self.prepareInputs(warpRefList)
448 self.log.info("Found %d %s", len(inputs.tempExpRefList),
449 self.getTempExpDatasetName(self.warpType))
450 if len(inputs.tempExpRefList) == 0:
451 self.log.warn("No coadd temporary exposures found")
452 return
454 supplementaryData = self.makeSupplementaryDataGen3(butlerQC, inputRefs, outputRefs)
455 retStruct = self.run(inputData['skyInfo'], inputs.tempExpRefList, inputs.imageScalerList,
456 inputs.weightList, supplementaryData=supplementaryData)
458 inputData.setdefault('brightObjectMask', None)
459 self.processResults(retStruct.coaddExposure, inputData['brightObjectMask'], outputDataId)
461 if self.config.doWrite:
462 butlerQC.put(retStruct, outputRefs)
463 return retStruct
465 @pipeBase.timeMethod
466 def runDataRef(self, dataRef, selectDataList=None, warpRefList=None):
467 """Assemble a coadd from a set of Warps.
469 Pipebase.CmdlineTask entry point to Coadd a set of Warps.
470 Compute weights to be applied to each Warp and
471 find scalings to match the photometric zeropoint to a reference Warp.
472 Assemble the Warps using `run`. Interpolate over NaNs and
473 optionally write the coadd to disk. Return the coadded exposure.
475 Parameters
476 ----------
477 dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
478 Data reference defining the patch for coaddition and the
479 reference Warp (if ``config.autoReference=False``).
480 Used to access the following data products:
481 - ``self.config.coaddName + "Coadd_skyMap"``
482 - ``self.config.coaddName + "Coadd_ + <warpType> + "Warp"`` (optionally)
483 - ``self.config.coaddName + "Coadd"``
484 selectDataList : `list`
485 List of data references to Calexps. Data to be coadded will be
486 selected from this list based on overlap with the patch defined
487 by dataRef, grouped by visit, and converted to a list of data
488 references to warps.
489 warpRefList : `list`
490 List of data references to Warps to be coadded.
491 Note: `warpRefList` is just the new name for `tempExpRefList`.
493 Returns
494 -------
495 retStruct : `lsst.pipe.base.Struct`
496 Result struct with components:
498 - ``coaddExposure``: coadded exposure (``Exposure``).
499 - ``nImage``: exposure count image (``Image``).
500 """
501 if selectDataList and warpRefList:
502 raise RuntimeError("runDataRef received both a selectDataList and warpRefList, "
503 "and which to use is ambiguous. Please pass only one.")
505 skyInfo = self.getSkyInfo(dataRef)
506 if warpRefList is None:
507 calExpRefList = self.selectExposures(dataRef, skyInfo, selectDataList=selectDataList)
508 if len(calExpRefList) == 0:
509 self.log.warn("No exposures to coadd")
510 return
511 self.log.info("Coadding %d exposures", len(calExpRefList))
513 warpRefList = self.getTempExpRefList(dataRef, calExpRefList)
515 inputData = self.prepareInputs(warpRefList)
516 self.log.info("Found %d %s", len(inputData.tempExpRefList),
517 self.getTempExpDatasetName(self.warpType))
518 if len(inputData.tempExpRefList) == 0:
519 self.log.warn("No coadd temporary exposures found")
520 return
522 supplementaryData = self.makeSupplementaryData(dataRef, warpRefList=inputData.tempExpRefList)
524 retStruct = self.run(skyInfo, inputData.tempExpRefList, inputData.imageScalerList,
525 inputData.weightList, supplementaryData=supplementaryData)
527 brightObjects = self.readBrightObjectMasks(dataRef) if self.config.doMaskBrightObjects else None
528 self.processResults(retStruct.coaddExposure, brightObjectMasks=brightObjects, dataId=dataRef.dataId)
530 if self.config.doWrite:
531 if self.getCoaddDatasetName(self.warpType) == "deepCoadd" and self.config.hasFakes:
532 coaddDatasetName = "fakes_" + self.getCoaddDatasetName(self.warpType)
533 else:
534 coaddDatasetName = self.getCoaddDatasetName(self.warpType)
535 self.log.info("Persisting %s" % coaddDatasetName)
536 dataRef.put(retStruct.coaddExposure, coaddDatasetName)
537 if self.config.doNImage and retStruct.nImage is not None:
538 dataRef.put(retStruct.nImage, self.getCoaddDatasetName(self.warpType) + '_nImage')
540 return retStruct
542 def processResults(self, coaddExposure, brightObjectMasks=None, dataId=None):
543 """Interpolate over missing data and mask bright stars.
545 Parameters
546 ----------
547 coaddExposure : `lsst.afw.image.Exposure`
548 The coadded exposure to process.
549 dataRef : `lsst.daf.persistence.ButlerDataRef`
550 Butler data reference for supplementary data.
551 """
552 if self.config.doInterp:
553 self.interpImage.run(coaddExposure.getMaskedImage(), planeName="NO_DATA")
554 # The variance must be positive; work around for DM-3201.
555 varArray = coaddExposure.variance.array
556 with numpy.errstate(invalid="ignore"):
557 varArray[:] = numpy.where(varArray > 0, varArray, numpy.inf)
559 if self.config.doMaskBrightObjects:
560 self.setBrightObjectMasks(coaddExposure, brightObjectMasks, dataId)
562 def makeSupplementaryData(self, dataRef, selectDataList=None, warpRefList=None):
563 """Make additional inputs to run() specific to subclasses (Gen2)
565 Duplicates interface of `runDataRef` method
566 Available to be implemented by subclasses only if they need the
567 coadd dataRef for performing preliminary processing before
568 assembling the coadd.
570 Parameters
571 ----------
572 dataRef : `lsst.daf.persistence.ButlerDataRef`
573 Butler data reference for supplementary data.
574 selectDataList : `list` (optional)
575 Optional List of data references to Calexps.
576 warpRefList : `list` (optional)
577 Optional List of data references to Warps.
578 """
579 return pipeBase.Struct()
581 def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs):
582 """Make additional inputs to run() specific to subclasses (Gen3)
584 Duplicates interface of `runQuantum` method.
585 Available to be implemented by subclasses only if they need the
586 coadd dataRef for performing preliminary processing before
587 assembling the coadd.
589 Parameters
590 ----------
591 butlerQC : `lsst.pipe.base.ButlerQuantumContext`
592 Gen3 Butler object for fetching additional data products before
593 running the Task specialized for quantum being processed
594 inputRefs : `lsst.pipe.base.InputQuantizedConnection`
595 Attributes are the names of the connections describing input dataset types.
596 Values are DatasetRefs that task consumes for corresponding dataset type.
597 DataIds are guaranteed to match data objects in ``inputData``.
598 outputRefs : `lsst.pipe.base.OutputQuantizedConnection`
599 Attributes are the names of the connections describing output dataset types.
600 Values are DatasetRefs that task is to produce
601 for corresponding dataset type.
602 """
603 return pipeBase.Struct()
605 def getTempExpRefList(self, patchRef, calExpRefList):
606 """Generate list data references corresponding to warped exposures
607 that lie within the patch to be coadded.
609 Parameters
610 ----------
611 patchRef : `dataRef`
612 Data reference for patch.
613 calExpRefList : `list`
614 List of data references for input calexps.
616 Returns
617 -------
618 tempExpRefList : `list`
619 List of Warp/CoaddTempExp data references.
620 """
621 butler = patchRef.getButler()
622 groupData = groupPatchExposures(patchRef, calExpRefList, self.getCoaddDatasetName(self.warpType),
623 self.getTempExpDatasetName(self.warpType))
624 tempExpRefList = [getGroupDataRef(butler, self.getTempExpDatasetName(self.warpType),
625 g, groupData.keys) for
626 g in groupData.groups.keys()]
627 return tempExpRefList
629 def prepareInputs(self, refList):
630 """Prepare the input warps for coaddition by measuring the weight for
631 each warp and the scaling for the photometric zero point.
633 Each Warp has its own photometric zeropoint and background variance.
634 Before coadding these Warps together, compute a scale factor to
635 normalize the photometric zeropoint and compute the weight for each Warp.
637 Parameters
638 ----------
639 refList : `list`
640 List of data references to tempExp
642 Returns
643 -------
644 result : `lsst.pipe.base.Struct`
645 Result struct with components:
647 - ``tempExprefList``: `list` of data references to tempExp.
648 - ``weightList``: `list` of weightings.
649 - ``imageScalerList``: `list` of image scalers.
650 """
651 statsCtrl = afwMath.StatisticsControl()
652 statsCtrl.setNumSigmaClip(self.config.sigmaClip)
653 statsCtrl.setNumIter(self.config.clipIter)
654 statsCtrl.setAndMask(self.getBadPixelMask())
655 statsCtrl.setNanSafe(True)
656 # compute tempExpRefList: a list of tempExpRef that actually exist
657 # and weightList: a list of the weight of the associated coadd tempExp
658 # and imageScalerList: a list of scale factors for the associated coadd tempExp
659 tempExpRefList = []
660 weightList = []
661 imageScalerList = []
662 tempExpName = self.getTempExpDatasetName(self.warpType)
663 for tempExpRef in refList:
664 # Gen3's DeferredDatasetHandles are guaranteed to exist and
665 # therefore have no datasetExists() method
666 if not isinstance(tempExpRef, DeferredDatasetHandle):
667 if not tempExpRef.datasetExists(tempExpName):
668 self.log.warn("Could not find %s %s; skipping it", tempExpName, tempExpRef.dataId)
669 continue
671 tempExp = tempExpRef.get(datasetType=tempExpName, immediate=True)
672 # Ignore any input warp that is empty of data
673 if numpy.isnan(tempExp.image.array).all():
674 continue
675 maskedImage = tempExp.getMaskedImage()
676 imageScaler = self.scaleZeroPoint.computeImageScaler(
677 exposure=tempExp,
678 dataRef=tempExpRef,
679 )
680 try:
681 imageScaler.scaleMaskedImage(maskedImage)
682 except Exception as e:
683 self.log.warn("Scaling failed for %s (skipping it): %s", tempExpRef.dataId, e)
684 continue
685 statObj = afwMath.makeStatistics(maskedImage.getVariance(), maskedImage.getMask(),
686 afwMath.MEANCLIP, statsCtrl)
687 meanVar, meanVarErr = statObj.getResult(afwMath.MEANCLIP)
688 weight = 1.0 / float(meanVar)
689 if not numpy.isfinite(weight):
690 self.log.warn("Non-finite weight for %s: skipping", tempExpRef.dataId)
691 continue
692 self.log.info("Weight of %s %s = %0.3f", tempExpName, tempExpRef.dataId, weight)
694 del maskedImage
695 del tempExp
697 tempExpRefList.append(tempExpRef)
698 weightList.append(weight)
699 imageScalerList.append(imageScaler)
701 return pipeBase.Struct(tempExpRefList=tempExpRefList, weightList=weightList,
702 imageScalerList=imageScalerList)
704 def prepareStats(self, mask=None):
705 """Prepare the statistics for coadding images.
707 Parameters
708 ----------
709 mask : `int`, optional
710 Bit mask value to exclude from coaddition.
712 Returns
713 -------
714 stats : `lsst.pipe.base.Struct`
715 Statistics structure with the following fields:
717 - ``statsCtrl``: Statistics control object for coadd
718 (`lsst.afw.math.StatisticsControl`)
719 - ``statsFlags``: Statistic for coadd (`lsst.afw.math.Property`)
720 """
721 if mask is None:
722 mask = self.getBadPixelMask()
723 statsCtrl = afwMath.StatisticsControl()
724 statsCtrl.setNumSigmaClip(self.config.sigmaClip)
725 statsCtrl.setNumIter(self.config.clipIter)
726 statsCtrl.setAndMask(mask)
727 statsCtrl.setNanSafe(True)
728 statsCtrl.setWeighted(True)
729 statsCtrl.setCalcErrorFromInputVariance(self.config.calcErrorFromInputVariance)
730 for plane, threshold in self.config.maskPropagationThresholds.items():
731 bit = afwImage.Mask.getMaskPlane(plane)
732 statsCtrl.setMaskPropagationThreshold(bit, threshold)
733 statsFlags = afwMath.stringToStatisticsProperty(self.config.statistic)
734 return pipeBase.Struct(ctrl=statsCtrl, flags=statsFlags)
736 @pipeBase.timeMethod
737 def run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
738 altMaskList=None, mask=None, supplementaryData=None):
739 """Assemble a coadd from input warps
741 Assemble the coadd using the provided list of coaddTempExps. Since
742 the full coadd covers a patch (a large area), the assembly is
743 performed over small areas on the image at a time in order to
744 conserve memory usage. Iterate over subregions within the outer
745 bbox of the patch using `assembleSubregion` to stack the corresponding
746 subregions from the coaddTempExps with the statistic specified.
747 Set the edge bits the coadd mask based on the weight map.
749 Parameters
750 ----------
751 skyInfo : `lsst.pipe.base.Struct`
752 Struct with geometric information about the patch.
753 tempExpRefList : `list`
754 List of data references to Warps (previously called CoaddTempExps).
755 imageScalerList : `list`
756 List of image scalers.
757 weightList : `list`
758 List of weights
759 altMaskList : `list`, optional
760 List of alternate masks to use rather than those stored with
761 tempExp.
762 mask : `int`, optional
763 Bit mask value to exclude from coaddition.
764 supplementaryData : lsst.pipe.base.Struct, optional
765 Struct with additional data products needed to assemble coadd.
766 Only used by subclasses that implement `makeSupplementaryData`
767 and override `run`.
769 Returns
770 -------
771 result : `lsst.pipe.base.Struct`
772 Result struct with components:
774 - ``coaddExposure``: coadded exposure (``lsst.afw.image.Exposure``).
775 - ``nImage``: exposure count image (``lsst.afw.image.Image``), if requested.
776 - ``warpRefList``: input list of refs to the warps (
777 ``lsst.daf.butler.DeferredDatasetHandle`` or
778 ``lsst.daf.persistence.ButlerDataRef``)
779 (unmodified)
780 - ``imageScalerList``: input list of image scalers (unmodified)
781 - ``weightList``: input list of weights (unmodified)
782 """
783 tempExpName = self.getTempExpDatasetName(self.warpType)
784 self.log.info("Assembling %s %s", len(tempExpRefList), tempExpName)
785 stats = self.prepareStats(mask=mask)
787 if altMaskList is None:
788 altMaskList = [None]*len(tempExpRefList)
790 coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
791 coaddExposure.setPhotoCalib(self.scaleZeroPoint.getPhotoCalib())
792 coaddExposure.getInfo().setCoaddInputs(self.inputRecorder.makeCoaddInputs())
793 self.assembleMetadata(coaddExposure, tempExpRefList, weightList)
794 coaddMaskedImage = coaddExposure.getMaskedImage()
795 subregionSizeArr = self.config.subregionSize
796 subregionSize = geom.Extent2I(subregionSizeArr[0], subregionSizeArr[1])
797 # if nImage is requested, create a zero one which can be passed to assembleSubregion
798 if self.config.doNImage:
799 nImage = afwImage.ImageU(skyInfo.bbox)
800 else:
801 nImage = None
802 for subBBox in self._subBBoxIter(skyInfo.bbox, subregionSize):
803 try:
804 self.assembleSubregion(coaddExposure, subBBox, tempExpRefList, imageScalerList,
805 weightList, altMaskList, stats.flags, stats.ctrl,
806 nImage=nImage)
807 except Exception as e:
808 self.log.fatal("Cannot compute coadd %s: %s", subBBox, e)
810 self.setInexactPsf(coaddMaskedImage.getMask())
811 # Despite the name, the following doesn't really deal with "EDGE" pixels: it identifies
812 # pixels that didn't receive any unmasked inputs (as occurs around the edge of the field).
813 coaddUtils.setCoaddEdgeBits(coaddMaskedImage.getMask(), coaddMaskedImage.getVariance())
814 return pipeBase.Struct(coaddExposure=coaddExposure, nImage=nImage,
815 warpRefList=tempExpRefList, imageScalerList=imageScalerList,
816 weightList=weightList)
818 def assembleMetadata(self, coaddExposure, tempExpRefList, weightList):
819 """Set the metadata for the coadd.
821 This basic implementation sets the filter from the first input.
823 Parameters
824 ----------
825 coaddExposure : `lsst.afw.image.Exposure`
826 The target exposure for the coadd.
827 tempExpRefList : `list`
828 List of data references to tempExp.
829 weightList : `list`
830 List of weights.
831 """
832 assert len(tempExpRefList) == len(weightList), "Length mismatch"
833 tempExpName = self.getTempExpDatasetName(self.warpType)
834 # We load a single pixel of each coaddTempExp, because we just want to get at the metadata
835 # (and we need more than just the PropertySet that contains the header), which is not possible
836 # with the current butler (see #2777).
837 bbox = geom.Box2I(coaddExposure.getBBox().getMin(), geom.Extent2I(1, 1))
839 if isinstance(tempExpRefList[0], DeferredDatasetHandle):
840 # Gen 3 API
841 tempExpList = [tempExpRef.get(parameters={'bbox': bbox}) for tempExpRef in tempExpRefList]
842 else:
843 # Gen 2 API. Delete this when Gen 2 retired
844 tempExpList = [tempExpRef.get(tempExpName + "_sub", bbox=bbox, immediate=True)
845 for tempExpRef in tempExpRefList]
846 numCcds = sum(len(tempExp.getInfo().getCoaddInputs().ccds) for tempExp in tempExpList)
848 # Set the coadd FilterLabel to the band of the first input exposure:
849 # Coadds are calibrated, so the physical label is now meaningless.
850 coaddExposure.setFilterLabel(afwImage.FilterLabel(tempExpList[0].getFilterLabel().bandLabel))
851 coaddInputs = coaddExposure.getInfo().getCoaddInputs()
852 coaddInputs.ccds.reserve(numCcds)
853 coaddInputs.visits.reserve(len(tempExpList))
855 for tempExp, weight in zip(tempExpList, weightList):
856 self.inputRecorder.addVisitToCoadd(coaddInputs, tempExp, weight)
858 if self.config.doUsePsfMatchedPolygons:
859 self.shrinkValidPolygons(coaddInputs)
861 coaddInputs.visits.sort()
862 if self.warpType == "psfMatched":
863 # The modelPsf BBox for a psfMatchedWarp/coaddTempExp was dynamically defined by
864 # ModelPsfMatchTask as the square box bounding its spatially-variable, pre-matched WarpedPsf.
865 # Likewise, set the PSF of a PSF-Matched Coadd to the modelPsf
866 # having the maximum width (sufficient because square)
867 modelPsfList = [tempExp.getPsf() for tempExp in tempExpList]
868 modelPsfWidthList = [modelPsf.computeBBox().getWidth() for modelPsf in modelPsfList]
869 psf = modelPsfList[modelPsfWidthList.index(max(modelPsfWidthList))]
870 else:
871 psf = measAlg.CoaddPsf(coaddInputs.ccds, coaddExposure.getWcs(),
872 self.config.coaddPsf.makeControl())
873 coaddExposure.setPsf(psf)
874 apCorrMap = measAlg.makeCoaddApCorrMap(coaddInputs.ccds, coaddExposure.getBBox(afwImage.PARENT),
875 coaddExposure.getWcs())
876 coaddExposure.getInfo().setApCorrMap(apCorrMap)
877 if self.config.doAttachTransmissionCurve:
878 transmissionCurve = measAlg.makeCoaddTransmissionCurve(coaddExposure.getWcs(), coaddInputs.ccds)
879 coaddExposure.getInfo().setTransmissionCurve(transmissionCurve)
881 def assembleSubregion(self, coaddExposure, bbox, tempExpRefList, imageScalerList, weightList,
882 altMaskList, statsFlags, statsCtrl, nImage=None):
883 """Assemble the coadd for a sub-region.
885 For each coaddTempExp, check for (and swap in) an alternative mask
886 if one is passed. Remove mask planes listed in
887 `config.removeMaskPlanes`. Finally, stack the actual exposures using
888 `lsst.afw.math.statisticsStack` with the statistic specified by
889 statsFlags. Typically, the statsFlag will be one of lsst.afw.math.MEAN for
890 a mean-stack or `lsst.afw.math.MEANCLIP` for outlier rejection using
891 an N-sigma clipped mean where N and iterations are specified by
892 statsCtrl. Assign the stacked subregion back to the coadd.
894 Parameters
895 ----------
896 coaddExposure : `lsst.afw.image.Exposure`
897 The target exposure for the coadd.
898 bbox : `lsst.geom.Box`
899 Sub-region to coadd.
900 tempExpRefList : `list`
901 List of data reference to tempExp.
902 imageScalerList : `list`
903 List of image scalers.
904 weightList : `list`
905 List of weights.
906 altMaskList : `list`
907 List of alternate masks to use rather than those stored with
908 tempExp, or None. Each element is dict with keys = mask plane
909 name to which to add the spans.
910 statsFlags : `lsst.afw.math.Property`
911 Property object for statistic for coadd.
912 statsCtrl : `lsst.afw.math.StatisticsControl`
913 Statistics control object for coadd.
914 nImage : `lsst.afw.image.ImageU`, optional
915 Keeps track of exposure count for each pixel.
916 """
917 self.log.debug("Computing coadd over %s", bbox)
918 tempExpName = self.getTempExpDatasetName(self.warpType)
919 coaddExposure.mask.addMaskPlane("REJECTED")
920 coaddExposure.mask.addMaskPlane("CLIPPED")
921 coaddExposure.mask.addMaskPlane("SENSOR_EDGE")
922 maskMap = self.setRejectedMaskMapping(statsCtrl)
923 clipped = afwImage.Mask.getPlaneBitMask("CLIPPED")
924 maskedImageList = []
925 if nImage is not None:
926 subNImage = afwImage.ImageU(bbox.getWidth(), bbox.getHeight())
927 for tempExpRef, imageScaler, altMask in zip(tempExpRefList, imageScalerList, altMaskList):
929 if isinstance(tempExpRef, DeferredDatasetHandle):
930 # Gen 3 API
931 exposure = tempExpRef.get(parameters={'bbox': bbox})
932 else:
933 # Gen 2 API. Delete this when Gen 2 retired
934 exposure = tempExpRef.get(tempExpName + "_sub", bbox=bbox)
936 maskedImage = exposure.getMaskedImage()
937 mask = maskedImage.getMask()
938 if altMask is not None:
939 self.applyAltMaskPlanes(mask, altMask)
940 imageScaler.scaleMaskedImage(maskedImage)
942 # Add 1 for each pixel which is not excluded by the exclude mask.
943 # In legacyCoadd, pixels may also be excluded by afwMath.statisticsStack.
944 if nImage is not None:
945 subNImage.getArray()[maskedImage.getMask().getArray() & statsCtrl.getAndMask() == 0] += 1
946 if self.config.removeMaskPlanes:
947 self.removeMaskPlanes(maskedImage)
948 maskedImageList.append(maskedImage)
950 with self.timer("stack"):
951 coaddSubregion = afwMath.statisticsStack(maskedImageList, statsFlags, statsCtrl, weightList,
952 clipped, # also set output to CLIPPED if sigma-clipped
953 maskMap)
954 coaddExposure.maskedImage.assign(coaddSubregion, bbox)
955 if nImage is not None:
956 nImage.assign(subNImage, bbox)
958 def removeMaskPlanes(self, maskedImage):
959 """Unset the mask of an image for mask planes specified in the config.
961 Parameters
962 ----------
963 maskedImage : `lsst.afw.image.MaskedImage`
964 The masked image to be modified.
965 """
966 mask = maskedImage.getMask()
967 for maskPlane in self.config.removeMaskPlanes:
968 try:
969 mask &= ~mask.getPlaneBitMask(maskPlane)
970 except pexExceptions.InvalidParameterError:
971 self.log.debug("Unable to remove mask plane %s: no mask plane with that name was found.",
972 maskPlane)
974 @staticmethod
975 def setRejectedMaskMapping(statsCtrl):
976 """Map certain mask planes of the warps to new planes for the coadd.
978 If a pixel is rejected due to a mask value other than EDGE, NO_DATA,
979 or CLIPPED, set it to REJECTED on the coadd.
980 If a pixel is rejected due to EDGE, set the coadd pixel to SENSOR_EDGE.
981 If a pixel is rejected due to CLIPPED, set the coadd pixel to CLIPPED.
983 Parameters
984 ----------
985 statsCtrl : `lsst.afw.math.StatisticsControl`
986 Statistics control object for coadd
988 Returns
989 -------
990 maskMap : `list` of `tuple` of `int`
991 A list of mappings of mask planes of the warped exposures to
992 mask planes of the coadd.
993 """
994 edge = afwImage.Mask.getPlaneBitMask("EDGE")
995 noData = afwImage.Mask.getPlaneBitMask("NO_DATA")
996 clipped = afwImage.Mask.getPlaneBitMask("CLIPPED")
997 toReject = statsCtrl.getAndMask() & (~noData) & (~edge) & (~clipped)
998 maskMap = [(toReject, afwImage.Mask.getPlaneBitMask("REJECTED")),
999 (edge, afwImage.Mask.getPlaneBitMask("SENSOR_EDGE")),
1000 (clipped, clipped)]
1001 return maskMap
1003 def applyAltMaskPlanes(self, mask, altMaskSpans):
1004 """Apply in place alt mask formatted as SpanSets to a mask.
1006 Parameters
1007 ----------
1008 mask : `lsst.afw.image.Mask`
1009 Original mask.
1010 altMaskSpans : `dict`
1011 SpanSet lists to apply. Each element contains the new mask
1012 plane name (e.g. "CLIPPED and/or "NO_DATA") as the key,
1013 and list of SpanSets to apply to the mask.
1015 Returns
1016 -------
1017 mask : `lsst.afw.image.Mask`
1018 Updated mask.
1019 """
1020 if self.config.doUsePsfMatchedPolygons:
1021 if ("NO_DATA" in altMaskSpans) and ("NO_DATA" in self.config.badMaskPlanes):
1022 # Clear away any other masks outside the validPolygons. These pixels are no longer
1023 # contributing to inexact PSFs, and will still be rejected because of NO_DATA
1024 # self.config.doUsePsfMatchedPolygons should be True only in CompareWarpAssemble
1025 # This mask-clearing step must only occur *before* applying the new masks below
1026 for spanSet in altMaskSpans['NO_DATA']:
1027 spanSet.clippedTo(mask.getBBox()).clearMask(mask, self.getBadPixelMask())
1029 for plane, spanSetList in altMaskSpans.items():
1030 maskClipValue = mask.addMaskPlane(plane)
1031 for spanSet in spanSetList:
1032 spanSet.clippedTo(mask.getBBox()).setMask(mask, 2**maskClipValue)
1033 return mask
1035 def shrinkValidPolygons(self, coaddInputs):
1036 """Shrink coaddInputs' ccds' ValidPolygons in place.
1038 Either modify each ccd's validPolygon in place, or if CoaddInputs
1039 does not have a validPolygon, create one from its bbox.
1041 Parameters
1042 ----------
1043 coaddInputs : `lsst.afw.image.coaddInputs`
1044 Original mask.
1046 """
1047 for ccd in coaddInputs.ccds:
1048 polyOrig = ccd.getValidPolygon()
1049 validPolyBBox = polyOrig.getBBox() if polyOrig else ccd.getBBox()
1050 validPolyBBox.grow(-self.config.matchingKernelSize//2)
1051 if polyOrig:
1052 validPolygon = polyOrig.intersectionSingle(validPolyBBox)
1053 else:
1054 validPolygon = afwGeom.polygon.Polygon(geom.Box2D(validPolyBBox))
1055 ccd.setValidPolygon(validPolygon)
1057 def readBrightObjectMasks(self, dataRef):
1058 """Retrieve the bright object masks.
1060 Returns None on failure.
1062 Parameters
1063 ----------
1064 dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
1065 A Butler dataRef.
1067 Returns
1068 -------
1069 result : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
1070 Bright object mask from the Butler object, or None if it cannot
1071 be retrieved.
1072 """
1073 try:
1074 return dataRef.get(datasetType="brightObjectMask", immediate=True)
1075 except Exception as e:
1076 self.log.warn("Unable to read brightObjectMask for %s: %s", dataRef.dataId, e)
1077 return None
1079 def setBrightObjectMasks(self, exposure, brightObjectMasks, dataId=None):
1080 """Set the bright object masks.
1082 Parameters
1083 ----------
1084 exposure : `lsst.afw.image.Exposure`
1085 Exposure under consideration.
1086 dataId : `lsst.daf.persistence.dataId`
1087 Data identifier dict for patch.
1088 brightObjectMasks : `lsst.afw.table`
1089 Table of bright objects to mask.
1090 """
1092 if brightObjectMasks is None:
1093 self.log.warn("Unable to apply bright object mask: none supplied")
1094 return
1095 self.log.info("Applying %d bright object masks to %s", len(brightObjectMasks), dataId)
1096 mask = exposure.getMaskedImage().getMask()
1097 wcs = exposure.getWcs()
1098 plateScale = wcs.getPixelScale().asArcseconds()
1100 for rec in brightObjectMasks:
1101 center = geom.PointI(wcs.skyToPixel(rec.getCoord()))
1102 if rec["type"] == "box":
1103 assert rec["angle"] == 0.0, ("Angle != 0 for mask object %s" % rec["id"])
1104 width = rec["width"].asArcseconds()/plateScale # convert to pixels
1105 height = rec["height"].asArcseconds()/plateScale # convert to pixels
1107 halfSize = geom.ExtentI(0.5*width, 0.5*height)
1108 bbox = geom.Box2I(center - halfSize, center + halfSize)
1110 bbox = geom.BoxI(geom.PointI(int(center[0] - 0.5*width), int(center[1] - 0.5*height)),
1111 geom.PointI(int(center[0] + 0.5*width), int(center[1] + 0.5*height)))
1112 spans = afwGeom.SpanSet(bbox)
1113 elif rec["type"] == "circle":
1114 radius = int(rec["radius"].asArcseconds()/plateScale) # convert to pixels
1115 spans = afwGeom.SpanSet.fromShape(radius, offset=center)
1116 else:
1117 self.log.warn("Unexpected region type %s at %s" % rec["type"], center)
1118 continue
1119 spans.clippedTo(mask.getBBox()).setMask(mask, self.brightObjectBitmask)
1121 def setInexactPsf(self, mask):
1122 """Set INEXACT_PSF mask plane.
1124 If any of the input images isn't represented in the coadd (due to
1125 clipped pixels or chip gaps), the `CoaddPsf` will be inexact. Flag
1126 these pixels.
1128 Parameters
1129 ----------
1130 mask : `lsst.afw.image.Mask`
1131 Coadded exposure's mask, modified in-place.
1132 """
1133 mask.addMaskPlane("INEXACT_PSF")
1134 inexactPsf = mask.getPlaneBitMask("INEXACT_PSF")
1135 sensorEdge = mask.getPlaneBitMask("SENSOR_EDGE") # chip edges (so PSF is discontinuous)
1136 clipped = mask.getPlaneBitMask("CLIPPED") # pixels clipped from coadd
1137 rejected = mask.getPlaneBitMask("REJECTED") # pixels rejected from coadd due to masks
1138 array = mask.getArray()
1139 selected = array & (sensorEdge | clipped | rejected) > 0
1140 array[selected] |= inexactPsf
1142 @classmethod
1143 def _makeArgumentParser(cls):
1144 """Create an argument parser.
1145 """
1146 parser = pipeBase.ArgumentParser(name=cls._DefaultName)
1147 parser.add_id_argument("--id", cls.ConfigClass().coaddName + "Coadd_"
1148 + cls.ConfigClass().warpType + "Warp",
1149 help="data ID, e.g. --id tract=12345 patch=1,2",
1150 ContainerClass=AssembleCoaddDataIdContainer)
1151 parser.add_id_argument("--selectId", "calexp", help="data ID, e.g. --selectId visit=6789 ccd=0..9",
1152 ContainerClass=SelectDataIdContainer)
1153 return parser
1155 @staticmethod
1156 def _subBBoxIter(bbox, subregionSize):
1157 """Iterate over subregions of a bbox.
1159 Parameters
1160 ----------
1161 bbox : `lsst.geom.Box2I`
1162 Bounding box over which to iterate.
1163 subregionSize: `lsst.geom.Extent2I`
1164 Size of sub-bboxes.
1166 Yields
1167 ------
1168 subBBox : `lsst.geom.Box2I`
1169 Next sub-bounding box of size ``subregionSize`` or smaller; each ``subBBox``
1170 is contained within ``bbox``, so it may be smaller than ``subregionSize`` at
1171 the edges of ``bbox``, but it will never be empty.
1172 """
1173 if bbox.isEmpty():
1174 raise RuntimeError("bbox %s is empty" % (bbox,))
1175 if subregionSize[0] < 1 or subregionSize[1] < 1:
1176 raise RuntimeError("subregionSize %s must be nonzero" % (subregionSize,))
1178 for rowShift in range(0, bbox.getHeight(), subregionSize[1]):
1179 for colShift in range(0, bbox.getWidth(), subregionSize[0]):
1180 subBBox = geom.Box2I(bbox.getMin() + geom.Extent2I(colShift, rowShift), subregionSize)
1181 subBBox.clip(bbox)
1182 if subBBox.isEmpty():
1183 raise RuntimeError("Bug: empty bbox! bbox=%s, subregionSize=%s, "
1184 "colShift=%s, rowShift=%s" %
1185 (bbox, subregionSize, colShift, rowShift))
1186 yield subBBox
1188 def filterWarps(self, inputs, goodVisits):
1189 """Return list of only inputRefs with visitId in goodVisits ordered by goodVisit
1191 Parameters
1192 ----------
1193 inputs : list
1194 List of `lsst.pipe.base.connections.DeferredDatasetRef` with dataId containing visit
1195 goodVisit : `dict`
1196 Dictionary with good visitIds as the keys. Value ignored.
1198 Returns:
1199 --------
1200 filteredInputs : `list`
1201 Filtered and sorted list of `lsst.pipe.base.connections.DeferredDatasetRef`
1202 """
1203 inputWarpDict = {inputRef.ref.dataId['visit']: inputRef for inputRef in inputs}
1204 filteredInputs = []
1205 for visit in goodVisits.keys():
1206 filteredInputs.append(inputWarpDict[visit])
1207 return filteredInputs
1210class AssembleCoaddDataIdContainer(pipeBase.DataIdContainer):
1211 """A version of `lsst.pipe.base.DataIdContainer` specialized for assembleCoadd.
1212 """
1214 def makeDataRefList(self, namespace):
1215 """Make self.refList from self.idList.
1217 Parameters
1218 ----------
1219 namespace
1220 Results of parsing command-line (with ``butler`` and ``log`` elements).
1221 """
1222 datasetType = namespace.config.coaddName + "Coadd"
1223 keysCoadd = namespace.butler.getKeys(datasetType=datasetType, level=self.level)
1225 for dataId in self.idList:
1226 # tract and patch are required
1227 for key in keysCoadd:
1228 if key not in dataId:
1229 raise RuntimeError("--id must include " + key)
1231 dataRef = namespace.butler.dataRef(
1232 datasetType=datasetType,
1233 dataId=dataId,
1234 )
1235 self.refList.append(dataRef)
1238def countMaskFromFootprint(mask, footprint, bitmask, ignoreMask):
1239 """Function to count the number of pixels with a specific mask in a
1240 footprint.
1242 Find the intersection of mask & footprint. Count all pixels in the mask
1243 that are in the intersection that have bitmask set but do not have
1244 ignoreMask set. Return the count.
1246 Parameters
1247 ----------
1248 mask : `lsst.afw.image.Mask`
1249 Mask to define intersection region by.
1250 footprint : `lsst.afw.detection.Footprint`
1251 Footprint to define the intersection region by.
1252 bitmask
1253 Specific mask that we wish to count the number of occurances of.
1254 ignoreMask
1255 Pixels to not consider.
1257 Returns
1258 -------
1259 result : `int`
1260 Count of number of pixels in footprint with specified mask.
1261 """
1262 bbox = footprint.getBBox()
1263 bbox.clip(mask.getBBox(afwImage.PARENT))
1264 fp = afwImage.Mask(bbox)
1265 subMask = mask.Factory(mask, bbox, afwImage.PARENT)
1266 footprint.spans.setMask(fp, bitmask)
1267 return numpy.logical_and((subMask.getArray() & fp.getArray()) > 0,
1268 (subMask.getArray() & ignoreMask) == 0).sum()
1271class SafeClipAssembleCoaddConfig(AssembleCoaddConfig, pipelineConnections=AssembleCoaddConnections):
1272 """Configuration parameters for the SafeClipAssembleCoaddTask.
1273 """
1274 clipDetection = pexConfig.ConfigurableField(
1275 target=SourceDetectionTask,
1276 doc="Detect sources on difference between unclipped and clipped coadd")
1277 minClipFootOverlap = pexConfig.Field(
1278 doc="Minimum fractional overlap of clipped footprint with visit DETECTED to be clipped",
1279 dtype=float,
1280 default=0.6
1281 )
1282 minClipFootOverlapSingle = pexConfig.Field(
1283 doc="Minimum fractional overlap of clipped footprint with visit DETECTED to be "
1284 "clipped when only one visit overlaps",
1285 dtype=float,
1286 default=0.5
1287 )
1288 minClipFootOverlapDouble = pexConfig.Field(
1289 doc="Minimum fractional overlap of clipped footprints with visit DETECTED to be "
1290 "clipped when two visits overlap",
1291 dtype=float,
1292 default=0.45
1293 )
1294 maxClipFootOverlapDouble = pexConfig.Field(
1295 doc="Maximum fractional overlap of clipped footprints with visit DETECTED when "
1296 "considering two visits",
1297 dtype=float,
1298 default=0.15
1299 )
1300 minBigOverlap = pexConfig.Field(
1301 doc="Minimum number of pixels in footprint to use DETECTED mask from the single visits "
1302 "when labeling clipped footprints",
1303 dtype=int,
1304 default=100
1305 )
1307 def setDefaults(self):
1308 """Set default values for clipDetection.
1310 Notes
1311 -----
1312 The numeric values for these configuration parameters were
1313 empirically determined, future work may further refine them.
1314 """
1315 AssembleCoaddConfig.setDefaults(self)
1316 self.clipDetection.doTempLocalBackground = False
1317 self.clipDetection.reEstimateBackground = False
1318 self.clipDetection.returnOriginalFootprints = False
1319 self.clipDetection.thresholdPolarity = "both"
1320 self.clipDetection.thresholdValue = 2
1321 self.clipDetection.nSigmaToGrow = 2
1322 self.clipDetection.minPixels = 4
1323 self.clipDetection.isotropicGrow = True
1324 self.clipDetection.thresholdType = "pixel_stdev"
1325 self.sigmaClip = 1.5
1326 self.clipIter = 3
1327 self.statistic = "MEAN"
1329 def validate(self):
1330 if self.doSigmaClip:
1331 log.warn("Additional Sigma-clipping not allowed in Safe-clipped Coadds. "
1332 "Ignoring doSigmaClip.")
1333 self.doSigmaClip = False
1334 if self.statistic != "MEAN":
1335 raise ValueError("Only MEAN statistic allowed for final stacking in SafeClipAssembleCoadd "
1336 "(%s chosen). Please set statistic to MEAN."
1337 % (self.statistic))
1338 AssembleCoaddTask.ConfigClass.validate(self)
1341class SafeClipAssembleCoaddTask(AssembleCoaddTask):
1342 """Assemble a coadded image from a set of coadded temporary exposures,
1343 being careful to clip & flag areas with potential artifacts.
1345 In ``AssembleCoaddTask``, we compute the coadd as an clipped mean (i.e.,
1346 we clip outliers). The problem with doing this is that when computing the
1347 coadd PSF at a given location, individual visit PSFs from visits with
1348 outlier pixels contribute to the coadd PSF and cannot be treated correctly.
1349 In this task, we correct for this behavior by creating a new
1350 ``badMaskPlane`` 'CLIPPED'. We populate this plane on the input
1351 coaddTempExps and the final coadd where
1353 i. difference imaging suggests that there is an outlier and
1354 ii. this outlier appears on only one or two images.
1356 Such regions will not contribute to the final coadd. Furthermore, any
1357 routine to determine the coadd PSF can now be cognizant of clipped regions.
1358 Note that the algorithm implemented by this task is preliminary and works
1359 correctly for HSC data. Parameter modifications and or considerable
1360 redesigning of the algorithm is likley required for other surveys.
1362 ``SafeClipAssembleCoaddTask`` uses a ``SourceDetectionTask``
1363 "clipDetection" subtask and also sub-classes ``AssembleCoaddTask``.
1364 You can retarget the ``SourceDetectionTask`` "clipDetection" subtask
1365 if you wish.
1367 Notes
1368 -----
1369 The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a
1370 flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``;
1371 see `baseDebug` for more about ``debug.py`` files.
1372 `SafeClipAssembleCoaddTask` has no debug variables of its own.
1373 The ``SourceDetectionTask`` "clipDetection" subtasks may support debug
1374 variables. See the documetation for `SourceDetectionTask` "clipDetection"
1375 for further information.
1377 Examples
1378 --------
1379 `SafeClipAssembleCoaddTask` assembles a set of warped ``coaddTempExp``
1380 images into a coadded image. The `SafeClipAssembleCoaddTask` is invoked by
1381 running assembleCoadd.py *without* the flag '--legacyCoadd'.
1383 Usage of ``assembleCoadd.py`` expects a data reference to the tract patch
1384 and filter to be coadded (specified using
1385 '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]')
1386 along with a list of coaddTempExps to attempt to coadd (specified using
1387 '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]').
1388 Only the coaddTempExps that cover the specified tract and patch will be
1389 coadded. A list of the available optional arguments can be obtained by
1390 calling assembleCoadd.py with the --help command line argument:
1392 .. code-block:: none
1394 assembleCoadd.py --help
1396 To demonstrate usage of the `SafeClipAssembleCoaddTask` in the larger
1397 context of multi-band processing, we will generate the HSC-I & -R band
1398 coadds from HSC engineering test data provided in the ci_hsc package.
1399 To begin, assuming that the lsst stack has been already set up, we must
1400 set up the obs_subaru and ci_hsc packages. This defines the environment
1401 variable $CI_HSC_DIR and points at the location of the package. The raw
1402 HSC data live in the ``$CI_HSC_DIR/raw`` directory. To begin assembling
1403 the coadds, we must first
1405 - ``processCcd``
1406 process the individual ccds in $CI_HSC_RAW to produce calibrated exposures
1407 - ``makeSkyMap``
1408 create a skymap that covers the area of the sky present in the raw exposures
1409 - ``makeCoaddTempExp``
1410 warp the individual calibrated exposures to the tangent plane of the coadd</DD>
1412 We can perform all of these steps by running
1414 .. code-block:: none
1416 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
1418 This will produce warped coaddTempExps for each visit. To coadd the
1419 warped data, we call ``assembleCoadd.py`` as follows:
1421 .. code-block:: none
1423 assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \
1424 --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \
1425 --selectId visit=903986 ccd=100--selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \
1426 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \
1427 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \
1428 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \
1429 --selectId visit=903988 ccd=24
1431 This will process the HSC-I band data. The results are written in
1432 ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``.
1434 You may also choose to run:
1436 .. code-block:: none
1438 scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346 nnn
1439 assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R --selectId visit=903334 ccd=16 \
1440 --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 --selectId visit=903334 ccd=100 \
1441 --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 --selectId visit=903338 ccd=18 \
1442 --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 --selectId visit=903342 ccd=10 \
1443 --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 --selectId visit=903344 ccd=5 \
1444 --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 --selectId visit=903346 ccd=6 \
1445 --selectId visit=903346 ccd=12
1447 to generate the coadd for the HSC-R band if you are interested in following
1448 multiBand Coadd processing as discussed in ``pipeTasks_multiBand``.
1449 """
1450 ConfigClass = SafeClipAssembleCoaddConfig
1451 _DefaultName = "safeClipAssembleCoadd"
1453 def __init__(self, *args, **kwargs):
1454 AssembleCoaddTask.__init__(self, *args, **kwargs)
1455 schema = afwTable.SourceTable.makeMinimalSchema()
1456 self.makeSubtask("clipDetection", schema=schema)
1458 @utils.inheritDoc(AssembleCoaddTask)
1459 @pipeBase.timeMethod
1460 def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, *args, **kwargs):
1461 """Assemble the coadd for a region.
1463 Compute the difference of coadds created with and without outlier
1464 rejection to identify coadd pixels that have outlier values in some
1465 individual visits.
1466 Detect clipped regions on the difference image and mark these regions
1467 on the one or two individual coaddTempExps where they occur if there
1468 is significant overlap between the clipped region and a source. This
1469 leaves us with a set of footprints from the difference image that have
1470 been identified as having occured on just one or two individual visits.
1471 However, these footprints were generated from a difference image. It
1472 is conceivable for a large diffuse source to have become broken up
1473 into multiple footprints acrosss the coadd difference in this process.
1474 Determine the clipped region from all overlapping footprints from the
1475 detected sources in each visit - these are big footprints.
1476 Combine the small and big clipped footprints and mark them on a new
1477 bad mask plane.
1478 Generate the coadd using `AssembleCoaddTask.run` without outlier
1479 removal. Clipped footprints will no longer make it into the coadd
1480 because they are marked in the new bad mask plane.
1482 Notes
1483 -----
1484 args and kwargs are passed but ignored in order to match the call
1485 signature expected by the parent task.
1486 """
1487 exp = self.buildDifferenceImage(skyInfo, tempExpRefList, imageScalerList, weightList)
1488 mask = exp.getMaskedImage().getMask()
1489 mask.addMaskPlane("CLIPPED")
1491 result = self.detectClip(exp, tempExpRefList)
1493 self.log.info('Found %d clipped objects', len(result.clipFootprints))
1495 maskClipValue = mask.getPlaneBitMask("CLIPPED")
1496 maskDetValue = mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE")
1497 # Append big footprints from individual Warps to result.clipSpans
1498 bigFootprints = self.detectClipBig(result.clipSpans, result.clipFootprints, result.clipIndices,
1499 result.detectionFootprints, maskClipValue, maskDetValue,
1500 exp.getBBox())
1501 # Create mask of the current clipped footprints
1502 maskClip = mask.Factory(mask.getBBox(afwImage.PARENT))
1503 afwDet.setMaskFromFootprintList(maskClip, result.clipFootprints, maskClipValue)
1505 maskClipBig = maskClip.Factory(mask.getBBox(afwImage.PARENT))
1506 afwDet.setMaskFromFootprintList(maskClipBig, bigFootprints, maskClipValue)
1507 maskClip |= maskClipBig
1509 # Assemble coadd from base class, but ignoring CLIPPED pixels
1510 badMaskPlanes = self.config.badMaskPlanes[:]
1511 badMaskPlanes.append("CLIPPED")
1512 badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes)
1513 return AssembleCoaddTask.run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
1514 result.clipSpans, mask=badPixelMask)
1516 def buildDifferenceImage(self, skyInfo, tempExpRefList, imageScalerList, weightList):
1517 """Return an exposure that contains the difference between unclipped
1518 and clipped coadds.
1520 Generate a difference image between clipped and unclipped coadds.
1521 Compute the difference image by subtracting an outlier-clipped coadd
1522 from an outlier-unclipped coadd. Return the difference image.
1524 Parameters
1525 ----------
1526 skyInfo : `lsst.pipe.base.Struct`
1527 Patch geometry information, from getSkyInfo
1528 tempExpRefList : `list`
1529 List of data reference to tempExp
1530 imageScalerList : `list`
1531 List of image scalers
1532 weightList : `list`
1533 List of weights
1535 Returns
1536 -------
1537 exp : `lsst.afw.image.Exposure`
1538 Difference image of unclipped and clipped coadd wrapped in an Exposure
1539 """
1540 config = AssembleCoaddConfig()
1541 # getattr necessary because subtasks do not survive Config.toDict()
1542 # exclude connections because the class of self.config.connections is not
1543 # the same as AssembleCoaddConfig.connections, and the connections are not
1544 # needed to run this task anyway.
1545 configIntersection = {k: getattr(self.config, k)
1546 for k, v in self.config.toDict().items()
1547 if (k in config.keys() and k != "connections")}
1548 config.update(**configIntersection)
1550 # statistic MEAN copied from self.config.statistic, but for clarity explicitly assign
1551 config.statistic = 'MEAN'
1552 task = AssembleCoaddTask(config=config)
1553 coaddMean = task.run(skyInfo, tempExpRefList, imageScalerList, weightList).coaddExposure
1555 config.statistic = 'MEANCLIP'
1556 task = AssembleCoaddTask(config=config)
1557 coaddClip = task.run(skyInfo, tempExpRefList, imageScalerList, weightList).coaddExposure
1559 coaddDiff = coaddMean.getMaskedImage().Factory(coaddMean.getMaskedImage())
1560 coaddDiff -= coaddClip.getMaskedImage()
1561 exp = afwImage.ExposureF(coaddDiff)
1562 exp.setPsf(coaddMean.getPsf())
1563 return exp
1565 def detectClip(self, exp, tempExpRefList):
1566 """Detect clipped regions on an exposure and set the mask on the
1567 individual tempExp masks.
1569 Detect footprints in the difference image after smoothing the
1570 difference image with a Gaussian kernal. Identify footprints that
1571 overlap with one or two input ``coaddTempExps`` by comparing the
1572 computed overlap fraction to thresholds set in the config. A different
1573 threshold is applied depending on the number of overlapping visits
1574 (restricted to one or two). If the overlap exceeds the thresholds,
1575 the footprint is considered "CLIPPED" and is marked as such on the
1576 coaddTempExp. Return a struct with the clipped footprints, the indices
1577 of the ``coaddTempExps`` that end up overlapping with the clipped
1578 footprints, and a list of new masks for the ``coaddTempExps``.
1580 Parameters
1581 ----------
1582 exp : `lsst.afw.image.Exposure`
1583 Exposure to run detection on.
1584 tempExpRefList : `list`
1585 List of data reference to tempExp.
1587 Returns
1588 -------
1589 result : `lsst.pipe.base.Struct`
1590 Result struct with components:
1592 - ``clipFootprints``: list of clipped footprints.
1593 - ``clipIndices``: indices for each ``clippedFootprint`` in
1594 ``tempExpRefList``.
1595 - ``clipSpans``: List of dictionaries containing spanSet lists
1596 to clip. Each element contains the new maskplane name
1597 ("CLIPPED") as the key and list of ``SpanSets`` as the value.
1598 - ``detectionFootprints``: List of DETECTED/DETECTED_NEGATIVE plane
1599 compressed into footprints.
1600 """
1601 mask = exp.getMaskedImage().getMask()
1602 maskDetValue = mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE")
1603 fpSet = self.clipDetection.detectFootprints(exp, doSmooth=True, clearMask=True)
1604 # Merge positive and negative together footprints together
1605 fpSet.positive.merge(fpSet.negative)
1606 footprints = fpSet.positive
1607 self.log.info('Found %d potential clipped objects', len(footprints.getFootprints()))
1608 ignoreMask = self.getBadPixelMask()
1610 clipFootprints = []
1611 clipIndices = []
1612 artifactSpanSets = [{'CLIPPED': list()} for _ in tempExpRefList]
1614 # for use by detectClipBig
1615 visitDetectionFootprints = []
1617 dims = [len(tempExpRefList), len(footprints.getFootprints())]
1618 overlapDetArr = numpy.zeros(dims, dtype=numpy.uint16)
1619 ignoreArr = numpy.zeros(dims, dtype=numpy.uint16)
1621 # Loop over masks once and extract/store only relevant overlap metrics and detection footprints
1622 for i, warpRef in enumerate(tempExpRefList):
1623 tmpExpMask = warpRef.get(datasetType=self.getTempExpDatasetName(self.warpType),
1624 immediate=True).getMaskedImage().getMask()
1625 maskVisitDet = tmpExpMask.Factory(tmpExpMask, tmpExpMask.getBBox(afwImage.PARENT),
1626 afwImage.PARENT, True)
1627 maskVisitDet &= maskDetValue
1628 visitFootprints = afwDet.FootprintSet(maskVisitDet, afwDet.Threshold(1))
1629 visitDetectionFootprints.append(visitFootprints)
1631 for j, footprint in enumerate(footprints.getFootprints()):
1632 ignoreArr[i, j] = countMaskFromFootprint(tmpExpMask, footprint, ignoreMask, 0x0)
1633 overlapDetArr[i, j] = countMaskFromFootprint(tmpExpMask, footprint, maskDetValue, ignoreMask)
1635 # build a list of clipped spans for each visit
1636 for j, footprint in enumerate(footprints.getFootprints()):
1637 nPixel = footprint.getArea()
1638 overlap = [] # hold the overlap with each visit
1639 indexList = [] # index of visit in global list
1640 for i in range(len(tempExpRefList)):
1641 ignore = ignoreArr[i, j]
1642 overlapDet = overlapDetArr[i, j]
1643 totPixel = nPixel - ignore
1645 # If we have more bad pixels than detection skip
1646 if ignore > overlapDet or totPixel <= 0.5*nPixel or overlapDet == 0:
1647 continue
1648 overlap.append(overlapDet/float(totPixel))
1649 indexList.append(i)
1651 overlap = numpy.array(overlap)
1652 if not len(overlap):
1653 continue
1655 keep = False # Should this footprint be marked as clipped?
1656 keepIndex = [] # Which tempExps does the clipped footprint belong to
1658 # If footprint only has one overlap use a lower threshold
1659 if len(overlap) == 1:
1660 if overlap[0] > self.config.minClipFootOverlapSingle:
1661 keep = True
1662 keepIndex = [0]
1663 else:
1664 # This is the general case where only visit should be clipped
1665 clipIndex = numpy.where(overlap > self.config.minClipFootOverlap)[0]
1666 if len(clipIndex) == 1:
1667 keep = True
1668 keepIndex = [clipIndex[0]]
1670 # Test if there are clipped objects that overlap two different visits
1671 clipIndex = numpy.where(overlap > self.config.minClipFootOverlapDouble)[0]
1672 if len(clipIndex) == 2 and len(overlap) > 3:
1673 clipIndexComp = numpy.where(overlap <= self.config.minClipFootOverlapDouble)[0]
1674 if numpy.max(overlap[clipIndexComp]) <= self.config.maxClipFootOverlapDouble:
1675 keep = True
1676 keepIndex = clipIndex
1678 if not keep:
1679 continue
1681 for index in keepIndex:
1682 globalIndex = indexList[index]
1683 artifactSpanSets[globalIndex]['CLIPPED'].append(footprint.spans)
1685 clipIndices.append(numpy.array(indexList)[keepIndex])
1686 clipFootprints.append(footprint)
1688 return pipeBase.Struct(clipFootprints=clipFootprints, clipIndices=clipIndices,
1689 clipSpans=artifactSpanSets, detectionFootprints=visitDetectionFootprints)
1691 def detectClipBig(self, clipList, clipFootprints, clipIndices, detectionFootprints,
1692 maskClipValue, maskDetValue, coaddBBox):
1693 """Return individual warp footprints for large artifacts and append
1694 them to ``clipList`` in place.
1696 Identify big footprints composed of many sources in the coadd
1697 difference that may have originated in a large diffuse source in the
1698 coadd. We do this by indentifying all clipped footprints that overlap
1699 significantly with each source in all the coaddTempExps.
1701 Parameters
1702 ----------
1703 clipList : `list`
1704 List of alt mask SpanSets with clipping information. Modified.
1705 clipFootprints : `list`
1706 List of clipped footprints.
1707 clipIndices : `list`
1708 List of which entries in tempExpClipList each footprint belongs to.
1709 maskClipValue
1710 Mask value of clipped pixels.
1711 maskDetValue
1712 Mask value of detected pixels.
1713 coaddBBox : `lsst.geom.Box`
1714 BBox of the coadd and warps.
1716 Returns
1717 -------
1718 bigFootprintsCoadd : `list`
1719 List of big footprints
1720 """
1721 bigFootprintsCoadd = []
1722 ignoreMask = self.getBadPixelMask()
1723 for index, (clippedSpans, visitFootprints) in enumerate(zip(clipList, detectionFootprints)):
1724 maskVisitDet = afwImage.MaskX(coaddBBox, 0x0)
1725 for footprint in visitFootprints.getFootprints():
1726 footprint.spans.setMask(maskVisitDet, maskDetValue)
1728 # build a mask of clipped footprints that are in this visit
1729 clippedFootprintsVisit = []
1730 for foot, clipIndex in zip(clipFootprints, clipIndices):
1731 if index not in clipIndex:
1732 continue
1733 clippedFootprintsVisit.append(foot)
1734 maskVisitClip = maskVisitDet.Factory(maskVisitDet.getBBox(afwImage.PARENT))
1735 afwDet.setMaskFromFootprintList(maskVisitClip, clippedFootprintsVisit, maskClipValue)
1737 bigFootprintsVisit = []
1738 for foot in visitFootprints.getFootprints():
1739 if foot.getArea() < self.config.minBigOverlap:
1740 continue
1741 nCount = countMaskFromFootprint(maskVisitClip, foot, maskClipValue, ignoreMask)
1742 if nCount > self.config.minBigOverlap:
1743 bigFootprintsVisit.append(foot)
1744 bigFootprintsCoadd.append(foot)
1746 for footprint in bigFootprintsVisit:
1747 clippedSpans["CLIPPED"].append(footprint.spans)
1749 return bigFootprintsCoadd
1752class CompareWarpAssembleCoaddConnections(AssembleCoaddConnections):
1753 psfMatchedWarps = pipeBase.connectionTypes.Input(
1754 doc=("PSF-Matched Warps are required by CompareWarp regardless of the coadd type requested. "
1755 "Only PSF-Matched Warps make sense for image subtraction. "
1756 "Therefore, they must be an additional declared input."),
1757 name="{inputCoaddName}Coadd_psfMatchedWarp",
1758 storageClass="ExposureF",
1759 dimensions=("tract", "patch", "skymap", "visit"),
1760 deferLoad=True,
1761 multiple=True
1762 )
1763 templateCoadd = pipeBase.connectionTypes.Output(
1764 doc=("Model of the static sky, used to find temporal artifacts. Typically a PSF-Matched, "
1765 "sigma-clipped coadd. Written if and only if assembleStaticSkyModel.doWrite=True"),
1766 name="{fakesType}{outputCoaddName}CoaddPsfMatched",
1767 storageClass="ExposureF",
1768 dimensions=("tract", "patch", "skymap", "band"),
1769 )
1771 def __init__(self, *, config=None):
1772 super().__init__(config=config)
1773 if not config.assembleStaticSkyModel.doWrite:
1774 self.outputs.remove("templateCoadd")
1775 config.validate()
1778class CompareWarpAssembleCoaddConfig(AssembleCoaddConfig,
1779 pipelineConnections=CompareWarpAssembleCoaddConnections):
1780 assembleStaticSkyModel = pexConfig.ConfigurableField(
1781 target=AssembleCoaddTask,
1782 doc="Task to assemble an artifact-free, PSF-matched Coadd to serve as a"
1783 " naive/first-iteration model of the static sky.",
1784 )
1785 detect = pexConfig.ConfigurableField(
1786 target=SourceDetectionTask,
1787 doc="Detect outlier sources on difference between each psfMatched warp and static sky model"
1788 )
1789 detectTemplate = pexConfig.ConfigurableField(
1790 target=SourceDetectionTask,
1791 doc="Detect sources on static sky model. Only used if doPreserveContainedBySource is True"
1792 )
1793 maskStreaks = pexConfig.ConfigurableField(
1794 target=MaskStreaksTask,
1795 doc="Detect streaks on difference between each psfMatched warp and static sky model. Only used if "
1796 "doFilterMorphological is True. Adds a mask plane to an exposure, with the mask plane name set by"
1797 "streakMaskName"
1798 )
1799 streakMaskName = pexConfig.Field(
1800 dtype=str,
1801 default="STREAK",
1802 doc="Name of mask bit used for streaks"
1803 )
1804 maxNumEpochs = pexConfig.Field(
1805 doc="Charactistic maximum local number of epochs/visits in which an artifact candidate can appear "
1806 "and still be masked. The effective maxNumEpochs is a broken linear function of local "
1807 "number of epochs (N): min(maxFractionEpochsLow*N, maxNumEpochs + maxFractionEpochsHigh*N). "
1808 "For each footprint detected on the image difference between the psfMatched warp and static sky "
1809 "model, if a significant fraction of pixels (defined by spatialThreshold) are residuals in more "
1810 "than the computed effective maxNumEpochs, the artifact candidate is deemed persistant rather "
1811 "than transient and not masked.",
1812 dtype=int,
1813 default=2
1814 )
1815 maxFractionEpochsLow = pexConfig.RangeField(
1816 doc="Fraction of local number of epochs (N) to use as effective maxNumEpochs for low N. "
1817 "Effective maxNumEpochs = "
1818 "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)",
1819 dtype=float,
1820 default=0.4,
1821 min=0., max=1.,
1822 )
1823 maxFractionEpochsHigh = pexConfig.RangeField(
1824 doc="Fraction of local number of epochs (N) to use as effective maxNumEpochs for high N. "
1825 "Effective maxNumEpochs = "
1826 "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)",
1827 dtype=float,
1828 default=0.03,
1829 min=0., max=1.,
1830 )
1831 spatialThreshold = pexConfig.RangeField(
1832 doc="Unitless fraction of pixels defining how much of the outlier region has to meet the "
1833 "temporal criteria. If 0, clip all. If 1, clip none.",
1834 dtype=float,
1835 default=0.5,
1836 min=0., max=1.,
1837 inclusiveMin=True, inclusiveMax=True
1838 )
1839 doScaleWarpVariance = pexConfig.Field(
1840 doc="Rescale Warp variance plane using empirical noise?",
1841 dtype=bool,
1842 default=True,
1843 )
1844 scaleWarpVariance = pexConfig.ConfigurableField(
1845 target=ScaleVarianceTask,
1846 doc="Rescale variance on warps",
1847 )
1848 doPreserveContainedBySource = pexConfig.Field(
1849 doc="Rescue artifacts from clipping that completely lie within a footprint detected"
1850 "on the PsfMatched Template Coadd. Replicates a behavior of SafeClip.",
1851 dtype=bool,
1852 default=True,
1853 )
1854 doPrefilterArtifacts = pexConfig.Field(
1855 doc="Ignore artifact candidates that are mostly covered by the bad pixel mask, "
1856 "because they will be excluded anyway. This prevents them from contributing "
1857 "to the outlier epoch count image and potentially being labeled as persistant."
1858 "'Mostly' is defined by the config 'prefilterArtifactsRatio'.",
1859 dtype=bool,
1860 default=True
1861 )
1862 prefilterArtifactsMaskPlanes = pexConfig.ListField(
1863 doc="Prefilter artifact candidates that are mostly covered by these bad mask planes.",
1864 dtype=str,
1865 default=('NO_DATA', 'BAD', 'SAT', 'SUSPECT'),
1866 )
1867 prefilterArtifactsRatio = pexConfig.Field(
1868 doc="Prefilter artifact candidates with less than this fraction overlapping good pixels",
1869 dtype=float,
1870 default=0.05
1871 )
1872 doFilterMorphological = pexConfig.Field(
1873 doc="Filter artifact candidates based on morphological criteria, i.g. those that appear to "
1874 "be streaks.",
1875 dtype=bool,
1876 default=False
1877 )
1879 def setDefaults(self):
1880 AssembleCoaddConfig.setDefaults(self)
1881 self.statistic = 'MEAN'
1882 self.doUsePsfMatchedPolygons = True
1884 # Real EDGE removed by psfMatched NO_DATA border half the width of the matching kernel
1885 # CompareWarp applies psfMatched EDGE pixels to directWarps before assembling
1886 if "EDGE" in self.badMaskPlanes:
1887 self.badMaskPlanes.remove('EDGE')
1888 self.removeMaskPlanes.append('EDGE')
1889 self.assembleStaticSkyModel.badMaskPlanes = ["NO_DATA", ]
1890 self.assembleStaticSkyModel.warpType = 'psfMatched'
1891 self.assembleStaticSkyModel.connections.warpType = 'psfMatched'
1892 self.assembleStaticSkyModel.statistic = 'MEANCLIP'
1893 self.assembleStaticSkyModel.sigmaClip = 2.5
1894 self.assembleStaticSkyModel.clipIter = 3
1895 self.assembleStaticSkyModel.calcErrorFromInputVariance = False
1896 self.assembleStaticSkyModel.doWrite = False
1897 self.detect.doTempLocalBackground = False
1898 self.detect.reEstimateBackground = False
1899 self.detect.returnOriginalFootprints = False
1900 self.detect.thresholdPolarity = "both"
1901 self.detect.thresholdValue = 5
1902 self.detect.minPixels = 4
1903 self.detect.isotropicGrow = True
1904 self.detect.thresholdType = "pixel_stdev"
1905 self.detect.nSigmaToGrow = 0.4
1906 # The default nSigmaToGrow for SourceDetectionTask is already 2.4,
1907 # Explicitly restating because ratio with detect.nSigmaToGrow matters
1908 self.detectTemplate.nSigmaToGrow = 2.4
1909 self.detectTemplate.doTempLocalBackground = False
1910 self.detectTemplate.reEstimateBackground = False
1911 self.detectTemplate.returnOriginalFootprints = False
1913 def validate(self):
1914 super().validate()
1915 if self.assembleStaticSkyModel.doNImage:
1916 raise ValueError("No dataset type exists for a PSF-Matched Template N Image."
1917 "Please set assembleStaticSkyModel.doNImage=False")
1919 if self.assembleStaticSkyModel.doWrite and (self.warpType == self.assembleStaticSkyModel.warpType):
1920 raise ValueError("warpType (%s) == assembleStaticSkyModel.warpType (%s) and will compete for "
1921 "the same dataset name. Please set assembleStaticSkyModel.doWrite to False "
1922 "or warpType to 'direct'. assembleStaticSkyModel.warpType should ways be "
1923 "'PsfMatched'" % (self.warpType, self.assembleStaticSkyModel.warpType))
1926class CompareWarpAssembleCoaddTask(AssembleCoaddTask):
1927 """Assemble a compareWarp coadded image from a set of warps
1928 by masking artifacts detected by comparing PSF-matched warps.
1930 In ``AssembleCoaddTask``, we compute the coadd as an clipped mean (i.e.,
1931 we clip outliers). The problem with doing this is that when computing the
1932 coadd PSF at a given location, individual visit PSFs from visits with
1933 outlier pixels contribute to the coadd PSF and cannot be treated correctly.
1934 In this task, we correct for this behavior by creating a new badMaskPlane
1935 'CLIPPED' which marks pixels in the individual warps suspected to contain
1936 an artifact. We populate this plane on the input warps by comparing
1937 PSF-matched warps with a PSF-matched median coadd which serves as a
1938 model of the static sky. Any group of pixels that deviates from the
1939 PSF-matched template coadd by more than config.detect.threshold sigma,
1940 is an artifact candidate. The candidates are then filtered to remove
1941 variable sources and sources that are difficult to subtract such as
1942 bright stars. This filter is configured using the config parameters
1943 ``temporalThreshold`` and ``spatialThreshold``. The temporalThreshold is
1944 the maximum fraction of epochs that the deviation can appear in and still
1945 be considered an artifact. The spatialThreshold is the maximum fraction of
1946 pixels in the footprint of the deviation that appear in other epochs
1947 (where other epochs is defined by the temporalThreshold). If the deviant
1948 region meets this criteria of having a significant percentage of pixels
1949 that deviate in only a few epochs, these pixels have the 'CLIPPED' bit
1950 set in the mask. These regions will not contribute to the final coadd.
1951 Furthermore, any routine to determine the coadd PSF can now be cognizant
1952 of clipped regions. Note that the algorithm implemented by this task is
1953 preliminary and works correctly for HSC data. Parameter modifications and
1954 or considerable redesigning of the algorithm is likley required for other
1955 surveys.
1957 ``CompareWarpAssembleCoaddTask`` sub-classes
1958 ``AssembleCoaddTask`` and instantiates ``AssembleCoaddTask``
1959 as a subtask to generate the TemplateCoadd (the model of the static sky).
1961 Notes
1962 -----
1963 The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a
1964 flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``; see
1965 ``baseDebug`` for more about ``debug.py`` files.
1967 This task supports the following debug variables:
1969 - ``saveCountIm``
1970 If True then save the Epoch Count Image as a fits file in the `figPath`
1971 - ``figPath``
1972 Path to save the debug fits images and figures
1974 For example, put something like:
1976 .. code-block:: python
1978 import lsstDebug
1979 def DebugInfo(name):
1980 di = lsstDebug.getInfo(name)
1981 if name == "lsst.pipe.tasks.assembleCoadd":
1982 di.saveCountIm = True
1983 di.figPath = "/desired/path/to/debugging/output/images"
1984 return di
1985 lsstDebug.Info = DebugInfo
1987 into your ``debug.py`` file and run ``assemebleCoadd.py`` with the
1988 ``--debug`` flag. Some subtasks may have their own debug variables;
1989 see individual Task documentation.
1991 Examples
1992 --------
1993 ``CompareWarpAssembleCoaddTask`` assembles a set of warped images into a
1994 coadded image. The ``CompareWarpAssembleCoaddTask`` is invoked by running
1995 ``assembleCoadd.py`` with the flag ``--compareWarpCoadd``.
1996 Usage of ``assembleCoadd.py`` expects a data reference to the tract patch
1997 and filter to be coadded (specified using
1998 '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]')
1999 along with a list of coaddTempExps to attempt to coadd (specified using
2000 '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]').
2001 Only the warps that cover the specified tract and patch will be coadded.
2002 A list of the available optional arguments can be obtained by calling
2003 ``assembleCoadd.py`` with the ``--help`` command line argument:
2005 .. code-block:: none
2007 assembleCoadd.py --help
2009 To demonstrate usage of the ``CompareWarpAssembleCoaddTask`` in the larger
2010 context of multi-band processing, we will generate the HSC-I & -R band
2011 oadds from HSC engineering test data provided in the ``ci_hsc`` package.
2012 To begin, assuming that the lsst stack has been already set up, we must
2013 set up the ``obs_subaru`` and ``ci_hsc`` packages.
2014 This defines the environment variable ``$CI_HSC_DIR`` and points at the
2015 location of the package. The raw HSC data live in the ``$CI_HSC_DIR/raw``
2016 directory. To begin assembling the coadds, we must first
2018 - processCcd
2019 process the individual ccds in $CI_HSC_RAW to produce calibrated exposures
2020 - makeSkyMap
2021 create a skymap that covers the area of the sky present in the raw exposures
2022 - makeCoaddTempExp
2023 warp the individual calibrated exposures to the tangent plane of the coadd
2025 We can perform all of these steps by running
2027 .. code-block:: none
2029 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
2031 This will produce warped ``coaddTempExps`` for each visit. To coadd the
2032 warped data, we call ``assembleCoadd.py`` as follows:
2034 .. code-block:: none
2036 assembleCoadd.py --compareWarpCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \
2037 --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \
2038 --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \
2039 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \
2040 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \
2041 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \
2042 --selectId visit=903988 ccd=24
2044 This will process the HSC-I band data. The results are written in
2045 ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``.
2046 """
2047 ConfigClass = CompareWarpAssembleCoaddConfig
2048 _DefaultName = "compareWarpAssembleCoadd"
2050 def __init__(self, *args, **kwargs):
2051 AssembleCoaddTask.__init__(self, *args, **kwargs)
2052 self.makeSubtask("assembleStaticSkyModel")
2053 detectionSchema = afwTable.SourceTable.makeMinimalSchema()
2054 self.makeSubtask("detect", schema=detectionSchema)
2055 if self.config.doPreserveContainedBySource:
2056 self.makeSubtask("detectTemplate", schema=afwTable.SourceTable.makeMinimalSchema())
2057 if self.config.doScaleWarpVariance:
2058 self.makeSubtask("scaleWarpVariance")
2059 if self.config.doFilterMorphological:
2060 self.makeSubtask("maskStreaks")
2062 @utils.inheritDoc(AssembleCoaddTask)
2063 def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs):
2064 """
2065 Generate a templateCoadd to use as a naive model of static sky to
2066 subtract from PSF-Matched warps.
2068 Returns
2069 -------
2070 result : `lsst.pipe.base.Struct`
2071 Result struct with components:
2073 - ``templateCoadd`` : coadded exposure (``lsst.afw.image.Exposure``)
2074 - ``nImage`` : N Image (``lsst.afw.image.Image``)
2075 """
2076 # Ensure that psfMatchedWarps are used as input warps for template generation
2077 staticSkyModelInputRefs = copy.deepcopy(inputRefs)
2078 staticSkyModelInputRefs.inputWarps = inputRefs.psfMatchedWarps
2080 # Because subtasks don't have connections we have to make one.
2081 # The main task's `templateCoadd` is the subtask's `coaddExposure`
2082 staticSkyModelOutputRefs = copy.deepcopy(outputRefs)
2083 if self.config.assembleStaticSkyModel.doWrite:
2084 staticSkyModelOutputRefs.coaddExposure = staticSkyModelOutputRefs.templateCoadd
2085 # Remove template coadd from both subtask's and main tasks outputs,
2086 # because it is handled by the subtask as `coaddExposure`
2087 del outputRefs.templateCoadd
2088 del staticSkyModelOutputRefs.templateCoadd
2090 # A PSF-Matched nImage does not exist as a dataset type
2091 if 'nImage' in staticSkyModelOutputRefs.keys():
2092 del staticSkyModelOutputRefs.nImage
2094 templateCoadd = self.assembleStaticSkyModel.runQuantum(butlerQC, staticSkyModelInputRefs,
2095 staticSkyModelOutputRefs)
2096 if templateCoadd is None:
2097 raise RuntimeError(self._noTemplateMessage(self.assembleStaticSkyModel.warpType))
2099 return pipeBase.Struct(templateCoadd=templateCoadd.coaddExposure,
2100 nImage=templateCoadd.nImage,
2101 warpRefList=templateCoadd.warpRefList,
2102 imageScalerList=templateCoadd.imageScalerList,
2103 weightList=templateCoadd.weightList)
2105 @utils.inheritDoc(AssembleCoaddTask)
2106 def makeSupplementaryData(self, dataRef, selectDataList=None, warpRefList=None):
2107 """
2108 Generate a templateCoadd to use as a naive model of static sky to
2109 subtract from PSF-Matched warps.
2111 Returns
2112 -------
2113 result : `lsst.pipe.base.Struct`
2114 Result struct with components:
2116 - ``templateCoadd``: coadded exposure (``lsst.afw.image.Exposure``)
2117 - ``nImage``: N Image (``lsst.afw.image.Image``)
2118 """
2119 templateCoadd = self.assembleStaticSkyModel.runDataRef(dataRef, selectDataList, warpRefList)
2120 if templateCoadd is None:
2121 raise RuntimeError(self._noTemplateMessage(self.assembleStaticSkyModel.warpType))
2123 return pipeBase.Struct(templateCoadd=templateCoadd.coaddExposure,
2124 nImage=templateCoadd.nImage,
2125 warpRefList=templateCoadd.warpRefList,
2126 imageScalerList=templateCoadd.imageScalerList,
2127 weightList=templateCoadd.weightList)
2129 def _noTemplateMessage(self, warpType):
2130 warpName = (warpType[0].upper() + warpType[1:])
2131 message = """No %(warpName)s warps were found to build the template coadd which is
2132 required to run CompareWarpAssembleCoaddTask. To continue assembling this type of coadd,
2133 first either rerun makeCoaddTempExp with config.make%(warpName)s=True or
2134 coaddDriver with config.makeCoadTempExp.make%(warpName)s=True, before assembleCoadd.
2136 Alternatively, to use another algorithm with existing warps, retarget the CoaddDriverConfig to
2137 another algorithm like:
2139 from lsst.pipe.tasks.assembleCoadd import SafeClipAssembleCoaddTask
2140 config.assemble.retarget(SafeClipAssembleCoaddTask)
2141 """ % {"warpName": warpName}
2142 return message
2144 @utils.inheritDoc(AssembleCoaddTask)
2145 @pipeBase.timeMethod
2146 def run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
2147 supplementaryData, *args, **kwargs):
2148 """Assemble the coadd.
2150 Find artifacts and apply them to the warps' masks creating a list of
2151 alternative masks with a new "CLIPPED" plane and updated "NO_DATA"
2152 plane. Then pass these alternative masks to the base class's `run`
2153 method.
2155 The input parameters ``supplementaryData`` is a `lsst.pipe.base.Struct`
2156 that must contain a ``templateCoadd`` that serves as the
2157 model of the static sky.
2158 """
2160 # Check and match the order of the supplementaryData
2161 # (PSF-matched) inputs to the order of the direct inputs,
2162 # so that the artifact mask is applied to the right warp
2163 dataIds = [ref.dataId for ref in tempExpRefList]
2164 psfMatchedDataIds = [ref.dataId for ref in supplementaryData.warpRefList]
2166 if dataIds != psfMatchedDataIds:
2167 self.log.info("Reordering and or/padding PSF-matched visit input list")
2168 supplementaryData.warpRefList = reorderAndPadList(supplementaryData.warpRefList,
2169 psfMatchedDataIds, dataIds)
2170 supplementaryData.imageScalerList = reorderAndPadList(supplementaryData.imageScalerList,
2171 psfMatchedDataIds, dataIds)
2173 # Use PSF-Matched Warps (and corresponding scalers) and coadd to find artifacts
2174 spanSetMaskList = self.findArtifacts(supplementaryData.templateCoadd,
2175 supplementaryData.warpRefList,
2176 supplementaryData.imageScalerList)
2178 badMaskPlanes = self.config.badMaskPlanes[:]
2179 badMaskPlanes.append("CLIPPED")
2180 badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes)
2182 result = AssembleCoaddTask.run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
2183 spanSetMaskList, mask=badPixelMask)
2185 # Propagate PSF-matched EDGE pixels to coadd SENSOR_EDGE and INEXACT_PSF
2186 # Psf-Matching moves the real edge inwards
2187 self.applyAltEdgeMask(result.coaddExposure.maskedImage.mask, spanSetMaskList)
2188 return result
2190 def applyAltEdgeMask(self, mask, altMaskList):
2191 """Propagate alt EDGE mask to SENSOR_EDGE AND INEXACT_PSF planes.
2193 Parameters
2194 ----------
2195 mask : `lsst.afw.image.Mask`
2196 Original mask.
2197 altMaskList : `list`
2198 List of Dicts containing ``spanSet`` lists.
2199 Each element contains the new mask plane name (e.g. "CLIPPED
2200 and/or "NO_DATA") as the key, and list of ``SpanSets`` to apply to
2201 the mask.
2202 """
2203 maskValue = mask.getPlaneBitMask(["SENSOR_EDGE", "INEXACT_PSF"])
2204 for visitMask in altMaskList:
2205 if "EDGE" in visitMask:
2206 for spanSet in visitMask['EDGE']:
2207 spanSet.clippedTo(mask.getBBox()).setMask(mask, maskValue)
2209 def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList):
2210 """Find artifacts.
2212 Loop through warps twice. The first loop builds a map with the count
2213 of how many epochs each pixel deviates from the templateCoadd by more
2214 than ``config.chiThreshold`` sigma. The second loop takes each
2215 difference image and filters the artifacts detected in each using
2216 count map to filter out variable sources and sources that are
2217 difficult to subtract cleanly.
2219 Parameters
2220 ----------
2221 templateCoadd : `lsst.afw.image.Exposure`
2222 Exposure to serve as model of static sky.
2223 tempExpRefList : `list`
2224 List of data references to warps.
2225 imageScalerList : `list`
2226 List of image scalers.
2228 Returns
2229 -------
2230 altMasks : `list`
2231 List of dicts containing information about CLIPPED
2232 (i.e., artifacts), NO_DATA, and EDGE pixels.
2233 """
2235 self.log.debug("Generating Count Image, and mask lists.")
2236 coaddBBox = templateCoadd.getBBox()
2237 slateIm = afwImage.ImageU(coaddBBox)
2238 epochCountImage = afwImage.ImageU(coaddBBox)
2239 nImage = afwImage.ImageU(coaddBBox)
2240 spanSetArtifactList = []
2241 spanSetNoDataMaskList = []
2242 spanSetEdgeList = []
2243 spanSetBadMorphoList = []
2244 badPixelMask = self.getBadPixelMask()
2246 # mask of the warp diffs should = that of only the warp
2247 templateCoadd.mask.clearAllMaskPlanes()
2249 if self.config.doPreserveContainedBySource:
2250 templateFootprints = self.detectTemplate.detectFootprints(templateCoadd)
2251 else:
2252 templateFootprints = None
2254 for warpRef, imageScaler in zip(tempExpRefList, imageScalerList):
2255 warpDiffExp = self._readAndComputeWarpDiff(warpRef, imageScaler, templateCoadd)
2256 if warpDiffExp is not None:
2257 # This nImage only approximates the final nImage because it uses the PSF-matched mask
2258 nImage.array += (numpy.isfinite(warpDiffExp.image.array)
2259 * ((warpDiffExp.mask.array & badPixelMask) == 0)).astype(numpy.uint16)
2260 fpSet = self.detect.detectFootprints(warpDiffExp, doSmooth=False, clearMask=True)
2261 fpSet.positive.merge(fpSet.negative)
2262 footprints = fpSet.positive
2263 slateIm.set(0)
2264 spanSetList = [footprint.spans for footprint in footprints.getFootprints()]
2266 # Remove artifacts due to defects before they contribute to the epochCountImage
2267 if self.config.doPrefilterArtifacts:
2268 spanSetList = self.prefilterArtifacts(spanSetList, warpDiffExp)
2270 # Clear mask before adding prefiltered spanSets
2271 self.detect.clearMask(warpDiffExp.mask)
2272 for spans in spanSetList:
2273 spans.setImage(slateIm, 1, doClip=True)
2274 spans.setMask(warpDiffExp.mask, warpDiffExp.mask.getPlaneBitMask("DETECTED"))
2275 epochCountImage += slateIm
2277 if self.config.doFilterMorphological:
2278 maskName = self.config.streakMaskName
2279 _ = self.maskStreaks.run(warpDiffExp)
2280 streakMask = warpDiffExp.mask
2281 spanSetStreak = afwGeom.SpanSet.fromMask(streakMask,
2282 streakMask.getPlaneBitMask(maskName)).split()
2284 # PSF-Matched warps have less available area (~the matching kernel) because the calexps
2285 # undergo a second convolution. Pixels with data in the direct warp
2286 # but not in the PSF-matched warp will not have their artifacts detected.
2287 # NaNs from the PSF-matched warp therefore must be masked in the direct warp
2288 nans = numpy.where(numpy.isnan(warpDiffExp.maskedImage.image.array), 1, 0)
2289 nansMask = afwImage.makeMaskFromArray(nans.astype(afwImage.MaskPixel))
2290 nansMask.setXY0(warpDiffExp.getXY0())
2291 edgeMask = warpDiffExp.mask
2292 spanSetEdgeMask = afwGeom.SpanSet.fromMask(edgeMask,
2293 edgeMask.getPlaneBitMask("EDGE")).split()
2294 else:
2295 # If the directWarp has <1% coverage, the psfMatchedWarp can have 0% and not exist
2296 # In this case, mask the whole epoch
2297 nansMask = afwImage.MaskX(coaddBBox, 1)
2298 spanSetList = []
2299 spanSetEdgeMask = []
2300 spanSetStreak = []
2302 spanSetNoDataMask = afwGeom.SpanSet.fromMask(nansMask).split()
2304 spanSetNoDataMaskList.append(spanSetNoDataMask)
2305 spanSetArtifactList.append(spanSetList)
2306 spanSetEdgeList.append(spanSetEdgeMask)
2307 if self.config.doFilterMorphological:
2308 spanSetBadMorphoList.append(spanSetStreak)
2310 if lsstDebug.Info(__name__).saveCountIm:
2311 path = self._dataRef2DebugPath("epochCountIm", tempExpRefList[0], coaddLevel=True)
2312 epochCountImage.writeFits(path)
2314 for i, spanSetList in enumerate(spanSetArtifactList):
2315 if spanSetList:
2316 filteredSpanSetList = self.filterArtifacts(spanSetList, epochCountImage, nImage,
2317 templateFootprints)
2318 spanSetArtifactList[i] = filteredSpanSetList
2319 if self.config.doFilterMorphological:
2320 spanSetArtifactList[i] += spanSetBadMorphoList[i]
2322 altMasks = []
2323 for artifacts, noData, edge in zip(spanSetArtifactList, spanSetNoDataMaskList, spanSetEdgeList):
2324 altMasks.append({'CLIPPED': artifacts,
2325 'NO_DATA': noData,
2326 'EDGE': edge})
2327 return altMasks
2329 def prefilterArtifacts(self, spanSetList, exp):
2330 """Remove artifact candidates covered by bad mask plane.
2332 Any future editing of the candidate list that does not depend on
2333 temporal information should go in this method.
2335 Parameters
2336 ----------
2337 spanSetList : `list`
2338 List of SpanSets representing artifact candidates.
2339 exp : `lsst.afw.image.Exposure`
2340 Exposure containing mask planes used to prefilter.
2342 Returns
2343 -------
2344 returnSpanSetList : `list`
2345 List of SpanSets with artifacts.
2346 """
2347 badPixelMask = exp.mask.getPlaneBitMask(self.config.prefilterArtifactsMaskPlanes)
2348 goodArr = (exp.mask.array & badPixelMask) == 0
2349 returnSpanSetList = []
2350 bbox = exp.getBBox()
2351 x0, y0 = exp.getXY0()
2352 for i, span in enumerate(spanSetList):
2353 y, x = span.clippedTo(bbox).indices()
2354 yIndexLocal = numpy.array(y) - y0
2355 xIndexLocal = numpy.array(x) - x0
2356 goodRatio = numpy.count_nonzero(goodArr[yIndexLocal, xIndexLocal])/span.getArea()
2357 if goodRatio > self.config.prefilterArtifactsRatio:
2358 returnSpanSetList.append(span)
2359 return returnSpanSetList
2361 def filterArtifacts(self, spanSetList, epochCountImage, nImage, footprintsToExclude=None):
2362 """Filter artifact candidates.
2364 Parameters
2365 ----------
2366 spanSetList : `list`
2367 List of SpanSets representing artifact candidates.
2368 epochCountImage : `lsst.afw.image.Image`
2369 Image of accumulated number of warpDiff detections.
2370 nImage : `lsst.afw.image.Image`
2371 Image of the accumulated number of total epochs contributing.
2373 Returns
2374 -------
2375 maskSpanSetList : `list`
2376 List of SpanSets with artifacts.
2377 """
2379 maskSpanSetList = []
2380 x0, y0 = epochCountImage.getXY0()
2381 for i, span in enumerate(spanSetList):
2382 y, x = span.indices()
2383 yIdxLocal = [y1 - y0 for y1 in y]
2384 xIdxLocal = [x1 - x0 for x1 in x]
2385 outlierN = epochCountImage.array[yIdxLocal, xIdxLocal]
2386 totalN = nImage.array[yIdxLocal, xIdxLocal]
2388 # effectiveMaxNumEpochs is broken line (fraction of N) with characteristic config.maxNumEpochs
2389 effMaxNumEpochsHighN = (self.config.maxNumEpochs
2390 + self.config.maxFractionEpochsHigh*numpy.mean(totalN))
2391 effMaxNumEpochsLowN = self.config.maxFractionEpochsLow * numpy.mean(totalN)
2392 effectiveMaxNumEpochs = int(min(effMaxNumEpochsLowN, effMaxNumEpochsHighN))
2393 nPixelsBelowThreshold = numpy.count_nonzero((outlierN > 0)
2394 & (outlierN <= effectiveMaxNumEpochs))
2395 percentBelowThreshold = nPixelsBelowThreshold / len(outlierN)
2396 if percentBelowThreshold > self.config.spatialThreshold:
2397 maskSpanSetList.append(span)
2399 if self.config.doPreserveContainedBySource and footprintsToExclude is not None:
2400 # If a candidate is contained by a footprint on the template coadd, do not clip
2401 filteredMaskSpanSetList = []
2402 for span in maskSpanSetList:
2403 doKeep = True
2404 for footprint in footprintsToExclude.positive.getFootprints():
2405 if footprint.spans.contains(span):
2406 doKeep = False
2407 break
2408 if doKeep:
2409 filteredMaskSpanSetList.append(span)
2410 maskSpanSetList = filteredMaskSpanSetList
2412 return maskSpanSetList
2414 def _readAndComputeWarpDiff(self, warpRef, imageScaler, templateCoadd):
2415 """Fetch a warp from the butler and return a warpDiff.
2417 Parameters
2418 ----------
2419 warpRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
2420 Butler dataRef for the warp.
2421 imageScaler : `lsst.pipe.tasks.scaleZeroPoint.ImageScaler`
2422 An image scaler object.
2423 templateCoadd : `lsst.afw.image.Exposure`
2424 Exposure to be substracted from the scaled warp.
2426 Returns
2427 -------
2428 warp : `lsst.afw.image.Exposure`
2429 Exposure of the image difference between the warp and template.
2430 """
2432 # If the PSF-Matched warp did not exist for this direct warp
2433 # None is holding its place to maintain order in Gen 3
2434 if warpRef is None:
2435 return None
2436 # Warp comparison must use PSF-Matched Warps regardless of requested coadd warp type
2437 warpName = self.getTempExpDatasetName('psfMatched')
2438 if not isinstance(warpRef, DeferredDatasetHandle):
2439 if not warpRef.datasetExists(warpName):
2440 self.log.warn("Could not find %s %s; skipping it", warpName, warpRef.dataId)
2441 return None
2442 warp = warpRef.get(datasetType=warpName, immediate=True)
2443 # direct image scaler OK for PSF-matched Warp
2444 imageScaler.scaleMaskedImage(warp.getMaskedImage())
2445 mi = warp.getMaskedImage()
2446 if self.config.doScaleWarpVariance:
2447 try:
2448 self.scaleWarpVariance.run(mi)
2449 except Exception as exc:
2450 self.log.warn("Unable to rescale variance of warp (%s); leaving it as-is" % (exc,))
2451 mi -= templateCoadd.getMaskedImage()
2452 return warp
2454 def _dataRef2DebugPath(self, prefix, warpRef, coaddLevel=False):
2455 """Return a path to which to write debugging output.
2457 Creates a hyphen-delimited string of dataId values for simple filenames.
2459 Parameters
2460 ----------
2461 prefix : `str`
2462 Prefix for filename.
2463 warpRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
2464 Butler dataRef to make the path from.
2465 coaddLevel : `bool`, optional.
2466 If True, include only coadd-level keys (e.g., 'tract', 'patch',
2467 'filter', but no 'visit').
2469 Returns
2470 -------
2471 result : `str`
2472 Path for debugging output.
2473 """
2474 if coaddLevel:
2475 keys = warpRef.getButler().getKeys(self.getCoaddDatasetName(self.warpType))
2476 else:
2477 keys = warpRef.dataId.keys()
2478 keyList = sorted(keys, reverse=True)
2479 directory = lsstDebug.Info(__name__).figPath if lsstDebug.Info(__name__).figPath else "."
2480 filename = "%s-%s.fits" % (prefix, '-'.join([str(warpRef.dataId[k]) for k in keyList]))
2481 return os.path.join(directory, filename)