Coverage for python/lsst/pipe/tasks/assembleCoadd.py : 13%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of pipe_tasks.
2#
3# LSST Data Management System
4# This product includes software developed by the
5# LSST Project (http://www.lsst.org/).
6# See COPYRIGHT file at the top of the source tree.
7#
8# This program is free software: you can redistribute it and/or modify
9# it under the terms of the GNU General Public License as published by
10# the Free Software Foundation, either version 3 of the License, or
11# (at your option) any later version.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the LSST License Statement and
19# the GNU General Public License along with this program. If not,
20# see <https://www.lsstcorp.org/LegalNotices/>.
21#
22import os
23import copy
24import numpy
25import warnings
26import lsst.pex.config as pexConfig
27import lsst.pex.exceptions as pexExceptions
28import lsst.geom as geom
29import lsst.afw.geom as afwGeom
30import lsst.afw.image as afwImage
31import lsst.afw.math as afwMath
32import lsst.afw.table as afwTable
33import lsst.afw.detection as afwDet
34import lsst.coadd.utils as coaddUtils
35import lsst.pipe.base as pipeBase
36import lsst.meas.algorithms as measAlg
37import lsst.log as log
38import lsstDebug
39import lsst.utils as utils
40from lsst.skymap import BaseSkyMap
41from .coaddBase import CoaddBaseTask, SelectDataIdContainer, makeSkyInfo, makeCoaddSuffix
42from .interpImage import InterpImageTask
43from .scaleZeroPoint import ScaleZeroPointTask
44from .coaddHelpers import groupPatchExposures, getGroupDataRef
45from .scaleVariance import ScaleVarianceTask
46from .maskStreaks import MaskStreaksTask
47from lsst.meas.algorithms import SourceDetectionTask
48from lsst.daf.butler import DeferredDatasetHandle
50__all__ = ["AssembleCoaddTask", "AssembleCoaddConnections", "AssembleCoaddConfig",
51 "SafeClipAssembleCoaddTask", "SafeClipAssembleCoaddConfig",
52 "CompareWarpAssembleCoaddTask", "CompareWarpAssembleCoaddConfig"]
55class AssembleCoaddConnections(pipeBase.PipelineTaskConnections,
56 dimensions=("tract", "patch", "band", "skymap"),
57 defaultTemplates={"inputCoaddName": "deep",
58 "outputCoaddName": "deep",
59 "warpType": "direct",
60 "warpTypeSuffix": "",
61 "fakesType": ""}):
62 inputWarps = pipeBase.connectionTypes.Input(
63 doc=("Input list of warps to be assemebled i.e. stacked."
64 "WarpType (e.g. direct, psfMatched) is controlled by the warpType config parameter"),
65 name="{inputCoaddName}Coadd_{warpType}Warp",
66 storageClass="ExposureF",
67 dimensions=("tract", "patch", "skymap", "visit", "instrument"),
68 deferLoad=True,
69 multiple=True
70 )
71 skyMap = pipeBase.connectionTypes.Input(
72 doc="Input definition of geometry/bbox and projection/wcs for coadded exposures",
73 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
74 storageClass="SkyMap",
75 dimensions=("skymap", ),
76 )
77 brightObjectMask = pipeBase.connectionTypes.PrerequisiteInput(
78 doc=("Input Bright Object Mask mask produced with external catalogs to be applied to the mask plane"
79 " BRIGHT_OBJECT."),
80 name="brightObjectMask",
81 storageClass="ObjectMaskCatalog",
82 dimensions=("tract", "patch", "skymap", "band"),
83 )
84 coaddExposure = pipeBase.connectionTypes.Output(
85 doc="Output coadded exposure, produced by stacking input warps",
86 name="{fakesType}{outputCoaddName}Coadd{warpTypeSuffix}",
87 storageClass="ExposureF",
88 dimensions=("tract", "patch", "skymap", "band"),
89 )
90 nImage = pipeBase.connectionTypes.Output(
91 doc="Output image of number of input images per pixel",
92 name="{outputCoaddName}Coadd_nImage",
93 storageClass="ImageU",
94 dimensions=("tract", "patch", "skymap", "band"),
95 )
97 def __init__(self, *, config=None):
98 super().__init__(config=config)
100 # Override the connection's name template with config to replicate Gen2 behavior
101 # This duplicates some of the logic in the base class, due to wanting Gen2 and
102 # Gen3 configs to stay in sync. This should be removed when gen2 is deprecated
103 templateValues = {name: getattr(config.connections, name) for name in self.defaultTemplates}
104 templateValues['warpType'] = config.warpType
105 templateValues['warpTypeSuffix'] = makeCoaddSuffix(config.warpType)
106 if config.hasFakes:
107 templateValues['fakesType'] = "_fakes"
108 self._nameOverrides = {name: getattr(config.connections, name).format(**templateValues)
109 for name in self.allConnections}
110 self._typeNameToVarName = {v: k for k, v in self._nameOverrides.items()}
111 # End code to remove after deprecation
113 if not config.doMaskBrightObjects:
114 self.prerequisiteInputs.remove("brightObjectMask")
116 if not config.doNImage:
117 self.outputs.remove("nImage")
120class AssembleCoaddConfig(CoaddBaseTask.ConfigClass, pipeBase.PipelineTaskConfig,
121 pipelineConnections=AssembleCoaddConnections):
122 """Configuration parameters for the `AssembleCoaddTask`.
124 Notes
125 -----
126 The `doMaskBrightObjects` and `brightObjectMaskName` configuration options
127 only set the bitplane config.brightObjectMaskName. To make this useful you
128 *must* also configure the flags.pixel algorithm, for example by adding
130 .. code-block:: none
132 config.measurement.plugins["base_PixelFlags"].masksFpCenter.append("BRIGHT_OBJECT")
133 config.measurement.plugins["base_PixelFlags"].masksFpAnywhere.append("BRIGHT_OBJECT")
135 to your measureCoaddSources.py and forcedPhotCoadd.py config overrides.
136 """
137 warpType = pexConfig.Field(
138 doc="Warp name: one of 'direct' or 'psfMatched'",
139 dtype=str,
140 default="direct",
141 )
142 subregionSize = pexConfig.ListField(
143 dtype=int,
144 doc="Width, height of stack subregion size; "
145 "make small enough that a full stack of images will fit into memory at once.",
146 length=2,
147 default=(2000, 2000),
148 )
149 statistic = pexConfig.Field(
150 dtype=str,
151 doc="Main stacking statistic for aggregating over the epochs.",
152 default="MEANCLIP",
153 )
154 doSigmaClip = pexConfig.Field(
155 dtype=bool,
156 doc="Perform sigma clipped outlier rejection with MEANCLIP statistic? (DEPRECATED)",
157 default=False,
158 )
159 sigmaClip = pexConfig.Field(
160 dtype=float,
161 doc="Sigma for outlier rejection; ignored if non-clipping statistic selected.",
162 default=3.0,
163 )
164 clipIter = pexConfig.Field(
165 dtype=int,
166 doc="Number of iterations of outlier rejection; ignored if non-clipping statistic selected.",
167 default=2,
168 )
169 calcErrorFromInputVariance = pexConfig.Field(
170 dtype=bool,
171 doc="Calculate coadd variance from input variance by stacking statistic."
172 "Passed to StatisticsControl.setCalcErrorFromInputVariance()",
173 default=True,
174 )
175 scaleZeroPoint = pexConfig.ConfigurableField(
176 target=ScaleZeroPointTask,
177 doc="Task to adjust the photometric zero point of the coadd temp exposures",
178 )
179 doInterp = pexConfig.Field(
180 doc="Interpolate over NaN pixels? Also extrapolate, if necessary, but the results are ugly.",
181 dtype=bool,
182 default=True,
183 )
184 interpImage = pexConfig.ConfigurableField(
185 target=InterpImageTask,
186 doc="Task to interpolate (and extrapolate) over NaN pixels",
187 )
188 doWrite = pexConfig.Field(
189 doc="Persist coadd?",
190 dtype=bool,
191 default=True,
192 )
193 doNImage = pexConfig.Field(
194 doc="Create image of number of contributing exposures for each pixel",
195 dtype=bool,
196 default=False,
197 )
198 doUsePsfMatchedPolygons = pexConfig.Field(
199 doc="Use ValidPolygons from shrunk Psf-Matched Calexps? Should be set to True by CompareWarp only.",
200 dtype=bool,
201 default=False,
202 )
203 maskPropagationThresholds = pexConfig.DictField(
204 keytype=str,
205 itemtype=float,
206 doc=("Threshold (in fractional weight) of rejection at which we propagate a mask plane to "
207 "the coadd; that is, we set the mask bit on the coadd if the fraction the rejected frames "
208 "would have contributed exceeds this value."),
209 default={"SAT": 0.1},
210 )
211 removeMaskPlanes = pexConfig.ListField(dtype=str, default=["NOT_DEBLENDED"],
212 doc="Mask planes to remove before coadding")
213 doMaskBrightObjects = pexConfig.Field(dtype=bool, default=False,
214 doc="Set mask and flag bits for bright objects?")
215 brightObjectMaskName = pexConfig.Field(dtype=str, default="BRIGHT_OBJECT",
216 doc="Name of mask bit used for bright objects")
217 coaddPsf = pexConfig.ConfigField(
218 doc="Configuration for CoaddPsf",
219 dtype=measAlg.CoaddPsfConfig,
220 )
221 doAttachTransmissionCurve = pexConfig.Field(
222 dtype=bool, default=False, optional=False,
223 doc=("Attach a piecewise TransmissionCurve for the coadd? "
224 "(requires all input Exposures to have TransmissionCurves).")
225 )
226 hasFakes = pexConfig.Field(
227 dtype=bool,
228 default=False,
229 doc="Should be set to True if fake sources have been inserted into the input data."
230 )
232 def setDefaults(self):
233 super().setDefaults()
234 self.badMaskPlanes = ["NO_DATA", "BAD", "SAT", "EDGE"]
236 def validate(self):
237 super().validate()
238 if self.doPsfMatch:
239 # Backwards compatibility.
240 # Configs do not have loggers
241 log.warn("Config doPsfMatch deprecated. Setting warpType='psfMatched'")
242 self.warpType = 'psfMatched'
243 if self.doSigmaClip and self.statistic != "MEANCLIP":
244 log.warn('doSigmaClip deprecated. To replicate behavior, setting statistic to "MEANCLIP"')
245 self.statistic = "MEANCLIP"
246 if self.doInterp and self.statistic not in ['MEAN', 'MEDIAN', 'MEANCLIP', 'VARIANCE', 'VARIANCECLIP']:
247 raise ValueError("Must set doInterp=False for statistic=%s, which does not "
248 "compute and set a non-zero coadd variance estimate." % (self.statistic))
250 unstackableStats = ['NOTHING', 'ERROR', 'ORMASK']
251 if not hasattr(afwMath.Property, self.statistic) or self.statistic in unstackableStats:
252 stackableStats = [str(k) for k in afwMath.Property.__members__.keys()
253 if str(k) not in unstackableStats]
254 raise ValueError("statistic %s is not allowed. Please choose one of %s."
255 % (self.statistic, stackableStats))
258class AssembleCoaddTask(CoaddBaseTask, pipeBase.PipelineTask):
259 """Assemble a coadded image from a set of warps (coadded temporary exposures).
261 We want to assemble a coadded image from a set of Warps (also called
262 coadded temporary exposures or ``coaddTempExps``).
263 Each input Warp covers a patch on the sky and corresponds to a single
264 run/visit/exposure of the covered patch. We provide the task with a list
265 of Warps (``selectDataList``) from which it selects Warps that cover the
266 specified patch (pointed at by ``dataRef``).
267 Each Warp that goes into a coadd will typically have an independent
268 photometric zero-point. Therefore, we must scale each Warp to set it to
269 a common photometric zeropoint. WarpType may be one of 'direct' or
270 'psfMatched', and the boolean configs `config.makeDirect` and
271 `config.makePsfMatched` set which of the warp types will be coadded.
272 The coadd is computed as a mean with optional outlier rejection.
273 Criteria for outlier rejection are set in `AssembleCoaddConfig`.
274 Finally, Warps can have bad 'NaN' pixels which received no input from the
275 source calExps. We interpolate over these bad (NaN) pixels.
277 `AssembleCoaddTask` uses several sub-tasks. These are
279 - `ScaleZeroPointTask`
280 - create and use an ``imageScaler`` object to scale the photometric zeropoint for each Warp
281 - `InterpImageTask`
282 - interpolate across bad pixels (NaN) in the final coadd
284 You can retarget these subtasks if you wish.
286 Notes
287 -----
288 The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a
289 flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``; see
290 `baseDebug` for more about ``debug.py`` files. `AssembleCoaddTask` has
291 no debug variables of its own. Some of the subtasks may support debug
292 variables. See the documentation for the subtasks for further information.
294 Examples
295 --------
296 `AssembleCoaddTask` assembles a set of warped images into a coadded image.
297 The `AssembleCoaddTask` can be invoked by running ``assembleCoadd.py``
298 with the flag '--legacyCoadd'. Usage of assembleCoadd.py expects two
299 inputs: a data reference to the tract patch and filter to be coadded, and
300 a list of Warps to attempt to coadd. These are specified using ``--id`` and
301 ``--selectId``, respectively:
303 .. code-block:: none
305 --id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]
306 --selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]
308 Only the Warps that cover the specified tract and patch will be coadded.
309 A list of the available optional arguments can be obtained by calling
310 ``assembleCoadd.py`` with the ``--help`` command line argument:
312 .. code-block:: none
314 assembleCoadd.py --help
316 To demonstrate usage of the `AssembleCoaddTask` in the larger context of
317 multi-band processing, we will generate the HSC-I & -R band coadds from
318 HSC engineering test data provided in the ``ci_hsc`` package. To begin,
319 assuming that the lsst stack has been already set up, we must set up the
320 obs_subaru and ``ci_hsc`` packages. This defines the environment variable
321 ``$CI_HSC_DIR`` and points at the location of the package. The raw HSC
322 data live in the ``$CI_HSC_DIR/raw directory``. To begin assembling the
323 coadds, we must first
325 - processCcd
326 - process the individual ccds in $CI_HSC_RAW to produce calibrated exposures
327 - makeSkyMap
328 - create a skymap that covers the area of the sky present in the raw exposures
329 - makeCoaddTempExp
330 - warp the individual calibrated exposures to the tangent plane of the coadd
332 We can perform all of these steps by running
334 .. code-block:: none
336 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
338 This will produce warped exposures for each visit. To coadd the warped
339 data, we call assembleCoadd.py as follows:
341 .. code-block:: none
343 assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \
344 --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \
345 --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \
346 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \
347 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \
348 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \
349 --selectId visit=903988 ccd=24
351 that will process the HSC-I band data. The results are written in
352 ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``.
354 You may also choose to run:
356 .. code-block:: none
358 scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346
359 assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R \
360 --selectId visit=903334 ccd=16 --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 \
361 --selectId visit=903334 ccd=100 --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 \
362 --selectId visit=903338 ccd=18 --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 \
363 --selectId visit=903342 ccd=10 --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 \
364 --selectId visit=903344 ccd=5 --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 \
365 --selectId visit=903346 ccd=6 --selectId visit=903346 ccd=12
367 to generate the coadd for the HSC-R band if you are interested in
368 following multiBand Coadd processing as discussed in `pipeTasks_multiBand`
369 (but note that normally, one would use the `SafeClipAssembleCoaddTask`
370 rather than `AssembleCoaddTask` to make the coadd.
371 """
372 ConfigClass = AssembleCoaddConfig
373 _DefaultName = "assembleCoadd"
375 def __init__(self, *args, **kwargs):
376 # TODO: DM-17415 better way to handle previously allowed passed args e.g.`AssembleCoaddTask(config)`
377 if args:
378 argNames = ["config", "name", "parentTask", "log"]
379 kwargs.update({k: v for k, v in zip(argNames, args)})
380 warnings.warn("AssembleCoadd received positional args, and casting them as kwargs: %s. "
381 "PipelineTask will not take positional args" % argNames, FutureWarning)
383 super().__init__(**kwargs)
384 self.makeSubtask("interpImage")
385 self.makeSubtask("scaleZeroPoint")
387 if self.config.doMaskBrightObjects:
388 mask = afwImage.Mask()
389 try:
390 self.brightObjectBitmask = 1 << mask.addMaskPlane(self.config.brightObjectMaskName)
391 except pexExceptions.LsstCppException:
392 raise RuntimeError("Unable to define mask plane for bright objects; planes used are %s" %
393 mask.getMaskPlaneDict().keys())
394 del mask
396 self.warpType = self.config.warpType
398 @utils.inheritDoc(pipeBase.PipelineTask)
399 def runQuantum(self, butlerQC, inputRefs, outputRefs):
400 # Docstring to be formatted with info from PipelineTask.runQuantum
401 """
402 Notes
403 -----
404 Assemble a coadd from a set of Warps.
406 PipelineTask (Gen3) entry point to Coadd a set of Warps.
407 Analogous to `runDataRef`, it prepares all the data products to be
408 passed to `run`, and processes the results before returning a struct
409 of results to be written out. AssembleCoadd cannot fit all Warps in memory.
410 Therefore, its inputs are accessed subregion by subregion
411 by the Gen3 `DeferredDatasetHandle` that is analagous to the Gen2
412 `lsst.daf.persistence.ButlerDataRef`. Any updates to this method should
413 correspond to an update in `runDataRef` while both entry points
414 are used.
415 """
416 inputData = butlerQC.get(inputRefs)
418 # Construct skyInfo expected by run
419 # Do not remove skyMap from inputData in case makeSupplementaryDataGen3 needs it
420 skyMap = inputData["skyMap"]
421 outputDataId = butlerQC.quantum.dataId
423 inputData['skyInfo'] = makeSkyInfo(skyMap,
424 tractId=outputDataId['tract'],
425 patchId=outputDataId['patch'])
427 # Construct list of input Deferred Datasets
428 # These quack a bit like like Gen2 DataRefs
429 warpRefList = inputData['inputWarps']
430 # Perform same middle steps as `runDataRef` does
431 inputs = self.prepareInputs(warpRefList)
432 self.log.info("Found %d %s", len(inputs.tempExpRefList),
433 self.getTempExpDatasetName(self.warpType))
434 if len(inputs.tempExpRefList) == 0:
435 self.log.warn("No coadd temporary exposures found")
436 return
438 supplementaryData = self.makeSupplementaryDataGen3(butlerQC, inputRefs, outputRefs)
439 retStruct = self.run(inputData['skyInfo'], inputs.tempExpRefList, inputs.imageScalerList,
440 inputs.weightList, supplementaryData=supplementaryData)
442 inputData.setdefault('brightObjectMask', None)
443 self.processResults(retStruct.coaddExposure, inputData['brightObjectMask'], outputDataId)
445 if self.config.doWrite:
446 butlerQC.put(retStruct, outputRefs)
447 return retStruct
449 @pipeBase.timeMethod
450 def runDataRef(self, dataRef, selectDataList=None, warpRefList=None):
451 """Assemble a coadd from a set of Warps.
453 Pipebase.CmdlineTask entry point to Coadd a set of Warps.
454 Compute weights to be applied to each Warp and
455 find scalings to match the photometric zeropoint to a reference Warp.
456 Assemble the Warps using `run`. Interpolate over NaNs and
457 optionally write the coadd to disk. Return the coadded exposure.
459 Parameters
460 ----------
461 dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
462 Data reference defining the patch for coaddition and the
463 reference Warp (if ``config.autoReference=False``).
464 Used to access the following data products:
465 - ``self.config.coaddName + "Coadd_skyMap"``
466 - ``self.config.coaddName + "Coadd_ + <warpType> + "Warp"`` (optionally)
467 - ``self.config.coaddName + "Coadd"``
468 selectDataList : `list`
469 List of data references to Calexps. Data to be coadded will be
470 selected from this list based on overlap with the patch defined
471 by dataRef, grouped by visit, and converted to a list of data
472 references to warps.
473 warpRefList : `list`
474 List of data references to Warps to be coadded.
475 Note: `warpRefList` is just the new name for `tempExpRefList`.
477 Returns
478 -------
479 retStruct : `lsst.pipe.base.Struct`
480 Result struct with components:
482 - ``coaddExposure``: coadded exposure (``Exposure``).
483 - ``nImage``: exposure count image (``Image``).
484 """
485 if selectDataList and warpRefList:
486 raise RuntimeError("runDataRef received both a selectDataList and warpRefList, "
487 "and which to use is ambiguous. Please pass only one.")
489 skyInfo = self.getSkyInfo(dataRef)
490 if warpRefList is None:
491 calExpRefList = self.selectExposures(dataRef, skyInfo, selectDataList=selectDataList)
492 if len(calExpRefList) == 0:
493 self.log.warn("No exposures to coadd")
494 return
495 self.log.info("Coadding %d exposures", len(calExpRefList))
497 warpRefList = self.getTempExpRefList(dataRef, calExpRefList)
499 inputData = self.prepareInputs(warpRefList)
500 self.log.info("Found %d %s", len(inputData.tempExpRefList),
501 self.getTempExpDatasetName(self.warpType))
502 if len(inputData.tempExpRefList) == 0:
503 self.log.warn("No coadd temporary exposures found")
504 return
506 supplementaryData = self.makeSupplementaryData(dataRef, warpRefList=inputData.tempExpRefList)
508 retStruct = self.run(skyInfo, inputData.tempExpRefList, inputData.imageScalerList,
509 inputData.weightList, supplementaryData=supplementaryData)
511 brightObjects = self.readBrightObjectMasks(dataRef) if self.config.doMaskBrightObjects else None
512 self.processResults(retStruct.coaddExposure, brightObjectMasks=brightObjects, dataId=dataRef.dataId)
514 if self.config.doWrite:
515 if self.getCoaddDatasetName(self.warpType) == "deepCoadd" and self.config.hasFakes:
516 coaddDatasetName = "fakes_" + self.getCoaddDatasetName(self.warpType)
517 else:
518 coaddDatasetName = self.getCoaddDatasetName(self.warpType)
519 self.log.info("Persisting %s" % coaddDatasetName)
520 dataRef.put(retStruct.coaddExposure, coaddDatasetName)
521 if self.config.doNImage and retStruct.nImage is not None:
522 dataRef.put(retStruct.nImage, self.getCoaddDatasetName(self.warpType) + '_nImage')
524 return retStruct
526 def processResults(self, coaddExposure, brightObjectMasks=None, dataId=None):
527 """Interpolate over missing data and mask bright stars.
529 Parameters
530 ----------
531 coaddExposure : `lsst.afw.image.Exposure`
532 The coadded exposure to process.
533 dataRef : `lsst.daf.persistence.ButlerDataRef`
534 Butler data reference for supplementary data.
535 """
536 if self.config.doInterp:
537 self.interpImage.run(coaddExposure.getMaskedImage(), planeName="NO_DATA")
538 # The variance must be positive; work around for DM-3201.
539 varArray = coaddExposure.variance.array
540 with numpy.errstate(invalid="ignore"):
541 varArray[:] = numpy.where(varArray > 0, varArray, numpy.inf)
543 if self.config.doMaskBrightObjects:
544 self.setBrightObjectMasks(coaddExposure, brightObjectMasks, dataId)
546 def makeSupplementaryData(self, dataRef, selectDataList=None, warpRefList=None):
547 """Make additional inputs to run() specific to subclasses (Gen2)
549 Duplicates interface of `runDataRef` method
550 Available to be implemented by subclasses only if they need the
551 coadd dataRef for performing preliminary processing before
552 assembling the coadd.
554 Parameters
555 ----------
556 dataRef : `lsst.daf.persistence.ButlerDataRef`
557 Butler data reference for supplementary data.
558 selectDataList : `list` (optional)
559 Optional List of data references to Calexps.
560 warpRefList : `list` (optional)
561 Optional List of data references to Warps.
562 """
563 return pipeBase.Struct()
565 def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs):
566 """Make additional inputs to run() specific to subclasses (Gen3)
568 Duplicates interface of `runQuantum` method.
569 Available to be implemented by subclasses only if they need the
570 coadd dataRef for performing preliminary processing before
571 assembling the coadd.
573 Parameters
574 ----------
575 butlerQC : `lsst.pipe.base.ButlerQuantumContext`
576 Gen3 Butler object for fetching additional data products before
577 running the Task specialized for quantum being processed
578 inputRefs : `lsst.pipe.base.InputQuantizedConnection`
579 Attributes are the names of the connections describing input dataset types.
580 Values are DatasetRefs that task consumes for corresponding dataset type.
581 DataIds are guaranteed to match data objects in ``inputData``.
582 outputRefs : `lsst.pipe.base.OutputQuantizedConnection`
583 Attributes are the names of the connections describing output dataset types.
584 Values are DatasetRefs that task is to produce
585 for corresponding dataset type.
586 """
587 return pipeBase.Struct()
589 def getTempExpRefList(self, patchRef, calExpRefList):
590 """Generate list data references corresponding to warped exposures
591 that lie within the patch to be coadded.
593 Parameters
594 ----------
595 patchRef : `dataRef`
596 Data reference for patch.
597 calExpRefList : `list`
598 List of data references for input calexps.
600 Returns
601 -------
602 tempExpRefList : `list`
603 List of Warp/CoaddTempExp data references.
604 """
605 butler = patchRef.getButler()
606 groupData = groupPatchExposures(patchRef, calExpRefList, self.getCoaddDatasetName(self.warpType),
607 self.getTempExpDatasetName(self.warpType))
608 tempExpRefList = [getGroupDataRef(butler, self.getTempExpDatasetName(self.warpType),
609 g, groupData.keys) for
610 g in groupData.groups.keys()]
611 return tempExpRefList
613 def prepareInputs(self, refList):
614 """Prepare the input warps for coaddition by measuring the weight for
615 each warp and the scaling for the photometric zero point.
617 Each Warp has its own photometric zeropoint and background variance.
618 Before coadding these Warps together, compute a scale factor to
619 normalize the photometric zeropoint and compute the weight for each Warp.
621 Parameters
622 ----------
623 refList : `list`
624 List of data references to tempExp
626 Returns
627 -------
628 result : `lsst.pipe.base.Struct`
629 Result struct with components:
631 - ``tempExprefList``: `list` of data references to tempExp.
632 - ``weightList``: `list` of weightings.
633 - ``imageScalerList``: `list` of image scalers.
634 """
635 statsCtrl = afwMath.StatisticsControl()
636 statsCtrl.setNumSigmaClip(self.config.sigmaClip)
637 statsCtrl.setNumIter(self.config.clipIter)
638 statsCtrl.setAndMask(self.getBadPixelMask())
639 statsCtrl.setNanSafe(True)
640 # compute tempExpRefList: a list of tempExpRef that actually exist
641 # and weightList: a list of the weight of the associated coadd tempExp
642 # and imageScalerList: a list of scale factors for the associated coadd tempExp
643 tempExpRefList = []
644 weightList = []
645 imageScalerList = []
646 tempExpName = self.getTempExpDatasetName(self.warpType)
647 for tempExpRef in refList:
648 # Gen3's DeferredDatasetHandles are guaranteed to exist and
649 # therefore have no datasetExists() method
650 if not isinstance(tempExpRef, DeferredDatasetHandle):
651 if not tempExpRef.datasetExists(tempExpName):
652 self.log.warn("Could not find %s %s; skipping it", tempExpName, tempExpRef.dataId)
653 continue
655 tempExp = tempExpRef.get(datasetType=tempExpName, immediate=True)
656 # Ignore any input warp that is empty of data
657 if numpy.isnan(tempExp.image.array).all():
658 continue
659 maskedImage = tempExp.getMaskedImage()
660 imageScaler = self.scaleZeroPoint.computeImageScaler(
661 exposure=tempExp,
662 dataRef=tempExpRef,
663 )
664 try:
665 imageScaler.scaleMaskedImage(maskedImage)
666 except Exception as e:
667 self.log.warn("Scaling failed for %s (skipping it): %s", tempExpRef.dataId, e)
668 continue
669 statObj = afwMath.makeStatistics(maskedImage.getVariance(), maskedImage.getMask(),
670 afwMath.MEANCLIP, statsCtrl)
671 meanVar, meanVarErr = statObj.getResult(afwMath.MEANCLIP)
672 weight = 1.0 / float(meanVar)
673 if not numpy.isfinite(weight):
674 self.log.warn("Non-finite weight for %s: skipping", tempExpRef.dataId)
675 continue
676 self.log.info("Weight of %s %s = %0.3f", tempExpName, tempExpRef.dataId, weight)
678 del maskedImage
679 del tempExp
681 tempExpRefList.append(tempExpRef)
682 weightList.append(weight)
683 imageScalerList.append(imageScaler)
685 return pipeBase.Struct(tempExpRefList=tempExpRefList, weightList=weightList,
686 imageScalerList=imageScalerList)
688 def prepareStats(self, mask=None):
689 """Prepare the statistics for coadding images.
691 Parameters
692 ----------
693 mask : `int`, optional
694 Bit mask value to exclude from coaddition.
696 Returns
697 -------
698 stats : `lsst.pipe.base.Struct`
699 Statistics structure with the following fields:
701 - ``statsCtrl``: Statistics control object for coadd
702 (`lsst.afw.math.StatisticsControl`)
703 - ``statsFlags``: Statistic for coadd (`lsst.afw.math.Property`)
704 """
705 if mask is None:
706 mask = self.getBadPixelMask()
707 statsCtrl = afwMath.StatisticsControl()
708 statsCtrl.setNumSigmaClip(self.config.sigmaClip)
709 statsCtrl.setNumIter(self.config.clipIter)
710 statsCtrl.setAndMask(mask)
711 statsCtrl.setNanSafe(True)
712 statsCtrl.setWeighted(True)
713 statsCtrl.setCalcErrorFromInputVariance(self.config.calcErrorFromInputVariance)
714 for plane, threshold in self.config.maskPropagationThresholds.items():
715 bit = afwImage.Mask.getMaskPlane(plane)
716 statsCtrl.setMaskPropagationThreshold(bit, threshold)
717 statsFlags = afwMath.stringToStatisticsProperty(self.config.statistic)
718 return pipeBase.Struct(ctrl=statsCtrl, flags=statsFlags)
720 @pipeBase.timeMethod
721 def run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
722 altMaskList=None, mask=None, supplementaryData=None):
723 """Assemble a coadd from input warps
725 Assemble the coadd using the provided list of coaddTempExps. Since
726 the full coadd covers a patch (a large area), the assembly is
727 performed over small areas on the image at a time in order to
728 conserve memory usage. Iterate over subregions within the outer
729 bbox of the patch using `assembleSubregion` to stack the corresponding
730 subregions from the coaddTempExps with the statistic specified.
731 Set the edge bits the coadd mask based on the weight map.
733 Parameters
734 ----------
735 skyInfo : `lsst.pipe.base.Struct`
736 Struct with geometric information about the patch.
737 tempExpRefList : `list`
738 List of data references to Warps (previously called CoaddTempExps).
739 imageScalerList : `list`
740 List of image scalers.
741 weightList : `list`
742 List of weights
743 altMaskList : `list`, optional
744 List of alternate masks to use rather than those stored with
745 tempExp.
746 mask : `int`, optional
747 Bit mask value to exclude from coaddition.
748 supplementaryData : lsst.pipe.base.Struct, optional
749 Struct with additional data products needed to assemble coadd.
750 Only used by subclasses that implement `makeSupplementaryData`
751 and override `run`.
753 Returns
754 -------
755 result : `lsst.pipe.base.Struct`
756 Result struct with components:
758 - ``coaddExposure``: coadded exposure (``lsst.afw.image.Exposure``).
759 - ``nImage``: exposure count image (``lsst.afw.image.Image``), if requested.
760 - ``warpRefList``: input list of refs to the warps (
761 ``lsst.daf.butler.DeferredDatasetHandle`` or
762 ``lsst.daf.persistence.ButlerDataRef``)
763 (unmodified)
764 - ``imageScalerList``: input list of image scalers (unmodified)
765 - ``weightList``: input list of weights (unmodified)
766 """
767 tempExpName = self.getTempExpDatasetName(self.warpType)
768 self.log.info("Assembling %s %s", len(tempExpRefList), tempExpName)
769 stats = self.prepareStats(mask=mask)
771 if altMaskList is None:
772 altMaskList = [None]*len(tempExpRefList)
774 coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
775 coaddExposure.setPhotoCalib(self.scaleZeroPoint.getPhotoCalib())
776 coaddExposure.getInfo().setCoaddInputs(self.inputRecorder.makeCoaddInputs())
777 self.assembleMetadata(coaddExposure, tempExpRefList, weightList)
778 coaddMaskedImage = coaddExposure.getMaskedImage()
779 subregionSizeArr = self.config.subregionSize
780 subregionSize = geom.Extent2I(subregionSizeArr[0], subregionSizeArr[1])
781 # if nImage is requested, create a zero one which can be passed to assembleSubregion
782 if self.config.doNImage:
783 nImage = afwImage.ImageU(skyInfo.bbox)
784 else:
785 nImage = None
786 for subBBox in self._subBBoxIter(skyInfo.bbox, subregionSize):
787 try:
788 self.assembleSubregion(coaddExposure, subBBox, tempExpRefList, imageScalerList,
789 weightList, altMaskList, stats.flags, stats.ctrl,
790 nImage=nImage)
791 except Exception as e:
792 self.log.fatal("Cannot compute coadd %s: %s", subBBox, e)
794 self.setInexactPsf(coaddMaskedImage.getMask())
795 # Despite the name, the following doesn't really deal with "EDGE" pixels: it identifies
796 # pixels that didn't receive any unmasked inputs (as occurs around the edge of the field).
797 coaddUtils.setCoaddEdgeBits(coaddMaskedImage.getMask(), coaddMaskedImage.getVariance())
798 return pipeBase.Struct(coaddExposure=coaddExposure, nImage=nImage,
799 warpRefList=tempExpRefList, imageScalerList=imageScalerList,
800 weightList=weightList)
802 def assembleMetadata(self, coaddExposure, tempExpRefList, weightList):
803 """Set the metadata for the coadd.
805 This basic implementation sets the filter from the first input.
807 Parameters
808 ----------
809 coaddExposure : `lsst.afw.image.Exposure`
810 The target exposure for the coadd.
811 tempExpRefList : `list`
812 List of data references to tempExp.
813 weightList : `list`
814 List of weights.
815 """
816 assert len(tempExpRefList) == len(weightList), "Length mismatch"
817 tempExpName = self.getTempExpDatasetName(self.warpType)
818 # We load a single pixel of each coaddTempExp, because we just want to get at the metadata
819 # (and we need more than just the PropertySet that contains the header), which is not possible
820 # with the current butler (see #2777).
821 bbox = geom.Box2I(coaddExposure.getBBox().getMin(), geom.Extent2I(1, 1))
823 if isinstance(tempExpRefList[0], DeferredDatasetHandle):
824 # Gen 3 API
825 tempExpList = [tempExpRef.get(parameters={'bbox': bbox}) for tempExpRef in tempExpRefList]
826 else:
827 # Gen 2 API. Delete this when Gen 2 retired
828 tempExpList = [tempExpRef.get(tempExpName + "_sub", bbox=bbox, immediate=True)
829 for tempExpRef in tempExpRefList]
830 numCcds = sum(len(tempExp.getInfo().getCoaddInputs().ccds) for tempExp in tempExpList)
832 # Set the coadd FilterLabel to the band of the first input exposure:
833 # Coadds are calibrated, so the physical label is now meaningless.
834 coaddExposure.setFilterLabel(afwImage.FilterLabel(tempExpList[0].getFilterLabel().bandLabel))
835 coaddInputs = coaddExposure.getInfo().getCoaddInputs()
836 coaddInputs.ccds.reserve(numCcds)
837 coaddInputs.visits.reserve(len(tempExpList))
839 for tempExp, weight in zip(tempExpList, weightList):
840 self.inputRecorder.addVisitToCoadd(coaddInputs, tempExp, weight)
842 if self.config.doUsePsfMatchedPolygons:
843 self.shrinkValidPolygons(coaddInputs)
845 coaddInputs.visits.sort()
846 if self.warpType == "psfMatched":
847 # The modelPsf BBox for a psfMatchedWarp/coaddTempExp was dynamically defined by
848 # ModelPsfMatchTask as the square box bounding its spatially-variable, pre-matched WarpedPsf.
849 # Likewise, set the PSF of a PSF-Matched Coadd to the modelPsf
850 # having the maximum width (sufficient because square)
851 modelPsfList = [tempExp.getPsf() for tempExp in tempExpList]
852 modelPsfWidthList = [modelPsf.computeBBox().getWidth() for modelPsf in modelPsfList]
853 psf = modelPsfList[modelPsfWidthList.index(max(modelPsfWidthList))]
854 else:
855 psf = measAlg.CoaddPsf(coaddInputs.ccds, coaddExposure.getWcs(),
856 self.config.coaddPsf.makeControl())
857 coaddExposure.setPsf(psf)
858 apCorrMap = measAlg.makeCoaddApCorrMap(coaddInputs.ccds, coaddExposure.getBBox(afwImage.PARENT),
859 coaddExposure.getWcs())
860 coaddExposure.getInfo().setApCorrMap(apCorrMap)
861 if self.config.doAttachTransmissionCurve:
862 transmissionCurve = measAlg.makeCoaddTransmissionCurve(coaddExposure.getWcs(), coaddInputs.ccds)
863 coaddExposure.getInfo().setTransmissionCurve(transmissionCurve)
865 def assembleSubregion(self, coaddExposure, bbox, tempExpRefList, imageScalerList, weightList,
866 altMaskList, statsFlags, statsCtrl, nImage=None):
867 """Assemble the coadd for a sub-region.
869 For each coaddTempExp, check for (and swap in) an alternative mask
870 if one is passed. Remove mask planes listed in
871 `config.removeMaskPlanes`. Finally, stack the actual exposures using
872 `lsst.afw.math.statisticsStack` with the statistic specified by
873 statsFlags. Typically, the statsFlag will be one of lsst.afw.math.MEAN for
874 a mean-stack or `lsst.afw.math.MEANCLIP` for outlier rejection using
875 an N-sigma clipped mean where N and iterations are specified by
876 statsCtrl. Assign the stacked subregion back to the coadd.
878 Parameters
879 ----------
880 coaddExposure : `lsst.afw.image.Exposure`
881 The target exposure for the coadd.
882 bbox : `lsst.geom.Box`
883 Sub-region to coadd.
884 tempExpRefList : `list`
885 List of data reference to tempExp.
886 imageScalerList : `list`
887 List of image scalers.
888 weightList : `list`
889 List of weights.
890 altMaskList : `list`
891 List of alternate masks to use rather than those stored with
892 tempExp, or None. Each element is dict with keys = mask plane
893 name to which to add the spans.
894 statsFlags : `lsst.afw.math.Property`
895 Property object for statistic for coadd.
896 statsCtrl : `lsst.afw.math.StatisticsControl`
897 Statistics control object for coadd.
898 nImage : `lsst.afw.image.ImageU`, optional
899 Keeps track of exposure count for each pixel.
900 """
901 self.log.debug("Computing coadd over %s", bbox)
902 tempExpName = self.getTempExpDatasetName(self.warpType)
903 coaddExposure.mask.addMaskPlane("REJECTED")
904 coaddExposure.mask.addMaskPlane("CLIPPED")
905 coaddExposure.mask.addMaskPlane("SENSOR_EDGE")
906 maskMap = self.setRejectedMaskMapping(statsCtrl)
907 clipped = afwImage.Mask.getPlaneBitMask("CLIPPED")
908 maskedImageList = []
909 if nImage is not None:
910 subNImage = afwImage.ImageU(bbox.getWidth(), bbox.getHeight())
911 for tempExpRef, imageScaler, altMask in zip(tempExpRefList, imageScalerList, altMaskList):
913 if isinstance(tempExpRef, DeferredDatasetHandle):
914 # Gen 3 API
915 exposure = tempExpRef.get(parameters={'bbox': bbox})
916 else:
917 # Gen 2 API. Delete this when Gen 2 retired
918 exposure = tempExpRef.get(tempExpName + "_sub", bbox=bbox)
920 maskedImage = exposure.getMaskedImage()
921 mask = maskedImage.getMask()
922 if altMask is not None:
923 self.applyAltMaskPlanes(mask, altMask)
924 imageScaler.scaleMaskedImage(maskedImage)
926 # Add 1 for each pixel which is not excluded by the exclude mask.
927 # In legacyCoadd, pixels may also be excluded by afwMath.statisticsStack.
928 if nImage is not None:
929 subNImage.getArray()[maskedImage.getMask().getArray() & statsCtrl.getAndMask() == 0] += 1
930 if self.config.removeMaskPlanes:
931 self.removeMaskPlanes(maskedImage)
932 maskedImageList.append(maskedImage)
934 with self.timer("stack"):
935 coaddSubregion = afwMath.statisticsStack(maskedImageList, statsFlags, statsCtrl, weightList,
936 clipped, # also set output to CLIPPED if sigma-clipped
937 maskMap)
938 coaddExposure.maskedImage.assign(coaddSubregion, bbox)
939 if nImage is not None:
940 nImage.assign(subNImage, bbox)
942 def removeMaskPlanes(self, maskedImage):
943 """Unset the mask of an image for mask planes specified in the config.
945 Parameters
946 ----------
947 maskedImage : `lsst.afw.image.MaskedImage`
948 The masked image to be modified.
949 """
950 mask = maskedImage.getMask()
951 for maskPlane in self.config.removeMaskPlanes:
952 try:
953 mask &= ~mask.getPlaneBitMask(maskPlane)
954 except pexExceptions.InvalidParameterError:
955 self.log.debug("Unable to remove mask plane %s: no mask plane with that name was found.",
956 maskPlane)
958 @staticmethod
959 def setRejectedMaskMapping(statsCtrl):
960 """Map certain mask planes of the warps to new planes for the coadd.
962 If a pixel is rejected due to a mask value other than EDGE, NO_DATA,
963 or CLIPPED, set it to REJECTED on the coadd.
964 If a pixel is rejected due to EDGE, set the coadd pixel to SENSOR_EDGE.
965 If a pixel is rejected due to CLIPPED, set the coadd pixel to CLIPPED.
967 Parameters
968 ----------
969 statsCtrl : `lsst.afw.math.StatisticsControl`
970 Statistics control object for coadd
972 Returns
973 -------
974 maskMap : `list` of `tuple` of `int`
975 A list of mappings of mask planes of the warped exposures to
976 mask planes of the coadd.
977 """
978 edge = afwImage.Mask.getPlaneBitMask("EDGE")
979 noData = afwImage.Mask.getPlaneBitMask("NO_DATA")
980 clipped = afwImage.Mask.getPlaneBitMask("CLIPPED")
981 toReject = statsCtrl.getAndMask() & (~noData) & (~edge) & (~clipped)
982 maskMap = [(toReject, afwImage.Mask.getPlaneBitMask("REJECTED")),
983 (edge, afwImage.Mask.getPlaneBitMask("SENSOR_EDGE")),
984 (clipped, clipped)]
985 return maskMap
987 def applyAltMaskPlanes(self, mask, altMaskSpans):
988 """Apply in place alt mask formatted as SpanSets to a mask.
990 Parameters
991 ----------
992 mask : `lsst.afw.image.Mask`
993 Original mask.
994 altMaskSpans : `dict`
995 SpanSet lists to apply. Each element contains the new mask
996 plane name (e.g. "CLIPPED and/or "NO_DATA") as the key,
997 and list of SpanSets to apply to the mask.
999 Returns
1000 -------
1001 mask : `lsst.afw.image.Mask`
1002 Updated mask.
1003 """
1004 if self.config.doUsePsfMatchedPolygons:
1005 if ("NO_DATA" in altMaskSpans) and ("NO_DATA" in self.config.badMaskPlanes):
1006 # Clear away any other masks outside the validPolygons. These pixels are no longer
1007 # contributing to inexact PSFs, and will still be rejected because of NO_DATA
1008 # self.config.doUsePsfMatchedPolygons should be True only in CompareWarpAssemble
1009 # This mask-clearing step must only occur *before* applying the new masks below
1010 for spanSet in altMaskSpans['NO_DATA']:
1011 spanSet.clippedTo(mask.getBBox()).clearMask(mask, self.getBadPixelMask())
1013 for plane, spanSetList in altMaskSpans.items():
1014 maskClipValue = mask.addMaskPlane(plane)
1015 for spanSet in spanSetList:
1016 spanSet.clippedTo(mask.getBBox()).setMask(mask, 2**maskClipValue)
1017 return mask
1019 def shrinkValidPolygons(self, coaddInputs):
1020 """Shrink coaddInputs' ccds' ValidPolygons in place.
1022 Either modify each ccd's validPolygon in place, or if CoaddInputs
1023 does not have a validPolygon, create one from its bbox.
1025 Parameters
1026 ----------
1027 coaddInputs : `lsst.afw.image.coaddInputs`
1028 Original mask.
1030 """
1031 for ccd in coaddInputs.ccds:
1032 polyOrig = ccd.getValidPolygon()
1033 validPolyBBox = polyOrig.getBBox() if polyOrig else ccd.getBBox()
1034 validPolyBBox.grow(-self.config.matchingKernelSize//2)
1035 if polyOrig:
1036 validPolygon = polyOrig.intersectionSingle(validPolyBBox)
1037 else:
1038 validPolygon = afwGeom.polygon.Polygon(geom.Box2D(validPolyBBox))
1039 ccd.setValidPolygon(validPolygon)
1041 def readBrightObjectMasks(self, dataRef):
1042 """Retrieve the bright object masks.
1044 Returns None on failure.
1046 Parameters
1047 ----------
1048 dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
1049 A Butler dataRef.
1051 Returns
1052 -------
1053 result : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
1054 Bright object mask from the Butler object, or None if it cannot
1055 be retrieved.
1056 """
1057 try:
1058 return dataRef.get(datasetType="brightObjectMask", immediate=True)
1059 except Exception as e:
1060 self.log.warn("Unable to read brightObjectMask for %s: %s", dataRef.dataId, e)
1061 return None
1063 def setBrightObjectMasks(self, exposure, brightObjectMasks, dataId=None):
1064 """Set the bright object masks.
1066 Parameters
1067 ----------
1068 exposure : `lsst.afw.image.Exposure`
1069 Exposure under consideration.
1070 dataId : `lsst.daf.persistence.dataId`
1071 Data identifier dict for patch.
1072 brightObjectMasks : `lsst.afw.table`
1073 Table of bright objects to mask.
1074 """
1076 if brightObjectMasks is None:
1077 self.log.warn("Unable to apply bright object mask: none supplied")
1078 return
1079 self.log.info("Applying %d bright object masks to %s", len(brightObjectMasks), dataId)
1080 mask = exposure.getMaskedImage().getMask()
1081 wcs = exposure.getWcs()
1082 plateScale = wcs.getPixelScale().asArcseconds()
1084 for rec in brightObjectMasks:
1085 center = geom.PointI(wcs.skyToPixel(rec.getCoord()))
1086 if rec["type"] == "box":
1087 assert rec["angle"] == 0.0, ("Angle != 0 for mask object %s" % rec["id"])
1088 width = rec["width"].asArcseconds()/plateScale # convert to pixels
1089 height = rec["height"].asArcseconds()/plateScale # convert to pixels
1091 halfSize = geom.ExtentI(0.5*width, 0.5*height)
1092 bbox = geom.Box2I(center - halfSize, center + halfSize)
1094 bbox = geom.BoxI(geom.PointI(int(center[0] - 0.5*width), int(center[1] - 0.5*height)),
1095 geom.PointI(int(center[0] + 0.5*width), int(center[1] + 0.5*height)))
1096 spans = afwGeom.SpanSet(bbox)
1097 elif rec["type"] == "circle":
1098 radius = int(rec["radius"].asArcseconds()/plateScale) # convert to pixels
1099 spans = afwGeom.SpanSet.fromShape(radius, offset=center)
1100 else:
1101 self.log.warn("Unexpected region type %s at %s" % rec["type"], center)
1102 continue
1103 spans.clippedTo(mask.getBBox()).setMask(mask, self.brightObjectBitmask)
1105 def setInexactPsf(self, mask):
1106 """Set INEXACT_PSF mask plane.
1108 If any of the input images isn't represented in the coadd (due to
1109 clipped pixels or chip gaps), the `CoaddPsf` will be inexact. Flag
1110 these pixels.
1112 Parameters
1113 ----------
1114 mask : `lsst.afw.image.Mask`
1115 Coadded exposure's mask, modified in-place.
1116 """
1117 mask.addMaskPlane("INEXACT_PSF")
1118 inexactPsf = mask.getPlaneBitMask("INEXACT_PSF")
1119 sensorEdge = mask.getPlaneBitMask("SENSOR_EDGE") # chip edges (so PSF is discontinuous)
1120 clipped = mask.getPlaneBitMask("CLIPPED") # pixels clipped from coadd
1121 rejected = mask.getPlaneBitMask("REJECTED") # pixels rejected from coadd due to masks
1122 array = mask.getArray()
1123 selected = array & (sensorEdge | clipped | rejected) > 0
1124 array[selected] |= inexactPsf
1126 @classmethod
1127 def _makeArgumentParser(cls):
1128 """Create an argument parser.
1129 """
1130 parser = pipeBase.ArgumentParser(name=cls._DefaultName)
1131 parser.add_id_argument("--id", cls.ConfigClass().coaddName + "Coadd_"
1132 + cls.ConfigClass().warpType + "Warp",
1133 help="data ID, e.g. --id tract=12345 patch=1,2",
1134 ContainerClass=AssembleCoaddDataIdContainer)
1135 parser.add_id_argument("--selectId", "calexp", help="data ID, e.g. --selectId visit=6789 ccd=0..9",
1136 ContainerClass=SelectDataIdContainer)
1137 return parser
1139 @staticmethod
1140 def _subBBoxIter(bbox, subregionSize):
1141 """Iterate over subregions of a bbox.
1143 Parameters
1144 ----------
1145 bbox : `lsst.geom.Box2I`
1146 Bounding box over which to iterate.
1147 subregionSize: `lsst.geom.Extent2I`
1148 Size of sub-bboxes.
1150 Yields
1151 ------
1152 subBBox : `lsst.geom.Box2I`
1153 Next sub-bounding box of size ``subregionSize`` or smaller; each ``subBBox``
1154 is contained within ``bbox``, so it may be smaller than ``subregionSize`` at
1155 the edges of ``bbox``, but it will never be empty.
1156 """
1157 if bbox.isEmpty():
1158 raise RuntimeError("bbox %s is empty" % (bbox,))
1159 if subregionSize[0] < 1 or subregionSize[1] < 1:
1160 raise RuntimeError("subregionSize %s must be nonzero" % (subregionSize,))
1162 for rowShift in range(0, bbox.getHeight(), subregionSize[1]):
1163 for colShift in range(0, bbox.getWidth(), subregionSize[0]):
1164 subBBox = geom.Box2I(bbox.getMin() + geom.Extent2I(colShift, rowShift), subregionSize)
1165 subBBox.clip(bbox)
1166 if subBBox.isEmpty():
1167 raise RuntimeError("Bug: empty bbox! bbox=%s, subregionSize=%s, "
1168 "colShift=%s, rowShift=%s" %
1169 (bbox, subregionSize, colShift, rowShift))
1170 yield subBBox
1173class AssembleCoaddDataIdContainer(pipeBase.DataIdContainer):
1174 """A version of `lsst.pipe.base.DataIdContainer` specialized for assembleCoadd.
1175 """
1177 def makeDataRefList(self, namespace):
1178 """Make self.refList from self.idList.
1180 Parameters
1181 ----------
1182 namespace
1183 Results of parsing command-line (with ``butler`` and ``log`` elements).
1184 """
1185 datasetType = namespace.config.coaddName + "Coadd"
1186 keysCoadd = namespace.butler.getKeys(datasetType=datasetType, level=self.level)
1188 for dataId in self.idList:
1189 # tract and patch are required
1190 for key in keysCoadd:
1191 if key not in dataId:
1192 raise RuntimeError("--id must include " + key)
1194 dataRef = namespace.butler.dataRef(
1195 datasetType=datasetType,
1196 dataId=dataId,
1197 )
1198 self.refList.append(dataRef)
1201def countMaskFromFootprint(mask, footprint, bitmask, ignoreMask):
1202 """Function to count the number of pixels with a specific mask in a
1203 footprint.
1205 Find the intersection of mask & footprint. Count all pixels in the mask
1206 that are in the intersection that have bitmask set but do not have
1207 ignoreMask set. Return the count.
1209 Parameters
1210 ----------
1211 mask : `lsst.afw.image.Mask`
1212 Mask to define intersection region by.
1213 footprint : `lsst.afw.detection.Footprint`
1214 Footprint to define the intersection region by.
1215 bitmask
1216 Specific mask that we wish to count the number of occurances of.
1217 ignoreMask
1218 Pixels to not consider.
1220 Returns
1221 -------
1222 result : `int`
1223 Count of number of pixels in footprint with specified mask.
1224 """
1225 bbox = footprint.getBBox()
1226 bbox.clip(mask.getBBox(afwImage.PARENT))
1227 fp = afwImage.Mask(bbox)
1228 subMask = mask.Factory(mask, bbox, afwImage.PARENT)
1229 footprint.spans.setMask(fp, bitmask)
1230 return numpy.logical_and((subMask.getArray() & fp.getArray()) > 0,
1231 (subMask.getArray() & ignoreMask) == 0).sum()
1234class SafeClipAssembleCoaddConfig(AssembleCoaddConfig, pipelineConnections=AssembleCoaddConnections):
1235 """Configuration parameters for the SafeClipAssembleCoaddTask.
1236 """
1237 clipDetection = pexConfig.ConfigurableField(
1238 target=SourceDetectionTask,
1239 doc="Detect sources on difference between unclipped and clipped coadd")
1240 minClipFootOverlap = pexConfig.Field(
1241 doc="Minimum fractional overlap of clipped footprint with visit DETECTED to be clipped",
1242 dtype=float,
1243 default=0.6
1244 )
1245 minClipFootOverlapSingle = pexConfig.Field(
1246 doc="Minimum fractional overlap of clipped footprint with visit DETECTED to be "
1247 "clipped when only one visit overlaps",
1248 dtype=float,
1249 default=0.5
1250 )
1251 minClipFootOverlapDouble = pexConfig.Field(
1252 doc="Minimum fractional overlap of clipped footprints with visit DETECTED to be "
1253 "clipped when two visits overlap",
1254 dtype=float,
1255 default=0.45
1256 )
1257 maxClipFootOverlapDouble = pexConfig.Field(
1258 doc="Maximum fractional overlap of clipped footprints with visit DETECTED when "
1259 "considering two visits",
1260 dtype=float,
1261 default=0.15
1262 )
1263 minBigOverlap = pexConfig.Field(
1264 doc="Minimum number of pixels in footprint to use DETECTED mask from the single visits "
1265 "when labeling clipped footprints",
1266 dtype=int,
1267 default=100
1268 )
1270 def setDefaults(self):
1271 """Set default values for clipDetection.
1273 Notes
1274 -----
1275 The numeric values for these configuration parameters were
1276 empirically determined, future work may further refine them.
1277 """
1278 AssembleCoaddConfig.setDefaults(self)
1279 self.clipDetection.doTempLocalBackground = False
1280 self.clipDetection.reEstimateBackground = False
1281 self.clipDetection.returnOriginalFootprints = False
1282 self.clipDetection.thresholdPolarity = "both"
1283 self.clipDetection.thresholdValue = 2
1284 self.clipDetection.nSigmaToGrow = 2
1285 self.clipDetection.minPixels = 4
1286 self.clipDetection.isotropicGrow = True
1287 self.clipDetection.thresholdType = "pixel_stdev"
1288 self.sigmaClip = 1.5
1289 self.clipIter = 3
1290 self.statistic = "MEAN"
1292 def validate(self):
1293 if self.doSigmaClip:
1294 log.warn("Additional Sigma-clipping not allowed in Safe-clipped Coadds. "
1295 "Ignoring doSigmaClip.")
1296 self.doSigmaClip = False
1297 if self.statistic != "MEAN":
1298 raise ValueError("Only MEAN statistic allowed for final stacking in SafeClipAssembleCoadd "
1299 "(%s chosen). Please set statistic to MEAN."
1300 % (self.statistic))
1301 AssembleCoaddTask.ConfigClass.validate(self)
1304class SafeClipAssembleCoaddTask(AssembleCoaddTask):
1305 """Assemble a coadded image from a set of coadded temporary exposures,
1306 being careful to clip & flag areas with potential artifacts.
1308 In ``AssembleCoaddTask``, we compute the coadd as an clipped mean (i.e.,
1309 we clip outliers). The problem with doing this is that when computing the
1310 coadd PSF at a given location, individual visit PSFs from visits with
1311 outlier pixels contribute to the coadd PSF and cannot be treated correctly.
1312 In this task, we correct for this behavior by creating a new
1313 ``badMaskPlane`` 'CLIPPED'. We populate this plane on the input
1314 coaddTempExps and the final coadd where
1316 i. difference imaging suggests that there is an outlier and
1317 ii. this outlier appears on only one or two images.
1319 Such regions will not contribute to the final coadd. Furthermore, any
1320 routine to determine the coadd PSF can now be cognizant of clipped regions.
1321 Note that the algorithm implemented by this task is preliminary and works
1322 correctly for HSC data. Parameter modifications and or considerable
1323 redesigning of the algorithm is likley required for other surveys.
1325 ``SafeClipAssembleCoaddTask`` uses a ``SourceDetectionTask``
1326 "clipDetection" subtask and also sub-classes ``AssembleCoaddTask``.
1327 You can retarget the ``SourceDetectionTask`` "clipDetection" subtask
1328 if you wish.
1330 Notes
1331 -----
1332 The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a
1333 flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``;
1334 see `baseDebug` for more about ``debug.py`` files.
1335 `SafeClipAssembleCoaddTask` has no debug variables of its own.
1336 The ``SourceDetectionTask`` "clipDetection" subtasks may support debug
1337 variables. See the documetation for `SourceDetectionTask` "clipDetection"
1338 for further information.
1340 Examples
1341 --------
1342 `SafeClipAssembleCoaddTask` assembles a set of warped ``coaddTempExp``
1343 images into a coadded image. The `SafeClipAssembleCoaddTask` is invoked by
1344 running assembleCoadd.py *without* the flag '--legacyCoadd'.
1346 Usage of ``assembleCoadd.py`` expects a data reference to the tract patch
1347 and filter to be coadded (specified using
1348 '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]')
1349 along with a list of coaddTempExps to attempt to coadd (specified using
1350 '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]').
1351 Only the coaddTempExps that cover the specified tract and patch will be
1352 coadded. A list of the available optional arguments can be obtained by
1353 calling assembleCoadd.py with the --help command line argument:
1355 .. code-block:: none
1357 assembleCoadd.py --help
1359 To demonstrate usage of the `SafeClipAssembleCoaddTask` in the larger
1360 context of multi-band processing, we will generate the HSC-I & -R band
1361 coadds from HSC engineering test data provided in the ci_hsc package.
1362 To begin, assuming that the lsst stack has been already set up, we must
1363 set up the obs_subaru and ci_hsc packages. This defines the environment
1364 variable $CI_HSC_DIR and points at the location of the package. The raw
1365 HSC data live in the ``$CI_HSC_DIR/raw`` directory. To begin assembling
1366 the coadds, we must first
1368 - ``processCcd``
1369 process the individual ccds in $CI_HSC_RAW to produce calibrated exposures
1370 - ``makeSkyMap``
1371 create a skymap that covers the area of the sky present in the raw exposures
1372 - ``makeCoaddTempExp``
1373 warp the individual calibrated exposures to the tangent plane of the coadd</DD>
1375 We can perform all of these steps by running
1377 .. code-block:: none
1379 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
1381 This will produce warped coaddTempExps for each visit. To coadd the
1382 warped data, we call ``assembleCoadd.py`` as follows:
1384 .. code-block:: none
1386 assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \
1387 --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \
1388 --selectId visit=903986 ccd=100--selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \
1389 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \
1390 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \
1391 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \
1392 --selectId visit=903988 ccd=24
1394 This will process the HSC-I band data. The results are written in
1395 ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``.
1397 You may also choose to run:
1399 .. code-block:: none
1401 scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346 nnn
1402 assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R --selectId visit=903334 ccd=16 \
1403 --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 --selectId visit=903334 ccd=100 \
1404 --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 --selectId visit=903338 ccd=18 \
1405 --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 --selectId visit=903342 ccd=10 \
1406 --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 --selectId visit=903344 ccd=5 \
1407 --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 --selectId visit=903346 ccd=6 \
1408 --selectId visit=903346 ccd=12
1410 to generate the coadd for the HSC-R band if you are interested in following
1411 multiBand Coadd processing as discussed in ``pipeTasks_multiBand``.
1412 """
1413 ConfigClass = SafeClipAssembleCoaddConfig
1414 _DefaultName = "safeClipAssembleCoadd"
1416 def __init__(self, *args, **kwargs):
1417 AssembleCoaddTask.__init__(self, *args, **kwargs)
1418 schema = afwTable.SourceTable.makeMinimalSchema()
1419 self.makeSubtask("clipDetection", schema=schema)
1421 @utils.inheritDoc(AssembleCoaddTask)
1422 @pipeBase.timeMethod
1423 def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, *args, **kwargs):
1424 """Assemble the coadd for a region.
1426 Compute the difference of coadds created with and without outlier
1427 rejection to identify coadd pixels that have outlier values in some
1428 individual visits.
1429 Detect clipped regions on the difference image and mark these regions
1430 on the one or two individual coaddTempExps where they occur if there
1431 is significant overlap between the clipped region and a source. This
1432 leaves us with a set of footprints from the difference image that have
1433 been identified as having occured on just one or two individual visits.
1434 However, these footprints were generated from a difference image. It
1435 is conceivable for a large diffuse source to have become broken up
1436 into multiple footprints acrosss the coadd difference in this process.
1437 Determine the clipped region from all overlapping footprints from the
1438 detected sources in each visit - these are big footprints.
1439 Combine the small and big clipped footprints and mark them on a new
1440 bad mask plane.
1441 Generate the coadd using `AssembleCoaddTask.run` without outlier
1442 removal. Clipped footprints will no longer make it into the coadd
1443 because they are marked in the new bad mask plane.
1445 Notes
1446 -----
1447 args and kwargs are passed but ignored in order to match the call
1448 signature expected by the parent task.
1449 """
1450 exp = self.buildDifferenceImage(skyInfo, tempExpRefList, imageScalerList, weightList)
1451 mask = exp.getMaskedImage().getMask()
1452 mask.addMaskPlane("CLIPPED")
1454 result = self.detectClip(exp, tempExpRefList)
1456 self.log.info('Found %d clipped objects', len(result.clipFootprints))
1458 maskClipValue = mask.getPlaneBitMask("CLIPPED")
1459 maskDetValue = mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE")
1460 # Append big footprints from individual Warps to result.clipSpans
1461 bigFootprints = self.detectClipBig(result.clipSpans, result.clipFootprints, result.clipIndices,
1462 result.detectionFootprints, maskClipValue, maskDetValue,
1463 exp.getBBox())
1464 # Create mask of the current clipped footprints
1465 maskClip = mask.Factory(mask.getBBox(afwImage.PARENT))
1466 afwDet.setMaskFromFootprintList(maskClip, result.clipFootprints, maskClipValue)
1468 maskClipBig = maskClip.Factory(mask.getBBox(afwImage.PARENT))
1469 afwDet.setMaskFromFootprintList(maskClipBig, bigFootprints, maskClipValue)
1470 maskClip |= maskClipBig
1472 # Assemble coadd from base class, but ignoring CLIPPED pixels
1473 badMaskPlanes = self.config.badMaskPlanes[:]
1474 badMaskPlanes.append("CLIPPED")
1475 badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes)
1476 return AssembleCoaddTask.run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
1477 result.clipSpans, mask=badPixelMask)
1479 def buildDifferenceImage(self, skyInfo, tempExpRefList, imageScalerList, weightList):
1480 """Return an exposure that contains the difference between unclipped
1481 and clipped coadds.
1483 Generate a difference image between clipped and unclipped coadds.
1484 Compute the difference image by subtracting an outlier-clipped coadd
1485 from an outlier-unclipped coadd. Return the difference image.
1487 Parameters
1488 ----------
1489 skyInfo : `lsst.pipe.base.Struct`
1490 Patch geometry information, from getSkyInfo
1491 tempExpRefList : `list`
1492 List of data reference to tempExp
1493 imageScalerList : `list`
1494 List of image scalers
1495 weightList : `list`
1496 List of weights
1498 Returns
1499 -------
1500 exp : `lsst.afw.image.Exposure`
1501 Difference image of unclipped and clipped coadd wrapped in an Exposure
1502 """
1503 config = AssembleCoaddConfig()
1504 # getattr necessary because subtasks do not survive Config.toDict()
1505 # exclude connections because the class of self.config.connections is not
1506 # the same as AssembleCoaddConfig.connections, and the connections are not
1507 # needed to run this task anyway.
1508 configIntersection = {k: getattr(self.config, k)
1509 for k, v in self.config.toDict().items()
1510 if (k in config.keys() and k != "connections")}
1511 config.update(**configIntersection)
1513 # statistic MEAN copied from self.config.statistic, but for clarity explicitly assign
1514 config.statistic = 'MEAN'
1515 task = AssembleCoaddTask(config=config)
1516 coaddMean = task.run(skyInfo, tempExpRefList, imageScalerList, weightList).coaddExposure
1518 config.statistic = 'MEANCLIP'
1519 task = AssembleCoaddTask(config=config)
1520 coaddClip = task.run(skyInfo, tempExpRefList, imageScalerList, weightList).coaddExposure
1522 coaddDiff = coaddMean.getMaskedImage().Factory(coaddMean.getMaskedImage())
1523 coaddDiff -= coaddClip.getMaskedImage()
1524 exp = afwImage.ExposureF(coaddDiff)
1525 exp.setPsf(coaddMean.getPsf())
1526 return exp
1528 def detectClip(self, exp, tempExpRefList):
1529 """Detect clipped regions on an exposure and set the mask on the
1530 individual tempExp masks.
1532 Detect footprints in the difference image after smoothing the
1533 difference image with a Gaussian kernal. Identify footprints that
1534 overlap with one or two input ``coaddTempExps`` by comparing the
1535 computed overlap fraction to thresholds set in the config. A different
1536 threshold is applied depending on the number of overlapping visits
1537 (restricted to one or two). If the overlap exceeds the thresholds,
1538 the footprint is considered "CLIPPED" and is marked as such on the
1539 coaddTempExp. Return a struct with the clipped footprints, the indices
1540 of the ``coaddTempExps`` that end up overlapping with the clipped
1541 footprints, and a list of new masks for the ``coaddTempExps``.
1543 Parameters
1544 ----------
1545 exp : `lsst.afw.image.Exposure`
1546 Exposure to run detection on.
1547 tempExpRefList : `list`
1548 List of data reference to tempExp.
1550 Returns
1551 -------
1552 result : `lsst.pipe.base.Struct`
1553 Result struct with components:
1555 - ``clipFootprints``: list of clipped footprints.
1556 - ``clipIndices``: indices for each ``clippedFootprint`` in
1557 ``tempExpRefList``.
1558 - ``clipSpans``: List of dictionaries containing spanSet lists
1559 to clip. Each element contains the new maskplane name
1560 ("CLIPPED") as the key and list of ``SpanSets`` as the value.
1561 - ``detectionFootprints``: List of DETECTED/DETECTED_NEGATIVE plane
1562 compressed into footprints.
1563 """
1564 mask = exp.getMaskedImage().getMask()
1565 maskDetValue = mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE")
1566 fpSet = self.clipDetection.detectFootprints(exp, doSmooth=True, clearMask=True)
1567 # Merge positive and negative together footprints together
1568 fpSet.positive.merge(fpSet.negative)
1569 footprints = fpSet.positive
1570 self.log.info('Found %d potential clipped objects', len(footprints.getFootprints()))
1571 ignoreMask = self.getBadPixelMask()
1573 clipFootprints = []
1574 clipIndices = []
1575 artifactSpanSets = [{'CLIPPED': list()} for _ in tempExpRefList]
1577 # for use by detectClipBig
1578 visitDetectionFootprints = []
1580 dims = [len(tempExpRefList), len(footprints.getFootprints())]
1581 overlapDetArr = numpy.zeros(dims, dtype=numpy.uint16)
1582 ignoreArr = numpy.zeros(dims, dtype=numpy.uint16)
1584 # Loop over masks once and extract/store only relevant overlap metrics and detection footprints
1585 for i, warpRef in enumerate(tempExpRefList):
1586 tmpExpMask = warpRef.get(datasetType=self.getTempExpDatasetName(self.warpType),
1587 immediate=True).getMaskedImage().getMask()
1588 maskVisitDet = tmpExpMask.Factory(tmpExpMask, tmpExpMask.getBBox(afwImage.PARENT),
1589 afwImage.PARENT, True)
1590 maskVisitDet &= maskDetValue
1591 visitFootprints = afwDet.FootprintSet(maskVisitDet, afwDet.Threshold(1))
1592 visitDetectionFootprints.append(visitFootprints)
1594 for j, footprint in enumerate(footprints.getFootprints()):
1595 ignoreArr[i, j] = countMaskFromFootprint(tmpExpMask, footprint, ignoreMask, 0x0)
1596 overlapDetArr[i, j] = countMaskFromFootprint(tmpExpMask, footprint, maskDetValue, ignoreMask)
1598 # build a list of clipped spans for each visit
1599 for j, footprint in enumerate(footprints.getFootprints()):
1600 nPixel = footprint.getArea()
1601 overlap = [] # hold the overlap with each visit
1602 indexList = [] # index of visit in global list
1603 for i in range(len(tempExpRefList)):
1604 ignore = ignoreArr[i, j]
1605 overlapDet = overlapDetArr[i, j]
1606 totPixel = nPixel - ignore
1608 # If we have more bad pixels than detection skip
1609 if ignore > overlapDet or totPixel <= 0.5*nPixel or overlapDet == 0:
1610 continue
1611 overlap.append(overlapDet/float(totPixel))
1612 indexList.append(i)
1614 overlap = numpy.array(overlap)
1615 if not len(overlap):
1616 continue
1618 keep = False # Should this footprint be marked as clipped?
1619 keepIndex = [] # Which tempExps does the clipped footprint belong to
1621 # If footprint only has one overlap use a lower threshold
1622 if len(overlap) == 1:
1623 if overlap[0] > self.config.minClipFootOverlapSingle:
1624 keep = True
1625 keepIndex = [0]
1626 else:
1627 # This is the general case where only visit should be clipped
1628 clipIndex = numpy.where(overlap > self.config.minClipFootOverlap)[0]
1629 if len(clipIndex) == 1:
1630 keep = True
1631 keepIndex = [clipIndex[0]]
1633 # Test if there are clipped objects that overlap two different visits
1634 clipIndex = numpy.where(overlap > self.config.minClipFootOverlapDouble)[0]
1635 if len(clipIndex) == 2 and len(overlap) > 3:
1636 clipIndexComp = numpy.where(overlap <= self.config.minClipFootOverlapDouble)[0]
1637 if numpy.max(overlap[clipIndexComp]) <= self.config.maxClipFootOverlapDouble:
1638 keep = True
1639 keepIndex = clipIndex
1641 if not keep:
1642 continue
1644 for index in keepIndex:
1645 globalIndex = indexList[index]
1646 artifactSpanSets[globalIndex]['CLIPPED'].append(footprint.spans)
1648 clipIndices.append(numpy.array(indexList)[keepIndex])
1649 clipFootprints.append(footprint)
1651 return pipeBase.Struct(clipFootprints=clipFootprints, clipIndices=clipIndices,
1652 clipSpans=artifactSpanSets, detectionFootprints=visitDetectionFootprints)
1654 def detectClipBig(self, clipList, clipFootprints, clipIndices, detectionFootprints,
1655 maskClipValue, maskDetValue, coaddBBox):
1656 """Return individual warp footprints for large artifacts and append
1657 them to ``clipList`` in place.
1659 Identify big footprints composed of many sources in the coadd
1660 difference that may have originated in a large diffuse source in the
1661 coadd. We do this by indentifying all clipped footprints that overlap
1662 significantly with each source in all the coaddTempExps.
1664 Parameters
1665 ----------
1666 clipList : `list`
1667 List of alt mask SpanSets with clipping information. Modified.
1668 clipFootprints : `list`
1669 List of clipped footprints.
1670 clipIndices : `list`
1671 List of which entries in tempExpClipList each footprint belongs to.
1672 maskClipValue
1673 Mask value of clipped pixels.
1674 maskDetValue
1675 Mask value of detected pixels.
1676 coaddBBox : `lsst.geom.Box`
1677 BBox of the coadd and warps.
1679 Returns
1680 -------
1681 bigFootprintsCoadd : `list`
1682 List of big footprints
1683 """
1684 bigFootprintsCoadd = []
1685 ignoreMask = self.getBadPixelMask()
1686 for index, (clippedSpans, visitFootprints) in enumerate(zip(clipList, detectionFootprints)):
1687 maskVisitDet = afwImage.MaskX(coaddBBox, 0x0)
1688 for footprint in visitFootprints.getFootprints():
1689 footprint.spans.setMask(maskVisitDet, maskDetValue)
1691 # build a mask of clipped footprints that are in this visit
1692 clippedFootprintsVisit = []
1693 for foot, clipIndex in zip(clipFootprints, clipIndices):
1694 if index not in clipIndex:
1695 continue
1696 clippedFootprintsVisit.append(foot)
1697 maskVisitClip = maskVisitDet.Factory(maskVisitDet.getBBox(afwImage.PARENT))
1698 afwDet.setMaskFromFootprintList(maskVisitClip, clippedFootprintsVisit, maskClipValue)
1700 bigFootprintsVisit = []
1701 for foot in visitFootprints.getFootprints():
1702 if foot.getArea() < self.config.minBigOverlap:
1703 continue
1704 nCount = countMaskFromFootprint(maskVisitClip, foot, maskClipValue, ignoreMask)
1705 if nCount > self.config.minBigOverlap:
1706 bigFootprintsVisit.append(foot)
1707 bigFootprintsCoadd.append(foot)
1709 for footprint in bigFootprintsVisit:
1710 clippedSpans["CLIPPED"].append(footprint.spans)
1712 return bigFootprintsCoadd
1715class CompareWarpAssembleCoaddConnections(AssembleCoaddConnections):
1716 psfMatchedWarps = pipeBase.connectionTypes.Input(
1717 doc=("PSF-Matched Warps are required by CompareWarp regardless of the coadd type requested. "
1718 "Only PSF-Matched Warps make sense for image subtraction. "
1719 "Therefore, they must be an additional declared input."),
1720 name="{inputCoaddName}Coadd_psfMatchedWarp",
1721 storageClass="ExposureF",
1722 dimensions=("tract", "patch", "skymap", "visit"),
1723 deferLoad=True,
1724 multiple=True
1725 )
1726 templateCoadd = pipeBase.connectionTypes.Output(
1727 doc=("Model of the static sky, used to find temporal artifacts. Typically a PSF-Matched, "
1728 "sigma-clipped coadd. Written if and only if assembleStaticSkyModel.doWrite=True"),
1729 name="{fakesType}{outputCoaddName}CoaddPsfMatched",
1730 storageClass="ExposureF",
1731 dimensions=("tract", "patch", "skymap", "band"),
1732 )
1734 def __init__(self, *, config=None):
1735 super().__init__(config=config)
1736 if not config.assembleStaticSkyModel.doWrite:
1737 self.outputs.remove("templateCoadd")
1738 config.validate()
1741class CompareWarpAssembleCoaddConfig(AssembleCoaddConfig,
1742 pipelineConnections=CompareWarpAssembleCoaddConnections):
1743 assembleStaticSkyModel = pexConfig.ConfigurableField(
1744 target=AssembleCoaddTask,
1745 doc="Task to assemble an artifact-free, PSF-matched Coadd to serve as a"
1746 " naive/first-iteration model of the static sky.",
1747 )
1748 detect = pexConfig.ConfigurableField(
1749 target=SourceDetectionTask,
1750 doc="Detect outlier sources on difference between each psfMatched warp and static sky model"
1751 )
1752 detectTemplate = pexConfig.ConfigurableField(
1753 target=SourceDetectionTask,
1754 doc="Detect sources on static sky model. Only used if doPreserveContainedBySource is True"
1755 )
1756 maskStreaks = pexConfig.ConfigurableField(
1757 target=MaskStreaksTask,
1758 doc="Detect streaks on difference between each psfMatched warp and static sky model. Only used if "
1759 "doFilterMorphological is True. Adds a mask plane to an exposure, with the mask plane name set by"
1760 "streakMaskName"
1761 )
1762 streakMaskName = pexConfig.Field(
1763 dtype=str,
1764 default="STREAK",
1765 doc="Name of mask bit used for streaks"
1766 )
1767 maxNumEpochs = pexConfig.Field(
1768 doc="Charactistic maximum local number of epochs/visits in which an artifact candidate can appear "
1769 "and still be masked. The effective maxNumEpochs is a broken linear function of local "
1770 "number of epochs (N): min(maxFractionEpochsLow*N, maxNumEpochs + maxFractionEpochsHigh*N). "
1771 "For each footprint detected on the image difference between the psfMatched warp and static sky "
1772 "model, if a significant fraction of pixels (defined by spatialThreshold) are residuals in more "
1773 "than the computed effective maxNumEpochs, the artifact candidate is deemed persistant rather "
1774 "than transient and not masked.",
1775 dtype=int,
1776 default=2
1777 )
1778 maxFractionEpochsLow = pexConfig.RangeField(
1779 doc="Fraction of local number of epochs (N) to use as effective maxNumEpochs for low N. "
1780 "Effective maxNumEpochs = "
1781 "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)",
1782 dtype=float,
1783 default=0.4,
1784 min=0., max=1.,
1785 )
1786 maxFractionEpochsHigh = pexConfig.RangeField(
1787 doc="Fraction of local number of epochs (N) to use as effective maxNumEpochs for high N. "
1788 "Effective maxNumEpochs = "
1789 "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)",
1790 dtype=float,
1791 default=0.03,
1792 min=0., max=1.,
1793 )
1794 spatialThreshold = pexConfig.RangeField(
1795 doc="Unitless fraction of pixels defining how much of the outlier region has to meet the "
1796 "temporal criteria. If 0, clip all. If 1, clip none.",
1797 dtype=float,
1798 default=0.5,
1799 min=0., max=1.,
1800 inclusiveMin=True, inclusiveMax=True
1801 )
1802 doScaleWarpVariance = pexConfig.Field(
1803 doc="Rescale Warp variance plane using empirical noise?",
1804 dtype=bool,
1805 default=True,
1806 )
1807 scaleWarpVariance = pexConfig.ConfigurableField(
1808 target=ScaleVarianceTask,
1809 doc="Rescale variance on warps",
1810 )
1811 doPreserveContainedBySource = pexConfig.Field(
1812 doc="Rescue artifacts from clipping that completely lie within a footprint detected"
1813 "on the PsfMatched Template Coadd. Replicates a behavior of SafeClip.",
1814 dtype=bool,
1815 default=True,
1816 )
1817 doPrefilterArtifacts = pexConfig.Field(
1818 doc="Ignore artifact candidates that are mostly covered by the bad pixel mask, "
1819 "because they will be excluded anyway. This prevents them from contributing "
1820 "to the outlier epoch count image and potentially being labeled as persistant."
1821 "'Mostly' is defined by the config 'prefilterArtifactsRatio'.",
1822 dtype=bool,
1823 default=True
1824 )
1825 prefilterArtifactsMaskPlanes = pexConfig.ListField(
1826 doc="Prefilter artifact candidates that are mostly covered by these bad mask planes.",
1827 dtype=str,
1828 default=('NO_DATA', 'BAD', 'SAT', 'SUSPECT'),
1829 )
1830 prefilterArtifactsRatio = pexConfig.Field(
1831 doc="Prefilter artifact candidates with less than this fraction overlapping good pixels",
1832 dtype=float,
1833 default=0.05
1834 )
1835 doFilterMorphological = pexConfig.Field(
1836 doc="Filter artifact candidates based on morphological criteria, i.g. those that appear to "
1837 "be streaks.",
1838 dtype=bool,
1839 default=False
1840 )
1842 def setDefaults(self):
1843 AssembleCoaddConfig.setDefaults(self)
1844 self.statistic = 'MEAN'
1845 self.doUsePsfMatchedPolygons = True
1847 # Real EDGE removed by psfMatched NO_DATA border half the width of the matching kernel
1848 # CompareWarp applies psfMatched EDGE pixels to directWarps before assembling
1849 if "EDGE" in self.badMaskPlanes:
1850 self.badMaskPlanes.remove('EDGE')
1851 self.removeMaskPlanes.append('EDGE')
1852 self.assembleStaticSkyModel.badMaskPlanes = ["NO_DATA", ]
1853 self.assembleStaticSkyModel.warpType = 'psfMatched'
1854 self.assembleStaticSkyModel.connections.warpType = 'psfMatched'
1855 self.assembleStaticSkyModel.statistic = 'MEANCLIP'
1856 self.assembleStaticSkyModel.sigmaClip = 2.5
1857 self.assembleStaticSkyModel.clipIter = 3
1858 self.assembleStaticSkyModel.calcErrorFromInputVariance = False
1859 self.assembleStaticSkyModel.doWrite = False
1860 self.detect.doTempLocalBackground = False
1861 self.detect.reEstimateBackground = False
1862 self.detect.returnOriginalFootprints = False
1863 self.detect.thresholdPolarity = "both"
1864 self.detect.thresholdValue = 5
1865 self.detect.minPixels = 4
1866 self.detect.isotropicGrow = True
1867 self.detect.thresholdType = "pixel_stdev"
1868 self.detect.nSigmaToGrow = 0.4
1869 # The default nSigmaToGrow for SourceDetectionTask is already 2.4,
1870 # Explicitly restating because ratio with detect.nSigmaToGrow matters
1871 self.detectTemplate.nSigmaToGrow = 2.4
1872 self.detectTemplate.doTempLocalBackground = False
1873 self.detectTemplate.reEstimateBackground = False
1874 self.detectTemplate.returnOriginalFootprints = False
1876 def validate(self):
1877 super().validate()
1878 if self.assembleStaticSkyModel.doNImage:
1879 raise ValueError("No dataset type exists for a PSF-Matched Template N Image."
1880 "Please set assembleStaticSkyModel.doNImage=False")
1882 if self.assembleStaticSkyModel.doWrite and (self.warpType == self.assembleStaticSkyModel.warpType):
1883 raise ValueError("warpType (%s) == assembleStaticSkyModel.warpType (%s) and will compete for "
1884 "the same dataset name. Please set assembleStaticSkyModel.doWrite to False "
1885 "or warpType to 'direct'. assembleStaticSkyModel.warpType should ways be "
1886 "'PsfMatched'" % (self.warpType, self.assembleStaticSkyModel.warpType))
1889class CompareWarpAssembleCoaddTask(AssembleCoaddTask):
1890 """Assemble a compareWarp coadded image from a set of warps
1891 by masking artifacts detected by comparing PSF-matched warps.
1893 In ``AssembleCoaddTask``, we compute the coadd as an clipped mean (i.e.,
1894 we clip outliers). The problem with doing this is that when computing the
1895 coadd PSF at a given location, individual visit PSFs from visits with
1896 outlier pixels contribute to the coadd PSF and cannot be treated correctly.
1897 In this task, we correct for this behavior by creating a new badMaskPlane
1898 'CLIPPED' which marks pixels in the individual warps suspected to contain
1899 an artifact. We populate this plane on the input warps by comparing
1900 PSF-matched warps with a PSF-matched median coadd which serves as a
1901 model of the static sky. Any group of pixels that deviates from the
1902 PSF-matched template coadd by more than config.detect.threshold sigma,
1903 is an artifact candidate. The candidates are then filtered to remove
1904 variable sources and sources that are difficult to subtract such as
1905 bright stars. This filter is configured using the config parameters
1906 ``temporalThreshold`` and ``spatialThreshold``. The temporalThreshold is
1907 the maximum fraction of epochs that the deviation can appear in and still
1908 be considered an artifact. The spatialThreshold is the maximum fraction of
1909 pixels in the footprint of the deviation that appear in other epochs
1910 (where other epochs is defined by the temporalThreshold). If the deviant
1911 region meets this criteria of having a significant percentage of pixels
1912 that deviate in only a few epochs, these pixels have the 'CLIPPED' bit
1913 set in the mask. These regions will not contribute to the final coadd.
1914 Furthermore, any routine to determine the coadd PSF can now be cognizant
1915 of clipped regions. Note that the algorithm implemented by this task is
1916 preliminary and works correctly for HSC data. Parameter modifications and
1917 or considerable redesigning of the algorithm is likley required for other
1918 surveys.
1920 ``CompareWarpAssembleCoaddTask`` sub-classes
1921 ``AssembleCoaddTask`` and instantiates ``AssembleCoaddTask``
1922 as a subtask to generate the TemplateCoadd (the model of the static sky).
1924 Notes
1925 -----
1926 The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a
1927 flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``; see
1928 ``baseDebug`` for more about ``debug.py`` files.
1930 This task supports the following debug variables:
1932 - ``saveCountIm``
1933 If True then save the Epoch Count Image as a fits file in the `figPath`
1934 - ``figPath``
1935 Path to save the debug fits images and figures
1937 For example, put something like:
1939 .. code-block:: python
1941 import lsstDebug
1942 def DebugInfo(name):
1943 di = lsstDebug.getInfo(name)
1944 if name == "lsst.pipe.tasks.assembleCoadd":
1945 di.saveCountIm = True
1946 di.figPath = "/desired/path/to/debugging/output/images"
1947 return di
1948 lsstDebug.Info = DebugInfo
1950 into your ``debug.py`` file and run ``assemebleCoadd.py`` with the
1951 ``--debug`` flag. Some subtasks may have their own debug variables;
1952 see individual Task documentation.
1954 Examples
1955 --------
1956 ``CompareWarpAssembleCoaddTask`` assembles a set of warped images into a
1957 coadded image. The ``CompareWarpAssembleCoaddTask`` is invoked by running
1958 ``assembleCoadd.py`` with the flag ``--compareWarpCoadd``.
1959 Usage of ``assembleCoadd.py`` expects a data reference to the tract patch
1960 and filter to be coadded (specified using
1961 '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]')
1962 along with a list of coaddTempExps to attempt to coadd (specified using
1963 '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]').
1964 Only the warps that cover the specified tract and patch will be coadded.
1965 A list of the available optional arguments can be obtained by calling
1966 ``assembleCoadd.py`` with the ``--help`` command line argument:
1968 .. code-block:: none
1970 assembleCoadd.py --help
1972 To demonstrate usage of the ``CompareWarpAssembleCoaddTask`` in the larger
1973 context of multi-band processing, we will generate the HSC-I & -R band
1974 oadds from HSC engineering test data provided in the ``ci_hsc`` package.
1975 To begin, assuming that the lsst stack has been already set up, we must
1976 set up the ``obs_subaru`` and ``ci_hsc`` packages.
1977 This defines the environment variable ``$CI_HSC_DIR`` and points at the
1978 location of the package. The raw HSC data live in the ``$CI_HSC_DIR/raw``
1979 directory. To begin assembling the coadds, we must first
1981 - processCcd
1982 process the individual ccds in $CI_HSC_RAW to produce calibrated exposures
1983 - makeSkyMap
1984 create a skymap that covers the area of the sky present in the raw exposures
1985 - makeCoaddTempExp
1986 warp the individual calibrated exposures to the tangent plane of the coadd
1988 We can perform all of these steps by running
1990 .. code-block:: none
1992 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
1994 This will produce warped ``coaddTempExps`` for each visit. To coadd the
1995 warped data, we call ``assembleCoadd.py`` as follows:
1997 .. code-block:: none
1999 assembleCoadd.py --compareWarpCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \
2000 --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \
2001 --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \
2002 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \
2003 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \
2004 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \
2005 --selectId visit=903988 ccd=24
2007 This will process the HSC-I band data. The results are written in
2008 ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``.
2009 """
2010 ConfigClass = CompareWarpAssembleCoaddConfig
2011 _DefaultName = "compareWarpAssembleCoadd"
2013 def __init__(self, *args, **kwargs):
2014 AssembleCoaddTask.__init__(self, *args, **kwargs)
2015 self.makeSubtask("assembleStaticSkyModel")
2016 detectionSchema = afwTable.SourceTable.makeMinimalSchema()
2017 self.makeSubtask("detect", schema=detectionSchema)
2018 if self.config.doPreserveContainedBySource:
2019 self.makeSubtask("detectTemplate", schema=afwTable.SourceTable.makeMinimalSchema())
2020 if self.config.doScaleWarpVariance:
2021 self.makeSubtask("scaleWarpVariance")
2022 if self.config.doFilterMorphological:
2023 self.makeSubtask("maskStreaks")
2025 @utils.inheritDoc(AssembleCoaddTask)
2026 def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs):
2027 """
2028 Generate a templateCoadd to use as a naive model of static sky to
2029 subtract from PSF-Matched warps.
2031 Returns
2032 -------
2033 result : `lsst.pipe.base.Struct`
2034 Result struct with components:
2036 - ``templateCoadd`` : coadded exposure (``lsst.afw.image.Exposure``)
2037 - ``nImage`` : N Image (``lsst.afw.image.Image``)
2038 """
2039 # Ensure that psfMatchedWarps are used as input warps for template generation
2040 staticSkyModelInputRefs = copy.deepcopy(inputRefs)
2041 staticSkyModelInputRefs.inputWarps = inputRefs.psfMatchedWarps
2043 # Because subtasks don't have connections we have to make one.
2044 # The main task's `templateCoadd` is the subtask's `coaddExposure`
2045 staticSkyModelOutputRefs = copy.deepcopy(outputRefs)
2046 if self.config.assembleStaticSkyModel.doWrite:
2047 staticSkyModelOutputRefs.coaddExposure = staticSkyModelOutputRefs.templateCoadd
2048 # Remove template coadd from both subtask's and main tasks outputs,
2049 # because it is handled by the subtask as `coaddExposure`
2050 del outputRefs.templateCoadd
2051 del staticSkyModelOutputRefs.templateCoadd
2053 # A PSF-Matched nImage does not exist as a dataset type
2054 if 'nImage' in staticSkyModelOutputRefs.keys():
2055 del staticSkyModelOutputRefs.nImage
2057 templateCoadd = self.assembleStaticSkyModel.runQuantum(butlerQC, staticSkyModelInputRefs,
2058 staticSkyModelOutputRefs)
2059 if templateCoadd is None:
2060 raise RuntimeError(self._noTemplateMessage(self.assembleStaticSkyModel.warpType))
2062 return pipeBase.Struct(templateCoadd=templateCoadd.coaddExposure,
2063 nImage=templateCoadd.nImage,
2064 warpRefList=templateCoadd.warpRefList,
2065 imageScalerList=templateCoadd.imageScalerList,
2066 weightList=templateCoadd.weightList)
2068 @utils.inheritDoc(AssembleCoaddTask)
2069 def makeSupplementaryData(self, dataRef, selectDataList=None, warpRefList=None):
2070 """
2071 Generate a templateCoadd to use as a naive model of static sky to
2072 subtract from PSF-Matched warps.
2074 Returns
2075 -------
2076 result : `lsst.pipe.base.Struct`
2077 Result struct with components:
2079 - ``templateCoadd``: coadded exposure (``lsst.afw.image.Exposure``)
2080 - ``nImage``: N Image (``lsst.afw.image.Image``)
2081 """
2082 templateCoadd = self.assembleStaticSkyModel.runDataRef(dataRef, selectDataList, warpRefList)
2083 if templateCoadd is None:
2084 raise RuntimeError(self._noTemplateMessage(self.assembleStaticSkyModel.warpType))
2086 return pipeBase.Struct(templateCoadd=templateCoadd.coaddExposure,
2087 nImage=templateCoadd.nImage,
2088 warpRefList=templateCoadd.warpRefList,
2089 imageScalerList=templateCoadd.imageScalerList,
2090 weightList=templateCoadd.weightList)
2092 def _noTemplateMessage(self, warpType):
2093 warpName = (warpType[0].upper() + warpType[1:])
2094 message = """No %(warpName)s warps were found to build the template coadd which is
2095 required to run CompareWarpAssembleCoaddTask. To continue assembling this type of coadd,
2096 first either rerun makeCoaddTempExp with config.make%(warpName)s=True or
2097 coaddDriver with config.makeCoadTempExp.make%(warpName)s=True, before assembleCoadd.
2099 Alternatively, to use another algorithm with existing warps, retarget the CoaddDriverConfig to
2100 another algorithm like:
2102 from lsst.pipe.tasks.assembleCoadd import SafeClipAssembleCoaddTask
2103 config.assemble.retarget(SafeClipAssembleCoaddTask)
2104 """ % {"warpName": warpName}
2105 return message
2107 @utils.inheritDoc(AssembleCoaddTask)
2108 @pipeBase.timeMethod
2109 def run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
2110 supplementaryData, *args, **kwargs):
2111 """Assemble the coadd.
2113 Find artifacts and apply them to the warps' masks creating a list of
2114 alternative masks with a new "CLIPPED" plane and updated "NO_DATA"
2115 plane. Then pass these alternative masks to the base class's `run`
2116 method.
2118 The input parameters ``supplementaryData`` is a `lsst.pipe.base.Struct`
2119 that must contain a ``templateCoadd`` that serves as the
2120 model of the static sky.
2121 """
2123 # Check and match the order of the supplementaryData
2124 # (PSF-matched) inputs to the order of the direct inputs,
2125 # so that the artifact mask is applied to the right warp
2126 dataIds = [ref.dataId for ref in tempExpRefList]
2127 psfMatchedDataIds = [ref.dataId for ref in supplementaryData.warpRefList]
2129 if dataIds != psfMatchedDataIds:
2130 self.log.info("Reordering and or/padding PSF-matched visit input list")
2131 supplementaryData.warpRefList = reorderAndPadList(supplementaryData.warpRefList,
2132 psfMatchedDataIds, dataIds)
2133 supplementaryData.imageScalerList = reorderAndPadList(supplementaryData.imageScalerList,
2134 psfMatchedDataIds, dataIds)
2136 # Use PSF-Matched Warps (and corresponding scalers) and coadd to find artifacts
2137 spanSetMaskList = self.findArtifacts(supplementaryData.templateCoadd,
2138 supplementaryData.warpRefList,
2139 supplementaryData.imageScalerList)
2141 badMaskPlanes = self.config.badMaskPlanes[:]
2142 badMaskPlanes.append("CLIPPED")
2143 badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes)
2145 result = AssembleCoaddTask.run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
2146 spanSetMaskList, mask=badPixelMask)
2148 # Propagate PSF-matched EDGE pixels to coadd SENSOR_EDGE and INEXACT_PSF
2149 # Psf-Matching moves the real edge inwards
2150 self.applyAltEdgeMask(result.coaddExposure.maskedImage.mask, spanSetMaskList)
2151 return result
2153 def applyAltEdgeMask(self, mask, altMaskList):
2154 """Propagate alt EDGE mask to SENSOR_EDGE AND INEXACT_PSF planes.
2156 Parameters
2157 ----------
2158 mask : `lsst.afw.image.Mask`
2159 Original mask.
2160 altMaskList : `list`
2161 List of Dicts containing ``spanSet`` lists.
2162 Each element contains the new mask plane name (e.g. "CLIPPED
2163 and/or "NO_DATA") as the key, and list of ``SpanSets`` to apply to
2164 the mask.
2165 """
2166 maskValue = mask.getPlaneBitMask(["SENSOR_EDGE", "INEXACT_PSF"])
2167 for visitMask in altMaskList:
2168 if "EDGE" in visitMask:
2169 for spanSet in visitMask['EDGE']:
2170 spanSet.clippedTo(mask.getBBox()).setMask(mask, maskValue)
2172 def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList):
2173 """Find artifacts.
2175 Loop through warps twice. The first loop builds a map with the count
2176 of how many epochs each pixel deviates from the templateCoadd by more
2177 than ``config.chiThreshold`` sigma. The second loop takes each
2178 difference image and filters the artifacts detected in each using
2179 count map to filter out variable sources and sources that are
2180 difficult to subtract cleanly.
2182 Parameters
2183 ----------
2184 templateCoadd : `lsst.afw.image.Exposure`
2185 Exposure to serve as model of static sky.
2186 tempExpRefList : `list`
2187 List of data references to warps.
2188 imageScalerList : `list`
2189 List of image scalers.
2191 Returns
2192 -------
2193 altMasks : `list`
2194 List of dicts containing information about CLIPPED
2195 (i.e., artifacts), NO_DATA, and EDGE pixels.
2196 """
2198 self.log.debug("Generating Count Image, and mask lists.")
2199 coaddBBox = templateCoadd.getBBox()
2200 slateIm = afwImage.ImageU(coaddBBox)
2201 epochCountImage = afwImage.ImageU(coaddBBox)
2202 nImage = afwImage.ImageU(coaddBBox)
2203 spanSetArtifactList = []
2204 spanSetNoDataMaskList = []
2205 spanSetEdgeList = []
2206 spanSetBadMorphoList = []
2207 badPixelMask = self.getBadPixelMask()
2209 # mask of the warp diffs should = that of only the warp
2210 templateCoadd.mask.clearAllMaskPlanes()
2212 if self.config.doPreserveContainedBySource:
2213 templateFootprints = self.detectTemplate.detectFootprints(templateCoadd)
2214 else:
2215 templateFootprints = None
2217 for warpRef, imageScaler in zip(tempExpRefList, imageScalerList):
2218 warpDiffExp = self._readAndComputeWarpDiff(warpRef, imageScaler, templateCoadd)
2219 if warpDiffExp is not None:
2220 # This nImage only approximates the final nImage because it uses the PSF-matched mask
2221 nImage.array += (numpy.isfinite(warpDiffExp.image.array)
2222 * ((warpDiffExp.mask.array & badPixelMask) == 0)).astype(numpy.uint16)
2223 fpSet = self.detect.detectFootprints(warpDiffExp, doSmooth=False, clearMask=True)
2224 fpSet.positive.merge(fpSet.negative)
2225 footprints = fpSet.positive
2226 slateIm.set(0)
2227 spanSetList = [footprint.spans for footprint in footprints.getFootprints()]
2229 # Remove artifacts due to defects before they contribute to the epochCountImage
2230 if self.config.doPrefilterArtifacts:
2231 spanSetList = self.prefilterArtifacts(spanSetList, warpDiffExp)
2233 # Clear mask before adding prefiltered spanSets
2234 self.detect.clearMask(warpDiffExp.mask)
2235 for spans in spanSetList:
2236 spans.setImage(slateIm, 1, doClip=True)
2237 spans.setMask(warpDiffExp.mask, warpDiffExp.mask.getPlaneBitMask("DETECTED"))
2238 epochCountImage += slateIm
2240 if self.config.doFilterMorphological:
2241 maskName = self.config.streakMaskName
2242 _ = self.maskStreaks.run(warpDiffExp)
2243 streakMask = warpDiffExp.mask
2244 spanSetStreak = afwGeom.SpanSet.fromMask(streakMask,
2245 streakMask.getPlaneBitMask(maskName)).split()
2247 # PSF-Matched warps have less available area (~the matching kernel) because the calexps
2248 # undergo a second convolution. Pixels with data in the direct warp
2249 # but not in the PSF-matched warp will not have their artifacts detected.
2250 # NaNs from the PSF-matched warp therefore must be masked in the direct warp
2251 nans = numpy.where(numpy.isnan(warpDiffExp.maskedImage.image.array), 1, 0)
2252 nansMask = afwImage.makeMaskFromArray(nans.astype(afwImage.MaskPixel))
2253 nansMask.setXY0(warpDiffExp.getXY0())
2254 edgeMask = warpDiffExp.mask
2255 spanSetEdgeMask = afwGeom.SpanSet.fromMask(edgeMask,
2256 edgeMask.getPlaneBitMask("EDGE")).split()
2257 else:
2258 # If the directWarp has <1% coverage, the psfMatchedWarp can have 0% and not exist
2259 # In this case, mask the whole epoch
2260 nansMask = afwImage.MaskX(coaddBBox, 1)
2261 spanSetList = []
2262 spanSetEdgeMask = []
2263 spanSetStreak = []
2265 spanSetNoDataMask = afwGeom.SpanSet.fromMask(nansMask).split()
2267 spanSetNoDataMaskList.append(spanSetNoDataMask)
2268 spanSetArtifactList.append(spanSetList)
2269 spanSetEdgeList.append(spanSetEdgeMask)
2270 if self.config.doFilterMorphological:
2271 spanSetBadMorphoList.append(spanSetStreak)
2273 if lsstDebug.Info(__name__).saveCountIm:
2274 path = self._dataRef2DebugPath("epochCountIm", tempExpRefList[0], coaddLevel=True)
2275 epochCountImage.writeFits(path)
2277 for i, spanSetList in enumerate(spanSetArtifactList):
2278 if spanSetList:
2279 filteredSpanSetList = self.filterArtifacts(spanSetList, epochCountImage, nImage,
2280 templateFootprints)
2281 spanSetArtifactList[i] = filteredSpanSetList
2282 if self.config.doFilterMorphological:
2283 spanSetArtifactList[i] += spanSetBadMorphoList[i]
2285 altMasks = []
2286 for artifacts, noData, edge in zip(spanSetArtifactList, spanSetNoDataMaskList, spanSetEdgeList):
2287 altMasks.append({'CLIPPED': artifacts,
2288 'NO_DATA': noData,
2289 'EDGE': edge})
2290 return altMasks
2292 def prefilterArtifacts(self, spanSetList, exp):
2293 """Remove artifact candidates covered by bad mask plane.
2295 Any future editing of the candidate list that does not depend on
2296 temporal information should go in this method.
2298 Parameters
2299 ----------
2300 spanSetList : `list`
2301 List of SpanSets representing artifact candidates.
2302 exp : `lsst.afw.image.Exposure`
2303 Exposure containing mask planes used to prefilter.
2305 Returns
2306 -------
2307 returnSpanSetList : `list`
2308 List of SpanSets with artifacts.
2309 """
2310 badPixelMask = exp.mask.getPlaneBitMask(self.config.prefilterArtifactsMaskPlanes)
2311 goodArr = (exp.mask.array & badPixelMask) == 0
2312 returnSpanSetList = []
2313 bbox = exp.getBBox()
2314 x0, y0 = exp.getXY0()
2315 for i, span in enumerate(spanSetList):
2316 y, x = span.clippedTo(bbox).indices()
2317 yIndexLocal = numpy.array(y) - y0
2318 xIndexLocal = numpy.array(x) - x0
2319 goodRatio = numpy.count_nonzero(goodArr[yIndexLocal, xIndexLocal])/span.getArea()
2320 if goodRatio > self.config.prefilterArtifactsRatio:
2321 returnSpanSetList.append(span)
2322 return returnSpanSetList
2324 def filterArtifacts(self, spanSetList, epochCountImage, nImage, footprintsToExclude=None):
2325 """Filter artifact candidates.
2327 Parameters
2328 ----------
2329 spanSetList : `list`
2330 List of SpanSets representing artifact candidates.
2331 epochCountImage : `lsst.afw.image.Image`
2332 Image of accumulated number of warpDiff detections.
2333 nImage : `lsst.afw.image.Image`
2334 Image of the accumulated number of total epochs contributing.
2336 Returns
2337 -------
2338 maskSpanSetList : `list`
2339 List of SpanSets with artifacts.
2340 """
2342 maskSpanSetList = []
2343 x0, y0 = epochCountImage.getXY0()
2344 for i, span in enumerate(spanSetList):
2345 y, x = span.indices()
2346 yIdxLocal = [y1 - y0 for y1 in y]
2347 xIdxLocal = [x1 - x0 for x1 in x]
2348 outlierN = epochCountImage.array[yIdxLocal, xIdxLocal]
2349 totalN = nImage.array[yIdxLocal, xIdxLocal]
2351 # effectiveMaxNumEpochs is broken line (fraction of N) with characteristic config.maxNumEpochs
2352 effMaxNumEpochsHighN = (self.config.maxNumEpochs
2353 + self.config.maxFractionEpochsHigh*numpy.mean(totalN))
2354 effMaxNumEpochsLowN = self.config.maxFractionEpochsLow * numpy.mean(totalN)
2355 effectiveMaxNumEpochs = int(min(effMaxNumEpochsLowN, effMaxNumEpochsHighN))
2356 nPixelsBelowThreshold = numpy.count_nonzero((outlierN > 0)
2357 & (outlierN <= effectiveMaxNumEpochs))
2358 percentBelowThreshold = nPixelsBelowThreshold / len(outlierN)
2359 if percentBelowThreshold > self.config.spatialThreshold:
2360 maskSpanSetList.append(span)
2362 if self.config.doPreserveContainedBySource and footprintsToExclude is not None:
2363 # If a candidate is contained by a footprint on the template coadd, do not clip
2364 filteredMaskSpanSetList = []
2365 for span in maskSpanSetList:
2366 doKeep = True
2367 for footprint in footprintsToExclude.positive.getFootprints():
2368 if footprint.spans.contains(span):
2369 doKeep = False
2370 break
2371 if doKeep:
2372 filteredMaskSpanSetList.append(span)
2373 maskSpanSetList = filteredMaskSpanSetList
2375 return maskSpanSetList
2377 def _readAndComputeWarpDiff(self, warpRef, imageScaler, templateCoadd):
2378 """Fetch a warp from the butler and return a warpDiff.
2380 Parameters
2381 ----------
2382 warpRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
2383 Butler dataRef for the warp.
2384 imageScaler : `lsst.pipe.tasks.scaleZeroPoint.ImageScaler`
2385 An image scaler object.
2386 templateCoadd : `lsst.afw.image.Exposure`
2387 Exposure to be substracted from the scaled warp.
2389 Returns
2390 -------
2391 warp : `lsst.afw.image.Exposure`
2392 Exposure of the image difference between the warp and template.
2393 """
2395 # If the PSF-Matched warp did not exist for this direct warp
2396 # None is holding its place to maintain order in Gen 3
2397 if warpRef is None:
2398 return None
2399 # Warp comparison must use PSF-Matched Warps regardless of requested coadd warp type
2400 warpName = self.getTempExpDatasetName('psfMatched')
2401 if not isinstance(warpRef, DeferredDatasetHandle):
2402 if not warpRef.datasetExists(warpName):
2403 self.log.warn("Could not find %s %s; skipping it", warpName, warpRef.dataId)
2404 return None
2405 warp = warpRef.get(datasetType=warpName, immediate=True)
2406 # direct image scaler OK for PSF-matched Warp
2407 imageScaler.scaleMaskedImage(warp.getMaskedImage())
2408 mi = warp.getMaskedImage()
2409 if self.config.doScaleWarpVariance:
2410 try:
2411 self.scaleWarpVariance.run(mi)
2412 except Exception as exc:
2413 self.log.warn("Unable to rescale variance of warp (%s); leaving it as-is" % (exc,))
2414 mi -= templateCoadd.getMaskedImage()
2415 return warp
2417 def _dataRef2DebugPath(self, prefix, warpRef, coaddLevel=False):
2418 """Return a path to which to write debugging output.
2420 Creates a hyphen-delimited string of dataId values for simple filenames.
2422 Parameters
2423 ----------
2424 prefix : `str`
2425 Prefix for filename.
2426 warpRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
2427 Butler dataRef to make the path from.
2428 coaddLevel : `bool`, optional.
2429 If True, include only coadd-level keys (e.g., 'tract', 'patch',
2430 'filter', but no 'visit').
2432 Returns
2433 -------
2434 result : `str`
2435 Path for debugging output.
2436 """
2437 if coaddLevel:
2438 keys = warpRef.getButler().getKeys(self.getCoaddDatasetName(self.warpType))
2439 else:
2440 keys = warpRef.dataId.keys()
2441 keyList = sorted(keys, reverse=True)
2442 directory = lsstDebug.Info(__name__).figPath if lsstDebug.Info(__name__).figPath else "."
2443 filename = "%s-%s.fits" % (prefix, '-'.join([str(warpRef.dataId[k]) for k in keyList]))
2444 return os.path.join(directory, filename)
2447def reorderAndPadList(inputList, inputKeys, outputKeys, padWith=None):
2448 """Match the order of one list to another, padding if necessary
2450 Parameters
2451 ----------
2452 inputList : list
2453 List to be reordered and padded. Elements can be any type.
2454 inputKeys : iterable
2455 Iterable of values to be compared with outputKeys.
2456 Length must match `inputList`
2457 outputKeys : iterable
2458 Iterable of values to be compared with inputKeys.
2459 padWith :
2460 Any value to be inserted where inputKey not in outputKeys
2462 Returns
2463 -------
2464 list
2465 Copy of inputList reordered per outputKeys and padded with `padWith`
2466 so that the length matches length of outputKeys.
2467 """
2468 outputList = []
2469 for d in outputKeys:
2470 if d in inputKeys:
2471 outputList.append(inputList[inputKeys.index(d)])
2472 else:
2473 outputList.append(padWith)
2474 return outputList