Coverage for python/lsst/pipe/tasks/assembleCoadd.py : 13%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of pipe_tasks.
2#
3# LSST Data Management System
4# This product includes software developed by the
5# LSST Project (http://www.lsst.org/).
6# See COPYRIGHT file at the top of the source tree.
7#
8# This program is free software: you can redistribute it and/or modify
9# it under the terms of the GNU General Public License as published by
10# the Free Software Foundation, either version 3 of the License, or
11# (at your option) any later version.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the LSST License Statement and
19# the GNU General Public License along with this program. If not,
20# see <https://www.lsstcorp.org/LegalNotices/>.
21#
22import os
23import copy
24import numpy
25import warnings
26import lsst.pex.config as pexConfig
27import lsst.pex.exceptions as pexExceptions
28import lsst.geom as geom
29import lsst.afw.geom as afwGeom
30import lsst.afw.image as afwImage
31import lsst.afw.math as afwMath
32import lsst.afw.table as afwTable
33import lsst.afw.detection as afwDet
34import lsst.coadd.utils as coaddUtils
35import lsst.pipe.base as pipeBase
36import lsst.meas.algorithms as measAlg
37import lsst.log as log
38import lsstDebug
39import lsst.utils as utils
40from .coaddBase import CoaddBaseTask, SelectDataIdContainer, makeSkyInfo, makeCoaddSuffix
41from .interpImage import InterpImageTask
42from .scaleZeroPoint import ScaleZeroPointTask
43from .coaddHelpers import groupPatchExposures, getGroupDataRef
44from .scaleVariance import ScaleVarianceTask
45from lsst.meas.algorithms import SourceDetectionTask
46from lsst.daf.butler import DeferredDatasetHandle
48__all__ = ["AssembleCoaddTask", "AssembleCoaddConnections", "AssembleCoaddConfig",
49 "SafeClipAssembleCoaddTask", "SafeClipAssembleCoaddConfig",
50 "CompareWarpAssembleCoaddTask", "CompareWarpAssembleCoaddConfig"]
53class AssembleCoaddConnections(pipeBase.PipelineTaskConnections,
54 dimensions=("tract", "patch", "abstract_filter", "skymap"),
55 defaultTemplates={"inputCoaddName": "deep",
56 "outputCoaddName": "deep",
57 "warpType": "direct",
58 "warpTypeSuffix": "",
59 "fakesType": ""}):
60 inputWarps = pipeBase.connectionTypes.Input(
61 doc=("Input list of warps to be assemebled i.e. stacked."
62 "WarpType (e.g. direct, psfMatched) is controlled by the warpType config parameter"),
63 name="{inputCoaddName}Coadd_{warpType}Warp",
64 storageClass="ExposureF",
65 dimensions=("tract", "patch", "skymap", "visit", "instrument"),
66 deferLoad=True,
67 multiple=True
68 )
69 skyMap = pipeBase.connectionTypes.Input(
70 doc="Input definition of geometry/bbox and projection/wcs for coadded exposures",
71 name="{inputCoaddName}Coadd_skyMap",
72 storageClass="SkyMap",
73 dimensions=("skymap", ),
74 )
75 brightObjectMask = pipeBase.connectionTypes.PrerequisiteInput(
76 doc=("Input Bright Object Mask mask produced with external catalogs to be applied to the mask plane"
77 " BRIGHT_OBJECT."),
78 name="brightObjectMask",
79 storageClass="ObjectMaskCatalog",
80 dimensions=("tract", "patch", "skymap", "abstract_filter"),
81 )
82 coaddExposure = pipeBase.connectionTypes.Output(
83 doc="Output coadded exposure, produced by stacking input warps",
84 name="{fakesType}{outputCoaddName}Coadd{warpTypeSuffix}",
85 storageClass="ExposureF",
86 dimensions=("tract", "patch", "skymap", "abstract_filter"),
87 )
88 nImage = pipeBase.connectionTypes.Output(
89 doc="Output image of number of input images per pixel",
90 name="{outputCoaddName}Coadd_nImage",
91 storageClass="ImageU",
92 dimensions=("tract", "patch", "skymap", "abstract_filter"),
93 )
95 def __init__(self, *, config=None):
96 # Override the connection's name template with config to replicate Gen2 behavior
97 config.connections.warpType = config.warpType
98 config.connections.warpTypeSuffix = makeCoaddSuffix(config.warpType)
100 if config.hasFakes:
101 config.connections.fakesType = "_fakes"
103 super().__init__(config=config)
105 if not config.doMaskBrightObjects:
106 self.prerequisiteInputs.remove("brightObjectMask")
108 if not config.doNImage:
109 self.outputs.remove("nImage")
112class AssembleCoaddConfig(CoaddBaseTask.ConfigClass, pipeBase.PipelineTaskConfig,
113 pipelineConnections=AssembleCoaddConnections):
114 """Configuration parameters for the `AssembleCoaddTask`.
116 Notes
117 -----
118 The `doMaskBrightObjects` and `brightObjectMaskName` configuration options
119 only set the bitplane config.brightObjectMaskName. To make this useful you
120 *must* also configure the flags.pixel algorithm, for example by adding
122 .. code-block:: none
124 config.measurement.plugins["base_PixelFlags"].masksFpCenter.append("BRIGHT_OBJECT")
125 config.measurement.plugins["base_PixelFlags"].masksFpAnywhere.append("BRIGHT_OBJECT")
127 to your measureCoaddSources.py and forcedPhotCoadd.py config overrides.
128 """
129 warpType = pexConfig.Field(
130 doc="Warp name: one of 'direct' or 'psfMatched'",
131 dtype=str,
132 default="direct",
133 )
134 subregionSize = pexConfig.ListField(
135 dtype=int,
136 doc="Width, height of stack subregion size; "
137 "make small enough that a full stack of images will fit into memory at once.",
138 length=2,
139 default=(2000, 2000),
140 )
141 statistic = pexConfig.Field(
142 dtype=str,
143 doc="Main stacking statistic for aggregating over the epochs.",
144 default="MEANCLIP",
145 )
146 doSigmaClip = pexConfig.Field(
147 dtype=bool,
148 doc="Perform sigma clipped outlier rejection with MEANCLIP statistic? (DEPRECATED)",
149 default=False,
150 )
151 sigmaClip = pexConfig.Field(
152 dtype=float,
153 doc="Sigma for outlier rejection; ignored if non-clipping statistic selected.",
154 default=3.0,
155 )
156 clipIter = pexConfig.Field(
157 dtype=int,
158 doc="Number of iterations of outlier rejection; ignored if non-clipping statistic selected.",
159 default=2,
160 )
161 calcErrorFromInputVariance = pexConfig.Field(
162 dtype=bool,
163 doc="Calculate coadd variance from input variance by stacking statistic."
164 "Passed to StatisticsControl.setCalcErrorFromInputVariance()",
165 default=True,
166 )
167 scaleZeroPoint = pexConfig.ConfigurableField(
168 target=ScaleZeroPointTask,
169 doc="Task to adjust the photometric zero point of the coadd temp exposures",
170 )
171 doInterp = pexConfig.Field(
172 doc="Interpolate over NaN pixels? Also extrapolate, if necessary, but the results are ugly.",
173 dtype=bool,
174 default=True,
175 )
176 interpImage = pexConfig.ConfigurableField(
177 target=InterpImageTask,
178 doc="Task to interpolate (and extrapolate) over NaN pixels",
179 )
180 doWrite = pexConfig.Field(
181 doc="Persist coadd?",
182 dtype=bool,
183 default=True,
184 )
185 doNImage = pexConfig.Field(
186 doc="Create image of number of contributing exposures for each pixel",
187 dtype=bool,
188 default=False,
189 )
190 doUsePsfMatchedPolygons = pexConfig.Field(
191 doc="Use ValidPolygons from shrunk Psf-Matched Calexps? Should be set to True by CompareWarp only.",
192 dtype=bool,
193 default=False,
194 )
195 maskPropagationThresholds = pexConfig.DictField(
196 keytype=str,
197 itemtype=float,
198 doc=("Threshold (in fractional weight) of rejection at which we propagate a mask plane to "
199 "the coadd; that is, we set the mask bit on the coadd if the fraction the rejected frames "
200 "would have contributed exceeds this value."),
201 default={"SAT": 0.1},
202 )
203 removeMaskPlanes = pexConfig.ListField(dtype=str, default=["NOT_DEBLENDED"],
204 doc="Mask planes to remove before coadding")
205 doMaskBrightObjects = pexConfig.Field(dtype=bool, default=False,
206 doc="Set mask and flag bits for bright objects?")
207 brightObjectMaskName = pexConfig.Field(dtype=str, default="BRIGHT_OBJECT",
208 doc="Name of mask bit used for bright objects")
209 coaddPsf = pexConfig.ConfigField(
210 doc="Configuration for CoaddPsf",
211 dtype=measAlg.CoaddPsfConfig,
212 )
213 doAttachTransmissionCurve = pexConfig.Field(
214 dtype=bool, default=False, optional=False,
215 doc=("Attach a piecewise TransmissionCurve for the coadd? "
216 "(requires all input Exposures to have TransmissionCurves).")
217 )
218 hasFakes = pexConfig.Field(
219 dtype=bool,
220 default=False,
221 doc="Should be set to True if fake sources have been inserted into the input data."
222 )
224 def setDefaults(self):
225 super().setDefaults()
226 self.badMaskPlanes = ["NO_DATA", "BAD", "SAT", "EDGE"]
228 def validate(self):
229 super().validate()
230 if self.doPsfMatch:
231 # Backwards compatibility.
232 # Configs do not have loggers
233 log.warn("Config doPsfMatch deprecated. Setting warpType='psfMatched'")
234 self.warpType = 'psfMatched'
235 if self.doSigmaClip and self.statistic != "MEANCLIP":
236 log.warn('doSigmaClip deprecated. To replicate behavior, setting statistic to "MEANCLIP"')
237 self.statistic = "MEANCLIP"
238 if self.doInterp and self.statistic not in ['MEAN', 'MEDIAN', 'MEANCLIP', 'VARIANCE', 'VARIANCECLIP']:
239 raise ValueError("Must set doInterp=False for statistic=%s, which does not "
240 "compute and set a non-zero coadd variance estimate." % (self.statistic))
242 unstackableStats = ['NOTHING', 'ERROR', 'ORMASK']
243 if not hasattr(afwMath.Property, self.statistic) or self.statistic in unstackableStats:
244 stackableStats = [str(k) for k in afwMath.Property.__members__.keys()
245 if str(k) not in unstackableStats]
246 raise ValueError("statistic %s is not allowed. Please choose one of %s."
247 % (self.statistic, stackableStats))
250class AssembleCoaddTask(CoaddBaseTask, pipeBase.PipelineTask):
251 """Assemble a coadded image from a set of warps (coadded temporary exposures).
253 We want to assemble a coadded image from a set of Warps (also called
254 coadded temporary exposures or ``coaddTempExps``).
255 Each input Warp covers a patch on the sky and corresponds to a single
256 run/visit/exposure of the covered patch. We provide the task with a list
257 of Warps (``selectDataList``) from which it selects Warps that cover the
258 specified patch (pointed at by ``dataRef``).
259 Each Warp that goes into a coadd will typically have an independent
260 photometric zero-point. Therefore, we must scale each Warp to set it to
261 a common photometric zeropoint. WarpType may be one of 'direct' or
262 'psfMatched', and the boolean configs `config.makeDirect` and
263 `config.makePsfMatched` set which of the warp types will be coadded.
264 The coadd is computed as a mean with optional outlier rejection.
265 Criteria for outlier rejection are set in `AssembleCoaddConfig`.
266 Finally, Warps can have bad 'NaN' pixels which received no input from the
267 source calExps. We interpolate over these bad (NaN) pixels.
269 `AssembleCoaddTask` uses several sub-tasks. These are
271 - `ScaleZeroPointTask`
272 - create and use an ``imageScaler`` object to scale the photometric zeropoint for each Warp
273 - `InterpImageTask`
274 - interpolate across bad pixels (NaN) in the final coadd
276 You can retarget these subtasks if you wish.
278 Notes
279 -----
280 The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a
281 flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``; see
282 `baseDebug` for more about ``debug.py`` files. `AssembleCoaddTask` has
283 no debug variables of its own. Some of the subtasks may support debug
284 variables. See the documentation for the subtasks for further information.
286 Examples
287 --------
288 `AssembleCoaddTask` assembles a set of warped images into a coadded image.
289 The `AssembleCoaddTask` can be invoked by running ``assembleCoadd.py``
290 with the flag '--legacyCoadd'. Usage of assembleCoadd.py expects two
291 inputs: a data reference to the tract patch and filter to be coadded, and
292 a list of Warps to attempt to coadd. These are specified using ``--id`` and
293 ``--selectId``, respectively:
295 .. code-block:: none
297 --id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]
298 --selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]
300 Only the Warps that cover the specified tract and patch will be coadded.
301 A list of the available optional arguments can be obtained by calling
302 ``assembleCoadd.py`` with the ``--help`` command line argument:
304 .. code-block:: none
306 assembleCoadd.py --help
308 To demonstrate usage of the `AssembleCoaddTask` in the larger context of
309 multi-band processing, we will generate the HSC-I & -R band coadds from
310 HSC engineering test data provided in the ``ci_hsc`` package. To begin,
311 assuming that the lsst stack has been already set up, we must set up the
312 obs_subaru and ``ci_hsc`` packages. This defines the environment variable
313 ``$CI_HSC_DIR`` and points at the location of the package. The raw HSC
314 data live in the ``$CI_HSC_DIR/raw directory``. To begin assembling the
315 coadds, we must first
317 - processCcd
318 - process the individual ccds in $CI_HSC_RAW to produce calibrated exposures
319 - makeSkyMap
320 - create a skymap that covers the area of the sky present in the raw exposures
321 - makeCoaddTempExp
322 - warp the individual calibrated exposures to the tangent plane of the coadd
324 We can perform all of these steps by running
326 .. code-block:: none
328 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
330 This will produce warped exposures for each visit. To coadd the warped
331 data, we call assembleCoadd.py as follows:
333 .. code-block:: none
335 assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \
336 --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \
337 --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \
338 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \
339 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \
340 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \
341 --selectId visit=903988 ccd=24
343 that will process the HSC-I band data. The results are written in
344 ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``.
346 You may also choose to run:
348 .. code-block:: none
350 scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346
351 assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R \
352 --selectId visit=903334 ccd=16 --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 \
353 --selectId visit=903334 ccd=100 --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 \
354 --selectId visit=903338 ccd=18 --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 \
355 --selectId visit=903342 ccd=10 --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 \
356 --selectId visit=903344 ccd=5 --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 \
357 --selectId visit=903346 ccd=6 --selectId visit=903346 ccd=12
359 to generate the coadd for the HSC-R band if you are interested in
360 following multiBand Coadd processing as discussed in `pipeTasks_multiBand`
361 (but note that normally, one would use the `SafeClipAssembleCoaddTask`
362 rather than `AssembleCoaddTask` to make the coadd.
363 """
364 ConfigClass = AssembleCoaddConfig
365 _DefaultName = "assembleCoadd"
367 def __init__(self, *args, **kwargs):
368 # TODO: DM-17415 better way to handle previously allowed passed args e.g.`AssembleCoaddTask(config)`
369 if args:
370 argNames = ["config", "name", "parentTask", "log"]
371 kwargs.update({k: v for k, v in zip(argNames, args)})
372 warnings.warn("AssembleCoadd received positional args, and casting them as kwargs: %s. "
373 "PipelineTask will not take positional args" % argNames, FutureWarning)
375 super().__init__(**kwargs)
376 self.makeSubtask("interpImage")
377 self.makeSubtask("scaleZeroPoint")
379 if self.config.doMaskBrightObjects:
380 mask = afwImage.Mask()
381 try:
382 self.brightObjectBitmask = 1 << mask.addMaskPlane(self.config.brightObjectMaskName)
383 except pexExceptions.LsstCppException:
384 raise RuntimeError("Unable to define mask plane for bright objects; planes used are %s" %
385 mask.getMaskPlaneDict().keys())
386 del mask
388 self.warpType = self.config.warpType
390 @utils.inheritDoc(pipeBase.PipelineTask)
391 def runQuantum(self, butlerQC, inputRefs, outputRefs):
392 # Docstring to be formatted with info from PipelineTask.runQuantum
393 """
394 Notes
395 -----
396 Assemble a coadd from a set of Warps.
398 PipelineTask (Gen3) entry point to Coadd a set of Warps.
399 Analogous to `runDataRef`, it prepares all the data products to be
400 passed to `run`, and processes the results before returning a struct
401 of results to be written out. AssembleCoadd cannot fit all Warps in memory.
402 Therefore, its inputs are accessed subregion by subregion
403 by the Gen3 `DeferredDatasetHandle` that is analagous to the Gen2
404 `lsst.daf.persistence.ButlerDataRef`. Any updates to this method should
405 correspond to an update in `runDataRef` while both entry points
406 are used.
407 """
408 inputData = butlerQC.get(inputRefs)
410 # Construct skyInfo expected by run
411 # Do not remove skyMap from inputData in case makeSupplementaryDataGen3 needs it
412 skyMap = inputData["skyMap"]
413 outputDataId = butlerQC.quantum.dataId
415 inputData['skyInfo'] = makeSkyInfo(skyMap,
416 tractId=outputDataId['tract'],
417 patchId=outputDataId['patch'])
419 # Construct list of input Deferred Datasets
420 # These quack a bit like like Gen2 DataRefs
421 warpRefList = inputData['inputWarps']
422 # Perform same middle steps as `runDataRef` does
423 inputs = self.prepareInputs(warpRefList)
424 self.log.info("Found %d %s", len(inputs.tempExpRefList),
425 self.getTempExpDatasetName(self.warpType))
426 if len(inputs.tempExpRefList) == 0:
427 self.log.warn("No coadd temporary exposures found")
428 return
430 supplementaryData = self.makeSupplementaryDataGen3(butlerQC, inputRefs, outputRefs)
431 retStruct = self.run(inputData['skyInfo'], inputs.tempExpRefList, inputs.imageScalerList,
432 inputs.weightList, supplementaryData=supplementaryData)
434 inputData.setdefault('brightObjectMask', None)
435 self.processResults(retStruct.coaddExposure, inputData['brightObjectMask'], outputDataId)
437 if self.config.doWrite:
438 butlerQC.put(retStruct, outputRefs)
439 return retStruct
441 @pipeBase.timeMethod
442 def runDataRef(self, dataRef, selectDataList=None, warpRefList=None):
443 """Assemble a coadd from a set of Warps.
445 Pipebase.CmdlineTask entry point to Coadd a set of Warps.
446 Compute weights to be applied to each Warp and
447 find scalings to match the photometric zeropoint to a reference Warp.
448 Assemble the Warps using `run`. Interpolate over NaNs and
449 optionally write the coadd to disk. Return the coadded exposure.
451 Parameters
452 ----------
453 dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
454 Data reference defining the patch for coaddition and the
455 reference Warp (if ``config.autoReference=False``).
456 Used to access the following data products:
457 - ``self.config.coaddName + "Coadd_skyMap"``
458 - ``self.config.coaddName + "Coadd_ + <warpType> + "Warp"`` (optionally)
459 - ``self.config.coaddName + "Coadd"``
460 selectDataList : `list`
461 List of data references to Calexps. Data to be coadded will be
462 selected from this list based on overlap with the patch defined
463 by dataRef, grouped by visit, and converted to a list of data
464 references to warps.
465 warpRefList : `list`
466 List of data references to Warps to be coadded.
467 Note: `warpRefList` is just the new name for `tempExpRefList`.
469 Returns
470 -------
471 retStruct : `lsst.pipe.base.Struct`
472 Result struct with components:
474 - ``coaddExposure``: coadded exposure (``Exposure``).
475 - ``nImage``: exposure count image (``Image``).
476 """
477 if selectDataList and warpRefList:
478 raise RuntimeError("runDataRef received both a selectDataList and warpRefList, "
479 "and which to use is ambiguous. Please pass only one.")
481 skyInfo = self.getSkyInfo(dataRef)
482 if warpRefList is None:
483 calExpRefList = self.selectExposures(dataRef, skyInfo, selectDataList=selectDataList)
484 if len(calExpRefList) == 0:
485 self.log.warn("No exposures to coadd")
486 return
487 self.log.info("Coadding %d exposures", len(calExpRefList))
489 warpRefList = self.getTempExpRefList(dataRef, calExpRefList)
491 inputData = self.prepareInputs(warpRefList)
492 self.log.info("Found %d %s", len(inputData.tempExpRefList),
493 self.getTempExpDatasetName(self.warpType))
494 if len(inputData.tempExpRefList) == 0:
495 self.log.warn("No coadd temporary exposures found")
496 return
498 supplementaryData = self.makeSupplementaryData(dataRef, warpRefList=inputData.tempExpRefList)
500 retStruct = self.run(skyInfo, inputData.tempExpRefList, inputData.imageScalerList,
501 inputData.weightList, supplementaryData=supplementaryData)
503 brightObjects = self.readBrightObjectMasks(dataRef) if self.config.doMaskBrightObjects else None
504 self.processResults(retStruct.coaddExposure, brightObjectMasks=brightObjects, dataId=dataRef.dataId)
506 if self.config.doWrite:
507 if self.getCoaddDatasetName(self.warpType) == "deepCoadd" and self.config.hasFakes:
508 coaddDatasetName = "fakes_" + self.getCoaddDatasetName(self.warpType)
509 else:
510 coaddDatasetName = self.getCoaddDatasetName(self.warpType)
511 self.log.info("Persisting %s" % coaddDatasetName)
512 dataRef.put(retStruct.coaddExposure, coaddDatasetName)
513 if self.config.doNImage and retStruct.nImage is not None:
514 dataRef.put(retStruct.nImage, self.getCoaddDatasetName(self.warpType) + '_nImage')
516 return retStruct
518 def processResults(self, coaddExposure, brightObjectMasks=None, dataId=None):
519 """Interpolate over missing data and mask bright stars.
521 Parameters
522 ----------
523 coaddExposure : `lsst.afw.image.Exposure`
524 The coadded exposure to process.
525 dataRef : `lsst.daf.persistence.ButlerDataRef`
526 Butler data reference for supplementary data.
527 """
528 if self.config.doInterp:
529 self.interpImage.run(coaddExposure.getMaskedImage(), planeName="NO_DATA")
530 # The variance must be positive; work around for DM-3201.
531 varArray = coaddExposure.variance.array
532 with numpy.errstate(invalid="ignore"):
533 varArray[:] = numpy.where(varArray > 0, varArray, numpy.inf)
535 if self.config.doMaskBrightObjects:
536 self.setBrightObjectMasks(coaddExposure, brightObjectMasks, dataId)
538 def makeSupplementaryData(self, dataRef, selectDataList=None, warpRefList=None):
539 """Make additional inputs to run() specific to subclasses (Gen2)
541 Duplicates interface of `runDataRef` method
542 Available to be implemented by subclasses only if they need the
543 coadd dataRef for performing preliminary processing before
544 assembling the coadd.
546 Parameters
547 ----------
548 dataRef : `lsst.daf.persistence.ButlerDataRef`
549 Butler data reference for supplementary data.
550 selectDataList : `list` (optional)
551 Optional List of data references to Calexps.
552 warpRefList : `list` (optional)
553 Optional List of data references to Warps.
554 """
555 return pipeBase.Struct()
557 def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs):
558 """Make additional inputs to run() specific to subclasses (Gen3)
560 Duplicates interface of `runQuantum` method.
561 Available to be implemented by subclasses only if they need the
562 coadd dataRef for performing preliminary processing before
563 assembling the coadd.
565 Parameters
566 ----------
567 butlerQC : `lsst.pipe.base.ButlerQuantumContext`
568 Gen3 Butler object for fetching additional data products before
569 running the Task specialized for quantum being processed
570 inputRefs : `lsst.pipe.base.InputQuantizedConnection`
571 Attributes are the names of the connections describing input dataset types.
572 Values are DatasetRefs that task consumes for corresponding dataset type.
573 DataIds are guaranteed to match data objects in ``inputData``.
574 outputRefs : `lsst.pipe.base.OutputQuantizedConnection`
575 Attributes are the names of the connections describing output dataset types.
576 Values are DatasetRefs that task is to produce
577 for corresponding dataset type.
578 """
579 return pipeBase.Struct()
581 def getTempExpRefList(self, patchRef, calExpRefList):
582 """Generate list data references corresponding to warped exposures
583 that lie within the patch to be coadded.
585 Parameters
586 ----------
587 patchRef : `dataRef`
588 Data reference for patch.
589 calExpRefList : `list`
590 List of data references for input calexps.
592 Returns
593 -------
594 tempExpRefList : `list`
595 List of Warp/CoaddTempExp data references.
596 """
597 butler = patchRef.getButler()
598 groupData = groupPatchExposures(patchRef, calExpRefList, self.getCoaddDatasetName(self.warpType),
599 self.getTempExpDatasetName(self.warpType))
600 tempExpRefList = [getGroupDataRef(butler, self.getTempExpDatasetName(self.warpType),
601 g, groupData.keys) for
602 g in groupData.groups.keys()]
603 return tempExpRefList
605 def prepareInputs(self, refList):
606 """Prepare the input warps for coaddition by measuring the weight for
607 each warp and the scaling for the photometric zero point.
609 Each Warp has its own photometric zeropoint and background variance.
610 Before coadding these Warps together, compute a scale factor to
611 normalize the photometric zeropoint and compute the weight for each Warp.
613 Parameters
614 ----------
615 refList : `list`
616 List of data references to tempExp
618 Returns
619 -------
620 result : `lsst.pipe.base.Struct`
621 Result struct with components:
623 - ``tempExprefList``: `list` of data references to tempExp.
624 - ``weightList``: `list` of weightings.
625 - ``imageScalerList``: `list` of image scalers.
626 """
627 statsCtrl = afwMath.StatisticsControl()
628 statsCtrl.setNumSigmaClip(self.config.sigmaClip)
629 statsCtrl.setNumIter(self.config.clipIter)
630 statsCtrl.setAndMask(self.getBadPixelMask())
631 statsCtrl.setNanSafe(True)
632 # compute tempExpRefList: a list of tempExpRef that actually exist
633 # and weightList: a list of the weight of the associated coadd tempExp
634 # and imageScalerList: a list of scale factors for the associated coadd tempExp
635 tempExpRefList = []
636 weightList = []
637 imageScalerList = []
638 tempExpName = self.getTempExpDatasetName(self.warpType)
639 for tempExpRef in refList:
640 # Gen3's DeferredDatasetHandles are guaranteed to exist and
641 # therefore have no datasetExists() method
642 if not isinstance(tempExpRef, DeferredDatasetHandle):
643 if not tempExpRef.datasetExists(tempExpName):
644 self.log.warn("Could not find %s %s; skipping it", tempExpName, tempExpRef.dataId)
645 continue
647 tempExp = tempExpRef.get(datasetType=tempExpName, immediate=True)
648 # Ignore any input warp that is empty of data
649 if numpy.isnan(tempExp.image.array).all():
650 continue
651 maskedImage = tempExp.getMaskedImage()
652 imageScaler = self.scaleZeroPoint.computeImageScaler(
653 exposure=tempExp,
654 dataRef=tempExpRef,
655 )
656 try:
657 imageScaler.scaleMaskedImage(maskedImage)
658 except Exception as e:
659 self.log.warn("Scaling failed for %s (skipping it): %s", tempExpRef.dataId, e)
660 continue
661 statObj = afwMath.makeStatistics(maskedImage.getVariance(), maskedImage.getMask(),
662 afwMath.MEANCLIP, statsCtrl)
663 meanVar, meanVarErr = statObj.getResult(afwMath.MEANCLIP)
664 weight = 1.0 / float(meanVar)
665 if not numpy.isfinite(weight):
666 self.log.warn("Non-finite weight for %s: skipping", tempExpRef.dataId)
667 continue
668 self.log.info("Weight of %s %s = %0.3f", tempExpName, tempExpRef.dataId, weight)
670 del maskedImage
671 del tempExp
673 tempExpRefList.append(tempExpRef)
674 weightList.append(weight)
675 imageScalerList.append(imageScaler)
677 return pipeBase.Struct(tempExpRefList=tempExpRefList, weightList=weightList,
678 imageScalerList=imageScalerList)
680 def prepareStats(self, mask=None):
681 """Prepare the statistics for coadding images.
683 Parameters
684 ----------
685 mask : `int`, optional
686 Bit mask value to exclude from coaddition.
688 Returns
689 -------
690 stats : `lsst.pipe.base.Struct`
691 Statistics structure with the following fields:
693 - ``statsCtrl``: Statistics control object for coadd
694 (`lsst.afw.math.StatisticsControl`)
695 - ``statsFlags``: Statistic for coadd (`lsst.afw.math.Property`)
696 """
697 if mask is None:
698 mask = self.getBadPixelMask()
699 statsCtrl = afwMath.StatisticsControl()
700 statsCtrl.setNumSigmaClip(self.config.sigmaClip)
701 statsCtrl.setNumIter(self.config.clipIter)
702 statsCtrl.setAndMask(mask)
703 statsCtrl.setNanSafe(True)
704 statsCtrl.setWeighted(True)
705 statsCtrl.setCalcErrorFromInputVariance(self.config.calcErrorFromInputVariance)
706 for plane, threshold in self.config.maskPropagationThresholds.items():
707 bit = afwImage.Mask.getMaskPlane(plane)
708 statsCtrl.setMaskPropagationThreshold(bit, threshold)
709 statsFlags = afwMath.stringToStatisticsProperty(self.config.statistic)
710 return pipeBase.Struct(ctrl=statsCtrl, flags=statsFlags)
712 @pipeBase.timeMethod
713 def run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
714 altMaskList=None, mask=None, supplementaryData=None):
715 """Assemble a coadd from input warps
717 Assemble the coadd using the provided list of coaddTempExps. Since
718 the full coadd covers a patch (a large area), the assembly is
719 performed over small areas on the image at a time in order to
720 conserve memory usage. Iterate over subregions within the outer
721 bbox of the patch using `assembleSubregion` to stack the corresponding
722 subregions from the coaddTempExps with the statistic specified.
723 Set the edge bits the coadd mask based on the weight map.
725 Parameters
726 ----------
727 skyInfo : `lsst.pipe.base.Struct`
728 Struct with geometric information about the patch.
729 tempExpRefList : `list`
730 List of data references to Warps (previously called CoaddTempExps).
731 imageScalerList : `list`
732 List of image scalers.
733 weightList : `list`
734 List of weights
735 altMaskList : `list`, optional
736 List of alternate masks to use rather than those stored with
737 tempExp.
738 mask : `int`, optional
739 Bit mask value to exclude from coaddition.
740 supplementaryData : lsst.pipe.base.Struct, optional
741 Struct with additional data products needed to assemble coadd.
742 Only used by subclasses that implement `makeSupplementaryData`
743 and override `run`.
745 Returns
746 -------
747 result : `lsst.pipe.base.Struct`
748 Result struct with components:
750 - ``coaddExposure``: coadded exposure (``lsst.afw.image.Exposure``).
751 - ``nImage``: exposure count image (``lsst.afw.image.Image``), if requested.
752 - ``warpRefList``: input list of refs to the warps (
753 ``lsst.daf.butler.DeferredDatasetHandle`` or
754 ``lsst.daf.persistence.ButlerDataRef``)
755 (unmodified)
756 - ``imageScalerList``: input list of image scalers (unmodified)
757 - ``weightList``: input list of weights (unmodified)
758 """
759 tempExpName = self.getTempExpDatasetName(self.warpType)
760 self.log.info("Assembling %s %s", len(tempExpRefList), tempExpName)
761 stats = self.prepareStats(mask=mask)
763 if altMaskList is None:
764 altMaskList = [None]*len(tempExpRefList)
766 coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
767 coaddExposure.setPhotoCalib(self.scaleZeroPoint.getPhotoCalib())
768 coaddExposure.getInfo().setCoaddInputs(self.inputRecorder.makeCoaddInputs())
769 self.assembleMetadata(coaddExposure, tempExpRefList, weightList)
770 coaddMaskedImage = coaddExposure.getMaskedImage()
771 subregionSizeArr = self.config.subregionSize
772 subregionSize = geom.Extent2I(subregionSizeArr[0], subregionSizeArr[1])
773 # if nImage is requested, create a zero one which can be passed to assembleSubregion
774 if self.config.doNImage:
775 nImage = afwImage.ImageU(skyInfo.bbox)
776 else:
777 nImage = None
778 for subBBox in self._subBBoxIter(skyInfo.bbox, subregionSize):
779 try:
780 self.assembleSubregion(coaddExposure, subBBox, tempExpRefList, imageScalerList,
781 weightList, altMaskList, stats.flags, stats.ctrl,
782 nImage=nImage)
783 except Exception as e:
784 self.log.fatal("Cannot compute coadd %s: %s", subBBox, e)
786 self.setInexactPsf(coaddMaskedImage.getMask())
787 # Despite the name, the following doesn't really deal with "EDGE" pixels: it identifies
788 # pixels that didn't receive any unmasked inputs (as occurs around the edge of the field).
789 coaddUtils.setCoaddEdgeBits(coaddMaskedImage.getMask(), coaddMaskedImage.getVariance())
790 return pipeBase.Struct(coaddExposure=coaddExposure, nImage=nImage,
791 warpRefList=tempExpRefList, imageScalerList=imageScalerList,
792 weightList=weightList)
794 def assembleMetadata(self, coaddExposure, tempExpRefList, weightList):
795 """Set the metadata for the coadd.
797 This basic implementation sets the filter from the first input.
799 Parameters
800 ----------
801 coaddExposure : `lsst.afw.image.Exposure`
802 The target exposure for the coadd.
803 tempExpRefList : `list`
804 List of data references to tempExp.
805 weightList : `list`
806 List of weights.
807 """
808 assert len(tempExpRefList) == len(weightList), "Length mismatch"
809 tempExpName = self.getTempExpDatasetName(self.warpType)
810 # We load a single pixel of each coaddTempExp, because we just want to get at the metadata
811 # (and we need more than just the PropertySet that contains the header), which is not possible
812 # with the current butler (see #2777).
813 bbox = geom.Box2I(coaddExposure.getBBox().getMin(), geom.Extent2I(1, 1))
815 if isinstance(tempExpRefList[0], DeferredDatasetHandle):
816 # Gen 3 API
817 tempExpList = [tempExpRef.get(parameters={'bbox': bbox}) for tempExpRef in tempExpRefList]
818 else:
819 # Gen 2 API. Delete this when Gen 2 retired
820 tempExpList = [tempExpRef.get(tempExpName + "_sub", bbox=bbox, immediate=True)
821 for tempExpRef in tempExpRefList]
822 numCcds = sum(len(tempExp.getInfo().getCoaddInputs().ccds) for tempExp in tempExpList)
824 coaddExposure.setFilter(tempExpList[0].getFilter())
825 coaddInputs = coaddExposure.getInfo().getCoaddInputs()
826 coaddInputs.ccds.reserve(numCcds)
827 coaddInputs.visits.reserve(len(tempExpList))
829 for tempExp, weight in zip(tempExpList, weightList):
830 self.inputRecorder.addVisitToCoadd(coaddInputs, tempExp, weight)
832 if self.config.doUsePsfMatchedPolygons:
833 self.shrinkValidPolygons(coaddInputs)
835 coaddInputs.visits.sort()
836 if self.warpType == "psfMatched":
837 # The modelPsf BBox for a psfMatchedWarp/coaddTempExp was dynamically defined by
838 # ModelPsfMatchTask as the square box bounding its spatially-variable, pre-matched WarpedPsf.
839 # Likewise, set the PSF of a PSF-Matched Coadd to the modelPsf
840 # having the maximum width (sufficient because square)
841 modelPsfList = [tempExp.getPsf() for tempExp in tempExpList]
842 modelPsfWidthList = [modelPsf.computeBBox().getWidth() for modelPsf in modelPsfList]
843 psf = modelPsfList[modelPsfWidthList.index(max(modelPsfWidthList))]
844 else:
845 psf = measAlg.CoaddPsf(coaddInputs.ccds, coaddExposure.getWcs(),
846 self.config.coaddPsf.makeControl())
847 coaddExposure.setPsf(psf)
848 apCorrMap = measAlg.makeCoaddApCorrMap(coaddInputs.ccds, coaddExposure.getBBox(afwImage.PARENT),
849 coaddExposure.getWcs())
850 coaddExposure.getInfo().setApCorrMap(apCorrMap)
851 if self.config.doAttachTransmissionCurve:
852 transmissionCurve = measAlg.makeCoaddTransmissionCurve(coaddExposure.getWcs(), coaddInputs.ccds)
853 coaddExposure.getInfo().setTransmissionCurve(transmissionCurve)
855 def assembleSubregion(self, coaddExposure, bbox, tempExpRefList, imageScalerList, weightList,
856 altMaskList, statsFlags, statsCtrl, nImage=None):
857 """Assemble the coadd for a sub-region.
859 For each coaddTempExp, check for (and swap in) an alternative mask
860 if one is passed. Remove mask planes listed in
861 `config.removeMaskPlanes`. Finally, stack the actual exposures using
862 `lsst.afw.math.statisticsStack` with the statistic specified by
863 statsFlags. Typically, the statsFlag will be one of lsst.afw.math.MEAN for
864 a mean-stack or `lsst.afw.math.MEANCLIP` for outlier rejection using
865 an N-sigma clipped mean where N and iterations are specified by
866 statsCtrl. Assign the stacked subregion back to the coadd.
868 Parameters
869 ----------
870 coaddExposure : `lsst.afw.image.Exposure`
871 The target exposure for the coadd.
872 bbox : `lsst.geom.Box`
873 Sub-region to coadd.
874 tempExpRefList : `list`
875 List of data reference to tempExp.
876 imageScalerList : `list`
877 List of image scalers.
878 weightList : `list`
879 List of weights.
880 altMaskList : `list`
881 List of alternate masks to use rather than those stored with
882 tempExp, or None. Each element is dict with keys = mask plane
883 name to which to add the spans.
884 statsFlags : `lsst.afw.math.Property`
885 Property object for statistic for coadd.
886 statsCtrl : `lsst.afw.math.StatisticsControl`
887 Statistics control object for coadd.
888 nImage : `lsst.afw.image.ImageU`, optional
889 Keeps track of exposure count for each pixel.
890 """
891 self.log.debug("Computing coadd over %s", bbox)
892 tempExpName = self.getTempExpDatasetName(self.warpType)
893 coaddExposure.mask.addMaskPlane("REJECTED")
894 coaddExposure.mask.addMaskPlane("CLIPPED")
895 coaddExposure.mask.addMaskPlane("SENSOR_EDGE")
896 maskMap = self.setRejectedMaskMapping(statsCtrl)
897 clipped = afwImage.Mask.getPlaneBitMask("CLIPPED")
898 maskedImageList = []
899 if nImage is not None:
900 subNImage = afwImage.ImageU(bbox.getWidth(), bbox.getHeight())
901 for tempExpRef, imageScaler, altMask in zip(tempExpRefList, imageScalerList, altMaskList):
903 if isinstance(tempExpRef, DeferredDatasetHandle):
904 # Gen 3 API
905 exposure = tempExpRef.get(parameters={'bbox': bbox})
906 else:
907 # Gen 2 API. Delete this when Gen 2 retired
908 exposure = tempExpRef.get(tempExpName + "_sub", bbox=bbox)
910 maskedImage = exposure.getMaskedImage()
911 mask = maskedImage.getMask()
912 if altMask is not None:
913 self.applyAltMaskPlanes(mask, altMask)
914 imageScaler.scaleMaskedImage(maskedImage)
916 # Add 1 for each pixel which is not excluded by the exclude mask.
917 # In legacyCoadd, pixels may also be excluded by afwMath.statisticsStack.
918 if nImage is not None:
919 subNImage.getArray()[maskedImage.getMask().getArray() & statsCtrl.getAndMask() == 0] += 1
920 if self.config.removeMaskPlanes:
921 self.removeMaskPlanes(maskedImage)
922 maskedImageList.append(maskedImage)
924 with self.timer("stack"):
925 coaddSubregion = afwMath.statisticsStack(maskedImageList, statsFlags, statsCtrl, weightList,
926 clipped, # also set output to CLIPPED if sigma-clipped
927 maskMap)
928 coaddExposure.maskedImage.assign(coaddSubregion, bbox)
929 if nImage is not None:
930 nImage.assign(subNImage, bbox)
932 def removeMaskPlanes(self, maskedImage):
933 """Unset the mask of an image for mask planes specified in the config.
935 Parameters
936 ----------
937 maskedImage : `lsst.afw.image.MaskedImage`
938 The masked image to be modified.
939 """
940 mask = maskedImage.getMask()
941 for maskPlane in self.config.removeMaskPlanes:
942 try:
943 mask &= ~mask.getPlaneBitMask(maskPlane)
944 except pexExceptions.InvalidParameterError:
945 self.log.debug("Unable to remove mask plane %s: no mask plane with that name was found.",
946 maskPlane)
948 @staticmethod
949 def setRejectedMaskMapping(statsCtrl):
950 """Map certain mask planes of the warps to new planes for the coadd.
952 If a pixel is rejected due to a mask value other than EDGE, NO_DATA,
953 or CLIPPED, set it to REJECTED on the coadd.
954 If a pixel is rejected due to EDGE, set the coadd pixel to SENSOR_EDGE.
955 If a pixel is rejected due to CLIPPED, set the coadd pixel to CLIPPED.
957 Parameters
958 ----------
959 statsCtrl : `lsst.afw.math.StatisticsControl`
960 Statistics control object for coadd
962 Returns
963 -------
964 maskMap : `list` of `tuple` of `int`
965 A list of mappings of mask planes of the warped exposures to
966 mask planes of the coadd.
967 """
968 edge = afwImage.Mask.getPlaneBitMask("EDGE")
969 noData = afwImage.Mask.getPlaneBitMask("NO_DATA")
970 clipped = afwImage.Mask.getPlaneBitMask("CLIPPED")
971 toReject = statsCtrl.getAndMask() & (~noData) & (~edge) & (~clipped)
972 maskMap = [(toReject, afwImage.Mask.getPlaneBitMask("REJECTED")),
973 (edge, afwImage.Mask.getPlaneBitMask("SENSOR_EDGE")),
974 (clipped, clipped)]
975 return maskMap
977 def applyAltMaskPlanes(self, mask, altMaskSpans):
978 """Apply in place alt mask formatted as SpanSets to a mask.
980 Parameters
981 ----------
982 mask : `lsst.afw.image.Mask`
983 Original mask.
984 altMaskSpans : `dict`
985 SpanSet lists to apply. Each element contains the new mask
986 plane name (e.g. "CLIPPED and/or "NO_DATA") as the key,
987 and list of SpanSets to apply to the mask.
989 Returns
990 -------
991 mask : `lsst.afw.image.Mask`
992 Updated mask.
993 """
994 if self.config.doUsePsfMatchedPolygons:
995 if ("NO_DATA" in altMaskSpans) and ("NO_DATA" in self.config.badMaskPlanes):
996 # Clear away any other masks outside the validPolygons. These pixels are no longer
997 # contributing to inexact PSFs, and will still be rejected because of NO_DATA
998 # self.config.doUsePsfMatchedPolygons should be True only in CompareWarpAssemble
999 # This mask-clearing step must only occur *before* applying the new masks below
1000 for spanSet in altMaskSpans['NO_DATA']:
1001 spanSet.clippedTo(mask.getBBox()).clearMask(mask, self.getBadPixelMask())
1003 for plane, spanSetList in altMaskSpans.items():
1004 maskClipValue = mask.addMaskPlane(plane)
1005 for spanSet in spanSetList:
1006 spanSet.clippedTo(mask.getBBox()).setMask(mask, 2**maskClipValue)
1007 return mask
1009 def shrinkValidPolygons(self, coaddInputs):
1010 """Shrink coaddInputs' ccds' ValidPolygons in place.
1012 Either modify each ccd's validPolygon in place, or if CoaddInputs
1013 does not have a validPolygon, create one from its bbox.
1015 Parameters
1016 ----------
1017 coaddInputs : `lsst.afw.image.coaddInputs`
1018 Original mask.
1020 """
1021 for ccd in coaddInputs.ccds:
1022 polyOrig = ccd.getValidPolygon()
1023 validPolyBBox = polyOrig.getBBox() if polyOrig else ccd.getBBox()
1024 validPolyBBox.grow(-self.config.matchingKernelSize//2)
1025 if polyOrig:
1026 validPolygon = polyOrig.intersectionSingle(validPolyBBox)
1027 else:
1028 validPolygon = afwGeom.polygon.Polygon(geom.Box2D(validPolyBBox))
1029 ccd.setValidPolygon(validPolygon)
1031 def readBrightObjectMasks(self, dataRef):
1032 """Retrieve the bright object masks.
1034 Returns None on failure.
1036 Parameters
1037 ----------
1038 dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
1039 A Butler dataRef.
1041 Returns
1042 -------
1043 result : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
1044 Bright object mask from the Butler object, or None if it cannot
1045 be retrieved.
1046 """
1047 try:
1048 return dataRef.get(datasetType="brightObjectMask", immediate=True)
1049 except Exception as e:
1050 self.log.warn("Unable to read brightObjectMask for %s: %s", dataRef.dataId, e)
1051 return None
1053 def setBrightObjectMasks(self, exposure, brightObjectMasks, dataId=None):
1054 """Set the bright object masks.
1056 Parameters
1057 ----------
1058 exposure : `lsst.afw.image.Exposure`
1059 Exposure under consideration.
1060 dataId : `lsst.daf.persistence.dataId`
1061 Data identifier dict for patch.
1062 brightObjectMasks : `lsst.afw.table`
1063 Table of bright objects to mask.
1064 """
1066 if brightObjectMasks is None:
1067 self.log.warn("Unable to apply bright object mask: none supplied")
1068 return
1069 self.log.info("Applying %d bright object masks to %s", len(brightObjectMasks), dataId)
1070 mask = exposure.getMaskedImage().getMask()
1071 wcs = exposure.getWcs()
1072 plateScale = wcs.getPixelScale().asArcseconds()
1074 for rec in brightObjectMasks:
1075 center = geom.PointI(wcs.skyToPixel(rec.getCoord()))
1076 if rec["type"] == "box":
1077 assert rec["angle"] == 0.0, ("Angle != 0 for mask object %s" % rec["id"])
1078 width = rec["width"].asArcseconds()/plateScale # convert to pixels
1079 height = rec["height"].asArcseconds()/plateScale # convert to pixels
1081 halfSize = geom.ExtentI(0.5*width, 0.5*height)
1082 bbox = geom.Box2I(center - halfSize, center + halfSize)
1084 bbox = geom.BoxI(geom.PointI(int(center[0] - 0.5*width), int(center[1] - 0.5*height)),
1085 geom.PointI(int(center[0] + 0.5*width), int(center[1] + 0.5*height)))
1086 spans = afwGeom.SpanSet(bbox)
1087 elif rec["type"] == "circle":
1088 radius = int(rec["radius"].asArcseconds()/plateScale) # convert to pixels
1089 spans = afwGeom.SpanSet.fromShape(radius, offset=center)
1090 else:
1091 self.log.warn("Unexpected region type %s at %s" % rec["type"], center)
1092 continue
1093 spans.clippedTo(mask.getBBox()).setMask(mask, self.brightObjectBitmask)
1095 def setInexactPsf(self, mask):
1096 """Set INEXACT_PSF mask plane.
1098 If any of the input images isn't represented in the coadd (due to
1099 clipped pixels or chip gaps), the `CoaddPsf` will be inexact. Flag
1100 these pixels.
1102 Parameters
1103 ----------
1104 mask : `lsst.afw.image.Mask`
1105 Coadded exposure's mask, modified in-place.
1106 """
1107 mask.addMaskPlane("INEXACT_PSF")
1108 inexactPsf = mask.getPlaneBitMask("INEXACT_PSF")
1109 sensorEdge = mask.getPlaneBitMask("SENSOR_EDGE") # chip edges (so PSF is discontinuous)
1110 clipped = mask.getPlaneBitMask("CLIPPED") # pixels clipped from coadd
1111 rejected = mask.getPlaneBitMask("REJECTED") # pixels rejected from coadd due to masks
1112 array = mask.getArray()
1113 selected = array & (sensorEdge | clipped | rejected) > 0
1114 array[selected] |= inexactPsf
1116 @classmethod
1117 def _makeArgumentParser(cls):
1118 """Create an argument parser.
1119 """
1120 parser = pipeBase.ArgumentParser(name=cls._DefaultName)
1121 parser.add_id_argument("--id", cls.ConfigClass().coaddName + "Coadd_"
1122 + cls.ConfigClass().warpType + "Warp",
1123 help="data ID, e.g. --id tract=12345 patch=1,2",
1124 ContainerClass=AssembleCoaddDataIdContainer)
1125 parser.add_id_argument("--selectId", "calexp", help="data ID, e.g. --selectId visit=6789 ccd=0..9",
1126 ContainerClass=SelectDataIdContainer)
1127 return parser
1129 @staticmethod
1130 def _subBBoxIter(bbox, subregionSize):
1131 """Iterate over subregions of a bbox.
1133 Parameters
1134 ----------
1135 bbox : `lsst.geom.Box2I`
1136 Bounding box over which to iterate.
1137 subregionSize: `lsst.geom.Extent2I`
1138 Size of sub-bboxes.
1140 Yields
1141 ------
1142 subBBox : `lsst.geom.Box2I`
1143 Next sub-bounding box of size ``subregionSize`` or smaller; each ``subBBox``
1144 is contained within ``bbox``, so it may be smaller than ``subregionSize`` at
1145 the edges of ``bbox``, but it will never be empty.
1146 """
1147 if bbox.isEmpty():
1148 raise RuntimeError("bbox %s is empty" % (bbox,))
1149 if subregionSize[0] < 1 or subregionSize[1] < 1:
1150 raise RuntimeError("subregionSize %s must be nonzero" % (subregionSize,))
1152 for rowShift in range(0, bbox.getHeight(), subregionSize[1]):
1153 for colShift in range(0, bbox.getWidth(), subregionSize[0]):
1154 subBBox = geom.Box2I(bbox.getMin() + geom.Extent2I(colShift, rowShift), subregionSize)
1155 subBBox.clip(bbox)
1156 if subBBox.isEmpty():
1157 raise RuntimeError("Bug: empty bbox! bbox=%s, subregionSize=%s, "
1158 "colShift=%s, rowShift=%s" %
1159 (bbox, subregionSize, colShift, rowShift))
1160 yield subBBox
1163class AssembleCoaddDataIdContainer(pipeBase.DataIdContainer):
1164 """A version of `lsst.pipe.base.DataIdContainer` specialized for assembleCoadd.
1165 """
1167 def makeDataRefList(self, namespace):
1168 """Make self.refList from self.idList.
1170 Parameters
1171 ----------
1172 namespace
1173 Results of parsing command-line (with ``butler`` and ``log`` elements).
1174 """
1175 datasetType = namespace.config.coaddName + "Coadd"
1176 keysCoadd = namespace.butler.getKeys(datasetType=datasetType, level=self.level)
1178 for dataId in self.idList:
1179 # tract and patch are required
1180 for key in keysCoadd:
1181 if key not in dataId:
1182 raise RuntimeError("--id must include " + key)
1184 dataRef = namespace.butler.dataRef(
1185 datasetType=datasetType,
1186 dataId=dataId,
1187 )
1188 self.refList.append(dataRef)
1191def countMaskFromFootprint(mask, footprint, bitmask, ignoreMask):
1192 """Function to count the number of pixels with a specific mask in a
1193 footprint.
1195 Find the intersection of mask & footprint. Count all pixels in the mask
1196 that are in the intersection that have bitmask set but do not have
1197 ignoreMask set. Return the count.
1199 Parameters
1200 ----------
1201 mask : `lsst.afw.image.Mask`
1202 Mask to define intersection region by.
1203 footprint : `lsst.afw.detection.Footprint`
1204 Footprint to define the intersection region by.
1205 bitmask
1206 Specific mask that we wish to count the number of occurances of.
1207 ignoreMask
1208 Pixels to not consider.
1210 Returns
1211 -------
1212 result : `int`
1213 Count of number of pixels in footprint with specified mask.
1214 """
1215 bbox = footprint.getBBox()
1216 bbox.clip(mask.getBBox(afwImage.PARENT))
1217 fp = afwImage.Mask(bbox)
1218 subMask = mask.Factory(mask, bbox, afwImage.PARENT)
1219 footprint.spans.setMask(fp, bitmask)
1220 return numpy.logical_and((subMask.getArray() & fp.getArray()) > 0,
1221 (subMask.getArray() & ignoreMask) == 0).sum()
1224class SafeClipAssembleCoaddConfig(AssembleCoaddConfig, pipelineConnections=AssembleCoaddConnections):
1225 """Configuration parameters for the SafeClipAssembleCoaddTask.
1226 """
1227 assembleMeanCoadd = pexConfig.ConfigurableField(
1228 target=AssembleCoaddTask,
1229 doc="Task to assemble an initial Coadd using the MEAN statistic.",
1230 )
1231 assembleMeanClipCoadd = pexConfig.ConfigurableField(
1232 target=AssembleCoaddTask,
1233 doc="Task to assemble an initial Coadd using the MEANCLIP statistic.",
1234 )
1235 clipDetection = pexConfig.ConfigurableField(
1236 target=SourceDetectionTask,
1237 doc="Detect sources on difference between unclipped and clipped coadd")
1238 minClipFootOverlap = pexConfig.Field(
1239 doc="Minimum fractional overlap of clipped footprint with visit DETECTED to be clipped",
1240 dtype=float,
1241 default=0.6
1242 )
1243 minClipFootOverlapSingle = pexConfig.Field(
1244 doc="Minimum fractional overlap of clipped footprint with visit DETECTED to be "
1245 "clipped when only one visit overlaps",
1246 dtype=float,
1247 default=0.5
1248 )
1249 minClipFootOverlapDouble = pexConfig.Field(
1250 doc="Minimum fractional overlap of clipped footprints with visit DETECTED to be "
1251 "clipped when two visits overlap",
1252 dtype=float,
1253 default=0.45
1254 )
1255 maxClipFootOverlapDouble = pexConfig.Field(
1256 doc="Maximum fractional overlap of clipped footprints with visit DETECTED when "
1257 "considering two visits",
1258 dtype=float,
1259 default=0.15
1260 )
1261 minBigOverlap = pexConfig.Field(
1262 doc="Minimum number of pixels in footprint to use DETECTED mask from the single visits "
1263 "when labeling clipped footprints",
1264 dtype=int,
1265 default=100
1266 )
1268 def setDefaults(self):
1269 """Set default values for clipDetection.
1271 Notes
1272 -----
1273 The numeric values for these configuration parameters were
1274 empirically determined, future work may further refine them.
1275 """
1276 AssembleCoaddConfig.setDefaults(self)
1277 self.clipDetection.doTempLocalBackground = False
1278 self.clipDetection.reEstimateBackground = False
1279 self.clipDetection.returnOriginalFootprints = False
1280 self.clipDetection.thresholdPolarity = "both"
1281 self.clipDetection.thresholdValue = 2
1282 self.clipDetection.nSigmaToGrow = 2
1283 self.clipDetection.minPixels = 4
1284 self.clipDetection.isotropicGrow = True
1285 self.clipDetection.thresholdType = "pixel_stdev"
1286 self.sigmaClip = 1.5
1287 self.clipIter = 3
1288 self.statistic = "MEAN"
1289 self.assembleMeanCoadd.statistic = 'MEAN'
1290 self.assembleMeanClipCoadd.statistic = 'MEANCLIP'
1291 self.assembleMeanCoadd.doWrite = False
1292 self.assembleMeanClipCoadd.doWrite = False
1294 def validate(self):
1295 if self.doSigmaClip:
1296 log.warn("Additional Sigma-clipping not allowed in Safe-clipped Coadds. "
1297 "Ignoring doSigmaClip.")
1298 self.doSigmaClip = False
1299 if self.statistic != "MEAN":
1300 raise ValueError("Only MEAN statistic allowed for final stacking in SafeClipAssembleCoadd "
1301 "(%s chosen). Please set statistic to MEAN."
1302 % (self.statistic))
1303 AssembleCoaddTask.ConfigClass.validate(self)
1306class SafeClipAssembleCoaddTask(AssembleCoaddTask):
1307 """Assemble a coadded image from a set of coadded temporary exposures,
1308 being careful to clip & flag areas with potential artifacts.
1310 In ``AssembleCoaddTask``, we compute the coadd as an clipped mean (i.e.,
1311 we clip outliers). The problem with doing this is that when computing the
1312 coadd PSF at a given location, individual visit PSFs from visits with
1313 outlier pixels contribute to the coadd PSF and cannot be treated correctly.
1314 In this task, we correct for this behavior by creating a new
1315 ``badMaskPlane`` 'CLIPPED'. We populate this plane on the input
1316 coaddTempExps and the final coadd where
1318 i. difference imaging suggests that there is an outlier and
1319 ii. this outlier appears on only one or two images.
1321 Such regions will not contribute to the final coadd. Furthermore, any
1322 routine to determine the coadd PSF can now be cognizant of clipped regions.
1323 Note that the algorithm implemented by this task is preliminary and works
1324 correctly for HSC data. Parameter modifications and or considerable
1325 redesigning of the algorithm is likley required for other surveys.
1327 ``SafeClipAssembleCoaddTask`` uses a ``SourceDetectionTask``
1328 "clipDetection" subtask and also sub-classes ``AssembleCoaddTask``.
1329 You can retarget the ``SourceDetectionTask`` "clipDetection" subtask
1330 if you wish.
1332 Notes
1333 -----
1334 The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a
1335 flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``;
1336 see `baseDebug` for more about ``debug.py`` files.
1337 `SafeClipAssembleCoaddTask` has no debug variables of its own.
1338 The ``SourceDetectionTask`` "clipDetection" subtasks may support debug
1339 variables. See the documetation for `SourceDetectionTask` "clipDetection"
1340 for further information.
1342 Examples
1343 --------
1344 `SafeClipAssembleCoaddTask` assembles a set of warped ``coaddTempExp``
1345 images into a coadded image. The `SafeClipAssembleCoaddTask` is invoked by
1346 running assembleCoadd.py *without* the flag '--legacyCoadd'.
1348 Usage of ``assembleCoadd.py`` expects a data reference to the tract patch
1349 and filter to be coadded (specified using
1350 '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]')
1351 along with a list of coaddTempExps to attempt to coadd (specified using
1352 '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]').
1353 Only the coaddTempExps that cover the specified tract and patch will be
1354 coadded. A list of the available optional arguments can be obtained by
1355 calling assembleCoadd.py with the --help command line argument:
1357 .. code-block:: none
1359 assembleCoadd.py --help
1361 To demonstrate usage of the `SafeClipAssembleCoaddTask` in the larger
1362 context of multi-band processing, we will generate the HSC-I & -R band
1363 coadds from HSC engineering test data provided in the ci_hsc package.
1364 To begin, assuming that the lsst stack has been already set up, we must
1365 set up the obs_subaru and ci_hsc packages. This defines the environment
1366 variable $CI_HSC_DIR and points at the location of the package. The raw
1367 HSC data live in the ``$CI_HSC_DIR/raw`` directory. To begin assembling
1368 the coadds, we must first
1370 - ``processCcd``
1371 process the individual ccds in $CI_HSC_RAW to produce calibrated exposures
1372 - ``makeSkyMap``
1373 create a skymap that covers the area of the sky present in the raw exposures
1374 - ``makeCoaddTempExp``
1375 warp the individual calibrated exposures to the tangent plane of the coadd</DD>
1377 We can perform all of these steps by running
1379 .. code-block:: none
1381 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
1383 This will produce warped coaddTempExps for each visit. To coadd the
1384 warped data, we call ``assembleCoadd.py`` as follows:
1386 .. code-block:: none
1388 assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \
1389 --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \
1390 --selectId visit=903986 ccd=100--selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \
1391 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \
1392 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \
1393 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \
1394 --selectId visit=903988 ccd=24
1396 This will process the HSC-I band data. The results are written in
1397 ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``.
1399 You may also choose to run:
1401 .. code-block:: none
1403 scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346 nnn
1404 assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R --selectId visit=903334 ccd=16 \
1405 --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 --selectId visit=903334 ccd=100 \
1406 --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 --selectId visit=903338 ccd=18 \
1407 --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 --selectId visit=903342 ccd=10 \
1408 --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 --selectId visit=903344 ccd=5 \
1409 --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 --selectId visit=903346 ccd=6 \
1410 --selectId visit=903346 ccd=12
1412 to generate the coadd for the HSC-R band if you are interested in following
1413 multiBand Coadd processing as discussed in ``pipeTasks_multiBand``.
1414 """
1415 ConfigClass = SafeClipAssembleCoaddConfig
1416 _DefaultName = "safeClipAssembleCoadd"
1418 def __init__(self, *args, **kwargs):
1419 AssembleCoaddTask.__init__(self, *args, **kwargs)
1420 schema = afwTable.SourceTable.makeMinimalSchema()
1421 self.makeSubtask("clipDetection", schema=schema)
1422 self.makeSubtask("assembleMeanClipCoadd")
1423 self.makeSubtask("assembleMeanCoadd")
1425 @utils.inheritDoc(AssembleCoaddTask)
1426 @pipeBase.timeMethod
1427 def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, *args, **kwargs):
1428 """Assemble the coadd for a region.
1430 Compute the difference of coadds created with and without outlier
1431 rejection to identify coadd pixels that have outlier values in some
1432 individual visits.
1433 Detect clipped regions on the difference image and mark these regions
1434 on the one or two individual coaddTempExps where they occur if there
1435 is significant overlap between the clipped region and a source. This
1436 leaves us with a set of footprints from the difference image that have
1437 been identified as having occured on just one or two individual visits.
1438 However, these footprints were generated from a difference image. It
1439 is conceivable for a large diffuse source to have become broken up
1440 into multiple footprints acrosss the coadd difference in this process.
1441 Determine the clipped region from all overlapping footprints from the
1442 detected sources in each visit - these are big footprints.
1443 Combine the small and big clipped footprints and mark them on a new
1444 bad mask plane.
1445 Generate the coadd using `AssembleCoaddTask.run` without outlier
1446 removal. Clipped footprints will no longer make it into the coadd
1447 because they are marked in the new bad mask plane.
1449 Notes
1450 -----
1451 args and kwargs are passed but ignored in order to match the call
1452 signature expected by the parent task.
1453 """
1454 exp = self.buildDifferenceImage(skyInfo, tempExpRefList, imageScalerList, weightList)
1455 mask = exp.getMaskedImage().getMask()
1456 mask.addMaskPlane("CLIPPED")
1458 result = self.detectClip(exp, tempExpRefList)
1460 self.log.info('Found %d clipped objects', len(result.clipFootprints))
1462 maskClipValue = mask.getPlaneBitMask("CLIPPED")
1463 maskDetValue = mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE")
1464 # Append big footprints from individual Warps to result.clipSpans
1465 bigFootprints = self.detectClipBig(result.clipSpans, result.clipFootprints, result.clipIndices,
1466 result.detectionFootprints, maskClipValue, maskDetValue,
1467 exp.getBBox())
1468 # Create mask of the current clipped footprints
1469 maskClip = mask.Factory(mask.getBBox(afwImage.PARENT))
1470 afwDet.setMaskFromFootprintList(maskClip, result.clipFootprints, maskClipValue)
1472 maskClipBig = maskClip.Factory(mask.getBBox(afwImage.PARENT))
1473 afwDet.setMaskFromFootprintList(maskClipBig, bigFootprints, maskClipValue)
1474 maskClip |= maskClipBig
1476 # Assemble coadd from base class, but ignoring CLIPPED pixels
1477 badMaskPlanes = self.config.badMaskPlanes[:]
1478 badMaskPlanes.append("CLIPPED")
1479 badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes)
1480 return AssembleCoaddTask.run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
1481 result.clipSpans, mask=badPixelMask)
1483 def buildDifferenceImage(self, skyInfo, tempExpRefList, imageScalerList, weightList):
1484 """Return an exposure that contains the difference between unclipped
1485 and clipped coadds.
1487 Generate a difference image between clipped and unclipped coadds.
1488 Compute the difference image by subtracting an outlier-clipped coadd
1489 from an outlier-unclipped coadd. Return the difference image.
1491 Parameters
1492 ----------
1493 skyInfo : `lsst.pipe.base.Struct`
1494 Patch geometry information, from getSkyInfo
1495 tempExpRefList : `list`
1496 List of data reference to tempExp
1497 imageScalerList : `list`
1498 List of image scalers
1499 weightList : `list`
1500 List of weights
1502 Returns
1503 -------
1504 exp : `lsst.afw.image.Exposure`
1505 Difference image of unclipped and clipped coadd wrapped in an Exposure
1506 """
1507 coaddMean = self.assembleMeanCoadd.run(skyInfo, tempExpRefList,
1508 imageScalerList, weightList).coaddExposure
1510 coaddClip = self.assembleMeanClipCoadd.run(skyInfo, tempExpRefList,
1511 imageScalerList, weightList).coaddExposure
1513 coaddDiff = coaddMean.getMaskedImage().Factory(coaddMean.getMaskedImage())
1514 coaddDiff -= coaddClip.getMaskedImage()
1515 exp = afwImage.ExposureF(coaddDiff)
1516 exp.setPsf(coaddMean.getPsf())
1517 return exp
1519 def detectClip(self, exp, tempExpRefList):
1520 """Detect clipped regions on an exposure and set the mask on the
1521 individual tempExp masks.
1523 Detect footprints in the difference image after smoothing the
1524 difference image with a Gaussian kernal. Identify footprints that
1525 overlap with one or two input ``coaddTempExps`` by comparing the
1526 computed overlap fraction to thresholds set in the config. A different
1527 threshold is applied depending on the number of overlapping visits
1528 (restricted to one or two). If the overlap exceeds the thresholds,
1529 the footprint is considered "CLIPPED" and is marked as such on the
1530 coaddTempExp. Return a struct with the clipped footprints, the indices
1531 of the ``coaddTempExps`` that end up overlapping with the clipped
1532 footprints, and a list of new masks for the ``coaddTempExps``.
1534 Parameters
1535 ----------
1536 exp : `lsst.afw.image.Exposure`
1537 Exposure to run detection on.
1538 tempExpRefList : `list`
1539 List of data reference to tempExp.
1541 Returns
1542 -------
1543 result : `lsst.pipe.base.Struct`
1544 Result struct with components:
1546 - ``clipFootprints``: list of clipped footprints.
1547 - ``clipIndices``: indices for each ``clippedFootprint`` in
1548 ``tempExpRefList``.
1549 - ``clipSpans``: List of dictionaries containing spanSet lists
1550 to clip. Each element contains the new maskplane name
1551 ("CLIPPED") as the key and list of ``SpanSets`` as the value.
1552 - ``detectionFootprints``: List of DETECTED/DETECTED_NEGATIVE plane
1553 compressed into footprints.
1554 """
1555 mask = exp.getMaskedImage().getMask()
1556 maskDetValue = mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE")
1557 fpSet = self.clipDetection.detectFootprints(exp, doSmooth=True, clearMask=True)
1558 # Merge positive and negative together footprints together
1559 fpSet.positive.merge(fpSet.negative)
1560 footprints = fpSet.positive
1561 self.log.info('Found %d potential clipped objects', len(footprints.getFootprints()))
1562 ignoreMask = self.getBadPixelMask()
1564 clipFootprints = []
1565 clipIndices = []
1566 artifactSpanSets = [{'CLIPPED': list()} for _ in tempExpRefList]
1568 # for use by detectClipBig
1569 visitDetectionFootprints = []
1571 dims = [len(tempExpRefList), len(footprints.getFootprints())]
1572 overlapDetArr = numpy.zeros(dims, dtype=numpy.uint16)
1573 ignoreArr = numpy.zeros(dims, dtype=numpy.uint16)
1575 # Loop over masks once and extract/store only relevant overlap metrics and detection footprints
1576 for i, warpRef in enumerate(tempExpRefList):
1577 tmpExpMask = warpRef.get(datasetType=self.getTempExpDatasetName(self.warpType),
1578 immediate=True).getMaskedImage().getMask()
1579 maskVisitDet = tmpExpMask.Factory(tmpExpMask, tmpExpMask.getBBox(afwImage.PARENT),
1580 afwImage.PARENT, True)
1581 maskVisitDet &= maskDetValue
1582 visitFootprints = afwDet.FootprintSet(maskVisitDet, afwDet.Threshold(1))
1583 visitDetectionFootprints.append(visitFootprints)
1585 for j, footprint in enumerate(footprints.getFootprints()):
1586 ignoreArr[i, j] = countMaskFromFootprint(tmpExpMask, footprint, ignoreMask, 0x0)
1587 overlapDetArr[i, j] = countMaskFromFootprint(tmpExpMask, footprint, maskDetValue, ignoreMask)
1589 # build a list of clipped spans for each visit
1590 for j, footprint in enumerate(footprints.getFootprints()):
1591 nPixel = footprint.getArea()
1592 overlap = [] # hold the overlap with each visit
1593 indexList = [] # index of visit in global list
1594 for i in range(len(tempExpRefList)):
1595 ignore = ignoreArr[i, j]
1596 overlapDet = overlapDetArr[i, j]
1597 totPixel = nPixel - ignore
1599 # If we have more bad pixels than detection skip
1600 if ignore > overlapDet or totPixel <= 0.5*nPixel or overlapDet == 0:
1601 continue
1602 overlap.append(overlapDet/float(totPixel))
1603 indexList.append(i)
1605 overlap = numpy.array(overlap)
1606 if not len(overlap):
1607 continue
1609 keep = False # Should this footprint be marked as clipped?
1610 keepIndex = [] # Which tempExps does the clipped footprint belong to
1612 # If footprint only has one overlap use a lower threshold
1613 if len(overlap) == 1:
1614 if overlap[0] > self.config.minClipFootOverlapSingle:
1615 keep = True
1616 keepIndex = [0]
1617 else:
1618 # This is the general case where only visit should be clipped
1619 clipIndex = numpy.where(overlap > self.config.minClipFootOverlap)[0]
1620 if len(clipIndex) == 1:
1621 keep = True
1622 keepIndex = [clipIndex[0]]
1624 # Test if there are clipped objects that overlap two different visits
1625 clipIndex = numpy.where(overlap > self.config.minClipFootOverlapDouble)[0]
1626 if len(clipIndex) == 2 and len(overlap) > 3:
1627 clipIndexComp = numpy.where(overlap <= self.config.minClipFootOverlapDouble)[0]
1628 if numpy.max(overlap[clipIndexComp]) <= self.config.maxClipFootOverlapDouble:
1629 keep = True
1630 keepIndex = clipIndex
1632 if not keep:
1633 continue
1635 for index in keepIndex:
1636 globalIndex = indexList[index]
1637 artifactSpanSets[globalIndex]['CLIPPED'].append(footprint.spans)
1639 clipIndices.append(numpy.array(indexList)[keepIndex])
1640 clipFootprints.append(footprint)
1642 return pipeBase.Struct(clipFootprints=clipFootprints, clipIndices=clipIndices,
1643 clipSpans=artifactSpanSets, detectionFootprints=visitDetectionFootprints)
1645 def detectClipBig(self, clipList, clipFootprints, clipIndices, detectionFootprints,
1646 maskClipValue, maskDetValue, coaddBBox):
1647 """Return individual warp footprints for large artifacts and append
1648 them to ``clipList`` in place.
1650 Identify big footprints composed of many sources in the coadd
1651 difference that may have originated in a large diffuse source in the
1652 coadd. We do this by indentifying all clipped footprints that overlap
1653 significantly with each source in all the coaddTempExps.
1655 Parameters
1656 ----------
1657 clipList : `list`
1658 List of alt mask SpanSets with clipping information. Modified.
1659 clipFootprints : `list`
1660 List of clipped footprints.
1661 clipIndices : `list`
1662 List of which entries in tempExpClipList each footprint belongs to.
1663 maskClipValue
1664 Mask value of clipped pixels.
1665 maskDetValue
1666 Mask value of detected pixels.
1667 coaddBBox : `lsst.geom.Box`
1668 BBox of the coadd and warps.
1670 Returns
1671 -------
1672 bigFootprintsCoadd : `list`
1673 List of big footprints
1674 """
1675 bigFootprintsCoadd = []
1676 ignoreMask = self.getBadPixelMask()
1677 for index, (clippedSpans, visitFootprints) in enumerate(zip(clipList, detectionFootprints)):
1678 maskVisitDet = afwImage.MaskX(coaddBBox, 0x0)
1679 for footprint in visitFootprints.getFootprints():
1680 footprint.spans.setMask(maskVisitDet, maskDetValue)
1682 # build a mask of clipped footprints that are in this visit
1683 clippedFootprintsVisit = []
1684 for foot, clipIndex in zip(clipFootprints, clipIndices):
1685 if index not in clipIndex:
1686 continue
1687 clippedFootprintsVisit.append(foot)
1688 maskVisitClip = maskVisitDet.Factory(maskVisitDet.getBBox(afwImage.PARENT))
1689 afwDet.setMaskFromFootprintList(maskVisitClip, clippedFootprintsVisit, maskClipValue)
1691 bigFootprintsVisit = []
1692 for foot in visitFootprints.getFootprints():
1693 if foot.getArea() < self.config.minBigOverlap:
1694 continue
1695 nCount = countMaskFromFootprint(maskVisitClip, foot, maskClipValue, ignoreMask)
1696 if nCount > self.config.minBigOverlap:
1697 bigFootprintsVisit.append(foot)
1698 bigFootprintsCoadd.append(foot)
1700 for footprint in bigFootprintsVisit:
1701 clippedSpans["CLIPPED"].append(footprint.spans)
1703 return bigFootprintsCoadd
1706class CompareWarpAssembleCoaddConnections(AssembleCoaddConnections):
1707 psfMatchedWarps = pipeBase.connectionTypes.Input(
1708 doc=("PSF-Matched Warps are required by CompareWarp regardless of the coadd type requested. "
1709 "Only PSF-Matched Warps make sense for image subtraction. "
1710 "Therefore, they must be an additional declared input."),
1711 name="{inputCoaddName}Coadd_psfMatchedWarp",
1712 storageClass="ExposureF",
1713 dimensions=("tract", "patch", "skymap", "visit"),
1714 deferLoad=True,
1715 multiple=True
1716 )
1717 templateCoadd = pipeBase.connectionTypes.Output(
1718 doc=("Model of the static sky, used to find temporal artifacts. Typically a PSF-Matched, "
1719 "sigma-clipped coadd. Written if and only if assembleStaticSkyModel.doWrite=True"),
1720 name="{fakesType}{outputCoaddName}CoaddPsfMatched",
1721 storageClass="ExposureF",
1722 dimensions=("tract", "patch", "skymap", "abstract_filter"),
1723 )
1725 def __init__(self, *, config=None):
1726 super().__init__(config=config)
1727 if not config.assembleStaticSkyModel.doWrite:
1728 self.outputs.remove("templateCoadd")
1729 config.validate()
1732class CompareWarpAssembleCoaddConfig(AssembleCoaddConfig,
1733 pipelineConnections=CompareWarpAssembleCoaddConnections):
1734 assembleStaticSkyModel = pexConfig.ConfigurableField(
1735 target=AssembleCoaddTask,
1736 doc="Task to assemble an artifact-free, PSF-matched Coadd to serve as a"
1737 " naive/first-iteration model of the static sky.",
1738 )
1739 detect = pexConfig.ConfigurableField(
1740 target=SourceDetectionTask,
1741 doc="Detect outlier sources on difference between each psfMatched warp and static sky model"
1742 )
1743 detectTemplate = pexConfig.ConfigurableField(
1744 target=SourceDetectionTask,
1745 doc="Detect sources on static sky model. Only used if doPreserveContainedBySource is True"
1746 )
1747 maxNumEpochs = pexConfig.Field(
1748 doc="Charactistic maximum local number of epochs/visits in which an artifact candidate can appear "
1749 "and still be masked. The effective maxNumEpochs is a broken linear function of local "
1750 "number of epochs (N): min(maxFractionEpochsLow*N, maxNumEpochs + maxFractionEpochsHigh*N). "
1751 "For each footprint detected on the image difference between the psfMatched warp and static sky "
1752 "model, if a significant fraction of pixels (defined by spatialThreshold) are residuals in more "
1753 "than the computed effective maxNumEpochs, the artifact candidate is deemed persistant rather "
1754 "than transient and not masked.",
1755 dtype=int,
1756 default=2
1757 )
1758 maxFractionEpochsLow = pexConfig.RangeField(
1759 doc="Fraction of local number of epochs (N) to use as effective maxNumEpochs for low N. "
1760 "Effective maxNumEpochs = "
1761 "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)",
1762 dtype=float,
1763 default=0.4,
1764 min=0., max=1.,
1765 )
1766 maxFractionEpochsHigh = pexConfig.RangeField(
1767 doc="Fraction of local number of epochs (N) to use as effective maxNumEpochs for high N. "
1768 "Effective maxNumEpochs = "
1769 "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)",
1770 dtype=float,
1771 default=0.03,
1772 min=0., max=1.,
1773 )
1774 spatialThreshold = pexConfig.RangeField(
1775 doc="Unitless fraction of pixels defining how much of the outlier region has to meet the "
1776 "temporal criteria. If 0, clip all. If 1, clip none.",
1777 dtype=float,
1778 default=0.5,
1779 min=0., max=1.,
1780 inclusiveMin=True, inclusiveMax=True
1781 )
1782 doScaleWarpVariance = pexConfig.Field(
1783 doc="Rescale Warp variance plane using empirical noise?",
1784 dtype=bool,
1785 default=True,
1786 )
1787 scaleWarpVariance = pexConfig.ConfigurableField(
1788 target=ScaleVarianceTask,
1789 doc="Rescale variance on warps",
1790 )
1791 doPreserveContainedBySource = pexConfig.Field(
1792 doc="Rescue artifacts from clipping that completely lie within a footprint detected"
1793 "on the PsfMatched Template Coadd. Replicates a behavior of SafeClip.",
1794 dtype=bool,
1795 default=True,
1796 )
1797 doPrefilterArtifacts = pexConfig.Field(
1798 doc="Ignore artifact candidates that are mostly covered by the bad pixel mask, "
1799 "because they will be excluded anyway. This prevents them from contributing "
1800 "to the outlier epoch count image and potentially being labeled as persistant."
1801 "'Mostly' is defined by the config 'prefilterArtifactsRatio'.",
1802 dtype=bool,
1803 default=True
1804 )
1805 prefilterArtifactsMaskPlanes = pexConfig.ListField(
1806 doc="Prefilter artifact candidates that are mostly covered by these bad mask planes.",
1807 dtype=str,
1808 default=('NO_DATA', 'BAD', 'SAT', 'SUSPECT'),
1809 )
1810 prefilterArtifactsRatio = pexConfig.Field(
1811 doc="Prefilter artifact candidates with less than this fraction overlapping good pixels",
1812 dtype=float,
1813 default=0.05
1814 )
1816 def setDefaults(self):
1817 AssembleCoaddConfig.setDefaults(self)
1818 self.statistic = 'MEAN'
1819 self.doUsePsfMatchedPolygons = True
1821 # Real EDGE removed by psfMatched NO_DATA border half the width of the matching kernel
1822 # CompareWarp applies psfMatched EDGE pixels to directWarps before assembling
1823 if "EDGE" in self.badMaskPlanes:
1824 self.badMaskPlanes.remove('EDGE')
1825 self.removeMaskPlanes.append('EDGE')
1826 self.assembleStaticSkyModel.badMaskPlanes = ["NO_DATA", ]
1827 self.assembleStaticSkyModel.warpType = 'psfMatched'
1828 self.assembleStaticSkyModel.connections.warpType = 'psfMatched'
1829 self.assembleStaticSkyModel.statistic = 'MEANCLIP'
1830 self.assembleStaticSkyModel.sigmaClip = 2.5
1831 self.assembleStaticSkyModel.clipIter = 3
1832 self.assembleStaticSkyModel.calcErrorFromInputVariance = False
1833 self.assembleStaticSkyModel.doWrite = False
1834 self.detect.doTempLocalBackground = False
1835 self.detect.reEstimateBackground = False
1836 self.detect.returnOriginalFootprints = False
1837 self.detect.thresholdPolarity = "both"
1838 self.detect.thresholdValue = 5
1839 self.detect.minPixels = 4
1840 self.detect.isotropicGrow = True
1841 self.detect.thresholdType = "pixel_stdev"
1842 self.detect.nSigmaToGrow = 0.4
1843 # The default nSigmaToGrow for SourceDetectionTask is already 2.4,
1844 # Explicitly restating because ratio with detect.nSigmaToGrow matters
1845 self.detectTemplate.nSigmaToGrow = 2.4
1846 self.detectTemplate.doTempLocalBackground = False
1847 self.detectTemplate.reEstimateBackground = False
1848 self.detectTemplate.returnOriginalFootprints = False
1850 def validate(self):
1851 super().validate()
1852 if self.assembleStaticSkyModel.doNImage:
1853 raise ValueError("No dataset type exists for a PSF-Matched Template N Image."
1854 "Please set assembleStaticSkyModel.doNImage=False")
1856 if self.assembleStaticSkyModel.doWrite and (self.warpType == self.assembleStaticSkyModel.warpType):
1857 raise ValueError("warpType (%s) == assembleStaticSkyModel.warpType (%s) and will compete for "
1858 "the same dataset name. Please set assembleStaticSkyModel.doWrite to False "
1859 "or warpType to 'direct'. assembleStaticSkyModel.warpType should ways be "
1860 "'PsfMatched'" % (self.warpType, self.assembleStaticSkyModel.warpType))
1863class CompareWarpAssembleCoaddTask(AssembleCoaddTask):
1864 """Assemble a compareWarp coadded image from a set of warps
1865 by masking artifacts detected by comparing PSF-matched warps.
1867 In ``AssembleCoaddTask``, we compute the coadd as an clipped mean (i.e.,
1868 we clip outliers). The problem with doing this is that when computing the
1869 coadd PSF at a given location, individual visit PSFs from visits with
1870 outlier pixels contribute to the coadd PSF and cannot be treated correctly.
1871 In this task, we correct for this behavior by creating a new badMaskPlane
1872 'CLIPPED' which marks pixels in the individual warps suspected to contain
1873 an artifact. We populate this plane on the input warps by comparing
1874 PSF-matched warps with a PSF-matched median coadd which serves as a
1875 model of the static sky. Any group of pixels that deviates from the
1876 PSF-matched template coadd by more than config.detect.threshold sigma,
1877 is an artifact candidate. The candidates are then filtered to remove
1878 variable sources and sources that are difficult to subtract such as
1879 bright stars. This filter is configured using the config parameters
1880 ``temporalThreshold`` and ``spatialThreshold``. The temporalThreshold is
1881 the maximum fraction of epochs that the deviation can appear in and still
1882 be considered an artifact. The spatialThreshold is the maximum fraction of
1883 pixels in the footprint of the deviation that appear in other epochs
1884 (where other epochs is defined by the temporalThreshold). If the deviant
1885 region meets this criteria of having a significant percentage of pixels
1886 that deviate in only a few epochs, these pixels have the 'CLIPPED' bit
1887 set in the mask. These regions will not contribute to the final coadd.
1888 Furthermore, any routine to determine the coadd PSF can now be cognizant
1889 of clipped regions. Note that the algorithm implemented by this task is
1890 preliminary and works correctly for HSC data. Parameter modifications and
1891 or considerable redesigning of the algorithm is likley required for other
1892 surveys.
1894 ``CompareWarpAssembleCoaddTask`` sub-classes
1895 ``AssembleCoaddTask`` and instantiates ``AssembleCoaddTask``
1896 as a subtask to generate the TemplateCoadd (the model of the static sky).
1898 Notes
1899 -----
1900 The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a
1901 flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``; see
1902 ``baseDebug`` for more about ``debug.py`` files.
1904 This task supports the following debug variables:
1906 - ``saveCountIm``
1907 If True then save the Epoch Count Image as a fits file in the `figPath`
1908 - ``figPath``
1909 Path to save the debug fits images and figures
1911 For example, put something like:
1913 .. code-block:: python
1915 import lsstDebug
1916 def DebugInfo(name):
1917 di = lsstDebug.getInfo(name)
1918 if name == "lsst.pipe.tasks.assembleCoadd":
1919 di.saveCountIm = True
1920 di.figPath = "/desired/path/to/debugging/output/images"
1921 return di
1922 lsstDebug.Info = DebugInfo
1924 into your ``debug.py`` file and run ``assemebleCoadd.py`` with the
1925 ``--debug`` flag. Some subtasks may have their own debug variables;
1926 see individual Task documentation.
1928 Examples
1929 --------
1930 ``CompareWarpAssembleCoaddTask`` assembles a set of warped images into a
1931 coadded image. The ``CompareWarpAssembleCoaddTask`` is invoked by running
1932 ``assembleCoadd.py`` with the flag ``--compareWarpCoadd``.
1933 Usage of ``assembleCoadd.py`` expects a data reference to the tract patch
1934 and filter to be coadded (specified using
1935 '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]')
1936 along with a list of coaddTempExps to attempt to coadd (specified using
1937 '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]').
1938 Only the warps that cover the specified tract and patch will be coadded.
1939 A list of the available optional arguments can be obtained by calling
1940 ``assembleCoadd.py`` with the ``--help`` command line argument:
1942 .. code-block:: none
1944 assembleCoadd.py --help
1946 To demonstrate usage of the ``CompareWarpAssembleCoaddTask`` in the larger
1947 context of multi-band processing, we will generate the HSC-I & -R band
1948 oadds from HSC engineering test data provided in the ``ci_hsc`` package.
1949 To begin, assuming that the lsst stack has been already set up, we must
1950 set up the ``obs_subaru`` and ``ci_hsc`` packages.
1951 This defines the environment variable ``$CI_HSC_DIR`` and points at the
1952 location of the package. The raw HSC data live in the ``$CI_HSC_DIR/raw``
1953 directory. To begin assembling the coadds, we must first
1955 - processCcd
1956 process the individual ccds in $CI_HSC_RAW to produce calibrated exposures
1957 - makeSkyMap
1958 create a skymap that covers the area of the sky present in the raw exposures
1959 - makeCoaddTempExp
1960 warp the individual calibrated exposures to the tangent plane of the coadd
1962 We can perform all of these steps by running
1964 .. code-block:: none
1966 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
1968 This will produce warped ``coaddTempExps`` for each visit. To coadd the
1969 warped data, we call ``assembleCoadd.py`` as follows:
1971 .. code-block:: none
1973 assembleCoadd.py --compareWarpCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \
1974 --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \
1975 --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \
1976 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \
1977 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \
1978 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \
1979 --selectId visit=903988 ccd=24
1981 This will process the HSC-I band data. The results are written in
1982 ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``.
1983 """
1984 ConfigClass = CompareWarpAssembleCoaddConfig
1985 _DefaultName = "compareWarpAssembleCoadd"
1987 def __init__(self, *args, **kwargs):
1988 AssembleCoaddTask.__init__(self, *args, **kwargs)
1989 self.makeSubtask("assembleStaticSkyModel")
1990 detectionSchema = afwTable.SourceTable.makeMinimalSchema()
1991 self.makeSubtask("detect", schema=detectionSchema)
1992 if self.config.doPreserveContainedBySource:
1993 self.makeSubtask("detectTemplate", schema=afwTable.SourceTable.makeMinimalSchema())
1994 if self.config.doScaleWarpVariance:
1995 self.makeSubtask("scaleWarpVariance")
1997 @utils.inheritDoc(AssembleCoaddTask)
1998 def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs):
1999 """
2000 Generate a templateCoadd to use as a naive model of static sky to
2001 subtract from PSF-Matched warps.
2003 Returns
2004 -------
2005 result : `lsst.pipe.base.Struct`
2006 Result struct with components:
2008 - ``templateCoadd`` : coadded exposure (``lsst.afw.image.Exposure``)
2009 - ``nImage`` : N Image (``lsst.afw.image.Image``)
2010 """
2011 # Ensure that psfMatchedWarps are used as input warps for template generation
2012 staticSkyModelInputRefs = copy.deepcopy(inputRefs)
2013 staticSkyModelInputRefs.inputWarps = inputRefs.psfMatchedWarps
2015 # Because subtasks don't have connections we have to make one.
2016 # The main task's `templateCoadd` is the subtask's `coaddExposure`
2017 staticSkyModelOutputRefs = copy.deepcopy(outputRefs)
2018 if self.config.assembleStaticSkyModel.doWrite:
2019 staticSkyModelOutputRefs.coaddExposure = staticSkyModelOutputRefs.templateCoadd
2020 # Remove template coadd from both subtask's and main tasks outputs,
2021 # because it is handled by the subtask as `coaddExposure`
2022 del outputRefs.templateCoadd
2023 del staticSkyModelOutputRefs.templateCoadd
2025 # A PSF-Matched nImage does not exist as a dataset type
2026 if 'nImage' in staticSkyModelOutputRefs.keys():
2027 del staticSkyModelOutputRefs.nImage
2029 templateCoadd = self.assembleStaticSkyModel.runQuantum(butlerQC, staticSkyModelInputRefs,
2030 staticSkyModelOutputRefs)
2031 if templateCoadd is None:
2032 raise RuntimeError(self._noTemplateMessage(self.assembleStaticSkyModel.warpType))
2034 return pipeBase.Struct(templateCoadd=templateCoadd.coaddExposure,
2035 nImage=templateCoadd.nImage,
2036 warpRefList=templateCoadd.warpRefList,
2037 imageScalerList=templateCoadd.imageScalerList,
2038 weightList=templateCoadd.weightList)
2040 @utils.inheritDoc(AssembleCoaddTask)
2041 def makeSupplementaryData(self, dataRef, selectDataList=None, warpRefList=None):
2042 """
2043 Generate a templateCoadd to use as a naive model of static sky to
2044 subtract from PSF-Matched warps.
2046 Returns
2047 -------
2048 result : `lsst.pipe.base.Struct`
2049 Result struct with components:
2051 - ``templateCoadd``: coadded exposure (``lsst.afw.image.Exposure``)
2052 - ``nImage``: N Image (``lsst.afw.image.Image``)
2053 """
2054 templateCoadd = self.assembleStaticSkyModel.runDataRef(dataRef, selectDataList, warpRefList)
2055 if templateCoadd is None:
2056 raise RuntimeError(self._noTemplateMessage(self.assembleStaticSkyModel.warpType))
2058 return pipeBase.Struct(templateCoadd=templateCoadd.coaddExposure,
2059 nImage=templateCoadd.nImage,
2060 warpRefList=templateCoadd.warpRefList,
2061 imageScalerList=templateCoadd.imageScalerList,
2062 weightList=templateCoadd.weightList)
2064 def _noTemplateMessage(self, warpType):
2065 warpName = (warpType[0].upper() + warpType[1:])
2066 message = """No %(warpName)s warps were found to build the template coadd which is
2067 required to run CompareWarpAssembleCoaddTask. To continue assembling this type of coadd,
2068 first either rerun makeCoaddTempExp with config.make%(warpName)s=True or
2069 coaddDriver with config.makeCoadTempExp.make%(warpName)s=True, before assembleCoadd.
2071 Alternatively, to use another algorithm with existing warps, retarget the CoaddDriverConfig to
2072 another algorithm like:
2074 from lsst.pipe.tasks.assembleCoadd import SafeClipAssembleCoaddTask
2075 config.assemble.retarget(SafeClipAssembleCoaddTask)
2076 """ % {"warpName": warpName}
2077 return message
2079 @utils.inheritDoc(AssembleCoaddTask)
2080 @pipeBase.timeMethod
2081 def run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
2082 supplementaryData, *args, **kwargs):
2083 """Assemble the coadd.
2085 Find artifacts and apply them to the warps' masks creating a list of
2086 alternative masks with a new "CLIPPED" plane and updated "NO_DATA"
2087 plane. Then pass these alternative masks to the base class's `run`
2088 method.
2090 The input parameters ``supplementaryData`` is a `lsst.pipe.base.Struct`
2091 that must contain a ``templateCoadd`` that serves as the
2092 model of the static sky.
2093 """
2095 # Check and match the order of the supplementaryData
2096 # (PSF-matched) inputs to the order of the direct inputs,
2097 # so that the artifact mask is applied to the right warp
2098 dataIds = [ref.dataId for ref in tempExpRefList]
2099 psfMatchedDataIds = [ref.dataId for ref in supplementaryData.warpRefList]
2101 if dataIds != psfMatchedDataIds:
2102 self.log.info("Reordering and or/padding PSF-matched visit input list")
2103 supplementaryData.warpRefList = reorderAndPadList(supplementaryData.warpRefList,
2104 psfMatchedDataIds, dataIds)
2105 supplementaryData.imageScalerList = reorderAndPadList(supplementaryData.imageScalerList,
2106 psfMatchedDataIds, dataIds)
2108 # Use PSF-Matched Warps (and corresponding scalers) and coadd to find artifacts
2109 spanSetMaskList = self.findArtifacts(supplementaryData.templateCoadd,
2110 supplementaryData.warpRefList,
2111 supplementaryData.imageScalerList)
2113 badMaskPlanes = self.config.badMaskPlanes[:]
2114 badMaskPlanes.append("CLIPPED")
2115 badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes)
2117 result = AssembleCoaddTask.run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
2118 spanSetMaskList, mask=badPixelMask)
2120 # Propagate PSF-matched EDGE pixels to coadd SENSOR_EDGE and INEXACT_PSF
2121 # Psf-Matching moves the real edge inwards
2122 self.applyAltEdgeMask(result.coaddExposure.maskedImage.mask, spanSetMaskList)
2123 return result
2125 def applyAltEdgeMask(self, mask, altMaskList):
2126 """Propagate alt EDGE mask to SENSOR_EDGE AND INEXACT_PSF planes.
2128 Parameters
2129 ----------
2130 mask : `lsst.afw.image.Mask`
2131 Original mask.
2132 altMaskList : `list`
2133 List of Dicts containing ``spanSet`` lists.
2134 Each element contains the new mask plane name (e.g. "CLIPPED
2135 and/or "NO_DATA") as the key, and list of ``SpanSets`` to apply to
2136 the mask.
2137 """
2138 maskValue = mask.getPlaneBitMask(["SENSOR_EDGE", "INEXACT_PSF"])
2139 for visitMask in altMaskList:
2140 if "EDGE" in visitMask:
2141 for spanSet in visitMask['EDGE']:
2142 spanSet.clippedTo(mask.getBBox()).setMask(mask, maskValue)
2144 def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList):
2145 """Find artifacts.
2147 Loop through warps twice. The first loop builds a map with the count
2148 of how many epochs each pixel deviates from the templateCoadd by more
2149 than ``config.chiThreshold`` sigma. The second loop takes each
2150 difference image and filters the artifacts detected in each using
2151 count map to filter out variable sources and sources that are
2152 difficult to subtract cleanly.
2154 Parameters
2155 ----------
2156 templateCoadd : `lsst.afw.image.Exposure`
2157 Exposure to serve as model of static sky.
2158 tempExpRefList : `list`
2159 List of data references to warps.
2160 imageScalerList : `list`
2161 List of image scalers.
2163 Returns
2164 -------
2165 altMasks : `list`
2166 List of dicts containing information about CLIPPED
2167 (i.e., artifacts), NO_DATA, and EDGE pixels.
2168 """
2170 self.log.debug("Generating Count Image, and mask lists.")
2171 coaddBBox = templateCoadd.getBBox()
2172 slateIm = afwImage.ImageU(coaddBBox)
2173 epochCountImage = afwImage.ImageU(coaddBBox)
2174 nImage = afwImage.ImageU(coaddBBox)
2175 spanSetArtifactList = []
2176 spanSetNoDataMaskList = []
2177 spanSetEdgeList = []
2178 badPixelMask = self.getBadPixelMask()
2180 # mask of the warp diffs should = that of only the warp
2181 templateCoadd.mask.clearAllMaskPlanes()
2183 if self.config.doPreserveContainedBySource:
2184 templateFootprints = self.detectTemplate.detectFootprints(templateCoadd)
2185 else:
2186 templateFootprints = None
2188 for warpRef, imageScaler in zip(tempExpRefList, imageScalerList):
2189 warpDiffExp = self._readAndComputeWarpDiff(warpRef, imageScaler, templateCoadd)
2190 if warpDiffExp is not None:
2191 # This nImage only approximates the final nImage because it uses the PSF-matched mask
2192 nImage.array += (numpy.isfinite(warpDiffExp.image.array)
2193 * ((warpDiffExp.mask.array & badPixelMask) == 0)).astype(numpy.uint16)
2194 fpSet = self.detect.detectFootprints(warpDiffExp, doSmooth=False, clearMask=True)
2195 fpSet.positive.merge(fpSet.negative)
2196 footprints = fpSet.positive
2197 slateIm.set(0)
2198 spanSetList = [footprint.spans for footprint in footprints.getFootprints()]
2200 # Remove artifacts due to defects before they contribute to the epochCountImage
2201 if self.config.doPrefilterArtifacts:
2202 spanSetList = self.prefilterArtifacts(spanSetList, warpDiffExp)
2203 for spans in spanSetList:
2204 spans.setImage(slateIm, 1, doClip=True)
2205 epochCountImage += slateIm
2207 # PSF-Matched warps have less available area (~the matching kernel) because the calexps
2208 # undergo a second convolution. Pixels with data in the direct warp
2209 # but not in the PSF-matched warp will not have their artifacts detected.
2210 # NaNs from the PSF-matched warp therefore must be masked in the direct warp
2211 nans = numpy.where(numpy.isnan(warpDiffExp.maskedImage.image.array), 1, 0)
2212 nansMask = afwImage.makeMaskFromArray(nans.astype(afwImage.MaskPixel))
2213 nansMask.setXY0(warpDiffExp.getXY0())
2214 edgeMask = warpDiffExp.mask
2215 spanSetEdgeMask = afwGeom.SpanSet.fromMask(edgeMask,
2216 edgeMask.getPlaneBitMask("EDGE")).split()
2217 else:
2218 # If the directWarp has <1% coverage, the psfMatchedWarp can have 0% and not exist
2219 # In this case, mask the whole epoch
2220 nansMask = afwImage.MaskX(coaddBBox, 1)
2221 spanSetList = []
2222 spanSetEdgeMask = []
2224 spanSetNoDataMask = afwGeom.SpanSet.fromMask(nansMask).split()
2226 spanSetNoDataMaskList.append(spanSetNoDataMask)
2227 spanSetArtifactList.append(spanSetList)
2228 spanSetEdgeList.append(spanSetEdgeMask)
2230 if lsstDebug.Info(__name__).saveCountIm:
2231 path = self._dataRef2DebugPath("epochCountIm", tempExpRefList[0], coaddLevel=True)
2232 epochCountImage.writeFits(path)
2234 for i, spanSetList in enumerate(spanSetArtifactList):
2235 if spanSetList:
2236 filteredSpanSetList = self.filterArtifacts(spanSetList, epochCountImage, nImage,
2237 templateFootprints)
2238 spanSetArtifactList[i] = filteredSpanSetList
2240 altMasks = []
2241 for artifacts, noData, edge in zip(spanSetArtifactList, spanSetNoDataMaskList, spanSetEdgeList):
2242 altMasks.append({'CLIPPED': artifacts,
2243 'NO_DATA': noData,
2244 'EDGE': edge})
2245 return altMasks
2247 def prefilterArtifacts(self, spanSetList, exp):
2248 """Remove artifact candidates covered by bad mask plane.
2250 Any future editing of the candidate list that does not depend on
2251 temporal information should go in this method.
2253 Parameters
2254 ----------
2255 spanSetList : `list`
2256 List of SpanSets representing artifact candidates.
2257 exp : `lsst.afw.image.Exposure`
2258 Exposure containing mask planes used to prefilter.
2260 Returns
2261 -------
2262 returnSpanSetList : `list`
2263 List of SpanSets with artifacts.
2264 """
2265 badPixelMask = exp.mask.getPlaneBitMask(self.config.prefilterArtifactsMaskPlanes)
2266 goodArr = (exp.mask.array & badPixelMask) == 0
2267 returnSpanSetList = []
2268 bbox = exp.getBBox()
2269 x0, y0 = exp.getXY0()
2270 for i, span in enumerate(spanSetList):
2271 y, x = span.clippedTo(bbox).indices()
2272 yIndexLocal = numpy.array(y) - y0
2273 xIndexLocal = numpy.array(x) - x0
2274 goodRatio = numpy.count_nonzero(goodArr[yIndexLocal, xIndexLocal])/span.getArea()
2275 if goodRatio > self.config.prefilterArtifactsRatio:
2276 returnSpanSetList.append(span)
2277 return returnSpanSetList
2279 def filterArtifacts(self, spanSetList, epochCountImage, nImage, footprintsToExclude=None):
2280 """Filter artifact candidates.
2282 Parameters
2283 ----------
2284 spanSetList : `list`
2285 List of SpanSets representing artifact candidates.
2286 epochCountImage : `lsst.afw.image.Image`
2287 Image of accumulated number of warpDiff detections.
2288 nImage : `lsst.afw.image.Image`
2289 Image of the accumulated number of total epochs contributing.
2291 Returns
2292 -------
2293 maskSpanSetList : `list`
2294 List of SpanSets with artifacts.
2295 """
2297 maskSpanSetList = []
2298 x0, y0 = epochCountImage.getXY0()
2299 for i, span in enumerate(spanSetList):
2300 y, x = span.indices()
2301 yIdxLocal = [y1 - y0 for y1 in y]
2302 xIdxLocal = [x1 - x0 for x1 in x]
2303 outlierN = epochCountImage.array[yIdxLocal, xIdxLocal]
2304 totalN = nImage.array[yIdxLocal, xIdxLocal]
2306 # effectiveMaxNumEpochs is broken line (fraction of N) with characteristic config.maxNumEpochs
2307 effMaxNumEpochsHighN = (self.config.maxNumEpochs
2308 + self.config.maxFractionEpochsHigh*numpy.mean(totalN))
2309 effMaxNumEpochsLowN = self.config.maxFractionEpochsLow * numpy.mean(totalN)
2310 effectiveMaxNumEpochs = int(min(effMaxNumEpochsLowN, effMaxNumEpochsHighN))
2311 nPixelsBelowThreshold = numpy.count_nonzero((outlierN > 0)
2312 & (outlierN <= effectiveMaxNumEpochs))
2313 percentBelowThreshold = nPixelsBelowThreshold / len(outlierN)
2314 if percentBelowThreshold > self.config.spatialThreshold:
2315 maskSpanSetList.append(span)
2317 if self.config.doPreserveContainedBySource and footprintsToExclude is not None:
2318 # If a candidate is contained by a footprint on the template coadd, do not clip
2319 filteredMaskSpanSetList = []
2320 for span in maskSpanSetList:
2321 doKeep = True
2322 for footprint in footprintsToExclude.positive.getFootprints():
2323 if footprint.spans.contains(span):
2324 doKeep = False
2325 break
2326 if doKeep:
2327 filteredMaskSpanSetList.append(span)
2328 maskSpanSetList = filteredMaskSpanSetList
2330 return maskSpanSetList
2332 def _readAndComputeWarpDiff(self, warpRef, imageScaler, templateCoadd):
2333 """Fetch a warp from the butler and return a warpDiff.
2335 Parameters
2336 ----------
2337 warpRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
2338 Butler dataRef for the warp.
2339 imageScaler : `lsst.pipe.tasks.scaleZeroPoint.ImageScaler`
2340 An image scaler object.
2341 templateCoadd : `lsst.afw.image.Exposure`
2342 Exposure to be substracted from the scaled warp.
2344 Returns
2345 -------
2346 warp : `lsst.afw.image.Exposure`
2347 Exposure of the image difference between the warp and template.
2348 """
2350 # If the PSF-Matched warp did not exist for this direct warp
2351 # None is holding its place to maintain order in Gen 3
2352 if warpRef is None:
2353 return None
2354 # Warp comparison must use PSF-Matched Warps regardless of requested coadd warp type
2355 warpName = self.getTempExpDatasetName('psfMatched')
2356 if not isinstance(warpRef, DeferredDatasetHandle):
2357 if not warpRef.datasetExists(warpName):
2358 self.log.warn("Could not find %s %s; skipping it", warpName, warpRef.dataId)
2359 return None
2360 warp = warpRef.get(datasetType=warpName, immediate=True)
2361 # direct image scaler OK for PSF-matched Warp
2362 imageScaler.scaleMaskedImage(warp.getMaskedImage())
2363 mi = warp.getMaskedImage()
2364 if self.config.doScaleWarpVariance:
2365 try:
2366 self.scaleWarpVariance.run(mi)
2367 except Exception as exc:
2368 self.log.warn("Unable to rescale variance of warp (%s); leaving it as-is" % (exc,))
2369 mi -= templateCoadd.getMaskedImage()
2370 return warp
2372 def _dataRef2DebugPath(self, prefix, warpRef, coaddLevel=False):
2373 """Return a path to which to write debugging output.
2375 Creates a hyphen-delimited string of dataId values for simple filenames.
2377 Parameters
2378 ----------
2379 prefix : `str`
2380 Prefix for filename.
2381 warpRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
2382 Butler dataRef to make the path from.
2383 coaddLevel : `bool`, optional.
2384 If True, include only coadd-level keys (e.g., 'tract', 'patch',
2385 'filter', but no 'visit').
2387 Returns
2388 -------
2389 result : `str`
2390 Path for debugging output.
2391 """
2392 if coaddLevel:
2393 keys = warpRef.getButler().getKeys(self.getCoaddDatasetName(self.warpType))
2394 else:
2395 keys = warpRef.dataId.keys()
2396 keyList = sorted(keys, reverse=True)
2397 directory = lsstDebug.Info(__name__).figPath if lsstDebug.Info(__name__).figPath else "."
2398 filename = "%s-%s.fits" % (prefix, '-'.join([str(warpRef.dataId[k]) for k in keyList]))
2399 return os.path.join(directory, filename)
2402def reorderAndPadList(inputList, inputKeys, outputKeys, padWith=None):
2403 """Match the order of one list to another, padding if necessary
2405 Parameters
2406 ----------
2407 inputList : list
2408 List to be reordered and padded. Elements can be any type.
2409 inputKeys : iterable
2410 Iterable of values to be compared with outputKeys.
2411 Length must match `inputList`
2412 outputKeys : iterable
2413 Iterable of values to be compared with inputKeys.
2414 padWith :
2415 Any value to be inserted where inputKey not in outputKeys
2417 Returns
2418 -------
2419 list
2420 Copy of inputList reordered per outputKeys and padded with `padWith`
2421 so that the length matches length of outputKeys.
2422 """
2423 outputList = []
2424 for d in outputKeys:
2425 if d in inputKeys:
2426 outputList.append(inputList[inputKeys.index(d)])
2427 else:
2428 outputList.append(padWith)
2429 return outputList