Coverage for python/lsst/pipe/tasks/assembleCoadd.py : 72%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of pipe_tasks.
2#
3# LSST Data Management System
4# This product includes software developed by the
5# LSST Project (http://www.lsst.org/).
6# See COPYRIGHT file at the top of the source tree.
7#
8# This program is free software: you can redistribute it and/or modify
9# it under the terms of the GNU General Public License as published by
10# the Free Software Foundation, either version 3 of the License, or
11# (at your option) any later version.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the LSST License Statement and
19# the GNU General Public License along with this program. If not,
20# see <https://www.lsstcorp.org/LegalNotices/>.
21#
22import os
23import copy
24import numpy
25import warnings
26import lsst.pex.config as pexConfig
27import lsst.pex.exceptions as pexExceptions
28import lsst.geom as geom
29import lsst.afw.geom as afwGeom
30import lsst.afw.image as afwImage
31import lsst.afw.math as afwMath
32import lsst.afw.table as afwTable
33import lsst.afw.detection as afwDet
34import lsst.coadd.utils as coaddUtils
35import lsst.pipe.base as pipeBase
36import lsst.meas.algorithms as measAlg
37import lsst.log as log
38import lsstDebug
39import lsst.utils as utils
40from .coaddBase import CoaddBaseTask, SelectDataIdContainer, makeSkyInfo, makeCoaddSuffix
41from .interpImage import InterpImageTask
42from .scaleZeroPoint import ScaleZeroPointTask
43from .coaddHelpers import groupPatchExposures, getGroupDataRef
44from .scaleVariance import ScaleVarianceTask
45from .maskStreaks import MaskStreaksTask
46from lsst.meas.algorithms import SourceDetectionTask
47from lsst.daf.butler import DeferredDatasetHandle
49__all__ = ["AssembleCoaddTask", "AssembleCoaddConnections", "AssembleCoaddConfig",
50 "SafeClipAssembleCoaddTask", "SafeClipAssembleCoaddConfig",
51 "CompareWarpAssembleCoaddTask", "CompareWarpAssembleCoaddConfig"]
54class AssembleCoaddConnections(pipeBase.PipelineTaskConnections,
55 dimensions=("tract", "patch", "abstract_filter", "skymap"),
56 defaultTemplates={"inputCoaddName": "deep",
57 "outputCoaddName": "deep",
58 "warpType": "direct",
59 "warpTypeSuffix": "",
60 "fakesType": ""}):
61 inputWarps = pipeBase.connectionTypes.Input(
62 doc=("Input list of warps to be assemebled i.e. stacked."
63 "WarpType (e.g. direct, psfMatched) is controlled by the warpType config parameter"),
64 name="{inputCoaddName}Coadd_{warpType}Warp",
65 storageClass="ExposureF",
66 dimensions=("tract", "patch", "skymap", "visit", "instrument"),
67 deferLoad=True,
68 multiple=True
69 )
70 skyMap = pipeBase.connectionTypes.Input(
71 doc="Input definition of geometry/bbox and projection/wcs for coadded exposures",
72 name="{inputCoaddName}Coadd_skyMap",
73 storageClass="SkyMap",
74 dimensions=("skymap", ),
75 )
76 brightObjectMask = pipeBase.connectionTypes.PrerequisiteInput(
77 doc=("Input Bright Object Mask mask produced with external catalogs to be applied to the mask plane"
78 " BRIGHT_OBJECT."),
79 name="brightObjectMask",
80 storageClass="ObjectMaskCatalog",
81 dimensions=("tract", "patch", "skymap", "abstract_filter"),
82 )
83 coaddExposure = pipeBase.connectionTypes.Output(
84 doc="Output coadded exposure, produced by stacking input warps",
85 name="{fakesType}{outputCoaddName}Coadd{warpTypeSuffix}",
86 storageClass="ExposureF",
87 dimensions=("tract", "patch", "skymap", "abstract_filter"),
88 )
89 nImage = pipeBase.connectionTypes.Output(
90 doc="Output image of number of input images per pixel",
91 name="{outputCoaddName}Coadd_nImage",
92 storageClass="ImageU",
93 dimensions=("tract", "patch", "skymap", "abstract_filter"),
94 )
96 def __init__(self, *, config=None):
97 # Override the connection's name template with config to replicate Gen2 behavior
98 config.connections.warpType = config.warpType
99 config.connections.warpTypeSuffix = makeCoaddSuffix(config.warpType)
101 if config.hasFakes:
102 config.connections.fakesType = "_fakes"
104 super().__init__(config=config)
106 if not config.doMaskBrightObjects:
107 self.prerequisiteInputs.remove("brightObjectMask")
109 if not config.doNImage:
110 self.outputs.remove("nImage")
113class AssembleCoaddConfig(CoaddBaseTask.ConfigClass, pipeBase.PipelineTaskConfig,
114 pipelineConnections=AssembleCoaddConnections):
115 """Configuration parameters for the `AssembleCoaddTask`.
117 Notes
118 -----
119 The `doMaskBrightObjects` and `brightObjectMaskName` configuration options
120 only set the bitplane config.brightObjectMaskName. To make this useful you
121 *must* also configure the flags.pixel algorithm, for example by adding
123 .. code-block:: none
125 config.measurement.plugins["base_PixelFlags"].masksFpCenter.append("BRIGHT_OBJECT")
126 config.measurement.plugins["base_PixelFlags"].masksFpAnywhere.append("BRIGHT_OBJECT")
128 to your measureCoaddSources.py and forcedPhotCoadd.py config overrides.
129 """
130 warpType = pexConfig.Field(
131 doc="Warp name: one of 'direct' or 'psfMatched'",
132 dtype=str,
133 default="direct",
134 )
135 subregionSize = pexConfig.ListField(
136 dtype=int,
137 doc="Width, height of stack subregion size; "
138 "make small enough that a full stack of images will fit into memory at once.",
139 length=2,
140 default=(2000, 2000),
141 )
142 statistic = pexConfig.Field(
143 dtype=str,
144 doc="Main stacking statistic for aggregating over the epochs.",
145 default="MEANCLIP",
146 )
147 doSigmaClip = pexConfig.Field(
148 dtype=bool,
149 doc="Perform sigma clipped outlier rejection with MEANCLIP statistic? (DEPRECATED)",
150 default=False,
151 )
152 sigmaClip = pexConfig.Field(
153 dtype=float,
154 doc="Sigma for outlier rejection; ignored if non-clipping statistic selected.",
155 default=3.0,
156 )
157 clipIter = pexConfig.Field(
158 dtype=int,
159 doc="Number of iterations of outlier rejection; ignored if non-clipping statistic selected.",
160 default=2,
161 )
162 calcErrorFromInputVariance = pexConfig.Field(
163 dtype=bool,
164 doc="Calculate coadd variance from input variance by stacking statistic."
165 "Passed to StatisticsControl.setCalcErrorFromInputVariance()",
166 default=True,
167 )
168 scaleZeroPoint = pexConfig.ConfigurableField(
169 target=ScaleZeroPointTask,
170 doc="Task to adjust the photometric zero point of the coadd temp exposures",
171 )
172 doInterp = pexConfig.Field(
173 doc="Interpolate over NaN pixels? Also extrapolate, if necessary, but the results are ugly.",
174 dtype=bool,
175 default=True,
176 )
177 interpImage = pexConfig.ConfigurableField(
178 target=InterpImageTask,
179 doc="Task to interpolate (and extrapolate) over NaN pixels",
180 )
181 doWrite = pexConfig.Field(
182 doc="Persist coadd?",
183 dtype=bool,
184 default=True,
185 )
186 doNImage = pexConfig.Field(
187 doc="Create image of number of contributing exposures for each pixel",
188 dtype=bool,
189 default=False,
190 )
191 doUsePsfMatchedPolygons = pexConfig.Field(
192 doc="Use ValidPolygons from shrunk Psf-Matched Calexps? Should be set to True by CompareWarp only.",
193 dtype=bool,
194 default=False,
195 )
196 maskPropagationThresholds = pexConfig.DictField(
197 keytype=str,
198 itemtype=float,
199 doc=("Threshold (in fractional weight) of rejection at which we propagate a mask plane to "
200 "the coadd; that is, we set the mask bit on the coadd if the fraction the rejected frames "
201 "would have contributed exceeds this value."),
202 default={"SAT": 0.1},
203 )
204 removeMaskPlanes = pexConfig.ListField(dtype=str, default=["NOT_DEBLENDED"],
205 doc="Mask planes to remove before coadding")
206 doMaskBrightObjects = pexConfig.Field(dtype=bool, default=False,
207 doc="Set mask and flag bits for bright objects?")
208 brightObjectMaskName = pexConfig.Field(dtype=str, default="BRIGHT_OBJECT",
209 doc="Name of mask bit used for bright objects")
210 coaddPsf = pexConfig.ConfigField(
211 doc="Configuration for CoaddPsf",
212 dtype=measAlg.CoaddPsfConfig,
213 )
214 doAttachTransmissionCurve = pexConfig.Field(
215 dtype=bool, default=False, optional=False,
216 doc=("Attach a piecewise TransmissionCurve for the coadd? "
217 "(requires all input Exposures to have TransmissionCurves).")
218 )
219 hasFakes = pexConfig.Field(
220 dtype=bool,
221 default=False,
222 doc="Should be set to True if fake sources have been inserted into the input data."
223 )
225 def setDefaults(self):
226 super().setDefaults()
227 self.badMaskPlanes = ["NO_DATA", "BAD", "SAT", "EDGE"]
229 def validate(self):
230 super().validate()
231 if self.doPsfMatch:
232 # Backwards compatibility.
233 # Configs do not have loggers
234 log.warn("Config doPsfMatch deprecated. Setting warpType='psfMatched'")
235 self.warpType = 'psfMatched'
236 if self.doSigmaClip and self.statistic != "MEANCLIP":
237 log.warn('doSigmaClip deprecated. To replicate behavior, setting statistic to "MEANCLIP"')
238 self.statistic = "MEANCLIP"
239 if self.doInterp and self.statistic not in ['MEAN', 'MEDIAN', 'MEANCLIP', 'VARIANCE', 'VARIANCECLIP']:
240 raise ValueError("Must set doInterp=False for statistic=%s, which does not "
241 "compute and set a non-zero coadd variance estimate." % (self.statistic))
243 unstackableStats = ['NOTHING', 'ERROR', 'ORMASK']
244 if not hasattr(afwMath.Property, self.statistic) or self.statistic in unstackableStats:
245 stackableStats = [str(k) for k in afwMath.Property.__members__.keys()
246 if str(k) not in unstackableStats]
247 raise ValueError("statistic %s is not allowed. Please choose one of %s."
248 % (self.statistic, stackableStats))
251class AssembleCoaddTask(CoaddBaseTask, pipeBase.PipelineTask):
252 """Assemble a coadded image from a set of warps (coadded temporary exposures).
254 We want to assemble a coadded image from a set of Warps (also called
255 coadded temporary exposures or ``coaddTempExps``).
256 Each input Warp covers a patch on the sky and corresponds to a single
257 run/visit/exposure of the covered patch. We provide the task with a list
258 of Warps (``selectDataList``) from which it selects Warps that cover the
259 specified patch (pointed at by ``dataRef``).
260 Each Warp that goes into a coadd will typically have an independent
261 photometric zero-point. Therefore, we must scale each Warp to set it to
262 a common photometric zeropoint. WarpType may be one of 'direct' or
263 'psfMatched', and the boolean configs `config.makeDirect` and
264 `config.makePsfMatched` set which of the warp types will be coadded.
265 The coadd is computed as a mean with optional outlier rejection.
266 Criteria for outlier rejection are set in `AssembleCoaddConfig`.
267 Finally, Warps can have bad 'NaN' pixels which received no input from the
268 source calExps. We interpolate over these bad (NaN) pixels.
270 `AssembleCoaddTask` uses several sub-tasks. These are
272 - `ScaleZeroPointTask`
273 - create and use an ``imageScaler`` object to scale the photometric zeropoint for each Warp
274 - `InterpImageTask`
275 - interpolate across bad pixels (NaN) in the final coadd
277 You can retarget these subtasks if you wish.
279 Notes
280 -----
281 The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a
282 flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``; see
283 `baseDebug` for more about ``debug.py`` files. `AssembleCoaddTask` has
284 no debug variables of its own. Some of the subtasks may support debug
285 variables. See the documentation for the subtasks for further information.
287 Examples
288 --------
289 `AssembleCoaddTask` assembles a set of warped images into a coadded image.
290 The `AssembleCoaddTask` can be invoked by running ``assembleCoadd.py``
291 with the flag '--legacyCoadd'. Usage of assembleCoadd.py expects two
292 inputs: a data reference to the tract patch and filter to be coadded, and
293 a list of Warps to attempt to coadd. These are specified using ``--id`` and
294 ``--selectId``, respectively:
296 .. code-block:: none
298 --id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]
299 --selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]
301 Only the Warps that cover the specified tract and patch will be coadded.
302 A list of the available optional arguments can be obtained by calling
303 ``assembleCoadd.py`` with the ``--help`` command line argument:
305 .. code-block:: none
307 assembleCoadd.py --help
309 To demonstrate usage of the `AssembleCoaddTask` in the larger context of
310 multi-band processing, we will generate the HSC-I & -R band coadds from
311 HSC engineering test data provided in the ``ci_hsc`` package. To begin,
312 assuming that the lsst stack has been already set up, we must set up the
313 obs_subaru and ``ci_hsc`` packages. This defines the environment variable
314 ``$CI_HSC_DIR`` and points at the location of the package. The raw HSC
315 data live in the ``$CI_HSC_DIR/raw directory``. To begin assembling the
316 coadds, we must first
318 - processCcd
319 - process the individual ccds in $CI_HSC_RAW to produce calibrated exposures
320 - makeSkyMap
321 - create a skymap that covers the area of the sky present in the raw exposures
322 - makeCoaddTempExp
323 - warp the individual calibrated exposures to the tangent plane of the coadd
325 We can perform all of these steps by running
327 .. code-block:: none
329 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
331 This will produce warped exposures for each visit. To coadd the warped
332 data, we call assembleCoadd.py as follows:
334 .. code-block:: none
336 assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \
337 --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \
338 --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \
339 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \
340 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \
341 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \
342 --selectId visit=903988 ccd=24
344 that will process the HSC-I band data. The results are written in
345 ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``.
347 You may also choose to run:
349 .. code-block:: none
351 scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346
352 assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R \
353 --selectId visit=903334 ccd=16 --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 \
354 --selectId visit=903334 ccd=100 --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 \
355 --selectId visit=903338 ccd=18 --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 \
356 --selectId visit=903342 ccd=10 --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 \
357 --selectId visit=903344 ccd=5 --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 \
358 --selectId visit=903346 ccd=6 --selectId visit=903346 ccd=12
360 to generate the coadd for the HSC-R band if you are interested in
361 following multiBand Coadd processing as discussed in `pipeTasks_multiBand`
362 (but note that normally, one would use the `SafeClipAssembleCoaddTask`
363 rather than `AssembleCoaddTask` to make the coadd.
364 """
365 ConfigClass = AssembleCoaddConfig
366 _DefaultName = "assembleCoadd"
368 def __init__(self, *args, **kwargs):
369 # TODO: DM-17415 better way to handle previously allowed passed args e.g.`AssembleCoaddTask(config)`
370 if args: 370 ↛ 371line 370 didn't jump to line 371, because the condition on line 370 was never true
371 argNames = ["config", "name", "parentTask", "log"]
372 kwargs.update({k: v for k, v in zip(argNames, args)})
373 warnings.warn("AssembleCoadd received positional args, and casting them as kwargs: %s. "
374 "PipelineTask will not take positional args" % argNames, FutureWarning)
376 super().__init__(**kwargs)
377 self.makeSubtask("interpImage")
378 self.makeSubtask("scaleZeroPoint")
380 if self.config.doMaskBrightObjects: 380 ↛ 381line 380 didn't jump to line 381, because the condition on line 380 was never true
381 mask = afwImage.Mask()
382 try:
383 self.brightObjectBitmask = 1 << mask.addMaskPlane(self.config.brightObjectMaskName)
384 except pexExceptions.LsstCppException:
385 raise RuntimeError("Unable to define mask plane for bright objects; planes used are %s" %
386 mask.getMaskPlaneDict().keys())
387 del mask
389 self.warpType = self.config.warpType
391 @utils.inheritDoc(pipeBase.PipelineTask)
392 def runQuantum(self, butlerQC, inputRefs, outputRefs):
393 # Docstring to be formatted with info from PipelineTask.runQuantum
394 """
395 Notes
396 -----
397 Assemble a coadd from a set of Warps.
399 PipelineTask (Gen3) entry point to Coadd a set of Warps.
400 Analogous to `runDataRef`, it prepares all the data products to be
401 passed to `run`, and processes the results before returning a struct
402 of results to be written out. AssembleCoadd cannot fit all Warps in memory.
403 Therefore, its inputs are accessed subregion by subregion
404 by the Gen3 `DeferredDatasetHandle` that is analagous to the Gen2
405 `lsst.daf.persistence.ButlerDataRef`. Any updates to this method should
406 correspond to an update in `runDataRef` while both entry points
407 are used.
408 """
409 inputData = butlerQC.get(inputRefs)
411 # Construct skyInfo expected by run
412 # Do not remove skyMap from inputData in case makeSupplementaryDataGen3 needs it
413 skyMap = inputData["skyMap"]
414 outputDataId = butlerQC.quantum.dataId
416 inputData['skyInfo'] = makeSkyInfo(skyMap,
417 tractId=outputDataId['tract'],
418 patchId=outputDataId['patch'])
420 # Construct list of input Deferred Datasets
421 # These quack a bit like like Gen2 DataRefs
422 warpRefList = inputData['inputWarps']
423 # Perform same middle steps as `runDataRef` does
424 inputs = self.prepareInputs(warpRefList)
425 self.log.info("Found %d %s", len(inputs.tempExpRefList),
426 self.getTempExpDatasetName(self.warpType))
427 if len(inputs.tempExpRefList) == 0:
428 self.log.warn("No coadd temporary exposures found")
429 return
431 supplementaryData = self.makeSupplementaryDataGen3(butlerQC, inputRefs, outputRefs)
432 retStruct = self.run(inputData['skyInfo'], inputs.tempExpRefList, inputs.imageScalerList,
433 inputs.weightList, supplementaryData=supplementaryData)
435 inputData.setdefault('brightObjectMask', None)
436 self.processResults(retStruct.coaddExposure, inputData['brightObjectMask'], outputDataId)
438 if self.config.doWrite:
439 butlerQC.put(retStruct, outputRefs)
440 return retStruct
442 @pipeBase.timeMethod
443 def runDataRef(self, dataRef, selectDataList=None, warpRefList=None):
444 """Assemble a coadd from a set of Warps.
446 Pipebase.CmdlineTask entry point to Coadd a set of Warps.
447 Compute weights to be applied to each Warp and
448 find scalings to match the photometric zeropoint to a reference Warp.
449 Assemble the Warps using `run`. Interpolate over NaNs and
450 optionally write the coadd to disk. Return the coadded exposure.
452 Parameters
453 ----------
454 dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
455 Data reference defining the patch for coaddition and the
456 reference Warp (if ``config.autoReference=False``).
457 Used to access the following data products:
458 - ``self.config.coaddName + "Coadd_skyMap"``
459 - ``self.config.coaddName + "Coadd_ + <warpType> + "Warp"`` (optionally)
460 - ``self.config.coaddName + "Coadd"``
461 selectDataList : `list`
462 List of data references to Calexps. Data to be coadded will be
463 selected from this list based on overlap with the patch defined
464 by dataRef, grouped by visit, and converted to a list of data
465 references to warps.
466 warpRefList : `list`
467 List of data references to Warps to be coadded.
468 Note: `warpRefList` is just the new name for `tempExpRefList`.
470 Returns
471 -------
472 retStruct : `lsst.pipe.base.Struct`
473 Result struct with components:
475 - ``coaddExposure``: coadded exposure (``Exposure``).
476 - ``nImage``: exposure count image (``Image``).
477 """
478 if selectDataList and warpRefList: 478 ↛ 479line 478 didn't jump to line 479, because the condition on line 478 was never true
479 raise RuntimeError("runDataRef received both a selectDataList and warpRefList, "
480 "and which to use is ambiguous. Please pass only one.")
482 skyInfo = self.getSkyInfo(dataRef)
483 if warpRefList is None:
484 calExpRefList = self.selectExposures(dataRef, skyInfo, selectDataList=selectDataList)
485 if len(calExpRefList) == 0: 485 ↛ 486line 485 didn't jump to line 486, because the condition on line 485 was never true
486 self.log.warn("No exposures to coadd")
487 return
488 self.log.info("Coadding %d exposures", len(calExpRefList))
490 warpRefList = self.getTempExpRefList(dataRef, calExpRefList)
492 inputData = self.prepareInputs(warpRefList)
493 self.log.info("Found %d %s", len(inputData.tempExpRefList),
494 self.getTempExpDatasetName(self.warpType))
495 if len(inputData.tempExpRefList) == 0: 495 ↛ 496line 495 didn't jump to line 496, because the condition on line 495 was never true
496 self.log.warn("No coadd temporary exposures found")
497 return
499 supplementaryData = self.makeSupplementaryData(dataRef, warpRefList=inputData.tempExpRefList)
501 retStruct = self.run(skyInfo, inputData.tempExpRefList, inputData.imageScalerList,
502 inputData.weightList, supplementaryData=supplementaryData)
504 brightObjects = self.readBrightObjectMasks(dataRef) if self.config.doMaskBrightObjects else None
505 self.processResults(retStruct.coaddExposure, brightObjectMasks=brightObjects, dataId=dataRef.dataId)
507 if self.config.doWrite:
508 if self.getCoaddDatasetName(self.warpType) == "deepCoadd" and self.config.hasFakes: 508 ↛ 509line 508 didn't jump to line 509, because the condition on line 508 was never true
509 coaddDatasetName = "fakes_" + self.getCoaddDatasetName(self.warpType)
510 else:
511 coaddDatasetName = self.getCoaddDatasetName(self.warpType)
512 self.log.info("Persisting %s" % coaddDatasetName)
513 dataRef.put(retStruct.coaddExposure, coaddDatasetName)
514 if self.config.doNImage and retStruct.nImage is not None: 514 ↛ 515line 514 didn't jump to line 515, because the condition on line 514 was never true
515 dataRef.put(retStruct.nImage, self.getCoaddDatasetName(self.warpType) + '_nImage')
517 return retStruct
519 def processResults(self, coaddExposure, brightObjectMasks=None, dataId=None):
520 """Interpolate over missing data and mask bright stars.
522 Parameters
523 ----------
524 coaddExposure : `lsst.afw.image.Exposure`
525 The coadded exposure to process.
526 dataRef : `lsst.daf.persistence.ButlerDataRef`
527 Butler data reference for supplementary data.
528 """
529 if self.config.doInterp: 529 ↛ 536line 529 didn't jump to line 536, because the condition on line 529 was never false
530 self.interpImage.run(coaddExposure.getMaskedImage(), planeName="NO_DATA")
531 # The variance must be positive; work around for DM-3201.
532 varArray = coaddExposure.variance.array
533 with numpy.errstate(invalid="ignore"):
534 varArray[:] = numpy.where(varArray > 0, varArray, numpy.inf)
536 if self.config.doMaskBrightObjects: 536 ↛ 537line 536 didn't jump to line 537, because the condition on line 536 was never true
537 self.setBrightObjectMasks(coaddExposure, brightObjectMasks, dataId)
539 def makeSupplementaryData(self, dataRef, selectDataList=None, warpRefList=None):
540 """Make additional inputs to run() specific to subclasses (Gen2)
542 Duplicates interface of `runDataRef` method
543 Available to be implemented by subclasses only if they need the
544 coadd dataRef for performing preliminary processing before
545 assembling the coadd.
547 Parameters
548 ----------
549 dataRef : `lsst.daf.persistence.ButlerDataRef`
550 Butler data reference for supplementary data.
551 selectDataList : `list` (optional)
552 Optional List of data references to Calexps.
553 warpRefList : `list` (optional)
554 Optional List of data references to Warps.
555 """
556 return pipeBase.Struct()
558 def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs):
559 """Make additional inputs to run() specific to subclasses (Gen3)
561 Duplicates interface of `runQuantum` method.
562 Available to be implemented by subclasses only if they need the
563 coadd dataRef for performing preliminary processing before
564 assembling the coadd.
566 Parameters
567 ----------
568 butlerQC : `lsst.pipe.base.ButlerQuantumContext`
569 Gen3 Butler object for fetching additional data products before
570 running the Task specialized for quantum being processed
571 inputRefs : `lsst.pipe.base.InputQuantizedConnection`
572 Attributes are the names of the connections describing input dataset types.
573 Values are DatasetRefs that task consumes for corresponding dataset type.
574 DataIds are guaranteed to match data objects in ``inputData``.
575 outputRefs : `lsst.pipe.base.OutputQuantizedConnection`
576 Attributes are the names of the connections describing output dataset types.
577 Values are DatasetRefs that task is to produce
578 for corresponding dataset type.
579 """
580 return pipeBase.Struct()
582 def getTempExpRefList(self, patchRef, calExpRefList):
583 """Generate list data references corresponding to warped exposures
584 that lie within the patch to be coadded.
586 Parameters
587 ----------
588 patchRef : `dataRef`
589 Data reference for patch.
590 calExpRefList : `list`
591 List of data references for input calexps.
593 Returns
594 -------
595 tempExpRefList : `list`
596 List of Warp/CoaddTempExp data references.
597 """
598 butler = patchRef.getButler()
599 groupData = groupPatchExposures(patchRef, calExpRefList, self.getCoaddDatasetName(self.warpType),
600 self.getTempExpDatasetName(self.warpType))
601 tempExpRefList = [getGroupDataRef(butler, self.getTempExpDatasetName(self.warpType),
602 g, groupData.keys) for
603 g in groupData.groups.keys()]
604 return tempExpRefList
606 def prepareInputs(self, refList):
607 """Prepare the input warps for coaddition by measuring the weight for
608 each warp and the scaling for the photometric zero point.
610 Each Warp has its own photometric zeropoint and background variance.
611 Before coadding these Warps together, compute a scale factor to
612 normalize the photometric zeropoint and compute the weight for each Warp.
614 Parameters
615 ----------
616 refList : `list`
617 List of data references to tempExp
619 Returns
620 -------
621 result : `lsst.pipe.base.Struct`
622 Result struct with components:
624 - ``tempExprefList``: `list` of data references to tempExp.
625 - ``weightList``: `list` of weightings.
626 - ``imageScalerList``: `list` of image scalers.
627 """
628 statsCtrl = afwMath.StatisticsControl()
629 statsCtrl.setNumSigmaClip(self.config.sigmaClip)
630 statsCtrl.setNumIter(self.config.clipIter)
631 statsCtrl.setAndMask(self.getBadPixelMask())
632 statsCtrl.setNanSafe(True)
633 # compute tempExpRefList: a list of tempExpRef that actually exist
634 # and weightList: a list of the weight of the associated coadd tempExp
635 # and imageScalerList: a list of scale factors for the associated coadd tempExp
636 tempExpRefList = []
637 weightList = []
638 imageScalerList = []
639 tempExpName = self.getTempExpDatasetName(self.warpType)
640 for tempExpRef in refList:
641 # Gen3's DeferredDatasetHandles are guaranteed to exist and
642 # therefore have no datasetExists() method
643 if not isinstance(tempExpRef, DeferredDatasetHandle):
644 if not tempExpRef.datasetExists(tempExpName):
645 self.log.warn("Could not find %s %s; skipping it", tempExpName, tempExpRef.dataId)
646 continue
648 tempExp = tempExpRef.get(datasetType=tempExpName, immediate=True)
649 # Ignore any input warp that is empty of data
650 if numpy.isnan(tempExp.image.array).all(): 650 ↛ 651line 650 didn't jump to line 651, because the condition on line 650 was never true
651 continue
652 maskedImage = tempExp.getMaskedImage()
653 imageScaler = self.scaleZeroPoint.computeImageScaler(
654 exposure=tempExp,
655 dataRef=tempExpRef,
656 )
657 try:
658 imageScaler.scaleMaskedImage(maskedImage)
659 except Exception as e:
660 self.log.warn("Scaling failed for %s (skipping it): %s", tempExpRef.dataId, e)
661 continue
662 statObj = afwMath.makeStatistics(maskedImage.getVariance(), maskedImage.getMask(),
663 afwMath.MEANCLIP, statsCtrl)
664 meanVar, meanVarErr = statObj.getResult(afwMath.MEANCLIP)
665 weight = 1.0 / float(meanVar)
666 if not numpy.isfinite(weight): 666 ↛ 667line 666 didn't jump to line 667, because the condition on line 666 was never true
667 self.log.warn("Non-finite weight for %s: skipping", tempExpRef.dataId)
668 continue
669 self.log.info("Weight of %s %s = %0.3f", tempExpName, tempExpRef.dataId, weight)
671 del maskedImage
672 del tempExp
674 tempExpRefList.append(tempExpRef)
675 weightList.append(weight)
676 imageScalerList.append(imageScaler)
678 return pipeBase.Struct(tempExpRefList=tempExpRefList, weightList=weightList,
679 imageScalerList=imageScalerList)
681 def prepareStats(self, mask=None):
682 """Prepare the statistics for coadding images.
684 Parameters
685 ----------
686 mask : `int`, optional
687 Bit mask value to exclude from coaddition.
689 Returns
690 -------
691 stats : `lsst.pipe.base.Struct`
692 Statistics structure with the following fields:
694 - ``statsCtrl``: Statistics control object for coadd
695 (`lsst.afw.math.StatisticsControl`)
696 - ``statsFlags``: Statistic for coadd (`lsst.afw.math.Property`)
697 """
698 if mask is None:
699 mask = self.getBadPixelMask()
700 statsCtrl = afwMath.StatisticsControl()
701 statsCtrl.setNumSigmaClip(self.config.sigmaClip)
702 statsCtrl.setNumIter(self.config.clipIter)
703 statsCtrl.setAndMask(mask)
704 statsCtrl.setNanSafe(True)
705 statsCtrl.setWeighted(True)
706 statsCtrl.setCalcErrorFromInputVariance(self.config.calcErrorFromInputVariance)
707 for plane, threshold in self.config.maskPropagationThresholds.items():
708 bit = afwImage.Mask.getMaskPlane(plane)
709 statsCtrl.setMaskPropagationThreshold(bit, threshold)
710 statsFlags = afwMath.stringToStatisticsProperty(self.config.statistic)
711 return pipeBase.Struct(ctrl=statsCtrl, flags=statsFlags)
713 @pipeBase.timeMethod
714 def run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
715 altMaskList=None, mask=None, supplementaryData=None):
716 """Assemble a coadd from input warps
718 Assemble the coadd using the provided list of coaddTempExps. Since
719 the full coadd covers a patch (a large area), the assembly is
720 performed over small areas on the image at a time in order to
721 conserve memory usage. Iterate over subregions within the outer
722 bbox of the patch using `assembleSubregion` to stack the corresponding
723 subregions from the coaddTempExps with the statistic specified.
724 Set the edge bits the coadd mask based on the weight map.
726 Parameters
727 ----------
728 skyInfo : `lsst.pipe.base.Struct`
729 Struct with geometric information about the patch.
730 tempExpRefList : `list`
731 List of data references to Warps (previously called CoaddTempExps).
732 imageScalerList : `list`
733 List of image scalers.
734 weightList : `list`
735 List of weights
736 altMaskList : `list`, optional
737 List of alternate masks to use rather than those stored with
738 tempExp.
739 mask : `int`, optional
740 Bit mask value to exclude from coaddition.
741 supplementaryData : lsst.pipe.base.Struct, optional
742 Struct with additional data products needed to assemble coadd.
743 Only used by subclasses that implement `makeSupplementaryData`
744 and override `run`.
746 Returns
747 -------
748 result : `lsst.pipe.base.Struct`
749 Result struct with components:
751 - ``coaddExposure``: coadded exposure (``lsst.afw.image.Exposure``).
752 - ``nImage``: exposure count image (``lsst.afw.image.Image``), if requested.
753 - ``warpRefList``: input list of refs to the warps (
754 ``lsst.daf.butler.DeferredDatasetHandle`` or
755 ``lsst.daf.persistence.ButlerDataRef``)
756 (unmodified)
757 - ``imageScalerList``: input list of image scalers (unmodified)
758 - ``weightList``: input list of weights (unmodified)
759 """
760 tempExpName = self.getTempExpDatasetName(self.warpType)
761 self.log.info("Assembling %s %s", len(tempExpRefList), tempExpName)
762 stats = self.prepareStats(mask=mask)
764 if altMaskList is None:
765 altMaskList = [None]*len(tempExpRefList)
767 coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
768 coaddExposure.setPhotoCalib(self.scaleZeroPoint.getPhotoCalib())
769 coaddExposure.getInfo().setCoaddInputs(self.inputRecorder.makeCoaddInputs())
770 self.assembleMetadata(coaddExposure, tempExpRefList, weightList)
771 coaddMaskedImage = coaddExposure.getMaskedImage()
772 subregionSizeArr = self.config.subregionSize
773 subregionSize = geom.Extent2I(subregionSizeArr[0], subregionSizeArr[1])
774 # if nImage is requested, create a zero one which can be passed to assembleSubregion
775 if self.config.doNImage: 775 ↛ 776line 775 didn't jump to line 776, because the condition on line 775 was never true
776 nImage = afwImage.ImageU(skyInfo.bbox)
777 else:
778 nImage = None
779 for subBBox in self._subBBoxIter(skyInfo.bbox, subregionSize):
780 try:
781 self.assembleSubregion(coaddExposure, subBBox, tempExpRefList, imageScalerList,
782 weightList, altMaskList, stats.flags, stats.ctrl,
783 nImage=nImage)
784 except Exception as e:
785 self.log.fatal("Cannot compute coadd %s: %s", subBBox, e)
787 self.setInexactPsf(coaddMaskedImage.getMask())
788 # Despite the name, the following doesn't really deal with "EDGE" pixels: it identifies
789 # pixels that didn't receive any unmasked inputs (as occurs around the edge of the field).
790 coaddUtils.setCoaddEdgeBits(coaddMaskedImage.getMask(), coaddMaskedImage.getVariance())
791 return pipeBase.Struct(coaddExposure=coaddExposure, nImage=nImage,
792 warpRefList=tempExpRefList, imageScalerList=imageScalerList,
793 weightList=weightList)
795 def assembleMetadata(self, coaddExposure, tempExpRefList, weightList):
796 """Set the metadata for the coadd.
798 This basic implementation sets the filter from the first input.
800 Parameters
801 ----------
802 coaddExposure : `lsst.afw.image.Exposure`
803 The target exposure for the coadd.
804 tempExpRefList : `list`
805 List of data references to tempExp.
806 weightList : `list`
807 List of weights.
808 """
809 assert len(tempExpRefList) == len(weightList), "Length mismatch"
810 tempExpName = self.getTempExpDatasetName(self.warpType)
811 # We load a single pixel of each coaddTempExp, because we just want to get at the metadata
812 # (and we need more than just the PropertySet that contains the header), which is not possible
813 # with the current butler (see #2777).
814 bbox = geom.Box2I(coaddExposure.getBBox().getMin(), geom.Extent2I(1, 1))
816 if isinstance(tempExpRefList[0], DeferredDatasetHandle):
817 # Gen 3 API
818 tempExpList = [tempExpRef.get(parameters={'bbox': bbox}) for tempExpRef in tempExpRefList]
819 else:
820 # Gen 2 API. Delete this when Gen 2 retired
821 tempExpList = [tempExpRef.get(tempExpName + "_sub", bbox=bbox, immediate=True)
822 for tempExpRef in tempExpRefList]
823 numCcds = sum(len(tempExp.getInfo().getCoaddInputs().ccds) for tempExp in tempExpList)
825 coaddExposure.setFilter(tempExpList[0].getFilter())
826 coaddInputs = coaddExposure.getInfo().getCoaddInputs()
827 coaddInputs.ccds.reserve(numCcds)
828 coaddInputs.visits.reserve(len(tempExpList))
830 for tempExp, weight in zip(tempExpList, weightList):
831 self.inputRecorder.addVisitToCoadd(coaddInputs, tempExp, weight)
833 if self.config.doUsePsfMatchedPolygons:
834 self.shrinkValidPolygons(coaddInputs)
836 coaddInputs.visits.sort()
837 if self.warpType == "psfMatched":
838 # The modelPsf BBox for a psfMatchedWarp/coaddTempExp was dynamically defined by
839 # ModelPsfMatchTask as the square box bounding its spatially-variable, pre-matched WarpedPsf.
840 # Likewise, set the PSF of a PSF-Matched Coadd to the modelPsf
841 # having the maximum width (sufficient because square)
842 modelPsfList = [tempExp.getPsf() for tempExp in tempExpList]
843 modelPsfWidthList = [modelPsf.computeBBox().getWidth() for modelPsf in modelPsfList]
844 psf = modelPsfList[modelPsfWidthList.index(max(modelPsfWidthList))]
845 else:
846 psf = measAlg.CoaddPsf(coaddInputs.ccds, coaddExposure.getWcs(),
847 self.config.coaddPsf.makeControl())
848 coaddExposure.setPsf(psf)
849 apCorrMap = measAlg.makeCoaddApCorrMap(coaddInputs.ccds, coaddExposure.getBBox(afwImage.PARENT),
850 coaddExposure.getWcs())
851 coaddExposure.getInfo().setApCorrMap(apCorrMap)
852 if self.config.doAttachTransmissionCurve:
853 transmissionCurve = measAlg.makeCoaddTransmissionCurve(coaddExposure.getWcs(), coaddInputs.ccds)
854 coaddExposure.getInfo().setTransmissionCurve(transmissionCurve)
856 def assembleSubregion(self, coaddExposure, bbox, tempExpRefList, imageScalerList, weightList,
857 altMaskList, statsFlags, statsCtrl, nImage=None):
858 """Assemble the coadd for a sub-region.
860 For each coaddTempExp, check for (and swap in) an alternative mask
861 if one is passed. Remove mask planes listed in
862 `config.removeMaskPlanes`. Finally, stack the actual exposures using
863 `lsst.afw.math.statisticsStack` with the statistic specified by
864 statsFlags. Typically, the statsFlag will be one of lsst.afw.math.MEAN for
865 a mean-stack or `lsst.afw.math.MEANCLIP` for outlier rejection using
866 an N-sigma clipped mean where N and iterations are specified by
867 statsCtrl. Assign the stacked subregion back to the coadd.
869 Parameters
870 ----------
871 coaddExposure : `lsst.afw.image.Exposure`
872 The target exposure for the coadd.
873 bbox : `lsst.geom.Box`
874 Sub-region to coadd.
875 tempExpRefList : `list`
876 List of data reference to tempExp.
877 imageScalerList : `list`
878 List of image scalers.
879 weightList : `list`
880 List of weights.
881 altMaskList : `list`
882 List of alternate masks to use rather than those stored with
883 tempExp, or None. Each element is dict with keys = mask plane
884 name to which to add the spans.
885 statsFlags : `lsst.afw.math.Property`
886 Property object for statistic for coadd.
887 statsCtrl : `lsst.afw.math.StatisticsControl`
888 Statistics control object for coadd.
889 nImage : `lsst.afw.image.ImageU`, optional
890 Keeps track of exposure count for each pixel.
891 """
892 self.log.debug("Computing coadd over %s", bbox)
893 tempExpName = self.getTempExpDatasetName(self.warpType)
894 coaddExposure.mask.addMaskPlane("REJECTED")
895 coaddExposure.mask.addMaskPlane("CLIPPED")
896 coaddExposure.mask.addMaskPlane("SENSOR_EDGE")
897 maskMap = self.setRejectedMaskMapping(statsCtrl)
898 clipped = afwImage.Mask.getPlaneBitMask("CLIPPED")
899 maskedImageList = []
900 if nImage is not None: 900 ↛ 901line 900 didn't jump to line 901, because the condition on line 900 was never true
901 subNImage = afwImage.ImageU(bbox.getWidth(), bbox.getHeight())
902 for tempExpRef, imageScaler, altMask in zip(tempExpRefList, imageScalerList, altMaskList):
904 if isinstance(tempExpRef, DeferredDatasetHandle):
905 # Gen 3 API
906 exposure = tempExpRef.get(parameters={'bbox': bbox})
907 else:
908 # Gen 2 API. Delete this when Gen 2 retired
909 exposure = tempExpRef.get(tempExpName + "_sub", bbox=bbox)
911 maskedImage = exposure.getMaskedImage()
912 mask = maskedImage.getMask()
913 if altMask is not None:
914 self.applyAltMaskPlanes(mask, altMask)
915 imageScaler.scaleMaskedImage(maskedImage)
917 # Add 1 for each pixel which is not excluded by the exclude mask.
918 # In legacyCoadd, pixels may also be excluded by afwMath.statisticsStack.
919 if nImage is not None: 919 ↛ 920line 919 didn't jump to line 920, because the condition on line 919 was never true
920 subNImage.getArray()[maskedImage.getMask().getArray() & statsCtrl.getAndMask() == 0] += 1
921 if self.config.removeMaskPlanes: 921 ↛ 923line 921 didn't jump to line 923, because the condition on line 921 was never false
922 self.removeMaskPlanes(maskedImage)
923 maskedImageList.append(maskedImage)
925 with self.timer("stack"):
926 coaddSubregion = afwMath.statisticsStack(maskedImageList, statsFlags, statsCtrl, weightList,
927 clipped, # also set output to CLIPPED if sigma-clipped
928 maskMap)
929 coaddExposure.maskedImage.assign(coaddSubregion, bbox)
930 if nImage is not None: 930 ↛ 931line 930 didn't jump to line 931, because the condition on line 930 was never true
931 nImage.assign(subNImage, bbox)
933 def removeMaskPlanes(self, maskedImage):
934 """Unset the mask of an image for mask planes specified in the config.
936 Parameters
937 ----------
938 maskedImage : `lsst.afw.image.MaskedImage`
939 The masked image to be modified.
940 """
941 mask = maskedImage.getMask()
942 for maskPlane in self.config.removeMaskPlanes:
943 try:
944 mask &= ~mask.getPlaneBitMask(maskPlane)
945 except pexExceptions.InvalidParameterError:
946 self.log.debug("Unable to remove mask plane %s: no mask plane with that name was found.",
947 maskPlane)
949 @staticmethod
950 def setRejectedMaskMapping(statsCtrl):
951 """Map certain mask planes of the warps to new planes for the coadd.
953 If a pixel is rejected due to a mask value other than EDGE, NO_DATA,
954 or CLIPPED, set it to REJECTED on the coadd.
955 If a pixel is rejected due to EDGE, set the coadd pixel to SENSOR_EDGE.
956 If a pixel is rejected due to CLIPPED, set the coadd pixel to CLIPPED.
958 Parameters
959 ----------
960 statsCtrl : `lsst.afw.math.StatisticsControl`
961 Statistics control object for coadd
963 Returns
964 -------
965 maskMap : `list` of `tuple` of `int`
966 A list of mappings of mask planes of the warped exposures to
967 mask planes of the coadd.
968 """
969 edge = afwImage.Mask.getPlaneBitMask("EDGE")
970 noData = afwImage.Mask.getPlaneBitMask("NO_DATA")
971 clipped = afwImage.Mask.getPlaneBitMask("CLIPPED")
972 toReject = statsCtrl.getAndMask() & (~noData) & (~edge) & (~clipped)
973 maskMap = [(toReject, afwImage.Mask.getPlaneBitMask("REJECTED")),
974 (edge, afwImage.Mask.getPlaneBitMask("SENSOR_EDGE")),
975 (clipped, clipped)]
976 return maskMap
978 def applyAltMaskPlanes(self, mask, altMaskSpans):
979 """Apply in place alt mask formatted as SpanSets to a mask.
981 Parameters
982 ----------
983 mask : `lsst.afw.image.Mask`
984 Original mask.
985 altMaskSpans : `dict`
986 SpanSet lists to apply. Each element contains the new mask
987 plane name (e.g. "CLIPPED and/or "NO_DATA") as the key,
988 and list of SpanSets to apply to the mask.
990 Returns
991 -------
992 mask : `lsst.afw.image.Mask`
993 Updated mask.
994 """
995 if self.config.doUsePsfMatchedPolygons:
996 if ("NO_DATA" in altMaskSpans) and ("NO_DATA" in self.config.badMaskPlanes): 996 ↛ 1004line 996 didn't jump to line 1004, because the condition on line 996 was never false
997 # Clear away any other masks outside the validPolygons. These pixels are no longer
998 # contributing to inexact PSFs, and will still be rejected because of NO_DATA
999 # self.config.doUsePsfMatchedPolygons should be True only in CompareWarpAssemble
1000 # This mask-clearing step must only occur *before* applying the new masks below
1001 for spanSet in altMaskSpans['NO_DATA']:
1002 spanSet.clippedTo(mask.getBBox()).clearMask(mask, self.getBadPixelMask())
1004 for plane, spanSetList in altMaskSpans.items():
1005 maskClipValue = mask.addMaskPlane(plane)
1006 for spanSet in spanSetList:
1007 spanSet.clippedTo(mask.getBBox()).setMask(mask, 2**maskClipValue)
1008 return mask
1010 def shrinkValidPolygons(self, coaddInputs):
1011 """Shrink coaddInputs' ccds' ValidPolygons in place.
1013 Either modify each ccd's validPolygon in place, or if CoaddInputs
1014 does not have a validPolygon, create one from its bbox.
1016 Parameters
1017 ----------
1018 coaddInputs : `lsst.afw.image.coaddInputs`
1019 Original mask.
1021 """
1022 for ccd in coaddInputs.ccds:
1023 polyOrig = ccd.getValidPolygon()
1024 validPolyBBox = polyOrig.getBBox() if polyOrig else ccd.getBBox()
1025 validPolyBBox.grow(-self.config.matchingKernelSize//2)
1026 if polyOrig:
1027 validPolygon = polyOrig.intersectionSingle(validPolyBBox)
1028 else:
1029 validPolygon = afwGeom.polygon.Polygon(geom.Box2D(validPolyBBox))
1030 ccd.setValidPolygon(validPolygon)
1032 def readBrightObjectMasks(self, dataRef):
1033 """Retrieve the bright object masks.
1035 Returns None on failure.
1037 Parameters
1038 ----------
1039 dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
1040 A Butler dataRef.
1042 Returns
1043 -------
1044 result : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
1045 Bright object mask from the Butler object, or None if it cannot
1046 be retrieved.
1047 """
1048 try:
1049 return dataRef.get(datasetType="brightObjectMask", immediate=True)
1050 except Exception as e:
1051 self.log.warn("Unable to read brightObjectMask for %s: %s", dataRef.dataId, e)
1052 return None
1054 def setBrightObjectMasks(self, exposure, brightObjectMasks, dataId=None):
1055 """Set the bright object masks.
1057 Parameters
1058 ----------
1059 exposure : `lsst.afw.image.Exposure`
1060 Exposure under consideration.
1061 dataId : `lsst.daf.persistence.dataId`
1062 Data identifier dict for patch.
1063 brightObjectMasks : `lsst.afw.table`
1064 Table of bright objects to mask.
1065 """
1067 if brightObjectMasks is None:
1068 self.log.warn("Unable to apply bright object mask: none supplied")
1069 return
1070 self.log.info("Applying %d bright object masks to %s", len(brightObjectMasks), dataId)
1071 mask = exposure.getMaskedImage().getMask()
1072 wcs = exposure.getWcs()
1073 plateScale = wcs.getPixelScale().asArcseconds()
1075 for rec in brightObjectMasks:
1076 center = geom.PointI(wcs.skyToPixel(rec.getCoord()))
1077 if rec["type"] == "box":
1078 assert rec["angle"] == 0.0, ("Angle != 0 for mask object %s" % rec["id"])
1079 width = rec["width"].asArcseconds()/plateScale # convert to pixels
1080 height = rec["height"].asArcseconds()/plateScale # convert to pixels
1082 halfSize = geom.ExtentI(0.5*width, 0.5*height)
1083 bbox = geom.Box2I(center - halfSize, center + halfSize)
1085 bbox = geom.BoxI(geom.PointI(int(center[0] - 0.5*width), int(center[1] - 0.5*height)),
1086 geom.PointI(int(center[0] + 0.5*width), int(center[1] + 0.5*height)))
1087 spans = afwGeom.SpanSet(bbox)
1088 elif rec["type"] == "circle":
1089 radius = int(rec["radius"].asArcseconds()/plateScale) # convert to pixels
1090 spans = afwGeom.SpanSet.fromShape(radius, offset=center)
1091 else:
1092 self.log.warn("Unexpected region type %s at %s" % rec["type"], center)
1093 continue
1094 spans.clippedTo(mask.getBBox()).setMask(mask, self.brightObjectBitmask)
1096 def setInexactPsf(self, mask):
1097 """Set INEXACT_PSF mask plane.
1099 If any of the input images isn't represented in the coadd (due to
1100 clipped pixels or chip gaps), the `CoaddPsf` will be inexact. Flag
1101 these pixels.
1103 Parameters
1104 ----------
1105 mask : `lsst.afw.image.Mask`
1106 Coadded exposure's mask, modified in-place.
1107 """
1108 mask.addMaskPlane("INEXACT_PSF")
1109 inexactPsf = mask.getPlaneBitMask("INEXACT_PSF")
1110 sensorEdge = mask.getPlaneBitMask("SENSOR_EDGE") # chip edges (so PSF is discontinuous)
1111 clipped = mask.getPlaneBitMask("CLIPPED") # pixels clipped from coadd
1112 rejected = mask.getPlaneBitMask("REJECTED") # pixels rejected from coadd due to masks
1113 array = mask.getArray()
1114 selected = array & (sensorEdge | clipped | rejected) > 0
1115 array[selected] |= inexactPsf
1117 @classmethod
1118 def _makeArgumentParser(cls):
1119 """Create an argument parser.
1120 """
1121 parser = pipeBase.ArgumentParser(name=cls._DefaultName)
1122 parser.add_id_argument("--id", cls.ConfigClass().coaddName + "Coadd_"
1123 + cls.ConfigClass().warpType + "Warp",
1124 help="data ID, e.g. --id tract=12345 patch=1,2",
1125 ContainerClass=AssembleCoaddDataIdContainer)
1126 parser.add_id_argument("--selectId", "calexp", help="data ID, e.g. --selectId visit=6789 ccd=0..9",
1127 ContainerClass=SelectDataIdContainer)
1128 return parser
1130 @staticmethod
1131 def _subBBoxIter(bbox, subregionSize):
1132 """Iterate over subregions of a bbox.
1134 Parameters
1135 ----------
1136 bbox : `lsst.geom.Box2I`
1137 Bounding box over which to iterate.
1138 subregionSize: `lsst.geom.Extent2I`
1139 Size of sub-bboxes.
1141 Yields
1142 ------
1143 subBBox : `lsst.geom.Box2I`
1144 Next sub-bounding box of size ``subregionSize`` or smaller; each ``subBBox``
1145 is contained within ``bbox``, so it may be smaller than ``subregionSize`` at
1146 the edges of ``bbox``, but it will never be empty.
1147 """
1148 if bbox.isEmpty(): 1148 ↛ 1149line 1148 didn't jump to line 1149, because the condition on line 1148 was never true
1149 raise RuntimeError("bbox %s is empty" % (bbox,))
1150 if subregionSize[0] < 1 or subregionSize[1] < 1: 1150 ↛ 1151line 1150 didn't jump to line 1151, because the condition on line 1150 was never true
1151 raise RuntimeError("subregionSize %s must be nonzero" % (subregionSize,))
1153 for rowShift in range(0, bbox.getHeight(), subregionSize[1]):
1154 for colShift in range(0, bbox.getWidth(), subregionSize[0]):
1155 subBBox = geom.Box2I(bbox.getMin() + geom.Extent2I(colShift, rowShift), subregionSize)
1156 subBBox.clip(bbox)
1157 if subBBox.isEmpty(): 1157 ↛ 1158line 1157 didn't jump to line 1158, because the condition on line 1157 was never true
1158 raise RuntimeError("Bug: empty bbox! bbox=%s, subregionSize=%s, "
1159 "colShift=%s, rowShift=%s" %
1160 (bbox, subregionSize, colShift, rowShift))
1161 yield subBBox
1164class AssembleCoaddDataIdContainer(pipeBase.DataIdContainer):
1165 """A version of `lsst.pipe.base.DataIdContainer` specialized for assembleCoadd.
1166 """
1168 def makeDataRefList(self, namespace):
1169 """Make self.refList from self.idList.
1171 Parameters
1172 ----------
1173 namespace
1174 Results of parsing command-line (with ``butler`` and ``log`` elements).
1175 """
1176 datasetType = namespace.config.coaddName + "Coadd"
1177 keysCoadd = namespace.butler.getKeys(datasetType=datasetType, level=self.level)
1179 for dataId in self.idList:
1180 # tract and patch are required
1181 for key in keysCoadd:
1182 if key not in dataId:
1183 raise RuntimeError("--id must include " + key)
1185 dataRef = namespace.butler.dataRef(
1186 datasetType=datasetType,
1187 dataId=dataId,
1188 )
1189 self.refList.append(dataRef)
1192def countMaskFromFootprint(mask, footprint, bitmask, ignoreMask):
1193 """Function to count the number of pixels with a specific mask in a
1194 footprint.
1196 Find the intersection of mask & footprint. Count all pixels in the mask
1197 that are in the intersection that have bitmask set but do not have
1198 ignoreMask set. Return the count.
1200 Parameters
1201 ----------
1202 mask : `lsst.afw.image.Mask`
1203 Mask to define intersection region by.
1204 footprint : `lsst.afw.detection.Footprint`
1205 Footprint to define the intersection region by.
1206 bitmask
1207 Specific mask that we wish to count the number of occurances of.
1208 ignoreMask
1209 Pixels to not consider.
1211 Returns
1212 -------
1213 result : `int`
1214 Count of number of pixels in footprint with specified mask.
1215 """
1216 bbox = footprint.getBBox()
1217 bbox.clip(mask.getBBox(afwImage.PARENT))
1218 fp = afwImage.Mask(bbox)
1219 subMask = mask.Factory(mask, bbox, afwImage.PARENT)
1220 footprint.spans.setMask(fp, bitmask)
1221 return numpy.logical_and((subMask.getArray() & fp.getArray()) > 0,
1222 (subMask.getArray() & ignoreMask) == 0).sum()
1225class SafeClipAssembleCoaddConfig(AssembleCoaddConfig, pipelineConnections=AssembleCoaddConnections):
1226 """Configuration parameters for the SafeClipAssembleCoaddTask.
1227 """
1228 assembleMeanCoadd = pexConfig.ConfigurableField(
1229 target=AssembleCoaddTask,
1230 doc="Task to assemble an initial Coadd using the MEAN statistic.",
1231 )
1232 assembleMeanClipCoadd = pexConfig.ConfigurableField(
1233 target=AssembleCoaddTask,
1234 doc="Task to assemble an initial Coadd using the MEANCLIP statistic.",
1235 )
1236 clipDetection = pexConfig.ConfigurableField(
1237 target=SourceDetectionTask,
1238 doc="Detect sources on difference between unclipped and clipped coadd")
1239 minClipFootOverlap = pexConfig.Field(
1240 doc="Minimum fractional overlap of clipped footprint with visit DETECTED to be clipped",
1241 dtype=float,
1242 default=0.6
1243 )
1244 minClipFootOverlapSingle = pexConfig.Field(
1245 doc="Minimum fractional overlap of clipped footprint with visit DETECTED to be "
1246 "clipped when only one visit overlaps",
1247 dtype=float,
1248 default=0.5
1249 )
1250 minClipFootOverlapDouble = pexConfig.Field(
1251 doc="Minimum fractional overlap of clipped footprints with visit DETECTED to be "
1252 "clipped when two visits overlap",
1253 dtype=float,
1254 default=0.45
1255 )
1256 maxClipFootOverlapDouble = pexConfig.Field(
1257 doc="Maximum fractional overlap of clipped footprints with visit DETECTED when "
1258 "considering two visits",
1259 dtype=float,
1260 default=0.15
1261 )
1262 minBigOverlap = pexConfig.Field(
1263 doc="Minimum number of pixels in footprint to use DETECTED mask from the single visits "
1264 "when labeling clipped footprints",
1265 dtype=int,
1266 default=100
1267 )
1269 def setDefaults(self):
1270 """Set default values for clipDetection.
1272 Notes
1273 -----
1274 The numeric values for these configuration parameters were
1275 empirically determined, future work may further refine them.
1276 """
1277 AssembleCoaddConfig.setDefaults(self)
1278 self.clipDetection.doTempLocalBackground = False
1279 self.clipDetection.reEstimateBackground = False
1280 self.clipDetection.returnOriginalFootprints = False
1281 self.clipDetection.thresholdPolarity = "both"
1282 self.clipDetection.thresholdValue = 2
1283 self.clipDetection.nSigmaToGrow = 2
1284 self.clipDetection.minPixels = 4
1285 self.clipDetection.isotropicGrow = True
1286 self.clipDetection.thresholdType = "pixel_stdev"
1287 self.sigmaClip = 1.5
1288 self.clipIter = 3
1289 self.statistic = "MEAN"
1290 self.assembleMeanCoadd.statistic = 'MEAN'
1291 self.assembleMeanClipCoadd.statistic = 'MEANCLIP'
1292 self.assembleMeanCoadd.doWrite = False
1293 self.assembleMeanClipCoadd.doWrite = False
1295 def validate(self):
1296 if self.doSigmaClip:
1297 log.warn("Additional Sigma-clipping not allowed in Safe-clipped Coadds. "
1298 "Ignoring doSigmaClip.")
1299 self.doSigmaClip = False
1300 if self.statistic != "MEAN":
1301 raise ValueError("Only MEAN statistic allowed for final stacking in SafeClipAssembleCoadd "
1302 "(%s chosen). Please set statistic to MEAN."
1303 % (self.statistic))
1304 AssembleCoaddTask.ConfigClass.validate(self)
1307class SafeClipAssembleCoaddTask(AssembleCoaddTask):
1308 """Assemble a coadded image from a set of coadded temporary exposures,
1309 being careful to clip & flag areas with potential artifacts.
1311 In ``AssembleCoaddTask``, we compute the coadd as an clipped mean (i.e.,
1312 we clip outliers). The problem with doing this is that when computing the
1313 coadd PSF at a given location, individual visit PSFs from visits with
1314 outlier pixels contribute to the coadd PSF and cannot be treated correctly.
1315 In this task, we correct for this behavior by creating a new
1316 ``badMaskPlane`` 'CLIPPED'. We populate this plane on the input
1317 coaddTempExps and the final coadd where
1319 i. difference imaging suggests that there is an outlier and
1320 ii. this outlier appears on only one or two images.
1322 Such regions will not contribute to the final coadd. Furthermore, any
1323 routine to determine the coadd PSF can now be cognizant of clipped regions.
1324 Note that the algorithm implemented by this task is preliminary and works
1325 correctly for HSC data. Parameter modifications and or considerable
1326 redesigning of the algorithm is likley required for other surveys.
1328 ``SafeClipAssembleCoaddTask`` uses a ``SourceDetectionTask``
1329 "clipDetection" subtask and also sub-classes ``AssembleCoaddTask``.
1330 You can retarget the ``SourceDetectionTask`` "clipDetection" subtask
1331 if you wish.
1333 Notes
1334 -----
1335 The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a
1336 flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``;
1337 see `baseDebug` for more about ``debug.py`` files.
1338 `SafeClipAssembleCoaddTask` has no debug variables of its own.
1339 The ``SourceDetectionTask`` "clipDetection" subtasks may support debug
1340 variables. See the documetation for `SourceDetectionTask` "clipDetection"
1341 for further information.
1343 Examples
1344 --------
1345 `SafeClipAssembleCoaddTask` assembles a set of warped ``coaddTempExp``
1346 images into a coadded image. The `SafeClipAssembleCoaddTask` is invoked by
1347 running assembleCoadd.py *without* the flag '--legacyCoadd'.
1349 Usage of ``assembleCoadd.py`` expects a data reference to the tract patch
1350 and filter to be coadded (specified using
1351 '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]')
1352 along with a list of coaddTempExps to attempt to coadd (specified using
1353 '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]').
1354 Only the coaddTempExps that cover the specified tract and patch will be
1355 coadded. A list of the available optional arguments can be obtained by
1356 calling assembleCoadd.py with the --help command line argument:
1358 .. code-block:: none
1360 assembleCoadd.py --help
1362 To demonstrate usage of the `SafeClipAssembleCoaddTask` in the larger
1363 context of multi-band processing, we will generate the HSC-I & -R band
1364 coadds from HSC engineering test data provided in the ci_hsc package.
1365 To begin, assuming that the lsst stack has been already set up, we must
1366 set up the obs_subaru and ci_hsc packages. This defines the environment
1367 variable $CI_HSC_DIR and points at the location of the package. The raw
1368 HSC data live in the ``$CI_HSC_DIR/raw`` directory. To begin assembling
1369 the coadds, we must first
1371 - ``processCcd``
1372 process the individual ccds in $CI_HSC_RAW to produce calibrated exposures
1373 - ``makeSkyMap``
1374 create a skymap that covers the area of the sky present in the raw exposures
1375 - ``makeCoaddTempExp``
1376 warp the individual calibrated exposures to the tangent plane of the coadd</DD>
1378 We can perform all of these steps by running
1380 .. code-block:: none
1382 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
1384 This will produce warped coaddTempExps for each visit. To coadd the
1385 warped data, we call ``assembleCoadd.py`` as follows:
1387 .. code-block:: none
1389 assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \
1390 --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \
1391 --selectId visit=903986 ccd=100--selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \
1392 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \
1393 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \
1394 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \
1395 --selectId visit=903988 ccd=24
1397 This will process the HSC-I band data. The results are written in
1398 ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``.
1400 You may also choose to run:
1402 .. code-block:: none
1404 scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346 nnn
1405 assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R --selectId visit=903334 ccd=16 \
1406 --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 --selectId visit=903334 ccd=100 \
1407 --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 --selectId visit=903338 ccd=18 \
1408 --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 --selectId visit=903342 ccd=10 \
1409 --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 --selectId visit=903344 ccd=5 \
1410 --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 --selectId visit=903346 ccd=6 \
1411 --selectId visit=903346 ccd=12
1413 to generate the coadd for the HSC-R band if you are interested in following
1414 multiBand Coadd processing as discussed in ``pipeTasks_multiBand``.
1415 """
1416 ConfigClass = SafeClipAssembleCoaddConfig
1417 _DefaultName = "safeClipAssembleCoadd"
1419 def __init__(self, *args, **kwargs):
1420 AssembleCoaddTask.__init__(self, *args, **kwargs)
1421 schema = afwTable.SourceTable.makeMinimalSchema()
1422 self.makeSubtask("clipDetection", schema=schema)
1423 self.makeSubtask("assembleMeanClipCoadd")
1424 self.makeSubtask("assembleMeanCoadd")
1426 @utils.inheritDoc(AssembleCoaddTask)
1427 @pipeBase.timeMethod
1428 def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, *args, **kwargs):
1429 """Assemble the coadd for a region.
1431 Compute the difference of coadds created with and without outlier
1432 rejection to identify coadd pixels that have outlier values in some
1433 individual visits.
1434 Detect clipped regions on the difference image and mark these regions
1435 on the one or two individual coaddTempExps where they occur if there
1436 is significant overlap between the clipped region and a source. This
1437 leaves us with a set of footprints from the difference image that have
1438 been identified as having occured on just one or two individual visits.
1439 However, these footprints were generated from a difference image. It
1440 is conceivable for a large diffuse source to have become broken up
1441 into multiple footprints acrosss the coadd difference in this process.
1442 Determine the clipped region from all overlapping footprints from the
1443 detected sources in each visit - these are big footprints.
1444 Combine the small and big clipped footprints and mark them on a new
1445 bad mask plane.
1446 Generate the coadd using `AssembleCoaddTask.run` without outlier
1447 removal. Clipped footprints will no longer make it into the coadd
1448 because they are marked in the new bad mask plane.
1450 Notes
1451 -----
1452 args and kwargs are passed but ignored in order to match the call
1453 signature expected by the parent task.
1454 """
1455 exp = self.buildDifferenceImage(skyInfo, tempExpRefList, imageScalerList, weightList)
1456 mask = exp.getMaskedImage().getMask()
1457 mask.addMaskPlane("CLIPPED")
1459 result = self.detectClip(exp, tempExpRefList)
1461 self.log.info('Found %d clipped objects', len(result.clipFootprints))
1463 maskClipValue = mask.getPlaneBitMask("CLIPPED")
1464 maskDetValue = mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE")
1465 # Append big footprints from individual Warps to result.clipSpans
1466 bigFootprints = self.detectClipBig(result.clipSpans, result.clipFootprints, result.clipIndices,
1467 result.detectionFootprints, maskClipValue, maskDetValue,
1468 exp.getBBox())
1469 # Create mask of the current clipped footprints
1470 maskClip = mask.Factory(mask.getBBox(afwImage.PARENT))
1471 afwDet.setMaskFromFootprintList(maskClip, result.clipFootprints, maskClipValue)
1473 maskClipBig = maskClip.Factory(mask.getBBox(afwImage.PARENT))
1474 afwDet.setMaskFromFootprintList(maskClipBig, bigFootprints, maskClipValue)
1475 maskClip |= maskClipBig
1477 # Assemble coadd from base class, but ignoring CLIPPED pixels
1478 badMaskPlanes = self.config.badMaskPlanes[:]
1479 badMaskPlanes.append("CLIPPED")
1480 badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes)
1481 return AssembleCoaddTask.run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
1482 result.clipSpans, mask=badPixelMask)
1484 def buildDifferenceImage(self, skyInfo, tempExpRefList, imageScalerList, weightList):
1485 """Return an exposure that contains the difference between unclipped
1486 and clipped coadds.
1488 Generate a difference image between clipped and unclipped coadds.
1489 Compute the difference image by subtracting an outlier-clipped coadd
1490 from an outlier-unclipped coadd. Return the difference image.
1492 Parameters
1493 ----------
1494 skyInfo : `lsst.pipe.base.Struct`
1495 Patch geometry information, from getSkyInfo
1496 tempExpRefList : `list`
1497 List of data reference to tempExp
1498 imageScalerList : `list`
1499 List of image scalers
1500 weightList : `list`
1501 List of weights
1503 Returns
1504 -------
1505 exp : `lsst.afw.image.Exposure`
1506 Difference image of unclipped and clipped coadd wrapped in an Exposure
1507 """
1508 coaddMean = self.assembleMeanCoadd.run(skyInfo, tempExpRefList,
1509 imageScalerList, weightList).coaddExposure
1511 coaddClip = self.assembleMeanClipCoadd.run(skyInfo, tempExpRefList,
1512 imageScalerList, weightList).coaddExposure
1514 coaddDiff = coaddMean.getMaskedImage().Factory(coaddMean.getMaskedImage())
1515 coaddDiff -= coaddClip.getMaskedImage()
1516 exp = afwImage.ExposureF(coaddDiff)
1517 exp.setPsf(coaddMean.getPsf())
1518 return exp
1520 def detectClip(self, exp, tempExpRefList):
1521 """Detect clipped regions on an exposure and set the mask on the
1522 individual tempExp masks.
1524 Detect footprints in the difference image after smoothing the
1525 difference image with a Gaussian kernal. Identify footprints that
1526 overlap with one or two input ``coaddTempExps`` by comparing the
1527 computed overlap fraction to thresholds set in the config. A different
1528 threshold is applied depending on the number of overlapping visits
1529 (restricted to one or two). If the overlap exceeds the thresholds,
1530 the footprint is considered "CLIPPED" and is marked as such on the
1531 coaddTempExp. Return a struct with the clipped footprints, the indices
1532 of the ``coaddTempExps`` that end up overlapping with the clipped
1533 footprints, and a list of new masks for the ``coaddTempExps``.
1535 Parameters
1536 ----------
1537 exp : `lsst.afw.image.Exposure`
1538 Exposure to run detection on.
1539 tempExpRefList : `list`
1540 List of data reference to tempExp.
1542 Returns
1543 -------
1544 result : `lsst.pipe.base.Struct`
1545 Result struct with components:
1547 - ``clipFootprints``: list of clipped footprints.
1548 - ``clipIndices``: indices for each ``clippedFootprint`` in
1549 ``tempExpRefList``.
1550 - ``clipSpans``: List of dictionaries containing spanSet lists
1551 to clip. Each element contains the new maskplane name
1552 ("CLIPPED") as the key and list of ``SpanSets`` as the value.
1553 - ``detectionFootprints``: List of DETECTED/DETECTED_NEGATIVE plane
1554 compressed into footprints.
1555 """
1556 mask = exp.getMaskedImage().getMask()
1557 maskDetValue = mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE")
1558 fpSet = self.clipDetection.detectFootprints(exp, doSmooth=True, clearMask=True)
1559 # Merge positive and negative together footprints together
1560 fpSet.positive.merge(fpSet.negative)
1561 footprints = fpSet.positive
1562 self.log.info('Found %d potential clipped objects', len(footprints.getFootprints()))
1563 ignoreMask = self.getBadPixelMask()
1565 clipFootprints = []
1566 clipIndices = []
1567 artifactSpanSets = [{'CLIPPED': list()} for _ in tempExpRefList]
1569 # for use by detectClipBig
1570 visitDetectionFootprints = []
1572 dims = [len(tempExpRefList), len(footprints.getFootprints())]
1573 overlapDetArr = numpy.zeros(dims, dtype=numpy.uint16)
1574 ignoreArr = numpy.zeros(dims, dtype=numpy.uint16)
1576 # Loop over masks once and extract/store only relevant overlap metrics and detection footprints
1577 for i, warpRef in enumerate(tempExpRefList):
1578 tmpExpMask = warpRef.get(datasetType=self.getTempExpDatasetName(self.warpType),
1579 immediate=True).getMaskedImage().getMask()
1580 maskVisitDet = tmpExpMask.Factory(tmpExpMask, tmpExpMask.getBBox(afwImage.PARENT),
1581 afwImage.PARENT, True)
1582 maskVisitDet &= maskDetValue
1583 visitFootprints = afwDet.FootprintSet(maskVisitDet, afwDet.Threshold(1))
1584 visitDetectionFootprints.append(visitFootprints)
1586 for j, footprint in enumerate(footprints.getFootprints()):
1587 ignoreArr[i, j] = countMaskFromFootprint(tmpExpMask, footprint, ignoreMask, 0x0)
1588 overlapDetArr[i, j] = countMaskFromFootprint(tmpExpMask, footprint, maskDetValue, ignoreMask)
1590 # build a list of clipped spans for each visit
1591 for j, footprint in enumerate(footprints.getFootprints()):
1592 nPixel = footprint.getArea()
1593 overlap = [] # hold the overlap with each visit
1594 indexList = [] # index of visit in global list
1595 for i in range(len(tempExpRefList)):
1596 ignore = ignoreArr[i, j]
1597 overlapDet = overlapDetArr[i, j]
1598 totPixel = nPixel - ignore
1600 # If we have more bad pixels than detection skip
1601 if ignore > overlapDet or totPixel <= 0.5*nPixel or overlapDet == 0: 1601 ↛ 1603line 1601 didn't jump to line 1603, because the condition on line 1601 was never false
1602 continue
1603 overlap.append(overlapDet/float(totPixel))
1604 indexList.append(i)
1606 overlap = numpy.array(overlap)
1607 if not len(overlap): 1607 ↛ 1610line 1607 didn't jump to line 1610, because the condition on line 1607 was never false
1608 continue
1610 keep = False # Should this footprint be marked as clipped?
1611 keepIndex = [] # Which tempExps does the clipped footprint belong to
1613 # If footprint only has one overlap use a lower threshold
1614 if len(overlap) == 1:
1615 if overlap[0] > self.config.minClipFootOverlapSingle:
1616 keep = True
1617 keepIndex = [0]
1618 else:
1619 # This is the general case where only visit should be clipped
1620 clipIndex = numpy.where(overlap > self.config.minClipFootOverlap)[0]
1621 if len(clipIndex) == 1:
1622 keep = True
1623 keepIndex = [clipIndex[0]]
1625 # Test if there are clipped objects that overlap two different visits
1626 clipIndex = numpy.where(overlap > self.config.minClipFootOverlapDouble)[0]
1627 if len(clipIndex) == 2 and len(overlap) > 3:
1628 clipIndexComp = numpy.where(overlap <= self.config.minClipFootOverlapDouble)[0]
1629 if numpy.max(overlap[clipIndexComp]) <= self.config.maxClipFootOverlapDouble:
1630 keep = True
1631 keepIndex = clipIndex
1633 if not keep:
1634 continue
1636 for index in keepIndex:
1637 globalIndex = indexList[index]
1638 artifactSpanSets[globalIndex]['CLIPPED'].append(footprint.spans)
1640 clipIndices.append(numpy.array(indexList)[keepIndex])
1641 clipFootprints.append(footprint)
1643 return pipeBase.Struct(clipFootprints=clipFootprints, clipIndices=clipIndices,
1644 clipSpans=artifactSpanSets, detectionFootprints=visitDetectionFootprints)
1646 def detectClipBig(self, clipList, clipFootprints, clipIndices, detectionFootprints,
1647 maskClipValue, maskDetValue, coaddBBox):
1648 """Return individual warp footprints for large artifacts and append
1649 them to ``clipList`` in place.
1651 Identify big footprints composed of many sources in the coadd
1652 difference that may have originated in a large diffuse source in the
1653 coadd. We do this by indentifying all clipped footprints that overlap
1654 significantly with each source in all the coaddTempExps.
1656 Parameters
1657 ----------
1658 clipList : `list`
1659 List of alt mask SpanSets with clipping information. Modified.
1660 clipFootprints : `list`
1661 List of clipped footprints.
1662 clipIndices : `list`
1663 List of which entries in tempExpClipList each footprint belongs to.
1664 maskClipValue
1665 Mask value of clipped pixels.
1666 maskDetValue
1667 Mask value of detected pixels.
1668 coaddBBox : `lsst.geom.Box`
1669 BBox of the coadd and warps.
1671 Returns
1672 -------
1673 bigFootprintsCoadd : `list`
1674 List of big footprints
1675 """
1676 bigFootprintsCoadd = []
1677 ignoreMask = self.getBadPixelMask()
1678 for index, (clippedSpans, visitFootprints) in enumerate(zip(clipList, detectionFootprints)):
1679 maskVisitDet = afwImage.MaskX(coaddBBox, 0x0)
1680 for footprint in visitFootprints.getFootprints():
1681 footprint.spans.setMask(maskVisitDet, maskDetValue)
1683 # build a mask of clipped footprints that are in this visit
1684 clippedFootprintsVisit = []
1685 for foot, clipIndex in zip(clipFootprints, clipIndices): 1685 ↛ 1686line 1685 didn't jump to line 1686, because the loop on line 1685 never started
1686 if index not in clipIndex:
1687 continue
1688 clippedFootprintsVisit.append(foot)
1689 maskVisitClip = maskVisitDet.Factory(maskVisitDet.getBBox(afwImage.PARENT))
1690 afwDet.setMaskFromFootprintList(maskVisitClip, clippedFootprintsVisit, maskClipValue)
1692 bigFootprintsVisit = []
1693 for foot in visitFootprints.getFootprints():
1694 if foot.getArea() < self.config.minBigOverlap:
1695 continue
1696 nCount = countMaskFromFootprint(maskVisitClip, foot, maskClipValue, ignoreMask)
1697 if nCount > self.config.minBigOverlap: 1697 ↛ 1698line 1697 didn't jump to line 1698, because the condition on line 1697 was never true
1698 bigFootprintsVisit.append(foot)
1699 bigFootprintsCoadd.append(foot)
1701 for footprint in bigFootprintsVisit: 1701 ↛ 1702line 1701 didn't jump to line 1702, because the loop on line 1701 never started
1702 clippedSpans["CLIPPED"].append(footprint.spans)
1704 return bigFootprintsCoadd
1707class CompareWarpAssembleCoaddConnections(AssembleCoaddConnections):
1708 psfMatchedWarps = pipeBase.connectionTypes.Input(
1709 doc=("PSF-Matched Warps are required by CompareWarp regardless of the coadd type requested. "
1710 "Only PSF-Matched Warps make sense for image subtraction. "
1711 "Therefore, they must be an additional declared input."),
1712 name="{inputCoaddName}Coadd_psfMatchedWarp",
1713 storageClass="ExposureF",
1714 dimensions=("tract", "patch", "skymap", "visit"),
1715 deferLoad=True,
1716 multiple=True
1717 )
1718 templateCoadd = pipeBase.connectionTypes.Output(
1719 doc=("Model of the static sky, used to find temporal artifacts. Typically a PSF-Matched, "
1720 "sigma-clipped coadd. Written if and only if assembleStaticSkyModel.doWrite=True"),
1721 name="{fakesType}{outputCoaddName}CoaddPsfMatched",
1722 storageClass="ExposureF",
1723 dimensions=("tract", "patch", "skymap", "abstract_filter"),
1724 )
1726 def __init__(self, *, config=None):
1727 super().__init__(config=config)
1728 if not config.assembleStaticSkyModel.doWrite:
1729 self.outputs.remove("templateCoadd")
1730 config.validate()
1733class CompareWarpAssembleCoaddConfig(AssembleCoaddConfig,
1734 pipelineConnections=CompareWarpAssembleCoaddConnections):
1735 assembleStaticSkyModel = pexConfig.ConfigurableField(
1736 target=AssembleCoaddTask,
1737 doc="Task to assemble an artifact-free, PSF-matched Coadd to serve as a"
1738 " naive/first-iteration model of the static sky.",
1739 )
1740 detect = pexConfig.ConfigurableField(
1741 target=SourceDetectionTask,
1742 doc="Detect outlier sources on difference between each psfMatched warp and static sky model"
1743 )
1744 detectTemplate = pexConfig.ConfigurableField(
1745 target=SourceDetectionTask,
1746 doc="Detect sources on static sky model. Only used if doPreserveContainedBySource is True"
1747 )
1748 maskStreaks = pexConfig.ConfigurableField(
1749 target=MaskStreaksTask,
1750 doc="Detect streaks on difference between each psfMatched warp and static sky model. Only used if "
1751 "doFilterMorphological is True. Adds a mask plane to an exposure, with the mask plane name set by"
1752 "streakMaskName"
1753 )
1754 streakMaskName = pexConfig.Field(
1755 dtype=str,
1756 default="STREAK",
1757 doc="Name of mask bit used for streaks"
1758 )
1759 maxNumEpochs = pexConfig.Field(
1760 doc="Charactistic maximum local number of epochs/visits in which an artifact candidate can appear "
1761 "and still be masked. The effective maxNumEpochs is a broken linear function of local "
1762 "number of epochs (N): min(maxFractionEpochsLow*N, maxNumEpochs + maxFractionEpochsHigh*N). "
1763 "For each footprint detected on the image difference between the psfMatched warp and static sky "
1764 "model, if a significant fraction of pixels (defined by spatialThreshold) are residuals in more "
1765 "than the computed effective maxNumEpochs, the artifact candidate is deemed persistant rather "
1766 "than transient and not masked.",
1767 dtype=int,
1768 default=2
1769 )
1770 maxFractionEpochsLow = pexConfig.RangeField(
1771 doc="Fraction of local number of epochs (N) to use as effective maxNumEpochs for low N. "
1772 "Effective maxNumEpochs = "
1773 "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)",
1774 dtype=float,
1775 default=0.4,
1776 min=0., max=1.,
1777 )
1778 maxFractionEpochsHigh = pexConfig.RangeField(
1779 doc="Fraction of local number of epochs (N) to use as effective maxNumEpochs for high N. "
1780 "Effective maxNumEpochs = "
1781 "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)",
1782 dtype=float,
1783 default=0.03,
1784 min=0., max=1.,
1785 )
1786 spatialThreshold = pexConfig.RangeField(
1787 doc="Unitless fraction of pixels defining how much of the outlier region has to meet the "
1788 "temporal criteria. If 0, clip all. If 1, clip none.",
1789 dtype=float,
1790 default=0.5,
1791 min=0., max=1.,
1792 inclusiveMin=True, inclusiveMax=True
1793 )
1794 doScaleWarpVariance = pexConfig.Field(
1795 doc="Rescale Warp variance plane using empirical noise?",
1796 dtype=bool,
1797 default=True,
1798 )
1799 scaleWarpVariance = pexConfig.ConfigurableField(
1800 target=ScaleVarianceTask,
1801 doc="Rescale variance on warps",
1802 )
1803 doPreserveContainedBySource = pexConfig.Field(
1804 doc="Rescue artifacts from clipping that completely lie within a footprint detected"
1805 "on the PsfMatched Template Coadd. Replicates a behavior of SafeClip.",
1806 dtype=bool,
1807 default=True,
1808 )
1809 doPrefilterArtifacts = pexConfig.Field(
1810 doc="Ignore artifact candidates that are mostly covered by the bad pixel mask, "
1811 "because they will be excluded anyway. This prevents them from contributing "
1812 "to the outlier epoch count image and potentially being labeled as persistant."
1813 "'Mostly' is defined by the config 'prefilterArtifactsRatio'.",
1814 dtype=bool,
1815 default=True
1816 )
1817 prefilterArtifactsMaskPlanes = pexConfig.ListField(
1818 doc="Prefilter artifact candidates that are mostly covered by these bad mask planes.",
1819 dtype=str,
1820 default=('NO_DATA', 'BAD', 'SAT', 'SUSPECT'),
1821 )
1822 prefilterArtifactsRatio = pexConfig.Field(
1823 doc="Prefilter artifact candidates with less than this fraction overlapping good pixels",
1824 dtype=float,
1825 default=0.05
1826 )
1827 doFilterMorphological = pexConfig.Field(
1828 doc="Filter artifact candidates based on morphological criteria, i.g. those that appear to "
1829 "be streaks.",
1830 dtype=bool,
1831 default=False
1832 )
1834 def setDefaults(self):
1835 AssembleCoaddConfig.setDefaults(self)
1836 self.statistic = 'MEAN'
1837 self.doUsePsfMatchedPolygons = True
1839 # Real EDGE removed by psfMatched NO_DATA border half the width of the matching kernel
1840 # CompareWarp applies psfMatched EDGE pixels to directWarps before assembling
1841 if "EDGE" in self.badMaskPlanes: 1841 ↛ 1843line 1841 didn't jump to line 1843, because the condition on line 1841 was never false
1842 self.badMaskPlanes.remove('EDGE')
1843 self.removeMaskPlanes.append('EDGE')
1844 self.assembleStaticSkyModel.badMaskPlanes = ["NO_DATA", ]
1845 self.assembleStaticSkyModel.warpType = 'psfMatched'
1846 self.assembleStaticSkyModel.connections.warpType = 'psfMatched'
1847 self.assembleStaticSkyModel.statistic = 'MEANCLIP'
1848 self.assembleStaticSkyModel.sigmaClip = 2.5
1849 self.assembleStaticSkyModel.clipIter = 3
1850 self.assembleStaticSkyModel.calcErrorFromInputVariance = False
1851 self.assembleStaticSkyModel.doWrite = False
1852 self.detect.doTempLocalBackground = False
1853 self.detect.reEstimateBackground = False
1854 self.detect.returnOriginalFootprints = False
1855 self.detect.thresholdPolarity = "both"
1856 self.detect.thresholdValue = 5
1857 self.detect.minPixels = 4
1858 self.detect.isotropicGrow = True
1859 self.detect.thresholdType = "pixel_stdev"
1860 self.detect.nSigmaToGrow = 0.4
1861 # The default nSigmaToGrow for SourceDetectionTask is already 2.4,
1862 # Explicitly restating because ratio with detect.nSigmaToGrow matters
1863 self.detectTemplate.nSigmaToGrow = 2.4
1864 self.detectTemplate.doTempLocalBackground = False
1865 self.detectTemplate.reEstimateBackground = False
1866 self.detectTemplate.returnOriginalFootprints = False
1868 def validate(self):
1869 super().validate()
1870 if self.assembleStaticSkyModel.doNImage:
1871 raise ValueError("No dataset type exists for a PSF-Matched Template N Image."
1872 "Please set assembleStaticSkyModel.doNImage=False")
1874 if self.assembleStaticSkyModel.doWrite and (self.warpType == self.assembleStaticSkyModel.warpType):
1875 raise ValueError("warpType (%s) == assembleStaticSkyModel.warpType (%s) and will compete for "
1876 "the same dataset name. Please set assembleStaticSkyModel.doWrite to False "
1877 "or warpType to 'direct'. assembleStaticSkyModel.warpType should ways be "
1878 "'PsfMatched'" % (self.warpType, self.assembleStaticSkyModel.warpType))
1881class CompareWarpAssembleCoaddTask(AssembleCoaddTask):
1882 """Assemble a compareWarp coadded image from a set of warps
1883 by masking artifacts detected by comparing PSF-matched warps.
1885 In ``AssembleCoaddTask``, we compute the coadd as an clipped mean (i.e.,
1886 we clip outliers). The problem with doing this is that when computing the
1887 coadd PSF at a given location, individual visit PSFs from visits with
1888 outlier pixels contribute to the coadd PSF and cannot be treated correctly.
1889 In this task, we correct for this behavior by creating a new badMaskPlane
1890 'CLIPPED' which marks pixels in the individual warps suspected to contain
1891 an artifact. We populate this plane on the input warps by comparing
1892 PSF-matched warps with a PSF-matched median coadd which serves as a
1893 model of the static sky. Any group of pixels that deviates from the
1894 PSF-matched template coadd by more than config.detect.threshold sigma,
1895 is an artifact candidate. The candidates are then filtered to remove
1896 variable sources and sources that are difficult to subtract such as
1897 bright stars. This filter is configured using the config parameters
1898 ``temporalThreshold`` and ``spatialThreshold``. The temporalThreshold is
1899 the maximum fraction of epochs that the deviation can appear in and still
1900 be considered an artifact. The spatialThreshold is the maximum fraction of
1901 pixels in the footprint of the deviation that appear in other epochs
1902 (where other epochs is defined by the temporalThreshold). If the deviant
1903 region meets this criteria of having a significant percentage of pixels
1904 that deviate in only a few epochs, these pixels have the 'CLIPPED' bit
1905 set in the mask. These regions will not contribute to the final coadd.
1906 Furthermore, any routine to determine the coadd PSF can now be cognizant
1907 of clipped regions. Note that the algorithm implemented by this task is
1908 preliminary and works correctly for HSC data. Parameter modifications and
1909 or considerable redesigning of the algorithm is likley required for other
1910 surveys.
1912 ``CompareWarpAssembleCoaddTask`` sub-classes
1913 ``AssembleCoaddTask`` and instantiates ``AssembleCoaddTask``
1914 as a subtask to generate the TemplateCoadd (the model of the static sky).
1916 Notes
1917 -----
1918 The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a
1919 flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``; see
1920 ``baseDebug`` for more about ``debug.py`` files.
1922 This task supports the following debug variables:
1924 - ``saveCountIm``
1925 If True then save the Epoch Count Image as a fits file in the `figPath`
1926 - ``figPath``
1927 Path to save the debug fits images and figures
1929 For example, put something like:
1931 .. code-block:: python
1933 import lsstDebug
1934 def DebugInfo(name):
1935 di = lsstDebug.getInfo(name)
1936 if name == "lsst.pipe.tasks.assembleCoadd":
1937 di.saveCountIm = True
1938 di.figPath = "/desired/path/to/debugging/output/images"
1939 return di
1940 lsstDebug.Info = DebugInfo
1942 into your ``debug.py`` file and run ``assemebleCoadd.py`` with the
1943 ``--debug`` flag. Some subtasks may have their own debug variables;
1944 see individual Task documentation.
1946 Examples
1947 --------
1948 ``CompareWarpAssembleCoaddTask`` assembles a set of warped images into a
1949 coadded image. The ``CompareWarpAssembleCoaddTask`` is invoked by running
1950 ``assembleCoadd.py`` with the flag ``--compareWarpCoadd``.
1951 Usage of ``assembleCoadd.py`` expects a data reference to the tract patch
1952 and filter to be coadded (specified using
1953 '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]')
1954 along with a list of coaddTempExps to attempt to coadd (specified using
1955 '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]').
1956 Only the warps that cover the specified tract and patch will be coadded.
1957 A list of the available optional arguments can be obtained by calling
1958 ``assembleCoadd.py`` with the ``--help`` command line argument:
1960 .. code-block:: none
1962 assembleCoadd.py --help
1964 To demonstrate usage of the ``CompareWarpAssembleCoaddTask`` in the larger
1965 context of multi-band processing, we will generate the HSC-I & -R band
1966 oadds from HSC engineering test data provided in the ``ci_hsc`` package.
1967 To begin, assuming that the lsst stack has been already set up, we must
1968 set up the ``obs_subaru`` and ``ci_hsc`` packages.
1969 This defines the environment variable ``$CI_HSC_DIR`` and points at the
1970 location of the package. The raw HSC data live in the ``$CI_HSC_DIR/raw``
1971 directory. To begin assembling the coadds, we must first
1973 - processCcd
1974 process the individual ccds in $CI_HSC_RAW to produce calibrated exposures
1975 - makeSkyMap
1976 create a skymap that covers the area of the sky present in the raw exposures
1977 - makeCoaddTempExp
1978 warp the individual calibrated exposures to the tangent plane of the coadd
1980 We can perform all of these steps by running
1982 .. code-block:: none
1984 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
1986 This will produce warped ``coaddTempExps`` for each visit. To coadd the
1987 warped data, we call ``assembleCoadd.py`` as follows:
1989 .. code-block:: none
1991 assembleCoadd.py --compareWarpCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \
1992 --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \
1993 --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \
1994 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \
1995 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \
1996 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \
1997 --selectId visit=903988 ccd=24
1999 This will process the HSC-I band data. The results are written in
2000 ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``.
2001 """
2002 ConfigClass = CompareWarpAssembleCoaddConfig
2003 _DefaultName = "compareWarpAssembleCoadd"
2005 def __init__(self, *args, **kwargs):
2006 AssembleCoaddTask.__init__(self, *args, **kwargs)
2007 self.makeSubtask("assembleStaticSkyModel")
2008 detectionSchema = afwTable.SourceTable.makeMinimalSchema()
2009 self.makeSubtask("detect", schema=detectionSchema)
2010 if self.config.doPreserveContainedBySource: 2010 ↛ 2012line 2010 didn't jump to line 2012, because the condition on line 2010 was never false
2011 self.makeSubtask("detectTemplate", schema=afwTable.SourceTable.makeMinimalSchema())
2012 if self.config.doScaleWarpVariance: 2012 ↛ 2014line 2012 didn't jump to line 2014, because the condition on line 2012 was never false
2013 self.makeSubtask("scaleWarpVariance")
2014 if self.config.doFilterMorphological: 2014 ↛ 2015line 2014 didn't jump to line 2015, because the condition on line 2014 was never true
2015 self.makeSubtask("maskStreaks")
2017 @utils.inheritDoc(AssembleCoaddTask)
2018 def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs):
2019 """
2020 Generate a templateCoadd to use as a naive model of static sky to
2021 subtract from PSF-Matched warps.
2023 Returns
2024 -------
2025 result : `lsst.pipe.base.Struct`
2026 Result struct with components:
2028 - ``templateCoadd`` : coadded exposure (``lsst.afw.image.Exposure``)
2029 - ``nImage`` : N Image (``lsst.afw.image.Image``)
2030 """
2031 # Ensure that psfMatchedWarps are used as input warps for template generation
2032 staticSkyModelInputRefs = copy.deepcopy(inputRefs)
2033 staticSkyModelInputRefs.inputWarps = inputRefs.psfMatchedWarps
2035 # Because subtasks don't have connections we have to make one.
2036 # The main task's `templateCoadd` is the subtask's `coaddExposure`
2037 staticSkyModelOutputRefs = copy.deepcopy(outputRefs)
2038 if self.config.assembleStaticSkyModel.doWrite:
2039 staticSkyModelOutputRefs.coaddExposure = staticSkyModelOutputRefs.templateCoadd
2040 # Remove template coadd from both subtask's and main tasks outputs,
2041 # because it is handled by the subtask as `coaddExposure`
2042 del outputRefs.templateCoadd
2043 del staticSkyModelOutputRefs.templateCoadd
2045 # A PSF-Matched nImage does not exist as a dataset type
2046 if 'nImage' in staticSkyModelOutputRefs.keys():
2047 del staticSkyModelOutputRefs.nImage
2049 templateCoadd = self.assembleStaticSkyModel.runQuantum(butlerQC, staticSkyModelInputRefs,
2050 staticSkyModelOutputRefs)
2051 if templateCoadd is None:
2052 raise RuntimeError(self._noTemplateMessage(self.assembleStaticSkyModel.warpType))
2054 return pipeBase.Struct(templateCoadd=templateCoadd.coaddExposure,
2055 nImage=templateCoadd.nImage,
2056 warpRefList=templateCoadd.warpRefList,
2057 imageScalerList=templateCoadd.imageScalerList,
2058 weightList=templateCoadd.weightList)
2060 @utils.inheritDoc(AssembleCoaddTask)
2061 def makeSupplementaryData(self, dataRef, selectDataList=None, warpRefList=None):
2062 """
2063 Generate a templateCoadd to use as a naive model of static sky to
2064 subtract from PSF-Matched warps.
2066 Returns
2067 -------
2068 result : `lsst.pipe.base.Struct`
2069 Result struct with components:
2071 - ``templateCoadd``: coadded exposure (``lsst.afw.image.Exposure``)
2072 - ``nImage``: N Image (``lsst.afw.image.Image``)
2073 """
2074 templateCoadd = self.assembleStaticSkyModel.runDataRef(dataRef, selectDataList, warpRefList)
2075 if templateCoadd is None: 2075 ↛ 2076line 2075 didn't jump to line 2076, because the condition on line 2075 was never true
2076 raise RuntimeError(self._noTemplateMessage(self.assembleStaticSkyModel.warpType))
2078 return pipeBase.Struct(templateCoadd=templateCoadd.coaddExposure,
2079 nImage=templateCoadd.nImage,
2080 warpRefList=templateCoadd.warpRefList,
2081 imageScalerList=templateCoadd.imageScalerList,
2082 weightList=templateCoadd.weightList)
2084 def _noTemplateMessage(self, warpType):
2085 warpName = (warpType[0].upper() + warpType[1:])
2086 message = """No %(warpName)s warps were found to build the template coadd which is
2087 required to run CompareWarpAssembleCoaddTask. To continue assembling this type of coadd,
2088 first either rerun makeCoaddTempExp with config.make%(warpName)s=True or
2089 coaddDriver with config.makeCoadTempExp.make%(warpName)s=True, before assembleCoadd.
2091 Alternatively, to use another algorithm with existing warps, retarget the CoaddDriverConfig to
2092 another algorithm like:
2094 from lsst.pipe.tasks.assembleCoadd import SafeClipAssembleCoaddTask
2095 config.assemble.retarget(SafeClipAssembleCoaddTask)
2096 """ % {"warpName": warpName}
2097 return message
2099 @utils.inheritDoc(AssembleCoaddTask)
2100 @pipeBase.timeMethod
2101 def run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
2102 supplementaryData, *args, **kwargs):
2103 """Assemble the coadd.
2105 Find artifacts and apply them to the warps' masks creating a list of
2106 alternative masks with a new "CLIPPED" plane and updated "NO_DATA"
2107 plane. Then pass these alternative masks to the base class's `run`
2108 method.
2110 The input parameters ``supplementaryData`` is a `lsst.pipe.base.Struct`
2111 that must contain a ``templateCoadd`` that serves as the
2112 model of the static sky.
2113 """
2115 # Check and match the order of the supplementaryData
2116 # (PSF-matched) inputs to the order of the direct inputs,
2117 # so that the artifact mask is applied to the right warp
2118 dataIds = [ref.dataId for ref in tempExpRefList]
2119 psfMatchedDataIds = [ref.dataId for ref in supplementaryData.warpRefList]
2121 if dataIds != psfMatchedDataIds:
2122 self.log.info("Reordering and or/padding PSF-matched visit input list")
2123 supplementaryData.warpRefList = reorderAndPadList(supplementaryData.warpRefList,
2124 psfMatchedDataIds, dataIds)
2125 supplementaryData.imageScalerList = reorderAndPadList(supplementaryData.imageScalerList,
2126 psfMatchedDataIds, dataIds)
2128 # Use PSF-Matched Warps (and corresponding scalers) and coadd to find artifacts
2129 spanSetMaskList = self.findArtifacts(supplementaryData.templateCoadd,
2130 supplementaryData.warpRefList,
2131 supplementaryData.imageScalerList)
2133 badMaskPlanes = self.config.badMaskPlanes[:]
2134 badMaskPlanes.append("CLIPPED")
2135 badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes)
2137 result = AssembleCoaddTask.run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
2138 spanSetMaskList, mask=badPixelMask)
2140 # Propagate PSF-matched EDGE pixels to coadd SENSOR_EDGE and INEXACT_PSF
2141 # Psf-Matching moves the real edge inwards
2142 self.applyAltEdgeMask(result.coaddExposure.maskedImage.mask, spanSetMaskList)
2143 return result
2145 def applyAltEdgeMask(self, mask, altMaskList):
2146 """Propagate alt EDGE mask to SENSOR_EDGE AND INEXACT_PSF planes.
2148 Parameters
2149 ----------
2150 mask : `lsst.afw.image.Mask`
2151 Original mask.
2152 altMaskList : `list`
2153 List of Dicts containing ``spanSet`` lists.
2154 Each element contains the new mask plane name (e.g. "CLIPPED
2155 and/or "NO_DATA") as the key, and list of ``SpanSets`` to apply to
2156 the mask.
2157 """
2158 maskValue = mask.getPlaneBitMask(["SENSOR_EDGE", "INEXACT_PSF"])
2159 for visitMask in altMaskList:
2160 if "EDGE" in visitMask: 2160 ↛ 2159line 2160 didn't jump to line 2159, because the condition on line 2160 was never false
2161 for spanSet in visitMask['EDGE']:
2162 spanSet.clippedTo(mask.getBBox()).setMask(mask, maskValue)
2164 def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList):
2165 """Find artifacts.
2167 Loop through warps twice. The first loop builds a map with the count
2168 of how many epochs each pixel deviates from the templateCoadd by more
2169 than ``config.chiThreshold`` sigma. The second loop takes each
2170 difference image and filters the artifacts detected in each using
2171 count map to filter out variable sources and sources that are
2172 difficult to subtract cleanly.
2174 Parameters
2175 ----------
2176 templateCoadd : `lsst.afw.image.Exposure`
2177 Exposure to serve as model of static sky.
2178 tempExpRefList : `list`
2179 List of data references to warps.
2180 imageScalerList : `list`
2181 List of image scalers.
2183 Returns
2184 -------
2185 altMasks : `list`
2186 List of dicts containing information about CLIPPED
2187 (i.e., artifacts), NO_DATA, and EDGE pixels.
2188 """
2190 self.log.debug("Generating Count Image, and mask lists.")
2191 coaddBBox = templateCoadd.getBBox()
2192 slateIm = afwImage.ImageU(coaddBBox)
2193 epochCountImage = afwImage.ImageU(coaddBBox)
2194 nImage = afwImage.ImageU(coaddBBox)
2195 spanSetArtifactList = []
2196 spanSetNoDataMaskList = []
2197 spanSetEdgeList = []
2198 spanSetBadMorphoList = []
2199 badPixelMask = self.getBadPixelMask()
2201 # mask of the warp diffs should = that of only the warp
2202 templateCoadd.mask.clearAllMaskPlanes()
2204 if self.config.doPreserveContainedBySource: 2204 ↛ 2207line 2204 didn't jump to line 2207, because the condition on line 2204 was never false
2205 templateFootprints = self.detectTemplate.detectFootprints(templateCoadd)
2206 else:
2207 templateFootprints = None
2209 for warpRef, imageScaler in zip(tempExpRefList, imageScalerList):
2210 warpDiffExp = self._readAndComputeWarpDiff(warpRef, imageScaler, templateCoadd)
2211 if warpDiffExp is not None:
2212 # This nImage only approximates the final nImage because it uses the PSF-matched mask
2213 nImage.array += (numpy.isfinite(warpDiffExp.image.array)
2214 * ((warpDiffExp.mask.array & badPixelMask) == 0)).astype(numpy.uint16)
2215 fpSet = self.detect.detectFootprints(warpDiffExp, doSmooth=False, clearMask=True)
2216 fpSet.positive.merge(fpSet.negative)
2217 footprints = fpSet.positive
2218 slateIm.set(0)
2219 spanSetList = [footprint.spans for footprint in footprints.getFootprints()]
2221 # Remove artifacts due to defects before they contribute to the epochCountImage
2222 if self.config.doPrefilterArtifacts: 2222 ↛ 2226line 2222 didn't jump to line 2226, because the condition on line 2222 was never false
2223 spanSetList = self.prefilterArtifacts(spanSetList, warpDiffExp)
2225 # Clear mask before adding prefiltered spanSets
2226 self.detect.clearMask(warpDiffExp.mask)
2227 for spans in spanSetList:
2228 spans.setImage(slateIm, 1, doClip=True)
2229 spans.setMask(warpDiffExp.mask, warpDiffExp.mask.getPlaneBitMask("DETECTED"))
2230 epochCountImage += slateIm
2232 if self.config.doFilterMorphological: 2232 ↛ 2233line 2232 didn't jump to line 2233, because the condition on line 2232 was never true
2233 maskName = self.config.streakMaskName
2234 _ = self.maskStreaks.run(warpDiffExp)
2235 streakMask = warpDiffExp.mask
2236 spanSetStreak = afwGeom.SpanSet.fromMask(streakMask,
2237 streakMask.getPlaneBitMask(maskName)).split()
2239 # PSF-Matched warps have less available area (~the matching kernel) because the calexps
2240 # undergo a second convolution. Pixels with data in the direct warp
2241 # but not in the PSF-matched warp will not have their artifacts detected.
2242 # NaNs from the PSF-matched warp therefore must be masked in the direct warp
2243 nans = numpy.where(numpy.isnan(warpDiffExp.maskedImage.image.array), 1, 0)
2244 nansMask = afwImage.makeMaskFromArray(nans.astype(afwImage.MaskPixel))
2245 nansMask.setXY0(warpDiffExp.getXY0())
2246 edgeMask = warpDiffExp.mask
2247 spanSetEdgeMask = afwGeom.SpanSet.fromMask(edgeMask,
2248 edgeMask.getPlaneBitMask("EDGE")).split()
2249 else:
2250 # If the directWarp has <1% coverage, the psfMatchedWarp can have 0% and not exist
2251 # In this case, mask the whole epoch
2252 nansMask = afwImage.MaskX(coaddBBox, 1)
2253 spanSetList = []
2254 spanSetEdgeMask = []
2255 spanSetStreak = []
2257 spanSetNoDataMask = afwGeom.SpanSet.fromMask(nansMask).split()
2259 spanSetNoDataMaskList.append(spanSetNoDataMask)
2260 spanSetArtifactList.append(spanSetList)
2261 spanSetEdgeList.append(spanSetEdgeMask)
2262 if self.config.doFilterMorphological: 2262 ↛ 2263line 2262 didn't jump to line 2263, because the condition on line 2262 was never true
2263 spanSetBadMorphoList.append(spanSetStreak)
2265 if lsstDebug.Info(__name__).saveCountIm: 2265 ↛ 2266line 2265 didn't jump to line 2266, because the condition on line 2265 was never true
2266 path = self._dataRef2DebugPath("epochCountIm", tempExpRefList[0], coaddLevel=True)
2267 epochCountImage.writeFits(path)
2269 for i, spanSetList in enumerate(spanSetArtifactList):
2270 if spanSetList:
2271 filteredSpanSetList = self.filterArtifacts(spanSetList, epochCountImage, nImage,
2272 templateFootprints)
2273 spanSetArtifactList[i] = filteredSpanSetList
2274 if self.config.doFilterMorphological: 2274 ↛ 2275line 2274 didn't jump to line 2275, because the condition on line 2274 was never true
2275 spanSetArtifactList[i] += spanSetBadMorphoList[i]
2277 altMasks = []
2278 for artifacts, noData, edge in zip(spanSetArtifactList, spanSetNoDataMaskList, spanSetEdgeList):
2279 altMasks.append({'CLIPPED': artifacts,
2280 'NO_DATA': noData,
2281 'EDGE': edge})
2282 return altMasks
2284 def prefilterArtifacts(self, spanSetList, exp):
2285 """Remove artifact candidates covered by bad mask plane.
2287 Any future editing of the candidate list that does not depend on
2288 temporal information should go in this method.
2290 Parameters
2291 ----------
2292 spanSetList : `list`
2293 List of SpanSets representing artifact candidates.
2294 exp : `lsst.afw.image.Exposure`
2295 Exposure containing mask planes used to prefilter.
2297 Returns
2298 -------
2299 returnSpanSetList : `list`
2300 List of SpanSets with artifacts.
2301 """
2302 badPixelMask = exp.mask.getPlaneBitMask(self.config.prefilterArtifactsMaskPlanes)
2303 goodArr = (exp.mask.array & badPixelMask) == 0
2304 returnSpanSetList = []
2305 bbox = exp.getBBox()
2306 x0, y0 = exp.getXY0()
2307 for i, span in enumerate(spanSetList):
2308 y, x = span.clippedTo(bbox).indices()
2309 yIndexLocal = numpy.array(y) - y0
2310 xIndexLocal = numpy.array(x) - x0
2311 goodRatio = numpy.count_nonzero(goodArr[yIndexLocal, xIndexLocal])/span.getArea()
2312 if goodRatio > self.config.prefilterArtifactsRatio: 2312 ↛ 2307line 2312 didn't jump to line 2307, because the condition on line 2312 was never false
2313 returnSpanSetList.append(span)
2314 return returnSpanSetList
2316 def filterArtifacts(self, spanSetList, epochCountImage, nImage, footprintsToExclude=None):
2317 """Filter artifact candidates.
2319 Parameters
2320 ----------
2321 spanSetList : `list`
2322 List of SpanSets representing artifact candidates.
2323 epochCountImage : `lsst.afw.image.Image`
2324 Image of accumulated number of warpDiff detections.
2325 nImage : `lsst.afw.image.Image`
2326 Image of the accumulated number of total epochs contributing.
2328 Returns
2329 -------
2330 maskSpanSetList : `list`
2331 List of SpanSets with artifacts.
2332 """
2334 maskSpanSetList = []
2335 x0, y0 = epochCountImage.getXY0()
2336 for i, span in enumerate(spanSetList):
2337 y, x = span.indices()
2338 yIdxLocal = [y1 - y0 for y1 in y]
2339 xIdxLocal = [x1 - x0 for x1 in x]
2340 outlierN = epochCountImage.array[yIdxLocal, xIdxLocal]
2341 totalN = nImage.array[yIdxLocal, xIdxLocal]
2343 # effectiveMaxNumEpochs is broken line (fraction of N) with characteristic config.maxNumEpochs
2344 effMaxNumEpochsHighN = (self.config.maxNumEpochs
2345 + self.config.maxFractionEpochsHigh*numpy.mean(totalN))
2346 effMaxNumEpochsLowN = self.config.maxFractionEpochsLow * numpy.mean(totalN)
2347 effectiveMaxNumEpochs = int(min(effMaxNumEpochsLowN, effMaxNumEpochsHighN))
2348 nPixelsBelowThreshold = numpy.count_nonzero((outlierN > 0)
2349 & (outlierN <= effectiveMaxNumEpochs))
2350 percentBelowThreshold = nPixelsBelowThreshold / len(outlierN)
2351 if percentBelowThreshold > self.config.spatialThreshold:
2352 maskSpanSetList.append(span)
2354 if self.config.doPreserveContainedBySource and footprintsToExclude is not None: 2354 ↛ 2367line 2354 didn't jump to line 2367, because the condition on line 2354 was never false
2355 # If a candidate is contained by a footprint on the template coadd, do not clip
2356 filteredMaskSpanSetList = []
2357 for span in maskSpanSetList:
2358 doKeep = True
2359 for footprint in footprintsToExclude.positive.getFootprints():
2360 if footprint.spans.contains(span):
2361 doKeep = False
2362 break
2363 if doKeep:
2364 filteredMaskSpanSetList.append(span)
2365 maskSpanSetList = filteredMaskSpanSetList
2367 return maskSpanSetList
2369 def _readAndComputeWarpDiff(self, warpRef, imageScaler, templateCoadd):
2370 """Fetch a warp from the butler and return a warpDiff.
2372 Parameters
2373 ----------
2374 warpRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
2375 Butler dataRef for the warp.
2376 imageScaler : `lsst.pipe.tasks.scaleZeroPoint.ImageScaler`
2377 An image scaler object.
2378 templateCoadd : `lsst.afw.image.Exposure`
2379 Exposure to be substracted from the scaled warp.
2381 Returns
2382 -------
2383 warp : `lsst.afw.image.Exposure`
2384 Exposure of the image difference between the warp and template.
2385 """
2387 # If the PSF-Matched warp did not exist for this direct warp
2388 # None is holding its place to maintain order in Gen 3
2389 if warpRef is None:
2390 return None
2391 # Warp comparison must use PSF-Matched Warps regardless of requested coadd warp type
2392 warpName = self.getTempExpDatasetName('psfMatched')
2393 if not isinstance(warpRef, DeferredDatasetHandle):
2394 if not warpRef.datasetExists(warpName): 2394 ↛ 2395line 2394 didn't jump to line 2395, because the condition on line 2394 was never true
2395 self.log.warn("Could not find %s %s; skipping it", warpName, warpRef.dataId)
2396 return None
2397 warp = warpRef.get(datasetType=warpName, immediate=True)
2398 # direct image scaler OK for PSF-matched Warp
2399 imageScaler.scaleMaskedImage(warp.getMaskedImage())
2400 mi = warp.getMaskedImage()
2401 if self.config.doScaleWarpVariance: 2401 ↛ 2406line 2401 didn't jump to line 2406, because the condition on line 2401 was never false
2402 try:
2403 self.scaleWarpVariance.run(mi)
2404 except Exception as exc:
2405 self.log.warn("Unable to rescale variance of warp (%s); leaving it as-is" % (exc,))
2406 mi -= templateCoadd.getMaskedImage()
2407 return warp
2409 def _dataRef2DebugPath(self, prefix, warpRef, coaddLevel=False):
2410 """Return a path to which to write debugging output.
2412 Creates a hyphen-delimited string of dataId values for simple filenames.
2414 Parameters
2415 ----------
2416 prefix : `str`
2417 Prefix for filename.
2418 warpRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
2419 Butler dataRef to make the path from.
2420 coaddLevel : `bool`, optional.
2421 If True, include only coadd-level keys (e.g., 'tract', 'patch',
2422 'filter', but no 'visit').
2424 Returns
2425 -------
2426 result : `str`
2427 Path for debugging output.
2428 """
2429 if coaddLevel:
2430 keys = warpRef.getButler().getKeys(self.getCoaddDatasetName(self.warpType))
2431 else:
2432 keys = warpRef.dataId.keys()
2433 keyList = sorted(keys, reverse=True)
2434 directory = lsstDebug.Info(__name__).figPath if lsstDebug.Info(__name__).figPath else "."
2435 filename = "%s-%s.fits" % (prefix, '-'.join([str(warpRef.dataId[k]) for k in keyList]))
2436 return os.path.join(directory, filename)
2439def reorderAndPadList(inputList, inputKeys, outputKeys, padWith=None):
2440 """Match the order of one list to another, padding if necessary
2442 Parameters
2443 ----------
2444 inputList : list
2445 List to be reordered and padded. Elements can be any type.
2446 inputKeys : iterable
2447 Iterable of values to be compared with outputKeys.
2448 Length must match `inputList`
2449 outputKeys : iterable
2450 Iterable of values to be compared with inputKeys.
2451 padWith :
2452 Any value to be inserted where inputKey not in outputKeys
2454 Returns
2455 -------
2456 list
2457 Copy of inputList reordered per outputKeys and padded with `padWith`
2458 so that the length matches length of outputKeys.
2459 """
2460 outputList = []
2461 for d in outputKeys:
2462 if d in inputKeys:
2463 outputList.append(inputList[inputKeys.index(d)])
2464 else:
2465 outputList.append(padWith)
2466 return outputList