Coverage for python/lsst/drp/tasks/assemble_coadd.py: 16%
633 statements
« prev ^ index » next coverage.py v7.3.2, created at 2023-11-15 13:29 +0000
« prev ^ index » next coverage.py v7.3.2, created at 2023-11-15 13:29 +0000
1# This file is part of drp_tasks.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22__all__ = [
23 "AssembleCoaddTask",
24 "AssembleCoaddConnections",
25 "AssembleCoaddConfig",
26 "CompareWarpAssembleCoaddTask",
27 "CompareWarpAssembleCoaddConfig",
28]
30import copy
31import logging
32import warnings
34import lsst.afw.geom as afwGeom
35import lsst.afw.image as afwImage
36import lsst.afw.math as afwMath
37import lsst.afw.table as afwTable
38import lsst.coadd.utils as coaddUtils
39import lsst.geom as geom
40import lsst.meas.algorithms as measAlg
41import lsst.pex.config as pexConfig
42import lsst.pex.exceptions as pexExceptions
43import lsst.pipe.base as pipeBase
44import lsst.utils as utils
45import lsstDebug
46import numpy
47from deprecated.sphinx import deprecated
48from lsst.meas.algorithms import AccumulatorMeanStack, ScaleVarianceTask, SourceDetectionTask
49from lsst.pipe.tasks.coaddBase import CoaddBaseTask, makeSkyInfo, reorderAndPadList, subBBoxIter
50from lsst.pipe.tasks.healSparseMapping import HealSparseInputMapTask
51from lsst.pipe.tasks.interpImage import InterpImageTask
52from lsst.pipe.tasks.maskStreaks import MaskStreaksTask
53from lsst.pipe.tasks.scaleZeroPoint import ScaleZeroPointTask
54from lsst.skymap import BaseSkyMap
55from lsst.utils.timer import timeMethod
57log = logging.getLogger(__name__)
60class AssembleCoaddConnections(
61 pipeBase.PipelineTaskConnections,
62 dimensions=("tract", "patch", "band", "skymap"),
63 defaultTemplates={
64 "inputCoaddName": "deep",
65 "outputCoaddName": "deep",
66 "warpType": "direct",
67 "warpTypeSuffix": "",
68 },
69):
70 inputWarps = pipeBase.connectionTypes.Input(
71 doc=(
72 "Input list of warps to be assemebled i.e. stacked."
73 "WarpType (e.g. direct, psfMatched) is controlled by the "
74 "warpType config parameter"
75 ),
76 name="{inputCoaddName}Coadd_{warpType}Warp",
77 storageClass="ExposureF",
78 dimensions=("tract", "patch", "skymap", "visit", "instrument"),
79 deferLoad=True,
80 multiple=True,
81 )
82 skyMap = pipeBase.connectionTypes.Input(
83 doc="Input definition of geometry/bbox and projection/wcs for coadded " "exposures",
84 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
85 storageClass="SkyMap",
86 dimensions=("skymap",),
87 )
88 selectedVisits = pipeBase.connectionTypes.Input(
89 doc="Selected visits to be coadded.",
90 name="{outputCoaddName}Visits",
91 storageClass="StructuredDataDict",
92 dimensions=("instrument", "tract", "patch", "skymap", "band"),
93 )
94 brightObjectMask = pipeBase.connectionTypes.PrerequisiteInput(
95 doc=(
96 "Input Bright Object Mask mask produced with external catalogs "
97 "to be applied to the mask plane BRIGHT_OBJECT."
98 ),
99 name="brightObjectMask",
100 storageClass="ObjectMaskCatalog",
101 dimensions=("tract", "patch", "skymap", "band"),
102 minimum=0,
103 )
104 coaddExposure = pipeBase.connectionTypes.Output(
105 doc="Output coadded exposure, produced by stacking input warps",
106 name="{outputCoaddName}Coadd{warpTypeSuffix}",
107 storageClass="ExposureF",
108 dimensions=("tract", "patch", "skymap", "band"),
109 )
110 nImage = pipeBase.connectionTypes.Output(
111 doc="Output image of number of input images per pixel",
112 name="{outputCoaddName}Coadd_nImage",
113 storageClass="ImageU",
114 dimensions=("tract", "patch", "skymap", "band"),
115 )
116 inputMap = pipeBase.connectionTypes.Output(
117 doc="Output healsparse map of input images",
118 name="{outputCoaddName}Coadd_inputMap",
119 storageClass="HealSparseMap",
120 dimensions=("tract", "patch", "skymap", "band"),
121 )
123 def __init__(self, *, config=None):
124 super().__init__(config=config)
126 if not config.doMaskBrightObjects:
127 self.prerequisiteInputs.remove("brightObjectMask")
129 if not config.doSelectVisits:
130 self.inputs.remove("selectedVisits")
132 if not config.doNImage:
133 self.outputs.remove("nImage")
135 if not self.config.doInputMap:
136 self.outputs.remove("inputMap")
139class AssembleCoaddConfig(
140 CoaddBaseTask.ConfigClass, pipeBase.PipelineTaskConfig, pipelineConnections=AssembleCoaddConnections
141):
142 warpType = pexConfig.Field(
143 doc="Warp name: one of 'direct' or 'psfMatched'",
144 dtype=str,
145 default="direct",
146 )
147 subregionSize = pexConfig.ListField(
148 dtype=int,
149 doc="Width, height of stack subregion size; "
150 "make small enough that a full stack of images will fit into memory "
151 " at once.",
152 length=2,
153 default=(2000, 2000),
154 )
155 statistic = pexConfig.Field(
156 dtype=str,
157 doc="Main stacking statistic for aggregating over the epochs.",
158 default="MEANCLIP",
159 )
160 doOnlineForMean = pexConfig.Field(
161 dtype=bool,
162 doc='Perform online coaddition when statistic="MEAN" to save memory?',
163 default=False,
164 )
165 doSigmaClip = pexConfig.Field(
166 dtype=bool,
167 doc="Perform sigma clipped outlier rejection with MEANCLIP statistic?",
168 deprecated=True,
169 default=False,
170 )
171 sigmaClip = pexConfig.Field(
172 dtype=float,
173 doc="Sigma for outlier rejection; ignored if non-clipping statistic " "selected.",
174 default=3.0,
175 )
176 clipIter = pexConfig.Field(
177 dtype=int,
178 doc="Number of iterations of outlier rejection; ignored if " "non-clipping statistic selected.",
179 default=2,
180 )
181 calcErrorFromInputVariance = pexConfig.Field(
182 dtype=bool,
183 doc="Calculate coadd variance from input variance by stacking "
184 "statistic. Passed to "
185 "StatisticsControl.setCalcErrorFromInputVariance()",
186 default=True,
187 )
188 scaleZeroPoint = pexConfig.ConfigurableField(
189 target=ScaleZeroPointTask,
190 doc="Task to adjust the photometric zero point of the coadd temp " "exposures",
191 )
192 doInterp = pexConfig.Field(
193 doc="Interpolate over NaN pixels? Also extrapolate, if necessary, but " "the results are ugly.",
194 dtype=bool,
195 default=True,
196 )
197 interpImage = pexConfig.ConfigurableField(
198 target=InterpImageTask,
199 doc="Task to interpolate (and extrapolate) over NaN pixels",
200 )
201 doWrite = pexConfig.Field(
202 doc="Persist coadd?",
203 dtype=bool,
204 default=True,
205 )
206 doNImage = pexConfig.Field(
207 doc="Create image of number of contributing exposures for each pixel",
208 dtype=bool,
209 default=False,
210 )
211 doUsePsfMatchedPolygons = pexConfig.Field(
212 doc="Use ValidPolygons from shrunk Psf-Matched Calexps? Should be set "
213 "to True by CompareWarp only.",
214 dtype=bool,
215 default=False,
216 )
217 maskPropagationThresholds = pexConfig.DictField(
218 keytype=str,
219 itemtype=float,
220 doc=(
221 "Threshold (in fractional weight) of rejection at which we "
222 "propagate a mask plane to the coadd; that is, we set the mask "
223 "bit on the coadd if the fraction the rejected frames "
224 "would have contributed exceeds this value."
225 ),
226 default={"SAT": 0.1},
227 )
228 removeMaskPlanes = pexConfig.ListField(
229 dtype=str,
230 default=["NOT_DEBLENDED"],
231 doc="Mask planes to remove before coadding",
232 )
233 doMaskBrightObjects = pexConfig.Field(
234 dtype=bool,
235 default=False,
236 doc="Set mask and flag bits for bright objects?",
237 )
238 brightObjectMaskName = pexConfig.Field(
239 dtype=str,
240 default="BRIGHT_OBJECT",
241 doc="Name of mask bit used for bright objects",
242 )
243 coaddPsf = pexConfig.ConfigField(
244 doc="Configuration for CoaddPsf",
245 dtype=measAlg.CoaddPsfConfig,
246 )
247 doAttachTransmissionCurve = pexConfig.Field(
248 dtype=bool,
249 default=False,
250 optional=False,
251 doc=(
252 "Attach a piecewise TransmissionCurve for the coadd? "
253 "(requires all input Exposures to have TransmissionCurves)."
254 ),
255 )
256 hasFakes = pexConfig.Field(
257 dtype=bool,
258 default=False,
259 doc="Should be set to True if fake sources have been inserted into the input data.",
260 )
261 doSelectVisits = pexConfig.Field(
262 doc="Coadd only visits selected by a SelectVisitsTask",
263 dtype=bool,
264 default=False,
265 )
266 doInputMap = pexConfig.Field(
267 doc="Create a bitwise map of coadd inputs",
268 dtype=bool,
269 default=False,
270 )
271 inputMapper = pexConfig.ConfigurableField(
272 doc="Input map creation subtask.",
273 target=HealSparseInputMapTask,
274 )
276 def setDefaults(self):
277 super().setDefaults()
278 self.badMaskPlanes = ["NO_DATA", "BAD", "SAT", "EDGE"]
280 def validate(self):
281 super().validate()
282 if self.doPsfMatch: # TODO: Remove this in DM-39841
283 # Backwards compatibility.
284 # Configs do not have loggers
285 log.warning("Config doPsfMatch deprecated. Setting warpType='psfMatched'")
286 self.warpType = "psfMatched"
287 if self.doSigmaClip and self.statistic != "MEANCLIP":
288 log.warning('doSigmaClip deprecated. To replicate behavior, setting statistic to "MEANCLIP"')
289 self.statistic = "MEANCLIP"
290 if self.doInterp and self.statistic not in ["MEAN", "MEDIAN", "MEANCLIP", "VARIANCE", "VARIANCECLIP"]:
291 raise ValueError(
292 "Must set doInterp=False for statistic=%s, which does not "
293 "compute and set a non-zero coadd variance estimate." % (self.statistic)
294 )
296 unstackableStats = ["NOTHING", "ERROR", "ORMASK"]
297 if not hasattr(afwMath.Property, self.statistic) or self.statistic in unstackableStats:
298 stackableStats = [
299 str(k) for k in afwMath.Property.__members__.keys() if str(k) not in unstackableStats
300 ]
301 raise ValueError(
302 "statistic %s is not allowed. Please choose one of %s." % (self.statistic, stackableStats)
303 )
306class AssembleCoaddTask(CoaddBaseTask, pipeBase.PipelineTask):
307 """Assemble a coadded image from a set of warps.
309 Each Warp that goes into a coadd will typically have an independent
310 photometric zero-point. Therefore, we must scale each Warp to set it to
311 a common photometric zeropoint. WarpType may be one of 'direct' or
312 'psfMatched', and the boolean configs `config.makeDirect` and
313 `config.makePsfMatched` set which of the warp types will be coadded.
314 The coadd is computed as a mean with optional outlier rejection.
315 Criteria for outlier rejection are set in `AssembleCoaddConfig`.
316 Finally, Warps can have bad 'NaN' pixels which received no input from the
317 source calExps. We interpolate over these bad (NaN) pixels.
319 `AssembleCoaddTask` uses several sub-tasks. These are
321 - `~lsst.pipe.tasks.ScaleZeroPointTask`
322 - create and use an ``imageScaler`` object to scale the photometric
323 zeropoint for each Warp
324 - `~lsst.pipe.tasks.InterpImageTask`
325 - interpolate across bad pixels (NaN) in the final coadd
327 You can retarget these subtasks if you wish.
329 Raises
330 ------
331 RuntimeError
332 Raised if unable to define mask plane for bright objects.
334 Notes
335 -----
336 Debugging:
337 `AssembleCoaddTask` has no debug variables of its own. Some of the
338 subtasks may support `~lsst.base.lsstDebug` variables. See the
339 documentation for the subtasks for further information.
340 """
342 ConfigClass = AssembleCoaddConfig
343 _DefaultName = "assembleCoadd"
345 def __init__(self, *args, **kwargs):
346 # TODO: DM-17415 better way to handle previously allowed passed args
347 # e.g.`AssembleCoaddTask(config)`
348 if args:
349 argNames = ["config", "name", "parentTask", "log"]
350 kwargs.update({k: v for k, v in zip(argNames, args)})
351 warnings.warn(
352 "AssembleCoadd received positional args, and casting them as kwargs: %s. "
353 "PipelineTask will not take positional args" % argNames,
354 FutureWarning,
355 stacklevel=2,
356 )
358 super().__init__(**kwargs)
359 self.makeSubtask("interpImage")
360 self.makeSubtask("scaleZeroPoint")
362 if self.config.doMaskBrightObjects:
363 mask = afwImage.Mask()
364 try:
365 self.brightObjectBitmask = 1 << mask.addMaskPlane(self.config.brightObjectMaskName)
366 except pexExceptions.LsstCppException:
367 raise RuntimeError(
368 "Unable to define mask plane for bright objects; planes used are %s"
369 % mask.getMaskPlaneDict().keys()
370 )
371 del mask
373 if self.config.doInputMap:
374 self.makeSubtask("inputMapper")
376 self.warpType = self.config.warpType
378 def runQuantum(self, butlerQC, inputRefs, outputRefs):
379 inputData = butlerQC.get(inputRefs)
381 # Construct skyInfo expected by run
382 # Do not remove skyMap from inputData in case _makeSupplementaryData
383 # needs it
384 skyMap = inputData["skyMap"]
385 outputDataId = butlerQC.quantum.dataId
387 inputData["skyInfo"] = makeSkyInfo(
388 skyMap, tractId=outputDataId["tract"], patchId=outputDataId["patch"]
389 )
391 if self.config.doSelectVisits:
392 warpRefList = self.filterWarps(inputData["inputWarps"], inputData["selectedVisits"])
393 else:
394 warpRefList = inputData["inputWarps"]
396 inputs = self.prepareInputs(warpRefList)
397 self.log.info("Found %d %s", len(inputs.tempExpRefList), self.getTempExpDatasetName(self.warpType))
398 if len(inputs.tempExpRefList) == 0:
399 raise pipeBase.NoWorkFound("No coadd temporary exposures found")
401 supplementaryData = self._makeSupplementaryData(butlerQC, inputRefs, outputRefs)
402 retStruct = self.run(
403 inputData["skyInfo"],
404 inputs.tempExpRefList,
405 inputs.imageScalerList,
406 inputs.weightList,
407 supplementaryData=supplementaryData,
408 )
410 inputData.setdefault("brightObjectMask", None)
411 if self.config.doMaskBrightObjects and inputData["brightObjectMask"] is None:
412 log.warning("doMaskBrightObjects is set to True, but brightObjectMask not loaded")
413 self.processResults(retStruct.coaddExposure, inputData["brightObjectMask"], outputDataId)
415 if self.config.doWrite:
416 butlerQC.put(retStruct, outputRefs)
417 return retStruct
419 def processResults(self, coaddExposure, brightObjectMasks=None, dataId=None):
420 """Interpolate over missing data and mask bright stars.
422 Parameters
423 ----------
424 coaddExposure : `lsst.afw.image.Exposure`
425 The coadded exposure to process.
426 brightObjectMasks : `lsst.afw.table` or `None`, optional
427 Table of bright objects to mask.
428 dataId : `lsst.daf.butler.DataId` or `None`, optional
429 Data identification.
430 """
431 if self.config.doInterp:
432 self.interpImage.run(coaddExposure.getMaskedImage(), planeName="NO_DATA")
433 # The variance must be positive; work around for DM-3201.
434 varArray = coaddExposure.variance.array
435 with numpy.errstate(invalid="ignore"):
436 varArray[:] = numpy.where(varArray > 0, varArray, numpy.inf)
438 if self.config.doMaskBrightObjects:
439 self.setBrightObjectMasks(coaddExposure, brightObjectMasks, dataId)
441 def _makeSupplementaryData(self, butlerQC, inputRefs, outputRefs):
442 """Make additional inputs to run() specific to subclasses (Gen3).
444 Duplicates interface of `runQuantum` method.
445 Available to be implemented by subclasses only if they need the
446 coadd dataRef for performing preliminary processing before
447 assembling the coadd.
449 Parameters
450 ----------
451 butlerQC : `~lsst.pipe.base.ButlerQuantumContext`
452 Gen3 Butler object for fetching additional data products before
453 running the Task specialized for quantum being processed.
454 inputRefs : `~lsst.pipe.base.InputQuantizedConnection`
455 Attributes are the names of the connections describing input
456 dataset types. Values are DatasetRefs that task consumes for the
457 corresponding dataset type. DataIds are guaranteed to match data
458 objects in ``inputData``.
459 outputRefs : `~lsst.pipe.base.OutputQuantizedConnection`
460 Attributes are the names of the connections describing output
461 dataset types. Values are DatasetRefs that task is to produce
462 for the corresponding dataset type.
463 """
464 return pipeBase.Struct()
466 @deprecated(
467 reason="makeSupplementaryDataGen3 is deprecated in favor of _makeSupplementaryData",
468 version="v25.0",
469 category=FutureWarning,
470 )
471 def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs):
472 return self._makeSupplementaryData(butlerQC, inputRefs, outputRefs)
474 def prepareInputs(self, refList):
475 """Prepare the input warps for coaddition by measuring the weight for
476 each warp and the scaling for the photometric zero point.
478 Each Warp has its own photometric zeropoint and background variance.
479 Before coadding these Warps together, compute a scale factor to
480 normalize the photometric zeropoint and compute the weight for each
481 Warp.
483 Parameters
484 ----------
485 refList : `list`
486 List of data references to tempExp.
488 Returns
489 -------
490 result : `~lsst.pipe.base.Struct`
491 Results as a struct with attributes:
493 ``tempExprefList``
494 `list` of data references to tempExp.
495 ``weightList``
496 `list` of weightings.
497 ``imageScalerList``
498 `list` of image scalers.
499 """
500 statsCtrl = afwMath.StatisticsControl()
501 statsCtrl.setNumSigmaClip(self.config.sigmaClip)
502 statsCtrl.setNumIter(self.config.clipIter)
503 statsCtrl.setAndMask(self.getBadPixelMask())
504 statsCtrl.setNanSafe(True)
505 # compute tempExpRefList: a list of tempExpRef that actually exist
506 # and weightList: a list of the weight of the associated coadd tempExp
507 # and imageScalerList: a list of scale factors for the associated coadd
508 # tempExp.
509 tempExpRefList = []
510 weightList = []
511 imageScalerList = []
512 tempExpName = self.getTempExpDatasetName(self.warpType)
513 for tempExpRef in refList:
514 tempExp = tempExpRef.get()
515 # Ignore any input warp that is empty of data
516 if numpy.isnan(tempExp.image.array).all():
517 continue
518 maskedImage = tempExp.getMaskedImage()
519 imageScaler = self.scaleZeroPoint.computeImageScaler(
520 exposure=tempExp,
521 dataRef=tempExpRef, # FIXME
522 )
523 try:
524 imageScaler.scaleMaskedImage(maskedImage)
525 except Exception as e:
526 self.log.warning("Scaling failed for %s (skipping it): %s", tempExpRef.dataId, e)
527 continue
528 statObj = afwMath.makeStatistics(
529 maskedImage.getVariance(), maskedImage.getMask(), afwMath.MEANCLIP, statsCtrl
530 )
531 meanVar, meanVarErr = statObj.getResult(afwMath.MEANCLIP)
532 weight = 1.0 / float(meanVar)
533 if not numpy.isfinite(weight):
534 self.log.warning("Non-finite weight for %s: skipping", tempExpRef.dataId)
535 continue
536 self.log.info("Weight of %s %s = %0.3f", tempExpName, tempExpRef.dataId, weight)
538 del maskedImage
539 del tempExp
541 tempExpRefList.append(tempExpRef)
542 weightList.append(weight)
543 imageScalerList.append(imageScaler)
545 return pipeBase.Struct(
546 tempExpRefList=tempExpRefList, weightList=weightList, imageScalerList=imageScalerList
547 )
549 def prepareStats(self, mask=None):
550 """Prepare the statistics for coadding images.
552 Parameters
553 ----------
554 mask : `int`, optional
555 Bit mask value to exclude from coaddition.
557 Returns
558 -------
559 stats : `~lsst.pipe.base.Struct`
560 Statistics as a struct with attributes:
562 ``statsCtrl``
563 Statistics control object for coadd
564 (`~lsst.afw.math.StatisticsControl`).
565 ``statsFlags``
566 Statistic for coadd (`~lsst.afw.math.Property`).
567 """
568 if mask is None:
569 mask = self.getBadPixelMask()
570 statsCtrl = afwMath.StatisticsControl()
571 statsCtrl.setNumSigmaClip(self.config.sigmaClip)
572 statsCtrl.setNumIter(self.config.clipIter)
573 statsCtrl.setAndMask(mask)
574 statsCtrl.setNanSafe(True)
575 statsCtrl.setWeighted(True)
576 statsCtrl.setCalcErrorFromInputVariance(self.config.calcErrorFromInputVariance)
577 for plane, threshold in self.config.maskPropagationThresholds.items():
578 bit = afwImage.Mask.getMaskPlane(plane)
579 statsCtrl.setMaskPropagationThreshold(bit, threshold)
580 statsFlags = afwMath.stringToStatisticsProperty(self.config.statistic)
581 return pipeBase.Struct(ctrl=statsCtrl, flags=statsFlags)
583 @timeMethod
584 def run(
585 self,
586 skyInfo,
587 tempExpRefList,
588 imageScalerList,
589 weightList,
590 altMaskList=None,
591 mask=None,
592 supplementaryData=None,
593 ):
594 """Assemble a coadd from input warps.
596 Assemble the coadd using the provided list of coaddTempExps. Since
597 the full coadd covers a patch (a large area), the assembly is
598 performed over small areas on the image at a time in order to
599 conserve memory usage. Iterate over subregions within the outer
600 bbox of the patch using `assembleSubregion` to stack the corresponding
601 subregions from the coaddTempExps with the statistic specified.
602 Set the edge bits the coadd mask based on the weight map.
604 Parameters
605 ----------
606 skyInfo : `~lsst.pipe.base.Struct`
607 Struct with geometric information about the patch.
608 tempExpRefList : `list`
609 List of data references to Warps (previously called CoaddTempExps).
610 imageScalerList : `list`
611 List of image scalers.
612 weightList : `list`
613 List of weights.
614 altMaskList : `list`, optional
615 List of alternate masks to use rather than those stored with
616 tempExp.
617 mask : `int`, optional
618 Bit mask value to exclude from coaddition.
619 supplementaryData : `~lsst.pipe.base.Struct`, optional
620 Struct with additional data products needed to assemble coadd.
621 Only used by subclasses that implement ``_makeSupplementaryData``
622 and override `run`.
624 Returns
625 -------
626 result : `~lsst.pipe.base.Struct`
627 Results as a struct with attributes:
629 ``coaddExposure``
630 Coadded exposure (`~lsst.afw.image.Exposure`).
631 ``nImage``
632 Exposure count image (`~lsst.afw.image.Image`), if requested.
633 ``inputMap``
634 Bit-wise map of inputs, if requested.
635 ``warpRefList``
636 Input list of refs to the warps
637 (`~lsst.daf.butler.DeferredDatasetHandle`) (unmodified).
638 ``imageScalerList``
639 Input list of image scalers (`list`) (unmodified).
640 ``weightList``
641 Input list of weights (`list`) (unmodified).
643 Raises
644 ------
645 lsst.pipe.base.NoWorkFound
646 Raised if no data references are provided.
647 """
648 tempExpName = self.getTempExpDatasetName(self.warpType)
649 self.log.info("Assembling %s %s", len(tempExpRefList), tempExpName)
650 if not tempExpRefList:
651 raise pipeBase.NoWorkFound("No exposures provided for co-addition.")
653 stats = self.prepareStats(mask=mask)
655 if altMaskList is None:
656 altMaskList = [None] * len(tempExpRefList)
658 coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
659 coaddExposure.setPhotoCalib(self.scaleZeroPoint.getPhotoCalib())
660 coaddExposure.getInfo().setCoaddInputs(self.inputRecorder.makeCoaddInputs())
661 self.assembleMetadata(coaddExposure, tempExpRefList, weightList)
662 coaddMaskedImage = coaddExposure.getMaskedImage()
663 subregionSizeArr = self.config.subregionSize
664 subregionSize = geom.Extent2I(subregionSizeArr[0], subregionSizeArr[1])
665 # if nImage is requested, create a zero one which can be passed to
666 # assembleSubregion.
667 if self.config.doNImage:
668 nImage = afwImage.ImageU(skyInfo.bbox)
669 else:
670 nImage = None
671 # If inputMap is requested, create the initial version that can be
672 # masked in assembleSubregion.
673 if self.config.doInputMap:
674 self.inputMapper.build_ccd_input_map(
675 skyInfo.bbox, skyInfo.wcs, coaddExposure.getInfo().getCoaddInputs().ccds
676 )
678 if self.config.doOnlineForMean and self.config.statistic == "MEAN":
679 try:
680 self.assembleOnlineMeanCoadd(
681 coaddExposure,
682 tempExpRefList,
683 imageScalerList,
684 weightList,
685 altMaskList,
686 stats.ctrl,
687 nImage=nImage,
688 )
689 except Exception as e:
690 self.log.exception("Cannot compute online coadd %s", e)
691 raise
692 else:
693 for subBBox in subBBoxIter(skyInfo.bbox, subregionSize):
694 try:
695 self.assembleSubregion(
696 coaddExposure,
697 subBBox,
698 tempExpRefList,
699 imageScalerList,
700 weightList,
701 altMaskList,
702 stats.flags,
703 stats.ctrl,
704 nImage=nImage,
705 )
706 except Exception as e:
707 self.log.exception("Cannot compute coadd %s: %s", subBBox, e)
708 raise
710 # If inputMap is requested, we must finalize the map after the
711 # accumulation.
712 if self.config.doInputMap:
713 self.inputMapper.finalize_ccd_input_map_mask()
714 inputMap = self.inputMapper.ccd_input_map
715 else:
716 inputMap = None
718 self.setInexactPsf(coaddMaskedImage.getMask())
719 # Despite the name, the following doesn't really deal with "EDGE"
720 # pixels: it identifies pixels that didn't receive any unmasked inputs
721 # (as occurs around the edge of the field).
722 coaddUtils.setCoaddEdgeBits(coaddMaskedImage.getMask(), coaddMaskedImage.getVariance())
723 return pipeBase.Struct(
724 coaddExposure=coaddExposure,
725 nImage=nImage,
726 warpRefList=tempExpRefList,
727 imageScalerList=imageScalerList,
728 weightList=weightList,
729 inputMap=inputMap,
730 )
732 def assembleMetadata(self, coaddExposure, tempExpRefList, weightList):
733 """Set the metadata for the coadd.
735 This basic implementation sets the filter from the first input.
737 Parameters
738 ----------
739 coaddExposure : `lsst.afw.image.Exposure`
740 The target exposure for the coadd.
741 tempExpRefList : `list`
742 List of data references to tempExp.
743 weightList : `list`
744 List of weights.
746 Raises
747 ------
748 AssertionError
749 Raised if there is a length mismatch.
750 """
751 assert len(tempExpRefList) == len(weightList), "Length mismatch"
753 # We load a single pixel of each coaddTempExp, because we just want to
754 # get at the metadata (and we need more than just the PropertySet that
755 # contains the header), which is not possible with the current butler
756 # (see #2777).
757 bbox = geom.Box2I(coaddExposure.getBBox().getMin(), geom.Extent2I(1, 1))
759 tempExpList = [tempExpRef.get(parameters={"bbox": bbox}) for tempExpRef in tempExpRefList]
761 numCcds = sum(len(tempExp.getInfo().getCoaddInputs().ccds) for tempExp in tempExpList)
763 # Set the coadd FilterLabel to the band of the first input exposure:
764 # Coadds are calibrated, so the physical label is now meaningless.
765 coaddExposure.setFilter(afwImage.FilterLabel(tempExpList[0].getFilter().bandLabel))
766 coaddInputs = coaddExposure.getInfo().getCoaddInputs()
767 coaddInputs.ccds.reserve(numCcds)
768 coaddInputs.visits.reserve(len(tempExpList))
770 for tempExp, weight in zip(tempExpList, weightList):
771 self.inputRecorder.addVisitToCoadd(coaddInputs, tempExp, weight)
773 if self.config.doUsePsfMatchedPolygons:
774 self.shrinkValidPolygons(coaddInputs)
776 coaddInputs.visits.sort()
777 coaddInputs.ccds.sort()
778 if self.warpType == "psfMatched":
779 # The modelPsf BBox for a psfMatchedWarp/coaddTempExp was
780 # dynamically defined by ModelPsfMatchTask as the square box
781 # bounding its spatially-variable, pre-matched WarpedPsf.
782 # Likewise, set the PSF of a PSF-Matched Coadd to the modelPsf
783 # having the maximum width (sufficient because square)
784 modelPsfList = [tempExp.getPsf() for tempExp in tempExpList]
785 modelPsfWidthList = [
786 modelPsf.computeBBox(modelPsf.getAveragePosition()).getWidth() for modelPsf in modelPsfList
787 ]
788 psf = modelPsfList[modelPsfWidthList.index(max(modelPsfWidthList))]
789 else:
790 psf = measAlg.CoaddPsf(
791 coaddInputs.ccds, coaddExposure.getWcs(), self.config.coaddPsf.makeControl()
792 )
793 coaddExposure.setPsf(psf)
794 apCorrMap = measAlg.makeCoaddApCorrMap(
795 coaddInputs.ccds, coaddExposure.getBBox(afwImage.PARENT), coaddExposure.getWcs()
796 )
797 coaddExposure.getInfo().setApCorrMap(apCorrMap)
798 if self.config.doAttachTransmissionCurve:
799 transmissionCurve = measAlg.makeCoaddTransmissionCurve(coaddExposure.getWcs(), coaddInputs.ccds)
800 coaddExposure.getInfo().setTransmissionCurve(transmissionCurve)
802 def assembleSubregion(
803 self,
804 coaddExposure,
805 bbox,
806 tempExpRefList,
807 imageScalerList,
808 weightList,
809 altMaskList,
810 statsFlags,
811 statsCtrl,
812 nImage=None,
813 ):
814 """Assemble the coadd for a sub-region.
816 For each coaddTempExp, check for (and swap in) an alternative mask
817 if one is passed. Remove mask planes listed in
818 `config.removeMaskPlanes`. Finally, stack the actual exposures using
819 `lsst.afw.math.statisticsStack` with the statistic specified by
820 statsFlags. Typically, the statsFlag will be one of lsst.afw.math.MEAN
821 for a mean-stack or `lsst.afw.math.MEANCLIP` for outlier rejection
822 using an N-sigma clipped mean where N and iterations are specified by
823 statsCtrl. Assign the stacked subregion back to the coadd.
825 Parameters
826 ----------
827 coaddExposure : `lsst.afw.image.Exposure`
828 The target exposure for the coadd.
829 bbox : `lsst.geom.Box`
830 Sub-region to coadd.
831 tempExpRefList : `list`
832 List of data reference to tempExp.
833 imageScalerList : `list`
834 List of image scalers.
835 weightList : `list`
836 List of weights.
837 altMaskList : `list`
838 List of alternate masks to use rather than those stored with
839 tempExp, or None. Each element is dict with keys = mask plane
840 name to which to add the spans.
841 statsFlags : `lsst.afw.math.Property`
842 Property object for statistic for coadd.
843 statsCtrl : `lsst.afw.math.StatisticsControl`
844 Statistics control object for coadd.
845 nImage : `lsst.afw.image.ImageU`, optional
846 Keeps track of exposure count for each pixel.
847 """
848 self.log.debug("Computing coadd over %s", bbox)
850 coaddExposure.mask.addMaskPlane("REJECTED")
851 coaddExposure.mask.addMaskPlane("CLIPPED")
852 coaddExposure.mask.addMaskPlane("SENSOR_EDGE")
853 maskMap = self.setRejectedMaskMapping(statsCtrl)
854 clipped = afwImage.Mask.getPlaneBitMask("CLIPPED")
855 maskedImageList = []
856 if nImage is not None:
857 subNImage = afwImage.ImageU(bbox.getWidth(), bbox.getHeight())
858 for tempExpRef, imageScaler, altMask in zip(tempExpRefList, imageScalerList, altMaskList):
859 exposure = tempExpRef.get(parameters={"bbox": bbox})
861 maskedImage = exposure.getMaskedImage()
862 mask = maskedImage.getMask()
863 if altMask is not None:
864 self.applyAltMaskPlanes(mask, altMask)
865 imageScaler.scaleMaskedImage(maskedImage)
867 # Add 1 for each pixel which is not excluded by the exclude mask.
868 # In legacyCoadd, pixels may also be excluded by
869 # afwMath.statisticsStack.
870 if nImage is not None:
871 subNImage.getArray()[maskedImage.getMask().getArray() & statsCtrl.getAndMask() == 0] += 1
872 if self.config.removeMaskPlanes:
873 self.removeMaskPlanes(maskedImage)
874 maskedImageList.append(maskedImage)
876 if self.config.doInputMap:
877 visit = exposure.getInfo().getCoaddInputs().visits[0].getId()
878 self.inputMapper.mask_warp_bbox(bbox, visit, mask, statsCtrl.getAndMask())
880 with self.timer("stack"):
881 coaddSubregion = afwMath.statisticsStack(
882 maskedImageList,
883 statsFlags,
884 statsCtrl,
885 weightList,
886 clipped, # also set output to CLIPPED if sigma-clipped
887 maskMap,
888 )
889 coaddExposure.maskedImage.assign(coaddSubregion, bbox)
890 if nImage is not None:
891 nImage.assign(subNImage, bbox)
893 def assembleOnlineMeanCoadd(
894 self, coaddExposure, tempExpRefList, imageScalerList, weightList, altMaskList, statsCtrl, nImage=None
895 ):
896 """Assemble the coadd using the "online" method.
898 This method takes a running sum of images and weights to save memory.
899 It only works for MEAN statistics.
901 Parameters
902 ----------
903 coaddExposure : `lsst.afw.image.Exposure`
904 The target exposure for the coadd.
905 tempExpRefList : `list`
906 List of data reference to tempExp.
907 imageScalerList : `list`
908 List of image scalers.
909 weightList : `list`
910 List of weights.
911 altMaskList : `list`
912 List of alternate masks to use rather than those stored with
913 tempExp, or None. Each element is dict with keys = mask plane
914 name to which to add the spans.
915 statsCtrl : `lsst.afw.math.StatisticsControl`
916 Statistics control object for coadd.
917 nImage : `lsst.afw.image.ImageU`, optional
918 Keeps track of exposure count for each pixel.
919 """
920 self.log.debug("Computing online coadd.")
922 coaddExposure.mask.addMaskPlane("REJECTED")
923 coaddExposure.mask.addMaskPlane("CLIPPED")
924 coaddExposure.mask.addMaskPlane("SENSOR_EDGE")
925 maskMap = self.setRejectedMaskMapping(statsCtrl)
926 thresholdDict = AccumulatorMeanStack.stats_ctrl_to_threshold_dict(statsCtrl)
928 bbox = coaddExposure.maskedImage.getBBox()
930 stacker = AccumulatorMeanStack(
931 coaddExposure.image.array.shape,
932 statsCtrl.getAndMask(),
933 mask_threshold_dict=thresholdDict,
934 mask_map=maskMap,
935 no_good_pixels_mask=statsCtrl.getNoGoodPixelsMask(),
936 calc_error_from_input_variance=self.config.calcErrorFromInputVariance,
937 compute_n_image=(nImage is not None),
938 )
940 for tempExpRef, imageScaler, altMask, weight in zip(
941 tempExpRefList, imageScalerList, altMaskList, weightList
942 ):
943 exposure = tempExpRef.get()
944 maskedImage = exposure.getMaskedImage()
945 mask = maskedImage.getMask()
946 if altMask is not None:
947 self.applyAltMaskPlanes(mask, altMask)
948 imageScaler.scaleMaskedImage(maskedImage)
949 if self.config.removeMaskPlanes:
950 self.removeMaskPlanes(maskedImage)
952 stacker.add_masked_image(maskedImage, weight=weight)
954 if self.config.doInputMap:
955 visit = exposure.getInfo().getCoaddInputs().visits[0].getId()
956 self.inputMapper.mask_warp_bbox(bbox, visit, mask, statsCtrl.getAndMask())
958 stacker.fill_stacked_masked_image(coaddExposure.maskedImage)
960 if nImage is not None:
961 nImage.array[:, :] = stacker.n_image
963 def removeMaskPlanes(self, maskedImage):
964 """Unset the mask of an image for mask planes specified in the config.
966 Parameters
967 ----------
968 maskedImage : `lsst.afw.image.MaskedImage`
969 The masked image to be modified.
971 Raises
972 ------
973 InvalidParameterError
974 Raised if no mask plane with that name was found.
975 """
976 mask = maskedImage.getMask()
977 for maskPlane in self.config.removeMaskPlanes:
978 try:
979 mask &= ~mask.getPlaneBitMask(maskPlane)
980 except pexExceptions.InvalidParameterError:
981 self.log.debug(
982 "Unable to remove mask plane %s: no mask plane with that name was found.", maskPlane
983 )
985 @staticmethod
986 def setRejectedMaskMapping(statsCtrl):
987 """Map certain mask planes of the warps to new planes for the coadd.
989 If a pixel is rejected due to a mask value other than EDGE, NO_DATA,
990 or CLIPPED, set it to REJECTED on the coadd.
991 If a pixel is rejected due to EDGE, set the coadd pixel to SENSOR_EDGE.
992 If a pixel is rejected due to CLIPPED, set the coadd pixel to CLIPPED.
994 Parameters
995 ----------
996 statsCtrl : `lsst.afw.math.StatisticsControl`
997 Statistics control object for coadd.
999 Returns
1000 -------
1001 maskMap : `list` of `tuple` of `int`
1002 A list of mappings of mask planes of the warped exposures to
1003 mask planes of the coadd.
1004 """
1005 edge = afwImage.Mask.getPlaneBitMask("EDGE")
1006 noData = afwImage.Mask.getPlaneBitMask("NO_DATA")
1007 clipped = afwImage.Mask.getPlaneBitMask("CLIPPED")
1008 toReject = statsCtrl.getAndMask() & (~noData) & (~edge) & (~clipped)
1009 maskMap = [
1010 (toReject, afwImage.Mask.getPlaneBitMask("REJECTED")),
1011 (edge, afwImage.Mask.getPlaneBitMask("SENSOR_EDGE")),
1012 (clipped, clipped),
1013 ]
1014 return maskMap
1016 def applyAltMaskPlanes(self, mask, altMaskSpans):
1017 """Apply in place alt mask formatted as SpanSets to a mask.
1019 Parameters
1020 ----------
1021 mask : `lsst.afw.image.Mask`
1022 Original mask.
1023 altMaskSpans : `dict`
1024 SpanSet lists to apply. Each element contains the new mask
1025 plane name (e.g. "CLIPPED and/or "NO_DATA") as the key,
1026 and list of SpanSets to apply to the mask.
1028 Returns
1029 -------
1030 mask : `lsst.afw.image.Mask`
1031 Updated mask.
1032 """
1033 if self.config.doUsePsfMatchedPolygons:
1034 if ("NO_DATA" in altMaskSpans) and ("NO_DATA" in self.config.badMaskPlanes):
1035 # Clear away any other masks outside the validPolygons. These
1036 # pixels are no longer contributing to inexact PSFs, and will
1037 # still be rejected because of NO_DATA.
1038 # self.config.doUsePsfMatchedPolygons should be True only in
1039 # CompareWarpAssemble. This mask-clearing step must only occur
1040 # *before* applying the new masks below.
1041 for spanSet in altMaskSpans["NO_DATA"]:
1042 spanSet.clippedTo(mask.getBBox()).clearMask(mask, self.getBadPixelMask())
1044 for plane, spanSetList in altMaskSpans.items():
1045 maskClipValue = mask.addMaskPlane(plane)
1046 for spanSet in spanSetList:
1047 spanSet.clippedTo(mask.getBBox()).setMask(mask, 2**maskClipValue)
1048 return mask
1050 def shrinkValidPolygons(self, coaddInputs):
1051 """Shrink coaddInputs' ccds' ValidPolygons in place.
1053 Either modify each ccd's validPolygon in place, or if CoaddInputs
1054 does not have a validPolygon, create one from its bbox.
1056 Parameters
1057 ----------
1058 coaddInputs : `lsst.afw.image.coaddInputs`
1059 Original mask.
1060 """
1061 for ccd in coaddInputs.ccds:
1062 polyOrig = ccd.getValidPolygon()
1063 validPolyBBox = polyOrig.getBBox() if polyOrig else ccd.getBBox()
1064 validPolyBBox.grow(-self.config.matchingKernelSize // 2)
1065 if polyOrig:
1066 validPolygon = polyOrig.intersectionSingle(validPolyBBox)
1067 else:
1068 validPolygon = afwGeom.polygon.Polygon(geom.Box2D(validPolyBBox))
1069 ccd.setValidPolygon(validPolygon)
1071 def setBrightObjectMasks(self, exposure, brightObjectMasks, dataId=None):
1072 """Set the bright object masks.
1074 Parameters
1075 ----------
1076 exposure : `lsst.afw.image.Exposure`
1077 Exposure under consideration.
1078 brightObjectMasks : `lsst.afw.table`
1079 Table of bright objects to mask.
1080 dataId : `lsst.daf.butler.DataId`, optional
1081 Data identifier dict for patch.
1082 """
1083 if brightObjectMasks is None:
1084 self.log.warning("Unable to apply bright object mask: none supplied")
1085 return
1086 self.log.info("Applying %d bright object masks to %s", len(brightObjectMasks), dataId)
1087 mask = exposure.getMaskedImage().getMask()
1088 wcs = exposure.getWcs()
1089 plateScale = wcs.getPixelScale().asArcseconds()
1091 for rec in brightObjectMasks:
1092 center = geom.PointI(wcs.skyToPixel(rec.getCoord()))
1093 if rec["type"] == "box":
1094 assert rec["angle"] == 0.0, "Angle != 0 for mask object %s" % rec["id"]
1095 width = rec["width"].asArcseconds() / plateScale # convert to pixels
1096 height = rec["height"].asArcseconds() / plateScale # convert to pixels
1098 halfSize = geom.ExtentI(0.5 * width, 0.5 * height)
1099 bbox = geom.Box2I(center - halfSize, center + halfSize)
1101 bbox = geom.BoxI(
1102 geom.PointI(int(center[0] - 0.5 * width), int(center[1] - 0.5 * height)),
1103 geom.PointI(int(center[0] + 0.5 * width), int(center[1] + 0.5 * height)),
1104 )
1105 spans = afwGeom.SpanSet(bbox)
1106 elif rec["type"] == "circle":
1107 radius = int(rec["radius"].asArcseconds() / plateScale) # convert to pixels
1108 spans = afwGeom.SpanSet.fromShape(radius, offset=center)
1109 else:
1110 self.log.warning("Unexpected region type %s at %s", rec["type"], center)
1111 continue
1112 spans.clippedTo(mask.getBBox()).setMask(mask, self.brightObjectBitmask)
1114 def setInexactPsf(self, mask):
1115 """Set INEXACT_PSF mask plane.
1117 If any of the input images isn't represented in the coadd (due to
1118 clipped pixels or chip gaps), the `CoaddPsf` will be inexact. Flag
1119 these pixels.
1121 Parameters
1122 ----------
1123 mask : `lsst.afw.image.Mask`
1124 Coadded exposure's mask, modified in-place.
1125 """
1126 mask.addMaskPlane("INEXACT_PSF")
1127 inexactPsf = mask.getPlaneBitMask("INEXACT_PSF")
1128 sensorEdge = mask.getPlaneBitMask("SENSOR_EDGE") # chip edges (so PSF is discontinuous)
1129 clipped = mask.getPlaneBitMask("CLIPPED") # pixels clipped from coadd
1130 rejected = mask.getPlaneBitMask("REJECTED") # pixels rejected from coadd due to masks
1131 array = mask.getArray()
1132 selected = array & (sensorEdge | clipped | rejected) > 0
1133 array[selected] |= inexactPsf
1135 def filterWarps(self, inputs, goodVisits):
1136 """Return list of only inputRefs with visitId in goodVisits ordered by
1137 goodVisit.
1139 Parameters
1140 ----------
1141 inputs : `list` of `~lsst.pipe.base.connections.DeferredDatasetRef`
1142 List of `lsst.pipe.base.connections.DeferredDatasetRef` with dataId
1143 containing visit.
1144 goodVisit : `dict`
1145 Dictionary with good visitIds as the keys. Value ignored.
1147 Returns
1148 -------
1149 filterInputs : `list` [`lsst.pipe.base.connections.DeferredDatasetRef`]
1150 Filtered and sorted list of inputRefs with visitId in goodVisits
1151 ordered by goodVisit.
1152 """
1153 inputWarpDict = {inputRef.ref.dataId["visit"]: inputRef for inputRef in inputs}
1154 filteredInputs = []
1155 for visit in goodVisits.keys():
1156 if visit in inputWarpDict:
1157 filteredInputs.append(inputWarpDict[visit])
1158 return filteredInputs
1161def countMaskFromFootprint(mask, footprint, bitmask, ignoreMask):
1162 """Function to count the number of pixels with a specific mask in a
1163 footprint.
1165 Find the intersection of mask & footprint. Count all pixels in the mask
1166 that are in the intersection that have bitmask set but do not have
1167 ignoreMask set. Return the count.
1169 Parameters
1170 ----------
1171 mask : `lsst.afw.image.Mask`
1172 Mask to define intersection region by.
1173 footprint : `lsst.afw.detection.Footprint`
1174 Footprint to define the intersection region by.
1175 bitmask : `Unknown`
1176 Specific mask that we wish to count the number of occurances of.
1177 ignoreMask : `Unknown`
1178 Pixels to not consider.
1180 Returns
1181 -------
1182 result : `int`
1183 Number of pixels in footprint with specified mask.
1184 """
1185 bbox = footprint.getBBox()
1186 bbox.clip(mask.getBBox(afwImage.PARENT))
1187 fp = afwImage.Mask(bbox)
1188 subMask = mask.Factory(mask, bbox, afwImage.PARENT)
1189 footprint.spans.setMask(fp, bitmask)
1190 return numpy.logical_and(
1191 (subMask.getArray() & fp.getArray()) > 0, (subMask.getArray() & ignoreMask) == 0
1192 ).sum()
1195class CompareWarpAssembleCoaddConnections(AssembleCoaddConnections):
1196 psfMatchedWarps = pipeBase.connectionTypes.Input(
1197 doc=(
1198 "PSF-Matched Warps are required by CompareWarp regardless of the coadd type requested. "
1199 "Only PSF-Matched Warps make sense for image subtraction. "
1200 "Therefore, they must be an additional declared input."
1201 ),
1202 name="{inputCoaddName}Coadd_psfMatchedWarp",
1203 storageClass="ExposureF",
1204 dimensions=("tract", "patch", "skymap", "visit"),
1205 deferLoad=True,
1206 multiple=True,
1207 )
1208 templateCoadd = pipeBase.connectionTypes.Output(
1209 doc=(
1210 "Model of the static sky, used to find temporal artifacts. Typically a PSF-Matched, "
1211 "sigma-clipped coadd. Written if and only if assembleStaticSkyModel.doWrite=True"
1212 ),
1213 name="{outputCoaddName}CoaddPsfMatched",
1214 storageClass="ExposureF",
1215 dimensions=("tract", "patch", "skymap", "band"),
1216 )
1218 def __init__(self, *, config=None):
1219 super().__init__(config=config)
1220 if not config.assembleStaticSkyModel.doWrite:
1221 self.outputs.remove("templateCoadd")
1222 config.validate()
1225class CompareWarpAssembleCoaddConfig(
1226 AssembleCoaddConfig, pipelineConnections=CompareWarpAssembleCoaddConnections
1227):
1228 assembleStaticSkyModel = pexConfig.ConfigurableField(
1229 target=AssembleCoaddTask,
1230 doc="Task to assemble an artifact-free, PSF-matched Coadd to serve as "
1231 "a naive/first-iteration model of the static sky.",
1232 )
1233 detect = pexConfig.ConfigurableField(
1234 target=SourceDetectionTask,
1235 doc="Detect outlier sources on difference between each psfMatched warp and static sky model",
1236 )
1237 detectTemplate = pexConfig.ConfigurableField(
1238 target=SourceDetectionTask,
1239 doc="Detect sources on static sky model. Only used if doPreserveContainedBySource is True",
1240 )
1241 maskStreaks = pexConfig.ConfigurableField(
1242 target=MaskStreaksTask,
1243 doc="Detect streaks on difference between each psfMatched warp and static sky model. Only used if "
1244 "doFilterMorphological is True. Adds a mask plane to an exposure, with the mask plane name set by"
1245 "streakMaskName",
1246 )
1247 streakMaskName = pexConfig.Field(dtype=str, default="STREAK", doc="Name of mask bit used for streaks")
1248 maxNumEpochs = pexConfig.Field(
1249 doc="Charactistic maximum local number of epochs/visits in which an artifact candidate can appear "
1250 "and still be masked. The effective maxNumEpochs is a broken linear function of local "
1251 "number of epochs (N): min(maxFractionEpochsLow*N, maxNumEpochs + maxFractionEpochsHigh*N). "
1252 "For each footprint detected on the image difference between the psfMatched warp and static sky "
1253 "model, if a significant fraction of pixels (defined by spatialThreshold) are residuals in more "
1254 "than the computed effective maxNumEpochs, the artifact candidate is deemed persistant rather "
1255 "than transient and not masked.",
1256 dtype=int,
1257 default=2,
1258 )
1259 maxFractionEpochsLow = pexConfig.RangeField(
1260 doc="Fraction of local number of epochs (N) to use as effective maxNumEpochs for low N. "
1261 "Effective maxNumEpochs = "
1262 "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)",
1263 dtype=float,
1264 default=0.4,
1265 min=0.0,
1266 max=1.0,
1267 )
1268 maxFractionEpochsHigh = pexConfig.RangeField(
1269 doc="Fraction of local number of epochs (N) to use as effective maxNumEpochs for high N. "
1270 "Effective maxNumEpochs = "
1271 "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)",
1272 dtype=float,
1273 default=0.03,
1274 min=0.0,
1275 max=1.0,
1276 )
1277 spatialThreshold = pexConfig.RangeField(
1278 doc="Unitless fraction of pixels defining how much of the outlier region has to meet the "
1279 "temporal criteria. If 0, clip all. If 1, clip none.",
1280 dtype=float,
1281 default=0.5,
1282 min=0.0,
1283 max=1.0,
1284 inclusiveMin=True,
1285 inclusiveMax=True,
1286 )
1287 doScaleWarpVariance = pexConfig.Field(
1288 doc="Rescale Warp variance plane using empirical noise?",
1289 dtype=bool,
1290 default=True,
1291 )
1292 scaleWarpVariance = pexConfig.ConfigurableField(
1293 target=ScaleVarianceTask,
1294 doc="Rescale variance on warps",
1295 )
1296 doPreserveContainedBySource = pexConfig.Field(
1297 doc="Rescue artifacts from clipping that completely lie within a footprint detected"
1298 "on the PsfMatched Template Coadd. Replicates a behavior of SafeClip.",
1299 dtype=bool,
1300 default=True,
1301 )
1302 doPrefilterArtifacts = pexConfig.Field(
1303 doc="Ignore artifact candidates that are mostly covered by the bad pixel mask, "
1304 "because they will be excluded anyway. This prevents them from contributing "
1305 "to the outlier epoch count image and potentially being labeled as persistant."
1306 "'Mostly' is defined by the config 'prefilterArtifactsRatio'.",
1307 dtype=bool,
1308 default=True,
1309 )
1310 prefilterArtifactsMaskPlanes = pexConfig.ListField(
1311 doc="Prefilter artifact candidates that are mostly covered by these bad mask planes.",
1312 dtype=str,
1313 default=("NO_DATA", "BAD", "SAT", "SUSPECT"),
1314 )
1315 prefilterArtifactsRatio = pexConfig.Field(
1316 doc="Prefilter artifact candidates with less than this fraction overlapping good pixels",
1317 dtype=float,
1318 default=0.05,
1319 )
1320 doFilterMorphological = pexConfig.Field(
1321 doc="Filter artifact candidates based on morphological criteria, i.g. those that appear to "
1322 "be streaks.",
1323 dtype=bool,
1324 default=False,
1325 )
1326 growStreakFp = pexConfig.Field(
1327 doc="Grow streak footprints by this number multiplied by the PSF width", dtype=float, default=5
1328 )
1330 def setDefaults(self):
1331 AssembleCoaddConfig.setDefaults(self)
1332 self.statistic = "MEAN"
1333 self.doUsePsfMatchedPolygons = True
1335 # Real EDGE removed by psfMatched NO_DATA border half the width of the
1336 # matching kernel. CompareWarp applies psfMatched EDGE pixels to
1337 # directWarps before assembling.
1338 if "EDGE" in self.badMaskPlanes:
1339 self.badMaskPlanes.remove("EDGE")
1340 self.removeMaskPlanes.append("EDGE")
1341 self.assembleStaticSkyModel.badMaskPlanes = [
1342 "NO_DATA",
1343 ]
1344 self.assembleStaticSkyModel.warpType = "psfMatched"
1345 self.assembleStaticSkyModel.connections.warpType = "psfMatched"
1346 self.assembleStaticSkyModel.statistic = "MEANCLIP"
1347 self.assembleStaticSkyModel.sigmaClip = 2.5
1348 self.assembleStaticSkyModel.clipIter = 3
1349 self.assembleStaticSkyModel.calcErrorFromInputVariance = False
1350 self.assembleStaticSkyModel.doWrite = False
1351 self.detect.doTempLocalBackground = False
1352 self.detect.reEstimateBackground = False
1353 self.detect.returnOriginalFootprints = False
1354 self.detect.thresholdPolarity = "both"
1355 self.detect.thresholdValue = 5
1356 self.detect.minPixels = 4
1357 self.detect.isotropicGrow = True
1358 self.detect.thresholdType = "pixel_stdev"
1359 self.detect.nSigmaToGrow = 0.4
1360 # The default nSigmaToGrow for SourceDetectionTask is already 2.4,
1361 # Explicitly restating because ratio with detect.nSigmaToGrow matters
1362 self.detectTemplate.nSigmaToGrow = 2.4
1363 self.detectTemplate.doTempLocalBackground = False
1364 self.detectTemplate.reEstimateBackground = False
1365 self.detectTemplate.returnOriginalFootprints = False
1367 def validate(self):
1368 super().validate()
1369 if self.assembleStaticSkyModel.doNImage:
1370 raise ValueError(
1371 "No dataset type exists for a PSF-Matched Template N Image."
1372 "Please set assembleStaticSkyModel.doNImage=False"
1373 )
1375 if self.assembleStaticSkyModel.doWrite and (self.warpType == self.assembleStaticSkyModel.warpType):
1376 raise ValueError(
1377 "warpType (%s) == assembleStaticSkyModel.warpType (%s) and will compete for "
1378 "the same dataset name. Please set assembleStaticSkyModel.doWrite to False "
1379 "or warpType to 'direct'. assembleStaticSkyModel.warpType should ways be "
1380 "'PsfMatched'" % (self.warpType, self.assembleStaticSkyModel.warpType)
1381 )
1384class CompareWarpAssembleCoaddTask(AssembleCoaddTask):
1385 """Assemble a compareWarp coadded image from a set of warps
1386 by masking artifacts detected by comparing PSF-matched warps.
1388 In ``AssembleCoaddTask``, we compute the coadd as an clipped mean (i.e.,
1389 we clip outliers). The problem with doing this is that when computing the
1390 coadd PSF at a given location, individual visit PSFs from visits with
1391 outlier pixels contribute to the coadd PSF and cannot be treated correctly.
1392 In this task, we correct for this behavior by creating a new badMaskPlane
1393 'CLIPPED' which marks pixels in the individual warps suspected to contain
1394 an artifact. We populate this plane on the input warps by comparing
1395 PSF-matched warps with a PSF-matched median coadd which serves as a
1396 model of the static sky. Any group of pixels that deviates from the
1397 PSF-matched template coadd by more than config.detect.threshold sigma,
1398 is an artifact candidate. The candidates are then filtered to remove
1399 variable sources and sources that are difficult to subtract such as
1400 bright stars. This filter is configured using the config parameters
1401 ``temporalThreshold`` and ``spatialThreshold``. The temporalThreshold is
1402 the maximum fraction of epochs that the deviation can appear in and still
1403 be considered an artifact. The spatialThreshold is the maximum fraction of
1404 pixels in the footprint of the deviation that appear in other epochs
1405 (where other epochs is defined by the temporalThreshold). If the deviant
1406 region meets this criteria of having a significant percentage of pixels
1407 that deviate in only a few epochs, these pixels have the 'CLIPPED' bit
1408 set in the mask. These regions will not contribute to the final coadd.
1409 Furthermore, any routine to determine the coadd PSF can now be cognizant
1410 of clipped regions. Note that the algorithm implemented by this task is
1411 preliminary and works correctly for HSC data. Parameter modifications and
1412 or considerable redesigning of the algorithm is likley required for other
1413 surveys.
1415 ``CompareWarpAssembleCoaddTask`` sub-classes
1416 ``AssembleCoaddTask`` and instantiates ``AssembleCoaddTask``
1417 as a subtask to generate the TemplateCoadd (the model of the static sky).
1419 Notes
1420 -----
1421 Debugging:
1422 This task supports the following debug variables:
1423 - ``saveCountIm``
1424 If True then save the Epoch Count Image as a fits file in the `figPath`
1425 - ``figPath``
1426 Path to save the debug fits images and figures
1427 """
1429 ConfigClass = CompareWarpAssembleCoaddConfig
1430 _DefaultName = "compareWarpAssembleCoadd"
1432 def __init__(self, *args, **kwargs):
1433 AssembleCoaddTask.__init__(self, *args, **kwargs)
1434 self.makeSubtask("assembleStaticSkyModel")
1435 detectionSchema = afwTable.SourceTable.makeMinimalSchema()
1436 self.makeSubtask("detect", schema=detectionSchema)
1437 if self.config.doPreserveContainedBySource:
1438 self.makeSubtask("detectTemplate", schema=afwTable.SourceTable.makeMinimalSchema())
1439 if self.config.doScaleWarpVariance:
1440 self.makeSubtask("scaleWarpVariance")
1441 if self.config.doFilterMorphological:
1442 self.makeSubtask("maskStreaks")
1444 @utils.inheritDoc(AssembleCoaddTask)
1445 def _makeSupplementaryData(self, butlerQC, inputRefs, outputRefs):
1446 """Generate a templateCoadd to use as a naive model of static sky to
1447 subtract from PSF-Matched warps.
1449 Returns
1450 -------
1451 result : `~lsst.pipe.base.Struct`
1452 Results as a struct with attributes:
1454 ``templateCoadd``
1455 Coadded exposure (`lsst.afw.image.Exposure`).
1456 ``nImage``
1457 Keeps track of exposure count for each pixel
1458 (`lsst.afw.image.ImageU`).
1460 Raises
1461 ------
1462 RuntimeError
1463 Raised if ``templateCoadd`` is `None`.
1464 """
1465 # Ensure that psfMatchedWarps are used as input warps for template
1466 # generation.
1467 staticSkyModelInputRefs = copy.deepcopy(inputRefs)
1468 staticSkyModelInputRefs.inputWarps = inputRefs.psfMatchedWarps
1470 # Because subtasks don't have connections we have to make one.
1471 # The main task's `templateCoadd` is the subtask's `coaddExposure`
1472 staticSkyModelOutputRefs = copy.deepcopy(outputRefs)
1473 if self.config.assembleStaticSkyModel.doWrite:
1474 staticSkyModelOutputRefs.coaddExposure = staticSkyModelOutputRefs.templateCoadd
1475 # Remove template coadd from both subtask's and main tasks outputs,
1476 # because it is handled by the subtask as `coaddExposure`
1477 del outputRefs.templateCoadd
1478 del staticSkyModelOutputRefs.templateCoadd
1480 # A PSF-Matched nImage does not exist as a dataset type
1481 if "nImage" in staticSkyModelOutputRefs.keys():
1482 del staticSkyModelOutputRefs.nImage
1484 templateCoadd = self.assembleStaticSkyModel.runQuantum(
1485 butlerQC, staticSkyModelInputRefs, staticSkyModelOutputRefs
1486 )
1487 if templateCoadd is None:
1488 raise RuntimeError(self._noTemplateMessage(self.assembleStaticSkyModel.warpType))
1490 return pipeBase.Struct(
1491 templateCoadd=templateCoadd.coaddExposure,
1492 nImage=templateCoadd.nImage,
1493 warpRefList=templateCoadd.warpRefList,
1494 imageScalerList=templateCoadd.imageScalerList,
1495 weightList=templateCoadd.weightList,
1496 )
1498 def _noTemplateMessage(self, warpType):
1499 warpName = warpType[0].upper() + warpType[1:]
1500 message = """No %(warpName)s warps were found to build the template coadd which is
1501 required to run CompareWarpAssembleCoaddTask. To continue assembling this type of coadd,
1502 first either rerun makeCoaddTempExp with config.make%(warpName)s=True or
1503 coaddDriver with config.makeCoadTempExp.make%(warpName)s=True, before assembleCoadd.
1505 Alternatively, to use another algorithm with existing warps, retarget the CoaddDriverConfig to
1506 another algorithm like:
1508 from lsst.pipe.tasks.assembleCoadd import SafeClipAssembleCoaddTask
1509 config.assemble.retarget(SafeClipAssembleCoaddTask)
1510 """ % {
1511 "warpName": warpName
1512 }
1513 return message
1515 @utils.inheritDoc(AssembleCoaddTask)
1516 @timeMethod
1517 def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, supplementaryData):
1518 """Notes
1519 -----
1520 Assemble the coadd.
1522 Find artifacts and apply them to the warps' masks creating a list of
1523 alternative masks with a new "CLIPPED" plane and updated "NO_DATA"
1524 plane. Then pass these alternative masks to the base class's ``run``
1525 method.
1526 """
1527 # Check and match the order of the supplementaryData
1528 # (PSF-matched) inputs to the order of the direct inputs,
1529 # so that the artifact mask is applied to the right warp
1530 dataIds = [ref.dataId for ref in tempExpRefList]
1531 psfMatchedDataIds = [ref.dataId for ref in supplementaryData.warpRefList]
1533 if dataIds != psfMatchedDataIds:
1534 self.log.info("Reordering and or/padding PSF-matched visit input list")
1535 supplementaryData.warpRefList = reorderAndPadList(
1536 supplementaryData.warpRefList, psfMatchedDataIds, dataIds
1537 )
1538 supplementaryData.imageScalerList = reorderAndPadList(
1539 supplementaryData.imageScalerList, psfMatchedDataIds, dataIds
1540 )
1542 # Use PSF-Matched Warps (and corresponding scalers) and coadd to find
1543 # artifacts.
1544 spanSetMaskList = self.findArtifacts(
1545 supplementaryData.templateCoadd, supplementaryData.warpRefList, supplementaryData.imageScalerList
1546 )
1548 badMaskPlanes = self.config.badMaskPlanes[:]
1549 badMaskPlanes.append("CLIPPED")
1550 badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes)
1552 result = AssembleCoaddTask.run(
1553 self, skyInfo, tempExpRefList, imageScalerList, weightList, spanSetMaskList, mask=badPixelMask
1554 )
1556 # Propagate PSF-matched EDGE pixels to coadd SENSOR_EDGE and
1557 # INEXACT_PSF. Psf-Matching moves the real edge inwards.
1558 self.applyAltEdgeMask(result.coaddExposure.maskedImage.mask, spanSetMaskList)
1559 return result
1561 def applyAltEdgeMask(self, mask, altMaskList):
1562 """Propagate alt EDGE mask to SENSOR_EDGE AND INEXACT_PSF planes.
1564 Parameters
1565 ----------
1566 mask : `lsst.afw.image.Mask`
1567 Original mask.
1568 altMaskList : `list` of `dict`
1569 List of Dicts containing ``spanSet`` lists.
1570 Each element contains the new mask plane name (e.g. "CLIPPED
1571 and/or "NO_DATA") as the key, and list of ``SpanSets`` to apply to
1572 the mask.
1573 """
1574 maskValue = mask.getPlaneBitMask(["SENSOR_EDGE", "INEXACT_PSF"])
1575 for visitMask in altMaskList:
1576 if "EDGE" in visitMask:
1577 for spanSet in visitMask["EDGE"]:
1578 spanSet.clippedTo(mask.getBBox()).setMask(mask, maskValue)
1580 def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList):
1581 """Find artifacts.
1583 Loop through warps twice. The first loop builds a map with the count
1584 of how many epochs each pixel deviates from the templateCoadd by more
1585 than ``config.chiThreshold`` sigma. The second loop takes each
1586 difference image and filters the artifacts detected in each using
1587 count map to filter out variable sources and sources that are
1588 difficult to subtract cleanly.
1590 Parameters
1591 ----------
1592 templateCoadd : `lsst.afw.image.Exposure`
1593 Exposure to serve as model of static sky.
1594 tempExpRefList : `list`
1595 List of data references to warps.
1596 imageScalerList : `list`
1597 List of image scalers.
1599 Returns
1600 -------
1601 altMasks : `list` of `dict`
1602 List of dicts containing information about CLIPPED
1603 (i.e., artifacts), NO_DATA, and EDGE pixels.
1604 """
1605 self.log.debug("Generating Count Image, and mask lists.")
1606 coaddBBox = templateCoadd.getBBox()
1607 slateIm = afwImage.ImageU(coaddBBox)
1608 epochCountImage = afwImage.ImageU(coaddBBox)
1609 nImage = afwImage.ImageU(coaddBBox)
1610 spanSetArtifactList = []
1611 spanSetNoDataMaskList = []
1612 spanSetEdgeList = []
1613 spanSetBadMorphoList = []
1614 badPixelMask = self.getBadPixelMask()
1616 # mask of the warp diffs should = that of only the warp
1617 templateCoadd.mask.clearAllMaskPlanes()
1619 if self.config.doPreserveContainedBySource:
1620 templateFootprints = self.detectTemplate.detectFootprints(templateCoadd)
1621 else:
1622 templateFootprints = None
1624 for warpRef, imageScaler in zip(tempExpRefList, imageScalerList):
1625 warpDiffExp = self._readAndComputeWarpDiff(warpRef, imageScaler, templateCoadd)
1626 if warpDiffExp is not None:
1627 # This nImage only approximates the final nImage because it
1628 # uses the PSF-matched mask.
1629 nImage.array += (
1630 numpy.isfinite(warpDiffExp.image.array) * ((warpDiffExp.mask.array & badPixelMask) == 0)
1631 ).astype(numpy.uint16)
1632 fpSet = self.detect.detectFootprints(warpDiffExp, doSmooth=False, clearMask=True)
1633 fpSet.positive.merge(fpSet.negative)
1634 footprints = fpSet.positive
1635 slateIm.set(0)
1636 spanSetList = [footprint.spans for footprint in footprints.getFootprints()]
1638 # Remove artifacts due to defects before they contribute to
1639 # the epochCountImage.
1640 if self.config.doPrefilterArtifacts:
1641 spanSetList = self.prefilterArtifacts(spanSetList, warpDiffExp)
1643 # Clear mask before adding prefiltered spanSets
1644 self.detect.clearMask(warpDiffExp.mask)
1645 for spans in spanSetList:
1646 spans.setImage(slateIm, 1, doClip=True)
1647 spans.setMask(warpDiffExp.mask, warpDiffExp.mask.getPlaneBitMask("DETECTED"))
1648 epochCountImage += slateIm
1650 if self.config.doFilterMorphological:
1651 maskName = self.config.streakMaskName
1652 _ = self.maskStreaks.run(warpDiffExp)
1653 streakMask = warpDiffExp.mask
1654 spanSetStreak = afwGeom.SpanSet.fromMask(
1655 streakMask, streakMask.getPlaneBitMask(maskName)
1656 ).split()
1657 # Pad the streaks to account for low-surface brightness
1658 # wings.
1659 psf = warpDiffExp.getPsf()
1660 for s, sset in enumerate(spanSetStreak):
1661 psfShape = psf.computeShape(sset.computeCentroid())
1662 dilation = self.config.growStreakFp * psfShape.getDeterminantRadius()
1663 sset_dilated = sset.dilated(int(dilation))
1664 spanSetStreak[s] = sset_dilated
1666 # PSF-Matched warps have less available area (~the matching
1667 # kernel) because the calexps undergo a second convolution.
1668 # Pixels with data in the direct warp but not in the
1669 # PSF-matched warp will not have their artifacts detected.
1670 # NaNs from the PSF-matched warp therefore must be masked in
1671 # the direct warp.
1672 nans = numpy.where(numpy.isnan(warpDiffExp.maskedImage.image.array), 1, 0)
1673 nansMask = afwImage.makeMaskFromArray(nans.astype(afwImage.MaskPixel))
1674 nansMask.setXY0(warpDiffExp.getXY0())
1675 edgeMask = warpDiffExp.mask
1676 spanSetEdgeMask = afwGeom.SpanSet.fromMask(edgeMask, edgeMask.getPlaneBitMask("EDGE")).split()
1677 else:
1678 # If the directWarp has <1% coverage, the psfMatchedWarp can
1679 # have 0% and not exist. In this case, mask the whole epoch.
1680 nansMask = afwImage.MaskX(coaddBBox, 1)
1681 spanSetList = []
1682 spanSetEdgeMask = []
1683 spanSetStreak = []
1685 spanSetNoDataMask = afwGeom.SpanSet.fromMask(nansMask).split()
1687 spanSetNoDataMaskList.append(spanSetNoDataMask)
1688 spanSetArtifactList.append(spanSetList)
1689 spanSetEdgeList.append(spanSetEdgeMask)
1690 if self.config.doFilterMorphological:
1691 spanSetBadMorphoList.append(spanSetStreak)
1693 if lsstDebug.Info(__name__).saveCountIm:
1694 path = self._dataRef2DebugPath("epochCountIm", tempExpRefList[0], coaddLevel=True)
1695 epochCountImage.writeFits(path)
1697 for i, spanSetList in enumerate(spanSetArtifactList):
1698 if spanSetList:
1699 filteredSpanSetList = self.filterArtifacts(
1700 spanSetList, epochCountImage, nImage, templateFootprints
1701 )
1702 spanSetArtifactList[i] = filteredSpanSetList
1703 if self.config.doFilterMorphological:
1704 spanSetArtifactList[i] += spanSetBadMorphoList[i]
1706 altMasks = []
1707 for artifacts, noData, edge in zip(spanSetArtifactList, spanSetNoDataMaskList, spanSetEdgeList):
1708 altMasks.append({"CLIPPED": artifacts, "NO_DATA": noData, "EDGE": edge})
1709 return altMasks
1711 def prefilterArtifacts(self, spanSetList, exp):
1712 """Remove artifact candidates covered by bad mask plane.
1714 Any future editing of the candidate list that does not depend on
1715 temporal information should go in this method.
1717 Parameters
1718 ----------
1719 spanSetList : `list` [`lsst.afw.geom.SpanSet`]
1720 List of SpanSets representing artifact candidates.
1721 exp : `lsst.afw.image.Exposure`
1722 Exposure containing mask planes used to prefilter.
1724 Returns
1725 -------
1726 returnSpanSetList : `list` [`lsst.afw.geom.SpanSet`]
1727 List of SpanSets with artifacts.
1728 """
1729 badPixelMask = exp.mask.getPlaneBitMask(self.config.prefilterArtifactsMaskPlanes)
1730 goodArr = (exp.mask.array & badPixelMask) == 0
1731 returnSpanSetList = []
1732 bbox = exp.getBBox()
1733 x0, y0 = exp.getXY0()
1734 for i, span in enumerate(spanSetList):
1735 y, x = span.clippedTo(bbox).indices()
1736 yIndexLocal = numpy.array(y) - y0
1737 xIndexLocal = numpy.array(x) - x0
1738 goodRatio = numpy.count_nonzero(goodArr[yIndexLocal, xIndexLocal]) / span.getArea()
1739 if goodRatio > self.config.prefilterArtifactsRatio:
1740 returnSpanSetList.append(span)
1741 return returnSpanSetList
1743 def filterArtifacts(self, spanSetList, epochCountImage, nImage, footprintsToExclude=None):
1744 """Filter artifact candidates.
1746 Parameters
1747 ----------
1748 spanSetList : `list` [`lsst.afw.geom.SpanSet`]
1749 List of SpanSets representing artifact candidates.
1750 epochCountImage : `lsst.afw.image.Image`
1751 Image of accumulated number of warpDiff detections.
1752 nImage : `lsst.afw.image.ImageU`
1753 Image of the accumulated number of total epochs contributing.
1755 Returns
1756 -------
1757 maskSpanSetList : `list` [`lsst.afw.geom.SpanSet`]
1758 List of SpanSets with artifacts.
1759 """
1760 maskSpanSetList = []
1761 x0, y0 = epochCountImage.getXY0()
1762 for i, span in enumerate(spanSetList):
1763 y, x = span.indices()
1764 yIdxLocal = [y1 - y0 for y1 in y]
1765 xIdxLocal = [x1 - x0 for x1 in x]
1766 outlierN = epochCountImage.array[yIdxLocal, xIdxLocal]
1767 totalN = nImage.array[yIdxLocal, xIdxLocal]
1769 # effectiveMaxNumEpochs is broken line (fraction of N) with
1770 # characteristic config.maxNumEpochs.
1771 effMaxNumEpochsHighN = self.config.maxNumEpochs + self.config.maxFractionEpochsHigh * numpy.mean(
1772 totalN
1773 )
1774 effMaxNumEpochsLowN = self.config.maxFractionEpochsLow * numpy.mean(totalN)
1775 effectiveMaxNumEpochs = int(min(effMaxNumEpochsLowN, effMaxNumEpochsHighN))
1776 nPixelsBelowThreshold = numpy.count_nonzero((outlierN > 0) & (outlierN <= effectiveMaxNumEpochs))
1777 percentBelowThreshold = nPixelsBelowThreshold / len(outlierN)
1778 if percentBelowThreshold > self.config.spatialThreshold:
1779 maskSpanSetList.append(span)
1781 if self.config.doPreserveContainedBySource and footprintsToExclude is not None:
1782 # If a candidate is contained by a footprint on the template coadd,
1783 # do not clip.
1784 filteredMaskSpanSetList = []
1785 for span in maskSpanSetList:
1786 doKeep = True
1787 for footprint in footprintsToExclude.positive.getFootprints():
1788 if footprint.spans.contains(span):
1789 doKeep = False
1790 break
1791 if doKeep:
1792 filteredMaskSpanSetList.append(span)
1793 maskSpanSetList = filteredMaskSpanSetList
1795 return maskSpanSetList
1797 def _readAndComputeWarpDiff(self, warpRef, imageScaler, templateCoadd):
1798 """Fetch a warp from the butler and return a warpDiff.
1800 Parameters
1801 ----------
1802 warpRef : `lsst.daf.butler.DeferredDatasetHandle`
1803 Handle for the warp.
1804 imageScaler : `lsst.pipe.tasks.scaleZeroPoint.ImageScaler`
1805 An image scaler object.
1806 templateCoadd : `lsst.afw.image.Exposure`
1807 Exposure to be substracted from the scaled warp.
1809 Returns
1810 -------
1811 warp : `lsst.afw.image.Exposure`
1812 Exposure of the image difference between the warp and template.
1813 """
1814 # If the PSF-Matched warp did not exist for this direct warp
1815 # None is holding its place to maintain order in Gen 3
1816 if warpRef is None:
1817 return None
1819 warp = warpRef.get()
1820 # direct image scaler OK for PSF-matched Warp
1821 imageScaler.scaleMaskedImage(warp.getMaskedImage())
1822 mi = warp.getMaskedImage()
1823 if self.config.doScaleWarpVariance:
1824 try:
1825 self.scaleWarpVariance.run(mi)
1826 except Exception as exc:
1827 self.log.warning("Unable to rescale variance of warp (%s); leaving it as-is", exc)
1828 mi -= templateCoadd.getMaskedImage()
1829 return warp