Coverage for python/lsst/pipe/tasks/assembleCoadd.py: 16%
643 statements
« prev ^ index » next coverage.py v6.4.4, created at 2022-08-26 03:18 -0700
« prev ^ index » next coverage.py v6.4.4, created at 2022-08-26 03:18 -0700
1# This file is part of pipe_tasks.
2#
3# LSST Data Management System
4# This product includes software developed by the
5# LSST Project (http://www.lsst.org/).
6# See COPYRIGHT file at the top of the source tree.
7#
8# This program is free software: you can redistribute it and/or modify
9# it under the terms of the GNU General Public License as published by
10# the Free Software Foundation, either version 3 of the License, or
11# (at your option) any later version.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the LSST License Statement and
19# the GNU General Public License along with this program. If not,
20# see <https://www.lsstcorp.org/LegalNotices/>.
21#
22import copy
23import numpy
24import warnings
25import logging
26import lsst.pex.config as pexConfig
27import lsst.pex.exceptions as pexExceptions
28import lsst.geom as geom
29import lsst.afw.geom as afwGeom
30import lsst.afw.image as afwImage
31import lsst.afw.math as afwMath
32import lsst.afw.table as afwTable
33import lsst.coadd.utils as coaddUtils
34import lsst.pipe.base as pipeBase
35import lsst.meas.algorithms as measAlg
36import lsstDebug
37import lsst.utils as utils
38from lsst.skymap import BaseSkyMap
39from .coaddBase import CoaddBaseTask, makeSkyInfo, reorderAndPadList
40from .interpImage import InterpImageTask
41from .scaleZeroPoint import ScaleZeroPointTask
42from .maskStreaks import MaskStreaksTask
43from .healSparseMapping import HealSparseInputMapTask
44from lsst.meas.algorithms import SourceDetectionTask, AccumulatorMeanStack, ScaleVarianceTask
45from lsst.utils.timer import timeMethod
46from deprecated.sphinx import deprecated
48__all__ = ["AssembleCoaddTask", "AssembleCoaddConnections", "AssembleCoaddConfig",
49 "CompareWarpAssembleCoaddTask", "CompareWarpAssembleCoaddConfig"]
51log = logging.getLogger(__name__)
54class AssembleCoaddConnections(pipeBase.PipelineTaskConnections,
55 dimensions=("tract", "patch", "band", "skymap"),
56 defaultTemplates={"inputCoaddName": "deep",
57 "outputCoaddName": "deep",
58 "warpType": "direct",
59 "warpTypeSuffix": ""}):
61 inputWarps = pipeBase.connectionTypes.Input(
62 doc=("Input list of warps to be assemebled i.e. stacked."
63 "WarpType (e.g. direct, psfMatched) is controlled by the warpType config parameter"),
64 name="{inputCoaddName}Coadd_{warpType}Warp",
65 storageClass="ExposureF",
66 dimensions=("tract", "patch", "skymap", "visit", "instrument"),
67 deferLoad=True,
68 multiple=True
69 )
70 skyMap = pipeBase.connectionTypes.Input(
71 doc="Input definition of geometry/bbox and projection/wcs for coadded exposures",
72 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
73 storageClass="SkyMap",
74 dimensions=("skymap", ),
75 )
76 selectedVisits = pipeBase.connectionTypes.Input(
77 doc="Selected visits to be coadded.",
78 name="{outputCoaddName}Visits",
79 storageClass="StructuredDataDict",
80 dimensions=("instrument", "tract", "patch", "skymap", "band")
81 )
82 brightObjectMask = pipeBase.connectionTypes.PrerequisiteInput(
83 doc=("Input Bright Object Mask mask produced with external catalogs to be applied to the mask plane"
84 " BRIGHT_OBJECT."),
85 name="brightObjectMask",
86 storageClass="ObjectMaskCatalog",
87 dimensions=("tract", "patch", "skymap", "band"),
88 )
89 coaddExposure = pipeBase.connectionTypes.Output(
90 doc="Output coadded exposure, produced by stacking input warps",
91 name="{outputCoaddName}Coadd{warpTypeSuffix}",
92 storageClass="ExposureF",
93 dimensions=("tract", "patch", "skymap", "band"),
94 )
95 nImage = pipeBase.connectionTypes.Output(
96 doc="Output image of number of input images per pixel",
97 name="{outputCoaddName}Coadd_nImage",
98 storageClass="ImageU",
99 dimensions=("tract", "patch", "skymap", "band"),
100 )
101 inputMap = pipeBase.connectionTypes.Output(
102 doc="Output healsparse map of input images",
103 name="{outputCoaddName}Coadd_inputMap",
104 storageClass="HealSparseMap",
105 dimensions=("tract", "patch", "skymap", "band"),
106 )
108 def __init__(self, *, config=None):
109 super().__init__(config=config)
111 if not config.doMaskBrightObjects:
112 self.prerequisiteInputs.remove("brightObjectMask")
114 if not config.doSelectVisits:
115 self.inputs.remove("selectedVisits")
117 if not config.doNImage:
118 self.outputs.remove("nImage")
120 if not self.config.doInputMap:
121 self.outputs.remove("inputMap")
124class AssembleCoaddConfig(CoaddBaseTask.ConfigClass, pipeBase.PipelineTaskConfig,
125 pipelineConnections=AssembleCoaddConnections):
126 """Configuration parameters for the `AssembleCoaddTask`.
128 Notes
129 -----
130 The `doMaskBrightObjects` and `brightObjectMaskName` configuration options
131 only set the bitplane config.brightObjectMaskName. To make this useful you
132 *must* also configure the flags.pixel algorithm, for example by adding
134 .. code-block:: none
136 config.measurement.plugins["base_PixelFlags"].masksFpCenter.append("BRIGHT_OBJECT")
137 config.measurement.plugins["base_PixelFlags"].masksFpAnywhere.append("BRIGHT_OBJECT")
139 to your measureCoaddSources.py and forcedPhotCoadd.py config overrides.
140 """
141 warpType = pexConfig.Field(
142 doc="Warp name: one of 'direct' or 'psfMatched'",
143 dtype=str,
144 default="direct",
145 )
146 subregionSize = pexConfig.ListField(
147 dtype=int,
148 doc="Width, height of stack subregion size; "
149 "make small enough that a full stack of images will fit into memory at once.",
150 length=2,
151 default=(2000, 2000),
152 )
153 statistic = pexConfig.Field(
154 dtype=str,
155 doc="Main stacking statistic for aggregating over the epochs.",
156 default="MEANCLIP",
157 )
158 doOnlineForMean = pexConfig.Field(
159 dtype=bool,
160 doc="Perform online coaddition when statistic=\"MEAN\" to save memory?",
161 default=False,
162 )
163 doSigmaClip = pexConfig.Field(
164 dtype=bool,
165 doc="Perform sigma clipped outlier rejection with MEANCLIP statistic? (DEPRECATED)",
166 default=False,
167 )
168 sigmaClip = pexConfig.Field(
169 dtype=float,
170 doc="Sigma for outlier rejection; ignored if non-clipping statistic selected.",
171 default=3.0,
172 )
173 clipIter = pexConfig.Field(
174 dtype=int,
175 doc="Number of iterations of outlier rejection; ignored if non-clipping statistic selected.",
176 default=2,
177 )
178 calcErrorFromInputVariance = pexConfig.Field(
179 dtype=bool,
180 doc="Calculate coadd variance from input variance by stacking statistic."
181 "Passed to StatisticsControl.setCalcErrorFromInputVariance()",
182 default=True,
183 )
184 scaleZeroPoint = pexConfig.ConfigurableField(
185 target=ScaleZeroPointTask,
186 doc="Task to adjust the photometric zero point of the coadd temp exposures",
187 )
188 doInterp = pexConfig.Field(
189 doc="Interpolate over NaN pixels? Also extrapolate, if necessary, but the results are ugly.",
190 dtype=bool,
191 default=True,
192 )
193 interpImage = pexConfig.ConfigurableField(
194 target=InterpImageTask,
195 doc="Task to interpolate (and extrapolate) over NaN pixels",
196 )
197 doWrite = pexConfig.Field(
198 doc="Persist coadd?",
199 dtype=bool,
200 default=True,
201 )
202 doNImage = pexConfig.Field(
203 doc="Create image of number of contributing exposures for each pixel",
204 dtype=bool,
205 default=False,
206 )
207 doUsePsfMatchedPolygons = pexConfig.Field(
208 doc="Use ValidPolygons from shrunk Psf-Matched Calexps? Should be set to True by CompareWarp only.",
209 dtype=bool,
210 default=False,
211 )
212 maskPropagationThresholds = pexConfig.DictField(
213 keytype=str,
214 itemtype=float,
215 doc=("Threshold (in fractional weight) of rejection at which we propagate a mask plane to "
216 "the coadd; that is, we set the mask bit on the coadd if the fraction the rejected frames "
217 "would have contributed exceeds this value."),
218 default={"SAT": 0.1},
219 )
220 removeMaskPlanes = pexConfig.ListField(dtype=str, default=["NOT_DEBLENDED"],
221 doc="Mask planes to remove before coadding")
222 doMaskBrightObjects = pexConfig.Field(dtype=bool, default=False,
223 doc="Set mask and flag bits for bright objects?")
224 brightObjectMaskName = pexConfig.Field(dtype=str, default="BRIGHT_OBJECT",
225 doc="Name of mask bit used for bright objects")
226 coaddPsf = pexConfig.ConfigField(
227 doc="Configuration for CoaddPsf",
228 dtype=measAlg.CoaddPsfConfig,
229 )
230 doAttachTransmissionCurve = pexConfig.Field(
231 dtype=bool, default=False, optional=False,
232 doc=("Attach a piecewise TransmissionCurve for the coadd? "
233 "(requires all input Exposures to have TransmissionCurves).")
234 )
235 hasFakes = pexConfig.Field(
236 dtype=bool,
237 default=False,
238 doc="Should be set to True if fake sources have been inserted into the input data."
239 )
240 doSelectVisits = pexConfig.Field(
241 doc="Coadd only visits selected by a SelectVisitsTask",
242 dtype=bool,
243 default=False,
244 )
245 doInputMap = pexConfig.Field(
246 doc="Create a bitwise map of coadd inputs",
247 dtype=bool,
248 default=False,
249 )
250 inputMapper = pexConfig.ConfigurableField(
251 doc="Input map creation subtask.",
252 target=HealSparseInputMapTask,
253 )
255 def setDefaults(self):
256 super().setDefaults()
257 self.badMaskPlanes = ["NO_DATA", "BAD", "SAT", "EDGE"]
259 def validate(self):
260 super().validate()
261 if self.doPsfMatch:
262 # Backwards compatibility.
263 # Configs do not have loggers
264 log.warning("Config doPsfMatch deprecated. Setting warpType='psfMatched'")
265 self.warpType = 'psfMatched'
266 if self.doSigmaClip and self.statistic != "MEANCLIP":
267 log.warning('doSigmaClip deprecated. To replicate behavior, setting statistic to "MEANCLIP"')
268 self.statistic = "MEANCLIP"
269 if self.doInterp and self.statistic not in ['MEAN', 'MEDIAN', 'MEANCLIP', 'VARIANCE', 'VARIANCECLIP']:
270 raise ValueError("Must set doInterp=False for statistic=%s, which does not "
271 "compute and set a non-zero coadd variance estimate." % (self.statistic))
273 unstackableStats = ['NOTHING', 'ERROR', 'ORMASK']
274 if not hasattr(afwMath.Property, self.statistic) or self.statistic in unstackableStats:
275 stackableStats = [str(k) for k in afwMath.Property.__members__.keys()
276 if str(k) not in unstackableStats]
277 raise ValueError("statistic %s is not allowed. Please choose one of %s."
278 % (self.statistic, stackableStats))
281class AssembleCoaddTask(CoaddBaseTask, pipeBase.PipelineTask):
282 """Assemble a coadded image from a set of warps.
284 Each Warp that goes into a coadd will typically have an independent
285 photometric zero-point. Therefore, we must scale each Warp to set it to
286 a common photometric zeropoint. WarpType may be one of 'direct' or
287 'psfMatched', and the boolean configs `config.makeDirect` and
288 `config.makePsfMatched` set which of the warp types will be coadded.
289 The coadd is computed as a mean with optional outlier rejection.
290 Criteria for outlier rejection are set in `AssembleCoaddConfig`.
291 Finally, Warps can have bad 'NaN' pixels which received no input from the
292 source calExps. We interpolate over these bad (NaN) pixels.
293 """
294 ConfigClass = AssembleCoaddConfig
295 _DefaultName = "assembleCoadd"
297 def __init__(self, *args, **kwargs):
298 # TODO: DM-17415 better way to handle previously allowed passed args e.g.`AssembleCoaddTask(config)`
299 if args:
300 argNames = ["config", "name", "parentTask", "log"]
301 kwargs.update({k: v for k, v in zip(argNames, args)})
302 warnings.warn("AssembleCoadd received positional args, and casting them as kwargs: %s. "
303 "PipelineTask will not take positional args" % argNames, FutureWarning)
305 super().__init__(**kwargs)
306 self.makeSubtask("interpImage")
307 self.makeSubtask("scaleZeroPoint")
309 if self.config.doMaskBrightObjects:
310 mask = afwImage.Mask()
311 try:
312 self.brightObjectBitmask = 1 << mask.addMaskPlane(self.config.brightObjectMaskName)
313 except pexExceptions.LsstCppException:
314 raise RuntimeError("Unable to define mask plane for bright objects; planes used are %s" %
315 mask.getMaskPlaneDict().keys())
316 del mask
318 if self.config.doInputMap:
319 self.makeSubtask("inputMapper")
321 self.warpType = self.config.warpType
323 @utils.inheritDoc(pipeBase.PipelineTask)
324 def runQuantum(self, butlerQC, inputRefs, outputRefs):
325 inputData = butlerQC.get(inputRefs)
327 # Construct skyInfo expected by run
328 # Do not remove skyMap from inputData in case _makeSupplementaryData needs it
329 skyMap = inputData["skyMap"]
330 outputDataId = butlerQC.quantum.dataId
332 inputData['skyInfo'] = makeSkyInfo(skyMap,
333 tractId=outputDataId['tract'],
334 patchId=outputDataId['patch'])
336 if self.config.doSelectVisits:
337 warpRefList = self.filterWarps(inputData['inputWarps'], inputData['selectedVisits'])
338 else:
339 warpRefList = inputData['inputWarps']
341 inputs = self.prepareInputs(warpRefList)
342 self.log.info("Found %d %s", len(inputs.tempExpRefList),
343 self.getTempExpDatasetName(self.warpType))
344 if len(inputs.tempExpRefList) == 0:
345 raise pipeBase.NoWorkFound("No coadd temporary exposures found")
347 supplementaryData = self._makeSupplementaryData(butlerQC, inputRefs, outputRefs)
348 retStruct = self.run(inputData['skyInfo'], inputs.tempExpRefList, inputs.imageScalerList,
349 inputs.weightList, supplementaryData=supplementaryData)
351 inputData.setdefault('brightObjectMask', None)
352 self.processResults(retStruct.coaddExposure, inputData['brightObjectMask'], outputDataId)
354 if self.config.doWrite:
355 butlerQC.put(retStruct, outputRefs)
356 return retStruct
358 def processResults(self, coaddExposure, brightObjectMasks=None, dataId=None):
359 """Interpolate over missing data and mask bright stars.
361 Parameters
362 ----------
363 coaddExposure : `lsst.afw.image.Exposure`
364 The coadded exposure to process.
365 brightObjectMasks : `lsst.afw.table`, optional
366 Table of bright objects to mask.
367 dataId : `lsst.daf.butler.DataId`, optional
368 Data identification.
369 """
370 if self.config.doInterp:
371 self.interpImage.run(coaddExposure.getMaskedImage(), planeName="NO_DATA")
372 # The variance must be positive; work around for DM-3201.
373 varArray = coaddExposure.variance.array
374 with numpy.errstate(invalid="ignore"):
375 varArray[:] = numpy.where(varArray > 0, varArray, numpy.inf)
377 if self.config.doMaskBrightObjects:
378 self.setBrightObjectMasks(coaddExposure, brightObjectMasks, dataId)
380 def _makeSupplementaryData(self, butlerQC, inputRefs, outputRefs):
381 """Make additional inputs to run() specific to subclasses (Gen3)
383 Duplicates interface of `runQuantum` method.
384 Available to be implemented by subclasses only if they need the
385 coadd dataRef for performing preliminary processing before
386 assembling the coadd.
388 Parameters
389 ----------
390 butlerQC : `lsst.pipe.base.ButlerQuantumContext`
391 Gen3 Butler object for fetching additional data products before
392 running the Task specialized for quantum being processed
393 inputRefs : `lsst.pipe.base.InputQuantizedConnection`
394 Attributes are the names of the connections describing input dataset types.
395 Values are DatasetRefs that task consumes for corresponding dataset type.
396 DataIds are guaranteed to match data objects in ``inputData``.
397 outputRefs : `lsst.pipe.base.OutputQuantizedConnection`
398 Attributes are the names of the connections describing output dataset types.
399 Values are DatasetRefs that task is to produce
400 for corresponding dataset type.
401 """
402 return pipeBase.Struct()
404 @deprecated(
405 reason="makeSupplementaryDataGen3 is deprecated in favor of _makeSupplementaryData",
406 version="v25.0",
407 category=FutureWarning
408 )
409 def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs):
410 return self._makeSupplementaryData(butlerQC, inputRefs, outputRefs)
412 def prepareInputs(self, refList):
413 """Prepare the input warps for coaddition by measuring the weight for
414 each warp and the scaling for the photometric zero point.
416 Each Warp has its own photometric zeropoint and background variance.
417 Before coadding these Warps together, compute a scale factor to
418 normalize the photometric zeropoint and compute the weight for each Warp.
420 Parameters
421 ----------
422 refList : `list`
423 List of data references to tempExp
425 Returns
426 -------
427 result : `lsst.pipe.base.Struct`
428 Result struct with components:
430 - ``tempExprefList``: `list` of data references to tempExp.
431 - ``weightList``: `list` of weightings.
432 - ``imageScalerList``: `list` of image scalers.
433 """
434 statsCtrl = afwMath.StatisticsControl()
435 statsCtrl.setNumSigmaClip(self.config.sigmaClip)
436 statsCtrl.setNumIter(self.config.clipIter)
437 statsCtrl.setAndMask(self.getBadPixelMask())
438 statsCtrl.setNanSafe(True)
439 # compute tempExpRefList: a list of tempExpRef that actually exist
440 # and weightList: a list of the weight of the associated coadd tempExp
441 # and imageScalerList: a list of scale factors for the associated coadd tempExp
442 tempExpRefList = []
443 weightList = []
444 imageScalerList = []
445 tempExpName = self.getTempExpDatasetName(self.warpType)
446 for tempExpRef in refList:
447 tempExp = tempExpRef.get()
448 # Ignore any input warp that is empty of data
449 if numpy.isnan(tempExp.image.array).all():
450 continue
451 maskedImage = tempExp.getMaskedImage()
452 imageScaler = self.scaleZeroPoint.computeImageScaler(
453 exposure=tempExp,
454 dataRef=tempExpRef, # FIXME
455 )
456 try:
457 imageScaler.scaleMaskedImage(maskedImage)
458 except Exception as e:
459 self.log.warning("Scaling failed for %s (skipping it): %s", tempExpRef.dataId, e)
460 continue
461 statObj = afwMath.makeStatistics(maskedImage.getVariance(), maskedImage.getMask(),
462 afwMath.MEANCLIP, statsCtrl)
463 meanVar, meanVarErr = statObj.getResult(afwMath.MEANCLIP)
464 weight = 1.0 / float(meanVar)
465 if not numpy.isfinite(weight):
466 self.log.warning("Non-finite weight for %s: skipping", tempExpRef.dataId)
467 continue
468 self.log.info("Weight of %s %s = %0.3f", tempExpName, tempExpRef.dataId, weight)
470 del maskedImage
471 del tempExp
473 tempExpRefList.append(tempExpRef)
474 weightList.append(weight)
475 imageScalerList.append(imageScaler)
477 return pipeBase.Struct(tempExpRefList=tempExpRefList, weightList=weightList,
478 imageScalerList=imageScalerList)
480 def prepareStats(self, mask=None):
481 """Prepare the statistics for coadding images.
483 Parameters
484 ----------
485 mask : `int`, optional
486 Bit mask value to exclude from coaddition.
488 Returns
489 -------
490 stats : `lsst.pipe.base.Struct`
491 Statistics structure with the following fields:
493 - ``statsCtrl``: Statistics control object for coadd
494 (`lsst.afw.math.StatisticsControl`)
495 - ``statsFlags``: Statistic for coadd (`lsst.afw.math.Property`)
496 """
497 if mask is None:
498 mask = self.getBadPixelMask()
499 statsCtrl = afwMath.StatisticsControl()
500 statsCtrl.setNumSigmaClip(self.config.sigmaClip)
501 statsCtrl.setNumIter(self.config.clipIter)
502 statsCtrl.setAndMask(mask)
503 statsCtrl.setNanSafe(True)
504 statsCtrl.setWeighted(True)
505 statsCtrl.setCalcErrorFromInputVariance(self.config.calcErrorFromInputVariance)
506 for plane, threshold in self.config.maskPropagationThresholds.items():
507 bit = afwImage.Mask.getMaskPlane(plane)
508 statsCtrl.setMaskPropagationThreshold(bit, threshold)
509 statsFlags = afwMath.stringToStatisticsProperty(self.config.statistic)
510 return pipeBase.Struct(ctrl=statsCtrl, flags=statsFlags)
512 @timeMethod
513 def run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
514 altMaskList=None, mask=None, supplementaryData=None):
515 """Assemble a coadd from input warps
517 Assemble the coadd using the provided list of coaddTempExps. Since
518 the full coadd covers a patch (a large area), the assembly is
519 performed over small areas on the image at a time in order to
520 conserve memory usage. Iterate over subregions within the outer
521 bbox of the patch using `assembleSubregion` to stack the corresponding
522 subregions from the coaddTempExps with the statistic specified.
523 Set the edge bits the coadd mask based on the weight map.
525 Parameters
526 ----------
527 skyInfo : `lsst.pipe.base.Struct`
528 Struct with geometric information about the patch.
529 tempExpRefList : `list`
530 List of data references to Warps (previously called CoaddTempExps).
531 imageScalerList : `list`
532 List of image scalers.
533 weightList : `list`
534 List of weights
535 altMaskList : `list`, optional
536 List of alternate masks to use rather than those stored with
537 tempExp.
538 mask : `int`, optional
539 Bit mask value to exclude from coaddition.
540 supplementaryData : lsst.pipe.base.Struct, optional
541 Struct with additional data products needed to assemble coadd.
542 Only used by subclasses that implement `_makeSupplementaryData`
543 and override `run`.
545 Returns
546 -------
547 result : `lsst.pipe.base.Struct`
548 Result struct with components:
550 - ``coaddExposure``: coadded exposure (``lsst.afw.image.Exposure``).
551 - ``nImage``: exposure count image (``lsst.afw.image.Image``), if requested.
552 - ``inputMap``: bit-wise map of inputs, if requested.
553 - ``warpRefList``: input list of refs to the warps (
554 ``lsst.daf.butler.DeferredDatasetHandle``)
555 (unmodified)
556 - ``imageScalerList``: input list of image scalers (unmodified)
557 - ``weightList``: input list of weights (unmodified)
558 """
559 tempExpName = self.getTempExpDatasetName(self.warpType)
560 self.log.info("Assembling %s %s", len(tempExpRefList), tempExpName)
561 stats = self.prepareStats(mask=mask)
563 if altMaskList is None:
564 altMaskList = [None]*len(tempExpRefList)
566 coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
567 coaddExposure.setPhotoCalib(self.scaleZeroPoint.getPhotoCalib())
568 coaddExposure.getInfo().setCoaddInputs(self.inputRecorder.makeCoaddInputs())
569 self.assembleMetadata(coaddExposure, tempExpRefList, weightList)
570 coaddMaskedImage = coaddExposure.getMaskedImage()
571 subregionSizeArr = self.config.subregionSize
572 subregionSize = geom.Extent2I(subregionSizeArr[0], subregionSizeArr[1])
573 # if nImage is requested, create a zero one which can be passed to assembleSubregion
574 if self.config.doNImage:
575 nImage = afwImage.ImageU(skyInfo.bbox)
576 else:
577 nImage = None
578 # If inputMap is requested, create the initial version that can be masked in
579 # assembleSubregion.
580 if self.config.doInputMap:
581 self.inputMapper.build_ccd_input_map(skyInfo.bbox,
582 skyInfo.wcs,
583 coaddExposure.getInfo().getCoaddInputs().ccds)
585 if self.config.doOnlineForMean and self.config.statistic == "MEAN":
586 try:
587 self.assembleOnlineMeanCoadd(coaddExposure, tempExpRefList, imageScalerList,
588 weightList, altMaskList, stats.ctrl,
589 nImage=nImage)
590 except Exception as e:
591 self.log.exception("Cannot compute online coadd %s", e)
592 raise
593 else:
594 for subBBox in self._subBBoxIter(skyInfo.bbox, subregionSize):
595 try:
596 self.assembleSubregion(coaddExposure, subBBox, tempExpRefList, imageScalerList,
597 weightList, altMaskList, stats.flags, stats.ctrl,
598 nImage=nImage)
599 except Exception as e:
600 self.log.exception("Cannot compute coadd %s: %s", subBBox, e)
601 raise
603 # If inputMap is requested, we must finalize the map after the accumulation.
604 if self.config.doInputMap:
605 self.inputMapper.finalize_ccd_input_map_mask()
606 inputMap = self.inputMapper.ccd_input_map
607 else:
608 inputMap = None
610 self.setInexactPsf(coaddMaskedImage.getMask())
611 # Despite the name, the following doesn't really deal with "EDGE" pixels: it identifies
612 # pixels that didn't receive any unmasked inputs (as occurs around the edge of the field).
613 coaddUtils.setCoaddEdgeBits(coaddMaskedImage.getMask(), coaddMaskedImage.getVariance())
614 return pipeBase.Struct(coaddExposure=coaddExposure, nImage=nImage,
615 warpRefList=tempExpRefList, imageScalerList=imageScalerList,
616 weightList=weightList, inputMap=inputMap)
618 def assembleMetadata(self, coaddExposure, tempExpRefList, weightList):
619 """Set the metadata for the coadd.
621 This basic implementation sets the filter from the first input.
623 Parameters
624 ----------
625 coaddExposure : `lsst.afw.image.Exposure`
626 The target exposure for the coadd.
627 tempExpRefList : `list`
628 List of data references to tempExp.
629 weightList : `list`
630 List of weights.
631 """
632 assert len(tempExpRefList) == len(weightList), "Length mismatch"
634 # We load a single pixel of each coaddTempExp, because we just want to get at the metadata
635 # (and we need more than just the PropertySet that contains the header), which is not possible
636 # with the current butler (see #2777).
637 bbox = geom.Box2I(coaddExposure.getBBox().getMin(), geom.Extent2I(1, 1))
639 tempExpList = [tempExpRef.get(parameters={'bbox': bbox}) for tempExpRef in tempExpRefList]
641 numCcds = sum(len(tempExp.getInfo().getCoaddInputs().ccds) for tempExp in tempExpList)
643 # Set the coadd FilterLabel to the band of the first input exposure:
644 # Coadds are calibrated, so the physical label is now meaningless.
645 coaddExposure.setFilter(afwImage.FilterLabel(tempExpList[0].getFilter().bandLabel))
646 coaddInputs = coaddExposure.getInfo().getCoaddInputs()
647 coaddInputs.ccds.reserve(numCcds)
648 coaddInputs.visits.reserve(len(tempExpList))
650 for tempExp, weight in zip(tempExpList, weightList):
651 self.inputRecorder.addVisitToCoadd(coaddInputs, tempExp, weight)
653 if self.config.doUsePsfMatchedPolygons:
654 self.shrinkValidPolygons(coaddInputs)
656 coaddInputs.visits.sort()
657 coaddInputs.ccds.sort()
658 if self.warpType == "psfMatched":
659 # The modelPsf BBox for a psfMatchedWarp/coaddTempExp was dynamically defined by
660 # ModelPsfMatchTask as the square box bounding its spatially-variable, pre-matched WarpedPsf.
661 # Likewise, set the PSF of a PSF-Matched Coadd to the modelPsf
662 # having the maximum width (sufficient because square)
663 modelPsfList = [tempExp.getPsf() for tempExp in tempExpList]
664 modelPsfWidthList = [modelPsf.computeBBox(modelPsf.getAveragePosition()).getWidth()
665 for modelPsf in modelPsfList]
666 psf = modelPsfList[modelPsfWidthList.index(max(modelPsfWidthList))]
667 else:
668 psf = measAlg.CoaddPsf(coaddInputs.ccds, coaddExposure.getWcs(),
669 self.config.coaddPsf.makeControl())
670 coaddExposure.setPsf(psf)
671 apCorrMap = measAlg.makeCoaddApCorrMap(coaddInputs.ccds, coaddExposure.getBBox(afwImage.PARENT),
672 coaddExposure.getWcs())
673 coaddExposure.getInfo().setApCorrMap(apCorrMap)
674 if self.config.doAttachTransmissionCurve:
675 transmissionCurve = measAlg.makeCoaddTransmissionCurve(coaddExposure.getWcs(), coaddInputs.ccds)
676 coaddExposure.getInfo().setTransmissionCurve(transmissionCurve)
678 def assembleSubregion(self, coaddExposure, bbox, tempExpRefList, imageScalerList, weightList,
679 altMaskList, statsFlags, statsCtrl, nImage=None):
680 """Assemble the coadd for a sub-region.
682 For each coaddTempExp, check for (and swap in) an alternative mask
683 if one is passed. Remove mask planes listed in
684 `config.removeMaskPlanes`. Finally, stack the actual exposures using
685 `lsst.afw.math.statisticsStack` with the statistic specified by
686 statsFlags. Typically, the statsFlag will be one of lsst.afw.math.MEAN for
687 a mean-stack or `lsst.afw.math.MEANCLIP` for outlier rejection using
688 an N-sigma clipped mean where N and iterations are specified by
689 statsCtrl. Assign the stacked subregion back to the coadd.
691 Parameters
692 ----------
693 coaddExposure : `lsst.afw.image.Exposure`
694 The target exposure for the coadd.
695 bbox : `lsst.geom.Box`
696 Sub-region to coadd.
697 tempExpRefList : `list`
698 List of data reference to tempExp.
699 imageScalerList : `list`
700 List of image scalers.
701 weightList : `list`
702 List of weights.
703 altMaskList : `list`
704 List of alternate masks to use rather than those stored with
705 tempExp, or None. Each element is dict with keys = mask plane
706 name to which to add the spans.
707 statsFlags : `lsst.afw.math.Property`
708 Property object for statistic for coadd.
709 statsCtrl : `lsst.afw.math.StatisticsControl`
710 Statistics control object for coadd.
711 nImage : `lsst.afw.image.ImageU`, optional
712 Keeps track of exposure count for each pixel.
713 """
714 self.log.debug("Computing coadd over %s", bbox)
716 coaddExposure.mask.addMaskPlane("REJECTED")
717 coaddExposure.mask.addMaskPlane("CLIPPED")
718 coaddExposure.mask.addMaskPlane("SENSOR_EDGE")
719 maskMap = self.setRejectedMaskMapping(statsCtrl)
720 clipped = afwImage.Mask.getPlaneBitMask("CLIPPED")
721 maskedImageList = []
722 if nImage is not None:
723 subNImage = afwImage.ImageU(bbox.getWidth(), bbox.getHeight())
724 for tempExpRef, imageScaler, altMask in zip(tempExpRefList, imageScalerList, altMaskList):
726 exposure = tempExpRef.get(parameters={'bbox': bbox})
728 maskedImage = exposure.getMaskedImage()
729 mask = maskedImage.getMask()
730 if altMask is not None:
731 self.applyAltMaskPlanes(mask, altMask)
732 imageScaler.scaleMaskedImage(maskedImage)
734 # Add 1 for each pixel which is not excluded by the exclude mask.
735 # In legacyCoadd, pixels may also be excluded by afwMath.statisticsStack.
736 if nImage is not None:
737 subNImage.getArray()[maskedImage.getMask().getArray() & statsCtrl.getAndMask() == 0] += 1
738 if self.config.removeMaskPlanes:
739 self.removeMaskPlanes(maskedImage)
740 maskedImageList.append(maskedImage)
742 if self.config.doInputMap:
743 visit = exposure.getInfo().getCoaddInputs().visits[0].getId()
744 self.inputMapper.mask_warp_bbox(bbox, visit, mask, statsCtrl.getAndMask())
746 with self.timer("stack"):
747 coaddSubregion = afwMath.statisticsStack(maskedImageList, statsFlags, statsCtrl, weightList,
748 clipped, # also set output to CLIPPED if sigma-clipped
749 maskMap)
750 coaddExposure.maskedImage.assign(coaddSubregion, bbox)
751 if nImage is not None:
752 nImage.assign(subNImage, bbox)
754 def assembleOnlineMeanCoadd(self, coaddExposure, tempExpRefList, imageScalerList, weightList,
755 altMaskList, statsCtrl, nImage=None):
756 """Assemble the coadd using the "online" method.
758 This method takes a running sum of images and weights to save memory.
759 It only works for MEAN statistics.
761 Parameters
762 ----------
763 coaddExposure : `lsst.afw.image.Exposure`
764 The target exposure for the coadd.
765 tempExpRefList : `list`
766 List of data reference to tempExp.
767 imageScalerList : `list`
768 List of image scalers.
769 weightList : `list`
770 List of weights.
771 altMaskList : `list`
772 List of alternate masks to use rather than those stored with
773 tempExp, or None. Each element is dict with keys = mask plane
774 name to which to add the spans.
775 statsCtrl : `lsst.afw.math.StatisticsControl`
776 Statistics control object for coadd
777 nImage : `lsst.afw.image.ImageU`, optional
778 Keeps track of exposure count for each pixel.
779 """
780 self.log.debug("Computing online coadd.")
782 coaddExposure.mask.addMaskPlane("REJECTED")
783 coaddExposure.mask.addMaskPlane("CLIPPED")
784 coaddExposure.mask.addMaskPlane("SENSOR_EDGE")
785 maskMap = self.setRejectedMaskMapping(statsCtrl)
786 thresholdDict = AccumulatorMeanStack.stats_ctrl_to_threshold_dict(statsCtrl)
788 bbox = coaddExposure.maskedImage.getBBox()
790 stacker = AccumulatorMeanStack(
791 coaddExposure.image.array.shape,
792 statsCtrl.getAndMask(),
793 mask_threshold_dict=thresholdDict,
794 mask_map=maskMap,
795 no_good_pixels_mask=statsCtrl.getNoGoodPixelsMask(),
796 calc_error_from_input_variance=self.config.calcErrorFromInputVariance,
797 compute_n_image=(nImage is not None)
798 )
800 for tempExpRef, imageScaler, altMask, weight in zip(tempExpRefList,
801 imageScalerList,
802 altMaskList,
803 weightList):
804 exposure = tempExpRef.get()
805 maskedImage = exposure.getMaskedImage()
806 mask = maskedImage.getMask()
807 if altMask is not None:
808 self.applyAltMaskPlanes(mask, altMask)
809 imageScaler.scaleMaskedImage(maskedImage)
810 if self.config.removeMaskPlanes:
811 self.removeMaskPlanes(maskedImage)
813 stacker.add_masked_image(maskedImage, weight=weight)
815 if self.config.doInputMap:
816 visit = exposure.getInfo().getCoaddInputs().visits[0].getId()
817 self.inputMapper.mask_warp_bbox(bbox, visit, mask, statsCtrl.getAndMask())
819 stacker.fill_stacked_masked_image(coaddExposure.maskedImage)
821 if nImage is not None:
822 nImage.array[:, :] = stacker.n_image
824 def removeMaskPlanes(self, maskedImage):
825 """Unset the mask of an image for mask planes specified in the config.
827 Parameters
828 ----------
829 maskedImage : `lsst.afw.image.MaskedImage`
830 The masked image to be modified.
831 """
832 mask = maskedImage.getMask()
833 for maskPlane in self.config.removeMaskPlanes:
834 try:
835 mask &= ~mask.getPlaneBitMask(maskPlane)
836 except pexExceptions.InvalidParameterError:
837 self.log.debug("Unable to remove mask plane %s: no mask plane with that name was found.",
838 maskPlane)
840 @staticmethod
841 def setRejectedMaskMapping(statsCtrl):
842 """Map certain mask planes of the warps to new planes for the coadd.
844 If a pixel is rejected due to a mask value other than EDGE, NO_DATA,
845 or CLIPPED, set it to REJECTED on the coadd.
846 If a pixel is rejected due to EDGE, set the coadd pixel to SENSOR_EDGE.
847 If a pixel is rejected due to CLIPPED, set the coadd pixel to CLIPPED.
849 Parameters
850 ----------
851 statsCtrl : `lsst.afw.math.StatisticsControl`
852 Statistics control object for coadd
854 Returns
855 -------
856 maskMap : `list` of `tuple` of `int`
857 A list of mappings of mask planes of the warped exposures to
858 mask planes of the coadd.
859 """
860 edge = afwImage.Mask.getPlaneBitMask("EDGE")
861 noData = afwImage.Mask.getPlaneBitMask("NO_DATA")
862 clipped = afwImage.Mask.getPlaneBitMask("CLIPPED")
863 toReject = statsCtrl.getAndMask() & (~noData) & (~edge) & (~clipped)
864 maskMap = [(toReject, afwImage.Mask.getPlaneBitMask("REJECTED")),
865 (edge, afwImage.Mask.getPlaneBitMask("SENSOR_EDGE")),
866 (clipped, clipped)]
867 return maskMap
869 def applyAltMaskPlanes(self, mask, altMaskSpans):
870 """Apply in place alt mask formatted as SpanSets to a mask.
872 Parameters
873 ----------
874 mask : `lsst.afw.image.Mask`
875 Original mask.
876 altMaskSpans : `dict`
877 SpanSet lists to apply. Each element contains the new mask
878 plane name (e.g. "CLIPPED and/or "NO_DATA") as the key,
879 and list of SpanSets to apply to the mask.
881 Returns
882 -------
883 mask : `lsst.afw.image.Mask`
884 Updated mask.
885 """
886 if self.config.doUsePsfMatchedPolygons:
887 if ("NO_DATA" in altMaskSpans) and ("NO_DATA" in self.config.badMaskPlanes):
888 # Clear away any other masks outside the validPolygons. These pixels are no longer
889 # contributing to inexact PSFs, and will still be rejected because of NO_DATA
890 # self.config.doUsePsfMatchedPolygons should be True only in CompareWarpAssemble
891 # This mask-clearing step must only occur *before* applying the new masks below
892 for spanSet in altMaskSpans['NO_DATA']:
893 spanSet.clippedTo(mask.getBBox()).clearMask(mask, self.getBadPixelMask())
895 for plane, spanSetList in altMaskSpans.items():
896 maskClipValue = mask.addMaskPlane(plane)
897 for spanSet in spanSetList:
898 spanSet.clippedTo(mask.getBBox()).setMask(mask, 2**maskClipValue)
899 return mask
901 def shrinkValidPolygons(self, coaddInputs):
902 """Shrink coaddInputs' ccds' ValidPolygons in place.
904 Either modify each ccd's validPolygon in place, or if CoaddInputs
905 does not have a validPolygon, create one from its bbox.
907 Parameters
908 ----------
909 coaddInputs : `lsst.afw.image.coaddInputs`
910 Original mask.
912 """
913 for ccd in coaddInputs.ccds:
914 polyOrig = ccd.getValidPolygon()
915 validPolyBBox = polyOrig.getBBox() if polyOrig else ccd.getBBox()
916 validPolyBBox.grow(-self.config.matchingKernelSize//2)
917 if polyOrig:
918 validPolygon = polyOrig.intersectionSingle(validPolyBBox)
919 else:
920 validPolygon = afwGeom.polygon.Polygon(geom.Box2D(validPolyBBox))
921 ccd.setValidPolygon(validPolygon)
923 def setBrightObjectMasks(self, exposure, brightObjectMasks, dataId=None):
924 """Set the bright object masks.
926 Parameters
927 ----------
928 exposure : `lsst.afw.image.Exposure`
929 Exposure under consideration.
930 brightObjectMasks : `lsst.afw.table`
931 Table of bright objects to mask.
932 dataId : `lsst.daf.butler.DataId`, optional
933 Data identifier dict for patch.
934 """
936 if brightObjectMasks is None:
937 self.log.warning("Unable to apply bright object mask: none supplied")
938 return
939 self.log.info("Applying %d bright object masks to %s", len(brightObjectMasks), dataId)
940 mask = exposure.getMaskedImage().getMask()
941 wcs = exposure.getWcs()
942 plateScale = wcs.getPixelScale().asArcseconds()
944 for rec in brightObjectMasks:
945 center = geom.PointI(wcs.skyToPixel(rec.getCoord()))
946 if rec["type"] == "box":
947 assert rec["angle"] == 0.0, ("Angle != 0 for mask object %s" % rec["id"])
948 width = rec["width"].asArcseconds()/plateScale # convert to pixels
949 height = rec["height"].asArcseconds()/plateScale # convert to pixels
951 halfSize = geom.ExtentI(0.5*width, 0.5*height)
952 bbox = geom.Box2I(center - halfSize, center + halfSize)
954 bbox = geom.BoxI(geom.PointI(int(center[0] - 0.5*width), int(center[1] - 0.5*height)),
955 geom.PointI(int(center[0] + 0.5*width), int(center[1] + 0.5*height)))
956 spans = afwGeom.SpanSet(bbox)
957 elif rec["type"] == "circle":
958 radius = int(rec["radius"].asArcseconds()/plateScale) # convert to pixels
959 spans = afwGeom.SpanSet.fromShape(radius, offset=center)
960 else:
961 self.log.warning("Unexpected region type %s at %s", rec["type"], center)
962 continue
963 spans.clippedTo(mask.getBBox()).setMask(mask, self.brightObjectBitmask)
965 def setInexactPsf(self, mask):
966 """Set INEXACT_PSF mask plane.
968 If any of the input images isn't represented in the coadd (due to
969 clipped pixels or chip gaps), the `CoaddPsf` will be inexact. Flag
970 these pixels.
972 Parameters
973 ----------
974 mask : `lsst.afw.image.Mask`
975 Coadded exposure's mask, modified in-place.
976 """
977 mask.addMaskPlane("INEXACT_PSF")
978 inexactPsf = mask.getPlaneBitMask("INEXACT_PSF")
979 sensorEdge = mask.getPlaneBitMask("SENSOR_EDGE") # chip edges (so PSF is discontinuous)
980 clipped = mask.getPlaneBitMask("CLIPPED") # pixels clipped from coadd
981 rejected = mask.getPlaneBitMask("REJECTED") # pixels rejected from coadd due to masks
982 array = mask.getArray()
983 selected = array & (sensorEdge | clipped | rejected) > 0
984 array[selected] |= inexactPsf
986 @staticmethod
987 def _subBBoxIter(bbox, subregionSize):
988 """Iterate over subregions of a bbox.
990 Parameters
991 ----------
992 bbox : `lsst.geom.Box2I`
993 Bounding box over which to iterate.
994 subregionSize: `lsst.geom.Extent2I`
995 Size of sub-bboxes.
997 Yields
998 ------
999 subBBox : `lsst.geom.Box2I`
1000 Next sub-bounding box of size ``subregionSize`` or smaller; each ``subBBox``
1001 is contained within ``bbox``, so it may be smaller than ``subregionSize`` at
1002 the edges of ``bbox``, but it will never be empty.
1003 """
1004 if bbox.isEmpty():
1005 raise RuntimeError("bbox %s is empty" % (bbox,))
1006 if subregionSize[0] < 1 or subregionSize[1] < 1:
1007 raise RuntimeError("subregionSize %s must be nonzero" % (subregionSize,))
1009 for rowShift in range(0, bbox.getHeight(), subregionSize[1]):
1010 for colShift in range(0, bbox.getWidth(), subregionSize[0]):
1011 subBBox = geom.Box2I(bbox.getMin() + geom.Extent2I(colShift, rowShift), subregionSize)
1012 subBBox.clip(bbox)
1013 if subBBox.isEmpty():
1014 raise RuntimeError("Bug: empty bbox! bbox=%s, subregionSize=%s, "
1015 "colShift=%s, rowShift=%s" %
1016 (bbox, subregionSize, colShift, rowShift))
1017 yield subBBox
1019 def filterWarps(self, inputs, goodVisits):
1020 """Return list of only inputRefs with visitId in goodVisits ordered by goodVisit
1022 Parameters
1023 ----------
1024 inputs : list
1025 List of `lsst.pipe.base.connections.DeferredDatasetRef` with dataId containing visit
1026 goodVisit : `dict`
1027 Dictionary with good visitIds as the keys. Value ignored.
1029 Returns:
1030 --------
1031 filteredInputs : `list`
1032 Filtered and sorted list of `lsst.pipe.base.connections.DeferredDatasetRef`
1033 """
1034 inputWarpDict = {inputRef.ref.dataId['visit']: inputRef for inputRef in inputs}
1035 filteredInputs = []
1036 for visit in goodVisits.keys():
1037 if visit in inputWarpDict:
1038 filteredInputs.append(inputWarpDict[visit])
1039 return filteredInputs
1042def countMaskFromFootprint(mask, footprint, bitmask, ignoreMask):
1043 """Function to count the number of pixels with a specific mask in a
1044 footprint.
1046 Find the intersection of mask & footprint. Count all pixels in the mask
1047 that are in the intersection that have bitmask set but do not have
1048 ignoreMask set. Return the count.
1050 Parameters
1051 ----------
1052 mask : `lsst.afw.image.Mask`
1053 Mask to define intersection region by.
1054 footprint : `lsst.afw.detection.Footprint`
1055 Footprint to define the intersection region by.
1056 bitmask
1057 Specific mask that we wish to count the number of occurances of.
1058 ignoreMask
1059 Pixels to not consider.
1061 Returns
1062 -------
1063 result : `int`
1064 Count of number of pixels in footprint with specified mask.
1065 """
1066 bbox = footprint.getBBox()
1067 bbox.clip(mask.getBBox(afwImage.PARENT))
1068 fp = afwImage.Mask(bbox)
1069 subMask = mask.Factory(mask, bbox, afwImage.PARENT)
1070 footprint.spans.setMask(fp, bitmask)
1071 return numpy.logical_and((subMask.getArray() & fp.getArray()) > 0,
1072 (subMask.getArray() & ignoreMask) == 0).sum()
1075class CompareWarpAssembleCoaddConnections(AssembleCoaddConnections):
1076 psfMatchedWarps = pipeBase.connectionTypes.Input(
1077 doc=("PSF-Matched Warps are required by CompareWarp regardless of the coadd type requested. "
1078 "Only PSF-Matched Warps make sense for image subtraction. "
1079 "Therefore, they must be an additional declared input."),
1080 name="{inputCoaddName}Coadd_psfMatchedWarp",
1081 storageClass="ExposureF",
1082 dimensions=("tract", "patch", "skymap", "visit"),
1083 deferLoad=True,
1084 multiple=True
1085 )
1086 templateCoadd = pipeBase.connectionTypes.Output(
1087 doc=("Model of the static sky, used to find temporal artifacts. Typically a PSF-Matched, "
1088 "sigma-clipped coadd. Written if and only if assembleStaticSkyModel.doWrite=True"),
1089 name="{outputCoaddName}CoaddPsfMatched",
1090 storageClass="ExposureF",
1091 dimensions=("tract", "patch", "skymap", "band"),
1092 )
1094 def __init__(self, *, config=None):
1095 super().__init__(config=config)
1096 if not config.assembleStaticSkyModel.doWrite:
1097 self.outputs.remove("templateCoadd")
1098 config.validate()
1101class CompareWarpAssembleCoaddConfig(AssembleCoaddConfig,
1102 pipelineConnections=CompareWarpAssembleCoaddConnections):
1103 assembleStaticSkyModel = pexConfig.ConfigurableField(
1104 target=AssembleCoaddTask,
1105 doc="Task to assemble an artifact-free, PSF-matched Coadd to serve as a"
1106 " naive/first-iteration model of the static sky.",
1107 )
1108 detect = pexConfig.ConfigurableField(
1109 target=SourceDetectionTask,
1110 doc="Detect outlier sources on difference between each psfMatched warp and static sky model"
1111 )
1112 detectTemplate = pexConfig.ConfigurableField(
1113 target=SourceDetectionTask,
1114 doc="Detect sources on static sky model. Only used if doPreserveContainedBySource is True"
1115 )
1116 maskStreaks = pexConfig.ConfigurableField(
1117 target=MaskStreaksTask,
1118 doc="Detect streaks on difference between each psfMatched warp and static sky model. Only used if "
1119 "doFilterMorphological is True. Adds a mask plane to an exposure, with the mask plane name set by"
1120 "streakMaskName"
1121 )
1122 streakMaskName = pexConfig.Field(
1123 dtype=str,
1124 default="STREAK",
1125 doc="Name of mask bit used for streaks"
1126 )
1127 maxNumEpochs = pexConfig.Field(
1128 doc="Charactistic maximum local number of epochs/visits in which an artifact candidate can appear "
1129 "and still be masked. The effective maxNumEpochs is a broken linear function of local "
1130 "number of epochs (N): min(maxFractionEpochsLow*N, maxNumEpochs + maxFractionEpochsHigh*N). "
1131 "For each footprint detected on the image difference between the psfMatched warp and static sky "
1132 "model, if a significant fraction of pixels (defined by spatialThreshold) are residuals in more "
1133 "than the computed effective maxNumEpochs, the artifact candidate is deemed persistant rather "
1134 "than transient and not masked.",
1135 dtype=int,
1136 default=2
1137 )
1138 maxFractionEpochsLow = pexConfig.RangeField(
1139 doc="Fraction of local number of epochs (N) to use as effective maxNumEpochs for low N. "
1140 "Effective maxNumEpochs = "
1141 "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)",
1142 dtype=float,
1143 default=0.4,
1144 min=0., max=1.,
1145 )
1146 maxFractionEpochsHigh = pexConfig.RangeField(
1147 doc="Fraction of local number of epochs (N) to use as effective maxNumEpochs for high N. "
1148 "Effective maxNumEpochs = "
1149 "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)",
1150 dtype=float,
1151 default=0.03,
1152 min=0., max=1.,
1153 )
1154 spatialThreshold = pexConfig.RangeField(
1155 doc="Unitless fraction of pixels defining how much of the outlier region has to meet the "
1156 "temporal criteria. If 0, clip all. If 1, clip none.",
1157 dtype=float,
1158 default=0.5,
1159 min=0., max=1.,
1160 inclusiveMin=True, inclusiveMax=True
1161 )
1162 doScaleWarpVariance = pexConfig.Field(
1163 doc="Rescale Warp variance plane using empirical noise?",
1164 dtype=bool,
1165 default=True,
1166 )
1167 scaleWarpVariance = pexConfig.ConfigurableField(
1168 target=ScaleVarianceTask,
1169 doc="Rescale variance on warps",
1170 )
1171 doPreserveContainedBySource = pexConfig.Field(
1172 doc="Rescue artifacts from clipping that completely lie within a footprint detected"
1173 "on the PsfMatched Template Coadd. Replicates a behavior of SafeClip.",
1174 dtype=bool,
1175 default=True,
1176 )
1177 doPrefilterArtifacts = pexConfig.Field(
1178 doc="Ignore artifact candidates that are mostly covered by the bad pixel mask, "
1179 "because they will be excluded anyway. This prevents them from contributing "
1180 "to the outlier epoch count image and potentially being labeled as persistant."
1181 "'Mostly' is defined by the config 'prefilterArtifactsRatio'.",
1182 dtype=bool,
1183 default=True
1184 )
1185 prefilterArtifactsMaskPlanes = pexConfig.ListField(
1186 doc="Prefilter artifact candidates that are mostly covered by these bad mask planes.",
1187 dtype=str,
1188 default=('NO_DATA', 'BAD', 'SAT', 'SUSPECT'),
1189 )
1190 prefilterArtifactsRatio = pexConfig.Field(
1191 doc="Prefilter artifact candidates with less than this fraction overlapping good pixels",
1192 dtype=float,
1193 default=0.05
1194 )
1195 doFilterMorphological = pexConfig.Field(
1196 doc="Filter artifact candidates based on morphological criteria, i.g. those that appear to "
1197 "be streaks.",
1198 dtype=bool,
1199 default=False
1200 )
1201 growStreakFp = pexConfig.Field(
1202 doc="Grow streak footprints by this number multiplied by the PSF width",
1203 dtype=float,
1204 default=5
1205 )
1207 def setDefaults(self):
1208 AssembleCoaddConfig.setDefaults(self)
1209 self.statistic = 'MEAN'
1210 self.doUsePsfMatchedPolygons = True
1212 # Real EDGE removed by psfMatched NO_DATA border half the width of the matching kernel
1213 # CompareWarp applies psfMatched EDGE pixels to directWarps before assembling
1214 if "EDGE" in self.badMaskPlanes:
1215 self.badMaskPlanes.remove('EDGE')
1216 self.removeMaskPlanes.append('EDGE')
1217 self.assembleStaticSkyModel.badMaskPlanes = ["NO_DATA", ]
1218 self.assembleStaticSkyModel.warpType = 'psfMatched'
1219 self.assembleStaticSkyModel.connections.warpType = 'psfMatched'
1220 self.assembleStaticSkyModel.statistic = 'MEANCLIP'
1221 self.assembleStaticSkyModel.sigmaClip = 2.5
1222 self.assembleStaticSkyModel.clipIter = 3
1223 self.assembleStaticSkyModel.calcErrorFromInputVariance = False
1224 self.assembleStaticSkyModel.doWrite = False
1225 self.detect.doTempLocalBackground = False
1226 self.detect.reEstimateBackground = False
1227 self.detect.returnOriginalFootprints = False
1228 self.detect.thresholdPolarity = "both"
1229 self.detect.thresholdValue = 5
1230 self.detect.minPixels = 4
1231 self.detect.isotropicGrow = True
1232 self.detect.thresholdType = "pixel_stdev"
1233 self.detect.nSigmaToGrow = 0.4
1234 # The default nSigmaToGrow for SourceDetectionTask is already 2.4,
1235 # Explicitly restating because ratio with detect.nSigmaToGrow matters
1236 self.detectTemplate.nSigmaToGrow = 2.4
1237 self.detectTemplate.doTempLocalBackground = False
1238 self.detectTemplate.reEstimateBackground = False
1239 self.detectTemplate.returnOriginalFootprints = False
1241 def validate(self):
1242 super().validate()
1243 if self.assembleStaticSkyModel.doNImage:
1244 raise ValueError("No dataset type exists for a PSF-Matched Template N Image."
1245 "Please set assembleStaticSkyModel.doNImage=False")
1247 if self.assembleStaticSkyModel.doWrite and (self.warpType == self.assembleStaticSkyModel.warpType):
1248 raise ValueError("warpType (%s) == assembleStaticSkyModel.warpType (%s) and will compete for "
1249 "the same dataset name. Please set assembleStaticSkyModel.doWrite to False "
1250 "or warpType to 'direct'. assembleStaticSkyModel.warpType should ways be "
1251 "'PsfMatched'" % (self.warpType, self.assembleStaticSkyModel.warpType))
1254class CompareWarpAssembleCoaddTask(AssembleCoaddTask):
1255 """Assemble a compareWarp coadded image from a set of warps
1256 by masking artifacts detected by comparing PSF-matched warps.
1258 In ``AssembleCoaddTask``, we compute the coadd as an clipped mean (i.e.,
1259 we clip outliers). The problem with doing this is that when computing the
1260 coadd PSF at a given location, individual visit PSFs from visits with
1261 outlier pixels contribute to the coadd PSF and cannot be treated correctly.
1262 In this task, we correct for this behavior by creating a new badMaskPlane
1263 'CLIPPED' which marks pixels in the individual warps suspected to contain
1264 an artifact. We populate this plane on the input warps by comparing
1265 PSF-matched warps with a PSF-matched median coadd which serves as a
1266 model of the static sky. Any group of pixels that deviates from the
1267 PSF-matched template coadd by more than config.detect.threshold sigma,
1268 is an artifact candidate. The candidates are then filtered to remove
1269 variable sources and sources that are difficult to subtract such as
1270 bright stars. This filter is configured using the config parameters
1271 ``temporalThreshold`` and ``spatialThreshold``. The temporalThreshold is
1272 the maximum fraction of epochs that the deviation can appear in and still
1273 be considered an artifact. The spatialThreshold is the maximum fraction of
1274 pixels in the footprint of the deviation that appear in other epochs
1275 (where other epochs is defined by the temporalThreshold). If the deviant
1276 region meets this criteria of having a significant percentage of pixels
1277 that deviate in only a few epochs, these pixels have the 'CLIPPED' bit
1278 set in the mask. These regions will not contribute to the final coadd.
1279 Furthermore, any routine to determine the coadd PSF can now be cognizant
1280 of clipped regions. Note that the algorithm implemented by this task is
1281 preliminary and works correctly for HSC data. Parameter modifications and
1282 or considerable redesigning of the algorithm is likley required for other
1283 surveys.
1285 ``CompareWarpAssembleCoaddTask`` sub-classes
1286 ``AssembleCoaddTask`` and instantiates ``AssembleCoaddTask``
1287 as a subtask to generate the TemplateCoadd (the model of the static sky).
1288 """
1289 ConfigClass = CompareWarpAssembleCoaddConfig
1290 _DefaultName = "compareWarpAssembleCoadd"
1292 def __init__(self, *args, **kwargs):
1293 AssembleCoaddTask.__init__(self, *args, **kwargs)
1294 self.makeSubtask("assembleStaticSkyModel")
1295 detectionSchema = afwTable.SourceTable.makeMinimalSchema()
1296 self.makeSubtask("detect", schema=detectionSchema)
1297 if self.config.doPreserveContainedBySource:
1298 self.makeSubtask("detectTemplate", schema=afwTable.SourceTable.makeMinimalSchema())
1299 if self.config.doScaleWarpVariance:
1300 self.makeSubtask("scaleWarpVariance")
1301 if self.config.doFilterMorphological:
1302 self.makeSubtask("maskStreaks")
1304 @utils.inheritDoc(AssembleCoaddTask)
1305 def _makeSupplementaryData(self, butlerQC, inputRefs, outputRefs):
1306 """
1307 Generate a templateCoadd to use as a naive model of static sky to
1308 subtract from PSF-Matched warps.
1310 Returns
1311 -------
1312 result : `lsst.pipe.base.Struct`
1313 Result struct with components:
1315 - ``templateCoadd`` : coadded exposure (``lsst.afw.image.Exposure``)
1316 - ``nImage`` : N Image (``lsst.afw.image.Image``)
1317 """
1318 # Ensure that psfMatchedWarps are used as input warps for template generation
1319 staticSkyModelInputRefs = copy.deepcopy(inputRefs)
1320 staticSkyModelInputRefs.inputWarps = inputRefs.psfMatchedWarps
1322 # Because subtasks don't have connections we have to make one.
1323 # The main task's `templateCoadd` is the subtask's `coaddExposure`
1324 staticSkyModelOutputRefs = copy.deepcopy(outputRefs)
1325 if self.config.assembleStaticSkyModel.doWrite:
1326 staticSkyModelOutputRefs.coaddExposure = staticSkyModelOutputRefs.templateCoadd
1327 # Remove template coadd from both subtask's and main tasks outputs,
1328 # because it is handled by the subtask as `coaddExposure`
1329 del outputRefs.templateCoadd
1330 del staticSkyModelOutputRefs.templateCoadd
1332 # A PSF-Matched nImage does not exist as a dataset type
1333 if 'nImage' in staticSkyModelOutputRefs.keys():
1334 del staticSkyModelOutputRefs.nImage
1336 templateCoadd = self.assembleStaticSkyModel.runQuantum(butlerQC, staticSkyModelInputRefs,
1337 staticSkyModelOutputRefs)
1338 if templateCoadd is None:
1339 raise RuntimeError(self._noTemplateMessage(self.assembleStaticSkyModel.warpType))
1341 return pipeBase.Struct(templateCoadd=templateCoadd.coaddExposure,
1342 nImage=templateCoadd.nImage,
1343 warpRefList=templateCoadd.warpRefList,
1344 imageScalerList=templateCoadd.imageScalerList,
1345 weightList=templateCoadd.weightList)
1347 def _noTemplateMessage(self, warpType):
1348 warpName = (warpType[0].upper() + warpType[1:])
1349 message = """No %(warpName)s warps were found to build the template coadd which is
1350 required to run CompareWarpAssembleCoaddTask. To continue assembling this type of coadd,
1351 first either rerun makeCoaddTempExp with config.make%(warpName)s=True or
1352 coaddDriver with config.makeCoadTempExp.make%(warpName)s=True, before assembleCoadd.
1354 Alternatively, to use another algorithm with existing warps, retarget the CoaddDriverConfig to
1355 another algorithm like:
1357 from lsst.pipe.tasks.assembleCoadd import SafeClipAssembleCoaddTask
1358 config.assemble.retarget(SafeClipAssembleCoaddTask)
1359 """ % {"warpName": warpName}
1360 return message
1362 @utils.inheritDoc(AssembleCoaddTask)
1363 @timeMethod
1364 def run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
1365 supplementaryData, *args, **kwargs):
1366 """Assemble the coadd.
1368 Find artifacts and apply them to the warps' masks creating a list of
1369 alternative masks with a new "CLIPPED" plane and updated "NO_DATA"
1370 plane. Then pass these alternative masks to the base class's `run`
1371 method.
1373 The input parameters ``supplementaryData`` is a `lsst.pipe.base.Struct`
1374 that must contain a ``templateCoadd`` that serves as the
1375 model of the static sky.
1376 """
1378 # Check and match the order of the supplementaryData
1379 # (PSF-matched) inputs to the order of the direct inputs,
1380 # so that the artifact mask is applied to the right warp
1381 dataIds = [ref.dataId for ref in tempExpRefList]
1382 psfMatchedDataIds = [ref.dataId for ref in supplementaryData.warpRefList]
1384 if dataIds != psfMatchedDataIds:
1385 self.log.info("Reordering and or/padding PSF-matched visit input list")
1386 supplementaryData.warpRefList = reorderAndPadList(supplementaryData.warpRefList,
1387 psfMatchedDataIds, dataIds)
1388 supplementaryData.imageScalerList = reorderAndPadList(supplementaryData.imageScalerList,
1389 psfMatchedDataIds, dataIds)
1391 # Use PSF-Matched Warps (and corresponding scalers) and coadd to find artifacts
1392 spanSetMaskList = self.findArtifacts(supplementaryData.templateCoadd,
1393 supplementaryData.warpRefList,
1394 supplementaryData.imageScalerList)
1396 badMaskPlanes = self.config.badMaskPlanes[:]
1397 badMaskPlanes.append("CLIPPED")
1398 badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes)
1400 result = AssembleCoaddTask.run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
1401 spanSetMaskList, mask=badPixelMask)
1403 # Propagate PSF-matched EDGE pixels to coadd SENSOR_EDGE and INEXACT_PSF
1404 # Psf-Matching moves the real edge inwards
1405 self.applyAltEdgeMask(result.coaddExposure.maskedImage.mask, spanSetMaskList)
1406 return result
1408 def applyAltEdgeMask(self, mask, altMaskList):
1409 """Propagate alt EDGE mask to SENSOR_EDGE AND INEXACT_PSF planes.
1411 Parameters
1412 ----------
1413 mask : `lsst.afw.image.Mask`
1414 Original mask.
1415 altMaskList : `list`
1416 List of Dicts containing ``spanSet`` lists.
1417 Each element contains the new mask plane name (e.g. "CLIPPED
1418 and/or "NO_DATA") as the key, and list of ``SpanSets`` to apply to
1419 the mask.
1420 """
1421 maskValue = mask.getPlaneBitMask(["SENSOR_EDGE", "INEXACT_PSF"])
1422 for visitMask in altMaskList:
1423 if "EDGE" in visitMask:
1424 for spanSet in visitMask['EDGE']:
1425 spanSet.clippedTo(mask.getBBox()).setMask(mask, maskValue)
1427 def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList):
1428 """Find artifacts.
1430 Loop through warps twice. The first loop builds a map with the count
1431 of how many epochs each pixel deviates from the templateCoadd by more
1432 than ``config.chiThreshold`` sigma. The second loop takes each
1433 difference image and filters the artifacts detected in each using
1434 count map to filter out variable sources and sources that are
1435 difficult to subtract cleanly.
1437 Parameters
1438 ----------
1439 templateCoadd : `lsst.afw.image.Exposure`
1440 Exposure to serve as model of static sky.
1441 tempExpRefList : `list`
1442 List of data references to warps.
1443 imageScalerList : `list`
1444 List of image scalers.
1446 Returns
1447 -------
1448 altMasks : `list`
1449 List of dicts containing information about CLIPPED
1450 (i.e., artifacts), NO_DATA, and EDGE pixels.
1451 """
1453 self.log.debug("Generating Count Image, and mask lists.")
1454 coaddBBox = templateCoadd.getBBox()
1455 slateIm = afwImage.ImageU(coaddBBox)
1456 epochCountImage = afwImage.ImageU(coaddBBox)
1457 nImage = afwImage.ImageU(coaddBBox)
1458 spanSetArtifactList = []
1459 spanSetNoDataMaskList = []
1460 spanSetEdgeList = []
1461 spanSetBadMorphoList = []
1462 badPixelMask = self.getBadPixelMask()
1464 # mask of the warp diffs should = that of only the warp
1465 templateCoadd.mask.clearAllMaskPlanes()
1467 if self.config.doPreserveContainedBySource:
1468 templateFootprints = self.detectTemplate.detectFootprints(templateCoadd)
1469 else:
1470 templateFootprints = None
1472 for warpRef, imageScaler in zip(tempExpRefList, imageScalerList):
1473 warpDiffExp = self._readAndComputeWarpDiff(warpRef, imageScaler, templateCoadd)
1474 if warpDiffExp is not None:
1475 # This nImage only approximates the final nImage because it uses the PSF-matched mask
1476 nImage.array += (numpy.isfinite(warpDiffExp.image.array)
1477 * ((warpDiffExp.mask.array & badPixelMask) == 0)).astype(numpy.uint16)
1478 fpSet = self.detect.detectFootprints(warpDiffExp, doSmooth=False, clearMask=True)
1479 fpSet.positive.merge(fpSet.negative)
1480 footprints = fpSet.positive
1481 slateIm.set(0)
1482 spanSetList = [footprint.spans for footprint in footprints.getFootprints()]
1484 # Remove artifacts due to defects before they contribute to the epochCountImage
1485 if self.config.doPrefilterArtifacts:
1486 spanSetList = self.prefilterArtifacts(spanSetList, warpDiffExp)
1488 # Clear mask before adding prefiltered spanSets
1489 self.detect.clearMask(warpDiffExp.mask)
1490 for spans in spanSetList:
1491 spans.setImage(slateIm, 1, doClip=True)
1492 spans.setMask(warpDiffExp.mask, warpDiffExp.mask.getPlaneBitMask("DETECTED"))
1493 epochCountImage += slateIm
1495 if self.config.doFilterMorphological:
1496 maskName = self.config.streakMaskName
1497 _ = self.maskStreaks.run(warpDiffExp)
1498 streakMask = warpDiffExp.mask
1499 spanSetStreak = afwGeom.SpanSet.fromMask(streakMask,
1500 streakMask.getPlaneBitMask(maskName)).split()
1501 # Pad the streaks to account for low-surface brightness wings
1502 psf = warpDiffExp.getPsf()
1503 for s, sset in enumerate(spanSetStreak):
1504 psfShape = psf.computeShape(sset.computeCentroid())
1505 dilation = self.config.growStreakFp * psfShape.getDeterminantRadius()
1506 sset_dilated = sset.dilated(int(dilation))
1507 spanSetStreak[s] = sset_dilated
1509 # PSF-Matched warps have less available area (~the matching kernel) because the calexps
1510 # undergo a second convolution. Pixels with data in the direct warp
1511 # but not in the PSF-matched warp will not have their artifacts detected.
1512 # NaNs from the PSF-matched warp therefore must be masked in the direct warp
1513 nans = numpy.where(numpy.isnan(warpDiffExp.maskedImage.image.array), 1, 0)
1514 nansMask = afwImage.makeMaskFromArray(nans.astype(afwImage.MaskPixel))
1515 nansMask.setXY0(warpDiffExp.getXY0())
1516 edgeMask = warpDiffExp.mask
1517 spanSetEdgeMask = afwGeom.SpanSet.fromMask(edgeMask,
1518 edgeMask.getPlaneBitMask("EDGE")).split()
1519 else:
1520 # If the directWarp has <1% coverage, the psfMatchedWarp can have 0% and not exist
1521 # In this case, mask the whole epoch
1522 nansMask = afwImage.MaskX(coaddBBox, 1)
1523 spanSetList = []
1524 spanSetEdgeMask = []
1525 spanSetStreak = []
1527 spanSetNoDataMask = afwGeom.SpanSet.fromMask(nansMask).split()
1529 spanSetNoDataMaskList.append(spanSetNoDataMask)
1530 spanSetArtifactList.append(spanSetList)
1531 spanSetEdgeList.append(spanSetEdgeMask)
1532 if self.config.doFilterMorphological:
1533 spanSetBadMorphoList.append(spanSetStreak)
1535 if lsstDebug.Info(__name__).saveCountIm:
1536 path = self._dataRef2DebugPath("epochCountIm", tempExpRefList[0], coaddLevel=True)
1537 epochCountImage.writeFits(path)
1539 for i, spanSetList in enumerate(spanSetArtifactList):
1540 if spanSetList:
1541 filteredSpanSetList = self.filterArtifacts(spanSetList, epochCountImage, nImage,
1542 templateFootprints)
1543 spanSetArtifactList[i] = filteredSpanSetList
1544 if self.config.doFilterMorphological:
1545 spanSetArtifactList[i] += spanSetBadMorphoList[i]
1547 altMasks = []
1548 for artifacts, noData, edge in zip(spanSetArtifactList, spanSetNoDataMaskList, spanSetEdgeList):
1549 altMasks.append({'CLIPPED': artifacts,
1550 'NO_DATA': noData,
1551 'EDGE': edge})
1552 return altMasks
1554 def prefilterArtifacts(self, spanSetList, exp):
1555 """Remove artifact candidates covered by bad mask plane.
1557 Any future editing of the candidate list that does not depend on
1558 temporal information should go in this method.
1560 Parameters
1561 ----------
1562 spanSetList : `list`
1563 List of SpanSets representing artifact candidates.
1564 exp : `lsst.afw.image.Exposure`
1565 Exposure containing mask planes used to prefilter.
1567 Returns
1568 -------
1569 returnSpanSetList : `list`
1570 List of SpanSets with artifacts.
1571 """
1572 badPixelMask = exp.mask.getPlaneBitMask(self.config.prefilterArtifactsMaskPlanes)
1573 goodArr = (exp.mask.array & badPixelMask) == 0
1574 returnSpanSetList = []
1575 bbox = exp.getBBox()
1576 x0, y0 = exp.getXY0()
1577 for i, span in enumerate(spanSetList):
1578 y, x = span.clippedTo(bbox).indices()
1579 yIndexLocal = numpy.array(y) - y0
1580 xIndexLocal = numpy.array(x) - x0
1581 goodRatio = numpy.count_nonzero(goodArr[yIndexLocal, xIndexLocal])/span.getArea()
1582 if goodRatio > self.config.prefilterArtifactsRatio:
1583 returnSpanSetList.append(span)
1584 return returnSpanSetList
1586 def filterArtifacts(self, spanSetList, epochCountImage, nImage, footprintsToExclude=None):
1587 """Filter artifact candidates.
1589 Parameters
1590 ----------
1591 spanSetList : `list`
1592 List of SpanSets representing artifact candidates.
1593 epochCountImage : `lsst.afw.image.Image`
1594 Image of accumulated number of warpDiff detections.
1595 nImage : `lsst.afw.image.Image`
1596 Image of the accumulated number of total epochs contributing.
1598 Returns
1599 -------
1600 maskSpanSetList : `list`
1601 List of SpanSets with artifacts.
1602 """
1604 maskSpanSetList = []
1605 x0, y0 = epochCountImage.getXY0()
1606 for i, span in enumerate(spanSetList):
1607 y, x = span.indices()
1608 yIdxLocal = [y1 - y0 for y1 in y]
1609 xIdxLocal = [x1 - x0 for x1 in x]
1610 outlierN = epochCountImage.array[yIdxLocal, xIdxLocal]
1611 totalN = nImage.array[yIdxLocal, xIdxLocal]
1613 # effectiveMaxNumEpochs is broken line (fraction of N) with characteristic config.maxNumEpochs
1614 effMaxNumEpochsHighN = (self.config.maxNumEpochs
1615 + self.config.maxFractionEpochsHigh*numpy.mean(totalN))
1616 effMaxNumEpochsLowN = self.config.maxFractionEpochsLow * numpy.mean(totalN)
1617 effectiveMaxNumEpochs = int(min(effMaxNumEpochsLowN, effMaxNumEpochsHighN))
1618 nPixelsBelowThreshold = numpy.count_nonzero((outlierN > 0)
1619 & (outlierN <= effectiveMaxNumEpochs))
1620 percentBelowThreshold = nPixelsBelowThreshold / len(outlierN)
1621 if percentBelowThreshold > self.config.spatialThreshold:
1622 maskSpanSetList.append(span)
1624 if self.config.doPreserveContainedBySource and footprintsToExclude is not None:
1625 # If a candidate is contained by a footprint on the template coadd, do not clip
1626 filteredMaskSpanSetList = []
1627 for span in maskSpanSetList:
1628 doKeep = True
1629 for footprint in footprintsToExclude.positive.getFootprints():
1630 if footprint.spans.contains(span):
1631 doKeep = False
1632 break
1633 if doKeep:
1634 filteredMaskSpanSetList.append(span)
1635 maskSpanSetList = filteredMaskSpanSetList
1637 return maskSpanSetList
1639 def _readAndComputeWarpDiff(self, warpRef, imageScaler, templateCoadd):
1640 """Fetch a warp from the butler and return a warpDiff.
1641 Parameters
1642 ----------
1643 warpRef : `lsst.daf.butler.DeferredDatasetHandle`
1644 Handle for the warp.
1645 imageScaler : `lsst.pipe.tasks.scaleZeroPoint.ImageScaler`
1646 An image scaler object.
1647 templateCoadd : `lsst.afw.image.Exposure`
1648 Exposure to be substracted from the scaled warp.
1649 Returns
1650 -------
1651 warp : `lsst.afw.image.Exposure`
1652 Exposure of the image difference between the warp and template.
1653 """
1655 # If the PSF-Matched warp did not exist for this direct warp
1656 # None is holding its place to maintain order in Gen 3
1657 if warpRef is None:
1658 return None
1660 warp = warpRef.get()
1661 # direct image scaler OK for PSF-matched Warp
1662 imageScaler.scaleMaskedImage(warp.getMaskedImage())
1663 mi = warp.getMaskedImage()
1664 if self.config.doScaleWarpVariance:
1665 try:
1666 self.scaleWarpVariance.run(mi)
1667 except Exception as exc:
1668 self.log.warning("Unable to rescale variance of warp (%s); leaving it as-is", exc)
1669 mi -= templateCoadd.getMaskedImage()
1670 return warp