Coverage for python/lsst/drp/tasks/assemble_coadd.py: 16%
633 statements
« prev ^ index » next coverage.py v7.3.1, created at 2023-10-02 10:27 +0000
« prev ^ index » next coverage.py v7.3.1, created at 2023-10-02 10:27 +0000
1# This file is part of drp_tasks.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22__all__ = ["AssembleCoaddTask", "AssembleCoaddConnections", "AssembleCoaddConfig",
23 "CompareWarpAssembleCoaddTask", "CompareWarpAssembleCoaddConfig"]
25import copy
26import numpy
27import warnings
28import logging
29import lsst.pex.config as pexConfig
30import lsst.pex.exceptions as pexExceptions
31import lsst.geom as geom
32import lsst.afw.geom as afwGeom
33import lsst.afw.image as afwImage
34import lsst.afw.math as afwMath
35import lsst.afw.table as afwTable
36import lsst.coadd.utils as coaddUtils
37import lsst.pipe.base as pipeBase
38import lsst.meas.algorithms as measAlg
39import lsstDebug
40import lsst.utils as utils
41from lsst.skymap import BaseSkyMap
42from lsst.pipe.tasks.coaddBase import CoaddBaseTask, makeSkyInfo, reorderAndPadList, subBBoxIter
43from lsst.pipe.tasks.interpImage import InterpImageTask
44from lsst.pipe.tasks.scaleZeroPoint import ScaleZeroPointTask
45from lsst.pipe.tasks.maskStreaks import MaskStreaksTask
46from lsst.pipe.tasks.healSparseMapping import HealSparseInputMapTask
47from lsst.meas.algorithms import SourceDetectionTask, AccumulatorMeanStack, ScaleVarianceTask
48from lsst.utils.timer import timeMethod
49from deprecated.sphinx import deprecated
51log = logging.getLogger(__name__)
54class AssembleCoaddConnections(pipeBase.PipelineTaskConnections,
55 dimensions=("tract", "patch", "band", "skymap"),
56 defaultTemplates={"inputCoaddName": "deep",
57 "outputCoaddName": "deep",
58 "warpType": "direct",
59 "warpTypeSuffix": ""}):
61 inputWarps = pipeBase.connectionTypes.Input(
62 doc=("Input list of warps to be assemebled i.e. stacked."
63 "WarpType (e.g. direct, psfMatched) is controlled by the "
64 "warpType config parameter"),
65 name="{inputCoaddName}Coadd_{warpType}Warp",
66 storageClass="ExposureF",
67 dimensions=("tract", "patch", "skymap", "visit", "instrument"),
68 deferLoad=True,
69 multiple=True
70 )
71 skyMap = pipeBase.connectionTypes.Input(
72 doc="Input definition of geometry/bbox and projection/wcs for coadded "
73 "exposures",
74 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
75 storageClass="SkyMap",
76 dimensions=("skymap", ),
77 )
78 selectedVisits = pipeBase.connectionTypes.Input(
79 doc="Selected visits to be coadded.",
80 name="{outputCoaddName}Visits",
81 storageClass="StructuredDataDict",
82 dimensions=("instrument", "tract", "patch", "skymap", "band")
83 )
84 brightObjectMask = pipeBase.connectionTypes.PrerequisiteInput(
85 doc=("Input Bright Object Mask mask produced with external catalogs "
86 "to be applied to the mask plane BRIGHT_OBJECT."
87 ),
88 name="brightObjectMask",
89 storageClass="ObjectMaskCatalog",
90 dimensions=("tract", "patch", "skymap", "band"),
91 minimum=0,
92 )
93 coaddExposure = pipeBase.connectionTypes.Output(
94 doc="Output coadded exposure, produced by stacking input warps",
95 name="{outputCoaddName}Coadd{warpTypeSuffix}",
96 storageClass="ExposureF",
97 dimensions=("tract", "patch", "skymap", "band"),
98 )
99 nImage = pipeBase.connectionTypes.Output(
100 doc="Output image of number of input images per pixel",
101 name="{outputCoaddName}Coadd_nImage",
102 storageClass="ImageU",
103 dimensions=("tract", "patch", "skymap", "band"),
104 )
105 inputMap = pipeBase.connectionTypes.Output(
106 doc="Output healsparse map of input images",
107 name="{outputCoaddName}Coadd_inputMap",
108 storageClass="HealSparseMap",
109 dimensions=("tract", "patch", "skymap", "band"),
110 )
112 def __init__(self, *, config=None):
113 super().__init__(config=config)
115 if not config.doMaskBrightObjects:
116 self.prerequisiteInputs.remove("brightObjectMask")
118 if not config.doSelectVisits:
119 self.inputs.remove("selectedVisits")
121 if not config.doNImage:
122 self.outputs.remove("nImage")
124 if not self.config.doInputMap:
125 self.outputs.remove("inputMap")
128class AssembleCoaddConfig(CoaddBaseTask.ConfigClass, pipeBase.PipelineTaskConfig,
129 pipelineConnections=AssembleCoaddConnections):
130 warpType = pexConfig.Field(
131 doc="Warp name: one of 'direct' or 'psfMatched'",
132 dtype=str,
133 default="direct",
134 )
135 subregionSize = pexConfig.ListField(
136 dtype=int,
137 doc="Width, height of stack subregion size; "
138 "make small enough that a full stack of images will fit into memory "
139 " at once.",
140 length=2,
141 default=(2000, 2000),
142 )
143 statistic = pexConfig.Field(
144 dtype=str,
145 doc="Main stacking statistic for aggregating over the epochs.",
146 default="MEANCLIP",
147 )
148 doOnlineForMean = pexConfig.Field(
149 dtype=bool,
150 doc="Perform online coaddition when statistic=\"MEAN\" to save memory?",
151 default=False,
152 )
153 doSigmaClip = pexConfig.Field(
154 dtype=bool,
155 doc="Perform sigma clipped outlier rejection with MEANCLIP statistic?",
156 deprecated=True,
157 default=False,
158 )
159 sigmaClip = pexConfig.Field(
160 dtype=float,
161 doc="Sigma for outlier rejection; ignored if non-clipping statistic "
162 "selected.",
163 default=3.0,
164 )
165 clipIter = pexConfig.Field(
166 dtype=int,
167 doc="Number of iterations of outlier rejection; ignored if "
168 "non-clipping statistic selected.",
169 default=2,
170 )
171 calcErrorFromInputVariance = pexConfig.Field(
172 dtype=bool,
173 doc="Calculate coadd variance from input variance by stacking "
174 "statistic. Passed to "
175 "StatisticsControl.setCalcErrorFromInputVariance()",
176 default=True,
177 )
178 scaleZeroPoint = pexConfig.ConfigurableField(
179 target=ScaleZeroPointTask,
180 doc="Task to adjust the photometric zero point of the coadd temp "
181 "exposures",
182 )
183 doInterp = pexConfig.Field(
184 doc="Interpolate over NaN pixels? Also extrapolate, if necessary, but "
185 "the results are ugly.",
186 dtype=bool,
187 default=True,
188 )
189 interpImage = pexConfig.ConfigurableField(
190 target=InterpImageTask,
191 doc="Task to interpolate (and extrapolate) over NaN pixels",
192 )
193 doWrite = pexConfig.Field(
194 doc="Persist coadd?",
195 dtype=bool,
196 default=True,
197 )
198 doNImage = pexConfig.Field(
199 doc="Create image of number of contributing exposures for each pixel",
200 dtype=bool,
201 default=False,
202 )
203 doUsePsfMatchedPolygons = pexConfig.Field(
204 doc="Use ValidPolygons from shrunk Psf-Matched Calexps? Should be set "
205 "to True by CompareWarp only.",
206 dtype=bool,
207 default=False,
208 )
209 maskPropagationThresholds = pexConfig.DictField(
210 keytype=str,
211 itemtype=float,
212 doc=("Threshold (in fractional weight) of rejection at which we "
213 "propagate a mask plane to the coadd; that is, we set the mask "
214 "bit on the coadd if the fraction the rejected frames "
215 "would have contributed exceeds this value."),
216 default={"SAT": 0.1},
217 )
218 removeMaskPlanes = pexConfig.ListField(
219 dtype=str,
220 default=["NOT_DEBLENDED"],
221 doc="Mask planes to remove before coadding",
222 )
223 doMaskBrightObjects = pexConfig.Field(
224 dtype=bool,
225 default=False,
226 doc="Set mask and flag bits for bright objects?",
227 )
228 brightObjectMaskName = pexConfig.Field(
229 dtype=str,
230 default="BRIGHT_OBJECT",
231 doc="Name of mask bit used for bright objects",
232 )
233 coaddPsf = pexConfig.ConfigField(
234 doc="Configuration for CoaddPsf",
235 dtype=measAlg.CoaddPsfConfig,
236 )
237 doAttachTransmissionCurve = pexConfig.Field(
238 dtype=bool,
239 default=False,
240 optional=False,
241 doc=("Attach a piecewise TransmissionCurve for the coadd? "
242 "(requires all input Exposures to have TransmissionCurves).")
243 )
244 hasFakes = pexConfig.Field(
245 dtype=bool,
246 default=False,
247 doc="Should be set to True if fake sources have been inserted into the input data."
248 )
249 doSelectVisits = pexConfig.Field(
250 doc="Coadd only visits selected by a SelectVisitsTask",
251 dtype=bool,
252 default=False,
253 )
254 doInputMap = pexConfig.Field(
255 doc="Create a bitwise map of coadd inputs",
256 dtype=bool,
257 default=False,
258 )
259 inputMapper = pexConfig.ConfigurableField(
260 doc="Input map creation subtask.",
261 target=HealSparseInputMapTask,
262 )
264 def setDefaults(self):
265 super().setDefaults()
266 self.badMaskPlanes = ["NO_DATA", "BAD", "SAT", "EDGE"]
268 def validate(self):
269 super().validate()
270 if self.doPsfMatch: # TODO: Remove this in DM-39841
271 # Backwards compatibility.
272 # Configs do not have loggers
273 log.warning("Config doPsfMatch deprecated. Setting warpType='psfMatched'")
274 self.warpType = 'psfMatched'
275 if self.doSigmaClip and self.statistic != "MEANCLIP":
276 log.warning('doSigmaClip deprecated. To replicate behavior, setting statistic to "MEANCLIP"')
277 self.statistic = "MEANCLIP"
278 if self.doInterp and self.statistic not in ['MEAN', 'MEDIAN', 'MEANCLIP', 'VARIANCE', 'VARIANCECLIP']:
279 raise ValueError("Must set doInterp=False for statistic=%s, which does not "
280 "compute and set a non-zero coadd variance estimate." % (self.statistic))
282 unstackableStats = ['NOTHING', 'ERROR', 'ORMASK']
283 if not hasattr(afwMath.Property, self.statistic) or self.statistic in unstackableStats:
284 stackableStats = [str(k) for k in afwMath.Property.__members__.keys()
285 if str(k) not in unstackableStats]
286 raise ValueError("statistic %s is not allowed. Please choose one of %s."
287 % (self.statistic, stackableStats))
290class AssembleCoaddTask(CoaddBaseTask, pipeBase.PipelineTask):
291 """Assemble a coadded image from a set of warps.
293 Each Warp that goes into a coadd will typically have an independent
294 photometric zero-point. Therefore, we must scale each Warp to set it to
295 a common photometric zeropoint. WarpType may be one of 'direct' or
296 'psfMatched', and the boolean configs `config.makeDirect` and
297 `config.makePsfMatched` set which of the warp types will be coadded.
298 The coadd is computed as a mean with optional outlier rejection.
299 Criteria for outlier rejection are set in `AssembleCoaddConfig`.
300 Finally, Warps can have bad 'NaN' pixels which received no input from the
301 source calExps. We interpolate over these bad (NaN) pixels.
303 `AssembleCoaddTask` uses several sub-tasks. These are
305 - `~lsst.pipe.tasks.ScaleZeroPointTask`
306 - create and use an ``imageScaler`` object to scale the photometric
307 zeropoint for each Warp
308 - `~lsst.pipe.tasks.InterpImageTask`
309 - interpolate across bad pixels (NaN) in the final coadd
311 You can retarget these subtasks if you wish.
313 Raises
314 ------
315 RuntimeError
316 Raised if unable to define mask plane for bright objects.
318 Notes
319 -----
320 Debugging:
321 `AssembleCoaddTask` has no debug variables of its own. Some of the
322 subtasks may support `~lsst.base.lsstDebug` variables. See the
323 documentation for the subtasks for further information.
324 """
326 ConfigClass = AssembleCoaddConfig
327 _DefaultName = "assembleCoadd"
329 def __init__(self, *args, **kwargs):
330 # TODO: DM-17415 better way to handle previously allowed passed args
331 # e.g.`AssembleCoaddTask(config)`
332 if args:
333 argNames = ["config", "name", "parentTask", "log"]
334 kwargs.update({k: v for k, v in zip(argNames, args)})
335 warnings.warn("AssembleCoadd received positional args, and casting them as kwargs: %s. "
336 "PipelineTask will not take positional args" % argNames, FutureWarning,
337 stacklevel=2)
339 super().__init__(**kwargs)
340 self.makeSubtask("interpImage")
341 self.makeSubtask("scaleZeroPoint")
343 if self.config.doMaskBrightObjects:
344 mask = afwImage.Mask()
345 try:
346 self.brightObjectBitmask = 1 << mask.addMaskPlane(self.config.brightObjectMaskName)
347 except pexExceptions.LsstCppException:
348 raise RuntimeError("Unable to define mask plane for bright objects; planes used are %s" %
349 mask.getMaskPlaneDict().keys())
350 del mask
352 if self.config.doInputMap:
353 self.makeSubtask("inputMapper")
355 self.warpType = self.config.warpType
357 def runQuantum(self, butlerQC, inputRefs, outputRefs):
358 inputData = butlerQC.get(inputRefs)
360 # Construct skyInfo expected by run
361 # Do not remove skyMap from inputData in case _makeSupplementaryData
362 # needs it
363 skyMap = inputData["skyMap"]
364 outputDataId = butlerQC.quantum.dataId
366 inputData['skyInfo'] = makeSkyInfo(skyMap,
367 tractId=outputDataId['tract'],
368 patchId=outputDataId['patch'])
370 if self.config.doSelectVisits:
371 warpRefList = self.filterWarps(inputData['inputWarps'], inputData['selectedVisits'])
372 else:
373 warpRefList = inputData['inputWarps']
375 inputs = self.prepareInputs(warpRefList)
376 self.log.info("Found %d %s", len(inputs.tempExpRefList),
377 self.getTempExpDatasetName(self.warpType))
378 if len(inputs.tempExpRefList) == 0:
379 raise pipeBase.NoWorkFound("No coadd temporary exposures found")
381 supplementaryData = self._makeSupplementaryData(butlerQC, inputRefs, outputRefs)
382 retStruct = self.run(inputData['skyInfo'], inputs.tempExpRefList, inputs.imageScalerList,
383 inputs.weightList, supplementaryData=supplementaryData)
385 inputData.setdefault('brightObjectMask', None)
386 if self.config.doMaskBrightObjects and inputData["brightObjectMask"] is None:
387 log.warning("doMaskBrightObjects is set to True, but brightObjectMask not loaded")
388 self.processResults(retStruct.coaddExposure, inputData['brightObjectMask'], outputDataId)
390 if self.config.doWrite:
391 butlerQC.put(retStruct, outputRefs)
392 return retStruct
394 def processResults(self, coaddExposure, brightObjectMasks=None, dataId=None):
395 """Interpolate over missing data and mask bright stars.
397 Parameters
398 ----------
399 coaddExposure : `lsst.afw.image.Exposure`
400 The coadded exposure to process.
401 brightObjectMasks : `lsst.afw.table` or `None`, optional
402 Table of bright objects to mask.
403 dataId : `lsst.daf.butler.DataId` or `None`, optional
404 Data identification.
405 """
406 if self.config.doInterp:
407 self.interpImage.run(coaddExposure.getMaskedImage(), planeName="NO_DATA")
408 # The variance must be positive; work around for DM-3201.
409 varArray = coaddExposure.variance.array
410 with numpy.errstate(invalid="ignore"):
411 varArray[:] = numpy.where(varArray > 0, varArray, numpy.inf)
413 if self.config.doMaskBrightObjects:
414 self.setBrightObjectMasks(coaddExposure, brightObjectMasks, dataId)
416 def _makeSupplementaryData(self, butlerQC, inputRefs, outputRefs):
417 """Make additional inputs to run() specific to subclasses (Gen3).
419 Duplicates interface of `runQuantum` method.
420 Available to be implemented by subclasses only if they need the
421 coadd dataRef for performing preliminary processing before
422 assembling the coadd.
424 Parameters
425 ----------
426 butlerQC : `~lsst.pipe.base.ButlerQuantumContext`
427 Gen3 Butler object for fetching additional data products before
428 running the Task specialized for quantum being processed.
429 inputRefs : `~lsst.pipe.base.InputQuantizedConnection`
430 Attributes are the names of the connections describing input
431 dataset types. Values are DatasetRefs that task consumes for the
432 corresponding dataset type. DataIds are guaranteed to match data
433 objects in ``inputData``.
434 outputRefs : `~lsst.pipe.base.OutputQuantizedConnection`
435 Attributes are the names of the connections describing output
436 dataset types. Values are DatasetRefs that task is to produce
437 for the corresponding dataset type.
438 """
439 return pipeBase.Struct()
441 @deprecated(
442 reason="makeSupplementaryDataGen3 is deprecated in favor of _makeSupplementaryData",
443 version="v25.0",
444 category=FutureWarning
445 )
446 def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs):
447 return self._makeSupplementaryData(butlerQC, inputRefs, outputRefs)
449 def prepareInputs(self, refList):
450 """Prepare the input warps for coaddition by measuring the weight for
451 each warp and the scaling for the photometric zero point.
453 Each Warp has its own photometric zeropoint and background variance.
454 Before coadding these Warps together, compute a scale factor to
455 normalize the photometric zeropoint and compute the weight for each
456 Warp.
458 Parameters
459 ----------
460 refList : `list`
461 List of data references to tempExp.
463 Returns
464 -------
465 result : `~lsst.pipe.base.Struct`
466 Results as a struct with attributes:
468 ``tempExprefList``
469 `list` of data references to tempExp.
470 ``weightList``
471 `list` of weightings.
472 ``imageScalerList``
473 `list` of image scalers.
474 """
475 statsCtrl = afwMath.StatisticsControl()
476 statsCtrl.setNumSigmaClip(self.config.sigmaClip)
477 statsCtrl.setNumIter(self.config.clipIter)
478 statsCtrl.setAndMask(self.getBadPixelMask())
479 statsCtrl.setNanSafe(True)
480 # compute tempExpRefList: a list of tempExpRef that actually exist
481 # and weightList: a list of the weight of the associated coadd tempExp
482 # and imageScalerList: a list of scale factors for the associated coadd
483 # tempExp.
484 tempExpRefList = []
485 weightList = []
486 imageScalerList = []
487 tempExpName = self.getTempExpDatasetName(self.warpType)
488 for tempExpRef in refList:
489 tempExp = tempExpRef.get()
490 # Ignore any input warp that is empty of data
491 if numpy.isnan(tempExp.image.array).all():
492 continue
493 maskedImage = tempExp.getMaskedImage()
494 imageScaler = self.scaleZeroPoint.computeImageScaler(
495 exposure=tempExp,
496 dataRef=tempExpRef, # FIXME
497 )
498 try:
499 imageScaler.scaleMaskedImage(maskedImage)
500 except Exception as e:
501 self.log.warning("Scaling failed for %s (skipping it): %s", tempExpRef.dataId, e)
502 continue
503 statObj = afwMath.makeStatistics(maskedImage.getVariance(), maskedImage.getMask(),
504 afwMath.MEANCLIP, statsCtrl)
505 meanVar, meanVarErr = statObj.getResult(afwMath.MEANCLIP)
506 weight = 1.0 / float(meanVar)
507 if not numpy.isfinite(weight):
508 self.log.warning("Non-finite weight for %s: skipping", tempExpRef.dataId)
509 continue
510 self.log.info("Weight of %s %s = %0.3f", tempExpName, tempExpRef.dataId, weight)
512 del maskedImage
513 del tempExp
515 tempExpRefList.append(tempExpRef)
516 weightList.append(weight)
517 imageScalerList.append(imageScaler)
519 return pipeBase.Struct(tempExpRefList=tempExpRefList, weightList=weightList,
520 imageScalerList=imageScalerList)
522 def prepareStats(self, mask=None):
523 """Prepare the statistics for coadding images.
525 Parameters
526 ----------
527 mask : `int`, optional
528 Bit mask value to exclude from coaddition.
530 Returns
531 -------
532 stats : `~lsst.pipe.base.Struct`
533 Statistics as a struct with attributes:
535 ``statsCtrl``
536 Statistics control object for coadd
537 (`~lsst.afw.math.StatisticsControl`).
538 ``statsFlags``
539 Statistic for coadd (`~lsst.afw.math.Property`).
540 """
541 if mask is None:
542 mask = self.getBadPixelMask()
543 statsCtrl = afwMath.StatisticsControl()
544 statsCtrl.setNumSigmaClip(self.config.sigmaClip)
545 statsCtrl.setNumIter(self.config.clipIter)
546 statsCtrl.setAndMask(mask)
547 statsCtrl.setNanSafe(True)
548 statsCtrl.setWeighted(True)
549 statsCtrl.setCalcErrorFromInputVariance(self.config.calcErrorFromInputVariance)
550 for plane, threshold in self.config.maskPropagationThresholds.items():
551 bit = afwImage.Mask.getMaskPlane(plane)
552 statsCtrl.setMaskPropagationThreshold(bit, threshold)
553 statsFlags = afwMath.stringToStatisticsProperty(self.config.statistic)
554 return pipeBase.Struct(ctrl=statsCtrl, flags=statsFlags)
556 @timeMethod
557 def run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
558 altMaskList=None, mask=None, supplementaryData=None):
559 """Assemble a coadd from input warps.
561 Assemble the coadd using the provided list of coaddTempExps. Since
562 the full coadd covers a patch (a large area), the assembly is
563 performed over small areas on the image at a time in order to
564 conserve memory usage. Iterate over subregions within the outer
565 bbox of the patch using `assembleSubregion` to stack the corresponding
566 subregions from the coaddTempExps with the statistic specified.
567 Set the edge bits the coadd mask based on the weight map.
569 Parameters
570 ----------
571 skyInfo : `~lsst.pipe.base.Struct`
572 Struct with geometric information about the patch.
573 tempExpRefList : `list`
574 List of data references to Warps (previously called CoaddTempExps).
575 imageScalerList : `list`
576 List of image scalers.
577 weightList : `list`
578 List of weights.
579 altMaskList : `list`, optional
580 List of alternate masks to use rather than those stored with
581 tempExp.
582 mask : `int`, optional
583 Bit mask value to exclude from coaddition.
584 supplementaryData : `~lsst.pipe.base.Struct`, optional
585 Struct with additional data products needed to assemble coadd.
586 Only used by subclasses that implement ``_makeSupplementaryData``
587 and override `run`.
589 Returns
590 -------
591 result : `~lsst.pipe.base.Struct`
592 Results as a struct with attributes:
594 ``coaddExposure``
595 Coadded exposure (`~lsst.afw.image.Exposure`).
596 ``nImage``
597 Exposure count image (`~lsst.afw.image.Image`), if requested.
598 ``inputMap``
599 Bit-wise map of inputs, if requested.
600 ``warpRefList``
601 Input list of refs to the warps
602 (`~lsst.daf.butler.DeferredDatasetHandle`) (unmodified).
603 ``imageScalerList``
604 Input list of image scalers (`list`) (unmodified).
605 ``weightList``
606 Input list of weights (`list`) (unmodified).
608 Raises
609 ------
610 lsst.pipe.base.NoWorkFound
611 Raised if no data references are provided.
612 """
613 tempExpName = self.getTempExpDatasetName(self.warpType)
614 self.log.info("Assembling %s %s", len(tempExpRefList), tempExpName)
615 if not tempExpRefList:
616 raise pipeBase.NoWorkFound("No exposures provided for co-addition.")
618 stats = self.prepareStats(mask=mask)
620 if altMaskList is None:
621 altMaskList = [None]*len(tempExpRefList)
623 coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
624 coaddExposure.setPhotoCalib(self.scaleZeroPoint.getPhotoCalib())
625 coaddExposure.getInfo().setCoaddInputs(self.inputRecorder.makeCoaddInputs())
626 self.assembleMetadata(coaddExposure, tempExpRefList, weightList)
627 coaddMaskedImage = coaddExposure.getMaskedImage()
628 subregionSizeArr = self.config.subregionSize
629 subregionSize = geom.Extent2I(subregionSizeArr[0], subregionSizeArr[1])
630 # if nImage is requested, create a zero one which can be passed to
631 # assembleSubregion.
632 if self.config.doNImage:
633 nImage = afwImage.ImageU(skyInfo.bbox)
634 else:
635 nImage = None
636 # If inputMap is requested, create the initial version that can be
637 # masked in assembleSubregion.
638 if self.config.doInputMap:
639 self.inputMapper.build_ccd_input_map(skyInfo.bbox,
640 skyInfo.wcs,
641 coaddExposure.getInfo().getCoaddInputs().ccds)
643 if self.config.doOnlineForMean and self.config.statistic == "MEAN":
644 try:
645 self.assembleOnlineMeanCoadd(coaddExposure, tempExpRefList, imageScalerList,
646 weightList, altMaskList, stats.ctrl,
647 nImage=nImage)
648 except Exception as e:
649 self.log.exception("Cannot compute online coadd %s", e)
650 raise
651 else:
652 for subBBox in subBBoxIter(skyInfo.bbox, subregionSize):
653 try:
654 self.assembleSubregion(coaddExposure, subBBox, tempExpRefList, imageScalerList,
655 weightList, altMaskList, stats.flags, stats.ctrl,
656 nImage=nImage)
657 except Exception as e:
658 self.log.exception("Cannot compute coadd %s: %s", subBBox, e)
659 raise
661 # If inputMap is requested, we must finalize the map after the
662 # accumulation.
663 if self.config.doInputMap:
664 self.inputMapper.finalize_ccd_input_map_mask()
665 inputMap = self.inputMapper.ccd_input_map
666 else:
667 inputMap = None
669 self.setInexactPsf(coaddMaskedImage.getMask())
670 # Despite the name, the following doesn't really deal with "EDGE"
671 # pixels: it identifies pixels that didn't receive any unmasked inputs
672 # (as occurs around the edge of the field).
673 coaddUtils.setCoaddEdgeBits(coaddMaskedImage.getMask(), coaddMaskedImage.getVariance())
674 return pipeBase.Struct(coaddExposure=coaddExposure, nImage=nImage,
675 warpRefList=tempExpRefList, imageScalerList=imageScalerList,
676 weightList=weightList, inputMap=inputMap)
678 def assembleMetadata(self, coaddExposure, tempExpRefList, weightList):
679 """Set the metadata for the coadd.
681 This basic implementation sets the filter from the first input.
683 Parameters
684 ----------
685 coaddExposure : `lsst.afw.image.Exposure`
686 The target exposure for the coadd.
687 tempExpRefList : `list`
688 List of data references to tempExp.
689 weightList : `list`
690 List of weights.
692 Raises
693 ------
694 AssertionError
695 Raised if there is a length mismatch.
696 """
697 assert len(tempExpRefList) == len(weightList), "Length mismatch"
699 # We load a single pixel of each coaddTempExp, because we just want to
700 # get at the metadata (and we need more than just the PropertySet that
701 # contains the header), which is not possible with the current butler
702 # (see #2777).
703 bbox = geom.Box2I(coaddExposure.getBBox().getMin(), geom.Extent2I(1, 1))
705 tempExpList = [tempExpRef.get(parameters={'bbox': bbox}) for tempExpRef in tempExpRefList]
707 numCcds = sum(len(tempExp.getInfo().getCoaddInputs().ccds) for tempExp in tempExpList)
709 # Set the coadd FilterLabel to the band of the first input exposure:
710 # Coadds are calibrated, so the physical label is now meaningless.
711 coaddExposure.setFilter(afwImage.FilterLabel(tempExpList[0].getFilter().bandLabel))
712 coaddInputs = coaddExposure.getInfo().getCoaddInputs()
713 coaddInputs.ccds.reserve(numCcds)
714 coaddInputs.visits.reserve(len(tempExpList))
716 for tempExp, weight in zip(tempExpList, weightList):
717 self.inputRecorder.addVisitToCoadd(coaddInputs, tempExp, weight)
719 if self.config.doUsePsfMatchedPolygons:
720 self.shrinkValidPolygons(coaddInputs)
722 coaddInputs.visits.sort()
723 coaddInputs.ccds.sort()
724 if self.warpType == "psfMatched":
725 # The modelPsf BBox for a psfMatchedWarp/coaddTempExp was
726 # dynamically defined by ModelPsfMatchTask as the square box
727 # bounding its spatially-variable, pre-matched WarpedPsf.
728 # Likewise, set the PSF of a PSF-Matched Coadd to the modelPsf
729 # having the maximum width (sufficient because square)
730 modelPsfList = [tempExp.getPsf() for tempExp in tempExpList]
731 modelPsfWidthList = [modelPsf.computeBBox(modelPsf.getAveragePosition()).getWidth()
732 for modelPsf in modelPsfList]
733 psf = modelPsfList[modelPsfWidthList.index(max(modelPsfWidthList))]
734 else:
735 psf = measAlg.CoaddPsf(coaddInputs.ccds, coaddExposure.getWcs(),
736 self.config.coaddPsf.makeControl())
737 coaddExposure.setPsf(psf)
738 apCorrMap = measAlg.makeCoaddApCorrMap(coaddInputs.ccds, coaddExposure.getBBox(afwImage.PARENT),
739 coaddExposure.getWcs())
740 coaddExposure.getInfo().setApCorrMap(apCorrMap)
741 if self.config.doAttachTransmissionCurve:
742 transmissionCurve = measAlg.makeCoaddTransmissionCurve(coaddExposure.getWcs(), coaddInputs.ccds)
743 coaddExposure.getInfo().setTransmissionCurve(transmissionCurve)
745 def assembleSubregion(self, coaddExposure, bbox, tempExpRefList, imageScalerList, weightList,
746 altMaskList, statsFlags, statsCtrl, nImage=None):
747 """Assemble the coadd for a sub-region.
749 For each coaddTempExp, check for (and swap in) an alternative mask
750 if one is passed. Remove mask planes listed in
751 `config.removeMaskPlanes`. Finally, stack the actual exposures using
752 `lsst.afw.math.statisticsStack` with the statistic specified by
753 statsFlags. Typically, the statsFlag will be one of lsst.afw.math.MEAN
754 for a mean-stack or `lsst.afw.math.MEANCLIP` for outlier rejection
755 using an N-sigma clipped mean where N and iterations are specified by
756 statsCtrl. Assign the stacked subregion back to the coadd.
758 Parameters
759 ----------
760 coaddExposure : `lsst.afw.image.Exposure`
761 The target exposure for the coadd.
762 bbox : `lsst.geom.Box`
763 Sub-region to coadd.
764 tempExpRefList : `list`
765 List of data reference to tempExp.
766 imageScalerList : `list`
767 List of image scalers.
768 weightList : `list`
769 List of weights.
770 altMaskList : `list`
771 List of alternate masks to use rather than those stored with
772 tempExp, or None. Each element is dict with keys = mask plane
773 name to which to add the spans.
774 statsFlags : `lsst.afw.math.Property`
775 Property object for statistic for coadd.
776 statsCtrl : `lsst.afw.math.StatisticsControl`
777 Statistics control object for coadd.
778 nImage : `lsst.afw.image.ImageU`, optional
779 Keeps track of exposure count for each pixel.
780 """
781 self.log.debug("Computing coadd over %s", bbox)
783 coaddExposure.mask.addMaskPlane("REJECTED")
784 coaddExposure.mask.addMaskPlane("CLIPPED")
785 coaddExposure.mask.addMaskPlane("SENSOR_EDGE")
786 maskMap = self.setRejectedMaskMapping(statsCtrl)
787 clipped = afwImage.Mask.getPlaneBitMask("CLIPPED")
788 maskedImageList = []
789 if nImage is not None:
790 subNImage = afwImage.ImageU(bbox.getWidth(), bbox.getHeight())
791 for tempExpRef, imageScaler, altMask in zip(tempExpRefList, imageScalerList, altMaskList):
793 exposure = tempExpRef.get(parameters={'bbox': bbox})
795 maskedImage = exposure.getMaskedImage()
796 mask = maskedImage.getMask()
797 if altMask is not None:
798 self.applyAltMaskPlanes(mask, altMask)
799 imageScaler.scaleMaskedImage(maskedImage)
801 # Add 1 for each pixel which is not excluded by the exclude mask.
802 # In legacyCoadd, pixels may also be excluded by
803 # afwMath.statisticsStack.
804 if nImage is not None:
805 subNImage.getArray()[maskedImage.getMask().getArray() & statsCtrl.getAndMask() == 0] += 1
806 if self.config.removeMaskPlanes:
807 self.removeMaskPlanes(maskedImage)
808 maskedImageList.append(maskedImage)
810 if self.config.doInputMap:
811 visit = exposure.getInfo().getCoaddInputs().visits[0].getId()
812 self.inputMapper.mask_warp_bbox(bbox, visit, mask, statsCtrl.getAndMask())
814 with self.timer("stack"):
815 coaddSubregion = afwMath.statisticsStack(maskedImageList, statsFlags, statsCtrl, weightList,
816 clipped, # also set output to CLIPPED if sigma-clipped
817 maskMap)
818 coaddExposure.maskedImage.assign(coaddSubregion, bbox)
819 if nImage is not None:
820 nImage.assign(subNImage, bbox)
822 def assembleOnlineMeanCoadd(self, coaddExposure, tempExpRefList, imageScalerList, weightList,
823 altMaskList, statsCtrl, nImage=None):
824 """Assemble the coadd using the "online" method.
826 This method takes a running sum of images and weights to save memory.
827 It only works for MEAN statistics.
829 Parameters
830 ----------
831 coaddExposure : `lsst.afw.image.Exposure`
832 The target exposure for the coadd.
833 tempExpRefList : `list`
834 List of data reference to tempExp.
835 imageScalerList : `list`
836 List of image scalers.
837 weightList : `list`
838 List of weights.
839 altMaskList : `list`
840 List of alternate masks to use rather than those stored with
841 tempExp, or None. Each element is dict with keys = mask plane
842 name to which to add the spans.
843 statsCtrl : `lsst.afw.math.StatisticsControl`
844 Statistics control object for coadd.
845 nImage : `lsst.afw.image.ImageU`, optional
846 Keeps track of exposure count for each pixel.
847 """
848 self.log.debug("Computing online coadd.")
850 coaddExposure.mask.addMaskPlane("REJECTED")
851 coaddExposure.mask.addMaskPlane("CLIPPED")
852 coaddExposure.mask.addMaskPlane("SENSOR_EDGE")
853 maskMap = self.setRejectedMaskMapping(statsCtrl)
854 thresholdDict = AccumulatorMeanStack.stats_ctrl_to_threshold_dict(statsCtrl)
856 bbox = coaddExposure.maskedImage.getBBox()
858 stacker = AccumulatorMeanStack(
859 coaddExposure.image.array.shape,
860 statsCtrl.getAndMask(),
861 mask_threshold_dict=thresholdDict,
862 mask_map=maskMap,
863 no_good_pixels_mask=statsCtrl.getNoGoodPixelsMask(),
864 calc_error_from_input_variance=self.config.calcErrorFromInputVariance,
865 compute_n_image=(nImage is not None)
866 )
868 for tempExpRef, imageScaler, altMask, weight in zip(tempExpRefList,
869 imageScalerList,
870 altMaskList,
871 weightList):
872 exposure = tempExpRef.get()
873 maskedImage = exposure.getMaskedImage()
874 mask = maskedImage.getMask()
875 if altMask is not None:
876 self.applyAltMaskPlanes(mask, altMask)
877 imageScaler.scaleMaskedImage(maskedImage)
878 if self.config.removeMaskPlanes:
879 self.removeMaskPlanes(maskedImage)
881 stacker.add_masked_image(maskedImage, weight=weight)
883 if self.config.doInputMap:
884 visit = exposure.getInfo().getCoaddInputs().visits[0].getId()
885 self.inputMapper.mask_warp_bbox(bbox, visit, mask, statsCtrl.getAndMask())
887 stacker.fill_stacked_masked_image(coaddExposure.maskedImage)
889 if nImage is not None:
890 nImage.array[:, :] = stacker.n_image
892 def removeMaskPlanes(self, maskedImage):
893 """Unset the mask of an image for mask planes specified in the config.
895 Parameters
896 ----------
897 maskedImage : `lsst.afw.image.MaskedImage`
898 The masked image to be modified.
900 Raises
901 ------
902 InvalidParameterError
903 Raised if no mask plane with that name was found.
904 """
905 mask = maskedImage.getMask()
906 for maskPlane in self.config.removeMaskPlanes:
907 try:
908 mask &= ~mask.getPlaneBitMask(maskPlane)
909 except pexExceptions.InvalidParameterError:
910 self.log.debug("Unable to remove mask plane %s: no mask plane with that name was found.",
911 maskPlane)
913 @staticmethod
914 def setRejectedMaskMapping(statsCtrl):
915 """Map certain mask planes of the warps to new planes for the coadd.
917 If a pixel is rejected due to a mask value other than EDGE, NO_DATA,
918 or CLIPPED, set it to REJECTED on the coadd.
919 If a pixel is rejected due to EDGE, set the coadd pixel to SENSOR_EDGE.
920 If a pixel is rejected due to CLIPPED, set the coadd pixel to CLIPPED.
922 Parameters
923 ----------
924 statsCtrl : `lsst.afw.math.StatisticsControl`
925 Statistics control object for coadd.
927 Returns
928 -------
929 maskMap : `list` of `tuple` of `int`
930 A list of mappings of mask planes of the warped exposures to
931 mask planes of the coadd.
932 """
933 edge = afwImage.Mask.getPlaneBitMask("EDGE")
934 noData = afwImage.Mask.getPlaneBitMask("NO_DATA")
935 clipped = afwImage.Mask.getPlaneBitMask("CLIPPED")
936 toReject = statsCtrl.getAndMask() & (~noData) & (~edge) & (~clipped)
937 maskMap = [(toReject, afwImage.Mask.getPlaneBitMask("REJECTED")),
938 (edge, afwImage.Mask.getPlaneBitMask("SENSOR_EDGE")),
939 (clipped, clipped)]
940 return maskMap
942 def applyAltMaskPlanes(self, mask, altMaskSpans):
943 """Apply in place alt mask formatted as SpanSets to a mask.
945 Parameters
946 ----------
947 mask : `lsst.afw.image.Mask`
948 Original mask.
949 altMaskSpans : `dict`
950 SpanSet lists to apply. Each element contains the new mask
951 plane name (e.g. "CLIPPED and/or "NO_DATA") as the key,
952 and list of SpanSets to apply to the mask.
954 Returns
955 -------
956 mask : `lsst.afw.image.Mask`
957 Updated mask.
958 """
959 if self.config.doUsePsfMatchedPolygons:
960 if ("NO_DATA" in altMaskSpans) and ("NO_DATA" in self.config.badMaskPlanes):
961 # Clear away any other masks outside the validPolygons. These
962 # pixels are no longer contributing to inexact PSFs, and will
963 # still be rejected because of NO_DATA.
964 # self.config.doUsePsfMatchedPolygons should be True only in
965 # CompareWarpAssemble. This mask-clearing step must only occur
966 # *before* applying the new masks below.
967 for spanSet in altMaskSpans['NO_DATA']:
968 spanSet.clippedTo(mask.getBBox()).clearMask(mask, self.getBadPixelMask())
970 for plane, spanSetList in altMaskSpans.items():
971 maskClipValue = mask.addMaskPlane(plane)
972 for spanSet in spanSetList:
973 spanSet.clippedTo(mask.getBBox()).setMask(mask, 2**maskClipValue)
974 return mask
976 def shrinkValidPolygons(self, coaddInputs):
977 """Shrink coaddInputs' ccds' ValidPolygons in place.
979 Either modify each ccd's validPolygon in place, or if CoaddInputs
980 does not have a validPolygon, create one from its bbox.
982 Parameters
983 ----------
984 coaddInputs : `lsst.afw.image.coaddInputs`
985 Original mask.
986 """
987 for ccd in coaddInputs.ccds:
988 polyOrig = ccd.getValidPolygon()
989 validPolyBBox = polyOrig.getBBox() if polyOrig else ccd.getBBox()
990 validPolyBBox.grow(-self.config.matchingKernelSize//2)
991 if polyOrig:
992 validPolygon = polyOrig.intersectionSingle(validPolyBBox)
993 else:
994 validPolygon = afwGeom.polygon.Polygon(geom.Box2D(validPolyBBox))
995 ccd.setValidPolygon(validPolygon)
997 def setBrightObjectMasks(self, exposure, brightObjectMasks, dataId=None):
998 """Set the bright object masks.
1000 Parameters
1001 ----------
1002 exposure : `lsst.afw.image.Exposure`
1003 Exposure under consideration.
1004 brightObjectMasks : `lsst.afw.table`
1005 Table of bright objects to mask.
1006 dataId : `lsst.daf.butler.DataId`, optional
1007 Data identifier dict for patch.
1008 """
1009 if brightObjectMasks is None:
1010 self.log.warning("Unable to apply bright object mask: none supplied")
1011 return
1012 self.log.info("Applying %d bright object masks to %s", len(brightObjectMasks), dataId)
1013 mask = exposure.getMaskedImage().getMask()
1014 wcs = exposure.getWcs()
1015 plateScale = wcs.getPixelScale().asArcseconds()
1017 for rec in brightObjectMasks:
1018 center = geom.PointI(wcs.skyToPixel(rec.getCoord()))
1019 if rec["type"] == "box":
1020 assert rec["angle"] == 0.0, ("Angle != 0 for mask object %s" % rec["id"])
1021 width = rec["width"].asArcseconds()/plateScale # convert to pixels
1022 height = rec["height"].asArcseconds()/plateScale # convert to pixels
1024 halfSize = geom.ExtentI(0.5*width, 0.5*height)
1025 bbox = geom.Box2I(center - halfSize, center + halfSize)
1027 bbox = geom.BoxI(geom.PointI(int(center[0] - 0.5*width), int(center[1] - 0.5*height)),
1028 geom.PointI(int(center[0] + 0.5*width), int(center[1] + 0.5*height)))
1029 spans = afwGeom.SpanSet(bbox)
1030 elif rec["type"] == "circle":
1031 radius = int(rec["radius"].asArcseconds()/plateScale) # convert to pixels
1032 spans = afwGeom.SpanSet.fromShape(radius, offset=center)
1033 else:
1034 self.log.warning("Unexpected region type %s at %s", rec["type"], center)
1035 continue
1036 spans.clippedTo(mask.getBBox()).setMask(mask, self.brightObjectBitmask)
1038 def setInexactPsf(self, mask):
1039 """Set INEXACT_PSF mask plane.
1041 If any of the input images isn't represented in the coadd (due to
1042 clipped pixels or chip gaps), the `CoaddPsf` will be inexact. Flag
1043 these pixels.
1045 Parameters
1046 ----------
1047 mask : `lsst.afw.image.Mask`
1048 Coadded exposure's mask, modified in-place.
1049 """
1050 mask.addMaskPlane("INEXACT_PSF")
1051 inexactPsf = mask.getPlaneBitMask("INEXACT_PSF")
1052 sensorEdge = mask.getPlaneBitMask("SENSOR_EDGE") # chip edges (so PSF is discontinuous)
1053 clipped = mask.getPlaneBitMask("CLIPPED") # pixels clipped from coadd
1054 rejected = mask.getPlaneBitMask("REJECTED") # pixels rejected from coadd due to masks
1055 array = mask.getArray()
1056 selected = array & (sensorEdge | clipped | rejected) > 0
1057 array[selected] |= inexactPsf
1059 def filterWarps(self, inputs, goodVisits):
1060 """Return list of only inputRefs with visitId in goodVisits ordered by
1061 goodVisit.
1063 Parameters
1064 ----------
1065 inputs : `list` of `~lsst.pipe.base.connections.DeferredDatasetRef`
1066 List of `lsst.pipe.base.connections.DeferredDatasetRef` with dataId
1067 containing visit.
1068 goodVisit : `dict`
1069 Dictionary with good visitIds as the keys. Value ignored.
1071 Returns
1072 -------
1073 filteredInputs : `list` \
1074 [`~lsst.pipe.base.connections.DeferredDatasetRef`]
1075 Filtered and sorted list of inputRefs with visitId in goodVisits
1076 ordered by goodVisit.
1077 """
1078 inputWarpDict = {inputRef.ref.dataId['visit']: inputRef for inputRef in inputs}
1079 filteredInputs = []
1080 for visit in goodVisits.keys():
1081 if visit in inputWarpDict:
1082 filteredInputs.append(inputWarpDict[visit])
1083 return filteredInputs
1086def countMaskFromFootprint(mask, footprint, bitmask, ignoreMask):
1087 """Function to count the number of pixels with a specific mask in a
1088 footprint.
1090 Find the intersection of mask & footprint. Count all pixels in the mask
1091 that are in the intersection that have bitmask set but do not have
1092 ignoreMask set. Return the count.
1094 Parameters
1095 ----------
1096 mask : `lsst.afw.image.Mask`
1097 Mask to define intersection region by.
1098 footprint : `lsst.afw.detection.Footprint`
1099 Footprint to define the intersection region by.
1100 bitmask : `Unknown`
1101 Specific mask that we wish to count the number of occurances of.
1102 ignoreMask : `Unknown`
1103 Pixels to not consider.
1105 Returns
1106 -------
1107 result : `int`
1108 Number of pixels in footprint with specified mask.
1109 """
1110 bbox = footprint.getBBox()
1111 bbox.clip(mask.getBBox(afwImage.PARENT))
1112 fp = afwImage.Mask(bbox)
1113 subMask = mask.Factory(mask, bbox, afwImage.PARENT)
1114 footprint.spans.setMask(fp, bitmask)
1115 return numpy.logical_and((subMask.getArray() & fp.getArray()) > 0,
1116 (subMask.getArray() & ignoreMask) == 0).sum()
1119class CompareWarpAssembleCoaddConnections(AssembleCoaddConnections):
1120 psfMatchedWarps = pipeBase.connectionTypes.Input(
1121 doc=("PSF-Matched Warps are required by CompareWarp regardless of the coadd type requested. "
1122 "Only PSF-Matched Warps make sense for image subtraction. "
1123 "Therefore, they must be an additional declared input."),
1124 name="{inputCoaddName}Coadd_psfMatchedWarp",
1125 storageClass="ExposureF",
1126 dimensions=("tract", "patch", "skymap", "visit"),
1127 deferLoad=True,
1128 multiple=True
1129 )
1130 templateCoadd = pipeBase.connectionTypes.Output(
1131 doc=("Model of the static sky, used to find temporal artifacts. Typically a PSF-Matched, "
1132 "sigma-clipped coadd. Written if and only if assembleStaticSkyModel.doWrite=True"),
1133 name="{outputCoaddName}CoaddPsfMatched",
1134 storageClass="ExposureF",
1135 dimensions=("tract", "patch", "skymap", "band"),
1136 )
1138 def __init__(self, *, config=None):
1139 super().__init__(config=config)
1140 if not config.assembleStaticSkyModel.doWrite:
1141 self.outputs.remove("templateCoadd")
1142 config.validate()
1145class CompareWarpAssembleCoaddConfig(AssembleCoaddConfig,
1146 pipelineConnections=CompareWarpAssembleCoaddConnections):
1147 assembleStaticSkyModel = pexConfig.ConfigurableField(
1148 target=AssembleCoaddTask,
1149 doc="Task to assemble an artifact-free, PSF-matched Coadd to serve as "
1150 "a naive/first-iteration model of the static sky.",
1151 )
1152 detect = pexConfig.ConfigurableField(
1153 target=SourceDetectionTask,
1154 doc="Detect outlier sources on difference between each psfMatched warp and static sky model"
1155 )
1156 detectTemplate = pexConfig.ConfigurableField(
1157 target=SourceDetectionTask,
1158 doc="Detect sources on static sky model. Only used if doPreserveContainedBySource is True"
1159 )
1160 maskStreaks = pexConfig.ConfigurableField(
1161 target=MaskStreaksTask,
1162 doc="Detect streaks on difference between each psfMatched warp and static sky model. Only used if "
1163 "doFilterMorphological is True. Adds a mask plane to an exposure, with the mask plane name set by"
1164 "streakMaskName"
1165 )
1166 streakMaskName = pexConfig.Field(
1167 dtype=str,
1168 default="STREAK",
1169 doc="Name of mask bit used for streaks"
1170 )
1171 maxNumEpochs = pexConfig.Field(
1172 doc="Charactistic maximum local number of epochs/visits in which an artifact candidate can appear "
1173 "and still be masked. The effective maxNumEpochs is a broken linear function of local "
1174 "number of epochs (N): min(maxFractionEpochsLow*N, maxNumEpochs + maxFractionEpochsHigh*N). "
1175 "For each footprint detected on the image difference between the psfMatched warp and static sky "
1176 "model, if a significant fraction of pixels (defined by spatialThreshold) are residuals in more "
1177 "than the computed effective maxNumEpochs, the artifact candidate is deemed persistant rather "
1178 "than transient and not masked.",
1179 dtype=int,
1180 default=2
1181 )
1182 maxFractionEpochsLow = pexConfig.RangeField(
1183 doc="Fraction of local number of epochs (N) to use as effective maxNumEpochs for low N. "
1184 "Effective maxNumEpochs = "
1185 "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)",
1186 dtype=float,
1187 default=0.4,
1188 min=0., max=1.,
1189 )
1190 maxFractionEpochsHigh = pexConfig.RangeField(
1191 doc="Fraction of local number of epochs (N) to use as effective maxNumEpochs for high N. "
1192 "Effective maxNumEpochs = "
1193 "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)",
1194 dtype=float,
1195 default=0.03,
1196 min=0., max=1.,
1197 )
1198 spatialThreshold = pexConfig.RangeField(
1199 doc="Unitless fraction of pixels defining how much of the outlier region has to meet the "
1200 "temporal criteria. If 0, clip all. If 1, clip none.",
1201 dtype=float,
1202 default=0.5,
1203 min=0., max=1.,
1204 inclusiveMin=True, inclusiveMax=True
1205 )
1206 doScaleWarpVariance = pexConfig.Field(
1207 doc="Rescale Warp variance plane using empirical noise?",
1208 dtype=bool,
1209 default=True,
1210 )
1211 scaleWarpVariance = pexConfig.ConfigurableField(
1212 target=ScaleVarianceTask,
1213 doc="Rescale variance on warps",
1214 )
1215 doPreserveContainedBySource = pexConfig.Field(
1216 doc="Rescue artifacts from clipping that completely lie within a footprint detected"
1217 "on the PsfMatched Template Coadd. Replicates a behavior of SafeClip.",
1218 dtype=bool,
1219 default=True,
1220 )
1221 doPrefilterArtifacts = pexConfig.Field(
1222 doc="Ignore artifact candidates that are mostly covered by the bad pixel mask, "
1223 "because they will be excluded anyway. This prevents them from contributing "
1224 "to the outlier epoch count image and potentially being labeled as persistant."
1225 "'Mostly' is defined by the config 'prefilterArtifactsRatio'.",
1226 dtype=bool,
1227 default=True
1228 )
1229 prefilterArtifactsMaskPlanes = pexConfig.ListField(
1230 doc="Prefilter artifact candidates that are mostly covered by these bad mask planes.",
1231 dtype=str,
1232 default=('NO_DATA', 'BAD', 'SAT', 'SUSPECT'),
1233 )
1234 prefilterArtifactsRatio = pexConfig.Field(
1235 doc="Prefilter artifact candidates with less than this fraction overlapping good pixels",
1236 dtype=float,
1237 default=0.05
1238 )
1239 doFilterMorphological = pexConfig.Field(
1240 doc="Filter artifact candidates based on morphological criteria, i.g. those that appear to "
1241 "be streaks.",
1242 dtype=bool,
1243 default=False
1244 )
1245 growStreakFp = pexConfig.Field(
1246 doc="Grow streak footprints by this number multiplied by the PSF width",
1247 dtype=float,
1248 default=5
1249 )
1251 def setDefaults(self):
1252 AssembleCoaddConfig.setDefaults(self)
1253 self.statistic = 'MEAN'
1254 self.doUsePsfMatchedPolygons = True
1256 # Real EDGE removed by psfMatched NO_DATA border half the width of the
1257 # matching kernel. CompareWarp applies psfMatched EDGE pixels to
1258 # directWarps before assembling.
1259 if "EDGE" in self.badMaskPlanes:
1260 self.badMaskPlanes.remove('EDGE')
1261 self.removeMaskPlanes.append('EDGE')
1262 self.assembleStaticSkyModel.badMaskPlanes = ["NO_DATA", ]
1263 self.assembleStaticSkyModel.warpType = 'psfMatched'
1264 self.assembleStaticSkyModel.connections.warpType = 'psfMatched'
1265 self.assembleStaticSkyModel.statistic = 'MEANCLIP'
1266 self.assembleStaticSkyModel.sigmaClip = 2.5
1267 self.assembleStaticSkyModel.clipIter = 3
1268 self.assembleStaticSkyModel.calcErrorFromInputVariance = False
1269 self.assembleStaticSkyModel.doWrite = False
1270 self.detect.doTempLocalBackground = False
1271 self.detect.reEstimateBackground = False
1272 self.detect.returnOriginalFootprints = False
1273 self.detect.thresholdPolarity = "both"
1274 self.detect.thresholdValue = 5
1275 self.detect.minPixels = 4
1276 self.detect.isotropicGrow = True
1277 self.detect.thresholdType = "pixel_stdev"
1278 self.detect.nSigmaToGrow = 0.4
1279 # The default nSigmaToGrow for SourceDetectionTask is already 2.4,
1280 # Explicitly restating because ratio with detect.nSigmaToGrow matters
1281 self.detectTemplate.nSigmaToGrow = 2.4
1282 self.detectTemplate.doTempLocalBackground = False
1283 self.detectTemplate.reEstimateBackground = False
1284 self.detectTemplate.returnOriginalFootprints = False
1286 def validate(self):
1287 super().validate()
1288 if self.assembleStaticSkyModel.doNImage:
1289 raise ValueError("No dataset type exists for a PSF-Matched Template N Image."
1290 "Please set assembleStaticSkyModel.doNImage=False")
1292 if self.assembleStaticSkyModel.doWrite and (self.warpType == self.assembleStaticSkyModel.warpType):
1293 raise ValueError("warpType (%s) == assembleStaticSkyModel.warpType (%s) and will compete for "
1294 "the same dataset name. Please set assembleStaticSkyModel.doWrite to False "
1295 "or warpType to 'direct'. assembleStaticSkyModel.warpType should ways be "
1296 "'PsfMatched'" % (self.warpType, self.assembleStaticSkyModel.warpType))
1299class CompareWarpAssembleCoaddTask(AssembleCoaddTask):
1300 """Assemble a compareWarp coadded image from a set of warps
1301 by masking artifacts detected by comparing PSF-matched warps.
1303 In ``AssembleCoaddTask``, we compute the coadd as an clipped mean (i.e.,
1304 we clip outliers). The problem with doing this is that when computing the
1305 coadd PSF at a given location, individual visit PSFs from visits with
1306 outlier pixels contribute to the coadd PSF and cannot be treated correctly.
1307 In this task, we correct for this behavior by creating a new badMaskPlane
1308 'CLIPPED' which marks pixels in the individual warps suspected to contain
1309 an artifact. We populate this plane on the input warps by comparing
1310 PSF-matched warps with a PSF-matched median coadd which serves as a
1311 model of the static sky. Any group of pixels that deviates from the
1312 PSF-matched template coadd by more than config.detect.threshold sigma,
1313 is an artifact candidate. The candidates are then filtered to remove
1314 variable sources and sources that are difficult to subtract such as
1315 bright stars. This filter is configured using the config parameters
1316 ``temporalThreshold`` and ``spatialThreshold``. The temporalThreshold is
1317 the maximum fraction of epochs that the deviation can appear in and still
1318 be considered an artifact. The spatialThreshold is the maximum fraction of
1319 pixels in the footprint of the deviation that appear in other epochs
1320 (where other epochs is defined by the temporalThreshold). If the deviant
1321 region meets this criteria of having a significant percentage of pixels
1322 that deviate in only a few epochs, these pixels have the 'CLIPPED' bit
1323 set in the mask. These regions will not contribute to the final coadd.
1324 Furthermore, any routine to determine the coadd PSF can now be cognizant
1325 of clipped regions. Note that the algorithm implemented by this task is
1326 preliminary and works correctly for HSC data. Parameter modifications and
1327 or considerable redesigning of the algorithm is likley required for other
1328 surveys.
1330 ``CompareWarpAssembleCoaddTask`` sub-classes
1331 ``AssembleCoaddTask`` and instantiates ``AssembleCoaddTask``
1332 as a subtask to generate the TemplateCoadd (the model of the static sky).
1334 Notes
1335 -----
1336 Debugging:
1337 This task supports the following debug variables:
1338 - ``saveCountIm``
1339 If True then save the Epoch Count Image as a fits file in the `figPath`
1340 - ``figPath``
1341 Path to save the debug fits images and figures
1342 """
1344 ConfigClass = CompareWarpAssembleCoaddConfig
1345 _DefaultName = "compareWarpAssembleCoadd"
1347 def __init__(self, *args, **kwargs):
1348 AssembleCoaddTask.__init__(self, *args, **kwargs)
1349 self.makeSubtask("assembleStaticSkyModel")
1350 detectionSchema = afwTable.SourceTable.makeMinimalSchema()
1351 self.makeSubtask("detect", schema=detectionSchema)
1352 if self.config.doPreserveContainedBySource:
1353 self.makeSubtask("detectTemplate", schema=afwTable.SourceTable.makeMinimalSchema())
1354 if self.config.doScaleWarpVariance:
1355 self.makeSubtask("scaleWarpVariance")
1356 if self.config.doFilterMorphological:
1357 self.makeSubtask("maskStreaks")
1359 @utils.inheritDoc(AssembleCoaddTask)
1360 def _makeSupplementaryData(self, butlerQC, inputRefs, outputRefs):
1361 """Generate a templateCoadd to use as a naive model of static sky to
1362 subtract from PSF-Matched warps.
1364 Returns
1365 -------
1366 result : `~lsst.pipe.base.Struct`
1367 Results as a struct with attributes:
1369 ``templateCoadd``
1370 Coadded exposure (`lsst.afw.image.Exposure`).
1371 ``nImage``
1372 Keeps track of exposure count for each pixel
1373 (`lsst.afw.image.ImageU`).
1375 Raises
1376 ------
1377 RuntimeError
1378 Raised if ``templateCoadd`` is `None`.
1379 """
1380 # Ensure that psfMatchedWarps are used as input warps for template
1381 # generation.
1382 staticSkyModelInputRefs = copy.deepcopy(inputRefs)
1383 staticSkyModelInputRefs.inputWarps = inputRefs.psfMatchedWarps
1385 # Because subtasks don't have connections we have to make one.
1386 # The main task's `templateCoadd` is the subtask's `coaddExposure`
1387 staticSkyModelOutputRefs = copy.deepcopy(outputRefs)
1388 if self.config.assembleStaticSkyModel.doWrite:
1389 staticSkyModelOutputRefs.coaddExposure = staticSkyModelOutputRefs.templateCoadd
1390 # Remove template coadd from both subtask's and main tasks outputs,
1391 # because it is handled by the subtask as `coaddExposure`
1392 del outputRefs.templateCoadd
1393 del staticSkyModelOutputRefs.templateCoadd
1395 # A PSF-Matched nImage does not exist as a dataset type
1396 if 'nImage' in staticSkyModelOutputRefs.keys():
1397 del staticSkyModelOutputRefs.nImage
1399 templateCoadd = self.assembleStaticSkyModel.runQuantum(butlerQC, staticSkyModelInputRefs,
1400 staticSkyModelOutputRefs)
1401 if templateCoadd is None:
1402 raise RuntimeError(self._noTemplateMessage(self.assembleStaticSkyModel.warpType))
1404 return pipeBase.Struct(templateCoadd=templateCoadd.coaddExposure,
1405 nImage=templateCoadd.nImage,
1406 warpRefList=templateCoadd.warpRefList,
1407 imageScalerList=templateCoadd.imageScalerList,
1408 weightList=templateCoadd.weightList)
1410 def _noTemplateMessage(self, warpType):
1411 warpName = (warpType[0].upper() + warpType[1:])
1412 message = """No %(warpName)s warps were found to build the template coadd which is
1413 required to run CompareWarpAssembleCoaddTask. To continue assembling this type of coadd,
1414 first either rerun makeCoaddTempExp with config.make%(warpName)s=True or
1415 coaddDriver with config.makeCoadTempExp.make%(warpName)s=True, before assembleCoadd.
1417 Alternatively, to use another algorithm with existing warps, retarget the CoaddDriverConfig to
1418 another algorithm like:
1420 from lsst.pipe.tasks.assembleCoadd import SafeClipAssembleCoaddTask
1421 config.assemble.retarget(SafeClipAssembleCoaddTask)
1422 """ % {"warpName": warpName}
1423 return message
1425 @utils.inheritDoc(AssembleCoaddTask)
1426 @timeMethod
1427 def run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
1428 supplementaryData):
1429 """Notes
1430 -----
1431 Assemble the coadd.
1433 Find artifacts and apply them to the warps' masks creating a list of
1434 alternative masks with a new "CLIPPED" plane and updated "NO_DATA"
1435 plane. Then pass these alternative masks to the base class's ``run``
1436 method.
1437 """
1438 # Check and match the order of the supplementaryData
1439 # (PSF-matched) inputs to the order of the direct inputs,
1440 # so that the artifact mask is applied to the right warp
1441 dataIds = [ref.dataId for ref in tempExpRefList]
1442 psfMatchedDataIds = [ref.dataId for ref in supplementaryData.warpRefList]
1444 if dataIds != psfMatchedDataIds:
1445 self.log.info("Reordering and or/padding PSF-matched visit input list")
1446 supplementaryData.warpRefList = reorderAndPadList(supplementaryData.warpRefList,
1447 psfMatchedDataIds, dataIds)
1448 supplementaryData.imageScalerList = reorderAndPadList(supplementaryData.imageScalerList,
1449 psfMatchedDataIds, dataIds)
1451 # Use PSF-Matched Warps (and corresponding scalers) and coadd to find
1452 # artifacts.
1453 spanSetMaskList = self.findArtifacts(supplementaryData.templateCoadd,
1454 supplementaryData.warpRefList,
1455 supplementaryData.imageScalerList)
1457 badMaskPlanes = self.config.badMaskPlanes[:]
1458 badMaskPlanes.append("CLIPPED")
1459 badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes)
1461 result = AssembleCoaddTask.run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
1462 spanSetMaskList, mask=badPixelMask)
1464 # Propagate PSF-matched EDGE pixels to coadd SENSOR_EDGE and
1465 # INEXACT_PSF. Psf-Matching moves the real edge inwards.
1466 self.applyAltEdgeMask(result.coaddExposure.maskedImage.mask, spanSetMaskList)
1467 return result
1469 def applyAltEdgeMask(self, mask, altMaskList):
1470 """Propagate alt EDGE mask to SENSOR_EDGE AND INEXACT_PSF planes.
1472 Parameters
1473 ----------
1474 mask : `lsst.afw.image.Mask`
1475 Original mask.
1476 altMaskList : `list` of `dict`
1477 List of Dicts containing ``spanSet`` lists.
1478 Each element contains the new mask plane name (e.g. "CLIPPED
1479 and/or "NO_DATA") as the key, and list of ``SpanSets`` to apply to
1480 the mask.
1481 """
1482 maskValue = mask.getPlaneBitMask(["SENSOR_EDGE", "INEXACT_PSF"])
1483 for visitMask in altMaskList:
1484 if "EDGE" in visitMask:
1485 for spanSet in visitMask['EDGE']:
1486 spanSet.clippedTo(mask.getBBox()).setMask(mask, maskValue)
1488 def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList):
1489 """Find artifacts.
1491 Loop through warps twice. The first loop builds a map with the count
1492 of how many epochs each pixel deviates from the templateCoadd by more
1493 than ``config.chiThreshold`` sigma. The second loop takes each
1494 difference image and filters the artifacts detected in each using
1495 count map to filter out variable sources and sources that are
1496 difficult to subtract cleanly.
1498 Parameters
1499 ----------
1500 templateCoadd : `lsst.afw.image.Exposure`
1501 Exposure to serve as model of static sky.
1502 tempExpRefList : `list`
1503 List of data references to warps.
1504 imageScalerList : `list`
1505 List of image scalers.
1507 Returns
1508 -------
1509 altMasks : `list` of `dict`
1510 List of dicts containing information about CLIPPED
1511 (i.e., artifacts), NO_DATA, and EDGE pixels.
1512 """
1513 self.log.debug("Generating Count Image, and mask lists.")
1514 coaddBBox = templateCoadd.getBBox()
1515 slateIm = afwImage.ImageU(coaddBBox)
1516 epochCountImage = afwImage.ImageU(coaddBBox)
1517 nImage = afwImage.ImageU(coaddBBox)
1518 spanSetArtifactList = []
1519 spanSetNoDataMaskList = []
1520 spanSetEdgeList = []
1521 spanSetBadMorphoList = []
1522 badPixelMask = self.getBadPixelMask()
1524 # mask of the warp diffs should = that of only the warp
1525 templateCoadd.mask.clearAllMaskPlanes()
1527 if self.config.doPreserveContainedBySource:
1528 templateFootprints = self.detectTemplate.detectFootprints(templateCoadd)
1529 else:
1530 templateFootprints = None
1532 for warpRef, imageScaler in zip(tempExpRefList, imageScalerList):
1533 warpDiffExp = self._readAndComputeWarpDiff(warpRef, imageScaler, templateCoadd)
1534 if warpDiffExp is not None:
1535 # This nImage only approximates the final nImage because it
1536 # uses the PSF-matched mask.
1537 nImage.array += (numpy.isfinite(warpDiffExp.image.array)
1538 * ((warpDiffExp.mask.array & badPixelMask) == 0)).astype(numpy.uint16)
1539 fpSet = self.detect.detectFootprints(warpDiffExp, doSmooth=False, clearMask=True)
1540 fpSet.positive.merge(fpSet.negative)
1541 footprints = fpSet.positive
1542 slateIm.set(0)
1543 spanSetList = [footprint.spans for footprint in footprints.getFootprints()]
1545 # Remove artifacts due to defects before they contribute to
1546 # the epochCountImage.
1547 if self.config.doPrefilterArtifacts:
1548 spanSetList = self.prefilterArtifacts(spanSetList, warpDiffExp)
1550 # Clear mask before adding prefiltered spanSets
1551 self.detect.clearMask(warpDiffExp.mask)
1552 for spans in spanSetList:
1553 spans.setImage(slateIm, 1, doClip=True)
1554 spans.setMask(warpDiffExp.mask, warpDiffExp.mask.getPlaneBitMask("DETECTED"))
1555 epochCountImage += slateIm
1557 if self.config.doFilterMorphological:
1558 maskName = self.config.streakMaskName
1559 _ = self.maskStreaks.run(warpDiffExp)
1560 streakMask = warpDiffExp.mask
1561 spanSetStreak = afwGeom.SpanSet.fromMask(streakMask,
1562 streakMask.getPlaneBitMask(maskName)).split()
1563 # Pad the streaks to account for low-surface brightness
1564 # wings.
1565 psf = warpDiffExp.getPsf()
1566 for s, sset in enumerate(spanSetStreak):
1567 psfShape = psf.computeShape(sset.computeCentroid())
1568 dilation = self.config.growStreakFp * psfShape.getDeterminantRadius()
1569 sset_dilated = sset.dilated(int(dilation))
1570 spanSetStreak[s] = sset_dilated
1572 # PSF-Matched warps have less available area (~the matching
1573 # kernel) because the calexps undergo a second convolution.
1574 # Pixels with data in the direct warp but not in the
1575 # PSF-matched warp will not have their artifacts detected.
1576 # NaNs from the PSF-matched warp therefore must be masked in
1577 # the direct warp.
1578 nans = numpy.where(numpy.isnan(warpDiffExp.maskedImage.image.array), 1, 0)
1579 nansMask = afwImage.makeMaskFromArray(nans.astype(afwImage.MaskPixel))
1580 nansMask.setXY0(warpDiffExp.getXY0())
1581 edgeMask = warpDiffExp.mask
1582 spanSetEdgeMask = afwGeom.SpanSet.fromMask(edgeMask,
1583 edgeMask.getPlaneBitMask("EDGE")).split()
1584 else:
1585 # If the directWarp has <1% coverage, the psfMatchedWarp can
1586 # have 0% and not exist. In this case, mask the whole epoch.
1587 nansMask = afwImage.MaskX(coaddBBox, 1)
1588 spanSetList = []
1589 spanSetEdgeMask = []
1590 spanSetStreak = []
1592 spanSetNoDataMask = afwGeom.SpanSet.fromMask(nansMask).split()
1594 spanSetNoDataMaskList.append(spanSetNoDataMask)
1595 spanSetArtifactList.append(spanSetList)
1596 spanSetEdgeList.append(spanSetEdgeMask)
1597 if self.config.doFilterMorphological:
1598 spanSetBadMorphoList.append(spanSetStreak)
1600 if lsstDebug.Info(__name__).saveCountIm:
1601 path = self._dataRef2DebugPath("epochCountIm", tempExpRefList[0], coaddLevel=True)
1602 epochCountImage.writeFits(path)
1604 for i, spanSetList in enumerate(spanSetArtifactList):
1605 if spanSetList:
1606 filteredSpanSetList = self.filterArtifacts(spanSetList, epochCountImage, nImage,
1607 templateFootprints)
1608 spanSetArtifactList[i] = filteredSpanSetList
1609 if self.config.doFilterMorphological:
1610 spanSetArtifactList[i] += spanSetBadMorphoList[i]
1612 altMasks = []
1613 for artifacts, noData, edge in zip(spanSetArtifactList, spanSetNoDataMaskList, spanSetEdgeList):
1614 altMasks.append({'CLIPPED': artifacts,
1615 'NO_DATA': noData,
1616 'EDGE': edge})
1617 return altMasks
1619 def prefilterArtifacts(self, spanSetList, exp):
1620 """Remove artifact candidates covered by bad mask plane.
1622 Any future editing of the candidate list that does not depend on
1623 temporal information should go in this method.
1625 Parameters
1626 ----------
1627 spanSetList : `list` [`lsst.afw.geom.SpanSet`]
1628 List of SpanSets representing artifact candidates.
1629 exp : `lsst.afw.image.Exposure`
1630 Exposure containing mask planes used to prefilter.
1632 Returns
1633 -------
1634 returnSpanSetList : `list` [`lsst.afw.geom.SpanSet`]
1635 List of SpanSets with artifacts.
1636 """
1637 badPixelMask = exp.mask.getPlaneBitMask(self.config.prefilterArtifactsMaskPlanes)
1638 goodArr = (exp.mask.array & badPixelMask) == 0
1639 returnSpanSetList = []
1640 bbox = exp.getBBox()
1641 x0, y0 = exp.getXY0()
1642 for i, span in enumerate(spanSetList):
1643 y, x = span.clippedTo(bbox).indices()
1644 yIndexLocal = numpy.array(y) - y0
1645 xIndexLocal = numpy.array(x) - x0
1646 goodRatio = numpy.count_nonzero(goodArr[yIndexLocal, xIndexLocal])/span.getArea()
1647 if goodRatio > self.config.prefilterArtifactsRatio:
1648 returnSpanSetList.append(span)
1649 return returnSpanSetList
1651 def filterArtifacts(self, spanSetList, epochCountImage, nImage, footprintsToExclude=None):
1652 """Filter artifact candidates.
1654 Parameters
1655 ----------
1656 spanSetList : `list` [`lsst.afw.geom.SpanSet`]
1657 List of SpanSets representing artifact candidates.
1658 epochCountImage : `lsst.afw.image.Image`
1659 Image of accumulated number of warpDiff detections.
1660 nImage : `lsst.afw.image.ImageU`
1661 Image of the accumulated number of total epochs contributing.
1663 Returns
1664 -------
1665 maskSpanSetList : `list` [`lsst.afw.geom.SpanSet`]
1666 List of SpanSets with artifacts.
1667 """
1668 maskSpanSetList = []
1669 x0, y0 = epochCountImage.getXY0()
1670 for i, span in enumerate(spanSetList):
1671 y, x = span.indices()
1672 yIdxLocal = [y1 - y0 for y1 in y]
1673 xIdxLocal = [x1 - x0 for x1 in x]
1674 outlierN = epochCountImage.array[yIdxLocal, xIdxLocal]
1675 totalN = nImage.array[yIdxLocal, xIdxLocal]
1677 # effectiveMaxNumEpochs is broken line (fraction of N) with
1678 # characteristic config.maxNumEpochs.
1679 effMaxNumEpochsHighN = (self.config.maxNumEpochs
1680 + self.config.maxFractionEpochsHigh*numpy.mean(totalN))
1681 effMaxNumEpochsLowN = self.config.maxFractionEpochsLow * numpy.mean(totalN)
1682 effectiveMaxNumEpochs = int(min(effMaxNumEpochsLowN, effMaxNumEpochsHighN))
1683 nPixelsBelowThreshold = numpy.count_nonzero((outlierN > 0)
1684 & (outlierN <= effectiveMaxNumEpochs))
1685 percentBelowThreshold = nPixelsBelowThreshold / len(outlierN)
1686 if percentBelowThreshold > self.config.spatialThreshold:
1687 maskSpanSetList.append(span)
1689 if self.config.doPreserveContainedBySource and footprintsToExclude is not None:
1690 # If a candidate is contained by a footprint on the template coadd,
1691 # do not clip.
1692 filteredMaskSpanSetList = []
1693 for span in maskSpanSetList:
1694 doKeep = True
1695 for footprint in footprintsToExclude.positive.getFootprints():
1696 if footprint.spans.contains(span):
1697 doKeep = False
1698 break
1699 if doKeep:
1700 filteredMaskSpanSetList.append(span)
1701 maskSpanSetList = filteredMaskSpanSetList
1703 return maskSpanSetList
1705 def _readAndComputeWarpDiff(self, warpRef, imageScaler, templateCoadd):
1706 """Fetch a warp from the butler and return a warpDiff.
1708 Parameters
1709 ----------
1710 warpRef : `lsst.daf.butler.DeferredDatasetHandle`
1711 Handle for the warp.
1712 imageScaler : `lsst.pipe.tasks.scaleZeroPoint.ImageScaler`
1713 An image scaler object.
1714 templateCoadd : `lsst.afw.image.Exposure`
1715 Exposure to be substracted from the scaled warp.
1717 Returns
1718 -------
1719 warp : `lsst.afw.image.Exposure`
1720 Exposure of the image difference between the warp and template.
1721 """
1722 # If the PSF-Matched warp did not exist for this direct warp
1723 # None is holding its place to maintain order in Gen 3
1724 if warpRef is None:
1725 return None
1727 warp = warpRef.get()
1728 # direct image scaler OK for PSF-matched Warp
1729 imageScaler.scaleMaskedImage(warp.getMaskedImage())
1730 mi = warp.getMaskedImage()
1731 if self.config.doScaleWarpVariance:
1732 try:
1733 self.scaleWarpVariance.run(mi)
1734 except Exception as exc:
1735 self.log.warning("Unable to rescale variance of warp (%s); leaving it as-is", exc)
1736 mi -= templateCoadd.getMaskedImage()
1737 return warp