22__all__ = [
"AssembleCoaddTask",
"AssembleCoaddConnections",
"AssembleCoaddConfig",
23 "CompareWarpAssembleCoaddTask",
"CompareWarpAssembleCoaddConfig"]
40import lsst.utils
as utils
42from .coaddBase
import CoaddBaseTask, makeSkyInfo, reorderAndPadList
43from .interpImage
import InterpImageTask
44from .scaleZeroPoint
import ScaleZeroPointTask
45from .maskStreaks
import MaskStreaksTask
46from .healSparseMapping
import HealSparseInputMapTask
48from lsst.utils.timer
import timeMethod
49from deprecated.sphinx
import deprecated
51log = logging.getLogger(__name__)
55 dimensions=(
"tract",
"patch",
"band",
"skymap"),
56 defaultTemplates={
"inputCoaddName":
"deep",
57 "outputCoaddName":
"deep",
59 "warpTypeSuffix":
""}):
61 inputWarps = pipeBase.connectionTypes.Input(
62 doc=(
"Input list of warps to be assemebled i.e. stacked."
63 "WarpType (e.g. direct, psfMatched) is controlled by the warpType config parameter"),
64 name=
"{inputCoaddName}Coadd_{warpType}Warp",
65 storageClass=
"ExposureF",
66 dimensions=(
"tract",
"patch",
"skymap",
"visit",
"instrument"),
70 skyMap = pipeBase.connectionTypes.Input(
71 doc=
"Input definition of geometry/bbox and projection/wcs for coadded exposures",
72 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
73 storageClass=
"SkyMap",
74 dimensions=(
"skymap", ),
76 selectedVisits = pipeBase.connectionTypes.Input(
77 doc=
"Selected visits to be coadded.",
78 name=
"{outputCoaddName}Visits",
79 storageClass=
"StructuredDataDict",
80 dimensions=(
"instrument",
"tract",
"patch",
"skymap",
"band")
82 brightObjectMask = pipeBase.connectionTypes.PrerequisiteInput(
83 doc=(
"Input Bright Object Mask mask produced with external catalogs to be applied to the mask plane"
85 name=
"brightObjectMask",
86 storageClass=
"ObjectMaskCatalog",
87 dimensions=(
"tract",
"patch",
"skymap",
"band"),
90 coaddExposure = pipeBase.connectionTypes.Output(
91 doc=
"Output coadded exposure, produced by stacking input warps",
92 name=
"{outputCoaddName}Coadd{warpTypeSuffix}",
93 storageClass=
"ExposureF",
94 dimensions=(
"tract",
"patch",
"skymap",
"band"),
96 nImage = pipeBase.connectionTypes.Output(
97 doc=
"Output image of number of input images per pixel",
98 name=
"{outputCoaddName}Coadd_nImage",
99 storageClass=
"ImageU",
100 dimensions=(
"tract",
"patch",
"skymap",
"band"),
102 inputMap = pipeBase.connectionTypes.Output(
103 doc=
"Output healsparse map of input images",
104 name=
"{outputCoaddName}Coadd_inputMap",
105 storageClass=
"HealSparseMap",
106 dimensions=(
"tract",
"patch",
"skymap",
"band"),
109 def __init__(self, *, config=None):
110 super().__init__(config=config)
112 if not config.doMaskBrightObjects:
113 self.prerequisiteInputs.remove(
"brightObjectMask")
115 if not config.doSelectVisits:
116 self.inputs.remove(
"selectedVisits")
118 if not config.doNImage:
119 self.outputs.remove(
"nImage")
121 if not self.config.doInputMap:
122 self.outputs.remove(
"inputMap")
125class AssembleCoaddConfig(CoaddBaseTask.ConfigClass, pipeBase.PipelineTaskConfig,
126 pipelineConnections=AssembleCoaddConnections):
127 warpType = pexConfig.Field(
128 doc=
"Warp name: one of 'direct' or 'psfMatched'",
132 subregionSize = pexConfig.ListField(
134 doc=
"Width, height of stack subregion size; "
135 "make small enough that a full stack of images will fit into memory at once.",
137 default=(2000, 2000),
139 statistic = pexConfig.Field(
141 doc=
"Main stacking statistic for aggregating over the epochs.",
144 doOnlineForMean = pexConfig.Field(
146 doc=
"Perform online coaddition when statistic=\"MEAN\" to save memory?",
149 doSigmaClip = pexConfig.Field(
151 doc=
"Perform sigma clipped outlier rejection with MEANCLIP statistic? (DEPRECATED)",
154 sigmaClip = pexConfig.Field(
156 doc=
"Sigma for outlier rejection; ignored if non-clipping statistic selected.",
159 clipIter = pexConfig.Field(
161 doc=
"Number of iterations of outlier rejection; ignored if non-clipping statistic selected.",
164 calcErrorFromInputVariance = pexConfig.Field(
166 doc=
"Calculate coadd variance from input variance by stacking statistic."
167 "Passed to StatisticsControl.setCalcErrorFromInputVariance()",
170 scaleZeroPoint = pexConfig.ConfigurableField(
171 target=ScaleZeroPointTask,
172 doc=
"Task to adjust the photometric zero point of the coadd temp exposures",
174 doInterp = pexConfig.Field(
175 doc=
"Interpolate over NaN pixels? Also extrapolate, if necessary, but the results are ugly.",
179 interpImage = pexConfig.ConfigurableField(
180 target=InterpImageTask,
181 doc=
"Task to interpolate (and extrapolate) over NaN pixels",
183 doWrite = pexConfig.Field(
184 doc=
"Persist coadd?",
188 doNImage = pexConfig.Field(
189 doc=
"Create image of number of contributing exposures for each pixel",
193 doUsePsfMatchedPolygons = pexConfig.Field(
194 doc=
"Use ValidPolygons from shrunk Psf-Matched Calexps? Should be set to True by CompareWarp only.",
198 maskPropagationThresholds = pexConfig.DictField(
201 doc=(
"Threshold (in fractional weight) of rejection at which we propagate a mask plane to "
202 "the coadd; that is, we set the mask bit on the coadd if the fraction the rejected frames "
203 "would have contributed exceeds this value."),
204 default={
"SAT": 0.1},
206 removeMaskPlanes = pexConfig.ListField(dtype=str, default=[
"NOT_DEBLENDED"],
207 doc=
"Mask planes to remove before coadding")
208 doMaskBrightObjects = pexConfig.Field(dtype=bool, default=
False,
209 doc=
"Set mask and flag bits for bright objects?")
210 brightObjectMaskName = pexConfig.Field(dtype=str, default=
"BRIGHT_OBJECT",
211 doc=
"Name of mask bit used for bright objects")
212 coaddPsf = pexConfig.ConfigField(
213 doc=
"Configuration for CoaddPsf",
214 dtype=measAlg.CoaddPsfConfig,
216 doAttachTransmissionCurve = pexConfig.Field(
217 dtype=bool, default=
False, optional=
False,
218 doc=(
"Attach a piecewise TransmissionCurve for the coadd? "
219 "(requires all input Exposures to have TransmissionCurves).")
221 hasFakes = pexConfig.Field(
224 doc=
"Should be set to True if fake sources have been inserted into the input data."
226 doSelectVisits = pexConfig.Field(
227 doc=
"Coadd only visits selected by a SelectVisitsTask",
231 doInputMap = pexConfig.Field(
232 doc=
"Create a bitwise map of coadd inputs",
236 inputMapper = pexConfig.ConfigurableField(
237 doc=
"Input map creation subtask.",
238 target=HealSparseInputMapTask,
243 self.badMaskPlanes = [
"NO_DATA",
"BAD",
"SAT",
"EDGE"]
250 log.warning(
"Config doPsfMatch deprecated. Setting warpType='psfMatched'")
251 self.warpType =
'psfMatched'
252 if self.doSigmaClip
and self.statistic !=
"MEANCLIP":
253 log.warning(
'doSigmaClip deprecated. To replicate behavior, setting statistic to "MEANCLIP"')
254 self.statistic =
"MEANCLIP"
255 if self.doInterp
and self.statistic
not in [
'MEAN',
'MEDIAN',
'MEANCLIP',
'VARIANCE',
'VARIANCECLIP']:
256 raise ValueError(
"Must set doInterp=False for statistic=%s, which does not "
257 "compute and set a non-zero coadd variance estimate." % (self.statistic))
259 unstackableStats = [
'NOTHING',
'ERROR',
'ORMASK']
260 if not hasattr(afwMath.Property, self.statistic)
or self.statistic
in unstackableStats:
261 stackableStats = [
str(k)
for k
in afwMath.Property.__members__.keys()
262 if str(k)
not in unstackableStats]
263 raise ValueError(
"statistic %s is not allowed. Please choose one of %s."
264 % (self.statistic, stackableStats))
268 """Assemble a coadded image from a set of warps.
270 Each Warp that goes into a coadd will typically have an independent
271 photometric zero-point. Therefore, we must scale each Warp to set it to
272 a common photometric zeropoint. WarpType may be one of 'direct' or
273 'psfMatched',
and the boolean configs `config.makeDirect`
and
274 `config.makePsfMatched` set which of the warp types will be coadded.
275 The coadd
is computed
as a mean
with optional outlier rejection.
276 Criteria
for outlier rejection are set
in `AssembleCoaddConfig`.
277 Finally, Warps can have bad
'NaN' pixels which received no input
from the
278 source calExps. We interpolate over these bad (NaN) pixels.
280 `AssembleCoaddTask` uses several sub-tasks. These are
282 - `~lsst.pipe.tasks.ScaleZeroPointTask`
283 - create
and use an ``imageScaler`` object to scale the photometric zeropoint
for each Warp
284 - `~lsst.pipe.tasks.InterpImageTask`
285 - interpolate across bad pixels (NaN)
in the final coadd
287 You can retarget these subtasks
if you wish.
292 Raised
if unable to define mask plane
for bright objects.
297 `AssembleCoaddTask` has no debug variables of its own. Some of the
298 subtasks may support `~lsst.base.lsstDebug` variables. See the
299 documentation
for the subtasks
for further information.
303 `AssembleCoaddTask` assembles a set of warped images into a coadded image.
304 The `AssembleCoaddTask` can be invoked by running ``assembleCoadd.py``
305 with the flag
'--legacyCoadd'. Usage of assembleCoadd.py expects two
306 inputs: a data reference to the tract patch
and filter to be coadded,
and
307 a list of Warps to attempt to coadd. These are specified using ``--id``
and
308 ``--selectId``, respectively:
312 --id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]
313 --selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]
315 Only the Warps that cover the specified tract
and patch will be coadded.
316 A list of the available optional arguments can be obtained by calling
317 ``assembleCoadd.py``
with the ``--help`` command line argument:
321 assembleCoadd.py --help
323 To demonstrate usage of the `AssembleCoaddTask`
in the larger context of
324 multi-band processing, we will generate the HSC-I & -R band coadds
from
325 HSC engineering test data provided
in the ``ci_hsc`` package. To begin,
326 assuming that the lsst stack has been already set up, we must set up the
327 obs_subaru
and ``ci_hsc`` packages. This defines the environment variable
328 ``$CI_HSC_DIR``
and points at the location of the package. The raw HSC
329 data live
in the ``$CI_HSC_DIR/raw directory``. To begin assembling the
330 coadds, we must first run:
333 - process the individual ccds
in $CI_HSC_RAW to produce calibrated exposures
335 - create a skymap that covers the area of the sky present
in the raw exposures
337 - warp the individual calibrated exposures to the tangent plane of the coadd
339 We can perform all of these steps by running
343 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
345 This will produce warped exposures
for each visit. To coadd the warped
346 data, we call assembleCoadd.py
as follows:
350 assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \
351 --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \
352 --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \
353 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \
354 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \
355 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \
356 --selectId visit=903988 ccd=24
358 that will process the HSC-I band data. The results are written
in
359 ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``.
361 You may also choose to run:
365 scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346
366 assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R \
367 --selectId visit=903334 ccd=16 --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 \
368 --selectId visit=903334 ccd=100 --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 \
369 --selectId visit=903338 ccd=18 --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 \
370 --selectId visit=903342 ccd=10 --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 \
371 --selectId visit=903344 ccd=5 --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 \
372 --selectId visit=903346 ccd=6 --selectId visit=903346 ccd=12
374 to generate the coadd
for the HSC-R band
if you are interested
in
375 following multiBand Coadd processing
as discussed
in `pipeTasks_multiBand`
376 (but note that normally, one would use the `SafeClipAssembleCoaddTask`
377 rather than `AssembleCoaddTask` to make the coadd.
380 ConfigClass = AssembleCoaddConfig
381 _DefaultName = "assembleCoadd"
386 argNames = [
"config",
"name",
"parentTask",
"log"]
387 kwargs.update({k: v
for k, v
in zip(argNames, args)})
388 warnings.warn(
"AssembleCoadd received positional args, and casting them as kwargs: %s. "
389 "PipelineTask will not take positional args" % argNames, FutureWarning,
393 self.makeSubtask(
"interpImage")
394 self.makeSubtask(
"scaleZeroPoint")
396 if self.config.doMaskBrightObjects:
397 mask = afwImage.Mask()
400 except pexExceptions.LsstCppException:
401 raise RuntimeError(
"Unable to define mask plane for bright objects; planes used are %s" %
402 mask.getMaskPlaneDict().keys())
405 if self.config.doInputMap:
406 self.makeSubtask(
"inputMapper")
410 @utils.inheritDoc(pipeBase.PipelineTask)
412 inputData = butlerQC.get(inputRefs)
416 skyMap = inputData[
"skyMap"]
417 outputDataId = butlerQC.quantum.dataId
419 inputData[
'skyInfo'] = makeSkyInfo(skyMap,
420 tractId=outputDataId[
'tract'],
421 patchId=outputDataId[
'patch'])
423 if self.config.doSelectVisits:
424 warpRefList = self.
filterWarps(inputData[
'inputWarps'], inputData[
'selectedVisits'])
426 warpRefList = inputData[
'inputWarps']
429 self.log.info(
"Found %d %s", len(inputs.tempExpRefList),
431 if len(inputs.tempExpRefList) == 0:
432 raise pipeBase.NoWorkFound(
"No coadd temporary exposures found")
435 retStruct = self.
run(inputData[
'skyInfo'], inputs.tempExpRefList, inputs.imageScalerList,
436 inputs.weightList, supplementaryData=supplementaryData)
438 inputData.setdefault(
'brightObjectMask',
None)
439 if self.config.doMaskBrightObjects
and inputData[
"brightObjectMask"]
is None:
440 log.warning(
"doMaskBrightObjects is set to True, but brightObjectMask not loaded")
441 self.
processResults(retStruct.coaddExposure, inputData[
'brightObjectMask'], outputDataId)
443 if self.config.doWrite:
444 butlerQC.put(retStruct, outputRefs)
448 """Interpolate over missing data and mask bright stars.
453 The coadded exposure to process.
455 Table of bright objects to mask.
456 dataId : `lsst.daf.butler.DataId`
or `
None`, optional
459 if self.config.doInterp:
460 self.interpImage.run(coaddExposure.getMaskedImage(), planeName=
"NO_DATA")
462 varArray = coaddExposure.variance.array
463 with numpy.errstate(invalid=
"ignore"):
464 varArray[:] = numpy.where(varArray > 0, varArray, numpy.inf)
466 if self.config.doMaskBrightObjects:
469 def _makeSupplementaryData(self, butlerQC, inputRefs, outputRefs):
470 """Make additional inputs to run() specific to subclasses (Gen3).
472 Duplicates interface of `runQuantum` method.
473 Available to be implemented by subclasses only if they need the
474 coadd dataRef
for performing preliminary processing before
475 assembling the coadd.
479 butlerQC : `~lsst.pipe.base.ButlerQuantumContext`
480 Gen3 Butler object
for fetching additional data products before
481 running the Task specialized
for quantum being processed.
482 inputRefs : `~lsst.pipe.base.InputQuantizedConnection`
483 Attributes are the names of the connections describing input dataset types.
484 Values are DatasetRefs that task consumes
for corresponding dataset type.
485 DataIds are guaranteed to match data objects
in ``inputData``.
486 outputRefs : `~lsst.pipe.base.OutputQuantizedConnection`
487 Attributes are the names of the connections describing output dataset types.
488 Values are DatasetRefs that task
is to produce
489 for corresponding dataset type.
491 return pipeBase.Struct()
494 reason=
"makeSupplementaryDataGen3 is deprecated in favor of _makeSupplementaryData",
496 category=FutureWarning
502 """Prepare the input warps for coaddition by measuring the weight for
503 each warp and the scaling
for the photometric zero point.
505 Each Warp has its own photometric zeropoint
and background variance.
506 Before coadding these Warps together, compute a scale factor to
507 normalize the photometric zeropoint
and compute the weight
for each Warp.
512 List of data references to tempExp.
516 result : `~lsst.pipe.base.Struct`
517 Results
as a struct
with attributes:
520 `list` of data references to tempExp.
522 `list` of weightings.
524 `list` of image scalers.
526 statsCtrl = afwMath.StatisticsControl()
527 statsCtrl.setNumSigmaClip(self.config.sigmaClip)
528 statsCtrl.setNumIter(self.config.clipIter)
530 statsCtrl.setNanSafe(True)
538 for tempExpRef
in refList:
539 tempExp = tempExpRef.get()
541 if numpy.isnan(tempExp.image.array).all():
543 maskedImage = tempExp.getMaskedImage()
544 imageScaler = self.scaleZeroPoint.computeImageScaler(
549 imageScaler.scaleMaskedImage(maskedImage)
550 except Exception
as e:
551 self.log.warning(
"Scaling failed for %s (skipping it): %s", tempExpRef.dataId, e)
553 statObj = afwMath.makeStatistics(maskedImage.getVariance(), maskedImage.getMask(),
554 afwMath.MEANCLIP, statsCtrl)
555 meanVar, meanVarErr = statObj.getResult(afwMath.MEANCLIP)
556 weight = 1.0 / float(meanVar)
557 if not numpy.isfinite(weight):
558 self.log.warning(
"Non-finite weight for %s: skipping", tempExpRef.dataId)
560 self.log.info(
"Weight of %s %s = %0.3f", tempExpName, tempExpRef.dataId, weight)
565 tempExpRefList.append(tempExpRef)
566 weightList.append(weight)
567 imageScalerList.append(imageScaler)
569 return pipeBase.Struct(tempExpRefList=tempExpRefList, weightList=weightList,
570 imageScalerList=imageScalerList)
573 """Prepare the statistics for coadding images.
577 mask : `int`, optional
578 Bit mask value to exclude from coaddition.
582 stats : `~lsst.pipe.base.Struct`
583 Statistics
as a struct
with attributes:
588 Statistic
for coadd (`~lsst.afw.math.Property`).
592 statsCtrl = afwMath.StatisticsControl()
593 statsCtrl.setNumSigmaClip(self.config.sigmaClip)
594 statsCtrl.setNumIter(self.config.clipIter)
595 statsCtrl.setAndMask(mask)
596 statsCtrl.setNanSafe(
True)
597 statsCtrl.setWeighted(
True)
598 statsCtrl.setCalcErrorFromInputVariance(self.config.calcErrorFromInputVariance)
599 for plane, threshold
in self.config.maskPropagationThresholds.items():
600 bit = afwImage.Mask.getMaskPlane(plane)
601 statsCtrl.setMaskPropagationThreshold(bit, threshold)
602 statsFlags = afwMath.stringToStatisticsProperty(self.config.statistic)
603 return pipeBase.Struct(ctrl=statsCtrl, flags=statsFlags)
606 def run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
607 altMaskList=None, mask=None, supplementaryData=None):
608 """Assemble a coadd from input warps.
610 Assemble the coadd using the provided list of coaddTempExps. Since
611 the full coadd covers a patch (a large area), the assembly is
612 performed over small areas on the image at a time
in order to
613 conserve memory usage. Iterate over subregions within the outer
614 bbox of the patch using `assembleSubregion` to stack the corresponding
615 subregions
from the coaddTempExps
with the statistic specified.
616 Set the edge bits the coadd mask based on the weight map.
620 skyInfo : `~lsst.pipe.base.Struct`
621 Struct
with geometric information about the patch.
622 tempExpRefList : `list`
623 List of data references to Warps (previously called CoaddTempExps).
624 imageScalerList : `list`
625 List of image scalers.
628 altMaskList : `list`, optional
629 List of alternate masks to use rather than those stored
with
631 mask : `int`, optional
632 Bit mask value to exclude
from coaddition.
633 supplementaryData : `~lsst.pipe.base.Struct`, optional
634 Struct
with additional data products needed to assemble coadd.
635 Only used by subclasses that implement ``_makeSupplementaryData``
640 result : `~lsst.pipe.base.Struct`
641 Results
as a struct
with attributes:
648 Bit-wise map of inputs,
if requested.
650 Input list of refs to the warps (``lsst.daf.butler.DeferredDatasetHandle``)
653 Input list of image scalers (`list`) (unmodified).
655 Input list of weights (`list`) (unmodified).
658 self.log.info("Assembling %s %s", len(tempExpRefList), tempExpName)
661 if altMaskList
is None:
662 altMaskList = [
None]*len(tempExpRefList)
664 coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
665 coaddExposure.setPhotoCalib(self.scaleZeroPoint.getPhotoCalib())
666 coaddExposure.getInfo().setCoaddInputs(self.inputRecorder.makeCoaddInputs())
668 coaddMaskedImage = coaddExposure.getMaskedImage()
669 subregionSizeArr = self.config.subregionSize
670 subregionSize =
geom.Extent2I(subregionSizeArr[0], subregionSizeArr[1])
672 if self.config.doNImage:
673 nImage = afwImage.ImageU(skyInfo.bbox)
678 if self.config.doInputMap:
679 self.inputMapper.build_ccd_input_map(skyInfo.bbox,
681 coaddExposure.getInfo().getCoaddInputs().ccds)
683 if self.config.doOnlineForMean
and self.config.statistic ==
"MEAN":
686 weightList, altMaskList, stats.ctrl,
688 except Exception
as e:
689 self.log.exception(
"Cannot compute online coadd %s", e)
692 for subBBox
in self.
_subBBoxIter(skyInfo.bbox, subregionSize):
695 weightList, altMaskList, stats.flags, stats.ctrl,
697 except Exception
as e:
698 self.log.exception(
"Cannot compute coadd %s: %s", subBBox, e)
702 if self.config.doInputMap:
703 self.inputMapper.finalize_ccd_input_map_mask()
704 inputMap = self.inputMapper.ccd_input_map
711 coaddUtils.setCoaddEdgeBits(coaddMaskedImage.getMask(), coaddMaskedImage.getVariance())
712 return pipeBase.Struct(coaddExposure=coaddExposure, nImage=nImage,
713 warpRefList=tempExpRefList, imageScalerList=imageScalerList,
714 weightList=weightList, inputMap=inputMap)
717 """Set the metadata for the coadd.
719 This basic implementation sets the filter from the first input.
724 The target exposure
for the coadd.
725 tempExpRefList : `list`
726 List of data references to tempExp.
733 Raised
if there
is a length mismatch.
735 assert len(tempExpRefList) == len(weightList),
"Length mismatch"
742 tempExpList = [tempExpRef.get(parameters={
'bbox': bbox})
for tempExpRef
in tempExpRefList]
744 numCcds = sum(len(tempExp.getInfo().getCoaddInputs().ccds)
for tempExp
in tempExpList)
748 coaddExposure.setFilter(afwImage.FilterLabel(tempExpList[0].getFilter().bandLabel))
749 coaddInputs = coaddExposure.getInfo().getCoaddInputs()
750 coaddInputs.ccds.reserve(numCcds)
751 coaddInputs.visits.reserve(len(tempExpList))
753 for tempExp, weight
in zip(tempExpList, weightList):
754 self.inputRecorder.addVisitToCoadd(coaddInputs, tempExp, weight)
756 if self.config.doUsePsfMatchedPolygons:
759 coaddInputs.visits.sort()
760 coaddInputs.ccds.sort()
766 modelPsfList = [tempExp.getPsf()
for tempExp
in tempExpList]
767 modelPsfWidthList = [modelPsf.computeBBox(modelPsf.getAveragePosition()).getWidth()
768 for modelPsf
in modelPsfList]
769 psf = modelPsfList[modelPsfWidthList.index(max(modelPsfWidthList))]
771 psf = measAlg.CoaddPsf(coaddInputs.ccds, coaddExposure.getWcs(),
772 self.config.coaddPsf.makeControl())
773 coaddExposure.setPsf(psf)
774 apCorrMap = measAlg.makeCoaddApCorrMap(coaddInputs.ccds, coaddExposure.getBBox(afwImage.PARENT),
775 coaddExposure.getWcs())
776 coaddExposure.getInfo().setApCorrMap(apCorrMap)
777 if self.config.doAttachTransmissionCurve:
778 transmissionCurve = measAlg.makeCoaddTransmissionCurve(coaddExposure.getWcs(), coaddInputs.ccds)
779 coaddExposure.getInfo().setTransmissionCurve(transmissionCurve)
782 altMaskList, statsFlags, statsCtrl, nImage=None):
783 """Assemble the coadd for a sub-region.
785 For each coaddTempExp, check for (
and swap
in) an alternative mask
786 if one
is passed. Remove mask planes listed
in
787 `config.removeMaskPlanes`. Finally, stack the actual exposures using
788 `lsst.afw.math.statisticsStack`
with the statistic specified by
789 statsFlags. Typically, the statsFlag will be one of lsst.afw.math.MEAN
for
790 a mean-stack
or `lsst.afw.math.MEANCLIP`
for outlier rejection using
791 an N-sigma clipped mean where N
and iterations are specified by
792 statsCtrl. Assign the stacked subregion back to the coadd.
797 The target exposure
for the coadd.
798 bbox : `lsst.geom.Box`
800 tempExpRefList : `list`
801 List of data reference to tempExp.
802 imageScalerList : `list`
803 List of image scalers.
807 List of alternate masks to use rather than those stored
with
808 tempExp,
or None. Each element
is dict
with keys = mask plane
809 name to which to add the spans.
810 statsFlags : `lsst.afw.math.Property`
811 Property object
for statistic
for coadd.
813 Statistics control object
for coadd.
814 nImage : `lsst.afw.image.ImageU`, optional
815 Keeps track of exposure count
for each pixel.
817 self.log.debug("Computing coadd over %s", bbox)
819 coaddExposure.mask.addMaskPlane(
"REJECTED")
820 coaddExposure.mask.addMaskPlane(
"CLIPPED")
821 coaddExposure.mask.addMaskPlane(
"SENSOR_EDGE")
823 clipped = afwImage.Mask.getPlaneBitMask(
"CLIPPED")
825 if nImage
is not None:
826 subNImage = afwImage.ImageU(bbox.getWidth(), bbox.getHeight())
827 for tempExpRef, imageScaler, altMask
in zip(tempExpRefList, imageScalerList, altMaskList):
829 exposure = tempExpRef.get(parameters={
'bbox': bbox})
831 maskedImage = exposure.getMaskedImage()
832 mask = maskedImage.getMask()
833 if altMask
is not None:
835 imageScaler.scaleMaskedImage(maskedImage)
839 if nImage
is not None:
840 subNImage.getArray()[maskedImage.getMask().getArray() & statsCtrl.getAndMask() == 0] += 1
841 if self.config.removeMaskPlanes:
843 maskedImageList.append(maskedImage)
845 if self.config.doInputMap:
846 visit = exposure.getInfo().getCoaddInputs().visits[0].getId()
847 self.inputMapper.mask_warp_bbox(bbox, visit, mask, statsCtrl.getAndMask())
849 with self.timer(
"stack"):
850 coaddSubregion = afwMath.statisticsStack(maskedImageList, statsFlags, statsCtrl, weightList,
853 coaddExposure.maskedImage.assign(coaddSubregion, bbox)
854 if nImage
is not None:
855 nImage.assign(subNImage, bbox)
858 altMaskList, statsCtrl, nImage=None):
859 """Assemble the coadd using the "online" method.
861 This method takes a running sum of images and weights to save memory.
862 It only works
for MEAN statistics.
867 The target exposure
for the coadd.
868 tempExpRefList : `list`
869 List of data reference to tempExp.
870 imageScalerList : `list`
871 List of image scalers.
875 List of alternate masks to use rather than those stored
with
876 tempExp,
or None. Each element
is dict
with keys = mask plane
877 name to which to add the spans.
879 Statistics control object
for coadd.
880 nImage : `lsst.afw.image.ImageU`, optional
881 Keeps track of exposure count
for each pixel.
883 self.log.debug("Computing online coadd.")
885 coaddExposure.mask.addMaskPlane(
"REJECTED")
886 coaddExposure.mask.addMaskPlane(
"CLIPPED")
887 coaddExposure.mask.addMaskPlane(
"SENSOR_EDGE")
889 thresholdDict = AccumulatorMeanStack.stats_ctrl_to_threshold_dict(statsCtrl)
891 bbox = coaddExposure.maskedImage.getBBox()
893 stacker = AccumulatorMeanStack(
894 coaddExposure.image.array.shape,
895 statsCtrl.getAndMask(),
896 mask_threshold_dict=thresholdDict,
898 no_good_pixels_mask=statsCtrl.getNoGoodPixelsMask(),
899 calc_error_from_input_variance=self.config.calcErrorFromInputVariance,
900 compute_n_image=(nImage
is not None)
903 for tempExpRef, imageScaler, altMask, weight
in zip(tempExpRefList,
907 exposure = tempExpRef.get()
908 maskedImage = exposure.getMaskedImage()
909 mask = maskedImage.getMask()
910 if altMask
is not None:
912 imageScaler.scaleMaskedImage(maskedImage)
913 if self.config.removeMaskPlanes:
916 stacker.add_masked_image(maskedImage, weight=weight)
918 if self.config.doInputMap:
919 visit = exposure.getInfo().getCoaddInputs().visits[0].getId()
920 self.inputMapper.mask_warp_bbox(bbox, visit, mask, statsCtrl.getAndMask())
922 stacker.fill_stacked_masked_image(coaddExposure.maskedImage)
924 if nImage
is not None:
925 nImage.array[:, :] = stacker.n_image
928 """Unset the mask of an image for mask planes specified in the config.
933 The masked image to be modified.
937 InvalidParameterError
938 Raised if no mask plane
with that name was found.
940 mask = maskedImage.getMask()
941 for maskPlane
in self.config.removeMaskPlanes:
943 mask &= ~mask.getPlaneBitMask(maskPlane)
944 except pexExceptions.InvalidParameterError:
945 self.log.debug(
"Unable to remove mask plane %s: no mask plane with that name was found.",
950 """Map certain mask planes of the warps to new planes for the coadd.
952 If a pixel is rejected due to a mask value other than EDGE, NO_DATA,
953 or CLIPPED, set it to REJECTED on the coadd.
954 If a pixel
is rejected due to EDGE, set the coadd pixel to SENSOR_EDGE.
955 If a pixel
is rejected due to CLIPPED, set the coadd pixel to CLIPPED.
960 Statistics control object
for coadd.
964 maskMap : `list` of `tuple` of `int`
965 A list of mappings of mask planes of the warped exposures to
966 mask planes of the coadd.
968 edge = afwImage.Mask.getPlaneBitMask("EDGE")
969 noData = afwImage.Mask.getPlaneBitMask(
"NO_DATA")
970 clipped = afwImage.Mask.getPlaneBitMask(
"CLIPPED")
971 toReject = statsCtrl.getAndMask() & (~noData) & (~edge) & (~clipped)
972 maskMap = [(toReject, afwImage.Mask.getPlaneBitMask(
"REJECTED")),
973 (edge, afwImage.Mask.getPlaneBitMask(
"SENSOR_EDGE")),
978 """Apply in place alt mask formatted as SpanSets to a mask.
984 altMaskSpans : `dict`
985 SpanSet lists to apply. Each element contains the new mask
986 plane name (e.g. "CLIPPED and/or "NO_DATA
") as the key,
987 and list of SpanSets to apply to the mask.
994 if self.config.doUsePsfMatchedPolygons:
995 if (
"NO_DATA" in altMaskSpans)
and (
"NO_DATA" in self.config.badMaskPlanes):
1000 for spanSet
in altMaskSpans[
'NO_DATA']:
1001 spanSet.clippedTo(mask.getBBox()).clearMask(mask, self.
getBadPixelMask())
1003 for plane, spanSetList
in altMaskSpans.items():
1004 maskClipValue = mask.addMaskPlane(plane)
1005 for spanSet
in spanSetList:
1006 spanSet.clippedTo(mask.getBBox()).setMask(mask, 2**maskClipValue)
1010 """Shrink coaddInputs' ccds' ValidPolygons in place.
1012 Either modify each ccd's validPolygon in place, or if CoaddInputs
1013 does not have a validPolygon, create one
from its bbox.
1017 coaddInputs : `lsst.afw.image.coaddInputs`
1020 for ccd
in coaddInputs.ccds:
1021 polyOrig = ccd.getValidPolygon()
1022 validPolyBBox = polyOrig.getBBox()
if polyOrig
else ccd.getBBox()
1023 validPolyBBox.grow(-self.config.matchingKernelSize//2)
1025 validPolygon = polyOrig.intersectionSingle(validPolyBBox)
1027 validPolygon = afwGeom.polygon.Polygon(
geom.Box2D(validPolyBBox))
1028 ccd.setValidPolygon(validPolygon)
1031 """Set the bright object masks.
1036 Exposure under consideration.
1038 Table of bright objects to mask.
1039 dataId : `lsst.daf.butler.DataId`, optional
1040 Data identifier dict for patch.
1042 if brightObjectMasks
is None:
1043 self.log.warning(
"Unable to apply bright object mask: none supplied")
1045 self.log.info(
"Applying %d bright object masks to %s", len(brightObjectMasks), dataId)
1046 mask = exposure.getMaskedImage().getMask()
1047 wcs = exposure.getWcs()
1048 plateScale = wcs.getPixelScale().asArcseconds()
1050 for rec
in brightObjectMasks:
1051 center =
geom.PointI(wcs.skyToPixel(rec.getCoord()))
1052 if rec[
"type"] ==
"box":
1053 assert rec[
"angle"] == 0.0, (
"Angle != 0 for mask object %s" % rec[
"id"])
1054 width = rec[
"width"].asArcseconds()/plateScale
1055 height = rec[
"height"].asArcseconds()/plateScale
1058 bbox =
geom.Box2I(center - halfSize, center + halfSize)
1061 geom.PointI(int(center[0] + 0.5*width), int(center[1] + 0.5*height)))
1062 spans = afwGeom.SpanSet(bbox)
1063 elif rec[
"type"] ==
"circle":
1064 radius = int(rec[
"radius"].asArcseconds()/plateScale)
1065 spans = afwGeom.SpanSet.fromShape(radius, offset=center)
1067 self.log.warning(
"Unexpected region type %s at %s", rec[
"type"], center)
1072 """Set INEXACT_PSF mask plane.
1074 If any of the input images isn't represented in the coadd (due to
1075 clipped pixels or chip gaps), the `CoaddPsf` will be inexact. Flag
1081 Coadded exposure
's mask, modified in-place.
1083 mask.addMaskPlane("INEXACT_PSF")
1084 inexactPsf = mask.getPlaneBitMask(
"INEXACT_PSF")
1085 sensorEdge = mask.getPlaneBitMask(
"SENSOR_EDGE")
1086 clipped = mask.getPlaneBitMask(
"CLIPPED")
1087 rejected = mask.getPlaneBitMask(
"REJECTED")
1088 array = mask.getArray()
1089 selected = array & (sensorEdge | clipped | rejected) > 0
1090 array[selected] |= inexactPsf
1093 def _subBBoxIter(bbox, subregionSize):
1094 """Iterate over subregions of a bbox.
1099 Bounding box over which to iterate.
1106 Next sub-bounding box of size ``subregionSize`` or smaller; each ``subBBox``
1107 is contained within ``bbox``, so it may be smaller than ``subregionSize`` at
1108 the edges of ``bbox``, but it will never be empty.
1113 Raised
if any of the following occur:
1114 - The given bbox
is empty.
1115 - The subregionSize
is 0.
1118 raise RuntimeError(
"bbox %s is empty" % (bbox,))
1119 if subregionSize[0] < 1
or subregionSize[1] < 1:
1120 raise RuntimeError(
"subregionSize %s must be nonzero" % (subregionSize,))
1122 for rowShift
in range(0, bbox.getHeight(), subregionSize[1]):
1123 for colShift
in range(0, bbox.getWidth(), subregionSize[0]):
1126 if subBBox.isEmpty():
1127 raise RuntimeError(
"Bug: empty bbox! bbox=%s, subregionSize=%s, "
1128 "colShift=%s, rowShift=%s" %
1129 (bbox, subregionSize, colShift, rowShift))
1133 """Return list of only inputRefs with visitId in goodVisits ordered by goodVisit.
1137 inputs : `list` of `~lsst.pipe.base.connections.DeferredDatasetRef`
1138 List of `lsst.pipe.base.connections.DeferredDatasetRef` with dataId containing visit.
1140 Dictionary
with good visitIds
as the keys. Value ignored.
1144 filteredInputs : `list` of `~lsst.pipe.base.connections.DeferredDatasetRef`
1145 Filtered
and sorted list of inputRefs
with visitId
in goodVisits ordered by goodVisit.
1147 inputWarpDict = {inputRef.ref.dataId['visit']: inputRef
for inputRef
in inputs}
1149 for visit
in goodVisits.keys():
1150 if visit
in inputWarpDict:
1151 filteredInputs.append(inputWarpDict[visit])
1152 return filteredInputs
1156 """Function to count the number of pixels with a specific mask in a
1159 Find the intersection of mask & footprint. Count all pixels in the mask
1160 that are
in the intersection that have bitmask set but do
not have
1161 ignoreMask set. Return the count.
1166 Mask to define intersection region by.
1168 Footprint to define the intersection region by.
1170 Specific mask that we wish to count the number of occurances of.
1171 ignoreMask : `Unknown`
1172 Pixels to
not consider.
1177 Number of pixels
in footprint
with specified mask.
1179 bbox = footprint.getBBox()
1180 bbox.clip(mask.getBBox(afwImage.PARENT))
1181 fp = afwImage.Mask(bbox)
1182 subMask = mask.Factory(mask, bbox, afwImage.PARENT)
1183 footprint.spans.setMask(fp, bitmask)
1184 return numpy.logical_and((subMask.getArray() & fp.getArray()) > 0,
1185 (subMask.getArray() & ignoreMask) == 0).sum()
1189 psfMatchedWarps = pipeBase.connectionTypes.Input(
1190 doc=(
"PSF-Matched Warps are required by CompareWarp regardless of the coadd type requested. "
1191 "Only PSF-Matched Warps make sense for image subtraction. "
1192 "Therefore, they must be an additional declared input."),
1193 name=
"{inputCoaddName}Coadd_psfMatchedWarp",
1194 storageClass=
"ExposureF",
1195 dimensions=(
"tract",
"patch",
"skymap",
"visit"),
1199 templateCoadd = pipeBase.connectionTypes.Output(
1200 doc=(
"Model of the static sky, used to find temporal artifacts. Typically a PSF-Matched, "
1201 "sigma-clipped coadd. Written if and only if assembleStaticSkyModel.doWrite=True"),
1202 name=
"{outputCoaddName}CoaddPsfMatched",
1203 storageClass=
"ExposureF",
1204 dimensions=(
"tract",
"patch",
"skymap",
"band"),
1209 if not config.assembleStaticSkyModel.doWrite:
1210 self.outputs.remove(
"templateCoadd")
1215 pipelineConnections=CompareWarpAssembleCoaddConnections):
1216 assembleStaticSkyModel = pexConfig.ConfigurableField(
1217 target=AssembleCoaddTask,
1218 doc=
"Task to assemble an artifact-free, PSF-matched Coadd to serve as a"
1219 " naive/first-iteration model of the static sky.",
1221 detect = pexConfig.ConfigurableField(
1222 target=SourceDetectionTask,
1223 doc=
"Detect outlier sources on difference between each psfMatched warp and static sky model"
1225 detectTemplate = pexConfig.ConfigurableField(
1226 target=SourceDetectionTask,
1227 doc=
"Detect sources on static sky model. Only used if doPreserveContainedBySource is True"
1229 maskStreaks = pexConfig.ConfigurableField(
1230 target=MaskStreaksTask,
1231 doc=
"Detect streaks on difference between each psfMatched warp and static sky model. Only used if "
1232 "doFilterMorphological is True. Adds a mask plane to an exposure, with the mask plane name set by"
1235 streakMaskName = pexConfig.Field(
1238 doc=
"Name of mask bit used for streaks"
1240 maxNumEpochs = pexConfig.Field(
1241 doc=
"Charactistic maximum local number of epochs/visits in which an artifact candidate can appear "
1242 "and still be masked. The effective maxNumEpochs is a broken linear function of local "
1243 "number of epochs (N): min(maxFractionEpochsLow*N, maxNumEpochs + maxFractionEpochsHigh*N). "
1244 "For each footprint detected on the image difference between the psfMatched warp and static sky "
1245 "model, if a significant fraction of pixels (defined by spatialThreshold) are residuals in more "
1246 "than the computed effective maxNumEpochs, the artifact candidate is deemed persistant rather "
1247 "than transient and not masked.",
1251 maxFractionEpochsLow = pexConfig.RangeField(
1252 doc=
"Fraction of local number of epochs (N) to use as effective maxNumEpochs for low N. "
1253 "Effective maxNumEpochs = "
1254 "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)",
1259 maxFractionEpochsHigh = pexConfig.RangeField(
1260 doc=
"Fraction of local number of epochs (N) to use as effective maxNumEpochs for high N. "
1261 "Effective maxNumEpochs = "
1262 "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)",
1267 spatialThreshold = pexConfig.RangeField(
1268 doc=
"Unitless fraction of pixels defining how much of the outlier region has to meet the "
1269 "temporal criteria. If 0, clip all. If 1, clip none.",
1273 inclusiveMin=
True, inclusiveMax=
True
1275 doScaleWarpVariance = pexConfig.Field(
1276 doc=
"Rescale Warp variance plane using empirical noise?",
1280 scaleWarpVariance = pexConfig.ConfigurableField(
1281 target=ScaleVarianceTask,
1282 doc=
"Rescale variance on warps",
1284 doPreserveContainedBySource = pexConfig.Field(
1285 doc=
"Rescue artifacts from clipping that completely lie within a footprint detected"
1286 "on the PsfMatched Template Coadd. Replicates a behavior of SafeClip.",
1290 doPrefilterArtifacts = pexConfig.Field(
1291 doc=
"Ignore artifact candidates that are mostly covered by the bad pixel mask, "
1292 "because they will be excluded anyway. This prevents them from contributing "
1293 "to the outlier epoch count image and potentially being labeled as persistant."
1294 "'Mostly' is defined by the config 'prefilterArtifactsRatio'.",
1298 prefilterArtifactsMaskPlanes = pexConfig.ListField(
1299 doc=
"Prefilter artifact candidates that are mostly covered by these bad mask planes.",
1301 default=(
'NO_DATA',
'BAD',
'SAT',
'SUSPECT'),
1303 prefilterArtifactsRatio = pexConfig.Field(
1304 doc=
"Prefilter artifact candidates with less than this fraction overlapping good pixels",
1308 doFilterMorphological = pexConfig.Field(
1309 doc=
"Filter artifact candidates based on morphological criteria, i.g. those that appear to "
1314 growStreakFp = pexConfig.Field(
1315 doc=
"Grow streak footprints by this number multiplied by the PSF width",
1321 AssembleCoaddConfig.setDefaults(self)
1327 if "EDGE" in self.badMaskPlanes:
1328 self.badMaskPlanes.remove(
'EDGE')
1329 self.removeMaskPlanes.append(
'EDGE')
1338 self.
detect.doTempLocalBackground =
False
1339 self.
detect.reEstimateBackground =
False
1340 self.
detect.returnOriginalFootprints =
False
1341 self.
detect.thresholdPolarity =
"both"
1342 self.
detect.thresholdValue = 5
1343 self.
detect.minPixels = 4
1344 self.
detect.isotropicGrow =
True
1345 self.
detect.thresholdType =
"pixel_stdev"
1346 self.
detect.nSigmaToGrow = 0.4
1357 raise ValueError(
"No dataset type exists for a PSF-Matched Template N Image."
1358 "Please set assembleStaticSkyModel.doNImage=False")
1361 raise ValueError(
"warpType (%s) == assembleStaticSkyModel.warpType (%s) and will compete for "
1362 "the same dataset name. Please set assembleStaticSkyModel.doWrite to False "
1363 "or warpType to 'direct'. assembleStaticSkyModel.warpType should ways be "
1368 """Assemble a compareWarp coadded image from a set of warps
1369 by masking artifacts detected by comparing PSF-matched warps.
1371 In ``AssembleCoaddTask``, we compute the coadd as an clipped mean (i.e.,
1372 we clip outliers). The problem
with doing this
is that when computing the
1373 coadd PSF at a given location, individual visit PSFs
from visits
with
1374 outlier pixels contribute to the coadd PSF
and cannot be treated correctly.
1375 In this task, we correct
for this behavior by creating a new badMaskPlane
1376 'CLIPPED' which marks pixels
in the individual warps suspected to contain
1377 an artifact. We populate this plane on the input warps by comparing
1378 PSF-matched warps
with a PSF-matched median coadd which serves
as a
1379 model of the static sky. Any group of pixels that deviates
from the
1380 PSF-matched template coadd by more than config.detect.threshold sigma,
1381 is an artifact candidate. The candidates are then filtered to remove
1382 variable sources
and sources that are difficult to subtract such
as
1383 bright stars. This filter
is configured using the config parameters
1384 ``temporalThreshold``
and ``spatialThreshold``. The temporalThreshold
is
1385 the maximum fraction of epochs that the deviation can appear
in and still
1386 be considered an artifact. The spatialThreshold
is the maximum fraction of
1387 pixels
in the footprint of the deviation that appear
in other epochs
1388 (where other epochs
is defined by the temporalThreshold). If the deviant
1389 region meets this criteria of having a significant percentage of pixels
1390 that deviate
in only a few epochs, these pixels have the
'CLIPPED' bit
1391 set
in the mask. These regions will
not contribute to the final coadd.
1392 Furthermore, any routine to determine the coadd PSF can now be cognizant
1393 of clipped regions. Note that the algorithm implemented by this task
is
1394 preliminary
and works correctly
for HSC data. Parameter modifications
and
1395 or considerable redesigning of the algorithm
is likley required
for other
1398 ``CompareWarpAssembleCoaddTask`` sub-classes
1399 ``AssembleCoaddTask``
and instantiates ``AssembleCoaddTask``
1400 as a subtask to generate the TemplateCoadd (the model of the static sky).
1405 This task supports the following debug variables:
1407 If
True then save the Epoch Count Image
as a fits file
in the `figPath`
1409 Path to save the debug fits images
and figures
1412 ConfigClass = CompareWarpAssembleCoaddConfig
1413 _DefaultName = "compareWarpAssembleCoadd"
1416 AssembleCoaddTask.__init__(self, *args, **kwargs)
1417 self.makeSubtask(
"assembleStaticSkyModel")
1418 detectionSchema = afwTable.SourceTable.makeMinimalSchema()
1419 self.makeSubtask(
"detect", schema=detectionSchema)
1420 if self.config.doPreserveContainedBySource:
1421 self.makeSubtask(
"detectTemplate", schema=afwTable.SourceTable.makeMinimalSchema())
1422 if self.config.doScaleWarpVariance:
1423 self.makeSubtask(
"scaleWarpVariance")
1424 if self.config.doFilterMorphological:
1425 self.makeSubtask(
"maskStreaks")
1427 @utils.inheritDoc(AssembleCoaddTask)
1428 def _makeSupplementaryData(self, butlerQC, inputRefs, outputRefs):
1429 """Generate a templateCoadd to use as a naive model of static sky to
1430 subtract from PSF-Matched warps.
1434 result : `~lsst.pipe.base.Struct`
1435 Results
as a struct
with attributes:
1440 Keeps track of exposure count
for each pixel (`lsst.afw.image.ImageU`).
1445 Raised
if ``templateCoadd``
is `
None`.
1448 staticSkyModelInputRefs = copy.deepcopy(inputRefs)
1449 staticSkyModelInputRefs.inputWarps = inputRefs.psfMatchedWarps
1453 staticSkyModelOutputRefs = copy.deepcopy(outputRefs)
1454 if self.config.assembleStaticSkyModel.doWrite:
1455 staticSkyModelOutputRefs.coaddExposure = staticSkyModelOutputRefs.templateCoadd
1458 del outputRefs.templateCoadd
1459 del staticSkyModelOutputRefs.templateCoadd
1462 if 'nImage' in staticSkyModelOutputRefs.keys():
1463 del staticSkyModelOutputRefs.nImage
1465 templateCoadd = self.assembleStaticSkyModel.
runQuantum(butlerQC, staticSkyModelInputRefs,
1466 staticSkyModelOutputRefs)
1467 if templateCoadd
is None:
1470 return pipeBase.Struct(templateCoadd=templateCoadd.coaddExposure,
1471 nImage=templateCoadd.nImage,
1472 warpRefList=templateCoadd.warpRefList,
1473 imageScalerList=templateCoadd.imageScalerList,
1474 weightList=templateCoadd.weightList)
1476 def _noTemplateMessage(self, warpType):
1477 warpName = (warpType[0].upper() + warpType[1:])
1478 message =
"""No %(warpName)s warps were found to build the template coadd which is
1479 required to run CompareWarpAssembleCoaddTask. To continue assembling this type of coadd,
1480 first either rerun makeCoaddTempExp
with config.make%(warpName)s=
True or
1481 coaddDriver
with config.makeCoadTempExp.make%(warpName)s=
True, before assembleCoadd.
1483 Alternatively, to use another algorithm
with existing warps, retarget the CoaddDriverConfig to
1484 another algorithm like:
1487 config.assemble.retarget(SafeClipAssembleCoaddTask)
1488 """ % {"warpName": warpName}
1491 @utils.inheritDoc(AssembleCoaddTask)
1493 def run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
1495 """Assemble the coadd.
1497 Find artifacts and apply them to the warps
' masks creating a list of
1498 alternative masks with a new
"CLIPPED" plane
and updated
"NO_DATA"
1499 plane. Then
pass these alternative masks to the base
class's ``run``
1505 dataIds = [ref.dataId
for ref
in tempExpRefList]
1506 psfMatchedDataIds = [ref.dataId
for ref
in supplementaryData.warpRefList]
1508 if dataIds != psfMatchedDataIds:
1509 self.log.info(
"Reordering and or/padding PSF-matched visit input list")
1510 supplementaryData.warpRefList = reorderAndPadList(supplementaryData.warpRefList,
1511 psfMatchedDataIds, dataIds)
1512 supplementaryData.imageScalerList = reorderAndPadList(supplementaryData.imageScalerList,
1513 psfMatchedDataIds, dataIds)
1516 spanSetMaskList = self.
findArtifacts(supplementaryData.templateCoadd,
1517 supplementaryData.warpRefList,
1518 supplementaryData.imageScalerList)
1520 badMaskPlanes = self.config.badMaskPlanes[:]
1521 badMaskPlanes.append(
"CLIPPED")
1522 badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes)
1524 result = AssembleCoaddTask.run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
1525 spanSetMaskList, mask=badPixelMask)
1529 self.
applyAltEdgeMask(result.coaddExposure.maskedImage.mask, spanSetMaskList)
1533 """Propagate alt EDGE mask to SENSOR_EDGE AND INEXACT_PSF planes.
1539 altMaskList : `list` of `dict`
1540 List of Dicts containing ``spanSet`` lists.
1541 Each element contains the new mask plane name (e.g. "CLIPPED
1542 and/
or "NO_DATA")
as the key,
and list of ``SpanSets`` to apply to
1545 maskValue = mask.getPlaneBitMask(["SENSOR_EDGE",
"INEXACT_PSF"])
1546 for visitMask
in altMaskList:
1547 if "EDGE" in visitMask:
1548 for spanSet
in visitMask[
'EDGE']:
1549 spanSet.clippedTo(mask.getBBox()).setMask(mask, maskValue)
1554 Loop through warps twice. The first loop builds a map with the count
1555 of how many epochs each pixel deviates
from the templateCoadd by more
1556 than ``config.chiThreshold`` sigma. The second loop takes each
1557 difference image
and filters the artifacts detected
in each using
1558 count map to filter out variable sources
and sources that are
1559 difficult to subtract cleanly.
1564 Exposure to serve
as model of static sky.
1565 tempExpRefList : `list`
1566 List of data references to warps.
1567 imageScalerList : `list`
1568 List of image scalers.
1572 altMasks : `list` of `dict`
1573 List of dicts containing information about CLIPPED
1574 (i.e., artifacts), NO_DATA,
and EDGE pixels.
1576 self.log.debug("Generating Count Image, and mask lists.")
1577 coaddBBox = templateCoadd.getBBox()
1578 slateIm = afwImage.ImageU(coaddBBox)
1579 epochCountImage = afwImage.ImageU(coaddBBox)
1580 nImage = afwImage.ImageU(coaddBBox)
1581 spanSetArtifactList = []
1582 spanSetNoDataMaskList = []
1583 spanSetEdgeList = []
1584 spanSetBadMorphoList = []
1588 templateCoadd.mask.clearAllMaskPlanes()
1590 if self.config.doPreserveContainedBySource:
1591 templateFootprints = self.detectTemplate.detectFootprints(templateCoadd)
1593 templateFootprints =
None
1595 for warpRef, imageScaler
in zip(tempExpRefList, imageScalerList):
1597 if warpDiffExp
is not None:
1599 nImage.array += (numpy.isfinite(warpDiffExp.image.array)
1600 * ((warpDiffExp.mask.array & badPixelMask) == 0)).astype(numpy.uint16)
1601 fpSet = self.detect.detectFootprints(warpDiffExp, doSmooth=
False, clearMask=
True)
1602 fpSet.positive.merge(fpSet.negative)
1603 footprints = fpSet.positive
1605 spanSetList = [footprint.spans
for footprint
in footprints.getFootprints()]
1608 if self.config.doPrefilterArtifacts:
1612 self.detect.clearMask(warpDiffExp.mask)
1613 for spans
in spanSetList:
1614 spans.setImage(slateIm, 1, doClip=
True)
1615 spans.setMask(warpDiffExp.mask, warpDiffExp.mask.getPlaneBitMask(
"DETECTED"))
1616 epochCountImage += slateIm
1618 if self.config.doFilterMorphological:
1619 maskName = self.config.streakMaskName
1620 _ = self.maskStreaks.run(warpDiffExp)
1621 streakMask = warpDiffExp.mask
1622 spanSetStreak = afwGeom.SpanSet.fromMask(streakMask,
1623 streakMask.getPlaneBitMask(maskName)).split()
1625 psf = warpDiffExp.getPsf()
1626 for s, sset
in enumerate(spanSetStreak):
1627 psfShape = psf.computeShape(sset.computeCentroid())
1628 dilation = self.config.growStreakFp * psfShape.getDeterminantRadius()
1629 sset_dilated = sset.dilated(int(dilation))
1630 spanSetStreak[s] = sset_dilated
1636 nans = numpy.where(numpy.isnan(warpDiffExp.maskedImage.image.array), 1, 0)
1637 nansMask = afwImage.makeMaskFromArray(nans.astype(afwImage.MaskPixel))
1638 nansMask.setXY0(warpDiffExp.getXY0())
1639 edgeMask = warpDiffExp.mask
1640 spanSetEdgeMask = afwGeom.SpanSet.fromMask(edgeMask,
1641 edgeMask.getPlaneBitMask(
"EDGE")).split()
1645 nansMask = afwImage.MaskX(coaddBBox, 1)
1647 spanSetEdgeMask = []
1650 spanSetNoDataMask = afwGeom.SpanSet.fromMask(nansMask).split()
1652 spanSetNoDataMaskList.append(spanSetNoDataMask)
1653 spanSetArtifactList.append(spanSetList)
1654 spanSetEdgeList.append(spanSetEdgeMask)
1655 if self.config.doFilterMorphological:
1656 spanSetBadMorphoList.append(spanSetStreak)
1659 path = self._dataRef2DebugPath(
"epochCountIm", tempExpRefList[0], coaddLevel=
True)
1660 epochCountImage.writeFits(path)
1662 for i, spanSetList
in enumerate(spanSetArtifactList):
1664 filteredSpanSetList = self.
filterArtifacts(spanSetList, epochCountImage, nImage,
1666 spanSetArtifactList[i] = filteredSpanSetList
1667 if self.config.doFilterMorphological:
1668 spanSetArtifactList[i] += spanSetBadMorphoList[i]
1671 for artifacts, noData, edge
in zip(spanSetArtifactList, spanSetNoDataMaskList, spanSetEdgeList):
1672 altMasks.append({
'CLIPPED': artifacts,
1678 """Remove artifact candidates covered by bad mask plane.
1680 Any future editing of the candidate list that does not depend on
1681 temporal information should go
in this method.
1686 List of SpanSets representing artifact candidates.
1688 Exposure containing mask planes used to prefilter.
1693 List of SpanSets
with artifacts.
1695 badPixelMask = exp.mask.getPlaneBitMask(self.config.prefilterArtifactsMaskPlanes)
1696 goodArr = (exp.mask.array & badPixelMask) == 0
1697 returnSpanSetList = []
1698 bbox = exp.getBBox()
1699 x0, y0 = exp.getXY0()
1700 for i, span
in enumerate(spanSetList):
1701 y, x = span.clippedTo(bbox).indices()
1702 yIndexLocal = numpy.array(y) - y0
1703 xIndexLocal = numpy.array(x) - x0
1704 goodRatio = numpy.count_nonzero(goodArr[yIndexLocal, xIndexLocal])/span.getArea()
1705 if goodRatio > self.config.prefilterArtifactsRatio:
1706 returnSpanSetList.append(span)
1707 return returnSpanSetList
1709 def filterArtifacts(self, spanSetList, epochCountImage, nImage, footprintsToExclude=None):
1710 """Filter artifact candidates.
1715 List of SpanSets representing artifact candidates.
1717 Image of accumulated number of warpDiff detections.
1718 nImage : `lsst.afw.image.ImageU`
1719 Image of the accumulated number of total epochs contributing.
1723 maskSpanSetList : `list`
1724 List of SpanSets with artifacts.
1726 maskSpanSetList = []
1727 x0, y0 = epochCountImage.getXY0()
1728 for i, span
in enumerate(spanSetList):
1729 y, x = span.indices()
1730 yIdxLocal = [y1 - y0
for y1
in y]
1731 xIdxLocal = [x1 - x0
for x1
in x]
1732 outlierN = epochCountImage.array[yIdxLocal, xIdxLocal]
1733 totalN = nImage.array[yIdxLocal, xIdxLocal]
1736 effMaxNumEpochsHighN = (self.config.maxNumEpochs
1737 + self.config.maxFractionEpochsHigh*numpy.mean(totalN))
1738 effMaxNumEpochsLowN = self.config.maxFractionEpochsLow * numpy.mean(totalN)
1739 effectiveMaxNumEpochs = int(min(effMaxNumEpochsLowN, effMaxNumEpochsHighN))
1740 nPixelsBelowThreshold = numpy.count_nonzero((outlierN > 0)
1741 & (outlierN <= effectiveMaxNumEpochs))
1742 percentBelowThreshold = nPixelsBelowThreshold / len(outlierN)
1743 if percentBelowThreshold > self.config.spatialThreshold:
1744 maskSpanSetList.append(span)
1746 if self.config.doPreserveContainedBySource
and footprintsToExclude
is not None:
1748 filteredMaskSpanSetList = []
1749 for span
in maskSpanSetList:
1751 for footprint
in footprintsToExclude.positive.getFootprints():
1752 if footprint.spans.contains(span):
1756 filteredMaskSpanSetList.append(span)
1757 maskSpanSetList = filteredMaskSpanSetList
1759 return maskSpanSetList
1761 def _readAndComputeWarpDiff(self, warpRef, imageScaler, templateCoadd):
1762 """Fetch a warp from the butler and return a warpDiff.
1766 warpRef : `lsst.daf.butler.DeferredDatasetHandle`
1767 Handle for the warp.
1769 An image scaler object.
1771 Exposure to be substracted
from the scaled warp.
1776 Exposure of the image difference between the warp
and template.
1783 warp = warpRef.get()
1785 imageScaler.scaleMaskedImage(warp.getMaskedImage())
1786 mi = warp.getMaskedImage()
1787 if self.config.doScaleWarpVariance:
1789 self.scaleWarpVariance.run(mi)
1790 except Exception
as exc:
1791 self.log.warning(
"Unable to rescale variance of warp (%s); leaving it as-is", exc)
1792 mi -= templateCoadd.getMaskedImage()
def shrinkValidPolygons(self, coaddInputs)
def setRejectedMaskMapping(statsCtrl)
def assembleMetadata(self, coaddExposure, tempExpRefList, weightList)
def assembleOnlineMeanCoadd(self, coaddExposure, tempExpRefList, imageScalerList, weightList, altMaskList, statsCtrl, nImage=None)
def runQuantum(self, butlerQC, inputRefs, outputRefs)
def _makeSupplementaryData(self, butlerQC, inputRefs, outputRefs)
def processResults(self, coaddExposure, brightObjectMasks=None, dataId=None)
def prepareInputs(self, refList)
def setBrightObjectMasks(self, exposure, brightObjectMasks, dataId=None)
def removeMaskPlanes(self, maskedImage)
def __init__(self, *args, **kwargs)
def _subBBoxIter(bbox, subregionSize)
def assembleSubregion(self, coaddExposure, bbox, tempExpRefList, imageScalerList, weightList, altMaskList, statsFlags, statsCtrl, nImage=None)
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, altMaskList=None, mask=None, supplementaryData=None)
def applyAltMaskPlanes(self, mask, altMaskSpans)
def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs)
def filterWarps(self, inputs, goodVisits)
def prepareStats(self, mask=None)
def setInexactPsf(self, mask)
def __init__(self, *config=None)
def prefilterArtifacts(self, spanSetList, exp)
def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList)
def _readAndComputeWarpDiff(self, warpRef, imageScaler, templateCoadd)
def applyAltEdgeMask(self, mask, altMaskList)
def _noTemplateMessage(self, warpType)
def filterArtifacts(self, spanSetList, epochCountImage, nImage, footprintsToExclude=None)
def __init__(self, *args, **kwargs)
def getTempExpDatasetName(self, warpType="direct")
def getBadPixelMask(self)
def countMaskFromFootprint(mask, footprint, bitmask, ignoreMask)