22__all__ = [
"AssembleCoaddTask",
"AssembleCoaddConnections",
"AssembleCoaddConfig",
23 "CompareWarpAssembleCoaddTask",
"CompareWarpAssembleCoaddConfig"]
40import lsst.utils
as utils
42from .coaddBase
import CoaddBaseTask, makeSkyInfo, reorderAndPadList
43from .interpImage
import InterpImageTask
44from .scaleZeroPoint
import ScaleZeroPointTask
45from .maskStreaks
import MaskStreaksTask
46from .healSparseMapping
import HealSparseInputMapTask
48from lsst.utils.timer
import timeMethod
49from deprecated.sphinx
import deprecated
51log = logging.getLogger(__name__)
55 dimensions=(
"tract",
"patch",
"band",
"skymap"),
56 defaultTemplates={
"inputCoaddName":
"deep",
57 "outputCoaddName":
"deep",
59 "warpTypeSuffix":
""}):
61 inputWarps = pipeBase.connectionTypes.Input(
62 doc=(
"Input list of warps to be assemebled i.e. stacked."
63 "WarpType (e.g. direct, psfMatched) is controlled by the warpType config parameter"),
64 name=
"{inputCoaddName}Coadd_{warpType}Warp",
65 storageClass=
"ExposureF",
66 dimensions=(
"tract",
"patch",
"skymap",
"visit",
"instrument"),
70 skyMap = pipeBase.connectionTypes.Input(
71 doc=
"Input definition of geometry/bbox and projection/wcs for coadded exposures",
72 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
73 storageClass=
"SkyMap",
74 dimensions=(
"skymap", ),
76 selectedVisits = pipeBase.connectionTypes.Input(
77 doc=
"Selected visits to be coadded.",
78 name=
"{outputCoaddName}Visits",
79 storageClass=
"StructuredDataDict",
80 dimensions=(
"instrument",
"tract",
"patch",
"skymap",
"band")
82 brightObjectMask = pipeBase.connectionTypes.PrerequisiteInput(
83 doc=(
"Input Bright Object Mask mask produced with external catalogs to be applied to the mask plane"
85 name=
"brightObjectMask",
86 storageClass=
"ObjectMaskCatalog",
87 dimensions=(
"tract",
"patch",
"skymap",
"band"),
90 coaddExposure = pipeBase.connectionTypes.Output(
91 doc=
"Output coadded exposure, produced by stacking input warps",
92 name=
"{outputCoaddName}Coadd{warpTypeSuffix}",
93 storageClass=
"ExposureF",
94 dimensions=(
"tract",
"patch",
"skymap",
"band"),
96 nImage = pipeBase.connectionTypes.Output(
97 doc=
"Output image of number of input images per pixel",
98 name=
"{outputCoaddName}Coadd_nImage",
99 storageClass=
"ImageU",
100 dimensions=(
"tract",
"patch",
"skymap",
"band"),
102 inputMap = pipeBase.connectionTypes.Output(
103 doc=
"Output healsparse map of input images",
104 name=
"{outputCoaddName}Coadd_inputMap",
105 storageClass=
"HealSparseMap",
106 dimensions=(
"tract",
"patch",
"skymap",
"band"),
109 def __init__(self, *, config=None):
110 super().__init__(config=config)
112 if not config.doMaskBrightObjects:
113 self.prerequisiteInputs.remove(
"brightObjectMask")
115 if not config.doSelectVisits:
116 self.inputs.remove(
"selectedVisits")
118 if not config.doNImage:
119 self.outputs.remove(
"nImage")
121 if not self.config.doInputMap:
122 self.outputs.remove(
"inputMap")
125class AssembleCoaddConfig(CoaddBaseTask.ConfigClass, pipeBase.PipelineTaskConfig,
126 pipelineConnections=AssembleCoaddConnections):
127 warpType = pexConfig.Field(
128 doc=
"Warp name: one of 'direct' or 'psfMatched'",
132 subregionSize = pexConfig.ListField(
134 doc=
"Width, height of stack subregion size; "
135 "make small enough that a full stack of images will fit into memory at once.",
137 default=(2000, 2000),
139 statistic = pexConfig.Field(
141 doc=
"Main stacking statistic for aggregating over the epochs.",
144 doOnlineForMean = pexConfig.Field(
146 doc=
"Perform online coaddition when statistic=\"MEAN\" to save memory?",
149 doSigmaClip = pexConfig.Field(
151 doc=
"Perform sigma clipped outlier rejection with MEANCLIP statistic? (DEPRECATED)",
154 sigmaClip = pexConfig.Field(
156 doc=
"Sigma for outlier rejection; ignored if non-clipping statistic selected.",
159 clipIter = pexConfig.Field(
161 doc=
"Number of iterations of outlier rejection; ignored if non-clipping statistic selected.",
164 calcErrorFromInputVariance = pexConfig.Field(
166 doc=
"Calculate coadd variance from input variance by stacking statistic."
167 "Passed to StatisticsControl.setCalcErrorFromInputVariance()",
170 scaleZeroPoint = pexConfig.ConfigurableField(
171 target=ScaleZeroPointTask,
172 doc=
"Task to adjust the photometric zero point of the coadd temp exposures",
174 doInterp = pexConfig.Field(
175 doc=
"Interpolate over NaN pixels? Also extrapolate, if necessary, but the results are ugly.",
179 interpImage = pexConfig.ConfigurableField(
180 target=InterpImageTask,
181 doc=
"Task to interpolate (and extrapolate) over NaN pixels",
183 doWrite = pexConfig.Field(
184 doc=
"Persist coadd?",
188 doNImage = pexConfig.Field(
189 doc=
"Create image of number of contributing exposures for each pixel",
193 doUsePsfMatchedPolygons = pexConfig.Field(
194 doc=
"Use ValidPolygons from shrunk Psf-Matched Calexps? Should be set to True by CompareWarp only.",
198 maskPropagationThresholds = pexConfig.DictField(
201 doc=(
"Threshold (in fractional weight) of rejection at which we propagate a mask plane to "
202 "the coadd; that is, we set the mask bit on the coadd if the fraction the rejected frames "
203 "would have contributed exceeds this value."),
204 default={
"SAT": 0.1},
206 removeMaskPlanes = pexConfig.ListField(dtype=str, default=[
"NOT_DEBLENDED"],
207 doc=
"Mask planes to remove before coadding")
208 doMaskBrightObjects = pexConfig.Field(dtype=bool, default=
False,
209 doc=
"Set mask and flag bits for bright objects?")
210 brightObjectMaskName = pexConfig.Field(dtype=str, default=
"BRIGHT_OBJECT",
211 doc=
"Name of mask bit used for bright objects")
212 coaddPsf = pexConfig.ConfigField(
213 doc=
"Configuration for CoaddPsf",
214 dtype=measAlg.CoaddPsfConfig,
216 doAttachTransmissionCurve = pexConfig.Field(
217 dtype=bool, default=
False, optional=
False,
218 doc=(
"Attach a piecewise TransmissionCurve for the coadd? "
219 "(requires all input Exposures to have TransmissionCurves).")
221 hasFakes = pexConfig.Field(
224 doc=
"Should be set to True if fake sources have been inserted into the input data."
226 doSelectVisits = pexConfig.Field(
227 doc=
"Coadd only visits selected by a SelectVisitsTask",
231 doInputMap = pexConfig.Field(
232 doc=
"Create a bitwise map of coadd inputs",
236 inputMapper = pexConfig.ConfigurableField(
237 doc=
"Input map creation subtask.",
238 target=HealSparseInputMapTask,
243 self.badMaskPlanes = [
"NO_DATA",
"BAD",
"SAT",
"EDGE"]
250 log.warning(
"Config doPsfMatch deprecated. Setting warpType='psfMatched'")
251 self.warpType =
'psfMatched'
252 if self.doSigmaClip
and self.statistic !=
"MEANCLIP":
253 log.warning(
'doSigmaClip deprecated. To replicate behavior, setting statistic to "MEANCLIP"')
254 self.statistic =
"MEANCLIP"
255 if self.doInterp
and self.statistic
not in [
'MEAN',
'MEDIAN',
'MEANCLIP',
'VARIANCE',
'VARIANCECLIP']:
256 raise ValueError(
"Must set doInterp=False for statistic=%s, which does not "
257 "compute and set a non-zero coadd variance estimate." % (self.statistic))
259 unstackableStats = [
'NOTHING',
'ERROR',
'ORMASK']
260 if not hasattr(afwMath.Property, self.statistic)
or self.statistic
in unstackableStats:
261 stackableStats = [
str(k)
for k
in afwMath.Property.__members__.keys()
262 if str(k)
not in unstackableStats]
263 raise ValueError(
"statistic %s is not allowed. Please choose one of %s."
264 % (self.statistic, stackableStats))
268 """Assemble a coadded image from a set of warps.
270 Each Warp that goes into a coadd will typically have an independent
271 photometric zero-point. Therefore, we must scale each Warp to set it to
272 a common photometric zeropoint. WarpType may be one of 'direct' or
273 'psfMatched',
and the boolean configs `config.makeDirect`
and
274 `config.makePsfMatched` set which of the warp types will be coadded.
275 The coadd
is computed
as a mean
with optional outlier rejection.
276 Criteria
for outlier rejection are set
in `AssembleCoaddConfig`.
277 Finally, Warps can have bad
'NaN' pixels which received no input
from the
278 source calExps. We interpolate over these bad (NaN) pixels.
280 `AssembleCoaddTask` uses several sub-tasks. These are
282 - `~lsst.pipe.tasks.ScaleZeroPointTask`
283 - create
and use an ``imageScaler`` object to scale the photometric zeropoint
for each Warp
284 - `~lsst.pipe.tasks.InterpImageTask`
285 - interpolate across bad pixels (NaN)
in the final coadd
287 You can retarget these subtasks
if you wish.
292 Raised
if unable to define mask plane
for bright objects.
297 `AssembleCoaddTask` has no debug variables of its own. Some of the
298 subtasks may support `~lsst.base.lsstDebug` variables. See the
299 documentation
for the subtasks
for further information.
303 `AssembleCoaddTask` assembles a set of warped images into a coadded image.
304 The `AssembleCoaddTask` can be invoked by running ``assembleCoadd.py``
305 with the flag
'--legacyCoadd'. Usage of assembleCoadd.py expects two
306 inputs: a data reference to the tract patch
and filter to be coadded,
and
307 a list of Warps to attempt to coadd. These are specified using ``--id``
and
308 ``--selectId``, respectively:
312 --id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]
313 --selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]
315 Only the Warps that cover the specified tract
and patch will be coadded.
316 A list of the available optional arguments can be obtained by calling
317 ``assembleCoadd.py``
with the ``--help`` command line argument:
321 assembleCoadd.py --help
323 To demonstrate usage of the `AssembleCoaddTask`
in the larger context of
324 multi-band processing, we will generate the HSC-I & -R band coadds
from
325 HSC engineering test data provided
in the ``ci_hsc`` package. To begin,
326 assuming that the lsst stack has been already set up, we must set up the
327 obs_subaru
and ``ci_hsc`` packages. This defines the environment variable
328 ``$CI_HSC_DIR``
and points at the location of the package. The raw HSC
329 data live
in the ``$CI_HSC_DIR/raw directory``. To begin assembling the
330 coadds, we must first run:
333 - process the individual ccds
in $CI_HSC_RAW to produce calibrated exposures
335 - create a skymap that covers the area of the sky present
in the raw exposures
337 - warp the individual calibrated exposures to the tangent plane of the coadd
339 We can perform all of these steps by running
343 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
345 This will produce warped exposures
for each visit. To coadd the warped
346 data, we call assembleCoadd.py
as follows:
350 assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \
351 --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \
352 --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \
353 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \
354 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \
355 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \
356 --selectId visit=903988 ccd=24
358 that will process the HSC-I band data. The results are written
in
359 ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``.
361 You may also choose to run:
365 scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346
366 assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R \
367 --selectId visit=903334 ccd=16 --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 \
368 --selectId visit=903334 ccd=100 --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 \
369 --selectId visit=903338 ccd=18 --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 \
370 --selectId visit=903342 ccd=10 --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 \
371 --selectId visit=903344 ccd=5 --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 \
372 --selectId visit=903346 ccd=6 --selectId visit=903346 ccd=12
374 to generate the coadd
for the HSC-R band
if you are interested
in
375 following multiBand Coadd processing
as discussed
in `pipeTasks_multiBand`
376 (but note that normally, one would use the `SafeClipAssembleCoaddTask`
377 rather than `AssembleCoaddTask` to make the coadd.
380 ConfigClass = AssembleCoaddConfig
381 _DefaultName = "assembleCoadd"
386 argNames = [
"config",
"name",
"parentTask",
"log"]
387 kwargs.update({k: v
for k, v
in zip(argNames, args)})
388 warnings.warn(
"AssembleCoadd received positional args, and casting them as kwargs: %s. "
389 "PipelineTask will not take positional args" % argNames, FutureWarning)
392 self.makeSubtask(
"interpImage")
393 self.makeSubtask(
"scaleZeroPoint")
395 if self.config.doMaskBrightObjects:
396 mask = afwImage.Mask()
399 except pexExceptions.LsstCppException:
400 raise RuntimeError(
"Unable to define mask plane for bright objects; planes used are %s" %
401 mask.getMaskPlaneDict().keys())
404 if self.config.doInputMap:
405 self.makeSubtask(
"inputMapper")
409 @utils.inheritDoc(pipeBase.PipelineTask)
411 inputData = butlerQC.get(inputRefs)
415 skyMap = inputData[
"skyMap"]
416 outputDataId = butlerQC.quantum.dataId
418 inputData[
'skyInfo'] = makeSkyInfo(skyMap,
419 tractId=outputDataId[
'tract'],
420 patchId=outputDataId[
'patch'])
422 if self.config.doSelectVisits:
423 warpRefList = self.
filterWarps(inputData[
'inputWarps'], inputData[
'selectedVisits'])
425 warpRefList = inputData[
'inputWarps']
428 self.log.info(
"Found %d %s", len(inputs.tempExpRefList),
430 if len(inputs.tempExpRefList) == 0:
431 raise pipeBase.NoWorkFound(
"No coadd temporary exposures found")
434 retStruct = self.
run(inputData[
'skyInfo'], inputs.tempExpRefList, inputs.imageScalerList,
435 inputs.weightList, supplementaryData=supplementaryData)
437 inputData.setdefault(
'brightObjectMask',
None)
438 if self.config.doMaskBrightObjects
and inputData[
"brightObjectMask"]
is None:
439 log.warning(
"doMaskBrightObjects is set to True, but brightObjectMask not loaded")
440 self.
processResults(retStruct.coaddExposure, inputData[
'brightObjectMask'], outputDataId)
442 if self.config.doWrite:
443 butlerQC.put(retStruct, outputRefs)
447 """Interpolate over missing data and mask bright stars.
452 The coadded exposure to process.
454 Table of bright objects to mask.
455 dataId : `lsst.daf.butler.DataId`
or `
None`, optional
458 if self.config.doInterp:
459 self.interpImage.run(coaddExposure.getMaskedImage(), planeName=
"NO_DATA")
461 varArray = coaddExposure.variance.array
462 with numpy.errstate(invalid=
"ignore"):
463 varArray[:] = numpy.where(varArray > 0, varArray, numpy.inf)
465 if self.config.doMaskBrightObjects:
468 def _makeSupplementaryData(self, butlerQC, inputRefs, outputRefs):
469 """Make additional inputs to run() specific to subclasses (Gen3).
471 Duplicates interface of `runQuantum` method.
472 Available to be implemented by subclasses only if they need the
473 coadd dataRef
for performing preliminary processing before
474 assembling the coadd.
478 butlerQC : `~lsst.pipe.base.ButlerQuantumContext`
479 Gen3 Butler object
for fetching additional data products before
480 running the Task specialized
for quantum being processed.
481 inputRefs : `~lsst.pipe.base.InputQuantizedConnection`
482 Attributes are the names of the connections describing input dataset types.
483 Values are DatasetRefs that task consumes
for corresponding dataset type.
484 DataIds are guaranteed to match data objects
in ``inputData``.
485 outputRefs : `~lsst.pipe.base.OutputQuantizedConnection`
486 Attributes are the names of the connections describing output dataset types.
487 Values are DatasetRefs that task
is to produce
488 for corresponding dataset type.
490 return pipeBase.Struct()
493 reason=
"makeSupplementaryDataGen3 is deprecated in favor of _makeSupplementaryData",
495 category=FutureWarning
501 """Prepare the input warps for coaddition by measuring the weight for
502 each warp and the scaling
for the photometric zero point.
504 Each Warp has its own photometric zeropoint
and background variance.
505 Before coadding these Warps together, compute a scale factor to
506 normalize the photometric zeropoint
and compute the weight
for each Warp.
511 List of data references to tempExp.
515 result : `~lsst.pipe.base.Struct`
516 Results
as a struct
with attributes:
519 `list` of data references to tempExp.
521 `list` of weightings.
523 `list` of image scalers.
525 statsCtrl = afwMath.StatisticsControl()
526 statsCtrl.setNumSigmaClip(self.config.sigmaClip)
527 statsCtrl.setNumIter(self.config.clipIter)
529 statsCtrl.setNanSafe(True)
537 for tempExpRef
in refList:
538 tempExp = tempExpRef.get()
540 if numpy.isnan(tempExp.image.array).all():
542 maskedImage = tempExp.getMaskedImage()
543 imageScaler = self.scaleZeroPoint.computeImageScaler(
548 imageScaler.scaleMaskedImage(maskedImage)
549 except Exception
as e:
550 self.log.warning(
"Scaling failed for %s (skipping it): %s", tempExpRef.dataId, e)
552 statObj = afwMath.makeStatistics(maskedImage.getVariance(), maskedImage.getMask(),
553 afwMath.MEANCLIP, statsCtrl)
554 meanVar, meanVarErr = statObj.getResult(afwMath.MEANCLIP)
555 weight = 1.0 / float(meanVar)
556 if not numpy.isfinite(weight):
557 self.log.warning(
"Non-finite weight for %s: skipping", tempExpRef.dataId)
559 self.log.info(
"Weight of %s %s = %0.3f", tempExpName, tempExpRef.dataId, weight)
564 tempExpRefList.append(tempExpRef)
565 weightList.append(weight)
566 imageScalerList.append(imageScaler)
568 return pipeBase.Struct(tempExpRefList=tempExpRefList, weightList=weightList,
569 imageScalerList=imageScalerList)
572 """Prepare the statistics for coadding images.
576 mask : `int`, optional
577 Bit mask value to exclude from coaddition.
581 stats : `~lsst.pipe.base.Struct`
582 Statistics
as a struct
with attributes:
587 Statistic
for coadd (`~lsst.afw.math.Property`).
591 statsCtrl = afwMath.StatisticsControl()
592 statsCtrl.setNumSigmaClip(self.config.sigmaClip)
593 statsCtrl.setNumIter(self.config.clipIter)
594 statsCtrl.setAndMask(mask)
595 statsCtrl.setNanSafe(
True)
596 statsCtrl.setWeighted(
True)
597 statsCtrl.setCalcErrorFromInputVariance(self.config.calcErrorFromInputVariance)
598 for plane, threshold
in self.config.maskPropagationThresholds.items():
599 bit = afwImage.Mask.getMaskPlane(plane)
600 statsCtrl.setMaskPropagationThreshold(bit, threshold)
601 statsFlags = afwMath.stringToStatisticsProperty(self.config.statistic)
602 return pipeBase.Struct(ctrl=statsCtrl, flags=statsFlags)
605 def run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
606 altMaskList=None, mask=None, supplementaryData=None):
607 """Assemble a coadd from input warps.
609 Assemble the coadd using the provided list of coaddTempExps. Since
610 the full coadd covers a patch (a large area), the assembly is
611 performed over small areas on the image at a time
in order to
612 conserve memory usage. Iterate over subregions within the outer
613 bbox of the patch using `assembleSubregion` to stack the corresponding
614 subregions
from the coaddTempExps
with the statistic specified.
615 Set the edge bits the coadd mask based on the weight map.
619 skyInfo : `~lsst.pipe.base.Struct`
620 Struct
with geometric information about the patch.
621 tempExpRefList : `list`
622 List of data references to Warps (previously called CoaddTempExps).
623 imageScalerList : `list`
624 List of image scalers.
627 altMaskList : `list`, optional
628 List of alternate masks to use rather than those stored
with
630 mask : `int`, optional
631 Bit mask value to exclude
from coaddition.
632 supplementaryData : `~lsst.pipe.base.Struct`, optional
633 Struct
with additional data products needed to assemble coadd.
634 Only used by subclasses that implement ``_makeSupplementaryData``
639 result : `~lsst.pipe.base.Struct`
640 Results
as a struct
with attributes:
647 Bit-wise map of inputs,
if requested.
649 Input list of refs to the warps (``lsst.daf.butler.DeferredDatasetHandle``)
652 Input list of image scalers (`list`) (unmodified).
654 Input list of weights (`list`) (unmodified).
657 self.log.info("Assembling %s %s", len(tempExpRefList), tempExpName)
660 if altMaskList
is None:
661 altMaskList = [
None]*len(tempExpRefList)
663 coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
664 coaddExposure.setPhotoCalib(self.scaleZeroPoint.getPhotoCalib())
665 coaddExposure.getInfo().setCoaddInputs(self.inputRecorder.makeCoaddInputs())
667 coaddMaskedImage = coaddExposure.getMaskedImage()
668 subregionSizeArr = self.config.subregionSize
669 subregionSize =
geom.Extent2I(subregionSizeArr[0], subregionSizeArr[1])
671 if self.config.doNImage:
672 nImage = afwImage.ImageU(skyInfo.bbox)
677 if self.config.doInputMap:
678 self.inputMapper.build_ccd_input_map(skyInfo.bbox,
680 coaddExposure.getInfo().getCoaddInputs().ccds)
682 if self.config.doOnlineForMean
and self.config.statistic ==
"MEAN":
685 weightList, altMaskList, stats.ctrl,
687 except Exception
as e:
688 self.log.exception(
"Cannot compute online coadd %s", e)
691 for subBBox
in self.
_subBBoxIter(skyInfo.bbox, subregionSize):
694 weightList, altMaskList, stats.flags, stats.ctrl,
696 except Exception
as e:
697 self.log.exception(
"Cannot compute coadd %s: %s", subBBox, e)
701 if self.config.doInputMap:
702 self.inputMapper.finalize_ccd_input_map_mask()
703 inputMap = self.inputMapper.ccd_input_map
710 coaddUtils.setCoaddEdgeBits(coaddMaskedImage.getMask(), coaddMaskedImage.getVariance())
711 return pipeBase.Struct(coaddExposure=coaddExposure, nImage=nImage,
712 warpRefList=tempExpRefList, imageScalerList=imageScalerList,
713 weightList=weightList, inputMap=inputMap)
716 """Set the metadata for the coadd.
718 This basic implementation sets the filter from the first input.
723 The target exposure
for the coadd.
724 tempExpRefList : `list`
725 List of data references to tempExp.
732 Raised
if there
is a length mismatch.
734 assert len(tempExpRefList) == len(weightList),
"Length mismatch"
741 tempExpList = [tempExpRef.get(parameters={
'bbox': bbox})
for tempExpRef
in tempExpRefList]
743 numCcds = sum(len(tempExp.getInfo().getCoaddInputs().ccds)
for tempExp
in tempExpList)
747 coaddExposure.setFilter(afwImage.FilterLabel(tempExpList[0].getFilter().bandLabel))
748 coaddInputs = coaddExposure.getInfo().getCoaddInputs()
749 coaddInputs.ccds.reserve(numCcds)
750 coaddInputs.visits.reserve(len(tempExpList))
752 for tempExp, weight
in zip(tempExpList, weightList):
753 self.inputRecorder.addVisitToCoadd(coaddInputs, tempExp, weight)
755 if self.config.doUsePsfMatchedPolygons:
758 coaddInputs.visits.sort()
759 coaddInputs.ccds.sort()
765 modelPsfList = [tempExp.getPsf()
for tempExp
in tempExpList]
766 modelPsfWidthList = [modelPsf.computeBBox(modelPsf.getAveragePosition()).getWidth()
767 for modelPsf
in modelPsfList]
768 psf = modelPsfList[modelPsfWidthList.index(max(modelPsfWidthList))]
770 psf = measAlg.CoaddPsf(coaddInputs.ccds, coaddExposure.getWcs(),
771 self.config.coaddPsf.makeControl())
772 coaddExposure.setPsf(psf)
773 apCorrMap = measAlg.makeCoaddApCorrMap(coaddInputs.ccds, coaddExposure.getBBox(afwImage.PARENT),
774 coaddExposure.getWcs())
775 coaddExposure.getInfo().setApCorrMap(apCorrMap)
776 if self.config.doAttachTransmissionCurve:
777 transmissionCurve = measAlg.makeCoaddTransmissionCurve(coaddExposure.getWcs(), coaddInputs.ccds)
778 coaddExposure.getInfo().setTransmissionCurve(transmissionCurve)
781 altMaskList, statsFlags, statsCtrl, nImage=None):
782 """Assemble the coadd for a sub-region.
784 For each coaddTempExp, check for (
and swap
in) an alternative mask
785 if one
is passed. Remove mask planes listed
in
786 `config.removeMaskPlanes`. Finally, stack the actual exposures using
787 `lsst.afw.math.statisticsStack`
with the statistic specified by
788 statsFlags. Typically, the statsFlag will be one of lsst.afw.math.MEAN
for
789 a mean-stack
or `lsst.afw.math.MEANCLIP`
for outlier rejection using
790 an N-sigma clipped mean where N
and iterations are specified by
791 statsCtrl. Assign the stacked subregion back to the coadd.
796 The target exposure
for the coadd.
797 bbox : `lsst.geom.Box`
799 tempExpRefList : `list`
800 List of data reference to tempExp.
801 imageScalerList : `list`
802 List of image scalers.
806 List of alternate masks to use rather than those stored
with
807 tempExp,
or None. Each element
is dict
with keys = mask plane
808 name to which to add the spans.
809 statsFlags : `lsst.afw.math.Property`
810 Property object
for statistic
for coadd.
812 Statistics control object
for coadd.
813 nImage : `lsst.afw.image.ImageU`, optional
814 Keeps track of exposure count
for each pixel.
816 self.log.debug("Computing coadd over %s", bbox)
818 coaddExposure.mask.addMaskPlane(
"REJECTED")
819 coaddExposure.mask.addMaskPlane(
"CLIPPED")
820 coaddExposure.mask.addMaskPlane(
"SENSOR_EDGE")
822 clipped = afwImage.Mask.getPlaneBitMask(
"CLIPPED")
824 if nImage
is not None:
825 subNImage = afwImage.ImageU(bbox.getWidth(), bbox.getHeight())
826 for tempExpRef, imageScaler, altMask
in zip(tempExpRefList, imageScalerList, altMaskList):
828 exposure = tempExpRef.get(parameters={
'bbox': bbox})
830 maskedImage = exposure.getMaskedImage()
831 mask = maskedImage.getMask()
832 if altMask
is not None:
834 imageScaler.scaleMaskedImage(maskedImage)
838 if nImage
is not None:
839 subNImage.getArray()[maskedImage.getMask().getArray() & statsCtrl.getAndMask() == 0] += 1
840 if self.config.removeMaskPlanes:
842 maskedImageList.append(maskedImage)
844 if self.config.doInputMap:
845 visit = exposure.getInfo().getCoaddInputs().visits[0].getId()
846 self.inputMapper.mask_warp_bbox(bbox, visit, mask, statsCtrl.getAndMask())
848 with self.timer(
"stack"):
849 coaddSubregion = afwMath.statisticsStack(maskedImageList, statsFlags, statsCtrl, weightList,
852 coaddExposure.maskedImage.assign(coaddSubregion, bbox)
853 if nImage
is not None:
854 nImage.assign(subNImage, bbox)
857 altMaskList, statsCtrl, nImage=None):
858 """Assemble the coadd using the "online" method.
860 This method takes a running sum of images and weights to save memory.
861 It only works
for MEAN statistics.
866 The target exposure
for the coadd.
867 tempExpRefList : `list`
868 List of data reference to tempExp.
869 imageScalerList : `list`
870 List of image scalers.
874 List of alternate masks to use rather than those stored
with
875 tempExp,
or None. Each element
is dict
with keys = mask plane
876 name to which to add the spans.
878 Statistics control object
for coadd.
879 nImage : `lsst.afw.image.ImageU`, optional
880 Keeps track of exposure count
for each pixel.
882 self.log.debug("Computing online coadd.")
884 coaddExposure.mask.addMaskPlane(
"REJECTED")
885 coaddExposure.mask.addMaskPlane(
"CLIPPED")
886 coaddExposure.mask.addMaskPlane(
"SENSOR_EDGE")
888 thresholdDict = AccumulatorMeanStack.stats_ctrl_to_threshold_dict(statsCtrl)
890 bbox = coaddExposure.maskedImage.getBBox()
892 stacker = AccumulatorMeanStack(
893 coaddExposure.image.array.shape,
894 statsCtrl.getAndMask(),
895 mask_threshold_dict=thresholdDict,
897 no_good_pixels_mask=statsCtrl.getNoGoodPixelsMask(),
898 calc_error_from_input_variance=self.config.calcErrorFromInputVariance,
899 compute_n_image=(nImage
is not None)
902 for tempExpRef, imageScaler, altMask, weight
in zip(tempExpRefList,
906 exposure = tempExpRef.get()
907 maskedImage = exposure.getMaskedImage()
908 mask = maskedImage.getMask()
909 if altMask
is not None:
911 imageScaler.scaleMaskedImage(maskedImage)
912 if self.config.removeMaskPlanes:
915 stacker.add_masked_image(maskedImage, weight=weight)
917 if self.config.doInputMap:
918 visit = exposure.getInfo().getCoaddInputs().visits[0].getId()
919 self.inputMapper.mask_warp_bbox(bbox, visit, mask, statsCtrl.getAndMask())
921 stacker.fill_stacked_masked_image(coaddExposure.maskedImage)
923 if nImage
is not None:
924 nImage.array[:, :] = stacker.n_image
927 """Unset the mask of an image for mask planes specified in the config.
932 The masked image to be modified.
936 InvalidParameterError
937 Raised if no mask plane
with that name was found.
939 mask = maskedImage.getMask()
940 for maskPlane
in self.config.removeMaskPlanes:
942 mask &= ~mask.getPlaneBitMask(maskPlane)
943 except pexExceptions.InvalidParameterError:
944 self.log.debug(
"Unable to remove mask plane %s: no mask plane with that name was found.",
949 """Map certain mask planes of the warps to new planes for the coadd.
951 If a pixel is rejected due to a mask value other than EDGE, NO_DATA,
952 or CLIPPED, set it to REJECTED on the coadd.
953 If a pixel
is rejected due to EDGE, set the coadd pixel to SENSOR_EDGE.
954 If a pixel
is rejected due to CLIPPED, set the coadd pixel to CLIPPED.
959 Statistics control object
for coadd.
963 maskMap : `list` of `tuple` of `int`
964 A list of mappings of mask planes of the warped exposures to
965 mask planes of the coadd.
967 edge = afwImage.Mask.getPlaneBitMask("EDGE")
968 noData = afwImage.Mask.getPlaneBitMask(
"NO_DATA")
969 clipped = afwImage.Mask.getPlaneBitMask(
"CLIPPED")
970 toReject = statsCtrl.getAndMask() & (~noData) & (~edge) & (~clipped)
971 maskMap = [(toReject, afwImage.Mask.getPlaneBitMask(
"REJECTED")),
972 (edge, afwImage.Mask.getPlaneBitMask(
"SENSOR_EDGE")),
977 """Apply in place alt mask formatted as SpanSets to a mask.
983 altMaskSpans : `dict`
984 SpanSet lists to apply. Each element contains the new mask
985 plane name (e.g. "CLIPPED and/or "NO_DATA
") as the key,
986 and list of SpanSets to apply to the mask.
993 if self.config.doUsePsfMatchedPolygons:
994 if (
"NO_DATA" in altMaskSpans)
and (
"NO_DATA" in self.config.badMaskPlanes):
999 for spanSet
in altMaskSpans[
'NO_DATA']:
1000 spanSet.clippedTo(mask.getBBox()).clearMask(mask, self.
getBadPixelMask())
1002 for plane, spanSetList
in altMaskSpans.items():
1003 maskClipValue = mask.addMaskPlane(plane)
1004 for spanSet
in spanSetList:
1005 spanSet.clippedTo(mask.getBBox()).setMask(mask, 2**maskClipValue)
1009 """Shrink coaddInputs' ccds' ValidPolygons in place.
1011 Either modify each ccd's validPolygon in place, or if CoaddInputs
1012 does not have a validPolygon, create one
from its bbox.
1016 coaddInputs : `lsst.afw.image.coaddInputs`
1019 for ccd
in coaddInputs.ccds:
1020 polyOrig = ccd.getValidPolygon()
1021 validPolyBBox = polyOrig.getBBox()
if polyOrig
else ccd.getBBox()
1022 validPolyBBox.grow(-self.config.matchingKernelSize//2)
1024 validPolygon = polyOrig.intersectionSingle(validPolyBBox)
1026 validPolygon = afwGeom.polygon.Polygon(
geom.Box2D(validPolyBBox))
1027 ccd.setValidPolygon(validPolygon)
1030 """Set the bright object masks.
1035 Exposure under consideration.
1037 Table of bright objects to mask.
1038 dataId : `lsst.daf.butler.DataId`, optional
1039 Data identifier dict for patch.
1041 if brightObjectMasks
is None:
1042 self.log.warning(
"Unable to apply bright object mask: none supplied")
1044 self.log.info(
"Applying %d bright object masks to %s", len(brightObjectMasks), dataId)
1045 mask = exposure.getMaskedImage().getMask()
1046 wcs = exposure.getWcs()
1047 plateScale = wcs.getPixelScale().asArcseconds()
1049 for rec
in brightObjectMasks:
1050 center =
geom.PointI(wcs.skyToPixel(rec.getCoord()))
1051 if rec[
"type"] ==
"box":
1052 assert rec[
"angle"] == 0.0, (
"Angle != 0 for mask object %s" % rec[
"id"])
1053 width = rec[
"width"].asArcseconds()/plateScale
1054 height = rec[
"height"].asArcseconds()/plateScale
1057 bbox =
geom.Box2I(center - halfSize, center + halfSize)
1060 geom.PointI(int(center[0] + 0.5*width), int(center[1] + 0.5*height)))
1061 spans = afwGeom.SpanSet(bbox)
1062 elif rec[
"type"] ==
"circle":
1063 radius = int(rec[
"radius"].asArcseconds()/plateScale)
1064 spans = afwGeom.SpanSet.fromShape(radius, offset=center)
1066 self.log.warning(
"Unexpected region type %s at %s", rec[
"type"], center)
1071 """Set INEXACT_PSF mask plane.
1073 If any of the input images isn't represented in the coadd (due to
1074 clipped pixels or chip gaps), the `CoaddPsf` will be inexact. Flag
1080 Coadded exposure
's mask, modified in-place.
1082 mask.addMaskPlane("INEXACT_PSF")
1083 inexactPsf = mask.getPlaneBitMask(
"INEXACT_PSF")
1084 sensorEdge = mask.getPlaneBitMask(
"SENSOR_EDGE")
1085 clipped = mask.getPlaneBitMask(
"CLIPPED")
1086 rejected = mask.getPlaneBitMask(
"REJECTED")
1087 array = mask.getArray()
1088 selected = array & (sensorEdge | clipped | rejected) > 0
1089 array[selected] |= inexactPsf
1092 def _subBBoxIter(bbox, subregionSize):
1093 """Iterate over subregions of a bbox.
1098 Bounding box over which to iterate.
1105 Next sub-bounding box of size ``subregionSize`` or smaller; each ``subBBox``
1106 is contained within ``bbox``, so it may be smaller than ``subregionSize`` at
1107 the edges of ``bbox``, but it will never be empty.
1112 Raised
if any of the following occur:
1113 - The given bbox
is empty.
1114 - The subregionSize
is 0.
1117 raise RuntimeError(
"bbox %s is empty" % (bbox,))
1118 if subregionSize[0] < 1
or subregionSize[1] < 1:
1119 raise RuntimeError(
"subregionSize %s must be nonzero" % (subregionSize,))
1121 for rowShift
in range(0, bbox.getHeight(), subregionSize[1]):
1122 for colShift
in range(0, bbox.getWidth(), subregionSize[0]):
1125 if subBBox.isEmpty():
1126 raise RuntimeError(
"Bug: empty bbox! bbox=%s, subregionSize=%s, "
1127 "colShift=%s, rowShift=%s" %
1128 (bbox, subregionSize, colShift, rowShift))
1132 """Return list of only inputRefs with visitId in goodVisits ordered by goodVisit.
1136 inputs : `list` of `~lsst.pipe.base.connections.DeferredDatasetRef`
1137 List of `lsst.pipe.base.connections.DeferredDatasetRef` with dataId containing visit.
1139 Dictionary
with good visitIds
as the keys. Value ignored.
1143 filteredInputs : `list` of `~lsst.pipe.base.connections.DeferredDatasetRef`
1144 Filtered
and sorted list of inputRefs
with visitId
in goodVisits ordered by goodVisit.
1146 inputWarpDict = {inputRef.ref.dataId['visit']: inputRef
for inputRef
in inputs}
1148 for visit
in goodVisits.keys():
1149 if visit
in inputWarpDict:
1150 filteredInputs.append(inputWarpDict[visit])
1151 return filteredInputs
1155 """Function to count the number of pixels with a specific mask in a
1158 Find the intersection of mask & footprint. Count all pixels in the mask
1159 that are
in the intersection that have bitmask set but do
not have
1160 ignoreMask set. Return the count.
1165 Mask to define intersection region by.
1167 Footprint to define the intersection region by.
1169 Specific mask that we wish to count the number of occurances of.
1170 ignoreMask : `Unknown`
1171 Pixels to
not consider.
1176 Number of pixels
in footprint
with specified mask.
1178 bbox = footprint.getBBox()
1179 bbox.clip(mask.getBBox(afwImage.PARENT))
1180 fp = afwImage.Mask(bbox)
1181 subMask = mask.Factory(mask, bbox, afwImage.PARENT)
1182 footprint.spans.setMask(fp, bitmask)
1183 return numpy.logical_and((subMask.getArray() & fp.getArray()) > 0,
1184 (subMask.getArray() & ignoreMask) == 0).sum()
1188 psfMatchedWarps = pipeBase.connectionTypes.Input(
1189 doc=(
"PSF-Matched Warps are required by CompareWarp regardless of the coadd type requested. "
1190 "Only PSF-Matched Warps make sense for image subtraction. "
1191 "Therefore, they must be an additional declared input."),
1192 name=
"{inputCoaddName}Coadd_psfMatchedWarp",
1193 storageClass=
"ExposureF",
1194 dimensions=(
"tract",
"patch",
"skymap",
"visit"),
1198 templateCoadd = pipeBase.connectionTypes.Output(
1199 doc=(
"Model of the static sky, used to find temporal artifacts. Typically a PSF-Matched, "
1200 "sigma-clipped coadd. Written if and only if assembleStaticSkyModel.doWrite=True"),
1201 name=
"{outputCoaddName}CoaddPsfMatched",
1202 storageClass=
"ExposureF",
1203 dimensions=(
"tract",
"patch",
"skymap",
"band"),
1208 if not config.assembleStaticSkyModel.doWrite:
1209 self.outputs.remove(
"templateCoadd")
1214 pipelineConnections=CompareWarpAssembleCoaddConnections):
1215 assembleStaticSkyModel = pexConfig.ConfigurableField(
1216 target=AssembleCoaddTask,
1217 doc=
"Task to assemble an artifact-free, PSF-matched Coadd to serve as a"
1218 " naive/first-iteration model of the static sky.",
1220 detect = pexConfig.ConfigurableField(
1221 target=SourceDetectionTask,
1222 doc=
"Detect outlier sources on difference between each psfMatched warp and static sky model"
1224 detectTemplate = pexConfig.ConfigurableField(
1225 target=SourceDetectionTask,
1226 doc=
"Detect sources on static sky model. Only used if doPreserveContainedBySource is True"
1228 maskStreaks = pexConfig.ConfigurableField(
1229 target=MaskStreaksTask,
1230 doc=
"Detect streaks on difference between each psfMatched warp and static sky model. Only used if "
1231 "doFilterMorphological is True. Adds a mask plane to an exposure, with the mask plane name set by"
1234 streakMaskName = pexConfig.Field(
1237 doc=
"Name of mask bit used for streaks"
1239 maxNumEpochs = pexConfig.Field(
1240 doc=
"Charactistic maximum local number of epochs/visits in which an artifact candidate can appear "
1241 "and still be masked. The effective maxNumEpochs is a broken linear function of local "
1242 "number of epochs (N): min(maxFractionEpochsLow*N, maxNumEpochs + maxFractionEpochsHigh*N). "
1243 "For each footprint detected on the image difference between the psfMatched warp and static sky "
1244 "model, if a significant fraction of pixels (defined by spatialThreshold) are residuals in more "
1245 "than the computed effective maxNumEpochs, the artifact candidate is deemed persistant rather "
1246 "than transient and not masked.",
1250 maxFractionEpochsLow = pexConfig.RangeField(
1251 doc=
"Fraction of local number of epochs (N) to use as effective maxNumEpochs for low N. "
1252 "Effective maxNumEpochs = "
1253 "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)",
1258 maxFractionEpochsHigh = pexConfig.RangeField(
1259 doc=
"Fraction of local number of epochs (N) to use as effective maxNumEpochs for high N. "
1260 "Effective maxNumEpochs = "
1261 "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)",
1266 spatialThreshold = pexConfig.RangeField(
1267 doc=
"Unitless fraction of pixels defining how much of the outlier region has to meet the "
1268 "temporal criteria. If 0, clip all. If 1, clip none.",
1272 inclusiveMin=
True, inclusiveMax=
True
1274 doScaleWarpVariance = pexConfig.Field(
1275 doc=
"Rescale Warp variance plane using empirical noise?",
1279 scaleWarpVariance = pexConfig.ConfigurableField(
1280 target=ScaleVarianceTask,
1281 doc=
"Rescale variance on warps",
1283 doPreserveContainedBySource = pexConfig.Field(
1284 doc=
"Rescue artifacts from clipping that completely lie within a footprint detected"
1285 "on the PsfMatched Template Coadd. Replicates a behavior of SafeClip.",
1289 doPrefilterArtifacts = pexConfig.Field(
1290 doc=
"Ignore artifact candidates that are mostly covered by the bad pixel mask, "
1291 "because they will be excluded anyway. This prevents them from contributing "
1292 "to the outlier epoch count image and potentially being labeled as persistant."
1293 "'Mostly' is defined by the config 'prefilterArtifactsRatio'.",
1297 prefilterArtifactsMaskPlanes = pexConfig.ListField(
1298 doc=
"Prefilter artifact candidates that are mostly covered by these bad mask planes.",
1300 default=(
'NO_DATA',
'BAD',
'SAT',
'SUSPECT'),
1302 prefilterArtifactsRatio = pexConfig.Field(
1303 doc=
"Prefilter artifact candidates with less than this fraction overlapping good pixels",
1307 doFilterMorphological = pexConfig.Field(
1308 doc=
"Filter artifact candidates based on morphological criteria, i.g. those that appear to "
1313 growStreakFp = pexConfig.Field(
1314 doc=
"Grow streak footprints by this number multiplied by the PSF width",
1320 AssembleCoaddConfig.setDefaults(self)
1326 if "EDGE" in self.badMaskPlanes:
1327 self.badMaskPlanes.remove(
'EDGE')
1328 self.removeMaskPlanes.append(
'EDGE')
1337 self.
detect.doTempLocalBackground =
False
1338 self.
detect.reEstimateBackground =
False
1339 self.
detect.returnOriginalFootprints =
False
1340 self.
detect.thresholdPolarity =
"both"
1341 self.
detect.thresholdValue = 5
1342 self.
detect.minPixels = 4
1343 self.
detect.isotropicGrow =
True
1344 self.
detect.thresholdType =
"pixel_stdev"
1345 self.
detect.nSigmaToGrow = 0.4
1356 raise ValueError(
"No dataset type exists for a PSF-Matched Template N Image."
1357 "Please set assembleStaticSkyModel.doNImage=False")
1360 raise ValueError(
"warpType (%s) == assembleStaticSkyModel.warpType (%s) and will compete for "
1361 "the same dataset name. Please set assembleStaticSkyModel.doWrite to False "
1362 "or warpType to 'direct'. assembleStaticSkyModel.warpType should ways be "
1367 """Assemble a compareWarp coadded image from a set of warps
1368 by masking artifacts detected by comparing PSF-matched warps.
1370 In ``AssembleCoaddTask``, we compute the coadd as an clipped mean (i.e.,
1371 we clip outliers). The problem
with doing this
is that when computing the
1372 coadd PSF at a given location, individual visit PSFs
from visits
with
1373 outlier pixels contribute to the coadd PSF
and cannot be treated correctly.
1374 In this task, we correct
for this behavior by creating a new badMaskPlane
1375 'CLIPPED' which marks pixels
in the individual warps suspected to contain
1376 an artifact. We populate this plane on the input warps by comparing
1377 PSF-matched warps
with a PSF-matched median coadd which serves
as a
1378 model of the static sky. Any group of pixels that deviates
from the
1379 PSF-matched template coadd by more than config.detect.threshold sigma,
1380 is an artifact candidate. The candidates are then filtered to remove
1381 variable sources
and sources that are difficult to subtract such
as
1382 bright stars. This filter
is configured using the config parameters
1383 ``temporalThreshold``
and ``spatialThreshold``. The temporalThreshold
is
1384 the maximum fraction of epochs that the deviation can appear
in and still
1385 be considered an artifact. The spatialThreshold
is the maximum fraction of
1386 pixels
in the footprint of the deviation that appear
in other epochs
1387 (where other epochs
is defined by the temporalThreshold). If the deviant
1388 region meets this criteria of having a significant percentage of pixels
1389 that deviate
in only a few epochs, these pixels have the
'CLIPPED' bit
1390 set
in the mask. These regions will
not contribute to the final coadd.
1391 Furthermore, any routine to determine the coadd PSF can now be cognizant
1392 of clipped regions. Note that the algorithm implemented by this task
is
1393 preliminary
and works correctly
for HSC data. Parameter modifications
and
1394 or considerable redesigning of the algorithm
is likley required
for other
1397 ``CompareWarpAssembleCoaddTask`` sub-classes
1398 ``AssembleCoaddTask``
and instantiates ``AssembleCoaddTask``
1399 as a subtask to generate the TemplateCoadd (the model of the static sky).
1404 This task supports the following debug variables:
1406 If
True then save the Epoch Count Image
as a fits file
in the `figPath`
1408 Path to save the debug fits images
and figures
1411 ConfigClass = CompareWarpAssembleCoaddConfig
1412 _DefaultName = "compareWarpAssembleCoadd"
1415 AssembleCoaddTask.__init__(self, *args, **kwargs)
1416 self.makeSubtask(
"assembleStaticSkyModel")
1417 detectionSchema = afwTable.SourceTable.makeMinimalSchema()
1418 self.makeSubtask(
"detect", schema=detectionSchema)
1419 if self.config.doPreserveContainedBySource:
1420 self.makeSubtask(
"detectTemplate", schema=afwTable.SourceTable.makeMinimalSchema())
1421 if self.config.doScaleWarpVariance:
1422 self.makeSubtask(
"scaleWarpVariance")
1423 if self.config.doFilterMorphological:
1424 self.makeSubtask(
"maskStreaks")
1426 @utils.inheritDoc(AssembleCoaddTask)
1427 def _makeSupplementaryData(self, butlerQC, inputRefs, outputRefs):
1428 """Generate a templateCoadd to use as a naive model of static sky to
1429 subtract from PSF-Matched warps.
1433 result : `~lsst.pipe.base.Struct`
1434 Results
as a struct
with attributes:
1439 Keeps track of exposure count
for each pixel (`lsst.afw.image.ImageU`).
1444 Raised
if ``templateCoadd``
is `
None`.
1447 staticSkyModelInputRefs = copy.deepcopy(inputRefs)
1448 staticSkyModelInputRefs.inputWarps = inputRefs.psfMatchedWarps
1452 staticSkyModelOutputRefs = copy.deepcopy(outputRefs)
1453 if self.config.assembleStaticSkyModel.doWrite:
1454 staticSkyModelOutputRefs.coaddExposure = staticSkyModelOutputRefs.templateCoadd
1457 del outputRefs.templateCoadd
1458 del staticSkyModelOutputRefs.templateCoadd
1461 if 'nImage' in staticSkyModelOutputRefs.keys():
1462 del staticSkyModelOutputRefs.nImage
1464 templateCoadd = self.assembleStaticSkyModel.
runQuantum(butlerQC, staticSkyModelInputRefs,
1465 staticSkyModelOutputRefs)
1466 if templateCoadd
is None:
1469 return pipeBase.Struct(templateCoadd=templateCoadd.coaddExposure,
1470 nImage=templateCoadd.nImage,
1471 warpRefList=templateCoadd.warpRefList,
1472 imageScalerList=templateCoadd.imageScalerList,
1473 weightList=templateCoadd.weightList)
1475 def _noTemplateMessage(self, warpType):
1476 warpName = (warpType[0].upper() + warpType[1:])
1477 message =
"""No %(warpName)s warps were found to build the template coadd which is
1478 required to run CompareWarpAssembleCoaddTask. To continue assembling this type of coadd,
1479 first either rerun makeCoaddTempExp
with config.make%(warpName)s=
True or
1480 coaddDriver
with config.makeCoadTempExp.make%(warpName)s=
True, before assembleCoadd.
1482 Alternatively, to use another algorithm
with existing warps, retarget the CoaddDriverConfig to
1483 another algorithm like:
1486 config.assemble.retarget(SafeClipAssembleCoaddTask)
1487 """ % {"warpName": warpName}
1490 @utils.inheritDoc(AssembleCoaddTask)
1492 def run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
1494 """Assemble the coadd.
1496 Find artifacts and apply them to the warps
' masks creating a list of
1497 alternative masks with a new
"CLIPPED" plane
and updated
"NO_DATA"
1498 plane. Then
pass these alternative masks to the base
class's ``run``
1504 dataIds = [ref.dataId
for ref
in tempExpRefList]
1505 psfMatchedDataIds = [ref.dataId
for ref
in supplementaryData.warpRefList]
1507 if dataIds != psfMatchedDataIds:
1508 self.log.info(
"Reordering and or/padding PSF-matched visit input list")
1509 supplementaryData.warpRefList = reorderAndPadList(supplementaryData.warpRefList,
1510 psfMatchedDataIds, dataIds)
1511 supplementaryData.imageScalerList = reorderAndPadList(supplementaryData.imageScalerList,
1512 psfMatchedDataIds, dataIds)
1515 spanSetMaskList = self.
findArtifacts(supplementaryData.templateCoadd,
1516 supplementaryData.warpRefList,
1517 supplementaryData.imageScalerList)
1519 badMaskPlanes = self.config.badMaskPlanes[:]
1520 badMaskPlanes.append(
"CLIPPED")
1521 badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes)
1523 result = AssembleCoaddTask.run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
1524 spanSetMaskList, mask=badPixelMask)
1528 self.
applyAltEdgeMask(result.coaddExposure.maskedImage.mask, spanSetMaskList)
1532 """Propagate alt EDGE mask to SENSOR_EDGE AND INEXACT_PSF planes.
1538 altMaskList : `list` of `dict`
1539 List of Dicts containing ``spanSet`` lists.
1540 Each element contains the new mask plane name (e.g. "CLIPPED
1541 and/
or "NO_DATA")
as the key,
and list of ``SpanSets`` to apply to
1544 maskValue = mask.getPlaneBitMask(["SENSOR_EDGE",
"INEXACT_PSF"])
1545 for visitMask
in altMaskList:
1546 if "EDGE" in visitMask:
1547 for spanSet
in visitMask[
'EDGE']:
1548 spanSet.clippedTo(mask.getBBox()).setMask(mask, maskValue)
1553 Loop through warps twice. The first loop builds a map with the count
1554 of how many epochs each pixel deviates
from the templateCoadd by more
1555 than ``config.chiThreshold`` sigma. The second loop takes each
1556 difference image
and filters the artifacts detected
in each using
1557 count map to filter out variable sources
and sources that are
1558 difficult to subtract cleanly.
1563 Exposure to serve
as model of static sky.
1564 tempExpRefList : `list`
1565 List of data references to warps.
1566 imageScalerList : `list`
1567 List of image scalers.
1571 altMasks : `list` of `dict`
1572 List of dicts containing information about CLIPPED
1573 (i.e., artifacts), NO_DATA,
and EDGE pixels.
1575 self.log.debug("Generating Count Image, and mask lists.")
1576 coaddBBox = templateCoadd.getBBox()
1577 slateIm = afwImage.ImageU(coaddBBox)
1578 epochCountImage = afwImage.ImageU(coaddBBox)
1579 nImage = afwImage.ImageU(coaddBBox)
1580 spanSetArtifactList = []
1581 spanSetNoDataMaskList = []
1582 spanSetEdgeList = []
1583 spanSetBadMorphoList = []
1587 templateCoadd.mask.clearAllMaskPlanes()
1589 if self.config.doPreserveContainedBySource:
1590 templateFootprints = self.detectTemplate.detectFootprints(templateCoadd)
1592 templateFootprints =
None
1594 for warpRef, imageScaler
in zip(tempExpRefList, imageScalerList):
1596 if warpDiffExp
is not None:
1598 nImage.array += (numpy.isfinite(warpDiffExp.image.array)
1599 * ((warpDiffExp.mask.array & badPixelMask) == 0)).astype(numpy.uint16)
1600 fpSet = self.detect.detectFootprints(warpDiffExp, doSmooth=
False, clearMask=
True)
1601 fpSet.positive.merge(fpSet.negative)
1602 footprints = fpSet.positive
1604 spanSetList = [footprint.spans
for footprint
in footprints.getFootprints()]
1607 if self.config.doPrefilterArtifacts:
1611 self.detect.clearMask(warpDiffExp.mask)
1612 for spans
in spanSetList:
1613 spans.setImage(slateIm, 1, doClip=
True)
1614 spans.setMask(warpDiffExp.mask, warpDiffExp.mask.getPlaneBitMask(
"DETECTED"))
1615 epochCountImage += slateIm
1617 if self.config.doFilterMorphological:
1618 maskName = self.config.streakMaskName
1619 _ = self.maskStreaks.run(warpDiffExp)
1620 streakMask = warpDiffExp.mask
1621 spanSetStreak = afwGeom.SpanSet.fromMask(streakMask,
1622 streakMask.getPlaneBitMask(maskName)).split()
1624 psf = warpDiffExp.getPsf()
1625 for s, sset
in enumerate(spanSetStreak):
1626 psfShape = psf.computeShape(sset.computeCentroid())
1627 dilation = self.config.growStreakFp * psfShape.getDeterminantRadius()
1628 sset_dilated = sset.dilated(int(dilation))
1629 spanSetStreak[s] = sset_dilated
1635 nans = numpy.where(numpy.isnan(warpDiffExp.maskedImage.image.array), 1, 0)
1636 nansMask = afwImage.makeMaskFromArray(nans.astype(afwImage.MaskPixel))
1637 nansMask.setXY0(warpDiffExp.getXY0())
1638 edgeMask = warpDiffExp.mask
1639 spanSetEdgeMask = afwGeom.SpanSet.fromMask(edgeMask,
1640 edgeMask.getPlaneBitMask(
"EDGE")).split()
1644 nansMask = afwImage.MaskX(coaddBBox, 1)
1646 spanSetEdgeMask = []
1649 spanSetNoDataMask = afwGeom.SpanSet.fromMask(nansMask).split()
1651 spanSetNoDataMaskList.append(spanSetNoDataMask)
1652 spanSetArtifactList.append(spanSetList)
1653 spanSetEdgeList.append(spanSetEdgeMask)
1654 if self.config.doFilterMorphological:
1655 spanSetBadMorphoList.append(spanSetStreak)
1658 path = self._dataRef2DebugPath(
"epochCountIm", tempExpRefList[0], coaddLevel=
True)
1659 epochCountImage.writeFits(path)
1661 for i, spanSetList
in enumerate(spanSetArtifactList):
1663 filteredSpanSetList = self.
filterArtifacts(spanSetList, epochCountImage, nImage,
1665 spanSetArtifactList[i] = filteredSpanSetList
1666 if self.config.doFilterMorphological:
1667 spanSetArtifactList[i] += spanSetBadMorphoList[i]
1670 for artifacts, noData, edge
in zip(spanSetArtifactList, spanSetNoDataMaskList, spanSetEdgeList):
1671 altMasks.append({
'CLIPPED': artifacts,
1677 """Remove artifact candidates covered by bad mask plane.
1679 Any future editing of the candidate list that does not depend on
1680 temporal information should go
in this method.
1685 List of SpanSets representing artifact candidates.
1687 Exposure containing mask planes used to prefilter.
1692 List of SpanSets
with artifacts.
1694 badPixelMask = exp.mask.getPlaneBitMask(self.config.prefilterArtifactsMaskPlanes)
1695 goodArr = (exp.mask.array & badPixelMask) == 0
1696 returnSpanSetList = []
1697 bbox = exp.getBBox()
1698 x0, y0 = exp.getXY0()
1699 for i, span
in enumerate(spanSetList):
1700 y, x = span.clippedTo(bbox).indices()
1701 yIndexLocal = numpy.array(y) - y0
1702 xIndexLocal = numpy.array(x) - x0
1703 goodRatio = numpy.count_nonzero(goodArr[yIndexLocal, xIndexLocal])/span.getArea()
1704 if goodRatio > self.config.prefilterArtifactsRatio:
1705 returnSpanSetList.append(span)
1706 return returnSpanSetList
1708 def filterArtifacts(self, spanSetList, epochCountImage, nImage, footprintsToExclude=None):
1709 """Filter artifact candidates.
1714 List of SpanSets representing artifact candidates.
1716 Image of accumulated number of warpDiff detections.
1717 nImage : `lsst.afw.image.ImageU`
1718 Image of the accumulated number of total epochs contributing.
1722 maskSpanSetList : `list`
1723 List of SpanSets with artifacts.
1725 maskSpanSetList = []
1726 x0, y0 = epochCountImage.getXY0()
1727 for i, span
in enumerate(spanSetList):
1728 y, x = span.indices()
1729 yIdxLocal = [y1 - y0
for y1
in y]
1730 xIdxLocal = [x1 - x0
for x1
in x]
1731 outlierN = epochCountImage.array[yIdxLocal, xIdxLocal]
1732 totalN = nImage.array[yIdxLocal, xIdxLocal]
1735 effMaxNumEpochsHighN = (self.config.maxNumEpochs
1736 + self.config.maxFractionEpochsHigh*numpy.mean(totalN))
1737 effMaxNumEpochsLowN = self.config.maxFractionEpochsLow * numpy.mean(totalN)
1738 effectiveMaxNumEpochs = int(min(effMaxNumEpochsLowN, effMaxNumEpochsHighN))
1739 nPixelsBelowThreshold = numpy.count_nonzero((outlierN > 0)
1740 & (outlierN <= effectiveMaxNumEpochs))
1741 percentBelowThreshold = nPixelsBelowThreshold / len(outlierN)
1742 if percentBelowThreshold > self.config.spatialThreshold:
1743 maskSpanSetList.append(span)
1745 if self.config.doPreserveContainedBySource
and footprintsToExclude
is not None:
1747 filteredMaskSpanSetList = []
1748 for span
in maskSpanSetList:
1750 for footprint
in footprintsToExclude.positive.getFootprints():
1751 if footprint.spans.contains(span):
1755 filteredMaskSpanSetList.append(span)
1756 maskSpanSetList = filteredMaskSpanSetList
1758 return maskSpanSetList
1760 def _readAndComputeWarpDiff(self, warpRef, imageScaler, templateCoadd):
1761 """Fetch a warp from the butler and return a warpDiff.
1765 warpRef : `lsst.daf.butler.DeferredDatasetHandle`
1766 Handle for the warp.
1768 An image scaler object.
1770 Exposure to be substracted
from the scaled warp.
1775 Exposure of the image difference between the warp
and template.
1782 warp = warpRef.get()
1784 imageScaler.scaleMaskedImage(warp.getMaskedImage())
1785 mi = warp.getMaskedImage()
1786 if self.config.doScaleWarpVariance:
1788 self.scaleWarpVariance.run(mi)
1789 except Exception
as exc:
1790 self.log.warning(
"Unable to rescale variance of warp (%s); leaving it as-is", exc)
1791 mi -= templateCoadd.getMaskedImage()
def shrinkValidPolygons(self, coaddInputs)
def setRejectedMaskMapping(statsCtrl)
def assembleMetadata(self, coaddExposure, tempExpRefList, weightList)
def assembleOnlineMeanCoadd(self, coaddExposure, tempExpRefList, imageScalerList, weightList, altMaskList, statsCtrl, nImage=None)
def runQuantum(self, butlerQC, inputRefs, outputRefs)
def _makeSupplementaryData(self, butlerQC, inputRefs, outputRefs)
def processResults(self, coaddExposure, brightObjectMasks=None, dataId=None)
def prepareInputs(self, refList)
def setBrightObjectMasks(self, exposure, brightObjectMasks, dataId=None)
def removeMaskPlanes(self, maskedImage)
def __init__(self, *args, **kwargs)
def _subBBoxIter(bbox, subregionSize)
def assembleSubregion(self, coaddExposure, bbox, tempExpRefList, imageScalerList, weightList, altMaskList, statsFlags, statsCtrl, nImage=None)
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, altMaskList=None, mask=None, supplementaryData=None)
def applyAltMaskPlanes(self, mask, altMaskSpans)
def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs)
def filterWarps(self, inputs, goodVisits)
def prepareStats(self, mask=None)
def setInexactPsf(self, mask)
def __init__(self, *config=None)
def prefilterArtifacts(self, spanSetList, exp)
def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList)
def _readAndComputeWarpDiff(self, warpRef, imageScaler, templateCoadd)
def applyAltEdgeMask(self, mask, altMaskList)
def _noTemplateMessage(self, warpType)
def filterArtifacts(self, spanSetList, epochCountImage, nImage, footprintsToExclude=None)
def __init__(self, *args, **kwargs)
def getTempExpDatasetName(self, warpType="direct")
def getBadPixelMask(self)
def countMaskFromFootprint(mask, footprint, bitmask, ignoreMask)