36 from .coaddBase
import CoaddBaseTask, SelectDataIdContainer
37 from .interpImage
import InterpImageTask
38 from .scaleZeroPoint
import ScaleZeroPointTask
39 from .coaddHelpers
import groupPatchExposures, getGroupDataRef
40 from .scaleVariance
import ScaleVarianceTask
43 __all__ = [
"AssembleCoaddTask",
"AssembleCoaddConfig",
"SafeClipAssembleCoaddTask",
44 "SafeClipAssembleCoaddConfig",
"CompareWarpAssembleCoaddTask",
"CompareWarpAssembleCoaddConfig"]
48 """Configuration parameters for the `AssembleCoaddTask`. 52 The `doMaskBrightObjects` and `brightObjectMaskName` configuration options 53 only set the bitplane config.brightObjectMaskName. To make this useful you 54 *must* also configure the flags.pixel algorithm, for example by adding 58 config.measurement.plugins["base_PixelFlags"].masksFpCenter.append("BRIGHT_OBJECT") 59 config.measurement.plugins["base_PixelFlags"].masksFpAnywhere.append("BRIGHT_OBJECT") 61 to your measureCoaddSources.py and forcedPhotCoadd.py config overrides. 63 warpType = pexConfig.Field(
64 doc=
"Warp name: one of 'direct' or 'psfMatched'",
68 subregionSize = pexConfig.ListField(
70 doc=
"Width, height of stack subregion size; " 71 "make small enough that a full stack of images will fit into memory at once.",
75 statistic = pexConfig.Field(
77 doc=
"Main stacking statistic for aggregating over the epochs.",
80 doSigmaClip = pexConfig.Field(
82 doc=
"Perform sigma clipped outlier rejection with MEANCLIP statistic? (DEPRECATED)",
85 sigmaClip = pexConfig.Field(
87 doc=
"Sigma for outlier rejection; ignored if non-clipping statistic selected.",
90 clipIter = pexConfig.Field(
92 doc=
"Number of iterations of outlier rejection; ignored if non-clipping statistic selected.",
95 calcErrorFromInputVariance = pexConfig.Field(
97 doc=
"Calculate coadd variance from input variance by stacking statistic." 98 "Passed to StatisticsControl.setCalcErrorFromInputVariance()",
101 scaleZeroPoint = pexConfig.ConfigurableField(
102 target=ScaleZeroPointTask,
103 doc=
"Task to adjust the photometric zero point of the coadd temp exposures",
105 doInterp = pexConfig.Field(
106 doc=
"Interpolate over NaN pixels? Also extrapolate, if necessary, but the results are ugly.",
110 interpImage = pexConfig.ConfigurableField(
111 target=InterpImageTask,
112 doc=
"Task to interpolate (and extrapolate) over NaN pixels",
114 doWrite = pexConfig.Field(
115 doc=
"Persist coadd?",
119 doNImage = pexConfig.Field(
120 doc=
"Create image of number of contributing exposures for each pixel",
124 doUsePsfMatchedPolygons = pexConfig.Field(
125 doc=
"Use ValidPolygons from shrunk Psf-Matched Calexps? Should be set to True by CompareWarp only.",
129 maskPropagationThresholds = pexConfig.DictField(
132 doc=(
"Threshold (in fractional weight) of rejection at which we propagate a mask plane to " 133 "the coadd; that is, we set the mask bit on the coadd if the fraction the rejected frames " 134 "would have contributed exceeds this value."),
135 default={
"SAT": 0.1},
137 removeMaskPlanes = pexConfig.ListField(dtype=str, default=[
"NOT_DEBLENDED"],
138 doc=
"Mask planes to remove before coadding")
139 doMaskBrightObjects = pexConfig.Field(dtype=bool, default=
False,
140 doc=
"Set mask and flag bits for bright objects?")
141 brightObjectMaskName = pexConfig.Field(dtype=str, default=
"BRIGHT_OBJECT",
142 doc=
"Name of mask bit used for bright objects")
143 coaddPsf = pexConfig.ConfigField(
144 doc=
"Configuration for CoaddPsf",
145 dtype=measAlg.CoaddPsfConfig,
147 doAttachTransmissionCurve = pexConfig.Field(
148 dtype=bool, default=
False, optional=
False,
149 doc=(
"Attach a piecewise TransmissionCurve for the coadd? " 150 "(requires all input Exposures to have TransmissionCurves).")
154 CoaddBaseTask.ConfigClass.setDefaults(self)
158 CoaddBaseTask.ConfigClass.validate(self)
162 log.warn(
"Config doPsfMatch deprecated. Setting warpType='psfMatched'")
165 log.warn(
'doSigmaClip deprecated. To replicate behavior, setting statistic to "MEANCLIP"')
167 if self.
doInterp and self.
statistic not in [
'MEAN',
'MEDIAN',
'MEANCLIP',
'VARIANCE',
'VARIANCECLIP']:
168 raise ValueError(
"Must set doInterp=False for statistic=%s, which does not " 169 "compute and set a non-zero coadd variance estimate." % (self.
statistic))
171 unstackableStats = [
'NOTHING',
'ERROR',
'ORMASK']
172 if not hasattr(afwMath.Property, self.
statistic)
or self.
statistic in unstackableStats:
173 stackableStats = [str(k)
for k
in afwMath.Property.__members__.keys()
174 if str(k)
not in unstackableStats]
175 raise ValueError(
"statistic %s is not allowed. Please choose one of %s." 180 """Assemble a coadded image from a set of warps (coadded temporary exposures). 182 We want to assemble a coadded image from a set of Warps (also called 183 coadded temporary exposures or ``coaddTempExps``). 184 Each input Warp covers a patch on the sky and corresponds to a single 185 run/visit/exposure of the covered patch. We provide the task with a list 186 of Warps (``selectDataList``) from which it selects Warps that cover the 187 specified patch (pointed at by ``dataRef``). 188 Each Warp that goes into a coadd will typically have an independent 189 photometric zero-point. Therefore, we must scale each Warp to set it to 190 a common photometric zeropoint. WarpType may be one of 'direct' or 191 'psfMatched', and the boolean configs `config.makeDirect` and 192 `config.makePsfMatched` set which of the warp types will be coadded. 193 The coadd is computed as a mean with optional outlier rejection. 194 Criteria for outlier rejection are set in `AssembleCoaddConfig`. 195 Finally, Warps can have bad 'NaN' pixels which received no input from the 196 source calExps. We interpolate over these bad (NaN) pixels. 198 `AssembleCoaddTask` uses several sub-tasks. These are 200 - `ScaleZeroPointTask` 201 - create and use an ``imageScaler`` object to scale the photometric zeropoint for each Warp 203 - interpolate across bad pixels (NaN) in the final coadd 205 You can retarget these subtasks if you wish. 209 The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a 210 flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``; see 211 `baseDebug` for more about ``debug.py`` files. `AssembleCoaddTask` has 212 no debug variables of its own. Some of the subtasks may support debug 213 variables. See the documentation for the subtasks for further information. 217 `AssembleCoaddTask` assembles a set of warped images into a coadded image. 218 The `AssembleCoaddTask` can be invoked by running ``assembleCoadd.py`` 219 with the flag '--legacyCoadd'. Usage of assembleCoadd.py expects two 220 inputs: a data reference to the tract patch and filter to be coadded, and 221 a list of Warps to attempt to coadd. These are specified using ``--id`` and 222 ``--selectId``, respectively: 226 --id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]] 227 --selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]] 229 Only the Warps that cover the specified tract and patch will be coadded. 230 A list of the available optional arguments can be obtained by calling 231 ``assembleCoadd.py`` with the ``--help`` command line argument: 235 assembleCoadd.py --help 237 To demonstrate usage of the `AssembleCoaddTask` in the larger context of 238 multi-band processing, we will generate the HSC-I & -R band coadds from 239 HSC engineering test data provided in the ``ci_hsc`` package. To begin, 240 assuming that the lsst stack has been already set up, we must set up the 241 obs_subaru and ``ci_hsc`` packages. This defines the environment variable 242 ``$CI_HSC_DIR`` and points at the location of the package. The raw HSC 243 data live in the ``$CI_HSC_DIR/raw directory``. To begin assembling the 244 coadds, we must first 247 - process the individual ccds in $CI_HSC_RAW to produce calibrated exposures 249 - create a skymap that covers the area of the sky present in the raw exposures 251 - warp the individual calibrated exposures to the tangent plane of the coadd 253 We can perform all of these steps by running 257 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988 259 This will produce warped exposures for each visit. To coadd the warped 260 data, we call assembleCoadd.py as follows: 264 assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \ 265 --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \ 266 --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \ 267 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \ 268 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \ 269 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \ 270 --selectId visit=903988 ccd=24 272 that will process the HSC-I band data. The results are written in 273 ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``. 275 You may also choose to run: 279 scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346 280 assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R \ 281 --selectId visit=903334 ccd=16 --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 \ 282 --selectId visit=903334 ccd=100 --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 \ 283 --selectId visit=903338 ccd=18 --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 \ 284 --selectId visit=903342 ccd=10 --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 \ 285 --selectId visit=903344 ccd=5 --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 \ 286 --selectId visit=903346 ccd=6 --selectId visit=903346 ccd=12 288 to generate the coadd for the HSC-R band if you are interested in 289 following multiBand Coadd processing as discussed in `pipeTasks_multiBand` 290 (but note that normally, one would use the `SafeClipAssembleCoaddTask` 291 rather than `AssembleCoaddTask` to make the coadd. 293 ConfigClass = AssembleCoaddConfig
294 _DefaultName =
"assembleCoadd" 297 CoaddBaseTask.__init__(self, *args, **kwargs)
298 self.makeSubtask(
"interpImage")
299 self.makeSubtask(
"scaleZeroPoint")
301 if self.config.doMaskBrightObjects:
302 mask = afwImage.Mask()
305 except pexExceptions.LsstCppException:
306 raise RuntimeError(
"Unable to define mask plane for bright objects; planes used are %s" %
307 mask.getMaskPlaneDict().keys())
313 def run(self, dataRef, selectDataList=[]):
314 """Assemble a coadd from a set of Warps. 316 Coadd a set of Warps. Compute weights to be applied to each Warp and 317 find scalings to match the photometric zeropoint to a reference Warp. 318 Assemble the Warps using `assemble`. Interpolate over NaNs and 319 optionally write the coadd to disk. Return the coadded exposure. 323 dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef` 324 Data reference defining the patch for coaddition and the 325 reference Warp (if ``config.autoReference=False``). 326 Used to access the following data products: 327 - ``self.config.coaddName + "Coadd_skyMap"`` 328 - ``self.config.coaddName + "Coadd_ + <warpType> + "Warp"`` (optionally) 329 - ``self.config.coaddName + "Coadd"`` 330 selectDataList : `list` 331 List of data references to Warps. Data to be coadded will be 332 selected from this list based on overlap with the patch defined 337 retStruct : `lsst.pipe.base.Struct` 338 Result struct with components: 340 - ``coaddExposure``: coadded exposure (``Exposure``). 341 - ``nImage``: exposure count image (``Image``). 344 calExpRefList = self.
selectExposures(dataRef, skyInfo, selectDataList=selectDataList)
345 if len(calExpRefList) == 0:
346 self.log.warn(
"No exposures to coadd")
348 self.log.info(
"Coadding %d exposures", len(calExpRefList))
352 self.log.info(
"Found %d %s", len(inputData.tempExpRefList),
354 if len(inputData.tempExpRefList) == 0:
355 self.log.warn(
"No coadd temporary exposures found")
360 retStruct = self.
assemble(skyInfo, inputData.tempExpRefList, inputData.imageScalerList,
361 inputData.weightList, supplementaryData=supplementaryData)
364 if self.config.doWrite:
367 if self.config.doNImage
and retStruct.nImage
is not None:
373 """Interpolate over missing data and mask bright stars. 377 coaddExposure : `lsst.afw.image.Exposure` 378 The coadded exposure to process. 379 dataRef : `lsst.daf.persistence.ButlerDataRef` 380 Butler data reference for supplementary data. 382 if self.config.doInterp:
383 self.interpImage.
run(coaddExposure.getMaskedImage(), planeName=
"NO_DATA")
385 varArray = coaddExposure.variance.array
386 with numpy.errstate(invalid=
"ignore"):
387 varArray[:] = numpy.where(varArray > 0, varArray, numpy.inf)
389 if self.config.doMaskBrightObjects:
394 """Make additional inputs to assemble() specific to subclasses. 396 Available to be implemented by subclasses only if they need the 397 coadd dataRef for performing preliminary processing before 398 assembling the coadd. 402 dataRef : `lsst.daf.persistence.ButlerDataRef` 403 Butler data reference for supplementary data. 404 selectDataList : `list` 405 List of data references to Warps. 410 """Generate list data references corresponding to warped exposures 411 that lie within the patch to be coadded. 416 Data reference for patch. 417 calExpRefList : `list` 418 List of data references for input calexps. 422 tempExpRefList : `list` 423 List of Warp/CoaddTempExp data references. 425 butler = patchRef.getButler()
426 groupData =
groupPatchExposures(patchRef, calExpRefList, self.getCoaddDatasetName(self.warpType),
427 self.getTempExpDatasetName(self.warpType))
428 tempExpRefList = [
getGroupDataRef(butler, self.getTempExpDatasetName(self.warpType),
429 g, groupData.keys)
for 430 g
in groupData.groups.keys()]
431 return tempExpRefList
434 """Prepare the input warps for coaddition by measuring the weight for 435 each warp and the scaling for the photometric zero point. 437 Each Warp has its own photometric zeropoint and background variance. 438 Before coadding these Warps together, compute a scale factor to 439 normalize the photometric zeropoint and compute the weight for each Warp. 444 List of data references to tempExp 448 result : `lsst.pipe.base.Struct` 449 Result struct with components: 451 - ``tempExprefList``: `list` of data references to tempExp. 452 - ``weightList``: `list` of weightings. 453 - ``imageScalerList``: `list` of image scalers. 455 statsCtrl = afwMath.StatisticsControl()
456 statsCtrl.setNumSigmaClip(self.config.sigmaClip)
457 statsCtrl.setNumIter(self.config.clipIter)
459 statsCtrl.setNanSafe(
True)
467 for tempExpRef
in refList:
468 if not tempExpRef.datasetExists(tempExpName):
469 self.log.warn(
"Could not find %s %s; skipping it", tempExpName, tempExpRef.dataId)
472 tempExp = tempExpRef.get(tempExpName, immediate=
True)
473 maskedImage = tempExp.getMaskedImage()
474 imageScaler = self.scaleZeroPoint.computeImageScaler(
479 imageScaler.scaleMaskedImage(maskedImage)
480 except Exception
as e:
481 self.log.warn(
"Scaling failed for %s (skipping it): %s", tempExpRef.dataId, e)
483 statObj = afwMath.makeStatistics(maskedImage.getVariance(), maskedImage.getMask(),
484 afwMath.MEANCLIP, statsCtrl)
485 meanVar, meanVarErr = statObj.getResult(afwMath.MEANCLIP)
486 weight = 1.0 / float(meanVar)
487 if not numpy.isfinite(weight):
488 self.log.warn(
"Non-finite weight for %s: skipping", tempExpRef.dataId)
490 self.log.info(
"Weight of %s %s = %0.3f", tempExpName, tempExpRef.dataId, weight)
495 tempExpRefList.append(tempExpRef)
496 weightList.append(weight)
497 imageScalerList.append(imageScaler)
499 return pipeBase.Struct(tempExpRefList=tempExpRefList, weightList=weightList,
500 imageScalerList=imageScalerList)
503 """Prepare the statistics for coadding images. 507 mask : `int`, optional 508 Bit mask value to exclude from coaddition. 512 stats : `lsst.pipe.base.Struct` 513 Statistics structure with the following fields: 515 - ``statsCtrl``: Statistics control object for coadd 516 (`lsst.afw.math.StatisticsControl`) 517 - ``statsFlags``: Statistic for coadd (`lsst.afw.math.Property`) 521 statsCtrl = afwMath.StatisticsControl()
522 statsCtrl.setNumSigmaClip(self.config.sigmaClip)
523 statsCtrl.setNumIter(self.config.clipIter)
524 statsCtrl.setAndMask(mask)
525 statsCtrl.setNanSafe(
True)
526 statsCtrl.setWeighted(
True)
527 statsCtrl.setCalcErrorFromInputVariance(self.config.calcErrorFromInputVariance)
528 for plane, threshold
in self.config.maskPropagationThresholds.items():
529 bit = afwImage.Mask.getMaskPlane(plane)
530 statsCtrl.setMaskPropagationThreshold(bit, threshold)
531 statsFlags = afwMath.stringToStatisticsProperty(self.config.statistic)
532 return pipeBase.Struct(ctrl=statsCtrl, flags=statsFlags)
534 def assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList,
535 altMaskList=None, mask=None, supplementaryData=None):
536 """Assemble a coadd from input warps. 538 Assemble the coadd using the provided list of coaddTempExps. Since 539 the full coadd covers a patch (a large area), the assembly is 540 performed over small areas on the image at a time in order to 541 conserve memory usage. Iterate over subregions within the outer 542 bbox of the patch using `assembleSubregion` to stack the corresponding 543 subregions from the coaddTempExps with the statistic specified. 544 Set the edge bits the coadd mask based on the weight map. 548 skyInfo : `lsst.pipe.base.Struct` 549 Struct with geometric information about the patch. 550 tempExpRefList : `list` 551 List of data references to Warps (previously called CoaddTempExps). 552 imageScalerList : `list` 553 List of image scalers. 556 altMaskList : `list`, optional 557 List of alternate masks to use rather than those stored with 559 mask : `lsst.afw.image.Mask`, optional 560 Mask to ignore when coadding 561 supplementaryData : lsst.pipe.base.Struct, optional 562 Struct with additional data products needed to assemble coadd. 563 Only used by subclasses that implement `makeSupplementaryData` 564 and override `assemble`. 568 result : `lsst.pipe.base.Struct` 569 Result struct with components: 571 - ``coaddExposure``: coadded exposure (``lsst.afw.image.Exposure``). 572 - ``nImage``: exposure count image (``lsst.afw.image.Image``). 575 self.log.info(
"Assembling %s %s", len(tempExpRefList), tempExpName)
578 if altMaskList
is None:
579 altMaskList = [
None]*len(tempExpRefList)
581 coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
582 coaddExposure.setCalib(self.scaleZeroPoint.getCalib())
583 coaddExposure.getInfo().setCoaddInputs(self.inputRecorder.makeCoaddInputs())
585 coaddMaskedImage = coaddExposure.getMaskedImage()
586 subregionSizeArr = self.config.subregionSize
587 subregionSize = afwGeom.Extent2I(subregionSizeArr[0], subregionSizeArr[1])
589 if self.config.doNImage:
590 nImage = afwImage.ImageU(skyInfo.bbox)
593 for subBBox
in self.
_subBBoxIter(skyInfo.bbox, subregionSize):
596 weightList, altMaskList, stats.flags, stats.ctrl,
598 except Exception
as e:
599 self.log.fatal(
"Cannot compute coadd %s: %s", subBBox, e)
604 coaddUtils.setCoaddEdgeBits(coaddMaskedImage.getMask(), coaddMaskedImage.getVariance())
605 return pipeBase.Struct(coaddExposure=coaddExposure, nImage=nImage)
608 """Set the metadata for the coadd. 610 This basic implementation sets the filter from the first input. 614 coaddExposure : `lsst.afw.image.Exposure` 615 The target exposure for the coadd. 616 tempExpRefList : `list` 617 List of data references to tempExp. 621 assert len(tempExpRefList) == len(weightList),
"Length mismatch" 626 tempExpList = [tempExpRef.get(tempExpName +
"_sub",
627 bbox=afwGeom.Box2I(afwGeom.Point2I(0, 0), afwGeom.Extent2I(1, 1)),
628 imageOrigin=
"LOCAL", immediate=
True)
for tempExpRef
in tempExpRefList]
629 numCcds = sum(len(tempExp.getInfo().getCoaddInputs().ccds)
for tempExp
in tempExpList)
631 coaddExposure.setFilter(tempExpList[0].getFilter())
632 coaddInputs = coaddExposure.getInfo().getCoaddInputs()
633 coaddInputs.ccds.reserve(numCcds)
634 coaddInputs.visits.reserve(len(tempExpList))
636 for tempExp, weight
in zip(tempExpList, weightList):
637 self.inputRecorder.addVisitToCoadd(coaddInputs, tempExp, weight)
639 if self.config.doUsePsfMatchedPolygons:
642 coaddInputs.visits.sort()
648 modelPsfList = [tempExp.getPsf()
for tempExp
in tempExpList]
649 modelPsfWidthList = [modelPsf.computeBBox().getWidth()
for modelPsf
in modelPsfList]
650 psf = modelPsfList[modelPsfWidthList.index(max(modelPsfWidthList))]
652 psf = measAlg.CoaddPsf(coaddInputs.ccds, coaddExposure.getWcs(),
653 self.config.coaddPsf.makeControl())
654 coaddExposure.setPsf(psf)
655 apCorrMap = measAlg.makeCoaddApCorrMap(coaddInputs.ccds, coaddExposure.getBBox(afwImage.PARENT),
656 coaddExposure.getWcs())
657 coaddExposure.getInfo().setApCorrMap(apCorrMap)
658 if self.config.doAttachTransmissionCurve:
659 transmissionCurve = measAlg.makeCoaddTransmissionCurve(coaddExposure.getWcs(), coaddInputs.ccds)
660 coaddExposure.getInfo().setTransmissionCurve(transmissionCurve)
662 def assembleSubregion(self, coaddExposure, bbox, tempExpRefList, imageScalerList, weightList,
663 altMaskList, statsFlags, statsCtrl, nImage=None):
664 """Assemble the coadd for a sub-region. 666 For each coaddTempExp, check for (and swap in) an alternative mask 667 if one is passed. Remove mask planes listed in 668 `config.removeMaskPlanes`. Finally, stack the actual exposures using 669 `lsst.afw.math.statisticsStack` with the statistic specified by 670 statsFlags. Typically, the statsFlag will be one of lsst.afw.math.MEAN for 671 a mean-stack or `lsst.afw.math.MEANCLIP` for outlier rejection using 672 an N-sigma clipped mean where N and iterations are specified by 673 statsCtrl. Assign the stacked subregion back to the coadd. 677 coaddExposure : `lsst.afw.image.Exposure` 678 The target exposure for the coadd. 679 bbox : `lsst.afw.geom.Box` 681 tempExpRefList : `list` 682 List of data reference to tempExp. 683 imageScalerList : `list` 684 List of image scalers. 688 List of alternate masks to use rather than those stored with 689 tempExp, or None. Each element is dict with keys = mask plane 690 name to which to add the spans. 691 statsFlags : `lsst.afw.math.Property` 692 Property object for statistic for coadd. 693 statsCtrl : `lsst.afw.math.StatisticsControl` 694 Statistics control object for coadd. 695 nImage : `lsst.afw.image.ImageU`, optional 696 Keeps track of exposure count for each pixel. 698 self.log.debug(
"Computing coadd over %s", bbox)
700 coaddExposure.mask.addMaskPlane(
"REJECTED")
701 coaddExposure.mask.addMaskPlane(
"CLIPPED")
702 coaddExposure.mask.addMaskPlane(
"SENSOR_EDGE")
704 clipped = afwImage.Mask.getPlaneBitMask(
"CLIPPED")
706 if nImage
is not None:
707 subNImage = afwImage.ImageU(bbox.getWidth(), bbox.getHeight())
708 for tempExpRef, imageScaler, altMask
in zip(tempExpRefList, imageScalerList, altMaskList):
709 exposure = tempExpRef.get(tempExpName +
"_sub", bbox=bbox)
710 maskedImage = exposure.getMaskedImage()
711 mask = maskedImage.getMask()
712 if altMask
is not None:
714 imageScaler.scaleMaskedImage(maskedImage)
718 if nImage
is not None:
719 subNImage.getArray()[maskedImage.getMask().getArray() & statsCtrl.getAndMask() == 0] += 1
720 if self.config.removeMaskPlanes:
722 maskedImageList.append(maskedImage)
724 with self.timer(
"stack"):
725 coaddSubregion = afwMath.statisticsStack(maskedImageList, statsFlags, statsCtrl, weightList,
728 coaddExposure.maskedImage.assign(coaddSubregion, bbox)
729 if nImage
is not None:
730 nImage.assign(subNImage, bbox)
733 """Unset the mask of an image for mask planes specified in the config. 737 maskedImage : `lsst.afw.image.MaskedImage` 738 The masked image to be modified. 740 mask = maskedImage.getMask()
741 for maskPlane
in self.config.removeMaskPlanes:
743 mask &= ~mask.getPlaneBitMask(maskPlane)
744 except Exception
as e:
745 self.log.warn(
"Unable to remove mask plane %s: %s", maskPlane, e.args[0])
749 """Map certain mask planes of the warps to new planes for the coadd. 751 If a pixel is rejected due to a mask value other than EDGE, NO_DATA, 752 or CLIPPED, set it to REJECTED on the coadd. 753 If a pixel is rejected due to EDGE, set the coadd pixel to SENSOR_EDGE. 754 If a pixel is rejected due to CLIPPED, set the coadd pixel to CLIPPED. 758 statsCtrl : `lsst.afw.math.StatisticsControl` 759 Statistics control object for coadd 763 maskMap : `list` of `tuple` of `int` 764 A list of mappings of mask planes of the warped exposures to 765 mask planes of the coadd. 767 edge = afwImage.Mask.getPlaneBitMask(
"EDGE")
768 noData = afwImage.Mask.getPlaneBitMask(
"NO_DATA")
769 clipped = afwImage.Mask.getPlaneBitMask(
"CLIPPED")
770 toReject = statsCtrl.getAndMask() & (~noData) & (~edge) & (~clipped)
771 maskMap = [(toReject, afwImage.Mask.getPlaneBitMask(
"REJECTED")),
772 (edge, afwImage.Mask.getPlaneBitMask(
"SENSOR_EDGE")),
777 """Apply in place alt mask formatted as SpanSets to a mask. 781 mask : `lsst.afw.image.Mask` 783 altMaskSpans : `dict` 784 SpanSet lists to apply. Each element contains the new mask 785 plane name (e.g. "CLIPPED and/or "NO_DATA") as the key, 786 and list of SpanSets to apply to the mask. 790 mask : `lsst.afw.image.Mask` 793 if self.config.doUsePsfMatchedPolygons:
794 if (
"NO_DATA" in altMaskSpans)
and (
"NO_DATA" in self.config.badMaskPlanes):
799 for spanSet
in altMaskSpans[
'NO_DATA']:
800 spanSet.clippedTo(mask.getBBox()).clearMask(mask, self.
getBadPixelMask())
802 for plane, spanSetList
in altMaskSpans.items():
803 maskClipValue = mask.addMaskPlane(plane)
804 for spanSet
in spanSetList:
805 spanSet.clippedTo(mask.getBBox()).setMask(mask, 2**maskClipValue)
809 """Shrink coaddInputs' ccds' ValidPolygons in place. 811 Either modify each ccd's validPolygon in place, or if CoaddInputs 812 does not have a validPolygon, create one from its bbox. 816 coaddInputs : `lsst.afw.image.coaddInputs` 820 for ccd
in coaddInputs.ccds:
821 polyOrig = ccd.getValidPolygon()
822 validPolyBBox = polyOrig.getBBox()
if polyOrig
else ccd.getBBox()
823 validPolyBBox.grow(-self.config.matchingKernelSize//2)
825 validPolygon = polyOrig.intersectionSingle(validPolyBBox)
827 validPolygon = afwGeom.polygon.Polygon(afwGeom.Box2D(validPolyBBox))
828 ccd.setValidPolygon(validPolygon)
831 """Retrieve the bright object masks. 833 Returns None on failure. 837 dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef` 842 result : `lsst.daf.persistence.butlerSubset.ButlerDataRef` 843 Bright object mask from the Butler object, or None if it cannot 847 return dataRef.get(
"brightObjectMask", immediate=
True)
848 except Exception
as e:
849 self.log.warn(
"Unable to read brightObjectMask for %s: %s", dataRef.dataId, e)
853 """Set the bright object masks. 857 exposure : `lsst.afw.image.Exposure` 858 Exposure under consideration. 859 dataId : `lsst.daf.persistence.dataId` 860 Data identifier dict for patch. 861 brightObjectMasks : `lsst.afw.table` 862 Table of bright objects to mask. 867 if brightObjectMasks
is None:
868 self.log.warn(
"Unable to apply bright object mask: none supplied")
870 self.log.info(
"Applying %d bright object masks to %s", len(brightObjectMasks), dataId)
871 md = brightObjectMasks.table.getMetadata()
874 self.log.warn(
"Expected to see %s in metadata", k)
876 if md.getScalar(k) != dataId[k]:
877 self.log.warn(
"Expected to see %s == %s in metadata, saw %s", k, md.get(k), dataId[k])
879 mask = exposure.getMaskedImage().getMask()
880 wcs = exposure.getWcs()
881 plateScale = wcs.getPixelScale().asArcseconds()
883 for rec
in brightObjectMasks:
884 center = afwGeom.PointI(wcs.skyToPixel(rec.getCoord()))
885 if rec[
"type"] ==
"box":
886 assert rec[
"angle"] == 0.0, (
"Angle != 0 for mask object %s" % rec[
"id"])
887 width = rec[
"width"].asArcseconds()/plateScale
888 height = rec[
"height"].asArcseconds()/plateScale
890 halfSize = afwGeom.ExtentI(0.5*width, 0.5*height)
891 bbox = afwGeom.Box2I(center - halfSize, center + halfSize)
893 bbox = afwGeom.BoxI(afwGeom.PointI(int(center[0] - 0.5*width), int(center[1] - 0.5*height)),
894 afwGeom.PointI(int(center[0] + 0.5*width), int(center[1] + 0.5*height)))
895 spans = afwGeom.SpanSet(bbox)
896 elif rec[
"type"] ==
"circle":
897 radius = int(rec[
"radius"].asArcseconds()/plateScale)
898 spans = afwGeom.SpanSet.fromShape(radius, offset=center)
900 self.log.warn(
"Unexpected region type %s at %s" % rec[
"type"], center)
905 """Set INEXACT_PSF mask plane. 907 If any of the input images isn't represented in the coadd (due to 908 clipped pixels or chip gaps), the `CoaddPsf` will be inexact. Flag 913 mask : `lsst.afw.image.Mask` 914 Coadded exposure's mask, modified in-place. 916 mask.addMaskPlane(
"INEXACT_PSF")
917 inexactPsf = mask.getPlaneBitMask(
"INEXACT_PSF")
918 sensorEdge = mask.getPlaneBitMask(
"SENSOR_EDGE")
919 clipped = mask.getPlaneBitMask(
"CLIPPED")
920 rejected = mask.getPlaneBitMask(
"REJECTED")
921 array = mask.getArray()
922 selected = array & (sensorEdge | clipped | rejected) > 0
923 array[selected] |= inexactPsf
926 def _makeArgumentParser(cls):
927 """Create an argument parser. 930 parser.add_id_argument(
"--id", cls.
ConfigClass().coaddName +
"Coadd_" +
932 help=
"data ID, e.g. --id tract=12345 patch=1,2",
933 ContainerClass=AssembleCoaddDataIdContainer)
934 parser.add_id_argument(
"--selectId",
"calexp", help=
"data ID, e.g. --selectId visit=6789 ccd=0..9",
935 ContainerClass=SelectDataIdContainer)
939 def _subBBoxIter(bbox, subregionSize):
940 """Iterate over subregions of a bbox. 944 bbox : `lsst.afw.geom.Box2I` 945 Bounding box over which to iterate. 946 subregionSize: `lsst.afw.geom.Extent2I` 951 subBBox : `lsst.afw.geom.Box2I` 952 Next sub-bounding box of size ``subregionSize`` or smaller; each ``subBBox`` 953 is contained within ``bbox``, so it may be smaller than ``subregionSize`` at 954 the edges of ``bbox``, but it will never be empty. 957 raise RuntimeError(
"bbox %s is empty" % (bbox,))
958 if subregionSize[0] < 1
or subregionSize[1] < 1:
959 raise RuntimeError(
"subregionSize %s must be nonzero" % (subregionSize,))
961 for rowShift
in range(0, bbox.getHeight(), subregionSize[1]):
962 for colShift
in range(0, bbox.getWidth(), subregionSize[0]):
963 subBBox = afwGeom.Box2I(bbox.getMin() + afwGeom.Extent2I(colShift, rowShift), subregionSize)
965 if subBBox.isEmpty():
966 raise RuntimeError(
"Bug: empty bbox! bbox=%s, subregionSize=%s, " 967 "colShift=%s, rowShift=%s" %
968 (bbox, subregionSize, colShift, rowShift))
973 """A version of `lsst.pipe.base.DataIdContainer` specialized for assembleCoadd. 977 """Make self.refList from self.idList. 982 Results of parsing command-line (with ``butler`` and ``log`` elements). 984 datasetType = namespace.config.coaddName +
"Coadd" 985 keysCoadd = namespace.butler.getKeys(datasetType=datasetType, level=self.level)
987 for dataId
in self.idList:
989 for key
in keysCoadd:
990 if key
not in dataId:
991 raise RuntimeError(
"--id must include " + key)
993 dataRef = namespace.butler.dataRef(
994 datasetType=datasetType,
997 self.refList.append(dataRef)
1001 """Function to count the number of pixels with a specific mask in a 1004 Find the intersection of mask & footprint. Count all pixels in the mask 1005 that are in the intersection that have bitmask set but do not have 1006 ignoreMask set. Return the count. 1010 mask : `lsst.afw.image.Mask` 1011 Mask to define intersection region by. 1012 footprint : `lsst.afw.detection.Footprint` 1013 Footprint to define the intersection region by. 1015 Specific mask that we wish to count the number of occurances of. 1017 Pixels to not consider. 1022 Count of number of pixels in footprint with specified mask. 1024 bbox = footprint.getBBox()
1025 bbox.clip(mask.getBBox(afwImage.PARENT))
1026 fp = afwImage.Mask(bbox)
1027 subMask = mask.Factory(mask, bbox, afwImage.PARENT)
1028 footprint.spans.setMask(fp, bitmask)
1029 return numpy.logical_and((subMask.getArray() & fp.getArray()) > 0,
1030 (subMask.getArray() & ignoreMask) == 0).sum()
1034 """Configuration parameters for the SafeClipAssembleCoaddTask. 1036 clipDetection = pexConfig.ConfigurableField(
1037 target=SourceDetectionTask,
1038 doc=
"Detect sources on difference between unclipped and clipped coadd")
1039 minClipFootOverlap = pexConfig.Field(
1040 doc=
"Minimum fractional overlap of clipped footprint with visit DETECTED to be clipped",
1044 minClipFootOverlapSingle = pexConfig.Field(
1045 doc=
"Minimum fractional overlap of clipped footprint with visit DETECTED to be " 1046 "clipped when only one visit overlaps",
1050 minClipFootOverlapDouble = pexConfig.Field(
1051 doc=
"Minimum fractional overlap of clipped footprints with visit DETECTED to be " 1052 "clipped when two visits overlap",
1056 maxClipFootOverlapDouble = pexConfig.Field(
1057 doc=
"Maximum fractional overlap of clipped footprints with visit DETECTED when " 1058 "considering two visits",
1062 minBigOverlap = pexConfig.Field(
1063 doc=
"Minimum number of pixels in footprint to use DETECTED mask from the single visits " 1064 "when labeling clipped footprints",
1070 """Set default values for clipDetection. 1074 The numeric values for these configuration parameters were 1075 empirically determined, future work may further refine them. 1077 AssembleCoaddConfig.setDefaults(self)
1093 log.warn(
"Additional Sigma-clipping not allowed in Safe-clipped Coadds. " 1094 "Ignoring doSigmaClip.")
1097 raise ValueError(
"Only MEAN statistic allowed for final stacking in SafeClipAssembleCoadd " 1098 "(%s chosen). Please set statistic to MEAN." 1100 AssembleCoaddTask.ConfigClass.validate(self)
1104 """Assemble a coadded image from a set of coadded temporary exposures, 1105 being careful to clip & flag areas with potential artifacts. 1107 In ``AssembleCoaddTask``, we compute the coadd as an clipped mean (i.e., 1108 we clip outliers). The problem with doing this is that when computing the 1109 coadd PSF at a given location, individual visit PSFs from visits with 1110 outlier pixels contribute to the coadd PSF and cannot be treated correctly. 1111 In this task, we correct for this behavior by creating a new 1112 ``badMaskPlane`` 'CLIPPED'. We populate this plane on the input 1113 coaddTempExps and the final coadd where 1115 i. difference imaging suggests that there is an outlier and 1116 ii. this outlier appears on only one or two images. 1118 Such regions will not contribute to the final coadd. Furthermore, any 1119 routine to determine the coadd PSF can now be cognizant of clipped regions. 1120 Note that the algorithm implemented by this task is preliminary and works 1121 correctly for HSC data. Parameter modifications and or considerable 1122 redesigning of the algorithm is likley required for other surveys. 1124 ``SafeClipAssembleCoaddTask`` uses a ``SourceDetectionTask`` 1125 "clipDetection" subtask and also sub-classes ``AssembleCoaddTask``. 1126 You can retarget the ``SourceDetectionTask`` "clipDetection" subtask 1131 The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a 1132 flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``; 1133 see `baseDebug` for more about ``debug.py`` files. 1134 `SafeClipAssembleCoaddTask` has no debug variables of its own. 1135 The ``SourceDetectionTask`` "clipDetection" subtasks may support debug 1136 variables. See the documetation for `SourceDetectionTask` "clipDetection" 1137 for further information. 1141 `SafeClipAssembleCoaddTask` assembles a set of warped ``coaddTempExp`` 1142 images into a coadded image. The `SafeClipAssembleCoaddTask` is invoked by 1143 running assembleCoadd.py *without* the flag '--legacyCoadd'. 1145 Usage of ``assembleCoadd.py`` expects a data reference to the tract patch 1146 and filter to be coadded (specified using 1147 '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]') 1148 along with a list of coaddTempExps to attempt to coadd (specified using 1149 '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]'). 1150 Only the coaddTempExps that cover the specified tract and patch will be 1151 coadded. A list of the available optional arguments can be obtained by 1152 calling assembleCoadd.py with the --help command line argument: 1154 .. code-block:: none 1156 assembleCoadd.py --help 1158 To demonstrate usage of the `SafeClipAssembleCoaddTask` in the larger 1159 context of multi-band processing, we will generate the HSC-I & -R band 1160 coadds from HSC engineering test data provided in the ci_hsc package. 1161 To begin, assuming that the lsst stack has been already set up, we must 1162 set up the obs_subaru and ci_hsc packages. This defines the environment 1163 variable $CI_HSC_DIR and points at the location of the package. The raw 1164 HSC data live in the ``$CI_HSC_DIR/raw`` directory. To begin assembling 1165 the coadds, we must first 1168 process the individual ccds in $CI_HSC_RAW to produce calibrated exposures 1170 create a skymap that covers the area of the sky present in the raw exposures 1171 - ``makeCoaddTempExp`` 1172 warp the individual calibrated exposures to the tangent plane of the coadd</DD> 1174 We can perform all of these steps by running 1176 .. code-block:: none 1178 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988 1180 This will produce warped coaddTempExps for each visit. To coadd the 1181 warped data, we call ``assembleCoadd.py`` as follows: 1183 .. code-block:: none 1185 assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \ 1186 --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \ 1187 --selectId visit=903986 ccd=100--selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \ 1188 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \ 1189 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \ 1190 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \ 1191 --selectId visit=903988 ccd=24 1193 This will process the HSC-I band data. The results are written in 1194 ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``. 1196 You may also choose to run: 1198 .. code-block:: none 1200 scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346 nnn 1201 assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R --selectId visit=903334 ccd=16 \ 1202 --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 --selectId visit=903334 ccd=100 \ 1203 --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 --selectId visit=903338 ccd=18 \ 1204 --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 --selectId visit=903342 ccd=10 \ 1205 --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 --selectId visit=903344 ccd=5 \ 1206 --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 --selectId visit=903346 ccd=6 \ 1207 --selectId visit=903346 ccd=12 1209 to generate the coadd for the HSC-R band if you are interested in following 1210 multiBand Coadd processing as discussed in ``pipeTasks_multiBand``. 1212 ConfigClass = SafeClipAssembleCoaddConfig
1213 _DefaultName =
"safeClipAssembleCoadd" 1216 AssembleCoaddTask.__init__(self, *args, **kwargs)
1217 schema = afwTable.SourceTable.makeMinimalSchema()
1218 self.makeSubtask(
"clipDetection", schema=schema)
1220 def assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList, *args, **kwargs):
1221 """Assemble the coadd for a region. 1223 Compute the difference of coadds created with and without outlier 1224 rejection to identify coadd pixels that have outlier values in some 1226 Detect clipped regions on the difference image and mark these regions 1227 on the one or two individual coaddTempExps where they occur if there 1228 is significant overlap between the clipped region and a source. This 1229 leaves us with a set of footprints from the difference image that have 1230 been identified as having occured on just one or two individual visits. 1231 However, these footprints were generated from a difference image. It 1232 is conceivable for a large diffuse source to have become broken up 1233 into multiple footprints acrosss the coadd difference in this process. 1234 Determine the clipped region from all overlapping footprints from the 1235 detected sources in each visit - these are big footprints. 1236 Combine the small and big clipped footprints and mark them on a new 1238 Generate the coadd using `AssembleCoaddTask.assemble` without outlier 1239 removal. Clipped footprints will no longer make it into the coadd 1240 because they are marked in the new bad mask plane. 1244 skyInfo : `lsst.pipe.base.Struct` 1245 Patch geometry information, from getSkyInfo 1246 tempExpRefList : `list` 1247 List of data reference to tempExp 1248 imageScalerList : `list` 1249 List of image scalers 1255 result : `lsst.pipe.base.Struct` 1256 Result struct with components: 1258 - ``coaddExposure``: coadded exposure (``lsst.afw.image.Exposure``). 1259 - ``nImage``: exposure count image (``lsst.afw.image.Image``). 1263 args and kwargs are passed but ignored in order to match the call 1264 signature expected by the parent task. 1267 mask = exp.getMaskedImage().getMask()
1268 mask.addMaskPlane(
"CLIPPED")
1270 result = self.
detectClip(exp, tempExpRefList)
1272 self.log.info(
'Found %d clipped objects', len(result.clipFootprints))
1274 maskClipValue = mask.getPlaneBitMask(
"CLIPPED")
1275 maskDetValue = mask.getPlaneBitMask(
"DETECTED") | mask.getPlaneBitMask(
"DETECTED_NEGATIVE")
1277 bigFootprints = self.
detectClipBig(result.clipSpans, result.clipFootprints, result.clipIndices,
1278 result.detectionFootprints, maskClipValue, maskDetValue,
1281 maskClip = mask.Factory(mask.getBBox(afwImage.PARENT))
1282 afwDet.setMaskFromFootprintList(maskClip, result.clipFootprints, maskClipValue)
1284 maskClipBig = maskClip.Factory(mask.getBBox(afwImage.PARENT))
1285 afwDet.setMaskFromFootprintList(maskClipBig, bigFootprints, maskClipValue)
1286 maskClip |= maskClipBig
1289 badMaskPlanes = self.config.badMaskPlanes[:]
1290 badMaskPlanes.append(
"CLIPPED")
1291 badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes)
1292 return AssembleCoaddTask.assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList,
1293 result.clipSpans, mask=badPixelMask)
1296 """Return an exposure that contains the difference between unclipped 1299 Generate a difference image between clipped and unclipped coadds. 1300 Compute the difference image by subtracting an outlier-clipped coadd 1301 from an outlier-unclipped coadd. Return the difference image. 1305 skyInfo : `lsst.pipe.base.Struct` 1306 Patch geometry information, from getSkyInfo 1307 tempExpRefList : `list` 1308 List of data reference to tempExp 1309 imageScalerList : `list` 1310 List of image scalers 1316 exp : `lsst.afw.image.Exposure` 1317 Difference image of unclipped and clipped coadd wrapped in an Exposure 1322 configIntersection = {k: getattr(self.config, k)
1323 for k, v
in self.config.toDict().items()
if (k
in config.keys())}
1324 config.update(**configIntersection)
1327 config.statistic =
'MEAN' 1329 coaddMean = task.assemble(skyInfo, tempExpRefList, imageScalerList, weightList).coaddExposure
1331 config.statistic =
'MEANCLIP' 1333 coaddClip = task.assemble(skyInfo, tempExpRefList, imageScalerList, weightList).coaddExposure
1335 coaddDiff = coaddMean.getMaskedImage().Factory(coaddMean.getMaskedImage())
1336 coaddDiff -= coaddClip.getMaskedImage()
1337 exp = afwImage.ExposureF(coaddDiff)
1338 exp.setPsf(coaddMean.getPsf())
1342 """Detect clipped regions on an exposure and set the mask on the 1343 individual tempExp masks. 1345 Detect footprints in the difference image after smoothing the 1346 difference image with a Gaussian kernal. Identify footprints that 1347 overlap with one or two input ``coaddTempExps`` by comparing the 1348 computed overlap fraction to thresholds set in the config. A different 1349 threshold is applied depending on the number of overlapping visits 1350 (restricted to one or two). If the overlap exceeds the thresholds, 1351 the footprint is considered "CLIPPED" and is marked as such on the 1352 coaddTempExp. Return a struct with the clipped footprints, the indices 1353 of the ``coaddTempExps`` that end up overlapping with the clipped 1354 footprints, and a list of new masks for the ``coaddTempExps``. 1358 exp : `lsst.afw.image.Exposure` 1359 Exposure to run detection on. 1360 tempExpRefList : `list` 1361 List of data reference to tempExp. 1365 result : `lsst.pipe.base.Struct` 1366 Result struct with components: 1368 - ``clipFootprints``: list of clipped footprints. 1369 - ``clipIndices``: indices for each ``clippedFootprint`` in 1371 - ``clipSpans``: List of dictionaries containing spanSet lists 1372 to clip. Each element contains the new maskplane name 1373 ("CLIPPED") as the key and list of ``SpanSets`` as the value. 1374 - ``detectionFootprints``: List of DETECTED/DETECTED_NEGATIVE plane 1375 compressed into footprints. 1377 mask = exp.getMaskedImage().getMask()
1378 maskDetValue = mask.getPlaneBitMask(
"DETECTED") | mask.getPlaneBitMask(
"DETECTED_NEGATIVE")
1379 fpSet = self.clipDetection.detectFootprints(exp, doSmooth=
True, clearMask=
True)
1381 fpSet.positive.merge(fpSet.negative)
1382 footprints = fpSet.positive
1383 self.log.info(
'Found %d potential clipped objects', len(footprints.getFootprints()))
1388 artifactSpanSets = [{
'CLIPPED': list()}
for _
in tempExpRefList]
1391 visitDetectionFootprints = []
1393 dims = [len(tempExpRefList), len(footprints.getFootprints())]
1394 overlapDetArr = numpy.zeros(dims, dtype=numpy.uint16)
1395 ignoreArr = numpy.zeros(dims, dtype=numpy.uint16)
1398 for i, warpRef
in enumerate(tempExpRefList):
1400 immediate=
True).getMaskedImage().getMask()
1401 maskVisitDet = tmpExpMask.Factory(tmpExpMask, tmpExpMask.getBBox(afwImage.PARENT),
1402 afwImage.PARENT,
True)
1403 maskVisitDet &= maskDetValue
1404 visitFootprints = afwDet.FootprintSet(maskVisitDet, afwDet.Threshold(1))
1405 visitDetectionFootprints.append(visitFootprints)
1407 for j, footprint
in enumerate(footprints.getFootprints()):
1412 for j, footprint
in enumerate(footprints.getFootprints()):
1413 nPixel = footprint.getArea()
1416 for i
in range(len(tempExpRefList)):
1417 ignore = ignoreArr[i, j]
1418 overlapDet = overlapDetArr[i, j]
1419 totPixel = nPixel - ignore
1422 if ignore > overlapDet
or totPixel <= 0.5*nPixel
or overlapDet == 0:
1424 overlap.append(overlapDet/float(totPixel))
1427 overlap = numpy.array(overlap)
1428 if not len(overlap):
1435 if len(overlap) == 1:
1436 if overlap[0] > self.config.minClipFootOverlapSingle:
1441 clipIndex = numpy.where(overlap > self.config.minClipFootOverlap)[0]
1442 if len(clipIndex) == 1:
1444 keepIndex = [clipIndex[0]]
1447 clipIndex = numpy.where(overlap > self.config.minClipFootOverlapDouble)[0]
1448 if len(clipIndex) == 2
and len(overlap) > 3:
1449 clipIndexComp = numpy.where(overlap <= self.config.minClipFootOverlapDouble)[0]
1450 if numpy.max(overlap[clipIndexComp]) <= self.config.maxClipFootOverlapDouble:
1452 keepIndex = clipIndex
1457 for index
in keepIndex:
1458 globalIndex = indexList[index]
1459 artifactSpanSets[globalIndex][
'CLIPPED'].append(footprint.spans)
1461 clipIndices.append(numpy.array(indexList)[keepIndex])
1462 clipFootprints.append(footprint)
1464 return pipeBase.Struct(clipFootprints=clipFootprints, clipIndices=clipIndices,
1465 clipSpans=artifactSpanSets, detectionFootprints=visitDetectionFootprints)
1467 def detectClipBig(self, clipList, clipFootprints, clipIndices, detectionFootprints,
1468 maskClipValue, maskDetValue, coaddBBox):
1469 """Return individual warp footprints for large artifacts and append 1470 them to ``clipList`` in place. 1472 Identify big footprints composed of many sources in the coadd 1473 difference that may have originated in a large diffuse source in the 1474 coadd. We do this by indentifying all clipped footprints that overlap 1475 significantly with each source in all the coaddTempExps. 1480 List of alt mask SpanSets with clipping information. Modified. 1481 clipFootprints : `list` 1482 List of clipped footprints. 1483 clipIndices : `list` 1484 List of which entries in tempExpClipList each footprint belongs to. 1486 Mask value of clipped pixels. 1488 Mask value of detected pixels. 1489 coaddBBox : `lsst.afw.geom.Box` 1490 BBox of the coadd and warps. 1494 bigFootprintsCoadd : `list` 1495 List of big footprints 1497 bigFootprintsCoadd = []
1499 for index, (clippedSpans, visitFootprints)
in enumerate(zip(clipList, detectionFootprints)):
1500 maskVisitDet = afwImage.MaskX(coaddBBox, 0x0)
1501 for footprint
in visitFootprints.getFootprints():
1502 footprint.spans.setMask(maskVisitDet, maskDetValue)
1505 clippedFootprintsVisit = []
1506 for foot, clipIndex
in zip(clipFootprints, clipIndices):
1507 if index
not in clipIndex:
1509 clippedFootprintsVisit.append(foot)
1510 maskVisitClip = maskVisitDet.Factory(maskVisitDet.getBBox(afwImage.PARENT))
1511 afwDet.setMaskFromFootprintList(maskVisitClip, clippedFootprintsVisit, maskClipValue)
1513 bigFootprintsVisit = []
1514 for foot
in visitFootprints.getFootprints():
1515 if foot.getArea() < self.config.minBigOverlap:
1518 if nCount > self.config.minBigOverlap:
1519 bigFootprintsVisit.append(foot)
1520 bigFootprintsCoadd.append(foot)
1522 for footprint
in bigFootprintsVisit:
1523 clippedSpans[
"CLIPPED"].append(footprint.spans)
1525 return bigFootprintsCoadd
1529 assembleStaticSkyModel = pexConfig.ConfigurableField(
1530 target=AssembleCoaddTask,
1531 doc=
"Task to assemble an artifact-free, PSF-matched Coadd to serve as a" 1532 " naive/first-iteration model of the static sky.",
1534 detect = pexConfig.ConfigurableField(
1535 target=SourceDetectionTask,
1536 doc=
"Detect outlier sources on difference between each psfMatched warp and static sky model" 1538 detectTemplate = pexConfig.ConfigurableField(
1539 target=SourceDetectionTask,
1540 doc=
"Detect sources on static sky model. Only used if doPreserveContainedBySource is True" 1542 maxNumEpochs = pexConfig.Field(
1543 doc=
"Charactistic maximum local number of epochs/visits in which an artifact candidate can appear " 1544 "and still be masked. The effective maxNumEpochs is a broken linear function of local " 1545 "number of epochs (N): min(maxFractionEpochsLow*N, maxNumEpochs + maxFractionEpochsHigh*N). " 1546 "For each footprint detected on the image difference between the psfMatched warp and static sky " 1547 "model, if a significant fraction of pixels (defined by spatialThreshold) are residuals in more " 1548 "than the computed effective maxNumEpochs, the artifact candidate is deemed persistant rather " 1549 "than transient and not masked.",
1553 maxFractionEpochsLow = pexConfig.RangeField(
1554 doc=
"Fraction of local number of epochs (N) to use as effective maxNumEpochs for low N. " 1555 "Effective maxNumEpochs = " 1556 "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)",
1561 maxFractionEpochsHigh = pexConfig.RangeField(
1562 doc=
"Fraction of local number of epochs (N) to use as effective maxNumEpochs for high N. " 1563 "Effective maxNumEpochs = " 1564 "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)",
1569 spatialThreshold = pexConfig.RangeField(
1570 doc=
"Unitless fraction of pixels defining how much of the outlier region has to meet the " 1571 "temporal criteria. If 0, clip all. If 1, clip none.",
1575 inclusiveMin=
True, inclusiveMax=
True 1577 doScaleWarpVariance = pexConfig.Field(
1578 doc=
"Rescale Warp variance plane using empirical noise?",
1582 scaleWarpVariance = pexConfig.ConfigurableField(
1583 target=ScaleVarianceTask,
1584 doc=
"Rescale variance on warps",
1586 doPreserveContainedBySource = pexConfig.Field(
1587 doc=
"Rescue artifacts from clipping that completely lie within a footprint detected" 1588 "on the PsfMatched Template Coadd. Replicates a behavior of SafeClip.",
1592 doPrefilterArtifacts = pexConfig.Field(
1593 doc=
"Ignore artifact candidates that are mostly covered by the bad pixel mask, " 1594 "because they will be excluded anyway. This prevents them from contributing " 1595 "to the outlier epoch count image and potentially being labeled as persistant." 1596 "'Mostly' is defined by the config 'prefilterArtifactsRatio'.",
1600 prefilterArtifactsMaskPlanes = pexConfig.ListField(
1601 doc=
"Prefilter artifact candidates that are mostly covered by these bad mask planes.",
1603 default=(
'NO_DATA',
'BAD',
'SAT',
'SUSPECT'),
1605 prefilterArtifactsRatio = pexConfig.Field(
1606 doc=
"Prefilter artifact candidates with less than this fraction overlapping good pixels",
1612 AssembleCoaddConfig.setDefaults(self)
1628 self.
detect.doTempLocalBackground =
False 1629 self.
detect.reEstimateBackground =
False 1630 self.
detect.returnOriginalFootprints =
False 1631 self.
detect.thresholdPolarity =
"both" 1632 self.
detect.thresholdValue = 5
1633 self.
detect.nSigmaToGrow = 2
1634 self.
detect.minPixels = 4
1635 self.
detect.isotropicGrow =
True 1636 self.
detect.thresholdType =
"pixel_stdev" 1644 """Assemble a compareWarp coadded image from a set of warps 1645 by masking artifacts detected by comparing PSF-matched warps. 1647 In ``AssembleCoaddTask``, we compute the coadd as an clipped mean (i.e., 1648 we clip outliers). The problem with doing this is that when computing the 1649 coadd PSF at a given location, individual visit PSFs from visits with 1650 outlier pixels contribute to the coadd PSF and cannot be treated correctly. 1651 In this task, we correct for this behavior by creating a new badMaskPlane 1652 'CLIPPED' which marks pixels in the individual warps suspected to contain 1653 an artifact. We populate this plane on the input warps by comparing 1654 PSF-matched warps with a PSF-matched median coadd which serves as a 1655 model of the static sky. Any group of pixels that deviates from the 1656 PSF-matched template coadd by more than config.detect.threshold sigma, 1657 is an artifact candidate. The candidates are then filtered to remove 1658 variable sources and sources that are difficult to subtract such as 1659 bright stars. This filter is configured using the config parameters 1660 ``temporalThreshold`` and ``spatialThreshold``. The temporalThreshold is 1661 the maximum fraction of epochs that the deviation can appear in and still 1662 be considered an artifact. The spatialThreshold is the maximum fraction of 1663 pixels in the footprint of the deviation that appear in other epochs 1664 (where other epochs is defined by the temporalThreshold). If the deviant 1665 region meets this criteria of having a significant percentage of pixels 1666 that deviate in only a few epochs, these pixels have the 'CLIPPED' bit 1667 set in the mask. These regions will not contribute to the final coadd. 1668 Furthermore, any routine to determine the coadd PSF can now be cognizant 1669 of clipped regions. Note that the algorithm implemented by this task is 1670 preliminary and works correctly for HSC data. Parameter modifications and 1671 or considerable redesigning of the algorithm is likley required for other 1674 ``CompareWarpAssembleCoaddTask`` sub-classes 1675 ``AssembleCoaddTask`` and instantiates ``AssembleCoaddTask`` 1676 as a subtask to generate the TemplateCoadd (the model of the static sky). 1680 The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a 1681 flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``; see 1682 ``baseDebug`` for more about ``debug.py`` files. 1684 This task supports the following debug variables: 1687 If True then save the Epoch Count Image as a fits file in the `figPath` 1689 Path to save the debug fits images and figures 1691 For example, put something like: 1693 .. code-block:: python 1696 def DebugInfo(name): 1697 di = lsstDebug.getInfo(name) 1698 if name == "lsst.pipe.tasks.assembleCoadd": 1699 di.saveCountIm = True 1700 di.figPath = "/desired/path/to/debugging/output/images" 1702 lsstDebug.Info = DebugInfo 1704 into your ``debug.py`` file and run ``assemebleCoadd.py`` with the 1705 ``--debug`` flag. Some subtasks may have their own debug variables; 1706 see individual Task documentation. 1710 ``CompareWarpAssembleCoaddTask`` assembles a set of warped images into a 1711 coadded image. The ``CompareWarpAssembleCoaddTask`` is invoked by running 1712 ``assembleCoadd.py`` with the flag ``--compareWarpCoadd``. 1713 Usage of ``assembleCoadd.py`` expects a data reference to the tract patch 1714 and filter to be coadded (specified using 1715 '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]') 1716 along with a list of coaddTempExps to attempt to coadd (specified using 1717 '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]'). 1718 Only the warps that cover the specified tract and patch will be coadded. 1719 A list of the available optional arguments can be obtained by calling 1720 ``assembleCoadd.py`` with the ``--help`` command line argument: 1722 .. code-block:: none 1724 assembleCoadd.py --help 1726 To demonstrate usage of the ``CompareWarpAssembleCoaddTask`` in the larger 1727 context of multi-band processing, we will generate the HSC-I & -R band 1728 oadds from HSC engineering test data provided in the ``ci_hsc`` package. 1729 To begin, assuming that the lsst stack has been already set up, we must 1730 set up the ``obs_subaru`` and ``ci_hsc`` packages. 1731 This defines the environment variable ``$CI_HSC_DIR`` and points at the 1732 location of the package. The raw HSC data live in the ``$CI_HSC_DIR/raw`` 1733 directory. To begin assembling the coadds, we must first 1736 process the individual ccds in $CI_HSC_RAW to produce calibrated exposures 1738 create a skymap that covers the area of the sky present in the raw exposures 1740 warp the individual calibrated exposures to the tangent plane of the coadd 1742 We can perform all of these steps by running 1744 .. code-block:: none 1746 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988 1748 This will produce warped ``coaddTempExps`` for each visit. To coadd the 1749 warped data, we call ``assembleCoadd.py`` as follows: 1751 .. code-block:: none 1753 assembleCoadd.py --compareWarpCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \ 1754 --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \ 1755 --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \ 1756 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \ 1757 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \ 1758 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \ 1759 --selectId visit=903988 ccd=24 1761 This will process the HSC-I band data. The results are written in 1762 ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``. 1764 ConfigClass = CompareWarpAssembleCoaddConfig
1765 _DefaultName =
"compareWarpAssembleCoadd" 1768 AssembleCoaddTask.__init__(self, *args, **kwargs)
1769 self.makeSubtask(
"assembleStaticSkyModel")
1770 detectionSchema = afwTable.SourceTable.makeMinimalSchema()
1771 self.makeSubtask(
"detect", schema=detectionSchema)
1772 if self.config.doPreserveContainedBySource:
1773 self.makeSubtask(
"detectTemplate", schema=afwTable.SourceTable.makeMinimalSchema())
1774 if self.config.doScaleWarpVariance:
1775 self.makeSubtask(
"scaleWarpVariance")
1778 """Make inputs specific to Subclass. 1780 Generate a templateCoadd to use as a native model of static sky to 1781 subtract from warps. 1785 dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef` 1786 Butler dataRef for supplementary data. 1787 selectDataList : `list` 1788 List of data references to Warps. 1792 result : `lsst.pipe.base.Struct` 1793 Result struct with components: 1795 - ``templateCoaddcoadd``: coadded exposure (``lsst.afw.image.Exposure``). 1797 templateCoadd = self.assembleStaticSkyModel.
run(dataRef, selectDataList)
1799 if templateCoadd
is None:
1800 warpName = (self.assembleStaticSkyModel.warpType[0].upper() +
1801 self.assembleStaticSkyModel.warpType[1:])
1802 message =
"""No %(warpName)s warps were found to build the template coadd which is 1803 required to run CompareWarpAssembleCoaddTask. To continue assembling this type of coadd, 1804 first either rerun makeCoaddTempExp with config.make%(warpName)s=True or 1805 coaddDriver with config.makeCoadTempExp.make%(warpName)s=True, before assembleCoadd. 1807 Alternatively, to use another algorithm with existing warps, retarget the CoaddDriverConfig to 1808 another algorithm like: 1810 from lsst.pipe.tasks.assembleCoadd import SafeClipAssembleCoaddTask 1811 config.assemble.retarget(SafeClipAssembleCoaddTask) 1812 """ % {
"warpName": warpName}
1813 raise RuntimeError(message)
1815 return pipeBase.Struct(templateCoadd=templateCoadd.coaddExposure,
1816 nImage=templateCoadd.nImage)
1818 def assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList,
1819 supplementaryData, *args, **kwargs):
1820 """Assemble the coadd. 1822 Find artifacts and apply them to the warps' masks creating a list of 1823 alternative masks with a new "CLIPPED" plane and updated "NO_DATA" 1824 plane. Then pass these alternative masks to the base class's assemble 1829 skyInfo : `lsst.pipe.base.Struct` 1830 Patch geometry information. 1831 tempExpRefList : `list` 1832 List of data references to warps. 1833 imageScalerList : `list` 1834 List of image scalers. 1837 supplementaryData : `lsst.pipe.base.Struct` 1838 This Struct must contain a ``templateCoadd`` that serves as the 1839 model of the static sky. 1843 result : `lsst.pipe.base.Struct` 1844 Result struct with components: 1846 - ``coaddExposure``: coadded exposure (``lsst.afw.image.Exposure``). 1847 - ``nImage``: exposure count image (``lsst.afw.image.Image``), if requested. 1849 templateCoadd = supplementaryData.templateCoadd
1850 spanSetMaskList = self.
findArtifacts(templateCoadd, tempExpRefList, imageScalerList)
1851 badMaskPlanes = self.config.badMaskPlanes[:]
1852 badMaskPlanes.append(
"CLIPPED")
1853 badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes)
1855 result = AssembleCoaddTask.assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList,
1856 spanSetMaskList, mask=badPixelMask)
1860 self.
applyAltEdgeMask(result.coaddExposure.maskedImage.mask, spanSetMaskList)
1864 """Propagate alt EDGE mask to SENSOR_EDGE AND INEXACT_PSF planes. 1868 mask : `lsst.afw.image.Mask` 1870 altMaskList : `list` 1871 List of Dicts containing ``spanSet`` lists. 1872 Each element contains the new mask plane name (e.g. "CLIPPED 1873 and/or "NO_DATA") as the key, and list of ``SpanSets`` to apply to 1876 maskValue = mask.getPlaneBitMask([
"SENSOR_EDGE",
"INEXACT_PSF"])
1877 for visitMask
in altMaskList:
1878 if "EDGE" in visitMask:
1879 for spanSet
in visitMask[
'EDGE']:
1880 spanSet.clippedTo(mask.getBBox()).setMask(mask, maskValue)
1885 Loop through warps twice. The first loop builds a map with the count 1886 of how many epochs each pixel deviates from the templateCoadd by more 1887 than ``config.chiThreshold`` sigma. The second loop takes each 1888 difference image and filters the artifacts detected in each using 1889 count map to filter out variable sources and sources that are 1890 difficult to subtract cleanly. 1894 templateCoadd : `lsst.afw.image.Exposure` 1895 Exposure to serve as model of static sky. 1896 tempExpRefList : `list` 1897 List of data references to warps. 1898 imageScalerList : `list` 1899 List of image scalers. 1904 List of dicts containing information about CLIPPED 1905 (i.e., artifacts), NO_DATA, and EDGE pixels. 1908 self.log.debug(
"Generating Count Image, and mask lists.")
1909 coaddBBox = templateCoadd.getBBox()
1910 slateIm = afwImage.ImageU(coaddBBox)
1911 epochCountImage = afwImage.ImageU(coaddBBox)
1912 nImage = afwImage.ImageU(coaddBBox)
1913 spanSetArtifactList = []
1914 spanSetNoDataMaskList = []
1915 spanSetEdgeList = []
1919 templateCoadd.mask.clearAllMaskPlanes()
1921 if self.config.doPreserveContainedBySource:
1922 templateFootprints = self.detectTemplate.detectFootprints(templateCoadd)
1924 templateFootprints =
None 1926 for warpRef, imageScaler
in zip(tempExpRefList, imageScalerList):
1928 if warpDiffExp
is not None:
1930 nImage.array += (numpy.isfinite(warpDiffExp.image.array) *
1931 ((warpDiffExp.mask.array & badPixelMask) == 0)).astype(numpy.uint16)
1932 fpSet = self.detect.detectFootprints(warpDiffExp, doSmooth=
False, clearMask=
True)
1933 fpSet.positive.merge(fpSet.negative)
1934 footprints = fpSet.positive
1936 spanSetList = [footprint.spans
for footprint
in footprints.getFootprints()]
1939 if self.config.doPrefilterArtifacts:
1941 for spans
in spanSetList:
1942 spans.setImage(slateIm, 1, doClip=
True)
1943 epochCountImage += slateIm
1949 nans = numpy.where(numpy.isnan(warpDiffExp.maskedImage.image.array), 1, 0)
1950 nansMask = afwImage.makeMaskFromArray(nans.astype(afwImage.MaskPixel))
1951 nansMask.setXY0(warpDiffExp.getXY0())
1952 edgeMask = warpDiffExp.mask
1953 spanSetEdgeMask = afwGeom.SpanSet.fromMask(edgeMask,
1954 edgeMask.getPlaneBitMask(
"EDGE")).split()
1958 nansMask = afwImage.MaskX(coaddBBox, 1)
1960 spanSetEdgeMask = []
1962 spanSetNoDataMask = afwGeom.SpanSet.fromMask(nansMask).split()
1964 spanSetNoDataMaskList.append(spanSetNoDataMask)
1965 spanSetArtifactList.append(spanSetList)
1966 spanSetEdgeList.append(spanSetEdgeMask)
1970 epochCountImage.writeFits(path)
1972 for i, spanSetList
in enumerate(spanSetArtifactList):
1974 filteredSpanSetList = self.
filterArtifacts(spanSetList, epochCountImage, nImage,
1976 spanSetArtifactList[i] = filteredSpanSetList
1979 for artifacts, noData, edge
in zip(spanSetArtifactList, spanSetNoDataMaskList, spanSetEdgeList):
1980 altMasks.append({
'CLIPPED': artifacts,
1986 """Remove artifact candidates covered by bad mask plane. 1988 Any future editing of the candidate list that does not depend on 1989 temporal information should go in this method. 1993 spanSetList : `list` 1994 List of SpanSets representing artifact candidates. 1995 exp : `lsst.afw.image.Exposure` 1996 Exposure containing mask planes used to prefilter. 2000 returnSpanSetList : `list` 2001 List of SpanSets with artifacts. 2003 badPixelMask = exp.mask.getPlaneBitMask(self.config.prefilterArtifactsMaskPlanes)
2004 goodArr = (exp.mask.array & badPixelMask) == 0
2005 returnSpanSetList = []
2006 bbox = exp.getBBox()
2007 x0, y0 = exp.getXY0()
2008 for i, span
in enumerate(spanSetList):
2009 y, x = span.clippedTo(bbox).indices()
2010 yIndexLocal = numpy.array(y) - y0
2011 xIndexLocal = numpy.array(x) - x0
2012 goodRatio = numpy.count_nonzero(goodArr[yIndexLocal, xIndexLocal])/span.getArea()
2013 if goodRatio > self.config.prefilterArtifactsRatio:
2014 returnSpanSetList.append(span)
2015 return returnSpanSetList
2017 def filterArtifacts(self, spanSetList, epochCountImage, nImage, footprintsToExclude=None):
2018 """Filter artifact candidates. 2022 spanSetList : `list` 2023 List of SpanSets representing artifact candidates. 2024 epochCountImage : `lsst.afw.image.Image` 2025 Image of accumulated number of warpDiff detections. 2026 nImage : `lsst.afw.image.Image` 2027 Image of the accumulated number of total epochs contributing. 2031 maskSpanSetList : `list` 2032 List of SpanSets with artifacts. 2035 maskSpanSetList = []
2036 x0, y0 = epochCountImage.getXY0()
2037 for i, span
in enumerate(spanSetList):
2038 y, x = span.indices()
2039 yIdxLocal = [y1 - y0
for y1
in y]
2040 xIdxLocal = [x1 - x0
for x1
in x]
2041 outlierN = epochCountImage.array[yIdxLocal, xIdxLocal]
2042 totalN = nImage.array[yIdxLocal, xIdxLocal]
2045 effMaxNumEpochsHighN = (self.config.maxNumEpochs +
2046 self.config.maxFractionEpochsHigh*numpy.mean(totalN))
2047 effMaxNumEpochsLowN = self.config.maxFractionEpochsLow * numpy.mean(totalN)
2048 effectiveMaxNumEpochs = int(min(effMaxNumEpochsLowN, effMaxNumEpochsHighN))
2049 nPixelsBelowThreshold = numpy.count_nonzero((outlierN > 0) &
2050 (outlierN <= effectiveMaxNumEpochs))
2051 percentBelowThreshold = nPixelsBelowThreshold / len(outlierN)
2052 if percentBelowThreshold > self.config.spatialThreshold:
2053 maskSpanSetList.append(span)
2055 if self.config.doPreserveContainedBySource
and footprintsToExclude
is not None:
2057 filteredMaskSpanSetList = []
2058 for span
in maskSpanSetList:
2060 for footprint
in footprintsToExclude.positive.getFootprints():
2061 if footprint.spans.contains(span):
2065 filteredMaskSpanSetList.append(span)
2066 maskSpanSetList = filteredMaskSpanSetList
2068 return maskSpanSetList
2070 def _readAndComputeWarpDiff(self, warpRef, imageScaler, templateCoadd):
2071 """Fetch a warp from the butler and return a warpDiff. 2075 warpRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef` 2076 Butler dataRef for the warp. 2077 imageScaler : `lsst.pipe.tasks.scaleZeroPoint.ImageScaler` 2078 An image scaler object. 2079 templateCoadd : `lsst.afw.image.Exposure` 2080 Exposure to be substracted from the scaled warp. 2084 warp : `lsst.afw.image.Exposure` 2085 Exposure of the image difference between the warp and template. 2090 if not warpRef.datasetExists(warpName):
2091 self.log.warn(
"Could not find %s %s; skipping it", warpName, warpRef.dataId)
2093 warp = warpRef.get(warpName, immediate=
True)
2095 imageScaler.scaleMaskedImage(warp.getMaskedImage())
2096 mi = warp.getMaskedImage()
2097 if self.config.doScaleWarpVariance:
2099 self.scaleWarpVariance.
run(mi)
2100 except Exception
as exc:
2101 self.log.warn(
"Unable to rescale variance of warp (%s); leaving it as-is" % (exc,))
2102 mi -= templateCoadd.getMaskedImage()
2105 def _dataRef2DebugPath(self, prefix, warpRef, coaddLevel=False):
2106 """Return a path to which to write debugging output. 2108 Creates a hyphen-delimited string of dataId values for simple filenames. 2113 Prefix for filename. 2114 warpRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef` 2115 Butler dataRef to make the path from. 2116 coaddLevel : `bool`, optional. 2117 If True, include only coadd-level keys (e.g., 'tract', 'patch', 2118 'filter', but no 'visit'). 2123 Path for debugging output. 2128 keys = warpRef.dataId.keys()
2129 keyList = sorted(keys, reverse=
True)
2131 filename =
"%s-%s.fits" % (prefix,
'-'.join([str(warpRef.dataId[k])
for k
in keyList]))
2132 return os.path.join(directory, filename)
def setBrightObjectMasks(self, exposure, dataId, brightObjectMasks)
def shrinkValidPolygons(self, coaddInputs)
def getCoaddDatasetName(self, warpType="direct")
def _dataRef2DebugPath(self, prefix, warpRef, coaddLevel=False)
def getGroupDataRef(butler, datasetType, groupTuple, keys)
Base class for coaddition.
def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList)
def assembleMetadata(self, coaddExposure, tempExpRefList, weightList)
def makeSupplementaryData(self, dataRef, selectDataList)
def makeSupplementaryData(self, dataRef, selectDataList)
def getTempExpRefList(self, patchRef, calExpRefList)
def removeMaskPlanes(self, maskedImage)
def assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList, args, kwargs)
def run(self, dataRef, selectDataList=[])
def _readAndComputeWarpDiff(self, warpRef, imageScaler, templateCoadd)
def prepareInputs(self, refList)
def applyAltMaskPlanes(self, mask, altMaskSpans)
def getSkyInfo(self, patchRef)
Use getSkyinfo to return the skyMap, tract and patch information, wcs and the outer bbox of the patch...
def getTempExpDatasetName(self, warpType="direct")
def __init__(self, args, kwargs)
def prepareStats(self, mask=None)
def makeDataRefList(self, namespace)
def getBadPixelMask(self)
Convenience method to provide the bitmask from the mask plane names.
def assembleSubregion(self, coaddExposure, bbox, tempExpRefList, imageScalerList, weightList, altMaskList, statsFlags, statsCtrl, nImage=None)
def detectClip(self, exp, tempExpRefList)
def setInexactPsf(self, mask)
def __init__(self, args, kwargs)
def filterArtifacts(self, spanSetList, epochCountImage, nImage, footprintsToExclude=None)
def buildDifferenceImage(self, skyInfo, tempExpRefList, imageScalerList, weightList)
def selectExposures(self, patchRef, skyInfo=None, selectDataList=[])
Select exposures to coadd.
def setRejectedMaskMapping(statsCtrl)
def applyAltEdgeMask(self, mask, altMaskList)
def readBrightObjectMasks(self, dataRef)
def processResults(self, coaddExposure, dataRef)
def __init__(self, args, kwargs)
def assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList, supplementaryData, args, kwargs)
def prefilterArtifacts(self, spanSetList, exp)
def _subBBoxIter(bbox, subregionSize)
def countMaskFromFootprint(mask, footprint, bitmask, ignoreMask)
def groupPatchExposures(patchDataRef, calexpDataRefList, coaddDatasetType="deepCoadd", tempExpDatasetType="deepCoadd_directWarp")
def assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList, altMaskList=None, mask=None, supplementaryData=None)
def detectClipBig(self, clipList, clipFootprints, clipIndices, detectionFootprints, maskClipValue, maskDetValue, coaddBBox)