36 from .coaddBase
import CoaddBaseTask, SelectDataIdContainer
37 from .interpImage
import InterpImageTask
38 from .scaleZeroPoint
import ScaleZeroPointTask
39 from .coaddHelpers
import groupPatchExposures, getGroupDataRef
40 from .scaleVariance
import ScaleVarianceTask
43 __all__ = [
"AssembleCoaddTask",
"AssembleCoaddConfig",
"SafeClipAssembleCoaddTask",
44 "SafeClipAssembleCoaddConfig",
"CompareWarpAssembleCoaddTask",
"CompareWarpAssembleCoaddConfig"]
48 """Configuration parameters for the `AssembleCoaddTask`. 52 The `doMaskBrightObjects` and `brightObjectMaskName` configuration options 53 only set the bitplane config.brightObjectMaskName. To make this useful you 54 *must* also configure the flags.pixel algorithm, for example by adding 58 config.measurement.plugins["base_PixelFlags"].masksFpCenter.append("BRIGHT_OBJECT") 59 config.measurement.plugins["base_PixelFlags"].masksFpAnywhere.append("BRIGHT_OBJECT") 61 to your measureCoaddSources.py and forcedPhotCoadd.py config overrides. 63 warpType = pexConfig.Field(
64 doc=
"Warp name: one of 'direct' or 'psfMatched'",
68 subregionSize = pexConfig.ListField(
70 doc=
"Width, height of stack subregion size; " 71 "make small enough that a full stack of images will fit into memory at once.",
75 statistic = pexConfig.Field(
77 doc=
"Main stacking statistic for aggregating over the epochs.",
80 doSigmaClip = pexConfig.Field(
82 doc=
"Perform sigma clipped outlier rejection with MEANCLIP statistic? (DEPRECATED)",
85 sigmaClip = pexConfig.Field(
87 doc=
"Sigma for outlier rejection; ignored if non-clipping statistic selected.",
90 clipIter = pexConfig.Field(
92 doc=
"Number of iterations of outlier rejection; ignored if non-clipping statistic selected.",
95 calcErrorFromInputVariance = pexConfig.Field(
97 doc=
"Calculate coadd variance from input variance by stacking statistic." 98 "Passed to StatisticsControl.setCalcErrorFromInputVariance()",
101 scaleZeroPoint = pexConfig.ConfigurableField(
102 target=ScaleZeroPointTask,
103 doc=
"Task to adjust the photometric zero point of the coadd temp exposures",
105 doInterp = pexConfig.Field(
106 doc=
"Interpolate over NaN pixels? Also extrapolate, if necessary, but the results are ugly.",
110 interpImage = pexConfig.ConfigurableField(
111 target=InterpImageTask,
112 doc=
"Task to interpolate (and extrapolate) over NaN pixels",
114 doWrite = pexConfig.Field(
115 doc=
"Persist coadd?",
119 doNImage = pexConfig.Field(
120 doc=
"Create image of number of contributing exposures for each pixel",
124 doUsePsfMatchedPolygons = pexConfig.Field(
125 doc=
"Use ValidPolygons from shrunk Psf-Matched Calexps? Should be set to True by CompareWarp only.",
129 maskPropagationThresholds = pexConfig.DictField(
132 doc=(
"Threshold (in fractional weight) of rejection at which we propagate a mask plane to " 133 "the coadd; that is, we set the mask bit on the coadd if the fraction the rejected frames " 134 "would have contributed exceeds this value."),
135 default={
"SAT": 0.1},
137 removeMaskPlanes = pexConfig.ListField(dtype=str, default=[
"NOT_DEBLENDED"],
138 doc=
"Mask planes to remove before coadding")
139 doMaskBrightObjects = pexConfig.Field(dtype=bool, default=
False,
140 doc=
"Set mask and flag bits for bright objects?")
141 brightObjectMaskName = pexConfig.Field(dtype=str, default=
"BRIGHT_OBJECT",
142 doc=
"Name of mask bit used for bright objects")
143 coaddPsf = pexConfig.ConfigField(
144 doc=
"Configuration for CoaddPsf",
145 dtype=measAlg.CoaddPsfConfig,
147 doAttachTransmissionCurve = pexConfig.Field(
148 dtype=bool, default=
False, optional=
False,
149 doc=(
"Attach a piecewise TransmissionCurve for the coadd? " 150 "(requires all input Exposures to have TransmissionCurves).")
154 CoaddBaseTask.ConfigClass.setDefaults(self)
158 CoaddBaseTask.ConfigClass.validate(self)
162 log.warn(
"Config doPsfMatch deprecated. Setting warpType='psfMatched'")
165 log.warn(
'doSigmaClip deprecated. To replicate behavior, setting statistic to "MEANCLIP"')
167 if self.
doInterp and self.
statistic not in [
'MEAN',
'MEDIAN',
'MEANCLIP',
'VARIANCE',
'VARIANCECLIP']:
168 raise ValueError(
"Must set doInterp=False for statistic=%s, which does not " 169 "compute and set a non-zero coadd variance estimate." % (self.
statistic))
171 unstackableStats = [
'NOTHING',
'ERROR',
'ORMASK']
172 if not hasattr(afwMath.Property, self.
statistic)
or self.
statistic in unstackableStats:
173 stackableStats = [str(k)
for k
in afwMath.Property.__members__.keys()
174 if str(k)
not in unstackableStats]
175 raise ValueError(
"statistic %s is not allowed. Please choose one of %s." 180 """Assemble a coadded image from a set of warps (coadded temporary exposures). 182 We want to assemble a coadded image from a set of Warps (also called 183 coadded temporary exposures or ``coaddTempExps``). 184 Each input Warp covers a patch on the sky and corresponds to a single 185 run/visit/exposure of the covered patch. We provide the task with a list 186 of Warps (``selectDataList``) from which it selects Warps that cover the 187 specified patch (pointed at by ``dataRef``). 188 Each Warp that goes into a coadd will typically have an independent 189 photometric zero-point. Therefore, we must scale each Warp to set it to 190 a common photometric zeropoint. WarpType may be one of 'direct' or 191 'psfMatched', and the boolean configs `config.makeDirect` and 192 `config.makePsfMatched` set which of the warp types will be coadded. 193 The coadd is computed as a mean with optional outlier rejection. 194 Criteria for outlier rejection are set in `AssembleCoaddConfig`. 195 Finally, Warps can have bad 'NaN' pixels which received no input from the 196 source calExps. We interpolate over these bad (NaN) pixels. 198 `AssembleCoaddTask` uses several sub-tasks. These are 200 - `ScaleZeroPointTask` 201 - create and use an ``imageScaler`` object to scale the photometric zeropoint for each Warp 203 - interpolate across bad pixels (NaN) in the final coadd 205 You can retarget these subtasks if you wish. 209 The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a 210 flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``; see 211 `baseDebug` for more about ``debug.py`` files. `AssembleCoaddTask` has 212 no debug variables of its own. Some of the subtasks may support debug 213 variables. See the documentation for the subtasks for further information. 217 `AssembleCoaddTask` assembles a set of warped images into a coadded image. 218 The `AssembleCoaddTask` can be invoked by running ``assembleCoadd.py`` 219 with the flag '--legacyCoadd'. Usage of assembleCoadd.py expects two 220 inputs: a data reference to the tract patch and filter to be coadded, and 221 a list of Warps to attempt to coadd. These are specified using ``--id`` and 222 ``--selectId``, respectively: 226 --id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]] 227 --selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]] 229 Only the Warps that cover the specified tract and patch will be coadded. 230 A list of the available optional arguments can be obtained by calling 231 ``assembleCoadd.py`` with the ``--help`` command line argument: 235 assembleCoadd.py --help 237 To demonstrate usage of the `AssembleCoaddTask` in the larger context of 238 multi-band processing, we will generate the HSC-I & -R band coadds from 239 HSC engineering test data provided in the ``ci_hsc`` package. To begin, 240 assuming that the lsst stack has been already set up, we must set up the 241 obs_subaru and ``ci_hsc`` packages. This defines the environment variable 242 ``$CI_HSC_DIR`` and points at the location of the package. The raw HSC 243 data live in the ``$CI_HSC_DIR/raw directory``. To begin assembling the 244 coadds, we must first 247 - process the individual ccds in $CI_HSC_RAW to produce calibrated exposures 249 - create a skymap that covers the area of the sky present in the raw exposures 251 - warp the individual calibrated exposures to the tangent plane of the coadd 253 We can perform all of these steps by running 257 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988 259 This will produce warped exposures for each visit. To coadd the warped 260 data, we call assembleCoadd.py as follows: 264 assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \ 265 --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \ 266 --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \ 267 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \ 268 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \ 269 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \ 270 --selectId visit=903988 ccd=24 272 that will process the HSC-I band data. The results are written in 273 ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``. 275 You may also choose to run: 279 scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346 280 assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R \ 281 --selectId visit=903334 ccd=16 --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 \ 282 --selectId visit=903334 ccd=100 --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 \ 283 --selectId visit=903338 ccd=18 --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 \ 284 --selectId visit=903342 ccd=10 --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 \ 285 --selectId visit=903344 ccd=5 --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 \ 286 --selectId visit=903346 ccd=6 --selectId visit=903346 ccd=12 288 to generate the coadd for the HSC-R band if you are interested in 289 following multiBand Coadd processing as discussed in `pipeTasks_multiBand` 290 (but note that normally, one would use the `SafeClipAssembleCoaddTask` 291 rather than `AssembleCoaddTask` to make the coadd. 293 ConfigClass = AssembleCoaddConfig
294 _DefaultName =
"assembleCoadd" 297 CoaddBaseTask.__init__(self, *args, **kwargs)
298 self.makeSubtask(
"interpImage")
299 self.makeSubtask(
"scaleZeroPoint")
301 if self.config.doMaskBrightObjects:
302 mask = afwImage.Mask()
305 except pexExceptions.LsstCppException:
306 raise RuntimeError(
"Unable to define mask plane for bright objects; planes used are %s" %
307 mask.getMaskPlaneDict().keys())
313 def run(self, dataRef, selectDataList=[]):
314 """Assemble a coadd from a set of Warps. 316 Coadd a set of Warps. Compute weights to be applied to each Warp and 317 find scalings to match the photometric zeropoint to a reference Warp. 318 Assemble the Warps using `assemble`. Interpolate over NaNs and 319 optionally write the coadd to disk. Return the coadded exposure. 323 dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef` 324 Data reference defining the patch for coaddition and the 325 reference Warp (if ``config.autoReference=False``). 326 Used to access the following data products: 327 - ``self.config.coaddName + "Coadd_skyMap"`` 328 - ``self.config.coaddName + "Coadd_ + <warpType> + "Warp"`` (optionally) 329 - ``self.config.coaddName + "Coadd"`` 330 selectDataList : `list` 331 List of data references to Warps. Data to be coadded will be 332 selected from this list based on overlap with the patch defined 337 retStruct : `lsst.pipe.base.Struct` 338 Result struct with components: 340 - ``coaddExposure``: coadded exposure (``Exposure``). 341 - ``nImage``: exposure count image (``Image``). 344 calExpRefList = self.
selectExposures(dataRef, skyInfo, selectDataList=selectDataList)
345 if len(calExpRefList) == 0:
346 self.log.warn(
"No exposures to coadd")
348 self.log.info(
"Coadding %d exposures", len(calExpRefList))
352 self.log.info(
"Found %d %s", len(inputData.tempExpRefList),
354 if len(inputData.tempExpRefList) == 0:
355 self.log.warn(
"No coadd temporary exposures found")
360 retStruct = self.
assemble(skyInfo, inputData.tempExpRefList, inputData.imageScalerList,
361 inputData.weightList, supplementaryData=supplementaryData)
363 if self.config.doInterp:
364 self.interpImage.
run(retStruct.coaddExposure.getMaskedImage(), planeName=
"NO_DATA")
366 varArray = retStruct.coaddExposure.getMaskedImage().getVariance().getArray()
367 with numpy.errstate(invalid=
"ignore"):
368 varArray[:] = numpy.where(varArray > 0, varArray, numpy.inf)
370 if self.config.doMaskBrightObjects:
374 if self.config.doWrite:
377 if self.config.doNImage
and retStruct.nImage
is not None:
383 """Make additional inputs to assemble() specific to subclasses. 385 Available to be implemented by subclasses only if they need the 386 coadd dataRef for performing preliminary processing before 387 assembling the coadd. 391 dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef` 392 Butler dataRef for supplementary data. 393 selectDataList : `list` 394 List of data references to Warps. 399 """Generate list data references corresponding to warped exposures 400 that lie within the patch to be coadded. 405 Data reference for patch. 406 calExpRefList : `list` 407 List of data references for input calexps. 411 tempExpRefList : `list` 412 List of Warp/CoaddTempExp data references. 414 butler = patchRef.getButler()
415 groupData =
groupPatchExposures(patchRef, calExpRefList, self.getCoaddDatasetName(self.warpType),
416 self.getTempExpDatasetName(self.warpType))
417 tempExpRefList = [
getGroupDataRef(butler, self.getTempExpDatasetName(self.warpType),
418 g, groupData.keys)
for 419 g
in groupData.groups.keys()]
420 return tempExpRefList
423 """Prepare the input warps for coaddition by measuring the weight for 424 each warp and the scaling for the photometric zero point. 426 Each Warp has its own photometric zeropoint and background variance. 427 Before coadding these Warps together, compute a scale factor to 428 normalize the photometric zeropoint and compute the weight for each Warp. 433 List of data references to tempExp 437 result : `lsst.pipe.base.Struct` 438 Result struct with components: 440 - ``tempExprefList``: `list` of data references to tempExp. 441 - ``weightList``: `list` of weightings. 442 - ``imageScalerList``: `list` of image scalers. 444 statsCtrl = afwMath.StatisticsControl()
445 statsCtrl.setNumSigmaClip(self.config.sigmaClip)
446 statsCtrl.setNumIter(self.config.clipIter)
448 statsCtrl.setNanSafe(
True)
456 for tempExpRef
in refList:
457 if not tempExpRef.datasetExists(tempExpName):
458 self.log.warn(
"Could not find %s %s; skipping it", tempExpName, tempExpRef.dataId)
461 tempExp = tempExpRef.get(tempExpName, immediate=
True)
462 maskedImage = tempExp.getMaskedImage()
463 imageScaler = self.scaleZeroPoint.computeImageScaler(
468 imageScaler.scaleMaskedImage(maskedImage)
469 except Exception
as e:
470 self.log.warn(
"Scaling failed for %s (skipping it): %s", tempExpRef.dataId, e)
472 statObj = afwMath.makeStatistics(maskedImage.getVariance(), maskedImage.getMask(),
473 afwMath.MEANCLIP, statsCtrl)
474 meanVar, meanVarErr = statObj.getResult(afwMath.MEANCLIP)
475 weight = 1.0 / float(meanVar)
476 if not numpy.isfinite(weight):
477 self.log.warn(
"Non-finite weight for %s: skipping", tempExpRef.dataId)
479 self.log.info(
"Weight of %s %s = %0.3f", tempExpName, tempExpRef.dataId, weight)
484 tempExpRefList.append(tempExpRef)
485 weightList.append(weight)
486 imageScalerList.append(imageScaler)
488 return pipeBase.Struct(tempExpRefList=tempExpRefList, weightList=weightList,
489 imageScalerList=imageScalerList)
491 def assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList,
492 altMaskList=None, mask=None, supplementaryData=None):
493 """Assemble a coadd from input warps 495 Assemble the coadd using the provided list of coaddTempExps. Since 496 the full coadd covers a patch (a large area), the assembly is 497 performed over small areas on the image at a time in order to 498 conserve memory usage. Iterate over subregions within the outer 499 bbox of the patch using `assembleSubregion` to stack the corresponding 500 subregions from the coaddTempExps with the statistic specified. 501 Set the edge bits the coadd mask based on the weight map. 505 skyInfo : `lsst.pipe.base.Struct` 506 Struct with geometric information about the patch. 507 tempExpRefList : `list` 508 List of data references to Warps (previously called CoaddTempExps). 509 imageScalerList : `list` 510 List of image scalers. 513 altMaskList : `list`, optional 514 List of alternate masks to use rather than those stored with 516 mask : `lsst.afw.image.Mask`, optional 517 Mask to ignore when coadding 518 supplementaryData : lsst.pipe.base.Struct, optional 519 Struct with additional data products needed to assemble coadd. 520 Only used by subclasses that implement `makeSupplementaryData` 521 and override `assemble`. 525 result : `lsst.pipe.base.Struct` 526 Result struct with components: 528 - ``coaddExposure``: coadded exposure (``lsst.afw.image.Exposure``). 529 - ``nImage``: exposure count image (``lsst.afw.image.Image``). 532 self.log.info(
"Assembling %s %s", len(tempExpRefList), tempExpName)
536 statsCtrl = afwMath.StatisticsControl()
537 statsCtrl.setNumSigmaClip(self.config.sigmaClip)
538 statsCtrl.setNumIter(self.config.clipIter)
539 statsCtrl.setAndMask(mask)
540 statsCtrl.setNanSafe(
True)
541 statsCtrl.setWeighted(
True)
542 statsCtrl.setCalcErrorFromInputVariance(self.config.calcErrorFromInputVariance)
543 for plane, threshold
in self.config.maskPropagationThresholds.items():
544 bit = afwImage.Mask.getMaskPlane(plane)
545 statsCtrl.setMaskPropagationThreshold(bit, threshold)
547 statsFlags = afwMath.stringToStatisticsProperty(self.config.statistic)
549 if altMaskList
is None:
550 altMaskList = [
None]*len(tempExpRefList)
552 coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
553 coaddExposure.setCalib(self.scaleZeroPoint.getCalib())
554 coaddExposure.getInfo().setCoaddInputs(self.inputRecorder.makeCoaddInputs())
556 coaddMaskedImage = coaddExposure.getMaskedImage()
557 subregionSizeArr = self.config.subregionSize
558 subregionSize = afwGeom.Extent2I(subregionSizeArr[0], subregionSizeArr[1])
560 if self.config.doNImage:
561 nImage = afwImage.ImageU(skyInfo.bbox)
564 for subBBox
in _subBBoxIter(skyInfo.bbox, subregionSize):
567 weightList, altMaskList, statsFlags, statsCtrl,
569 except Exception
as e:
570 self.log.fatal(
"Cannot compute coadd %s: %s", subBBox, e)
575 coaddUtils.setCoaddEdgeBits(coaddMaskedImage.getMask(), coaddMaskedImage.getVariance())
576 return pipeBase.Struct(coaddExposure=coaddExposure, nImage=nImage)
579 """Set the metadata for the coadd. 581 This basic implementation sets the filter from the first input. 585 coaddExposure : `lsst.afw.image.Exposure` 586 The target exposure for the coadd. 587 tempExpRefList : `list` 588 List of data references to tempExp. 592 assert len(tempExpRefList) == len(weightList),
"Length mismatch" 597 tempExpList = [tempExpRef.get(tempExpName +
"_sub",
598 bbox=afwGeom.Box2I(afwGeom.Point2I(0, 0), afwGeom.Extent2I(1, 1)),
599 imageOrigin=
"LOCAL", immediate=
True)
for tempExpRef
in tempExpRefList]
600 numCcds = sum(len(tempExp.getInfo().getCoaddInputs().ccds)
for tempExp
in tempExpList)
602 coaddExposure.setFilter(tempExpList[0].getFilter())
603 coaddInputs = coaddExposure.getInfo().getCoaddInputs()
604 coaddInputs.ccds.reserve(numCcds)
605 coaddInputs.visits.reserve(len(tempExpList))
607 for tempExp, weight
in zip(tempExpList, weightList):
608 self.inputRecorder.addVisitToCoadd(coaddInputs, tempExp, weight)
610 if self.config.doUsePsfMatchedPolygons:
613 coaddInputs.visits.sort()
619 modelPsfList = [tempExp.getPsf()
for tempExp
in tempExpList]
620 modelPsfWidthList = [modelPsf.computeBBox().getWidth()
for modelPsf
in modelPsfList]
621 psf = modelPsfList[modelPsfWidthList.index(max(modelPsfWidthList))]
623 psf = measAlg.CoaddPsf(coaddInputs.ccds, coaddExposure.getWcs(),
624 self.config.coaddPsf.makeControl())
625 coaddExposure.setPsf(psf)
626 apCorrMap = measAlg.makeCoaddApCorrMap(coaddInputs.ccds, coaddExposure.getBBox(afwImage.PARENT),
627 coaddExposure.getWcs())
628 coaddExposure.getInfo().setApCorrMap(apCorrMap)
629 if self.config.doAttachTransmissionCurve:
630 transmissionCurve = measAlg.makeCoaddTransmissionCurve(coaddExposure.getWcs(), coaddInputs.ccds)
631 coaddExposure.getInfo().setTransmissionCurve(transmissionCurve)
633 def assembleSubregion(self, coaddExposure, bbox, tempExpRefList, imageScalerList, weightList,
634 altMaskList, statsFlags, statsCtrl, nImage=None):
635 """Assemble the coadd for a sub-region. 637 For each coaddTempExp, check for (and swap in) an alternative mask 638 if one is passed. Remove mask planes listed in 639 `config.removeMaskPlanes`. Finally, stack the actual exposures using 640 `lsst.afw.math.statisticsStack` with the statistic specified by 641 statsFlags. Typically, the statsFlag will be one of lsst.afw.math.MEAN for 642 a mean-stack or `lsst.afw.math.MEANCLIP` for outlier rejection using 643 an N-sigma clipped mean where N and iterations are specified by 644 statsCtrl. Assign the stacked subregion back to the coadd. 648 coaddExposure : `lsst.afw.image.Exposure` 649 The target exposure for the coadd. 650 bbox : `lsst.afw.geom.Box` 652 tempExpRefList : `list` 653 List of data reference to tempExp. 654 imageScalerList : `list` 655 List of image scalers. 659 List of alternate masks to use rather than those stored with 660 tempExp, or None. Each element is dict with keys = mask plane 661 name to which to add the spans. 662 statsFlags : `lsst.afw.math.Property` 663 Property object for statistic for coadd. 664 statsCtrl : `lsst.afw.math.StatisticsControl` 665 Statistics control object for coadd. 666 nImage : `lsst.afw.image.ImageU`, optional 667 Keeps track of exposure count for each pixel. 669 self.log.debug(
"Computing coadd over %s", bbox)
671 coaddExposure.mask.addMaskPlane(
"REJECTED")
672 coaddExposure.mask.addMaskPlane(
"CLIPPED")
673 coaddExposure.mask.addMaskPlane(
"SENSOR_EDGE")
678 edge = afwImage.Mask.getPlaneBitMask(
"EDGE")
679 noData = afwImage.Mask.getPlaneBitMask(
"NO_DATA")
680 clipped = afwImage.Mask.getPlaneBitMask(
"CLIPPED")
681 toReject = statsCtrl.getAndMask() & (~noData) & (~edge) & (~clipped)
682 maskMap = [(toReject, coaddExposure.mask.getPlaneBitMask(
"REJECTED")),
683 (edge, coaddExposure.mask.getPlaneBitMask(
"SENSOR_EDGE")),
686 if nImage
is not None:
687 subNImage = afwImage.ImageU(bbox.getWidth(), bbox.getHeight())
688 for tempExpRef, imageScaler, altMask
in zip(tempExpRefList, imageScalerList, altMaskList):
689 exposure = tempExpRef.get(tempExpName +
"_sub", bbox=bbox)
690 maskedImage = exposure.getMaskedImage()
691 mask = maskedImage.getMask()
692 if altMask
is not None:
694 imageScaler.scaleMaskedImage(maskedImage)
698 if nImage
is not None:
699 subNImage.getArray()[maskedImage.getMask().getArray() & statsCtrl.getAndMask() == 0] += 1
700 if self.config.removeMaskPlanes:
701 mask = maskedImage.getMask()
702 for maskPlane
in self.config.removeMaskPlanes:
704 mask &= ~mask.getPlaneBitMask(maskPlane)
705 except Exception
as e:
706 self.log.warn(
"Unable to remove mask plane %s: %s", maskPlane, e.args[0])
708 maskedImageList.append(maskedImage)
710 with self.timer(
"stack"):
711 coaddSubregion = afwMath.statisticsStack(maskedImageList, statsFlags, statsCtrl, weightList,
714 coaddExposure.maskedImage.assign(coaddSubregion, bbox)
715 if nImage
is not None:
716 nImage.assign(subNImage, bbox)
719 """Apply in place alt mask formatted as SpanSets to a mask. 723 mask : `lsst.afw.image.Mask` 725 altMaskSpans : `dict` 726 SpanSet lists to apply. Each element contains the new mask 727 plane name (e.g. "CLIPPED and/or "NO_DATA") as the key, 728 and list of SpanSets to apply to the mask. 732 mask : `lsst.afw.image.Mask` 735 if self.config.doUsePsfMatchedPolygons:
736 if (
"NO_DATA" in altMaskSpans)
and (
"NO_DATA" in self.config.badMaskPlanes):
741 for spanSet
in altMaskSpans[
'NO_DATA']:
742 spanSet.clippedTo(mask.getBBox()).clearMask(mask, self.
getBadPixelMask())
744 for plane, spanSetList
in altMaskSpans.items():
745 maskClipValue = mask.addMaskPlane(plane)
746 for spanSet
in spanSetList:
747 spanSet.clippedTo(mask.getBBox()).setMask(mask, 2**maskClipValue)
751 """Shrink coaddInputs' ccds' ValidPolygons in place. 753 Either modify each ccd's validPolygon in place, or if CoaddInputs 754 does not have a validPolygon, create one from its bbox. 758 coaddInputs : `lsst.afw.image.coaddInputs` 762 for ccd
in coaddInputs.ccds:
763 polyOrig = ccd.getValidPolygon()
764 validPolyBBox = polyOrig.getBBox()
if polyOrig
else ccd.getBBox()
765 validPolyBBox.grow(-self.config.matchingKernelSize//2)
767 validPolygon = polyOrig.intersectionSingle(validPolyBBox)
769 validPolygon = afwGeom.polygon.Polygon(afwGeom.Box2D(validPolyBBox))
770 ccd.setValidPolygon(validPolygon)
773 """Retrieve the bright object masks. 775 Returns None on failure. 779 dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef` 784 result : `lsst.daf.persistence.butlerSubset.ButlerDataRef` 785 Bright object mask from the Butler object, or None if it cannot 789 return dataRef.get(
"brightObjectMask", immediate=
True)
790 except Exception
as e:
791 self.log.warn(
"Unable to read brightObjectMask for %s: %s", dataRef.dataId, e)
795 """Set the bright object masks. 799 exposure : `lsst.afw.image.Exposure` 800 Exposure under consideration. 801 dataId : `lsst.daf.persistence.dataId` 802 Data identifier dict for patch. 803 brightObjectMasks : `lsst.afw.table` 804 Table of bright objects to mask. 809 if brightObjectMasks
is None:
810 self.log.warn(
"Unable to apply bright object mask: none supplied")
812 self.log.info(
"Applying %d bright object masks to %s", len(brightObjectMasks), dataId)
813 md = brightObjectMasks.table.getMetadata()
816 self.log.warn(
"Expected to see %s in metadata", k)
818 if md.getScalar(k) != dataId[k]:
819 self.log.warn(
"Expected to see %s == %s in metadata, saw %s", k, md.get(k), dataId[k])
821 mask = exposure.getMaskedImage().getMask()
822 wcs = exposure.getWcs()
823 plateScale = wcs.getPixelScale().asArcseconds()
825 for rec
in brightObjectMasks:
826 center = afwGeom.PointI(wcs.skyToPixel(rec.getCoord()))
827 if rec[
"type"] ==
"box":
828 assert rec[
"angle"] == 0.0, (
"Angle != 0 for mask object %s" % rec[
"id"])
829 width = rec[
"width"].asArcseconds()/plateScale
830 height = rec[
"height"].asArcseconds()/plateScale
832 halfSize = afwGeom.ExtentI(0.5*width, 0.5*height)
833 bbox = afwGeom.Box2I(center - halfSize, center + halfSize)
835 bbox = afwGeom.BoxI(afwGeom.PointI(int(center[0] - 0.5*width), int(center[1] - 0.5*height)),
836 afwGeom.PointI(int(center[0] + 0.5*width), int(center[1] + 0.5*height)))
837 spans = afwGeom.SpanSet(bbox)
838 elif rec[
"type"] ==
"circle":
839 radius = int(rec[
"radius"].asArcseconds()/plateScale)
840 spans = afwGeom.SpanSet.fromShape(radius, offset=center)
842 self.log.warn(
"Unexpected region type %s at %s" % rec[
"type"], center)
847 """Set INEXACT_PSF mask plane. 849 If any of the input images isn't represented in the coadd (due to 850 clipped pixels or chip gaps), the `CoaddPsf` will be inexact. Flag 855 mask : `lsst.afw.image.Mask` 856 Coadded exposure's mask, modified in-place. 858 mask.addMaskPlane(
"INEXACT_PSF")
859 inexactPsf = mask.getPlaneBitMask(
"INEXACT_PSF")
860 sensorEdge = mask.getPlaneBitMask(
"SENSOR_EDGE")
861 clipped = mask.getPlaneBitMask(
"CLIPPED")
862 rejected = mask.getPlaneBitMask(
"REJECTED")
863 array = mask.getArray()
864 selected = array & (sensorEdge | clipped | rejected) > 0
865 array[selected] |= inexactPsf
868 def _makeArgumentParser(cls):
869 """Create an argument parser. 872 parser.add_id_argument(
"--id", cls.
ConfigClass().coaddName +
"Coadd_" +
874 help=
"data ID, e.g. --id tract=12345 patch=1,2",
875 ContainerClass=AssembleCoaddDataIdContainer)
876 parser.add_id_argument(
"--selectId",
"calexp", help=
"data ID, e.g. --selectId visit=6789 ccd=0..9",
877 ContainerClass=SelectDataIdContainer)
881 def _subBBoxIter(bbox, subregionSize):
882 """Iterate over subregions of a bbox. 886 bbox : `lsst.afw.geom.Box2I` 887 Bounding box over which to iterate. 888 subregionSize: `lsst.afw.geom.Extent2I` 893 subBBox : `lsst.afw.geom.Box2I` 894 Next sub-bounding box of size subregionSize or smaller; each subBBox 895 is contained within bbox, so it may be smaller than subregionSize at 896 the edges of bbox, but it will never be empty. 899 raise RuntimeError(
"bbox %s is empty" % (bbox,))
900 if subregionSize[0] < 1
or subregionSize[1] < 1:
901 raise RuntimeError(
"subregionSize %s must be nonzero" % (subregionSize,))
903 for rowShift
in range(0, bbox.getHeight(), subregionSize[1]):
904 for colShift
in range(0, bbox.getWidth(), subregionSize[0]):
905 subBBox = afwGeom.Box2I(bbox.getMin() + afwGeom.Extent2I(colShift, rowShift), subregionSize)
907 if subBBox.isEmpty():
908 raise RuntimeError(
"Bug: empty bbox! bbox=%s, subregionSize=%s, colShift=%s, rowShift=%s" %
909 (bbox, subregionSize, colShift, rowShift))
914 """A version of `lsst.pipe.base.DataIdContainer` specialized for assembleCoadd. 918 """Make self.refList from self.idList. 923 Results of parsing command-line (with ``butler`` and ``log`` elements). 925 datasetType = namespace.config.coaddName +
"Coadd" 926 keysCoadd = namespace.butler.getKeys(datasetType=datasetType, level=self.level)
928 for dataId
in self.idList:
930 for key
in keysCoadd:
931 if key
not in dataId:
932 raise RuntimeError(
"--id must include " + key)
934 dataRef = namespace.butler.dataRef(
935 datasetType=datasetType,
938 self.refList.append(dataRef)
942 """Function to count the number of pixels with a specific mask in a 945 Find the intersection of mask & footprint. Count all pixels in the mask 946 that are in the intersection that have bitmask set but do not have 947 ignoreMask set. Return the count. 951 mask : `lsst.afw.image.Mask` 952 Mask to define intersection region by. 953 footprint : `lsst.afw.detection.Footprint` 954 Footprint to define the intersection region by. 956 Specific mask that we wish to count the number of occurances of. 958 Pixels to not consider. 963 Count of number of pixels in footprint with specified mask. 965 bbox = footprint.getBBox()
966 bbox.clip(mask.getBBox(afwImage.PARENT))
967 fp = afwImage.Mask(bbox)
968 subMask = mask.Factory(mask, bbox, afwImage.PARENT)
969 footprint.spans.setMask(fp, bitmask)
970 return numpy.logical_and((subMask.getArray() & fp.getArray()) > 0,
971 (subMask.getArray() & ignoreMask) == 0).sum()
975 """Configuration parameters for the SafeClipAssembleCoaddTask. 977 clipDetection = pexConfig.ConfigurableField(
978 target=SourceDetectionTask,
979 doc=
"Detect sources on difference between unclipped and clipped coadd")
980 minClipFootOverlap = pexConfig.Field(
981 doc=
"Minimum fractional overlap of clipped footprint with visit DETECTED to be clipped",
985 minClipFootOverlapSingle = pexConfig.Field(
986 doc=
"Minimum fractional overlap of clipped footprint with visit DETECTED to be " 987 "clipped when only one visit overlaps",
991 minClipFootOverlapDouble = pexConfig.Field(
992 doc=
"Minimum fractional overlap of clipped footprints with visit DETECTED to be " 993 "clipped when two visits overlap",
997 maxClipFootOverlapDouble = pexConfig.Field(
998 doc=
"Maximum fractional overlap of clipped footprints with visit DETECTED when " 999 "considering two visits",
1003 minBigOverlap = pexConfig.Field(
1004 doc=
"Minimum number of pixels in footprint to use DETECTED mask from the single visits " 1005 "when labeling clipped footprints",
1011 """Set default values for clipDetection. 1015 The numeric values for these configuration parameters were 1016 empirically determined, future work may further refine them. 1018 AssembleCoaddConfig.setDefaults(self)
1034 log.warn(
"Additional Sigma-clipping not allowed in Safe-clipped Coadds. " 1035 "Ignoring doSigmaClip.")
1038 raise ValueError(
"Only MEAN statistic allowed for final stacking in SafeClipAssembleCoadd " 1039 "(%s chosen). Please set statistic to MEAN." 1041 AssembleCoaddTask.ConfigClass.validate(self)
1045 """Assemble a coadded image from a set of coadded temporary exposures, 1046 being careful to clip & flag areas with potential artifacts. 1048 In ``AssembleCoaddTask``, we compute the coadd as an clipped mean (i.e., 1049 we clip outliers). The problem with doing this is that when computing the 1050 coadd PSF at a given location, individual visit PSFs from visits with 1051 outlier pixels contribute to the coadd PSF and cannot be treated correctly. 1052 In this task, we correct for this behavior by creating a new 1053 ``badMaskPlane`` 'CLIPPED'. We populate this plane on the input 1054 coaddTempExps and the final coadd where 1056 i. difference imaging suggests that there is an outlier and 1057 ii. this outlier appears on only one or two images. 1059 Such regions will not contribute to the final coadd. Furthermore, any 1060 routine to determine the coadd PSF can now be cognizant of clipped regions. 1061 Note that the algorithm implemented by this task is preliminary and works 1062 correctly for HSC data. Parameter modifications and or considerable 1063 redesigning of the algorithm is likley required for other surveys. 1065 ``SafeClipAssembleCoaddTask`` uses a ``SourceDetectionTask`` 1066 "clipDetection" subtask and also sub-classes ``AssembleCoaddTask``. 1067 You can retarget the ``SourceDetectionTask`` "clipDetection" subtask 1072 The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a 1073 flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``; 1074 see `baseDebug` for more about ``debug.py`` files. 1075 `SafeClipAssembleCoaddTask` has no debug variables of its own. 1076 The ``SourceDetectionTask`` "clipDetection" subtasks may support debug 1077 variables. See the documetation for `SourceDetectionTask` "clipDetection" 1078 for further information. 1082 `SafeClipAssembleCoaddTask` assembles a set of warped ``coaddTempExp`` 1083 images into a coadded image. The `SafeClipAssembleCoaddTask` is invoked by 1084 running assembleCoadd.py *without* the flag '--legacyCoadd'. 1086 Usage of ``assembleCoadd.py`` expects a data reference to the tract patch 1087 and filter to be coadded (specified using 1088 '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]') 1089 along with a list of coaddTempExps to attempt to coadd (specified using 1090 '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]'). 1091 Only the coaddTempExps that cover the specified tract and patch will be 1092 coadded. A list of the available optional arguments can be obtained by 1093 calling assembleCoadd.py with the --help command line argument: 1095 .. code-block:: none 1097 assembleCoadd.py --help 1099 To demonstrate usage of the `SafeClipAssembleCoaddTask` in the larger 1100 context of multi-band processing, we will generate the HSC-I & -R band 1101 coadds from HSC engineering test data provided in the ci_hsc package. 1102 To begin, assuming that the lsst stack has been already set up, we must 1103 set up the obs_subaru and ci_hsc packages. This defines the environment 1104 variable $CI_HSC_DIR and points at the location of the package. The raw 1105 HSC data live in the ``$CI_HSC_DIR/raw`` directory. To begin assembling 1106 the coadds, we must first 1109 process the individual ccds in $CI_HSC_RAW to produce calibrated exposures 1111 create a skymap that covers the area of the sky present in the raw exposures 1112 - ``makeCoaddTempExp`` 1113 warp the individual calibrated exposures to the tangent plane of the coadd</DD> 1115 We can perform all of these steps by running 1117 .. code-block:: none 1119 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988 1121 This will produce warped coaddTempExps for each visit. To coadd the 1122 warped data, we call ``assembleCoadd.py`` as follows: 1124 .. code-block:: none 1126 assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \ 1127 --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \ 1128 --selectId visit=903986 ccd=100--selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \ 1129 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \ 1130 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \ 1131 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \ 1132 --selectId visit=903988 ccd=24 1134 This will process the HSC-I band data. The results are written in 1135 ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``. 1137 You may also choose to run: 1139 .. code-block:: none 1141 scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346 nnn 1142 assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R --selectId visit=903334 ccd=16 \ 1143 --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 --selectId visit=903334 ccd=100 \ 1144 --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 --selectId visit=903338 ccd=18 \ 1145 --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 --selectId visit=903342 ccd=10 \ 1146 --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 --selectId visit=903344 ccd=5 \ 1147 --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 --selectId visit=903346 ccd=6 \ 1148 --selectId visit=903346 ccd=12 1150 to generate the coadd for the HSC-R band if you are interested in following 1151 multiBand Coadd processing as discussed in ``pipeTasks_multiBand``. 1153 ConfigClass = SafeClipAssembleCoaddConfig
1154 _DefaultName =
"safeClipAssembleCoadd" 1157 AssembleCoaddTask.__init__(self, *args, **kwargs)
1158 schema = afwTable.SourceTable.makeMinimalSchema()
1159 self.makeSubtask(
"clipDetection", schema=schema)
1161 def assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList, *args, **kwargs):
1162 """Assemble the coadd for a region. 1164 Compute the difference of coadds created with and without outlier 1165 rejection to identify coadd pixels that have outlier values in some 1167 Detect clipped regions on the difference image and mark these regions 1168 on the one or two individual coaddTempExps where they occur if there 1169 is significant overlap between the clipped region and a source. This 1170 leaves us with a set of footprints from the difference image that have 1171 been identified as having occured on just one or two individual visits. 1172 However, these footprints were generated from a difference image. It 1173 is conceivable for a large diffuse source to have become broken up 1174 into multiple footprints acrosss the coadd difference in this process. 1175 Determine the clipped region from all overlapping footprints from the 1176 detected sources in each visit - these are big footprints. 1177 Combine the small and big clipped footprints and mark them on a new 1179 Generate the coadd using `AssembleCoaddTask.assemble` without outlier 1180 removal. Clipped footprints will no longer make it into the coadd 1181 because they are marked in the new bad mask plane. 1185 skyInfo : `lsst.pipe.base.Struct` 1186 Patch geometry information, from getSkyInfo 1187 tempExpRefList : `list` 1188 List of data reference to tempExp 1189 imageScalerList : `list` 1190 List of image scalers 1196 result : `lsst.pipe.base.Struct` 1197 Result struct with components: 1199 - ``coaddExposure``: coadded exposure (``lsst.afw.image.Exposure``). 1200 - ``nImage``: exposure count image (``lsst.afw.image.Image``). 1204 args and kwargs are passed but ignored in order to match the call 1205 signature expected by the parent task. 1208 mask = exp.getMaskedImage().getMask()
1209 mask.addMaskPlane(
"CLIPPED")
1211 result = self.
detectClip(exp, tempExpRefList)
1213 self.log.info(
'Found %d clipped objects', len(result.clipFootprints))
1215 maskClipValue = mask.getPlaneBitMask(
"CLIPPED")
1216 maskDetValue = mask.getPlaneBitMask(
"DETECTED") | mask.getPlaneBitMask(
"DETECTED_NEGATIVE")
1218 bigFootprints = self.
detectClipBig(result.clipSpans, result.clipFootprints, result.clipIndices,
1219 result.detectionFootprints, maskClipValue, maskDetValue,
1222 maskClip = mask.Factory(mask.getBBox(afwImage.PARENT))
1223 afwDet.setMaskFromFootprintList(maskClip, result.clipFootprints, maskClipValue)
1225 maskClipBig = maskClip.Factory(mask.getBBox(afwImage.PARENT))
1226 afwDet.setMaskFromFootprintList(maskClipBig, bigFootprints, maskClipValue)
1227 maskClip |= maskClipBig
1230 badMaskPlanes = self.config.badMaskPlanes[:]
1231 badMaskPlanes.append(
"CLIPPED")
1232 badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes)
1233 return AssembleCoaddTask.assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList,
1234 result.clipSpans, mask=badPixelMask)
1237 """Return an exposure that contains the difference between unclipped 1240 Generate a difference image between clipped and unclipped coadds. 1241 Compute the difference image by subtracting an outlier-clipped coadd 1242 from an outlier-unclipped coadd. Return the difference image. 1246 skyInfo : `lsst.pipe.base.Struct` 1247 Patch geometry information, from getSkyInfo 1248 tempExpRefList : `list` 1249 List of data reference to tempExp 1250 imageScalerList : `list` 1251 List of image scalers 1257 exp : `lsst.afw.image.Exposure` 1258 Difference image of unclipped and clipped coadd wrapped in an Exposure 1263 configIntersection = {k: getattr(self.config, k)
1264 for k, v
in self.config.toDict().items()
if (k
in config.keys())}
1265 config.update(**configIntersection)
1268 config.statistic =
'MEAN' 1270 coaddMean = task.assemble(skyInfo, tempExpRefList, imageScalerList, weightList).coaddExposure
1272 config.statistic =
'MEANCLIP' 1274 coaddClip = task.assemble(skyInfo, tempExpRefList, imageScalerList, weightList).coaddExposure
1276 coaddDiff = coaddMean.getMaskedImage().Factory(coaddMean.getMaskedImage())
1277 coaddDiff -= coaddClip.getMaskedImage()
1278 exp = afwImage.ExposureF(coaddDiff)
1279 exp.setPsf(coaddMean.getPsf())
1283 """Detect clipped regions on an exposure and set the mask on the 1284 individual tempExp masks. 1286 Detect footprints in the difference image after smoothing the 1287 difference image with a Gaussian kernal. Identify footprints that 1288 overlap with one or two input ``coaddTempExps`` by comparing the 1289 computed overlap fraction to thresholds set in the config. A different 1290 threshold is applied depending on the number of overlapping visits 1291 (restricted to one or two). If the overlap exceeds the thresholds, 1292 the footprint is considered "CLIPPED" and is marked as such on the 1293 coaddTempExp. Return a struct with the clipped footprints, the indices 1294 of the ``coaddTempExps`` that end up overlapping with the clipped 1295 footprints, and a list of new masks for the ``coaddTempExps``. 1299 exp : `lsst.afw.image.Exposure` 1300 Exposure to run detection on. 1301 tempExpRefList : `list` 1302 List of data reference to tempExp. 1306 result : `lsst.pipe.base.Struct` 1307 Result struct with components: 1309 - ``clipFootprints``: list of clipped footprints. 1310 - ``clipIndices``: indices for each ``clippedFootprint`` in 1312 - ``clipSpans``: List of dictionaries containing spanSet lists 1313 to clip. Each element contains the new maskplane name 1314 ("CLIPPED") as the key and list of ``SpanSets`` as the value. 1315 - ``detectionFootprints``: List of DETECTED/DETECTED_NEGATIVE plane 1316 compressed into footprints. 1318 mask = exp.getMaskedImage().getMask()
1319 maskDetValue = mask.getPlaneBitMask(
"DETECTED") | mask.getPlaneBitMask(
"DETECTED_NEGATIVE")
1320 fpSet = self.clipDetection.detectFootprints(exp, doSmooth=
True, clearMask=
True)
1322 fpSet.positive.merge(fpSet.negative)
1323 footprints = fpSet.positive
1324 self.log.info(
'Found %d potential clipped objects', len(footprints.getFootprints()))
1329 artifactSpanSets = [{
'CLIPPED': list()}
for _
in tempExpRefList]
1332 visitDetectionFootprints = []
1334 dims = [len(tempExpRefList), len(footprints.getFootprints())]
1335 overlapDetArr = numpy.zeros(dims, dtype=numpy.uint16)
1336 ignoreArr = numpy.zeros(dims, dtype=numpy.uint16)
1339 for i, warpRef
in enumerate(tempExpRefList):
1341 immediate=
True).getMaskedImage().getMask()
1342 maskVisitDet = tmpExpMask.Factory(tmpExpMask, tmpExpMask.getBBox(afwImage.PARENT),
1343 afwImage.PARENT,
True)
1344 maskVisitDet &= maskDetValue
1345 visitFootprints = afwDet.FootprintSet(maskVisitDet, afwDet.Threshold(1))
1346 visitDetectionFootprints.append(visitFootprints)
1348 for j, footprint
in enumerate(footprints.getFootprints()):
1353 for j, footprint
in enumerate(footprints.getFootprints()):
1354 nPixel = footprint.getArea()
1357 for i
in range(len(tempExpRefList)):
1358 ignore = ignoreArr[i, j]
1359 overlapDet = overlapDetArr[i, j]
1360 totPixel = nPixel - ignore
1363 if ignore > overlapDet
or totPixel <= 0.5*nPixel
or overlapDet == 0:
1365 overlap.append(overlapDet/float(totPixel))
1368 overlap = numpy.array(overlap)
1369 if not len(overlap):
1376 if len(overlap) == 1:
1377 if overlap[0] > self.config.minClipFootOverlapSingle:
1382 clipIndex = numpy.where(overlap > self.config.minClipFootOverlap)[0]
1383 if len(clipIndex) == 1:
1385 keepIndex = [clipIndex[0]]
1388 clipIndex = numpy.where(overlap > self.config.minClipFootOverlapDouble)[0]
1389 if len(clipIndex) == 2
and len(overlap) > 3:
1390 clipIndexComp = numpy.where(overlap <= self.config.minClipFootOverlapDouble)[0]
1391 if numpy.max(overlap[clipIndexComp]) <= self.config.maxClipFootOverlapDouble:
1393 keepIndex = clipIndex
1398 for index
in keepIndex:
1399 globalIndex = indexList[index]
1400 artifactSpanSets[globalIndex][
'CLIPPED'].append(footprint.spans)
1402 clipIndices.append(numpy.array(indexList)[keepIndex])
1403 clipFootprints.append(footprint)
1405 return pipeBase.Struct(clipFootprints=clipFootprints, clipIndices=clipIndices,
1406 clipSpans=artifactSpanSets, detectionFootprints=visitDetectionFootprints)
1408 def detectClipBig(self, clipList, clipFootprints, clipIndices, detectionFootprints,
1409 maskClipValue, maskDetValue, coaddBBox):
1410 """Return individual warp footprints for large artifacts and append 1411 them to ``clipList`` in place. 1413 Identify big footprints composed of many sources in the coadd 1414 difference that may have originated in a large diffuse source in the 1415 coadd. We do this by indentifying all clipped footprints that overlap 1416 significantly with each source in all the coaddTempExps. 1421 List of alt mask SpanSets with clipping information. Modified. 1422 clipFootprints : `list` 1423 List of clipped footprints. 1424 clipIndices : `list` 1425 List of which entries in tempExpClipList each footprint belongs to. 1427 Mask value of clipped pixels. 1429 Mask value of detected pixels. 1430 coaddBBox : `lsst.afw.geom.Box` 1431 BBox of the coadd and warps. 1435 bigFootprintsCoadd : `list` 1436 List of big footprints 1438 bigFootprintsCoadd = []
1440 for index, (clippedSpans, visitFootprints)
in enumerate(zip(clipList, detectionFootprints)):
1441 maskVisitDet = afwImage.MaskX(coaddBBox, 0x0)
1442 for footprint
in visitFootprints.getFootprints():
1443 footprint.spans.setMask(maskVisitDet, maskDetValue)
1446 clippedFootprintsVisit = []
1447 for foot, clipIndex
in zip(clipFootprints, clipIndices):
1448 if index
not in clipIndex:
1450 clippedFootprintsVisit.append(foot)
1451 maskVisitClip = maskVisitDet.Factory(maskVisitDet.getBBox(afwImage.PARENT))
1452 afwDet.setMaskFromFootprintList(maskVisitClip, clippedFootprintsVisit, maskClipValue)
1454 bigFootprintsVisit = []
1455 for foot
in visitFootprints.getFootprints():
1456 if foot.getArea() < self.config.minBigOverlap:
1459 if nCount > self.config.minBigOverlap:
1460 bigFootprintsVisit.append(foot)
1461 bigFootprintsCoadd.append(foot)
1463 for footprint
in bigFootprintsVisit:
1464 clippedSpans[
"CLIPPED"].append(footprint.spans)
1466 return bigFootprintsCoadd
1470 assembleStaticSkyModel = pexConfig.ConfigurableField(
1471 target=AssembleCoaddTask,
1472 doc=
"Task to assemble an artifact-free, PSF-matched Coadd to serve as a" 1473 " naive/first-iteration model of the static sky.",
1475 detect = pexConfig.ConfigurableField(
1476 target=SourceDetectionTask,
1477 doc=
"Detect outlier sources on difference between each psfMatched warp and static sky model" 1479 detectTemplate = pexConfig.ConfigurableField(
1480 target=SourceDetectionTask,
1481 doc=
"Detect sources on static sky model. Only used if doPreserveContainedBySource is True" 1483 maxNumEpochs = pexConfig.Field(
1484 doc=
"Charactistic maximum local number of epochs/visits in which an artifact candidate can appear " 1485 "and still be masked. The effective maxNumEpochs is a broken linear function of local " 1486 "number of epochs (N): min(maxFractionEpochsLow*N, maxNumEpochs + maxFractionEpochsHigh*N). " 1487 "For each footprint detected on the image difference between the psfMatched warp and static sky " 1488 "model, if a significant fraction of pixels (defined by spatialThreshold) are residuals in more " 1489 "than the computed effective maxNumEpochs, the artifact candidate is deemed persistant rather " 1490 "than transient and not masked.",
1494 maxFractionEpochsLow = pexConfig.RangeField(
1495 doc=
"Fraction of local number of epochs (N) to use as effective maxNumEpochs for low N. " 1496 "Effective maxNumEpochs = " 1497 "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)",
1502 maxFractionEpochsHigh = pexConfig.RangeField(
1503 doc=
"Fraction of local number of epochs (N) to use as effective maxNumEpochs for high N. " 1504 "Effective maxNumEpochs = " 1505 "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)",
1510 spatialThreshold = pexConfig.RangeField(
1511 doc=
"Unitless fraction of pixels defining how much of the outlier region has to meet the " 1512 "temporal criteria. If 0, clip all. If 1, clip none.",
1516 inclusiveMin=
True, inclusiveMax=
True 1518 doScaleWarpVariance = pexConfig.Field(
1519 doc=
"Rescale Warp variance plane using empirical noise?",
1523 scaleWarpVariance = pexConfig.ConfigurableField(
1524 target=ScaleVarianceTask,
1525 doc=
"Rescale variance on warps",
1527 doPreserveContainedBySource = pexConfig.Field(
1528 doc=
"Rescue artifacts from clipping that completely lie within a footprint detected" 1529 "on the PsfMatched Template Coadd. Replicates a behavior of SafeClip.",
1533 doPrefilterArtifacts = pexConfig.Field(
1534 doc=
"Ignore artifact candidates that are mostly covered by the bad pixel mask, " 1535 "because they will be excluded anyway. This prevents them from contributing " 1536 "to the outlier epoch count image and potentially being labeled as persistant." 1537 "'Mostly' is defined by the config 'prefilterArtifactsRatio'.",
1541 prefilterArtifactsMaskPlanes = pexConfig.ListField(
1542 doc=
"Prefilter artifact candidates that are mostly covered by these bad mask planes.",
1544 default=(
'NO_DATA',
'BAD',
'SAT',
'SUSPECT'),
1546 prefilterArtifactsRatio = pexConfig.Field(
1547 doc=
"Prefilter artifact candidates with less than this fraction overlapping good pixels",
1553 AssembleCoaddConfig.setDefaults(self)
1569 self.
detect.doTempLocalBackground =
False 1570 self.
detect.reEstimateBackground =
False 1571 self.
detect.returnOriginalFootprints =
False 1572 self.
detect.thresholdPolarity =
"both" 1573 self.
detect.thresholdValue = 5
1574 self.
detect.nSigmaToGrow = 2
1575 self.
detect.minPixels = 4
1576 self.
detect.isotropicGrow =
True 1577 self.
detect.thresholdType =
"pixel_stdev" 1585 """Assemble a compareWarp coadded image from a set of warps 1586 by masking artifacts detected by comparing PSF-matched warps. 1588 In ``AssembleCoaddTask``, we compute the coadd as an clipped mean (i.e., 1589 we clip outliers). The problem with doing this is that when computing the 1590 coadd PSF at a given location, individual visit PSFs from visits with 1591 outlier pixels contribute to the coadd PSF and cannot be treated correctly. 1592 In this task, we correct for this behavior by creating a new badMaskPlane 1593 'CLIPPED' which marks pixels in the individual warps suspected to contain 1594 an artifact. We populate this plane on the input warps by comparing 1595 PSF-matched warps with a PSF-matched median coadd which serves as a 1596 model of the static sky. Any group of pixels that deviates from the 1597 PSF-matched template coadd by more than config.detect.threshold sigma, 1598 is an artifact candidate. The candidates are then filtered to remove 1599 variable sources and sources that are difficult to subtract such as 1600 bright stars. This filter is configured using the config parameters 1601 ``temporalThreshold`` and ``spatialThreshold``. The temporalThreshold is 1602 the maximum fraction of epochs that the deviation can appear in and still 1603 be considered an artifact. The spatialThreshold is the maximum fraction of 1604 pixels in the footprint of the deviation that appear in other epochs 1605 (where other epochs is defined by the temporalThreshold). If the deviant 1606 region meets this criteria of having a significant percentage of pixels 1607 that deviate in only a few epochs, these pixels have the 'CLIPPED' bit 1608 set in the mask. These regions will not contribute to the final coadd. 1609 Furthermore, any routine to determine the coadd PSF can now be cognizant 1610 of clipped regions. Note that the algorithm implemented by this task is 1611 preliminary and works correctly for HSC data. Parameter modifications and 1612 or considerable redesigning of the algorithm is likley required for other 1615 ``CompareWarpAssembleCoaddTask`` sub-classes 1616 ``AssembleCoaddTask`` and instantiates ``AssembleCoaddTask`` 1617 as a subtask to generate the TemplateCoadd (the model of the static sky). 1621 The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a 1622 flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``; see 1623 ``baseDebug`` for more about ``debug.py`` files. 1625 This task supports the following debug variables: 1628 If True then save the Epoch Count Image as a fits file in the `figPath` 1630 Path to save the debug fits images and figures 1632 For example, put something like: 1634 .. code-block:: python 1637 def DebugInfo(name): 1638 di = lsstDebug.getInfo(name) 1639 if name == "lsst.pipe.tasks.assembleCoadd": 1640 di.saveCountIm = True 1641 di.figPath = "/desired/path/to/debugging/output/images" 1643 lsstDebug.Info = DebugInfo 1645 into your ``debug.py`` file and run ``assemebleCoadd.py`` with the 1646 ``--debug`` flag. Some subtasks may have their own debug variables; 1647 see individual Task documentation. 1651 ``CompareWarpAssembleCoaddTask`` assembles a set of warped images into a 1652 coadded image. The ``CompareWarpAssembleCoaddTask`` is invoked by running 1653 ``assembleCoadd.py`` with the flag ``--compareWarpCoadd``. 1654 Usage of ``assembleCoadd.py`` expects a data reference to the tract patch 1655 and filter to be coadded (specified using 1656 '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]') 1657 along with a list of coaddTempExps to attempt to coadd (specified using 1658 '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]'). 1659 Only the warps that cover the specified tract and patch will be coadded. 1660 A list of the available optional arguments can be obtained by calling 1661 ``assembleCoadd.py`` with the ``--help`` command line argument: 1663 .. code-block:: none 1665 assembleCoadd.py --help 1667 To demonstrate usage of the ``CompareWarpAssembleCoaddTask`` in the larger 1668 context of multi-band processing, we will generate the HSC-I & -R band 1669 oadds from HSC engineering test data provided in the ``ci_hsc`` package. 1670 To begin, assuming that the lsst stack has been already set up, we must 1671 set up the ``obs_subaru`` and ``ci_hsc`` packages. 1672 This defines the environment variable ``$CI_HSC_DIR`` and points at the 1673 location of the package. The raw HSC data live in the ``$CI_HSC_DIR/raw`` 1674 directory. To begin assembling the coadds, we must first 1677 process the individual ccds in $CI_HSC_RAW to produce calibrated exposures 1679 create a skymap that covers the area of the sky present in the raw exposures 1681 warp the individual calibrated exposures to the tangent plane of the coadd 1683 We can perform all of these steps by running 1685 .. code-block:: none 1687 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988 1689 This will produce warped ``coaddTempExps`` for each visit. To coadd the 1690 warped data, we call ``assembleCoadd.py`` as follows: 1692 .. code-block:: none 1694 assembleCoadd.py --compareWarpCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \ 1695 --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \ 1696 --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \ 1697 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \ 1698 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \ 1699 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \ 1700 --selectId visit=903988 ccd=24 1702 This will process the HSC-I band data. The results are written in 1703 ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``. 1705 ConfigClass = CompareWarpAssembleCoaddConfig
1706 _DefaultName =
"compareWarpAssembleCoadd" 1709 AssembleCoaddTask.__init__(self, *args, **kwargs)
1710 self.makeSubtask(
"assembleStaticSkyModel")
1711 detectionSchema = afwTable.SourceTable.makeMinimalSchema()
1712 self.makeSubtask(
"detect", schema=detectionSchema)
1713 if self.config.doPreserveContainedBySource:
1714 self.makeSubtask(
"detectTemplate", schema=afwTable.SourceTable.makeMinimalSchema())
1715 if self.config.doScaleWarpVariance:
1716 self.makeSubtask(
"scaleWarpVariance")
1719 """Make inputs specific to Subclass. 1721 Generate a templateCoadd to use as a native model of static sky to 1722 subtract from warps. 1726 dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef` 1727 Butler dataRef for supplementary data. 1728 selectDataList : `list` 1729 List of data references to Warps. 1733 result : `lsst.pipe.base.Struct` 1734 Result struct with components: 1736 - ``templateCoaddcoadd``: coadded exposure (``lsst.afw.image.Exposure``). 1738 templateCoadd = self.assembleStaticSkyModel.
run(dataRef, selectDataList)
1740 if templateCoadd
is None:
1741 warpName = (self.assembleStaticSkyModel.warpType[0].upper() +
1742 self.assembleStaticSkyModel.warpType[1:])
1743 message =
"""No %(warpName)s warps were found to build the template coadd which is 1744 required to run CompareWarpAssembleCoaddTask. To continue assembling this type of coadd, 1745 first either rerun makeCoaddTempExp with config.make%(warpName)s=True or 1746 coaddDriver with config.makeCoadTempExp.make%(warpName)s=True, before assembleCoadd. 1748 Alternatively, to use another algorithm with existing warps, retarget the CoaddDriverConfig to 1749 another algorithm like: 1751 from lsst.pipe.tasks.assembleCoadd import SafeClipAssembleCoaddTask 1752 config.assemble.retarget(SafeClipAssembleCoaddTask) 1753 """ % {
"warpName": warpName}
1754 raise RuntimeError(message)
1756 return pipeBase.Struct(templateCoadd=templateCoadd.coaddExposure)
1758 def assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList,
1759 supplementaryData, *args, **kwargs):
1760 """Assemble the coadd. 1762 Find artifacts and apply them to the warps' masks creating a list of 1763 alternative masks with a new "CLIPPED" plane and updated "NO_DATA" 1764 plane. Then pass these alternative masks to the base class's assemble 1769 skyInfo : `lsst.pipe.base.Struct` 1770 Patch geometry information. 1771 tempExpRefList : `list` 1772 List of data references to warps. 1773 imageScalerList : `list` 1774 List of image scalers. 1777 supplementaryData : `lsst.pipe.base.Struct` 1778 This Struct must contain a ``templateCoadd`` that serves as the 1779 model of the static sky. 1783 result : `lsst.pipe.base.Struct` 1784 Result struct with components: 1786 - ``coaddExposure``: coadded exposure (``lsst.afw.image.Exposure``). 1787 - ``nImage``: exposure count image (``lsst.afw.image.Image``), if requested. 1789 templateCoadd = supplementaryData.templateCoadd
1790 spanSetMaskList = self.
findArtifacts(templateCoadd, tempExpRefList, imageScalerList)
1791 badMaskPlanes = self.config.badMaskPlanes[:]
1792 badMaskPlanes.append(
"CLIPPED")
1793 badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes)
1795 result = AssembleCoaddTask.assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList,
1796 spanSetMaskList, mask=badPixelMask)
1800 self.
applyAltEdgeMask(result.coaddExposure.maskedImage.mask, spanSetMaskList)
1804 """Propagate alt EDGE mask to SENSOR_EDGE AND INEXACT_PSF planes. 1808 mask : `lsst.afw.image.Mask` 1810 altMaskList : `list` 1811 List of Dicts containing ``spanSet`` lists. 1812 Each element contains the new mask plane name (e.g. "CLIPPED 1813 and/or "NO_DATA") as the key, and list of ``SpanSets`` to apply to 1816 maskValue = mask.getPlaneBitMask([
"SENSOR_EDGE",
"INEXACT_PSF"])
1817 for visitMask
in altMaskList:
1818 if "EDGE" in visitMask:
1819 for spanSet
in visitMask[
'EDGE']:
1820 spanSet.clippedTo(mask.getBBox()).setMask(mask, maskValue)
1825 Loop through warps twice. The first loop builds a map with the count 1826 of how many epochs each pixel deviates from the templateCoadd by more 1827 than ``config.chiThreshold`` sigma. The second loop takes each 1828 difference image and filters the artifacts detected in each using 1829 count map to filter out variable sources and sources that are 1830 difficult to subtract cleanly. 1834 templateCoadd : `lsst.afw.image.Exposure` 1835 Exposure to serve as model of static sky. 1836 tempExpRefList : `list` 1837 List of data references to warps. 1838 imageScalerList : `list` 1839 List of image scalers. 1844 List of dicts containing information about CLIPPED 1845 (i.e., artifacts), NO_DATA, and EDGE pixels. 1848 self.log.debug(
"Generating Count Image, and mask lists.")
1849 coaddBBox = templateCoadd.getBBox()
1850 slateIm = afwImage.ImageU(coaddBBox)
1851 epochCountImage = afwImage.ImageU(coaddBBox)
1852 nImage = afwImage.ImageU(coaddBBox)
1853 spanSetArtifactList = []
1854 spanSetNoDataMaskList = []
1855 spanSetEdgeList = []
1859 templateCoadd.mask.clearAllMaskPlanes()
1861 if self.config.doPreserveContainedBySource:
1862 templateFootprints = self.detectTemplate.detectFootprints(templateCoadd)
1864 templateFootprints =
None 1866 for warpRef, imageScaler
in zip(tempExpRefList, imageScalerList):
1868 if warpDiffExp
is not None:
1870 nImage.array += (numpy.isfinite(warpDiffExp.image.array) *
1871 ((warpDiffExp.mask.array & badPixelMask) == 0)).astype(numpy.uint16)
1872 fpSet = self.detect.detectFootprints(warpDiffExp, doSmooth=
False, clearMask=
True)
1873 fpSet.positive.merge(fpSet.negative)
1874 footprints = fpSet.positive
1876 spanSetList = [footprint.spans
for footprint
in footprints.getFootprints()]
1879 if self.config.doPrefilterArtifacts:
1881 for spans
in spanSetList:
1882 spans.setImage(slateIm, 1, doClip=
True)
1883 epochCountImage += slateIm
1889 nans = numpy.where(numpy.isnan(warpDiffExp.maskedImage.image.array), 1, 0)
1890 nansMask = afwImage.makeMaskFromArray(nans.astype(afwImage.MaskPixel))
1891 nansMask.setXY0(warpDiffExp.getXY0())
1892 edgeMask = warpDiffExp.mask
1893 spanSetEdgeMask = afwGeom.SpanSet.fromMask(edgeMask,
1894 edgeMask.getPlaneBitMask(
"EDGE")).split()
1898 nansMask = afwImage.MaskX(coaddBBox, 1)
1900 spanSetEdgeMask = []
1902 spanSetNoDataMask = afwGeom.SpanSet.fromMask(nansMask).split()
1904 spanSetNoDataMaskList.append(spanSetNoDataMask)
1905 spanSetArtifactList.append(spanSetList)
1906 spanSetEdgeList.append(spanSetEdgeMask)
1910 epochCountImage.writeFits(path)
1912 for i, spanSetList
in enumerate(spanSetArtifactList):
1914 filteredSpanSetList = self.
filterArtifacts(spanSetList, epochCountImage, nImage,
1916 spanSetArtifactList[i] = filteredSpanSetList
1919 for artifacts, noData, edge
in zip(spanSetArtifactList, spanSetNoDataMaskList, spanSetEdgeList):
1920 altMasks.append({
'CLIPPED': artifacts,
1926 """Remove artifact candidates covered by bad mask plane. 1928 Any future editing of the candidate list that does not depend on 1929 temporal information should go in this method. 1933 spanSetList : `list` 1934 List of SpanSets representing artifact candidates. 1935 exp : `lsst.afw.image.Exposure` 1936 Exposure containing mask planes used to prefilter. 1940 returnSpanSetList : `list` 1941 List of SpanSets with artifacts. 1943 badPixelMask = exp.mask.getPlaneBitMask(self.config.prefilterArtifactsMaskPlanes)
1944 goodArr = (exp.mask.array & badPixelMask) == 0
1945 returnSpanSetList = []
1946 bbox = exp.getBBox()
1947 x0, y0 = exp.getXY0()
1948 for i, span
in enumerate(spanSetList):
1949 y, x = span.clippedTo(bbox).indices()
1950 yIndexLocal = numpy.array(y) - y0
1951 xIndexLocal = numpy.array(x) - x0
1952 goodRatio = numpy.count_nonzero(goodArr[yIndexLocal, xIndexLocal])/span.getArea()
1953 if goodRatio > self.config.prefilterArtifactsRatio:
1954 returnSpanSetList.append(span)
1955 return returnSpanSetList
1957 def filterArtifacts(self, spanSetList, epochCountImage, nImage, footprintsToExclude=None):
1958 """Filter artifact candidates. 1962 spanSetList : `list` 1963 List of SpanSets representing artifact candidates. 1964 epochCountImage : `lsst.afw.image.Image` 1965 Image of accumulated number of warpDiff detections. 1966 nImage : `lsst.afw.image.Image` 1967 Image of the accumulated number of total epochs contributing. 1971 maskSpanSetList : `list` 1972 List of SpanSets with artifacts. 1975 maskSpanSetList = []
1976 x0, y0 = epochCountImage.getXY0()
1977 for i, span
in enumerate(spanSetList):
1978 y, x = span.indices()
1979 yIdxLocal = [y1 - y0
for y1
in y]
1980 xIdxLocal = [x1 - x0
for x1
in x]
1981 outlierN = epochCountImage.array[yIdxLocal, xIdxLocal]
1982 totalN = nImage.array[yIdxLocal, xIdxLocal]
1985 effMaxNumEpochsHighN = (self.config.maxNumEpochs +
1986 self.config.maxFractionEpochsHigh*numpy.mean(totalN))
1987 effMaxNumEpochsLowN = self.config.maxFractionEpochsLow * numpy.mean(totalN)
1988 effectiveMaxNumEpochs = int(min(effMaxNumEpochsLowN, effMaxNumEpochsHighN))
1989 nPixelsBelowThreshold = numpy.count_nonzero((outlierN > 0) &
1990 (outlierN <= effectiveMaxNumEpochs))
1991 percentBelowThreshold = nPixelsBelowThreshold / len(outlierN)
1992 if percentBelowThreshold > self.config.spatialThreshold:
1993 maskSpanSetList.append(span)
1995 if self.config.doPreserveContainedBySource
and footprintsToExclude
is not None:
1997 filteredMaskSpanSetList = []
1998 for span
in maskSpanSetList:
2000 for footprint
in footprintsToExclude.positive.getFootprints():
2001 if footprint.spans.contains(span):
2005 filteredMaskSpanSetList.append(span)
2006 maskSpanSetList = filteredMaskSpanSetList
2008 return maskSpanSetList
2010 def _readAndComputeWarpDiff(self, warpRef, imageScaler, templateCoadd):
2011 """Fetch a warp from the butler and return a warpDiff. 2015 warpRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef` 2016 Butler dataRef for the warp. 2017 imageScaler : `lsst.pipe.tasks.scaleZeroPoint.ImageScaler` 2018 An image scaler object. 2019 templateCoadd : `lsst.afw.image.Exposure` 2020 Exposure to be substracted from the scaled warp. 2024 warp : `lsst.afw.image.Exposure` 2025 Exposure of the image difference between the warp and template. 2030 if not warpRef.datasetExists(warpName):
2031 self.log.warn(
"Could not find %s %s; skipping it", warpName, warpRef.dataId)
2033 warp = warpRef.get(warpName, immediate=
True)
2035 imageScaler.scaleMaskedImage(warp.getMaskedImage())
2036 mi = warp.getMaskedImage()
2037 if self.config.doScaleWarpVariance:
2039 self.scaleWarpVariance.
run(mi)
2040 except Exception
as exc:
2041 self.log.warn(
"Unable to rescale variance of warp (%s); leaving it as-is" % (exc,))
2042 mi -= templateCoadd.getMaskedImage()
2045 def _dataRef2DebugPath(self, prefix, warpRef, coaddLevel=False):
2046 """Return a path to which to write debugging output. 2048 Creates a hyphen-delimited string of dataId values for simple filenames. 2053 Prefix for filename. 2054 warpRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef` 2055 Butler dataRef to make the path from. 2056 coaddLevel : `bool`, optional. 2057 If True, include only coadd-level keys (e.g., 'tract', 'patch', 2058 'filter', but no 'visit'). 2063 Path for debugging output. 2068 keys = warpRef.dataId.keys()
2069 keyList = sorted(keys, reverse=
True)
2071 filename =
"%s-%s.fits" % (prefix,
'-'.join([str(warpRef.dataId[k])
for k
in keyList]))
2072 return os.path.join(directory, filename)
def setBrightObjectMasks(self, exposure, dataId, brightObjectMasks)
def shrinkValidPolygons(self, coaddInputs)
def getCoaddDatasetName(self, warpType="direct")
def _dataRef2DebugPath(self, prefix, warpRef, coaddLevel=False)
def getGroupDataRef(butler, datasetType, groupTuple, keys)
Base class for coaddition.
def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList)
def assembleMetadata(self, coaddExposure, tempExpRefList, weightList)
def makeSupplementaryData(self, dataRef, selectDataList)
def makeSupplementaryData(self, dataRef, selectDataList)
def getTempExpRefList(self, patchRef, calExpRefList)
def assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList, args, kwargs)
def run(self, dataRef, selectDataList=[])
def _readAndComputeWarpDiff(self, warpRef, imageScaler, templateCoadd)
def prepareInputs(self, refList)
def applyAltMaskPlanes(self, mask, altMaskSpans)
def getSkyInfo(self, patchRef)
Use getSkyinfo to return the skyMap, tract and patch information, wcs and the outer bbox of the patch...
def getTempExpDatasetName(self, warpType="direct")
def __init__(self, args, kwargs)
def makeDataRefList(self, namespace)
def getBadPixelMask(self)
Convenience method to provide the bitmask from the mask plane names.
def assembleSubregion(self, coaddExposure, bbox, tempExpRefList, imageScalerList, weightList, altMaskList, statsFlags, statsCtrl, nImage=None)
def detectClip(self, exp, tempExpRefList)
def setInexactPsf(self, mask)
def __init__(self, args, kwargs)
def filterArtifacts(self, spanSetList, epochCountImage, nImage, footprintsToExclude=None)
def buildDifferenceImage(self, skyInfo, tempExpRefList, imageScalerList, weightList)
def selectExposures(self, patchRef, skyInfo=None, selectDataList=[])
Select exposures to coadd.
def applyAltEdgeMask(self, mask, altMaskList)
def readBrightObjectMasks(self, dataRef)
def __init__(self, args, kwargs)
def assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList, supplementaryData, args, kwargs)
def prefilterArtifacts(self, spanSetList, exp)
def countMaskFromFootprint(mask, footprint, bitmask, ignoreMask)
def groupPatchExposures(patchDataRef, calexpDataRefList, coaddDatasetType="deepCoadd", tempExpDatasetType="deepCoadd_directWarp")
def assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList, altMaskList=None, mask=None, supplementaryData=None)
def detectClipBig(self, clipList, clipFootprints, clipIndices, detectionFootprints, maskClipValue, maskDetValue, coaddBBox)