36 from .coaddBase
import CoaddBaseTask, SelectDataIdContainer
37 from .interpImage
import InterpImageTask
38 from .scaleZeroPoint
import ScaleZeroPointTask
39 from .coaddHelpers
import groupPatchExposures, getGroupDataRef
40 from .scaleVariance
import ScaleVarianceTask
43 __all__ = [
"AssembleCoaddTask",
"SafeClipAssembleCoaddTask",
"CompareWarpAssembleCoaddTask"]
48 \anchor AssembleCoaddConfig_ 50 \brief Configuration parameters for the \ref AssembleCoaddTask_ "AssembleCoaddTask" 52 warpType = pexConfig.Field(
53 doc=
"Warp name: one of 'direct' or 'psfMatched'",
57 subregionSize = pexConfig.ListField(
59 doc=
"Width, height of stack subregion size; " 60 "make small enough that a full stack of images will fit into memory at once.",
64 statistic = pexConfig.Field(
66 doc=
"Main stacking statistic for aggregating over the epochs.",
69 doSigmaClip = pexConfig.Field(
71 doc=
"Perform sigma clipped outlier rejection with MEANCLIP statistic? (DEPRECATED)",
74 sigmaClip = pexConfig.Field(
76 doc=
"Sigma for outlier rejection; ignored if non-clipping statistic selected.",
79 clipIter = pexConfig.Field(
81 doc=
"Number of iterations of outlier rejection; ignored if non-clipping statistic selected.",
84 calcErrorFromInputVariance = pexConfig.Field(
86 doc=
"Calculate coadd variance from input variance by stacking statistic." 87 "Passed to StatisticsControl.setCalcErrorFromInputVariance()",
90 scaleZeroPoint = pexConfig.ConfigurableField(
91 target=ScaleZeroPointTask,
92 doc=
"Task to adjust the photometric zero point of the coadd temp exposures",
94 doInterp = pexConfig.Field(
95 doc=
"Interpolate over NaN pixels? Also extrapolate, if necessary, but the results are ugly.",
99 interpImage = pexConfig.ConfigurableField(
100 target=InterpImageTask,
101 doc=
"Task to interpolate (and extrapolate) over NaN pixels",
103 doWrite = pexConfig.Field(
104 doc=
"Persist coadd?",
108 doNImage = pexConfig.Field(
109 doc=
"Create image of number of contributing exposures for each pixel",
113 doUsePsfMatchedPolygons = pexConfig.Field(
114 doc=
"Use ValidPolygons from shrunk Psf-Matched Calexps? Should be set to True by CompareWarp only.",
118 maskPropagationThresholds = pexConfig.DictField(
121 doc=(
"Threshold (in fractional weight) of rejection at which we propagate a mask plane to " 122 "the coadd; that is, we set the mask bit on the coadd if the fraction the rejected frames " 123 "would have contributed exceeds this value."),
124 default={
"SAT": 0.1},
126 removeMaskPlanes = pexConfig.ListField(dtype=str, default=[
"NOT_DEBLENDED"],
127 doc=
"Mask planes to remove before coadding")
136 doMaskBrightObjects = pexConfig.Field(dtype=bool, default=
False,
137 doc=
"Set mask and flag bits for bright objects?")
138 brightObjectMaskName = pexConfig.Field(dtype=str, default=
"BRIGHT_OBJECT",
139 doc=
"Name of mask bit used for bright objects")
141 coaddPsf = pexConfig.ConfigField(
142 doc=
"Configuration for CoaddPsf",
143 dtype=measAlg.CoaddPsfConfig,
145 doAttachTransmissionCurve = pexConfig.Field(
146 dtype=bool, default=
False, optional=
False,
147 doc=(
"Attach a piecewise TransmissionCurve for the coadd? " 148 "(requires all input Exposures to have TransmissionCurves).")
152 CoaddBaseTask.ConfigClass.setDefaults(self)
156 CoaddBaseTask.ConfigClass.validate(self)
160 log.warn(
"Config doPsfMatch deprecated. Setting warpType='psfMatched'")
163 log.warn(
'doSigmaClip deprecated. To replicate behavior, setting statistic to "MEANCLIP"')
165 if self.
doInterp and self.
statistic not in [
'MEAN',
'MEDIAN',
'MEANCLIP',
'VARIANCE',
'VARIANCECLIP']:
166 raise ValueError(
"Must set doInterp=False for statistic=%s, which does not " 167 "compute and set a non-zero coadd variance estimate." % (self.
statistic))
169 unstackableStats = [
'NOTHING',
'ERROR',
'ORMASK']
170 if not hasattr(afwMath.Property, self.
statistic)
or self.
statistic in unstackableStats:
171 stackableStats = [str(k)
for k
in afwMath.Property.__members__.keys()
172 if str(k)
not in unstackableStats]
173 raise ValueError(
"statistic %s is not allowed. Please choose one of %s." 185 \anchor AssembleCoaddTask_ 187 \brief Assemble a coadded image from a set of warps (coadded temporary exposures). 189 \section pipe_tasks_assembleCoadd_Contents Contents 190 - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Purpose 191 - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Initialize 192 - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Run 193 - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Config 194 - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Debug 195 - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Example 197 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Purpose Description 199 \copybrief AssembleCoaddTask_ 201 We want to assemble a coadded image from a set of Warps (also called 202 coadded temporary exposures or coaddTempExps. 203 Each input Warp covers a patch on the sky and corresponds to a single run/visit/exposure of the 204 covered patch. We provide the task with a list of Warps (selectDataList) from which it selects 205 Warps that cover the specified patch (pointed at by dataRef). 206 Each Warp that goes into a coadd will typically have an independent photometric zero-point. 207 Therefore, we must scale each Warp to set it to a common photometric zeropoint. 208 WarpType may be one of 'direct' or 'psfMatched', and the boolean configs config.makeDirect and 209 config.makePsfMatched set which of the warp types will be coadded. 210 The coadd is computed as a mean with optional outlier rejection. 211 Criteria for outlier rejection are set in \ref AssembleCoaddConfig. Finally, Warps can have bad 'NaN' 212 pixels which received no input from the source calExps. We interpolate over these bad (NaN) pixels. 214 AssembleCoaddTask uses several sub-tasks. These are 216 <DT>\ref ScaleZeroPointTask_ "ScaleZeroPointTask"</DT> 217 <DD> create and use an imageScaler object to scale the photometric zeropoint for each Warp</DD> 218 <DT>\ref InterpImageTask_ "InterpImageTask"</DT> 219 <DD>interpolate across bad pixels (NaN) in the final coadd</DD> 221 You can retarget these subtasks if you wish. 223 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Initialize Task initialization 224 \copydoc \_\_init\_\_ 226 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Run Invoking the Task 229 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Config Configuration parameters 230 See \ref AssembleCoaddConfig_ 232 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Debug Debug variables 233 The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a 234 flag \c -d to import \b debug.py from your \c PYTHONPATH; see \ref baseDebug for more about \b debug.py files. 235 AssembleCoaddTask has no debug variables of its own. Some of the subtasks may support debug variables. See 236 the documetation for the subtasks for further information. 238 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Example A complete example of using AssembleCoaddTask 240 AssembleCoaddTask assembles a set of warped images into a coadded image. The AssembleCoaddTask 241 can be invoked by running assembleCoadd.py with the flag '--legacyCoadd'. Usage of assembleCoadd.py expects 242 a data reference to the tract patch and filter to be coadded (specified using 243 '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]') along with a list of 244 Warps to attempt to coadd (specified using 245 '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]'). Only the Warps 246 that cover the specified tract and patch will be coadded. A list of the available optional 247 arguments can be obtained by calling assembleCoadd.py with the --help command line argument: 249 assembleCoadd.py --help 251 To demonstrate usage of the AssembleCoaddTask in the larger context of multi-band processing, we will generate 252 the HSC-I & -R band coadds from HSC engineering test data provided in the ci_hsc package. To begin, assuming 253 that the lsst stack has been already set up, we must set up the obs_subaru and ci_hsc packages. 254 This defines the environment variable $CI_HSC_DIR and points at the location of the package. The raw HSC 255 data live in the $CI_HSC_DIR/raw directory. To begin assembling the coadds, we must first 258 <DD> process the individual ccds in $CI_HSC_RAW to produce calibrated exposures</DD> 260 <DD> create a skymap that covers the area of the sky present in the raw exposures</DD> 261 <DT>makeCoaddTempExp</DT> 262 <DD> warp the individual calibrated exposures to the tangent plane of the coadd</DD> 264 We can perform all of these steps by running 266 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988 268 This will produce warped exposures for each visit. To coadd the warped data, we call assembleCoadd.py as 271 assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \ 272 --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \ 273 --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \ 274 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \ 275 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \ 276 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \ 277 --selectId visit=903988 ccd=24 279 that will process the HSC-I band data. The results are written in 280 `$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I`. 282 You may also choose to run: 284 scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346 285 assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R \ 286 --selectId visit=903334 ccd=16 --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 \ 287 --selectId visit=903334 ccd=100 --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 \ 288 --selectId visit=903338 ccd=18 --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 \ 289 --selectId visit=903342 ccd=10 --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 \ 290 --selectId visit=903344 ccd=5 --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 \ 291 --selectId visit=903346 ccd=6 --selectId visit=903346 ccd=12 293 to generate the coadd for the HSC-R band if you are interested in following multiBand Coadd processing as 294 discussed in \ref pipeTasks_multiBand (but note that normally, one would use the 295 \ref SafeClipAssembleCoaddTask_ "SafeClipAssembleCoaddTask" rather than AssembleCoaddTask to make the coadd. 297 ConfigClass = AssembleCoaddConfig
298 _DefaultName =
"assembleCoadd" 302 \brief Initialize the task. Create the \ref InterpImageTask "interpImage", 303 & \ref ScaleZeroPointTask "scaleZeroPoint" subtasks. 305 CoaddBaseTask.__init__(self, *args, **kwargs)
306 self.makeSubtask(
"interpImage")
307 self.makeSubtask(
"scaleZeroPoint")
309 if self.config.doMaskBrightObjects:
310 mask = afwImage.Mask()
313 except pexExceptions.LsstCppException:
314 raise RuntimeError(
"Unable to define mask plane for bright objects; planes used are %s" %
315 mask.getMaskPlaneDict().keys())
321 def run(self, dataRef, selectDataList=[]):
323 \brief Assemble a coadd from a set of Warps 325 Coadd a set of Warps. Compute weights to be applied to each Warp and find scalings to 326 match the photometric zeropoint to a reference Warp. Assemble the Warps using 327 \ref assemble. Interpolate over NaNs and optionally write the coadd to disk. Return the coadded 331 \param[in] dataRef: Data reference defining the patch for coaddition and the reference Warp 332 (if config.autoReference=False). Used to access the following data products: 333 - [in] self.config.coaddName + "Coadd_skyMap" 334 - [in] self.config.coaddName + "Coadd_ + <warpType> + "Warp" (optionally) 335 - [out] self.config.coaddName + "Coadd" 336 \param[in] selectDataList[in]: List of data references to Warps. Data to be coadded will be 337 selected from this list based on overlap with the patch defined by dataRef. 339 \return a pipeBase.Struct with fields: 340 - coaddExposure: coadded exposure 341 - nImage: exposure count image 344 calExpRefList = self.
selectExposures(dataRef, skyInfo, selectDataList=selectDataList)
345 if len(calExpRefList) == 0:
346 self.log.warn(
"No exposures to coadd")
348 self.log.info(
"Coadding %d exposures", len(calExpRefList))
352 self.log.info(
"Found %d %s", len(inputData.tempExpRefList),
354 if len(inputData.tempExpRefList) == 0:
355 self.log.warn(
"No coadd temporary exposures found")
360 retStruct = self.
assemble(skyInfo, inputData.tempExpRefList, inputData.imageScalerList,
361 inputData.weightList, supplementaryData=supplementaryData)
363 if self.config.doInterp:
364 self.interpImage.
run(retStruct.coaddExposure.getMaskedImage(), planeName=
"NO_DATA")
366 varArray = retStruct.coaddExposure.getMaskedImage().getVariance().getArray()
367 with numpy.errstate(invalid=
"ignore"):
368 varArray[:] = numpy.where(varArray > 0, varArray, numpy.inf)
370 if self.config.doMaskBrightObjects:
374 if self.config.doWrite:
377 if self.config.doNImage
and retStruct.nImage
is not None:
384 \brief Make additional inputs to assemble() specific to subclasses. 386 Available to be implemented by subclasses only if they need the 387 coadd dataRef for performing preliminary processing before 388 assembling the coadd. 394 \brief Generate list data references corresponding to warped exposures that lie within the 397 \param[in] patchRef: Data reference for patch 398 \param[in] calExpRefList: List of data references for input calexps 399 \return List of Warp/CoaddTempExp data references 401 butler = patchRef.getButler()
402 groupData =
groupPatchExposures(patchRef, calExpRefList, self.getCoaddDatasetName(self.warpType),
403 self.getTempExpDatasetName(self.warpType))
404 tempExpRefList = [
getGroupDataRef(butler, self.getTempExpDatasetName(self.warpType),
405 g, groupData.keys)
for 406 g
in groupData.groups.keys()]
407 return tempExpRefList
411 \brief Prepare the input warps for coaddition by measuring the weight for each warp and the scaling 412 for the photometric zero point. 414 Each Warp has its own photometric zeropoint and background variance. Before coadding these 415 Warps together, compute a scale factor to normalize the photometric zeropoint and compute the 416 weight for each Warp. 418 \param[in] refList: List of data references to tempExp 420 - tempExprefList: List of data references to tempExp 421 - weightList: List of weightings 422 - imageScalerList: List of image scalers 424 statsCtrl = afwMath.StatisticsControl()
425 statsCtrl.setNumSigmaClip(self.config.sigmaClip)
426 statsCtrl.setNumIter(self.config.clipIter)
428 statsCtrl.setNanSafe(
True)
436 for tempExpRef
in refList:
437 if not tempExpRef.datasetExists(tempExpName):
438 self.log.warn(
"Could not find %s %s; skipping it", tempExpName, tempExpRef.dataId)
441 tempExp = tempExpRef.get(tempExpName, immediate=
True)
442 maskedImage = tempExp.getMaskedImage()
443 imageScaler = self.scaleZeroPoint.computeImageScaler(
448 imageScaler.scaleMaskedImage(maskedImage)
449 except Exception
as e:
450 self.log.warn(
"Scaling failed for %s (skipping it): %s", tempExpRef.dataId, e)
452 statObj = afwMath.makeStatistics(maskedImage.getVariance(), maskedImage.getMask(),
453 afwMath.MEANCLIP, statsCtrl)
454 meanVar, meanVarErr = statObj.getResult(afwMath.MEANCLIP)
455 weight = 1.0 / float(meanVar)
456 if not numpy.isfinite(weight):
457 self.log.warn(
"Non-finite weight for %s: skipping", tempExpRef.dataId)
459 self.log.info(
"Weight of %s %s = %0.3f", tempExpName, tempExpRef.dataId, weight)
464 tempExpRefList.append(tempExpRef)
465 weightList.append(weight)
466 imageScalerList.append(imageScaler)
468 return pipeBase.Struct(tempExpRefList=tempExpRefList, weightList=weightList,
469 imageScalerList=imageScalerList)
471 def assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList,
472 altMaskList=None, mask=None, supplementaryData=None):
474 \anchor AssembleCoaddTask.assemble_ 476 \brief Assemble a coadd from input warps 478 Assemble the coadd using the provided list of coaddTempExps. Since the full coadd covers a patch (a 479 large area), the assembly is performed over small areas on the image at a time in order to 480 conserve memory usage. Iterate over subregions within the outer bbox of the patch using 481 \ref assembleSubregion to stack the corresponding subregions from the coaddTempExps with the 482 statistic specified. Set the edge bits the coadd mask based on the weight map. 484 \param[in] skyInfo: Patch geometry information, from getSkyInfo 485 \param[in] tempExpRefList: List of data references to Warps (previously called CoaddTempExps) 486 \param[in] imageScalerList: List of image scalers 487 \param[in] weightList: List of weights 488 \param[in] altMaskList: List of alternate masks to use rather than those stored with tempExp, or None 489 \param[in] mask: Mask to ignore when coadding 490 \param[in] supplementaryData: pipeBase.Struct with additional data products needed to assemble coadd. 491 Only used by subclasses that implement makeSupplementaryData and override assemble. 492 \return pipeBase.Struct with coaddExposure, nImage if requested 495 self.log.info(
"Assembling %s %s", len(tempExpRefList), tempExpName)
499 statsCtrl = afwMath.StatisticsControl()
500 statsCtrl.setNumSigmaClip(self.config.sigmaClip)
501 statsCtrl.setNumIter(self.config.clipIter)
502 statsCtrl.setAndMask(mask)
503 statsCtrl.setNanSafe(
True)
504 statsCtrl.setWeighted(
True)
505 statsCtrl.setCalcErrorFromInputVariance(self.config.calcErrorFromInputVariance)
506 for plane, threshold
in self.config.maskPropagationThresholds.items():
507 bit = afwImage.Mask.getMaskPlane(plane)
508 statsCtrl.setMaskPropagationThreshold(bit, threshold)
510 statsFlags = afwMath.stringToStatisticsProperty(self.config.statistic)
512 if altMaskList
is None:
513 altMaskList = [
None]*len(tempExpRefList)
515 coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
516 coaddExposure.setCalib(self.scaleZeroPoint.getCalib())
517 coaddExposure.getInfo().setCoaddInputs(self.inputRecorder.makeCoaddInputs())
519 coaddMaskedImage = coaddExposure.getMaskedImage()
520 subregionSizeArr = self.config.subregionSize
521 subregionSize = afwGeom.Extent2I(subregionSizeArr[0], subregionSizeArr[1])
523 if self.config.doNImage:
524 nImage = afwImage.ImageU(skyInfo.bbox)
527 for subBBox
in _subBBoxIter(skyInfo.bbox, subregionSize):
530 weightList, altMaskList, statsFlags, statsCtrl,
532 except Exception
as e:
533 self.log.fatal(
"Cannot compute coadd %s: %s", subBBox, e)
538 coaddUtils.setCoaddEdgeBits(coaddMaskedImage.getMask(), coaddMaskedImage.getVariance())
539 return pipeBase.Struct(coaddExposure=coaddExposure, nImage=nImage)
543 \brief Set the metadata for the coadd 545 This basic implementation simply sets the filter from the 548 \param[in] coaddExposure: The target image for the coadd 549 \param[in] tempExpRefList: List of data references to tempExp 550 \param[in] weightList: List of weights 552 assert len(tempExpRefList) == len(weightList),
"Length mismatch" 557 tempExpList = [tempExpRef.get(tempExpName +
"_sub",
558 bbox=afwGeom.Box2I(afwGeom.Point2I(0, 0), afwGeom.Extent2I(1, 1)),
559 imageOrigin=
"LOCAL", immediate=
True)
for tempExpRef
in tempExpRefList]
560 numCcds = sum(len(tempExp.getInfo().getCoaddInputs().ccds)
for tempExp
in tempExpList)
562 coaddExposure.setFilter(tempExpList[0].getFilter())
563 coaddInputs = coaddExposure.getInfo().getCoaddInputs()
564 coaddInputs.ccds.reserve(numCcds)
565 coaddInputs.visits.reserve(len(tempExpList))
567 for tempExp, weight
in zip(tempExpList, weightList):
568 self.inputRecorder.addVisitToCoadd(coaddInputs, tempExp, weight)
570 if self.config.doUsePsfMatchedPolygons:
573 coaddInputs.visits.sort()
579 modelPsfList = [tempExp.getPsf()
for tempExp
in tempExpList]
580 modelPsfWidthList = [modelPsf.computeBBox().getWidth()
for modelPsf
in modelPsfList]
581 psf = modelPsfList[modelPsfWidthList.index(max(modelPsfWidthList))]
583 psf = measAlg.CoaddPsf(coaddInputs.ccds, coaddExposure.getWcs(),
584 self.config.coaddPsf.makeControl())
585 coaddExposure.setPsf(psf)
586 apCorrMap = measAlg.makeCoaddApCorrMap(coaddInputs.ccds, coaddExposure.getBBox(afwImage.PARENT),
587 coaddExposure.getWcs())
588 coaddExposure.getInfo().setApCorrMap(apCorrMap)
589 if self.config.doAttachTransmissionCurve:
590 transmissionCurve = measAlg.makeCoaddTransmissionCurve(coaddExposure.getWcs(), coaddInputs.ccds)
591 coaddExposure.getInfo().setTransmissionCurve(transmissionCurve)
593 def assembleSubregion(self, coaddExposure, bbox, tempExpRefList, imageScalerList, weightList,
594 altMaskList, statsFlags, statsCtrl, nImage=None):
596 \brief Assemble the coadd for a sub-region. 598 For each coaddTempExp, check for (and swap in) an alternative mask if one is passed. Remove mask 599 planes listed in config.removeMaskPlanes, Finally, stack the actual exposures using 600 \ref afwMath.statisticsStack "statisticsStack" with the statistic specified 601 by statsFlags. Typically, the statsFlag will be one of afwMath.MEAN for a mean-stack or 602 afwMath.MEANCLIP for outlier rejection using an N-sigma clipped mean where N and iterations 603 are specified by statsCtrl. Assign the stacked subregion back to the coadd. 605 \param[in] coaddExposure: The target image for the coadd 606 \param[in] bbox: Sub-region to coadd 607 \param[in] tempExpRefList: List of data reference to tempExp 608 \param[in] imageScalerList: List of image scalers 609 \param[in] weightList: List of weights 610 \param[in] altMaskList: List of alternate masks to use rather than those stored with tempExp, or None 611 Each element is dict with keys = mask plane name to which to add the spans 612 \param[in] statsFlags: afwMath.Property object for statistic for coadd 613 \param[in] statsCtrl: Statistics control object for coadd 614 \param[in] nImage: optional ImageU keeps track of exposure count for each pixel 616 self.log.debug(
"Computing coadd over %s", bbox)
618 coaddExposure.mask.addMaskPlane(
"REJECTED")
619 coaddExposure.mask.addMaskPlane(
"CLIPPED")
620 coaddExposure.mask.addMaskPlane(
"SENSOR_EDGE")
625 edge = afwImage.Mask.getPlaneBitMask(
"EDGE")
626 noData = afwImage.Mask.getPlaneBitMask(
"NO_DATA")
627 clipped = afwImage.Mask.getPlaneBitMask(
"CLIPPED")
628 toReject = statsCtrl.getAndMask() & (~noData) & (~edge) & (~clipped)
629 maskMap = [(toReject, coaddExposure.mask.getPlaneBitMask(
"REJECTED")),
630 (edge, coaddExposure.mask.getPlaneBitMask(
"SENSOR_EDGE")),
633 if nImage
is not None:
634 subNImage = afwImage.ImageU(bbox.getWidth(), bbox.getHeight())
635 for tempExpRef, imageScaler, altMask
in zip(tempExpRefList, imageScalerList, altMaskList):
636 exposure = tempExpRef.get(tempExpName +
"_sub", bbox=bbox)
637 maskedImage = exposure.getMaskedImage()
638 mask = maskedImage.getMask()
639 if altMask
is not None:
641 imageScaler.scaleMaskedImage(maskedImage)
645 if nImage
is not None:
646 subNImage.getArray()[maskedImage.getMask().getArray() & statsCtrl.getAndMask() == 0] += 1
647 if self.config.removeMaskPlanes:
648 mask = maskedImage.getMask()
649 for maskPlane
in self.config.removeMaskPlanes:
651 mask &= ~mask.getPlaneBitMask(maskPlane)
652 except Exception
as e:
653 self.log.warn(
"Unable to remove mask plane %s: %s", maskPlane, e.args[0])
655 maskedImageList.append(maskedImage)
657 with self.timer(
"stack"):
658 coaddSubregion = afwMath.statisticsStack(maskedImageList, statsFlags, statsCtrl, weightList,
661 coaddExposure.maskedImage.assign(coaddSubregion, bbox)
662 if nImage
is not None:
663 nImage.assign(subNImage, bbox)
667 \brief Apply in place alt mask formatted as SpanSets to a mask 669 @param mask: original mask 670 @param altMaskSpans: Dict containing spanSet lists to apply. 671 Each element contains the new mask plane name 672 (e.g. "CLIPPED and/or "NO_DATA") as the key, 673 and list of SpanSets to apply to the mask 675 if self.config.doUsePsfMatchedPolygons:
676 if (
"NO_DATA" in altMaskSpans)
and (
"NO_DATA" in self.config.badMaskPlanes):
681 for spanSet
in altMaskSpans[
'NO_DATA']:
682 spanSet.clippedTo(mask.getBBox()).clearMask(mask, self.
getBadPixelMask())
684 for plane, spanSetList
in altMaskSpans.items():
685 maskClipValue = mask.addMaskPlane(plane)
686 for spanSet
in spanSetList:
687 spanSet.clippedTo(mask.getBBox()).setMask(mask, 2**maskClipValue)
692 \brief Shrink coaddInputs' ccds' ValidPolygons in place 694 @param coaddInputs: original mask 696 Either modify each ccd's validPolygon in place, or if CoaddInputs does not 697 have a validPolygon, create one from its bbox. 699 for ccd
in coaddInputs.ccds:
700 polyOrig = ccd.getValidPolygon()
701 validPolyBBox = polyOrig.getBBox()
if polyOrig
else ccd.getBBox()
702 validPolyBBox.grow(-self.config.matchingKernelSize//2)
704 validPolygon = polyOrig.intersectionSingle(validPolyBBox)
706 validPolygon = afwGeom.polygon.Polygon(afwGeom.Box2D(validPolyBBox))
707 ccd.setValidPolygon(validPolygon)
710 """Returns None on failure""" 712 return dataRef.get(
"brightObjectMask", immediate=
True)
713 except Exception
as e:
714 self.log.warn(
"Unable to read brightObjectMask for %s: %s", dataRef.dataId, e)
718 """Set the bright object masks 720 exposure: Exposure under consideration 721 dataId: Data identifier dict for patch 722 brightObjectMasks: afwTable of bright objects to mask 727 if brightObjectMasks
is None:
728 self.log.warn(
"Unable to apply bright object mask: none supplied")
730 self.log.info(
"Applying %d bright object masks to %s", len(brightObjectMasks), dataId)
731 md = brightObjectMasks.table.getMetadata()
734 self.log.warn(
"Expected to see %s in metadata", k)
736 if md.get(k) != dataId[k]:
737 self.log.warn(
"Expected to see %s == %s in metadata, saw %s", k, md.get(k), dataId[k])
739 mask = exposure.getMaskedImage().getMask()
740 wcs = exposure.getWcs()
741 plateScale = wcs.getPixelScale().asArcseconds()
743 for rec
in brightObjectMasks:
744 center = afwGeom.PointI(wcs.skyToPixel(rec.getCoord()))
745 if rec[
"type"] ==
"box":
746 assert rec[
"angle"] == 0.0, (
"Angle != 0 for mask object %s" % rec[
"id"])
747 width = rec[
"width"].asArcseconds()/plateScale
748 height = rec[
"height"].asArcseconds()/plateScale
750 halfSize = afwGeom.ExtentI(0.5*width, 0.5*height)
751 bbox = afwGeom.Box2I(center - halfSize, center + halfSize)
753 bbox = afwGeom.BoxI(afwGeom.PointI(int(center[0] - 0.5*width), int(center[1] - 0.5*height)),
754 afwGeom.PointI(int(center[0] + 0.5*width), int(center[1] + 0.5*height)))
755 spans = afwGeom.SpanSet(bbox)
756 elif rec[
"type"] ==
"circle":
757 radius = int(rec[
"radius"].asArcseconds()/plateScale)
758 spans = afwGeom.SpanSet.fromShape(radius, offset=center)
760 self.log.warn(
"Unexpected region type %s at %s" % rec[
"type"], center)
765 """Set INEXACT_PSF mask plane 767 If any of the input images isn't represented in the coadd (due to 768 clipped pixels or chip gaps), the `CoaddPsf` will be inexact. Flag 773 mask : `lsst.afw.image.Mask` 774 Coadded exposure's mask, modified in-place. 776 mask.addMaskPlane(
"INEXACT_PSF")
777 inexactPsf = mask.getPlaneBitMask(
"INEXACT_PSF")
778 sensorEdge = mask.getPlaneBitMask(
"SENSOR_EDGE")
779 clipped = mask.getPlaneBitMask(
"CLIPPED")
780 rejected = mask.getPlaneBitMask(
"REJECTED")
781 array = mask.getArray()
782 selected = array & (sensorEdge | clipped | rejected) > 0
783 array[selected] |= inexactPsf
786 def _makeArgumentParser(cls):
788 \brief Create an argument parser 791 parser.add_id_argument(
"--id", cls.
ConfigClass().coaddName +
"Coadd_" +
793 help=
"data ID, e.g. --id tract=12345 patch=1,2",
794 ContainerClass=AssembleCoaddDataIdContainer)
795 parser.add_id_argument(
"--selectId",
"calexp", help=
"data ID, e.g. --selectId visit=6789 ccd=0..9",
796 ContainerClass=SelectDataIdContainer)
800 def _subBBoxIter(bbox, subregionSize):
802 \brief Iterate over subregions of a bbox 804 \param[in] bbox: bounding box over which to iterate: afwGeom.Box2I 805 \param[in] subregionSize: size of sub-bboxes 807 \return subBBox: next sub-bounding box of size subregionSize or smaller; 808 each subBBox is contained within bbox, so it may be smaller than subregionSize at the edges of bbox, 809 but it will never be empty 812 raise RuntimeError(
"bbox %s is empty" % (bbox,))
813 if subregionSize[0] < 1
or subregionSize[1] < 1:
814 raise RuntimeError(
"subregionSize %s must be nonzero" % (subregionSize,))
816 for rowShift
in range(0, bbox.getHeight(), subregionSize[1]):
817 for colShift
in range(0, bbox.getWidth(), subregionSize[0]):
818 subBBox = afwGeom.Box2I(bbox.getMin() + afwGeom.Extent2I(colShift, rowShift), subregionSize)
820 if subBBox.isEmpty():
821 raise RuntimeError(
"Bug: empty bbox! bbox=%s, subregionSize=%s, colShift=%s, rowShift=%s" %
822 (bbox, subregionSize, colShift, rowShift))
828 \brief A version of lsst.pipe.base.DataIdContainer specialized for assembleCoadd. 833 \brief Make self.refList from self.idList. 835 datasetType = namespace.config.coaddName +
"Coadd" 836 keysCoadd = namespace.butler.getKeys(datasetType=datasetType, level=self.level)
838 for dataId
in self.idList:
840 for key
in keysCoadd:
841 if key
not in dataId:
842 raise RuntimeError(
"--id must include " + key)
844 dataRef = namespace.butler.dataRef(
845 datasetType=datasetType,
848 self.refList.append(dataRef)
853 \brief Function to count the number of pixels with a specific mask in a footprint. 855 Find the intersection of mask & footprint. Count all pixels in the mask that are in the intersection that 856 have bitmask set but do not have ignoreMask set. Return the count. 858 \param[in] mask: mask to define intersection region by. 859 \parma[in] footprint: footprint to define the intersection region by. 860 \param[in] bitmask: specific mask that we wish to count the number of occurances of. 861 \param[in] ignoreMask: pixels to not consider. 862 \return count of number of pixels in footprint with specified mask. 864 bbox = footprint.getBBox()
865 bbox.clip(mask.getBBox(afwImage.PARENT))
866 fp = afwImage.Mask(bbox)
867 subMask = mask.Factory(mask, bbox, afwImage.PARENT)
868 footprint.spans.setMask(fp, bitmask)
869 return numpy.logical_and((subMask.getArray() & fp.getArray()) > 0,
870 (subMask.getArray() & ignoreMask) == 0).sum()
875 \anchor SafeClipAssembleCoaddConfig 877 \brief Configuration parameters for the SafeClipAssembleCoaddTask 879 clipDetection = pexConfig.ConfigurableField(
880 target=SourceDetectionTask,
881 doc=
"Detect sources on difference between unclipped and clipped coadd")
882 minClipFootOverlap = pexConfig.Field(
883 doc=
"Minimum fractional overlap of clipped footprint with visit DETECTED to be clipped",
887 minClipFootOverlapSingle = pexConfig.Field(
888 doc=
"Minimum fractional overlap of clipped footprint with visit DETECTED to be " 889 "clipped when only one visit overlaps",
893 minClipFootOverlapDouble = pexConfig.Field(
894 doc=
"Minimum fractional overlap of clipped footprints with visit DETECTED to be " 895 "clipped when two visits overlap",
899 maxClipFootOverlapDouble = pexConfig.Field(
900 doc=
"Maximum fractional overlap of clipped footprints with visit DETECTED when " 901 "considering two visits",
905 minBigOverlap = pexConfig.Field(
906 doc=
"Minimum number of pixels in footprint to use DETECTED mask from the single visits " 907 "when labeling clipped footprints",
915 AssembleCoaddConfig.setDefaults(self)
931 log.warn(
"Additional Sigma-clipping not allowed in Safe-clipped Coadds. " 932 "Ignoring doSigmaClip.")
935 raise ValueError(
"Only MEAN statistic allowed for final stacking in SafeClipAssembleCoadd " 936 "(%s chosen). Please set statistic to MEAN." 938 AssembleCoaddTask.ConfigClass.validate(self)
951 \anchor SafeClipAssembleCoaddTask_ 953 \brief Assemble a coadded image from a set of coadded temporary exposures, 954 being careful to clip & flag areas with potential artifacts. 956 \section pipe_tasks_assembleCoadd_Contents Contents 957 - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Purpose 958 - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Initialize 959 - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Run 960 - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Config 961 - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Debug 962 - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Example 964 \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Purpose Description 966 \copybrief SafeClipAssembleCoaddTask 968 Read the documentation for \ref AssembleCoaddTask_ "AssembleCoaddTask" first since 969 SafeClipAssembleCoaddTask subtasks that task. 970 In \ref AssembleCoaddTask_ "AssembleCoaddTask", we compute the coadd as an clipped mean (i.e. we clip 972 The problem with doing this is that when computing the coadd PSF at a given location, individual visit 973 PSFs from visits with outlier pixels contribute to the coadd PSF and cannot be treated correctly. 974 In this task, we correct for this behavior by creating a new badMaskPlane 'CLIPPED'. 975 We populate this plane on the input coaddTempExps and the final coadd where i. difference imaging suggests 976 that there is an outlier and ii. this outlier appears on only one or two images. 977 Such regions will not contribute to the final coadd. 978 Furthermore, any routine to determine the coadd PSF can now be cognizant of clipped regions. 979 Note that the algorithm implemented by this task is preliminary and works correctly for HSC data. 980 Parameter modifications and or considerable redesigning of the algorithm is likley required for other 983 SafeClipAssembleCoaddTask uses a \ref SourceDetectionTask_ "clipDetection" subtask and also sub-classes 984 \ref AssembleCoaddTask_ "AssembleCoaddTask". You can retarget the 985 \ref SourceDetectionTask_ "clipDetection" subtask if you wish. 987 \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Initialize Task initialization 988 \copydoc \_\_init\_\_ 990 \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Run Invoking the Task 993 \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Config Configuration parameters 994 See \ref SafeClipAssembleCoaddConfig 996 \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Debug Debug variables 997 The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a 998 flag \c -d to import \b debug.py from your \c PYTHONPATH; see \ref baseDebug for more about \b debug.py 1000 SafeClipAssembleCoaddTask has no debug variables of its own. The \ref SourceDetectionTask_ "clipDetection" 1001 subtasks may support debug variables. See the documetation for \ref SourceDetectionTask_ "clipDetection" 1002 for further information. 1004 \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Example A complete example of using 1005 SafeClipAssembleCoaddTask 1007 SafeClipAssembleCoaddTask assembles a set of warped coaddTempExp images into a coadded image. 1008 The SafeClipAssembleCoaddTask is invoked by running assembleCoadd.py <em>without</em> the flag 1010 Usage of assembleCoadd.py expects a data reference to the tract patch and filter to be coadded 1011 (specified using '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]') along 1012 with a list of coaddTempExps to attempt to coadd (specified using 1013 '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]'). 1014 Only the coaddTempExps that cover the specified tract and patch will be coadded. 1015 A list of the available optional arguments can be obtained by calling assembleCoadd.py with the --help 1016 command line argument: 1018 assembleCoadd.py --help 1020 To demonstrate usage of the SafeClipAssembleCoaddTask in the larger context of multi-band processing, we 1021 will generate the HSC-I & -R band coadds from HSC engineering test data provided in the ci_hsc package. To 1022 begin, assuming that the lsst stack has been already set up, we must set up the obs_subaru and ci_hsc 1024 This defines the environment variable $CI_HSC_DIR and points at the location of the package. The raw HSC 1025 data live in the $CI_HSC_DIR/raw directory. To begin assembling the coadds, we must first 1028 <DD> process the individual ccds in $CI_HSC_RAW to produce calibrated exposures</DD> 1030 <DD> create a skymap that covers the area of the sky present in the raw exposures</DD> 1031 <DT>makeCoaddTempExp</DT> 1032 <DD> warp the individual calibrated exposures to the tangent plane of the coadd</DD> 1034 We can perform all of these steps by running 1036 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988 1038 This will produce warped coaddTempExps for each visit. To coadd the warped data, we call assembleCoadd.py 1041 assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \ 1042 --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \ 1043 --selectId visit=903986 ccd=100--selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \ 1044 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \ 1045 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \ 1046 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \ 1047 --selectId visit=903988 ccd=24 1049 This will process the HSC-I band data. The results are written in 1050 `$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I`. 1052 You may also choose to run: 1054 scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346 1055 assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R --selectId visit=903334 ccd=16 \ 1056 --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 --selectId visit=903334 ccd=100 \ 1057 --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 --selectId visit=903338 ccd=18 \ 1058 --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 --selectId visit=903342 ccd=10 \ 1059 --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 --selectId visit=903344 ccd=5 \ 1060 --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 --selectId visit=903346 ccd=6 \ 1061 --selectId visit=903346 ccd=12 1063 to generate the coadd for the HSC-R band if you are interested in following multiBand Coadd processing as 1064 discussed in \ref pipeTasks_multiBand. 1066 ConfigClass = SafeClipAssembleCoaddConfig
1067 _DefaultName =
"safeClipAssembleCoadd" 1071 \brief Initialize the task and make the \ref SourceDetectionTask_ "clipDetection" subtask. 1073 AssembleCoaddTask.__init__(self, *args, **kwargs)
1074 schema = afwTable.SourceTable.makeMinimalSchema()
1075 self.makeSubtask(
"clipDetection", schema=schema)
1077 def assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList, *args, **kwargs):
1079 \brief Assemble the coadd for a region 1081 Compute the difference of coadds created with and without outlier rejection to identify coadd pixels 1082 that have outlier values in some individual visits. Detect clipped regions on the difference image and 1083 mark these regions on the one or two individual coaddTempExps where they occur if there is significant 1084 overlap between the clipped region and a source. 1085 This leaves us with a set of footprints from the difference image that have been identified as having 1086 occured on just one or two individual visits. However, these footprints were generated from a 1087 difference image. It is conceivable for a large diffuse source to have become broken up into multiple 1088 footprints acrosss the coadd difference in this process. 1089 Determine the clipped region from all overlapping footprints from the detected sources in each visit - 1090 these are big footprints. 1091 Combine the small and big clipped footprints and mark them on a new bad mask plane 1092 Generate the coadd using \ref AssembleCoaddTask.assemble_ "AssembleCoaddTask.assemble" without outlier 1093 removal. Clipped footprints will no longer make it into the coadd because they are marked in the new 1096 N.b. *args and **kwargs are passed but ignored in order to match the call signature expected by the 1099 @param skyInfo: Patch geometry information, from getSkyInfo 1100 @param tempExpRefList: List of data reference to tempExp 1101 @param imageScalerList: List of image scalers 1102 @param weightList: List of weights 1103 return pipeBase.Struct with coaddExposure, nImage 1106 mask = exp.getMaskedImage().getMask()
1107 mask.addMaskPlane(
"CLIPPED")
1109 result = self.
detectClip(exp, tempExpRefList)
1111 self.log.info(
'Found %d clipped objects', len(result.clipFootprints))
1113 maskClipValue = mask.getPlaneBitMask(
"CLIPPED")
1114 maskDetValue = mask.getPlaneBitMask(
"DETECTED") | mask.getPlaneBitMask(
"DETECTED_NEGATIVE")
1116 bigFootprints = self.
detectClipBig(result.clipSpans, result.clipFootprints, result.clipIndices,
1117 result.detectionFootprints, maskClipValue, maskDetValue,
1120 maskClip = mask.Factory(mask.getBBox(afwImage.PARENT))
1121 afwDet.setMaskFromFootprintList(maskClip, result.clipFootprints, maskClipValue)
1123 maskClipBig = maskClip.Factory(mask.getBBox(afwImage.PARENT))
1124 afwDet.setMaskFromFootprintList(maskClipBig, bigFootprints, maskClipValue)
1125 maskClip |= maskClipBig
1128 badMaskPlanes = self.config.badMaskPlanes[:]
1129 badMaskPlanes.append(
"CLIPPED")
1130 badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes)
1131 return AssembleCoaddTask.assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList,
1132 result.clipSpans, mask=badPixelMask)
1136 \brief Return an exposure that contains the difference between and unclipped and clipped coadds. 1138 Generate a difference image between clipped and unclipped coadds. 1139 Compute the difference image by subtracting an outlier-clipped coadd from an outlier-unclipped coadd. 1140 Return the difference image. 1142 @param skyInfo: Patch geometry information, from getSkyInfo 1143 @param tempExpRefList: List of data reference to tempExp 1144 @param imageScalerList: List of image scalers 1145 @param weightList: List of weights 1146 @return Difference image of unclipped and clipped coadd wrapped in an Exposure 1151 configIntersection = {k: getattr(self.config, k)
1152 for k, v
in self.config.toDict().items()
if (k
in config.keys())}
1153 config.update(**configIntersection)
1156 config.statistic =
'MEAN' 1158 coaddMean = task.assemble(skyInfo, tempExpRefList, imageScalerList, weightList).coaddExposure
1160 config.statistic =
'MEANCLIP' 1162 coaddClip = task.assemble(skyInfo, tempExpRefList, imageScalerList, weightList).coaddExposure
1164 coaddDiff = coaddMean.getMaskedImage().Factory(coaddMean.getMaskedImage())
1165 coaddDiff -= coaddClip.getMaskedImage()
1166 exp = afwImage.ExposureF(coaddDiff)
1167 exp.setPsf(coaddMean.getPsf())
1172 \brief Detect clipped regions on an exposure and set the mask on the individual tempExp masks 1174 Detect footprints in the difference image after smoothing the difference image with a Gaussian kernal. 1175 Identify footprints that overlap with one or two input coaddTempExps by comparing the computed overlap 1176 fraction to thresholds set in the config. 1177 A different threshold is applied depending on the number of overlapping visits (restricted to one or 1179 If the overlap exceeds the thresholds, the footprint is considered "CLIPPED" and is marked as such on 1181 Return a struct with the clipped footprints, the indices of the coaddTempExps that end up overlapping 1182 with the clipped footprints and a list of new masks for the coaddTempExps. 1184 \param[in] exp: Exposure to run detection on 1185 \param[in] tempExpRefList: List of data reference to tempExp 1186 \return struct containing: 1187 - clipFootprints: list of clipped footprints 1188 - clipIndices: indices for each clippedFootprint in tempExpRefList 1189 - clipSpans: List of dictionaries containing spanSet lists to clip. Each element contains the new 1190 maskplane name ("CLIPPED")" as the key and list of SpanSets as value 1191 - detectionFootprints: List of DETECTED/DETECTED_NEGATIVE plane compressed into footprints 1193 mask = exp.getMaskedImage().getMask()
1194 maskDetValue = mask.getPlaneBitMask(
"DETECTED") | mask.getPlaneBitMask(
"DETECTED_NEGATIVE")
1195 fpSet = self.clipDetection.detectFootprints(exp, doSmooth=
True, clearMask=
True)
1197 fpSet.positive.merge(fpSet.negative)
1198 footprints = fpSet.positive
1199 self.log.info(
'Found %d potential clipped objects', len(footprints.getFootprints()))
1204 artifactSpanSets = [{
'CLIPPED': list()}
for _
in tempExpRefList]
1207 visitDetectionFootprints = []
1209 dims = [len(tempExpRefList), len(footprints.getFootprints())]
1210 overlapDetArr = numpy.zeros(dims, dtype=numpy.uint16)
1211 ignoreArr = numpy.zeros(dims, dtype=numpy.uint16)
1214 for i, warpRef
in enumerate(tempExpRefList):
1216 immediate=
True).getMaskedImage().getMask()
1217 maskVisitDet = tmpExpMask.Factory(tmpExpMask, tmpExpMask.getBBox(afwImage.PARENT),
1218 afwImage.PARENT,
True)
1219 maskVisitDet &= maskDetValue
1220 visitFootprints = afwDet.FootprintSet(maskVisitDet, afwDet.Threshold(1))
1221 visitDetectionFootprints.append(visitFootprints)
1223 for j, footprint
in enumerate(footprints.getFootprints()):
1228 for j, footprint
in enumerate(footprints.getFootprints()):
1229 nPixel = footprint.getArea()
1232 for i
in range(len(tempExpRefList)):
1233 ignore = ignoreArr[i, j]
1234 overlapDet = overlapDetArr[i, j]
1235 totPixel = nPixel - ignore
1238 if ignore > overlapDet
or totPixel <= 0.5*nPixel
or overlapDet == 0:
1240 overlap.append(overlapDet/float(totPixel))
1243 overlap = numpy.array(overlap)
1244 if not len(overlap):
1251 if len(overlap) == 1:
1252 if overlap[0] > self.config.minClipFootOverlapSingle:
1257 clipIndex = numpy.where(overlap > self.config.minClipFootOverlap)[0]
1258 if len(clipIndex) == 1:
1260 keepIndex = [clipIndex[0]]
1263 clipIndex = numpy.where(overlap > self.config.minClipFootOverlapDouble)[0]
1264 if len(clipIndex) == 2
and len(overlap) > 3:
1265 clipIndexComp = numpy.where(overlap <= self.config.minClipFootOverlapDouble)[0]
1266 if numpy.max(overlap[clipIndexComp]) <= self.config.maxClipFootOverlapDouble:
1268 keepIndex = clipIndex
1273 for index
in keepIndex:
1274 globalIndex = indexList[index]
1275 artifactSpanSets[globalIndex][
'CLIPPED'].append(footprint.spans)
1277 clipIndices.append(numpy.array(indexList)[keepIndex])
1278 clipFootprints.append(footprint)
1280 return pipeBase.Struct(clipFootprints=clipFootprints, clipIndices=clipIndices,
1281 clipSpans=artifactSpanSets, detectionFootprints=visitDetectionFootprints)
1283 def detectClipBig(self, clipList, clipFootprints, clipIndices, detectionFootprints,
1284 maskClipValue, maskDetValue, coaddBBox):
1286 \brief Return individual warp footprints for large artifacts and append them to clipList in place 1288 Identify big footprints composed of many sources in the coadd difference that may have originated in a 1289 large diffuse source in the coadd. We do this by indentifying all clipped footprints that overlap 1290 significantly with each source in all the coaddTempExps. 1291 \param[in] clipList: List of alt mask SpanSets with clipping information. Modified. 1292 \param[in] clipFootprints: List of clipped footprints 1293 \param[in] clipIndices: List of which entries in tempExpClipList each footprint belongs to 1294 \param[in] maskClipValue: Mask value of clipped pixels 1295 \param[in] maskDetValue: Mask value of detected pixels 1296 \param[in] coaddBBox: BBox of the coadd and warps 1297 \return list of big footprints 1299 bigFootprintsCoadd = []
1301 for index, (clippedSpans, visitFootprints)
in enumerate(zip(clipList, detectionFootprints)):
1302 maskVisitDet = afwImage.MaskX(coaddBBox, 0x0)
1303 for footprint
in visitFootprints.getFootprints():
1304 footprint.spans.setMask(maskVisitDet, maskDetValue)
1307 clippedFootprintsVisit = []
1308 for foot, clipIndex
in zip(clipFootprints, clipIndices):
1309 if index
not in clipIndex:
1311 clippedFootprintsVisit.append(foot)
1312 maskVisitClip = maskVisitDet.Factory(maskVisitDet.getBBox(afwImage.PARENT))
1313 afwDet.setMaskFromFootprintList(maskVisitClip, clippedFootprintsVisit, maskClipValue)
1315 bigFootprintsVisit = []
1316 for foot
in visitFootprints.getFootprints():
1317 if foot.getArea() < self.config.minBigOverlap:
1320 if nCount > self.config.minBigOverlap:
1321 bigFootprintsVisit.append(foot)
1322 bigFootprintsCoadd.append(foot)
1324 for footprint
in bigFootprintsVisit:
1325 clippedSpans[
"CLIPPED"].append(footprint.spans)
1327 return bigFootprintsCoadd
1331 assembleStaticSkyModel = pexConfig.ConfigurableField(
1332 target=AssembleCoaddTask,
1333 doc=
"Task to assemble an artifact-free, PSF-matched Coadd to serve as a" 1334 " naive/first-iteration model of the static sky.",
1336 detect = pexConfig.ConfigurableField(
1337 target=SourceDetectionTask,
1338 doc=
"Detect outlier sources on difference between each psfMatched warp and static sky model" 1340 detectTemplate = pexConfig.ConfigurableField(
1341 target=SourceDetectionTask,
1342 doc=
"Detect sources on static sky model. Only used if doPreserveContainedBySource is True" 1344 maxNumEpochs = pexConfig.Field(
1345 doc=
"Charactistic maximum local number of epochs/visits in which an artifact candidate can appear " 1346 "and still be masked. The effective maxNumEpochs is a broken linear function of local " 1347 "number of epochs (N): min(maxFractionEpochsLow*N, maxNumEpochs + maxFractionEpochsHigh*N). " 1348 "For each footprint detected on the image difference between the psfMatched warp and static sky " 1349 "model, if a significant fraction of pixels (defined by spatialThreshold) are residuals in more " 1350 "than the computed effective maxNumEpochs, the artifact candidate is deemed persistant rather " 1351 "than transient and not masked.",
1355 maxFractionEpochsLow = pexConfig.RangeField(
1356 doc=
"Fraction of local number of epochs (N) to use as effective maxNumEpochs for low N. " 1357 "Effective maxNumEpochs = " 1358 "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)",
1363 maxFractionEpochsHigh = pexConfig.RangeField(
1364 doc=
"Fraction of local number of epochs (N) to use as effective maxNumEpochs for high N. " 1365 "Effective maxNumEpochs = " 1366 "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)",
1371 spatialThreshold = pexConfig.RangeField(
1372 doc=
"Unitless fraction of pixels defining how much of the outlier region has to meet the " 1373 "temporal criteria. If 0, clip all. If 1, clip none.",
1377 inclusiveMin=
True, inclusiveMax=
True 1379 doScaleWarpVariance = pexConfig.Field(
1380 doc=
"Rescale Warp variance plane using empirical noise?",
1384 scaleWarpVariance = pexConfig.ConfigurableField(
1385 target=ScaleVarianceTask,
1386 doc=
"Rescale variance on warps",
1388 doPreserveContainedBySource = pexConfig.Field(
1389 doc=
"Rescue artifacts from clipping that completely lie within a footprint detected" 1390 "on the PsfMatched Template Coadd. Replicates a behavior of SafeClip.",
1394 doPrefilterArtifacts = pexConfig.Field(
1395 doc=
"Ignore artifact candidates that are mostly covered by the bad pixel mask, " 1396 "because they will be excluded anyway. This prevents them from contributing " 1397 "to the outlier epoch count image and potentially being labeled as persistant." 1398 "'Mostly' is defined by the config 'prefilterArtifactsRatio'.",
1402 prefilterArtifactsMaskPlanes = pexConfig.ListField(
1403 doc=
"Prefilter artifact candidates that are mostly covered by these bad mask planes.",
1405 default=(
'NO_DATA',
'BAD',
'SAT',
'SUSPECT'),
1407 prefilterArtifactsRatio = pexConfig.Field(
1408 doc=
"Prefilter artifact candidates with less than this fraction overlapping good pixels",
1414 AssembleCoaddConfig.setDefaults(self)
1430 self.
detect.doTempLocalBackground =
False 1431 self.
detect.reEstimateBackground =
False 1432 self.
detect.returnOriginalFootprints =
False 1433 self.
detect.thresholdPolarity =
"both" 1434 self.
detect.thresholdValue = 5
1435 self.
detect.nSigmaToGrow = 2
1436 self.
detect.minPixels = 4
1437 self.
detect.isotropicGrow =
True 1438 self.
detect.thresholdType =
"pixel_stdev" 1454 \anchor CompareWarpAssembleCoaddTask_ 1456 \brief Assemble a compareWarp coadded image from a set of warps 1457 by masking artifacts detected by comparing PSF-matched warps 1459 \section pipe_tasks_assembleCoadd_Contents Contents 1460 - \ref pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Purpose 1461 - \ref pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Initialize 1462 - \ref pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Run 1463 - \ref pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Config 1464 - \ref pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Debug 1465 - \ref pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Example 1467 \section pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Purpose Description 1469 \copybrief CompareWarpAssembleCoaddTask 1471 In \ref AssembleCoaddTask_ "AssembleCoaddTask", we compute the coadd as an clipped mean (i.e. we clip 1473 The problem with doing this is that when computing the coadd PSF at a given location, individual visit 1474 PSFs from visits with outlier pixels contribute to the coadd PSF and cannot be treated correctly. 1475 In this task, we correct for this behavior by creating a new badMaskPlane 'CLIPPED' which marks 1476 pixels in the individual warps suspected to contain an artifact. 1477 We populate this plane on the input warps by comparing PSF-matched warps with a PSF-matched median coadd 1478 which serves as a model of the static sky. Any group of pixels that deviates from the PSF-matched 1479 template coadd by more than config.detect.threshold sigma, is an artifact candidate. 1480 The candidates are then filtered to remove variable sources and sources that are difficult to subtract 1481 such as bright stars. 1482 This filter is configured using the config parameters temporalThreshold and spatialThreshold. 1483 The temporalThreshold is the maximum fraction of epochs that the deviation can 1484 appear in and still be considered an artifact. The spatialThreshold is the maximum fraction of pixels in 1485 the footprint of the deviation that appear in other epochs (where other epochs is defined by the 1486 temporalThreshold). If the deviant region meets this criteria of having a significant percentage of pixels 1487 that deviate in only a few epochs, these pixels have the 'CLIPPED' bit set in the mask. 1488 These regions will not contribute to the final coadd. 1489 Furthermore, any routine to determine the coadd PSF can now be cognizant of clipped regions. 1490 Note that the algorithm implemented by this task is preliminary and works correctly for HSC data. 1491 Parameter modifications and or considerable redesigning of the algorithm is likley required for other 1494 CompareWarpAssembleCoaddTask sub-classes 1495 \ref AssembleCoaddTask_ "AssembleCoaddTask" and instantiates \ref AssembleCoaddTask_ "AssembleCoaddTask" 1496 as a subtask to generate the TemplateCoadd (the model of the static sky) 1498 \section pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Initialize Task initialization 1499 \copydoc \_\_init\_\_ 1501 \section pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Run Invoking the Task 1504 \section pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Config Configuration parameters 1505 See \ref CompareWarpAssembleCoaddConfig 1507 \section pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Debug Debug variables 1508 The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a 1509 flag \c -d to import \b debug.py from your \c PYTHONPATH; see \ref baseDebug for more about \b debug.py 1512 This task supports the following debug variables: 1515 <dd> If True then save the Epoch Count Image as a fits file in the `figPath` 1517 <dd> Path to save the debug fits images and figures 1520 For example, put something like: 1523 def DebugInfo(name): 1524 di = lsstDebug.getInfo(name) 1525 if name == "lsst.pipe.tasks.assembleCoadd": 1526 di.saveCountIm = True 1527 di.figPath = "/desired/path/to/debugging/output/images" 1529 lsstDebug.Info = DebugInfo 1531 into your `debug.py` file and run `assemebleCoadd.py` with the `--debug` 1533 Some subtasks may have their own debug variables; see individual Task 1536 \section pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Example A complete example of using 1537 CompareWarpAssembleCoaddTask 1539 CompareWarpAssembleCoaddTask assembles a set of warped images into a coadded image. 1540 The CompareWarpAssembleCoaddTask is invoked by running assembleCoadd.py with the flag 1541 '--compareWarpCoadd'. 1542 Usage of assembleCoadd.py expects a data reference to the tract patch and filter to be coadded 1543 (specified using '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]') along 1544 with a list of coaddTempExps to attempt to coadd (specified using 1545 '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]'). 1546 Only the warps that cover the specified tract and patch will be coadded. 1547 A list of the available optional arguments can be obtained by calling assembleCoadd.py with the --help 1548 command line argument: 1550 assembleCoadd.py --help 1552 To demonstrate usage of the CompareWarpAssembleCoaddTask in the larger context of multi-band processing, 1553 we will generate the HSC-I & -R band coadds from HSC engineering test data provided in the ci_hsc package. 1554 To begin, assuming that the lsst stack has been already set up, we must set up the obs_subaru and ci_hsc 1556 This defines the environment variable $CI_HSC_DIR and points at the location of the package. The raw HSC 1557 data live in the $CI_HSC_DIR/raw directory. To begin assembling the coadds, we must first 1560 <DD> process the individual ccds in $CI_HSC_RAW to produce calibrated exposures</DD> 1562 <DD> create a skymap that covers the area of the sky present in the raw exposures</DD> 1563 <DT>makeCoaddTempExp</DT> 1564 <DD> warp the individual calibrated exposures to the tangent plane of the coadd</DD> 1566 We can perform all of these steps by running 1568 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988 1570 This will produce warped coaddTempExps for each visit. To coadd the warped data, we call assembleCoadd.py 1573 assembleCoadd.py --compareWarpCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \ 1574 --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \ 1575 --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \ 1576 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \ 1577 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \ 1578 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \ 1579 --selectId visit=903988 ccd=24 1581 This will process the HSC-I band data. The results are written in 1582 `$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I`. 1584 ConfigClass = CompareWarpAssembleCoaddConfig
1585 _DefaultName =
"compareWarpAssembleCoadd" 1589 \brief Initialize the task and make the \ref AssembleCoadd_ "assembleStaticSkyModel" subtask. 1591 AssembleCoaddTask.__init__(self, *args, **kwargs)
1592 self.makeSubtask(
"assembleStaticSkyModel")
1593 detectionSchema = afwTable.SourceTable.makeMinimalSchema()
1594 self.makeSubtask(
"detect", schema=detectionSchema)
1595 if self.config.doPreserveContainedBySource:
1596 self.makeSubtask(
"detectTemplate", schema=afwTable.SourceTable.makeMinimalSchema())
1597 if self.config.doScaleWarpVariance:
1598 self.makeSubtask(
"scaleWarpVariance")
1602 \brief Make inputs specific to Subclass 1604 Generate a templateCoadd to use as a native model of static sky to subtract from warps. 1606 templateCoadd = self.assembleStaticSkyModel.
run(dataRef, selectDataList)
1608 if templateCoadd
is None:
1609 warpName = (self.assembleStaticSkyModel.warpType[0].upper() +
1610 self.assembleStaticSkyModel.warpType[1:])
1611 message =
"""No %(warpName)s warps were found to build the template coadd which is 1612 required to run CompareWarpAssembleCoaddTask. To continue assembling this type of coadd, 1613 first either rerun makeCoaddTempExp with config.make%(warpName)s=True or 1614 coaddDriver with config.makeCoadTempExp.make%(warpName)s=True, before assembleCoadd. 1616 Alternatively, to use another algorithm with existing warps, retarget the CoaddDriverConfig to 1617 another algorithm like: 1619 from lsst.pipe.tasks.assembleCoadd import SafeClipAssembleCoaddTask 1620 config.assemble.retarget(SafeClipAssembleCoaddTask) 1621 """ % {
"warpName": warpName}
1622 raise RuntimeError(message)
1624 return pipeBase.Struct(templateCoadd=templateCoadd.coaddExposure)
1626 def assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList,
1627 supplementaryData, *args, **kwargs):
1629 \brief Assemble the coadd 1631 Requires additional inputs Struct `supplementaryData` to contain a `templateCoadd` that serves 1632 as the model of the static sky. 1634 Find artifacts and apply them to the warps' masks creating a list of alternative masks with a 1635 new "CLIPPED" plane and updated "NO_DATA" plane. 1636 Then pass these alternative masks to the base class's assemble method. 1638 @param skyInfo: Patch geometry information 1639 @param tempExpRefList: List of data references to warps 1640 @param imageScalerList: List of image scalers 1641 @param weightList: List of weights 1642 @param supplementaryData: PipeBase.Struct containing a templateCoadd 1644 return pipeBase.Struct with coaddExposure, nImage if requested 1646 templateCoadd = supplementaryData.templateCoadd
1647 spanSetMaskList = self.
findArtifacts(templateCoadd, tempExpRefList, imageScalerList)
1648 badMaskPlanes = self.config.badMaskPlanes[:]
1649 badMaskPlanes.append(
"CLIPPED")
1650 badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes)
1652 result = AssembleCoaddTask.assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList,
1653 spanSetMaskList, mask=badPixelMask)
1657 self.
applyAltEdgeMask(result.coaddExposure.maskedImage.mask, spanSetMaskList)
1662 \brief Propagate alt EDGE mask to SENSOR_EDGE AND INEXACT_PSF planes 1664 @param mask: original mask 1665 @param altMaskList: List of Dicts containing spanSet lists. 1666 Each element contains the new mask plane name 1667 (e.g. "CLIPPED and/or "NO_DATA") as the key, 1668 and list of SpanSets to apply to the mask. 1670 maskValue = mask.getPlaneBitMask([
"SENSOR_EDGE",
"INEXACT_PSF"])
1671 for visitMask
in altMaskList:
1672 if "EDGE" in visitMask:
1673 for spanSet
in visitMask[
'EDGE']:
1674 spanSet.clippedTo(mask.getBBox()).setMask(mask, maskValue)
1678 \brief Find artifacts 1680 Loop through warps twice. The first loop builds a map with the count of how many 1681 epochs each pixel deviates from the templateCoadd by more than config.chiThreshold sigma. 1682 The second loop takes each difference image and filters the artifacts detected 1683 in each using count map to filter out variable sources and sources that are difficult to 1686 @param templateCoadd: Exposure to serve as model of static sky 1687 @param tempExpRefList: List of data references to warps 1688 @param imageScalerList: List of image scalers 1691 self.log.debug(
"Generating Count Image, and mask lists.")
1692 coaddBBox = templateCoadd.getBBox()
1693 slateIm = afwImage.ImageU(coaddBBox)
1694 epochCountImage = afwImage.ImageU(coaddBBox)
1695 nImage = afwImage.ImageU(coaddBBox)
1696 spanSetArtifactList = []
1697 spanSetNoDataMaskList = []
1698 spanSetEdgeList = []
1702 templateCoadd.mask.clearAllMaskPlanes()
1704 if self.config.doPreserveContainedBySource:
1705 templateFootprints = self.detectTemplate.detectFootprints(templateCoadd)
1707 templateFootprints =
None 1709 for warpRef, imageScaler
in zip(tempExpRefList, imageScalerList):
1711 if warpDiffExp
is not None:
1713 nImage.array += (numpy.isfinite(warpDiffExp.image.array) *
1714 ((warpDiffExp.mask.array & badPixelMask) == 0)).astype(numpy.uint16)
1715 fpSet = self.detect.detectFootprints(warpDiffExp, doSmooth=
False, clearMask=
True)
1716 fpSet.positive.merge(fpSet.negative)
1717 footprints = fpSet.positive
1719 spanSetList = [footprint.spans
for footprint
in footprints.getFootprints()]
1722 if self.config.doPrefilterArtifacts:
1724 for spans
in spanSetList:
1725 spans.setImage(slateIm, 1, doClip=
True)
1726 epochCountImage += slateIm
1732 nans = numpy.where(numpy.isnan(warpDiffExp.maskedImage.image.array), 1, 0)
1733 nansMask = afwImage.makeMaskFromArray(nans.astype(afwImage.MaskPixel))
1734 nansMask.setXY0(warpDiffExp.getXY0())
1735 edgeMask = warpDiffExp.mask
1736 spanSetEdgeMask = afwGeom.SpanSet.fromMask(edgeMask,
1737 edgeMask.getPlaneBitMask(
"EDGE")).split()
1741 nansMask = afwImage.MaskX(coaddBBox, 1)
1743 spanSetEdgeMask = []
1745 spanSetNoDataMask = afwGeom.SpanSet.fromMask(nansMask).split()
1747 spanSetNoDataMaskList.append(spanSetNoDataMask)
1748 spanSetArtifactList.append(spanSetList)
1749 spanSetEdgeList.append(spanSetEdgeMask)
1753 epochCountImage.writeFits(path)
1755 for i, spanSetList
in enumerate(spanSetArtifactList):
1757 filteredSpanSetList = self.
filterArtifacts(spanSetList, epochCountImage, nImage,
1759 spanSetArtifactList[i] = filteredSpanSetList
1762 for artifacts, noData, edge
in zip(spanSetArtifactList, spanSetNoDataMaskList, spanSetEdgeList):
1763 altMasks.append({
'CLIPPED': artifacts,
1770 \brief Remove artifact candidates covered by bad mask plane 1772 Any future editing of the candidate list that does not depend on 1773 temporal information should go in this method. 1775 @param spanSetList: List of SpanSets representing artifact candidates 1776 @param exp: Exposure containing mask planes used to prefilter 1778 return List of SpanSets with artifacts 1780 badPixelMask = exp.mask.getPlaneBitMask(self.config.prefilterArtifactsMaskPlanes)
1781 goodArr = (exp.mask.array & badPixelMask) == 0
1782 returnSpanSetList = []
1783 bbox = exp.getBBox()
1784 x0, y0 = exp.getXY0()
1785 for i, span
in enumerate(spanSetList):
1786 y, x = span.clippedTo(bbox).indices()
1787 yIndexLocal = numpy.array(y) - y0
1788 xIndexLocal = numpy.array(x) - x0
1789 goodRatio = numpy.count_nonzero(goodArr[yIndexLocal, xIndexLocal])/span.getArea()
1790 if goodRatio > self.config.prefilterArtifactsRatio:
1791 returnSpanSetList.append(span)
1792 return returnSpanSetList
1794 def filterArtifacts(self, spanSetList, epochCountImage, nImage, footprintsToExclude=None):
1796 \brief Filter artifact candidates 1798 @param spanSetList: List of SpanSets representing artifact candidates 1799 @param epochCountImage: Image of accumulated number of warpDiff detections 1800 @param nImage: Image of the accumulated number of total epochs contributing 1802 return List of SpanSets with artifacts 1805 maskSpanSetList = []
1806 x0, y0 = epochCountImage.getXY0()
1807 for i, span
in enumerate(spanSetList):
1808 y, x = span.indices()
1809 yIdxLocal = [y1 - y0
for y1
in y]
1810 xIdxLocal = [x1 - x0
for x1
in x]
1811 outlierN = epochCountImage.array[yIdxLocal, xIdxLocal]
1812 totalN = nImage.array[yIdxLocal, xIdxLocal]
1815 effMaxNumEpochsHighN = (self.config.maxNumEpochs +
1816 self.config.maxFractionEpochsHigh*numpy.mean(totalN))
1817 effMaxNumEpochsLowN = self.config.maxFractionEpochsLow * numpy.mean(totalN)
1818 effectiveMaxNumEpochs = int(min(effMaxNumEpochsLowN, effMaxNumEpochsHighN))
1819 nPixelsBelowThreshold = numpy.count_nonzero((outlierN > 0) &
1820 (outlierN <= effectiveMaxNumEpochs))
1821 percentBelowThreshold = nPixelsBelowThreshold / len(outlierN)
1822 if percentBelowThreshold > self.config.spatialThreshold:
1823 maskSpanSetList.append(span)
1825 if self.config.doPreserveContainedBySource
and footprintsToExclude
is not None:
1827 filteredMaskSpanSetList = []
1828 for span
in maskSpanSetList:
1830 for footprint
in footprintsToExclude.positive.getFootprints():
1831 if footprint.spans.contains(span):
1835 filteredMaskSpanSetList.append(span)
1836 maskSpanSetList = filteredMaskSpanSetList
1838 return maskSpanSetList
1840 def _readAndComputeWarpDiff(self, warpRef, imageScaler, templateCoadd):
1842 \brief Fetch a warp from the butler and return a warpDiff 1844 @param warpRef: `Butler dataRef` for the warp 1845 @param imageScaler: `scaleZeroPoint.ImageScaler` object 1846 @param templateCoadd: Exposure to be substracted from the scaled warp 1848 return Exposure of the image difference between the warp and template 1853 if not warpRef.datasetExists(warpName):
1854 self.log.warn(
"Could not find %s %s; skipping it", warpName, warpRef.dataId)
1856 warp = warpRef.get(warpName, immediate=
True)
1858 imageScaler.scaleMaskedImage(warp.getMaskedImage())
1859 mi = warp.getMaskedImage()
1860 if self.config.doScaleWarpVariance:
1862 self.scaleWarpVariance.
run(mi)
1863 except Exception
as exc:
1864 self.log.warn(
"Unable to rescale variance of warp (%s); leaving it as-is" % (exc,))
1865 mi -= templateCoadd.getMaskedImage()
1868 def _dataRef2DebugPath(self, prefix, warpRef, coaddLevel=False):
1870 \brief Return a path to which to write debugging output 1872 @param prefix: string, prefix for filename 1873 @param warpRef: Butler dataRef 1874 @param coaddLevel: bool, optional. If True, include only coadd-level keys 1875 (e.g. 'tract', 'patch', 'filter', but no 'visit') 1877 Creates a hyphen-delimited string of dataId values for simple filenames. 1882 keys = warpRef.dataId.keys()
1883 keyList = sorted(keys, reverse=
True)
1885 filename =
"%s-%s.fits" % (prefix,
'-'.join([str(warpRef.dataId[k])
for k
in keyList]))
1886 return os.path.join(directory, filename)
def setBrightObjectMasks(self, exposure, dataId, brightObjectMasks)
def shrinkValidPolygons(self, coaddInputs)
Shrink coaddInputs' ccds' ValidPolygons in place.
def getCoaddDatasetName(self, warpType="direct")
def _dataRef2DebugPath(self, prefix, warpRef, coaddLevel=False)
Return a path to which to write debugging output.
def getGroupDataRef(butler, datasetType, groupTuple, keys)
Base class for coaddition.
def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList)
Find artifacts.
def assembleMetadata(self, coaddExposure, tempExpRefList, weightList)
Set the metadata for the coadd.
def makeSupplementaryData(self, dataRef, selectDataList)
Make additional inputs to assemble() specific to subclasses.
def makeSupplementaryData(self, dataRef, selectDataList)
Make inputs specific to Subclass.
def getTempExpRefList(self, patchRef, calExpRefList)
Generate list data references corresponding to warped exposures that lie within the patch to be coadd...
def assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList, args, kwargs)
Assemble the coadd for a region.
def run(self, dataRef, selectDataList=[])
Assemble a coadd from a set of Warps.
def _readAndComputeWarpDiff(self, warpRef, imageScaler, templateCoadd)
Fetch a warp from the butler and return a warpDiff.
def prepareInputs(self, refList)
Prepare the input warps for coaddition by measuring the weight for each warp and the scaling for the ...
def applyAltMaskPlanes(self, mask, altMaskSpans)
Apply in place alt mask formatted as SpanSets to a mask.
def getSkyInfo(self, patchRef)
Use getSkyinfo to return the skyMap, tract and patch information, wcs and the outer bbox of the patch...
def getTempExpDatasetName(self, warpType="direct")
def __init__(self, args, kwargs)
Initialize the task and make the assembleStaticSkyModel subtask.
def makeDataRefList(self, namespace)
Make self.refList from self.idList.
def getBadPixelMask(self)
Convenience method to provide the bitmask from the mask plane names.
def assembleSubregion(self, coaddExposure, bbox, tempExpRefList, imageScalerList, weightList, altMaskList, statsFlags, statsCtrl, nImage=None)
Assemble the coadd for a sub-region.
def detectClip(self, exp, tempExpRefList)
Detect clipped regions on an exposure and set the mask on the individual tempExp masks.
def setInexactPsf(self, mask)
Configuration parameters for the SafeClipAssembleCoaddTask.
def __init__(self, args, kwargs)
Initialize the task.
Assemble a coadded image from a set of warps (coadded temporary exposures).
def filterArtifacts(self, spanSetList, epochCountImage, nImage, footprintsToExclude=None)
Filter artifact candidates.
Assemble a coadded image from a set of coadded temporary exposures, being careful to clip & flag area...
def buildDifferenceImage(self, skyInfo, tempExpRefList, imageScalerList, weightList)
Return an exposure that contains the difference between and unclipped and clipped coadds...
def selectExposures(self, patchRef, skyInfo=None, selectDataList=[])
Select exposures to coadd.
def applyAltEdgeMask(self, mask, altMaskList)
Propagate alt EDGE mask to SENSOR_EDGE AND INEXACT_PSF planes.
def readBrightObjectMasks(self, dataRef)
Configuration parameters for the AssembleCoaddTask.
Assemble a compareWarp coadded image from a set of warps by masking artifacts detected by comparing P...
def __init__(self, args, kwargs)
Initialize the task and make the clipDetection subtask.
def assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList, supplementaryData, args, kwargs)
Assemble the coadd.
def prefilterArtifacts(self, spanSetList, exp)
Remove artifact candidates covered by bad mask plane.
A version of lsst.pipe.base.DataIdContainer specialized for assembleCoadd.
def countMaskFromFootprint(mask, footprint, bitmask, ignoreMask)
Function to count the number of pixels with a specific mask in a footprint.
def groupPatchExposures(patchDataRef, calexpDataRefList, coaddDatasetType="deepCoadd", tempExpDatasetType="deepCoadd_directWarp")
def assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList, altMaskList=None, mask=None, supplementaryData=None)
Assemble a coadd from input warps.
def detectClipBig(self, clipList, clipFootprints, clipIndices, detectionFootprints, maskClipValue, maskDetValue, coaddBBox)
Return individual warp footprints for large artifacts and append them to clipList in place...