1 from __future__
import absolute_import, division, print_function
2 from builtins
import zip
3 from builtins
import range
39 from .coaddBase
import CoaddBaseTask, SelectDataIdContainer
40 from .interpImage
import InterpImageTask
41 from .scaleZeroPoint
import ScaleZeroPointTask
42 from .coaddHelpers
import groupPatchExposures, getGroupDataRef
45 __all__ = [
"AssembleCoaddTask",
"SafeClipAssembleCoaddTask",
"CompareWarpAssembleCoaddTask"]
50 \anchor AssembleCoaddConfig_ 52 \brief Configuration parameters for the \ref AssembleCoaddTask_ "AssembleCoaddTask" 54 warpType = pexConfig.Field(
55 doc=
"Warp name: one of 'direct' or 'psfMatched'",
59 subregionSize = pexConfig.ListField(
61 doc=
"Width, height of stack subregion size; " 62 "make small enough that a full stack of images will fit into memory at once.",
66 statistic = pexConfig.Field(
68 doc=
"Main stacking statistic for aggregating over the epochs.",
71 doSigmaClip = pexConfig.Field(
73 doc=
"Perform sigma clipped outlier rejection with MEANCLIP statistic? (DEPRECATED)",
76 sigmaClip = pexConfig.Field(
78 doc=
"Sigma for outlier rejection; ignored if non-clipping statistic selected.",
81 clipIter = pexConfig.Field(
83 doc=
"Number of iterations of outlier rejection; ignored if non-clipping statistic selected.",
86 scaleZeroPoint = pexConfig.ConfigurableField(
87 target=ScaleZeroPointTask,
88 doc=
"Task to adjust the photometric zero point of the coadd temp exposures",
90 doInterp = pexConfig.Field(
91 doc=
"Interpolate over NaN pixels? Also extrapolate, if necessary, but the results are ugly.",
95 interpImage = pexConfig.ConfigurableField(
96 target=InterpImageTask,
97 doc=
"Task to interpolate (and extrapolate) over NaN pixels",
99 doWrite = pexConfig.Field(
100 doc=
"Persist coadd?",
104 doNImage = pexConfig.Field(
105 doc=
"Create image of number of contributing exposures for each pixel",
109 maskPropagationThresholds = pexConfig.DictField(
112 doc=(
"Threshold (in fractional weight) of rejection at which we propagate a mask plane to " 113 "the coadd; that is, we set the mask bit on the coadd if the fraction the rejected frames " 114 "would have contributed exceeds this value."),
115 default={
"SAT": 0.1},
117 removeMaskPlanes = pexConfig.ListField(dtype=str, default=[
"NOT_DEBLENDED"],
118 doc=
"Mask planes to remove before coadding")
127 doMaskBrightObjects = pexConfig.Field(dtype=bool, default=
False,
128 doc=
"Set mask and flag bits for bright objects?")
129 brightObjectMaskName = pexConfig.Field(dtype=str, default=
"BRIGHT_OBJECT",
130 doc=
"Name of mask bit used for bright objects")
132 coaddPsf = pexConfig.ConfigField(
133 doc=
"Configuration for CoaddPsf",
134 dtype=measAlg.CoaddPsfConfig,
138 CoaddBaseTask.ConfigClass.setDefaults(self)
142 CoaddBaseTask.ConfigClass.validate(self)
146 log.warn(
"Config doPsfMatch deprecated. Setting warpType='psfMatched'")
149 log.warn(
'doSigmaClip deprecated. To replicate behavior, setting statistic to "MEANCLIP"')
151 if self.
doInterp and self.
statistic not in [
'MEAN',
'MEDIAN',
'MEANCLIP',
'VARIANCE',
'VARIANCECLIP']:
152 raise ValueError(
"Must set doInterp=False for statistic=%s, which does not " 153 "compute and set a non-zero coadd variance estimate." % (self.
statistic))
155 unstackableStats = [
'NOTHING',
'ERROR',
'ORMASK']
156 if not hasattr(afwMath.Property, self.
statistic)
or self.
statistic in unstackableStats:
157 stackableStats = [str(k)
for k
in afwMath.Property.__members__.keys()
158 if str(k)
not in unstackableStats]
159 raise ValueError(
"statistic %s is not allowed. Please choose one of %s." 171 \anchor AssembleCoaddTask_ 173 \brief Assemble a coadded image from a set of warps (coadded temporary exposures). 175 \section pipe_tasks_assembleCoadd_Contents Contents 176 - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Purpose 177 - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Initialize 178 - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Run 179 - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Config 180 - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Debug 181 - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Example 183 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Purpose Description 185 \copybrief AssembleCoaddTask_ 187 We want to assemble a coadded image from a set of Warps (also called 188 coadded temporary exposures or coaddTempExps. 189 Each input Warp covers a patch on the sky and corresponds to a single run/visit/exposure of the 190 covered patch. We provide the task with a list of Warps (selectDataList) from which it selects 191 Warps that cover the specified patch (pointed at by dataRef). 192 Each Warp that goes into a coadd will typically have an independent photometric zero-point. 193 Therefore, we must scale each Warp to set it to a common photometric zeropoint. 194 WarpType may be one of 'direct' or 'psfMatched', and the boolean configs config.makeDirect and 195 config.makePsfMatched set which of the warp types will be coadded. 196 The coadd is computed as a mean with optional outlier rejection. 197 Criteria for outlier rejection are set in \ref AssembleCoaddConfig. Finally, Warps can have bad 'NaN' 198 pixels which received no input from the source calExps. We interpolate over these bad (NaN) pixels. 200 AssembleCoaddTask uses several sub-tasks. These are 202 <DT>\ref ScaleZeroPointTask_ "ScaleZeroPointTask"</DT> 203 <DD> create and use an imageScaler object to scale the photometric zeropoint for each Warp</DD> 204 <DT>\ref InterpImageTask_ "InterpImageTask"</DT> 205 <DD>interpolate across bad pixels (NaN) in the final coadd</DD> 207 You can retarget these subtasks if you wish. 209 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Initialize Task initialization 210 \copydoc \_\_init\_\_ 212 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Run Invoking the Task 215 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Config Configuration parameters 216 See \ref AssembleCoaddConfig_ 218 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Debug Debug variables 219 The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a 220 flag \c -d to import \b debug.py from your \c PYTHONPATH; see \ref baseDebug for more about \b debug.py files. 221 AssembleCoaddTask has no debug variables of its own. Some of the subtasks may support debug variables. See 222 the documetation for the subtasks for further information. 224 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Example A complete example of using AssembleCoaddTask 226 AssembleCoaddTask assembles a set of warped images into a coadded image. The AssembleCoaddTask 227 can be invoked by running assembleCoadd.py with the flag '--legacyCoadd'. Usage of assembleCoadd.py expects 228 a data reference to the tract patch and filter to be coadded (specified using 229 '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]') along with a list of 230 Warps to attempt to coadd (specified using 231 '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]'). Only the Warps 232 that cover the specified tract and patch will be coadded. A list of the available optional 233 arguments can be obtained by calling assembleCoadd.py with the --help command line argument: 235 assembleCoadd.py --help 237 To demonstrate usage of the AssembleCoaddTask in the larger context of multi-band processing, we will generate 238 the HSC-I & -R band coadds from HSC engineering test data provided in the ci_hsc package. To begin, assuming 239 that the lsst stack has been already set up, we must set up the obs_subaru and ci_hsc packages. 240 This defines the environment variable $CI_HSC_DIR and points at the location of the package. The raw HSC 241 data live in the $CI_HSC_DIR/raw directory. To begin assembling the coadds, we must first 244 <DD> process the individual ccds in $CI_HSC_RAW to produce calibrated exposures</DD> 246 <DD> create a skymap that covers the area of the sky present in the raw exposures</DD> 247 <DT>makeCoaddTempExp</DT> 248 <DD> warp the individual calibrated exposures to the tangent plane of the coadd</DD> 250 We can perform all of these steps by running 252 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988 254 This will produce warped exposures for each visit. To coadd the warped data, we call assembleCoadd.py as 257 assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \ 258 --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \ 259 --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \ 260 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \ 261 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \ 262 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \ 263 --selectId visit=903988 ccd=24 265 that will process the HSC-I band data. The results are written in 266 `$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I`. 268 You may also choose to run: 270 scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346 271 assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R \ 272 --selectId visit=903334 ccd=16 --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 \ 273 --selectId visit=903334 ccd=100 --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 \ 274 --selectId visit=903338 ccd=18 --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 \ 275 --selectId visit=903342 ccd=10 --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 \ 276 --selectId visit=903344 ccd=5 --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 \ 277 --selectId visit=903346 ccd=6 --selectId visit=903346 ccd=12 279 to generate the coadd for the HSC-R band if you are interested in following multiBand Coadd processing as 280 discussed in \ref pipeTasks_multiBand (but note that normally, one would use the 281 \ref SafeClipAssembleCoaddTask_ "SafeClipAssembleCoaddTask" rather than AssembleCoaddTask to make the coadd. 283 ConfigClass = AssembleCoaddConfig
284 _DefaultName =
"assembleCoadd" 288 \brief Initialize the task. Create the \ref InterpImageTask "interpImage", 289 & \ref ScaleZeroPointTask "scaleZeroPoint" subtasks. 291 CoaddBaseTask.__init__(self, *args, **kwargs)
292 self.makeSubtask(
"interpImage")
293 self.makeSubtask(
"scaleZeroPoint")
295 if self.config.doMaskBrightObjects:
296 mask = afwImage.Mask()
299 except pexExceptions.LsstCppException:
300 raise RuntimeError(
"Unable to define mask plane for bright objects; planes used are %s" %
301 mask.getMaskPlaneDict().keys())
307 def run(self, dataRef, selectDataList=[]):
309 \brief Assemble a coadd from a set of Warps 311 Coadd a set of Warps. Compute weights to be applied to each Warp and find scalings to 312 match the photometric zeropoint to a reference Warp. Assemble the Warps using 313 \ref assemble. Interpolate over NaNs and optionally write the coadd to disk. Return the coadded 317 \param[in] dataRef: Data reference defining the patch for coaddition and the reference Warp 318 (if config.autoReference=False). Used to access the following data products: 319 - [in] self.config.coaddName + "Coadd_skyMap" 320 - [in] self.config.coaddName + "Coadd_ + <warpType> + "Warp" (optionally) 321 - [out] self.config.coaddName + "Coadd" 322 \param[in] selectDataList[in]: List of data references to Warps. Data to be coadded will be 323 selected from this list based on overlap with the patch defined by dataRef. 325 \return a pipeBase.Struct with fields: 326 - coaddExposure: coadded exposure 327 - nImage: exposure count image 330 calExpRefList = self.
selectExposures(dataRef, skyInfo, selectDataList=selectDataList)
331 if len(calExpRefList) == 0:
332 self.log.warn(
"No exposures to coadd")
334 self.log.info(
"Coadding %d exposures", len(calExpRefList))
338 self.log.info(
"Found %d %s", len(inputData.tempExpRefList),
340 if len(inputData.tempExpRefList) == 0:
341 self.log.warn(
"No coadd temporary exposures found")
346 retStruct = self.
assemble(skyInfo, inputData.tempExpRefList, inputData.imageScalerList,
347 inputData.weightList, supplementaryData=supplementaryData)
349 if self.config.doInterp:
350 self.interpImage.
run(retStruct.coaddExposure.getMaskedImage(), planeName=
"NO_DATA")
352 varArray = retStruct.coaddExposure.getMaskedImage().getVariance().getArray()
353 varArray[:] = numpy.where(varArray > 0, varArray, numpy.inf)
355 if self.config.doMaskBrightObjects:
359 if self.config.doWrite:
362 if retStruct.nImage
is not None:
369 \brief Make additional inputs to assemble() specific to subclasses. 371 Available to be implemented by subclasses only if they need the 372 coadd dataRef for performing preliminary processing before 373 assembling the coadd. 379 \brief Generate list data references corresponding to warped exposures that lie within the 382 \param[in] patchRef: Data reference for patch 383 \param[in] calExpRefList: List of data references for input calexps 384 \return List of Warp/CoaddTempExp data references 386 butler = patchRef.getButler()
387 groupData =
groupPatchExposures(patchRef, calExpRefList, self.getCoaddDatasetName(self.warpType),
388 self.getTempExpDatasetName(self.warpType))
389 tempExpRefList = [
getGroupDataRef(butler, self.getTempExpDatasetName(self.warpType),
390 g, groupData.keys)
for 391 g
in groupData.groups.keys()]
392 return tempExpRefList
396 \brief Prepare the input warps for coaddition by measuring the weight for each warp and the scaling 397 for the photometric zero point. 399 Each Warp has its own photometric zeropoint and background variance. Before coadding these 400 Warps together, compute a scale factor to normalize the photometric zeropoint and compute the 401 weight for each Warp. 403 \param[in] refList: List of data references to tempExp 405 - tempExprefList: List of data references to tempExp 406 - weightList: List of weightings 407 - imageScalerList: List of image scalers 409 statsCtrl = afwMath.StatisticsControl()
410 statsCtrl.setNumSigmaClip(self.config.sigmaClip)
411 statsCtrl.setNumIter(self.config.clipIter)
413 statsCtrl.setNanSafe(
True)
421 for tempExpRef
in refList:
422 if not tempExpRef.datasetExists(tempExpName):
423 self.log.warn(
"Could not find %s %s; skipping it", tempExpName, tempExpRef.dataId)
426 tempExp = tempExpRef.get(tempExpName, immediate=
True)
427 maskedImage = tempExp.getMaskedImage()
428 imageScaler = self.scaleZeroPoint.computeImageScaler(
433 imageScaler.scaleMaskedImage(maskedImage)
434 except Exception
as e:
435 self.log.warn(
"Scaling failed for %s (skipping it): %s", tempExpRef.dataId, e)
437 statObj = afwMath.makeStatistics(maskedImage.getVariance(), maskedImage.getMask(),
438 afwMath.MEANCLIP, statsCtrl)
439 meanVar, meanVarErr = statObj.getResult(afwMath.MEANCLIP)
440 weight = 1.0 / float(meanVar)
441 if not numpy.isfinite(weight):
442 self.log.warn(
"Non-finite weight for %s: skipping", tempExpRef.dataId)
444 self.log.info(
"Weight of %s %s = %0.3f", tempExpName, tempExpRef.dataId, weight)
449 tempExpRefList.append(tempExpRef)
450 weightList.append(weight)
451 imageScalerList.append(imageScaler)
453 return pipeBase.Struct(tempExpRefList=tempExpRefList, weightList=weightList,
454 imageScalerList=imageScalerList)
456 def assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList,
457 altMaskList=None, mask=None, supplementaryData=None):
459 \anchor AssembleCoaddTask.assemble_ 461 \brief Assemble a coadd from input warps 463 Assemble the coadd using the provided list of coaddTempExps. Since the full coadd covers a patch (a 464 large area), the assembly is performed over small areas on the image at a time in order to 465 conserve memory usage. Iterate over subregions within the outer bbox of the patch using 466 \ref assembleSubregion to stack the corresponding subregions from the coaddTempExps with the 467 statistic specified. Set the edge bits the coadd mask based on the weight map. 469 \param[in] skyInfo: Patch geometry information, from getSkyInfo 470 \param[in] tempExpRefList: List of data references to Warps (previously called CoaddTempExps) 471 \param[in] imageScalerList: List of image scalers 472 \param[in] weightList: List of weights 473 \param[in] altMaskList: List of alternate masks to use rather than those stored with tempExp, or None 474 \param[in] mask: Mask to ignore when coadding 475 \param[in] supplementaryData: pipeBase.Struct with additional data products needed to assemble coadd. 476 Only used by subclasses that implement makeSupplementaryData and override assemble. 477 \return pipeBase.Struct with coaddExposure, nImage if requested 480 self.log.info(
"Assembling %s %s", len(tempExpRefList), tempExpName)
484 statsCtrl = afwMath.StatisticsControl()
485 statsCtrl.setNumSigmaClip(self.config.sigmaClip)
486 statsCtrl.setNumIter(self.config.clipIter)
487 statsCtrl.setAndMask(mask)
488 statsCtrl.setNanSafe(
True)
489 statsCtrl.setWeighted(
True)
490 statsCtrl.setCalcErrorFromInputVariance(
True)
491 for plane, threshold
in self.config.maskPropagationThresholds.items():
492 bit = afwImage.Mask.getMaskPlane(plane)
493 statsCtrl.setMaskPropagationThreshold(bit, threshold)
495 statsFlags = afwMath.stringToStatisticsProperty(self.config.statistic)
497 if altMaskList
is None:
498 altMaskList = [
None]*len(tempExpRefList)
500 coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
501 coaddExposure.setCalib(self.scaleZeroPoint.getCalib())
502 coaddExposure.getInfo().setCoaddInputs(self.inputRecorder.makeCoaddInputs())
504 coaddMaskedImage = coaddExposure.getMaskedImage()
505 subregionSizeArr = self.config.subregionSize
506 subregionSize = afwGeom.Extent2I(subregionSizeArr[0], subregionSizeArr[1])
508 if self.config.doNImage:
509 nImage = afwImage.ImageU(skyInfo.bbox)
512 for subBBox
in _subBBoxIter(skyInfo.bbox, subregionSize):
515 weightList, altMaskList, statsFlags, statsCtrl,
517 except Exception
as e:
518 self.log.fatal(
"Cannot compute coadd %s: %s", subBBox, e)
520 self.
setEdge(coaddMaskedImage.getMask())
524 coaddUtils.setCoaddEdgeBits(coaddMaskedImage.getMask(), coaddMaskedImage.getVariance())
525 return pipeBase.Struct(coaddExposure=coaddExposure, nImage=nImage)
529 \brief Set the metadata for the coadd 531 This basic implementation simply sets the filter from the 534 \param[in] coaddExposure: The target image for the coadd 535 \param[in] tempExpRefList: List of data references to tempExp 536 \param[in] weightList: List of weights 538 assert len(tempExpRefList) == len(weightList),
"Length mismatch" 543 tempExpList = [tempExpRef.get(tempExpName +
"_sub",
544 bbox=afwGeom.Box2I(afwGeom.Point2I(0, 0), afwGeom.Extent2I(1, 1)),
545 imageOrigin=
"LOCAL", immediate=
True)
for tempExpRef
in tempExpRefList]
546 numCcds = sum(len(tempExp.getInfo().getCoaddInputs().ccds)
for tempExp
in tempExpList)
548 coaddExposure.setFilter(tempExpList[0].getFilter())
549 coaddInputs = coaddExposure.getInfo().getCoaddInputs()
550 coaddInputs.ccds.reserve(numCcds)
551 coaddInputs.visits.reserve(len(tempExpList))
553 for tempExp, weight
in zip(tempExpList, weightList):
554 self.inputRecorder.addVisitToCoadd(coaddInputs, tempExp, weight)
555 coaddInputs.visits.sort()
561 modelPsfList = [tempExp.getPsf()
for tempExp
in tempExpList]
562 modelPsfWidthList = [modelPsf.computeBBox().getWidth()
for modelPsf
in modelPsfList]
563 psf = modelPsfList[modelPsfWidthList.index(max(modelPsfWidthList))]
565 psf = measAlg.CoaddPsf(coaddInputs.ccds, coaddExposure.getWcs(),
566 self.config.coaddPsf.makeControl())
567 coaddExposure.setPsf(psf)
568 apCorrMap = measAlg.makeCoaddApCorrMap(coaddInputs.ccds, coaddExposure.getBBox(afwImage.PARENT),
569 coaddExposure.getWcs())
570 coaddExposure.getInfo().setApCorrMap(apCorrMap)
572 def assembleSubregion(self, coaddExposure, bbox, tempExpRefList, imageScalerList, weightList,
573 altMaskList, statsFlags, statsCtrl, nImage=None):
575 \brief Assemble the coadd for a sub-region. 577 For each coaddTempExp, check for (and swap in) an alternative mask if one is passed. Remove mask 578 planes listed in config.removeMaskPlanes, Finally, stack the actual exposures using 579 \ref afwMath.statisticsStack "statisticsStack" with the statistic specified 580 by statsFlags. Typically, the statsFlag will be one of afwMath.MEAN for a mean-stack or 581 afwMath.MEANCLIP for outlier rejection using an N-sigma clipped mean where N and iterations 582 are specified by statsCtrl. Assign the stacked subregion back to the coadd. 584 \param[in] coaddExposure: The target image for the coadd 585 \param[in] bbox: Sub-region to coadd 586 \param[in] tempExpRefList: List of data reference to tempExp 587 \param[in] imageScalerList: List of image scalers 588 \param[in] weightList: List of weights 589 \param[in] altMaskList: List of alternate masks to use rather than those stored with tempExp, or None 590 \param[in] statsFlags: afwMath.Property object for statistic for coadd 591 \param[in] statsCtrl: Statistics control object for coadd 592 \param[in] nImage: optional ImageU keeps track of exposure count for each pixel 594 self.log.debug(
"Computing coadd over %s", bbox)
596 coaddMaskedImage = coaddExposure.getMaskedImage()
597 coaddMaskedImage.getMask().addMaskPlane(
"CLIPPED")
599 if nImage
is not None:
600 subNImage = afwImage.ImageU(bbox.getWidth(), bbox.getHeight())
601 for tempExpRef, imageScaler, altMask
in zip(tempExpRefList, imageScalerList, altMaskList):
602 exposure = tempExpRef.get(tempExpName +
"_sub", bbox=bbox)
603 maskedImage = exposure.getMaskedImage()
605 altMaskSub = altMask.Factory(altMask, bbox, afwImage.PARENT)
606 maskedImage.getMask().swap(altMaskSub)
607 imageScaler.scaleMaskedImage(maskedImage)
611 if nImage
is not None:
612 subNImage.getArray()[maskedImage.getMask().getArray() & statsCtrl.getAndMask() == 0] += 1
613 if self.config.removeMaskPlanes:
614 mask = maskedImage.getMask()
615 for maskPlane
in self.config.removeMaskPlanes:
617 mask &= ~mask.getPlaneBitMask(maskPlane)
618 except Exception
as e:
619 self.log.warn(
"Unable to remove mask plane %s: %s", maskPlane, e.args[0])
621 maskedImageList.append(maskedImage)
623 with self.timer(
"stack"):
624 coaddSubregion = afwMath.statisticsStack(maskedImageList, statsFlags, statsCtrl, weightList,
625 coaddMaskedImage.getMask().getPlaneBitMask(
"CLIPPED"),
626 coaddMaskedImage.getMask().getPlaneBitMask(
"NO_DATA"))
627 coaddMaskedImage.assign(coaddSubregion, bbox)
628 if nImage
is not None:
629 nImage.assign(subNImage, bbox)
632 """Returns None on failure""" 634 return dataRef.get(
"brightObjectMask", immediate=
True)
635 except Exception
as e:
636 self.log.warn(
"Unable to read brightObjectMask for %s: %s", dataRef.dataId, e)
640 """Set the bright object masks 642 exposure: Exposure under consideration 643 dataId: Data identifier dict for patch 644 brightObjectMasks: afwTable of bright objects to mask 649 if brightObjectMasks
is None:
650 self.log.warn(
"Unable to apply bright object mask: none supplied")
652 self.log.info(
"Applying %d bright object masks to %s", len(brightObjectMasks), dataId)
653 md = brightObjectMasks.table.getMetadata()
656 self.log.warn(
"Expected to see %s in metadata", k)
658 if md.get(k) != dataId[k]:
659 self.log.warn(
"Expected to see %s == %s in metadata, saw %s", k, md.get(k), dataId[k])
661 mask = exposure.getMaskedImage().getMask()
662 wcs = exposure.getWcs()
663 plateScale = wcs.pixelScale().asArcseconds()
665 for rec
in brightObjectMasks:
666 center = afwGeom.PointI(wcs.skyToPixel(rec.getCoord()))
667 if rec[
"type"] ==
"box":
668 assert rec[
"angle"] == 0.0, (
"Angle != 0 for mask object %s" % rec[
"id"])
669 width = rec[
"width"].asArcseconds()/plateScale
670 height = rec[
"height"].asArcseconds()/plateScale
672 halfSize = afwGeom.ExtentI(0.5*width, 0.5*height)
673 bbox = afwGeom.Box2I(center - halfSize, center + halfSize)
675 bbox = afwGeom.BoxI(afwGeom.PointI(int(center[0] - 0.5*width), int(center[1] - 0.5*height)),
676 afwGeom.PointI(int(center[0] + 0.5*width), int(center[1] + 0.5*height)))
677 spans = afwGeom.SpanSet(bbox)
678 elif rec[
"type"] ==
"circle":
679 radius = int(rec[
"radius"].asArcseconds()/plateScale)
680 spans = afwGeom.SpanSet.fromShape(radius, offset=center)
682 self.log.warn(
"Unexpected region type %s at %s" % rec[
"type"], center)
687 """Set EDGE bits as SENSOR_EDGE 689 The EDGE pixels from the individual CCDs have printed through into the 690 coadd, but EDGE means "we couldn't search for sources in this area 691 because we couldn't convolve by the PSF near the edge of the image", 692 so this mask plane needs to be converted to something else if we want 693 to keep them. We do want to be able to identify pixels near the edge 694 of the detector because they will have an inexact `CoaddPsf`. We 695 rename EDGE pixels as SENSOR_EDGE. 699 mask : `lsst.afw.image.Mask` 700 Coadded exposure's mask, modified in-place. 702 mask.addMaskPlane(
"SENSOR_EDGE")
703 edge = mask.getPlaneBitMask(
"EDGE")
704 sensorEdge = mask.getPlaneBitMask(
"SENSOR_EDGE")
705 array = mask.getArray()
706 selected = (array & edge > 0)
707 array[selected] |= sensorEdge
708 array[selected] &= ~edge
711 """Set INEXACT_PSF mask plane 713 If any of the input images isn't represented in the coadd (due to 714 clipped pixels or chip gaps), the `CoaddPsf` will be inexact. Flag 719 mask : `lsst.afw.image.Mask` 720 Coadded exposure's mask, modified in-place. 722 mask.addMaskPlane(
"INEXACT_PSF")
723 inexactPsf = mask.getPlaneBitMask(
"INEXACT_PSF")
724 sensorEdge = mask.getPlaneBitMask(
"SENSOR_EDGE")
725 clipped = mask.getPlaneBitMask(
"CLIPPED")
726 array = mask.getArray()
727 selected = array & (sensorEdge | clipped) > 0
728 array[selected] |= inexactPsf
731 def _makeArgumentParser(cls):
733 \brief Create an argument parser 736 parser.add_id_argument(
"--id", cls.
ConfigClass().coaddName +
"Coadd_" +
738 help=
"data ID, e.g. --id tract=12345 patch=1,2",
739 ContainerClass=AssembleCoaddDataIdContainer)
740 parser.add_id_argument(
"--selectId",
"calexp", help=
"data ID, e.g. --selectId visit=6789 ccd=0..9",
741 ContainerClass=SelectDataIdContainer)
745 def _subBBoxIter(bbox, subregionSize):
747 \brief Iterate over subregions of a bbox 749 \param[in] bbox: bounding box over which to iterate: afwGeom.Box2I 750 \param[in] subregionSize: size of sub-bboxes 752 \return subBBox: next sub-bounding box of size subregionSize or smaller; 753 each subBBox is contained within bbox, so it may be smaller than subregionSize at the edges of bbox, 754 but it will never be empty 757 raise RuntimeError(
"bbox %s is empty" % (bbox,))
758 if subregionSize[0] < 1
or subregionSize[1] < 1:
759 raise RuntimeError(
"subregionSize %s must be nonzero" % (subregionSize,))
761 for rowShift
in range(0, bbox.getHeight(), subregionSize[1]):
762 for colShift
in range(0, bbox.getWidth(), subregionSize[0]):
763 subBBox = afwGeom.Box2I(bbox.getMin() + afwGeom.Extent2I(colShift, rowShift), subregionSize)
765 if subBBox.isEmpty():
766 raise RuntimeError(
"Bug: empty bbox! bbox=%s, subregionSize=%s, colShift=%s, rowShift=%s" %
767 (bbox, subregionSize, colShift, rowShift))
773 \brief A version of lsst.pipe.base.DataIdContainer specialized for assembleCoadd. 778 \brief Make self.refList from self.idList. 780 datasetType = namespace.config.coaddName +
"Coadd" 781 keysCoadd = namespace.butler.getKeys(datasetType=datasetType, level=self.level)
783 for dataId
in self.idList:
785 for key
in keysCoadd:
786 if key
not in dataId:
787 raise RuntimeError(
"--id must include " + key)
789 dataRef = namespace.butler.dataRef(
790 datasetType=datasetType,
793 self.refList.append(dataRef)
798 \brief Function to count the number of pixels with a specific mask in a footprint. 800 Find the intersection of mask & footprint. Count all pixels in the mask that are in the intersection that 801 have bitmask set but do not have ignoreMask set. Return the count. 803 \param[in] mask: mask to define intersection region by. 804 \parma[in] footprint: footprint to define the intersection region by. 805 \param[in] bitmask: specific mask that we wish to count the number of occurances of. 806 \param[in] ignoreMask: pixels to not consider. 807 \return count of number of pixels in footprint with specified mask. 809 bbox = footprint.getBBox()
810 bbox.clip(mask.getBBox(afwImage.PARENT))
811 fp = afwImage.Mask(bbox)
812 subMask = mask.Factory(mask, bbox, afwImage.PARENT)
813 footprint.spans.setMask(fp, bitmask)
814 return numpy.logical_and((subMask.getArray() & fp.getArray()) > 0,
815 (subMask.getArray() & ignoreMask) == 0).sum()
820 \anchor SafeClipAssembleCoaddConfig 822 \brief Configuration parameters for the SafeClipAssembleCoaddTask 824 clipDetection = pexConfig.ConfigurableField(
825 target=SourceDetectionTask,
826 doc=
"Detect sources on difference between unclipped and clipped coadd")
827 minClipFootOverlap = pexConfig.Field(
828 doc=
"Minimum fractional overlap of clipped footprint with visit DETECTED to be clipped",
832 minClipFootOverlapSingle = pexConfig.Field(
833 doc=
"Minimum fractional overlap of clipped footprint with visit DETECTED to be " 834 "clipped when only one visit overlaps",
838 minClipFootOverlapDouble = pexConfig.Field(
839 doc=
"Minimum fractional overlap of clipped footprints with visit DETECTED to be " 840 "clipped when two visits overlap",
844 maxClipFootOverlapDouble = pexConfig.Field(
845 doc=
"Maximum fractional overlap of clipped footprints with visit DETECTED when " 846 "considering two visits",
850 minBigOverlap = pexConfig.Field(
851 doc=
"Minimum number of pixels in footprint to use DETECTED mask from the single visits " 852 "when labeling clipped footprints",
860 AssembleCoaddConfig.setDefaults(self)
876 log.warn(
"Additional Sigma-clipping not allowed in Safe-clipped Coadds. " 877 "Ignoring doSigmaClip.")
880 raise ValueError(
"Only MEAN statistic allowed for final stacking in SafeClipAssembleCoadd " 881 "(%s chosen). Please set statistic to MEAN." 883 AssembleCoaddTask.ConfigClass.validate(self)
896 \anchor SafeClipAssembleCoaddTask_ 898 \brief Assemble a coadded image from a set of coadded temporary exposures, 899 being careful to clip & flag areas with potential artifacts. 901 \section pipe_tasks_assembleCoadd_Contents Contents 902 - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Purpose 903 - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Initialize 904 - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Run 905 - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Config 906 - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Debug 907 - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Example 909 \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Purpose Description 911 \copybrief SafeClipAssembleCoaddTask 913 Read the documentation for \ref AssembleCoaddTask_ "AssembleCoaddTask" first since 914 SafeClipAssembleCoaddTask subtasks that task. 915 In \ref AssembleCoaddTask_ "AssembleCoaddTask", we compute the coadd as an clipped mean (i.e. we clip 917 The problem with doing this is that when computing the coadd PSF at a given location, individual visit 918 PSFs from visits with outlier pixels contribute to the coadd PSF and cannot be treated correctly. 919 In this task, we correct for this behavior by creating a new badMaskPlane 'CLIPPED'. 920 We populate this plane on the input coaddTempExps and the final coadd where i. difference imaging suggests 921 that there is an outlier and ii. this outlier appears on only one or two images. 922 Such regions will not contribute to the final coadd. 923 Furthermore, any routine to determine the coadd PSF can now be cognizant of clipped regions. 924 Note that the algorithm implemented by this task is preliminary and works correctly for HSC data. 925 Parameter modifications and or considerable redesigning of the algorithm is likley required for other 928 SafeClipAssembleCoaddTask uses a \ref SourceDetectionTask_ "clipDetection" subtask and also sub-classes 929 \ref AssembleCoaddTask_ "AssembleCoaddTask". You can retarget the 930 \ref SourceDetectionTask_ "clipDetection" subtask if you wish. 932 \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Initialize Task initialization 933 \copydoc \_\_init\_\_ 935 \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Run Invoking the Task 938 \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Config Configuration parameters 939 See \ref SafeClipAssembleCoaddConfig 941 \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Debug Debug variables 942 The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a 943 flag \c -d to import \b debug.py from your \c PYTHONPATH; see \ref baseDebug for more about \b debug.py 945 SafeClipAssembleCoaddTask has no debug variables of its own. The \ref SourceDetectionTask_ "clipDetection" 946 subtasks may support debug variables. See the documetation for \ref SourceDetectionTask_ "clipDetection" 947 for further information. 949 \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Example A complete example of using 950 SafeClipAssembleCoaddTask 952 SafeClipAssembleCoaddTask assembles a set of warped coaddTempExp images into a coadded image. 953 The SafeClipAssembleCoaddTask is invoked by running assembleCoadd.py <em>without</em> the flag 955 Usage of assembleCoadd.py expects a data reference to the tract patch and filter to be coadded 956 (specified using '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]') along 957 with a list of coaddTempExps to attempt to coadd (specified using 958 '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]'). 959 Only the coaddTempExps that cover the specified tract and patch will be coadded. 960 A list of the available optional arguments can be obtained by calling assembleCoadd.py with the --help 961 command line argument: 963 assembleCoadd.py --help 965 To demonstrate usage of the SafeClipAssembleCoaddTask in the larger context of multi-band processing, we 966 will generate the HSC-I & -R band coadds from HSC engineering test data provided in the ci_hsc package. To 967 begin, assuming that the lsst stack has been already set up, we must set up the obs_subaru and ci_hsc 969 This defines the environment variable $CI_HSC_DIR and points at the location of the package. The raw HSC 970 data live in the $CI_HSC_DIR/raw directory. To begin assembling the coadds, we must first 973 <DD> process the individual ccds in $CI_HSC_RAW to produce calibrated exposures</DD> 975 <DD> create a skymap that covers the area of the sky present in the raw exposures</DD> 976 <DT>makeCoaddTempExp</DT> 977 <DD> warp the individual calibrated exposures to the tangent plane of the coadd</DD> 979 We can perform all of these steps by running 981 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988 983 This will produce warped coaddTempExps for each visit. To coadd the warped data, we call assembleCoadd.py 986 assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \ 987 --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \ 988 --selectId visit=903986 ccd=100--selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \ 989 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \ 990 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \ 991 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \ 992 --selectId visit=903988 ccd=24 994 This will process the HSC-I band data. The results are written in 995 `$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I`. 997 You may also choose to run: 999 scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346 1000 assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R --selectId visit=903334 ccd=16 \ 1001 --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 --selectId visit=903334 ccd=100 \ 1002 --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 --selectId visit=903338 ccd=18 \ 1003 --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 --selectId visit=903342 ccd=10 \ 1004 --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 --selectId visit=903344 ccd=5 \ 1005 --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 --selectId visit=903346 ccd=6 \ 1006 --selectId visit=903346 ccd=12 1008 to generate the coadd for the HSC-R band if you are interested in following multiBand Coadd processing as 1009 discussed in \ref pipeTasks_multiBand. 1011 ConfigClass = SafeClipAssembleCoaddConfig
1012 _DefaultName =
"safeClipAssembleCoadd" 1016 \brief Initialize the task and make the \ref SourceDetectionTask_ "clipDetection" subtask. 1018 AssembleCoaddTask.__init__(self, *args, **kwargs)
1019 schema = afwTable.SourceTable.makeMinimalSchema()
1020 self.makeSubtask(
"clipDetection", schema=schema)
1022 def assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList, *args, **kwargs):
1024 \brief Assemble the coadd for a region 1026 Compute the difference of coadds created with and without outlier rejection to identify coadd pixels 1027 that have outlier values in some individual visits. Detect clipped regions on the difference image and 1028 mark these regions on the one or two individual coaddTempExps where they occur if there is significant 1029 overlap between the clipped region and a source. 1030 This leaves us with a set of footprints from the difference image that have been identified as having 1031 occured on just one or two individual visits. However, these footprints were generated from a 1032 difference image. It is conceivable for a large diffuse source to have become broken up into multiple 1033 footprints acrosss the coadd difference in this process. 1034 Determine the clipped region from all overlapping footprints from the detected sources in each visit - 1035 these are big footprints. 1036 Combine the small and big clipped footprints and mark them on a new bad mask plane 1037 Generate the coadd using \ref AssembleCoaddTask.assemble_ "AssembleCoaddTask.assemble" without outlier 1038 removal. Clipped footprints will no longer make it into the coadd because they are marked in the new 1041 N.b. *args and **kwargs are passed but ignored in order to match the call signature expected by the 1044 @param skyInfo: Patch geometry information, from getSkyInfo 1045 @param tempExpRefList: List of data reference to tempExp 1046 @param imageScalerList: List of image scalers 1047 @param weightList: List of weights 1048 return pipeBase.Struct with coaddExposure, nImage 1051 mask = exp.getMaskedImage().getMask()
1052 mask.addMaskPlane(
"CLIPPED")
1054 result = self.
detectClip(exp, tempExpRefList)
1056 self.log.info(
'Found %d clipped objects', len(result.clipFootprints))
1059 maskClipValue = mask.getPlaneBitMask(
"CLIPPED")
1060 maskDetValue = mask.getPlaneBitMask(
"DETECTED") | mask.getPlaneBitMask(
"DETECTED_NEGATIVE")
1061 bigFootprints = self.
detectClipBig(result.tempExpClipList, result.clipFootprints, result.clipIndices,
1062 maskClipValue, maskDetValue)
1065 maskClip = mask.Factory(mask.getBBox(afwImage.PARENT))
1066 afwDet.setMaskFromFootprintList(maskClip, result.clipFootprints, maskClipValue)
1068 maskClipBig = maskClip.Factory(mask.getBBox(afwImage.PARENT))
1069 afwDet.setMaskFromFootprintList(maskClipBig, bigFootprints, maskClipValue)
1070 maskClip |= maskClipBig
1073 badMaskPlanes = self.config.badMaskPlanes[:]
1074 badMaskPlanes.append(
"CLIPPED")
1075 badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes)
1076 return AssembleCoaddTask.assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList,
1077 result.tempExpClipList, mask=badPixelMask)
1081 \brief Return an exposure that contains the difference between and unclipped and clipped coadds. 1083 Generate a difference image between clipped and unclipped coadds. 1084 Compute the difference image by subtracting an outlier-clipped coadd from an outlier-unclipped coadd. 1085 Return the difference image. 1087 @param skyInfo: Patch geometry information, from getSkyInfo 1088 @param tempExpRefList: List of data reference to tempExp 1089 @param imageScalerList: List of image scalers 1090 @param weightList: List of weights 1091 @return Difference image of unclipped and clipped coadd wrapped in an Exposure 1096 configIntersection = {k: getattr(self.config, k)
1097 for k, v
in self.config.toDict().items()
if (k
in config.keys())}
1098 config.update(**configIntersection)
1101 config.statistic =
'MEAN' 1103 coaddMean = task.assemble(skyInfo, tempExpRefList, imageScalerList, weightList).coaddExposure
1105 config.statistic =
'MEANCLIP' 1107 coaddClip = task.assemble(skyInfo, tempExpRefList, imageScalerList, weightList).coaddExposure
1109 coaddDiff = coaddMean.getMaskedImage().Factory(coaddMean.getMaskedImage())
1110 coaddDiff -= coaddClip.getMaskedImage()
1111 exp = afwImage.ExposureF(coaddDiff)
1112 exp.setPsf(coaddMean.getPsf())
1117 \brief Detect clipped regions on an exposure and set the mask on the individual tempExp masks 1119 Detect footprints in the difference image after smoothing the difference image with a Gaussian kernal. 1120 Identify footprints that overlap with one or two input coaddTempExps by comparing the computed overlap 1121 fraction to thresholds set in the config. 1122 A different threshold is applied depending on the number of overlapping visits (restricted to one or 1124 If the overlap exceeds the thresholds, the footprint is considered "CLIPPED" and is marked as such on 1126 Return a struct with the clipped footprints, the indices of the coaddTempExps that end up overlapping 1127 with the clipped footprints and a list of new masks for the coaddTempExps. 1129 \param[in] exp: Exposure to run detection on 1130 \param[in] tempExpRefList: List of data reference to tempExp 1131 \return struct containing: 1132 - clippedFootprints: list of clipped footprints 1133 - clippedIndices: indices for each clippedFootprint in tempExpRefList 1134 - tempExpClipList: list of new masks for tempExp 1136 mask = exp.getMaskedImage().getMask()
1137 maskClipValue = mask.getPlaneBitMask(
"CLIPPED")
1138 maskDetValue = mask.getPlaneBitMask(
"DETECTED") | mask.getPlaneBitMask(
"DETECTED_NEGATIVE")
1139 fpSet = self.clipDetection.detectFootprints(exp, doSmooth=
True, clearMask=
True)
1141 fpSet.positive.merge(fpSet.negative)
1142 footprints = fpSet.positive
1143 self.log.info(
'Found %d potential clipped objects', len(footprints.getFootprints()))
1151 immediate=
True).getMaskedImage().getMask()
for 1152 tmpExpRef
in tempExpRefList]
1154 for footprint
in footprints.getFootprints():
1155 nPixel = footprint.getArea()
1159 for i, tmpExpMask
in enumerate(tempExpClipList):
1163 totPixel = nPixel - ignore
1166 if ignore > overlapDet
or totPixel <= 0.5*nPixel
or overlapDet == 0:
1168 overlap.append(overlapDet/float(totPixel))
1169 maskList.append(tmpExpMask)
1172 overlap = numpy.array(overlap)
1173 if not len(overlap):
1180 if len(overlap) == 1:
1181 if overlap[0] > self.config.minClipFootOverlapSingle:
1186 clipIndex = numpy.where(overlap > self.config.minClipFootOverlap)[0]
1187 if len(clipIndex) == 1:
1189 keepIndex = [clipIndex[0]]
1192 clipIndex = numpy.where(overlap > self.config.minClipFootOverlapDouble)[0]
1193 if len(clipIndex) == 2
and len(overlap) > 3:
1194 clipIndexComp = numpy.where(overlap <= self.config.minClipFootOverlapDouble)[0]
1195 if numpy.max(overlap[clipIndexComp]) <= self.config.maxClipFootOverlapDouble:
1197 keepIndex = clipIndex
1202 for index
in keepIndex:
1203 footprint.spans.setMask(maskList[index], maskClipValue)
1205 clipIndices.append(numpy.array(indexList)[keepIndex])
1206 clipFootprints.append(footprint)
1208 return pipeBase.Struct(clipFootprints=clipFootprints, clipIndices=clipIndices,
1209 tempExpClipList=tempExpClipList)
1211 def detectClipBig(self, tempExpClipList, clipFootprints, clipIndices, maskClipValue, maskDetValue):
1213 \brief Find footprints from individual tempExp footprints for large footprints. 1215 Identify big footprints composed of many sources in the coadd difference that may have originated in a 1216 large diffuse source in the coadd. We do this by indentifying all clipped footprints that overlap 1217 significantly with each source in all the coaddTempExps. 1219 \param[in] tempExpClipList: List of tempExp masks with clipping information 1220 \param[in] clipFootprints: List of clipped footprints 1221 \param[in] clipIndices: List of which entries in tempExpClipList each footprint belongs to 1222 \param[in] maskClipValue: Mask value of clipped pixels 1223 \param[in] maskClipValue: Mask value of detected pixels 1224 \return list of big footprints 1226 bigFootprintsCoadd = []
1228 for index, tmpExpMask
in enumerate(tempExpClipList):
1231 maskVisitDet = tmpExpMask.Factory(tmpExpMask, tmpExpMask.getBBox(afwImage.PARENT),
1232 afwImage.PARENT,
True)
1233 maskVisitDet &= maskDetValue
1234 visitFootprints = afwDet.FootprintSet(maskVisitDet, afwDet.Threshold(1))
1237 clippedFootprintsVisit = []
1238 for foot, clipIndex
in zip(clipFootprints, clipIndices):
1239 if index
not in clipIndex:
1241 clippedFootprintsVisit.append(foot)
1242 maskVisitClip = maskVisitDet.Factory(maskVisitDet.getBBox(afwImage.PARENT))
1243 afwDet.setMaskFromFootprintList(maskVisitClip, clippedFootprintsVisit, maskClipValue)
1245 bigFootprintsVisit = []
1246 for foot
in visitFootprints.getFootprints():
1247 if foot.getArea() < self.config.minBigOverlap:
1250 if nCount > self.config.minBigOverlap:
1251 bigFootprintsVisit.append(foot)
1252 bigFootprintsCoadd.append(foot)
1255 maskVisitClip.clearAllMaskPlanes()
1256 afwDet.setMaskFromFootprintList(maskVisitClip, bigFootprintsVisit, maskClipValue)
1257 tmpExpMask |= maskVisitClip
1259 return bigFootprintsCoadd
1263 assembleStaticSkyModel = pexConfig.ConfigurableField(
1264 target=AssembleCoaddTask,
1265 doc=
"Task to assemble an artifact-free, PSF-matched Coadd to serve as a" 1266 " naive/first-iteration model of the static sky.",
1268 detect = pexConfig.ConfigurableField(
1269 target=SourceDetectionTask,
1270 doc=
"Detect outlier sources on difference between each psfMatched warp and static sky model" 1272 maxNumEpochs = pexConfig.Field(
1273 doc=
"Maximum number of epochs/visits in which an artifact candidate can appear and still be masked. " 1274 "For each footprint detected on the image difference between the psfMatched warp and static sky " 1275 "model, if a significant fraction of pixels (defined by spatialThreshold) are residuals in more " 1276 "than maxNumEpochs, the artifact candidate is persistant rather than transient and not masked.",
1280 maxFractionEpochs = pexConfig.RangeField(
1281 doc=
"Fraction of local number of epochs (N) to use as maxNumEpochs. " 1282 "Effective maxNumEpochs is the lesser of floor(maxNumEpochsMinFraction*N) and maxNumEpochs. ",
1287 spatialThreshold = pexConfig.RangeField(
1288 doc=
"Unitless fraction of pixels defining how much of the outlier region has to meet the " 1289 "temporal criteria. If 0, clip all. If 1, clip none.",
1293 inclusiveMin=
True, inclusiveMax=
True 1297 AssembleCoaddConfig.setDefaults(self)
1305 self.
detect.doTempLocalBackground =
False 1306 self.
detect.reEstimateBackground =
False 1307 self.
detect.returnOriginalFootprints =
False 1308 self.
detect.thresholdPolarity =
"both" 1309 self.
detect.thresholdValue = 5
1310 self.
detect.nSigmaToGrow = 2
1311 self.
detect.minPixels = 4
1312 self.
detect.isotropicGrow =
True 1313 self.
detect.thresholdType =
"pixel_stdev" 1325 \anchor CompareWarpAssembleCoaddTask_ 1327 \brief Assemble a compareWarp coadded image from a set of warps 1328 by masking artifacts detected by comparing PSF-matched warps 1330 \section pipe_tasks_assembleCoadd_Contents Contents 1331 - \ref pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Purpose 1332 - \ref pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Initialize 1333 - \ref pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Run 1334 - \ref pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Config 1335 - \ref pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Debug 1336 - \ref pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Example 1338 \section pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Purpose Description 1340 \copybrief CompareWarpAssembleCoaddTask 1342 In \ref AssembleCoaddTask_ "AssembleCoaddTask", we compute the coadd as an clipped mean (i.e. we clip 1344 The problem with doing this is that when computing the coadd PSF at a given location, individual visit 1345 PSFs from visits with outlier pixels contribute to the coadd PSF and cannot be treated correctly. 1346 In this task, we correct for this behavior by creating a new badMaskPlane 'CLIPPED' which marks 1347 pixels in the individual warps suspected to contain an artifact. 1348 We populate this plane on the input warps by comparing PSF-matched warps with a PSF-matched median coadd 1349 which serves as a model of the static sky. Any group of pixels that deviates from the PSF-matched 1350 template coadd by more than config.detect.threshold sigma, is an artifact candidate. 1351 The candidates are then filtered to remove variable sources and sources that are difficult to subtract 1352 such as bright stars. 1353 This filter is configured using the config parameters temporalThreshold and spatialThreshold. 1354 The temporalThreshold is the maximum fraction of epochs that the deviation can 1355 appear in and still be considered an artifact. The spatialThreshold is the maximum fraction of pixels in 1356 the footprint of the deviation that appear in other epochs (where other epochs is defined by the 1357 temporalThreshold). If the deviant region meets this criteria of having a significant percentage of pixels 1358 that deviate in only a few epochs, these pixels have the 'CLIPPED' bit set in the mask. 1359 These regions will not contribute to the final coadd. 1360 Furthermore, any routine to determine the coadd PSF can now be cognizant of clipped regions. 1361 Note that the algorithm implemented by this task is preliminary and works correctly for HSC data. 1362 Parameter modifications and or considerable redesigning of the algorithm is likley required for other 1365 CompareWarpAssembleCoaddTask sub-classes 1366 \ref AssembleCoaddTask_ "AssembleCoaddTask" and instantiates \ref AssembleCoaddTask_ "AssembleCoaddTask" 1367 as a subtask to generate the TemplateCoadd (the model of the static sky) 1369 \section pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Initialize Task initialization 1370 \copydoc \_\_init\_\_ 1372 \section pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Run Invoking the Task 1375 \section pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Config Configuration parameters 1376 See \ref CompareWarpAssembleCoaddConfig 1378 \section pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Debug Debug variables 1379 The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a 1380 flag \c -d to import \b debug.py from your \c PYTHONPATH; see \ref baseDebug for more about \b debug.py 1383 This task supports the following debug variables: 1386 <dd> If True then save the Epoch Count Image as a fits file in the `figPath` 1388 <dd> If True then save the new masks with CLIPPED planes as fits files to the `figPath` 1390 <dd> Path to save the debug fits images and figures 1393 For example, put something like: 1396 def DebugInfo(name): 1397 di = lsstDebug.getInfo(name) 1398 if name == "lsst.pipe.tasks.assembleCoadd": 1399 di.saveCountIm = True 1400 di.saveAltMask = True 1401 di.figPath = "/desired/path/to/debugging/output/images" 1403 lsstDebug.Info = DebugInfo 1405 into your `debug.py` file and run `assemebleCoadd.py` with the `--debug` 1407 Some subtasks may have their own debug variables; see individual Task 1410 \section pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Example A complete example of using 1411 CompareWarpAssembleCoaddTask 1413 CompareWarpAssembleCoaddTask assembles a set of warped images into a coadded image. 1414 The CompareWarpAssembleCoaddTask is invoked by running assembleCoadd.py with the flag 1415 '--compareWarpCoadd'. 1416 Usage of assembleCoadd.py expects a data reference to the tract patch and filter to be coadded 1417 (specified using '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]') along 1418 with a list of coaddTempExps to attempt to coadd (specified using 1419 '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]'). 1420 Only the warps that cover the specified tract and patch will be coadded. 1421 A list of the available optional arguments can be obtained by calling assembleCoadd.py with the --help 1422 command line argument: 1424 assembleCoadd.py --help 1426 To demonstrate usage of the CompareWarpAssembleCoaddTask in the larger context of multi-band processing, 1427 we will generate the HSC-I & -R band coadds from HSC engineering test data provided in the ci_hsc package. 1428 To begin, assuming that the lsst stack has been already set up, we must set up the obs_subaru and ci_hsc 1430 This defines the environment variable $CI_HSC_DIR and points at the location of the package. The raw HSC 1431 data live in the $CI_HSC_DIR/raw directory. To begin assembling the coadds, we must first 1434 <DD> process the individual ccds in $CI_HSC_RAW to produce calibrated exposures</DD> 1436 <DD> create a skymap that covers the area of the sky present in the raw exposures</DD> 1437 <DT>makeCoaddTempExp</DT> 1438 <DD> warp the individual calibrated exposures to the tangent plane of the coadd</DD> 1440 We can perform all of these steps by running 1442 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988 1444 This will produce warped coaddTempExps for each visit. To coadd the warped data, we call assembleCoadd.py 1447 assembleCoadd.py --compareWarpCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \ 1448 --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \ 1449 --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \ 1450 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \ 1451 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \ 1452 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \ 1453 --selectId visit=903988 ccd=24 1455 This will process the HSC-I band data. The results are written in 1456 `$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I`. 1458 ConfigClass = CompareWarpAssembleCoaddConfig
1459 _DefaultName =
"compareWarpAssembleCoadd" 1463 \brief Initialize the task and make the \ref AssembleCoadd_ "assembleStaticSkyModel" subtask. 1465 AssembleCoaddTask.__init__(self, *args, **kwargs)
1466 self.makeSubtask(
"assembleStaticSkyModel")
1467 detectionSchema = afwTable.SourceTable.makeMinimalSchema()
1468 self.makeSubtask(
"detect", schema=detectionSchema)
1472 \brief Make inputs specific to Subclass 1474 Generate a templateCoadd to use as a native model of static sky to subtract from warps. 1476 templateCoadd = self.assembleStaticSkyModel.
run(dataRef, selectDataList)
1478 if templateCoadd
is None:
1479 warpName = (self.assembleStaticSkyModel.warpType[0].upper() +
1480 self.assembleStaticSkyModel.warpType[1:])
1481 message =
"""No %(warpName)s warps were found to build the template coadd which is 1482 required to run CompareWarpAssembleCoaddTask. To continue assembling this type of coadd, 1483 first either rerun makeCoaddTempExp with config.make%(warpName)s=True or 1484 coaddDriver with config.makeCoadTempExp.make%(warpName)s=True, before assembleCoadd. 1486 Alternatively, to use another algorithm with existing warps, retarget the CoaddDriverConfig to 1487 another algorithm like: 1489 from lsst.pipe.tasks.assembleCoadd import SafeClipAssembleCoaddTask 1490 config.assemble.retarget(SafeClipAssembleCoaddTask) 1491 """ % {
"warpName": warpName}
1492 raise RuntimeError(message)
1494 return pipeBase.Struct(templateCoadd=templateCoadd.coaddExposure)
1496 def assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList,
1497 supplementaryData, *args, **kwargs):
1499 \brief Assemble the coadd 1501 Requires additional inputs Struct `supplementaryData` to contain a `templateCoadd` that serves 1502 as the model of the static sky. 1504 Find artifacts and apply them to the warps' masks creating a list of alternative masks with a 1505 new "CLIPPED" plane and updated "NO_DATA" plane. 1506 Then pass these alternative masks to the base class's assemble method. 1508 @param skyInfo: Patch geometry information 1509 @param tempExpRefList: List of data references to warps 1510 @param imageScalerList: List of image scalers 1511 @param weightList: List of weights 1512 @param supplementaryData: PipeBase.Struct containing a templateCoadd 1514 return pipeBase.Struct with coaddExposure, nImage if requested 1516 templateCoadd = supplementaryData.templateCoadd
1517 spanSetMaskList = self.
findArtifacts(templateCoadd, tempExpRefList, imageScalerList)
1519 badMaskPlanes = self.config.badMaskPlanes[:]
1520 badMaskPlanes.append(
"CLIPPED")
1521 badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes)
1523 return AssembleCoaddTask.assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList,
1524 maskList, mask=badPixelMask)
1528 \brief Find artifacts 1530 Loop through warps twice. The first loop builds a map with the count of how many 1531 epochs each pixel deviates from the templateCoadd by more than config.chiThreshold sigma. 1532 The second loop takes each difference image and filters the artifacts detected 1533 in each using count map to filter out variable sources and sources that are difficult to 1536 @param templateCoadd: Exposure to serve as model of static sky 1537 @param tempExpRefList: List of data references to warps 1538 @param imageScalerList: List of image scalers 1541 self.log.debug(
"Generating Count Image, and mask lists.")
1542 coaddBBox = templateCoadd.getBBox()
1543 slateIm = afwImage.ImageU(coaddBBox)
1544 epochCountImage = afwImage.ImageU(coaddBBox)
1545 nImage = afwImage.ImageU(coaddBBox)
1546 spanSetArtifactList = []
1547 spanSetNoDataMaskList = []
1549 for warpRef, imageScaler
in zip(tempExpRefList, imageScalerList):
1551 if warpDiffExp
is not None:
1552 nImage.array += numpy.where(numpy.isnan(warpDiffExp.image.array),
1553 0, 1).astype(numpy.uint16)
1554 fpSet = self.detect.detectFootprints(warpDiffExp, doSmooth=
False, clearMask=
True)
1555 fpSet.positive.merge(fpSet.negative)
1556 footprints = fpSet.positive
1558 spanSetList = [footprint.spans
for footprint
in footprints.getFootprints()]
1559 for spans
in spanSetList:
1560 spans.setImage(slateIm, 1, doClip=
True)
1561 epochCountImage += slateIm
1567 nans = numpy.where(numpy.isnan(warpDiffExp.maskedImage.image.array), 1, 0)
1568 nansMask = afwImage.makeMaskFromArray(nans.astype(afwImage.MaskPixel))
1569 nansMask.setXY0(warpDiffExp.getXY0())
1573 nansMask = afwImage.MaskX(coaddBBox, 1)
1576 spanSetNoDataMask = afwGeom.SpanSet.fromMask(nansMask).split()
1578 spanSetNoDataMaskList.append(spanSetNoDataMask)
1579 spanSetArtifactList.append(spanSetList)
1583 epochCountImage.writeFits(path)
1585 for i, spanSetList
in enumerate(spanSetArtifactList):
1587 filteredSpanSetList = self.
_filterArtifacts(spanSetList, epochCountImage, nImage)
1588 spanSetArtifactList[i] = filteredSpanSetList
1590 return pipeBase.Struct(artifacts=spanSetArtifactList,
1591 noData=spanSetNoDataMaskList)
1595 \brief Apply artifact span set lists to masks 1597 @param tempExpRefList: List of data references to warps 1598 @param maskSpanSets: Struct containing artifact and noData spanSet lists to apply 1600 return List of alternative masks 1602 Add artifact span set list as "CLIPPED" plane and NaNs to existing "NO_DATA" plane 1604 spanSetMaskList = maskSpanSets.artifacts
1605 spanSetNoDataList = maskSpanSets.noData
1607 for warpRef, artifacts, noData
in zip(tempExpRefList, spanSetMaskList, spanSetNoDataList):
1609 mask = warp.maskedImage.mask
1610 maskClipValue = mask.addMaskPlane(
"CLIPPED")
1611 noDataValue = mask.addMaskPlane(
"NO_DATA")
1612 for artifact
in artifacts:
1613 artifact.clippedTo(mask.getBBox()).setMask(mask, 2**maskClipValue)
1614 for noDataRegion
in noData:
1615 noDataRegion.clippedTo(mask.getBBox()).setMask(mask, 2**noDataValue)
1616 altMaskList.append(mask)
1622 def _filterArtifacts(self, spanSetList, epochCountImage, nImage):
1624 \brief Filter artifact candidates 1626 @param spanSetList: List of SpanSets representing artifact candidates 1627 @param epochCountImage: Image of accumulated number of warpDiff detections 1628 @param nImage: Image of the accumulated number of total epochs contributing 1630 return List of SpanSets with artifacts 1633 maskSpanSetList = []
1634 x0, y0 = epochCountImage.getXY0()
1635 for i, span
in enumerate(spanSetList):
1636 y, x = span.indices()
1637 yIdxLocal = [y1 - y0
for y1
in y]
1638 xIdxLocal = [x1 - x0
for x1
in x]
1639 outlierN = epochCountImage.array[yIdxLocal, xIdxLocal]
1640 totalN = nImage.array[yIdxLocal, xIdxLocal]
1641 effectiveMaxNumEpochs = min(self.config.maxNumEpochs,
1642 int(self.config.maxFractionEpochs * numpy.mean(totalN)))
1643 nPixelsBelowThreshold = numpy.count_nonzero((outlierN > 0) &
1644 (outlierN <= effectiveMaxNumEpochs))
1645 percentBelowThreshold = nPixelsBelowThreshold / len(outlierN)
1646 if percentBelowThreshold > self.config.spatialThreshold:
1647 maskSpanSetList.append(span)
1648 return maskSpanSetList
1650 def _readAndComputeWarpDiff(self, warpRef, imageScaler, templateCoadd):
1652 \brief Fetch a warp from the butler and return a warpDiff 1654 @param warpRef: `Butler dataRef` for the warp 1655 @param imageScaler: `scaleZeroPoint.ImageScaler` object 1656 @param templateCoadd: Exposure to be substracted from the scaled warp 1658 return Exposure of the image difference between the warp and template 1663 if not warpRef.datasetExists(warpName):
1664 self.log.warn(
"Could not find %s %s; skipping it", warpName, warpRef.dataId)
1666 warp = warpRef.get(warpName, immediate=
True)
1668 imageScaler.scaleMaskedImage(warp.getMaskedImage())
1669 mi = warp.getMaskedImage()
1670 mi -= templateCoadd.getMaskedImage()
1673 def _dataRef2DebugPath(self, prefix, warpRef, coaddLevel=False):
1675 \brief Return a path to which to write debugging output 1677 @param prefix: string, prefix for filename 1678 @param warpRef: Butler dataRef 1679 @param coaddLevel: bool, optional. If True, include only coadd-level keys 1680 (e.g. 'tract', 'patch', 'filter', but no 'visit') 1682 Creates a hyphen-delimited string of dataId values for simple filenames. 1687 keys = warpRef.dataId.keys()
1688 keyList = sorted(keys, reverse=
True)
1690 filename =
"%s-%s.fits" % (prefix,
'-'.join([str(warpRef.dataId[k])
for k
in keyList]))
1691 return os.path.join(directory, filename)
def setBrightObjectMasks(self, exposure, dataId, brightObjectMasks)
def getCoaddDatasetName(self, warpType="direct")
def _dataRef2DebugPath(self, prefix, warpRef, coaddLevel=False)
Return a path to which to write debugging output.
def getGroupDataRef(butler, datasetType, groupTuple, keys)
Base class for coaddition.
def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList)
Find artifacts.
def assembleMetadata(self, coaddExposure, tempExpRefList, weightList)
Set the metadata for the coadd.
def makeSupplementaryData(self, dataRef, selectDataList)
Make additional inputs to assemble() specific to subclasses.
def makeSupplementaryData(self, dataRef, selectDataList)
Make inputs specific to Subclass.
def getTempExpRefList(self, patchRef, calExpRefList)
Generate list data references corresponding to warped exposures that lie within the patch to be coadd...
def assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList, args, kwargs)
Assemble the coadd for a region.
def run(self, dataRef, selectDataList=[])
Assemble a coadd from a set of Warps.
def _readAndComputeWarpDiff(self, warpRef, imageScaler, templateCoadd)
Fetch a warp from the butler and return a warpDiff.
def prepareInputs(self, refList)
Prepare the input warps for coaddition by measuring the weight for each warp and the scaling for the ...
def getSkyInfo(self, patchRef)
Use getSkyinfo to return the skyMap, tract and patch information, wcs and the outer bbox of the patch...
def getTempExpDatasetName(self, warpType="direct")
def __init__(self, args, kwargs)
Initialize the task and make the assembleStaticSkyModel subtask.
def makeDataRefList(self, namespace)
Make self.refList from self.idList.
def getBadPixelMask(self)
Convenience method to provide the bitmask from the mask plane names.
def assembleSubregion(self, coaddExposure, bbox, tempExpRefList, imageScalerList, weightList, altMaskList, statsFlags, statsCtrl, nImage=None)
Assemble the coadd for a sub-region.
def detectClip(self, exp, tempExpRefList)
Detect clipped regions on an exposure and set the mask on the individual tempExp masks.
def setInexactPsf(self, mask)
Configuration parameters for the SafeClipAssembleCoaddTask.
def __init__(self, args, kwargs)
Initialize the task.
Assemble a coadded image from a set of warps (coadded temporary exposures).
def computeAltMaskList(self, tempExpRefList, maskSpanSets)
Apply artifact span set lists to masks.
Assemble a coadded image from a set of coadded temporary exposures, being careful to clip & flag area...
def buildDifferenceImage(self, skyInfo, tempExpRefList, imageScalerList, weightList)
Return an exposure that contains the difference between and unclipped and clipped coadds...
def selectExposures(self, patchRef, skyInfo=None, selectDataList=[])
Select exposures to coadd.
def detectClipBig(self, tempExpClipList, clipFootprints, clipIndices, maskClipValue, maskDetValue)
Find footprints from individual tempExp footprints for large footprints.
def readBrightObjectMasks(self, dataRef)
Configuration parameters for the AssembleCoaddTask.
Assemble a compareWarp coadded image from a set of warps by masking artifacts detected by comparing P...
def __init__(self, args, kwargs)
Initialize the task and make the clipDetection subtask.
def assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList, supplementaryData, args, kwargs)
Assemble the coadd.
def _filterArtifacts(self, spanSetList, epochCountImage, nImage)
Filter artifact candidates.
A version of lsst.pipe.base.DataIdContainer specialized for assembleCoadd.
def countMaskFromFootprint(mask, footprint, bitmask, ignoreMask)
Function to count the number of pixels with a specific mask in a footprint.
def groupPatchExposures(patchDataRef, calexpDataRefList, coaddDatasetType="deepCoadd", tempExpDatasetType="deepCoadd_directWarp")
def assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList, altMaskList=None, mask=None, supplementaryData=None)
Assemble a coadd from input warps.