1 from __future__
import absolute_import, division, print_function
2 from builtins
import zip
3 from builtins
import range
26 import lsst.pex.config
as pexConfig
27 import lsst.pex.exceptions
as pexExceptions
28 import lsst.afw.detection
as afwDetect
29 import lsst.afw.geom
as afwGeom
30 import lsst.afw.image
as afwImage
31 import lsst.afw.math
as afwMath
32 import lsst.afw.table
as afwTable
33 import lsst.afw.detection
as afwDet
34 import lsst.coadd.utils
as coaddUtils
35 import lsst.pipe.base
as pipeBase
36 import lsst.meas.algorithms
as measAlg
37 import lsst.log
as log
38 from .coaddBase
import CoaddBaseTask, SelectDataIdContainer
39 from .interpImage
import InterpImageTask
40 from .matchBackgrounds
import MatchBackgroundsTask
41 from .scaleZeroPoint
import ScaleZeroPointTask
42 from .coaddHelpers
import groupPatchExposures, getGroupDataRef
43 from lsst.meas.algorithms
import SourceDetectionTask
45 __all__ = [
"AssembleCoaddTask",
"SafeClipAssembleCoaddTask"]
50 \anchor AssembleCoaddConfig_
52 \brief Configuration parameters for the \ref AssembleCoaddTask_ "AssembleCoaddTask"
54 subregionSize = pexConfig.ListField(
56 doc=
"Width, height of stack subregion size; "
57 "make small enough that a full stack of images will fit into memory at once.",
61 statistic = pexConfig.Field(
63 doc=
"Main stacking statistic for aggregating over the epochs.",
66 doSigmaClip = pexConfig.Field(
68 doc=
"Perform sigma clipped outlier rejection with MEANCLIP statistic? (DEPRECATED)",
71 sigmaClip = pexConfig.Field(
73 doc=
"Sigma for outlier rejection; ignored if non-clipping statistic selected.",
76 clipIter = pexConfig.Field(
78 doc=
"Number of iterations of outlier rejection; ignored if non-clipping statistic selected.",
81 scaleZeroPoint = pexConfig.ConfigurableField(
82 target=ScaleZeroPointTask,
83 doc=
"Task to adjust the photometric zero point of the coadd temp exposures",
85 doInterp = pexConfig.Field(
86 doc=
"Interpolate over NaN pixels? Also extrapolate, if necessary, but the results are ugly.",
90 interpImage = pexConfig.ConfigurableField(
91 target=InterpImageTask,
92 doc=
"Task to interpolate (and extrapolate) over NaN pixels",
94 matchBackgrounds = pexConfig.ConfigurableField(
95 target=MatchBackgroundsTask,
96 doc=
"Task to match backgrounds",
98 maxMatchResidualRatio = pexConfig.Field(
99 doc=
"Maximum ratio of the mean squared error of the background matching model to the variance "
100 "of the difference in backgrounds",
104 maxMatchResidualRMS = pexConfig.Field(
105 doc=
"Maximum RMS of residuals of the background offset fit in matchBackgrounds.",
109 doWrite = pexConfig.Field(
110 doc=
"Persist coadd?",
114 doMatchBackgrounds = pexConfig.Field(
115 doc=
"Match backgrounds of coadd temp exposures before coadding them? "
116 "If False, the coadd temp expsosures must already have been background subtracted or matched",
120 autoReference = pexConfig.Field(
121 doc=
"Automatically select the coadd temp exposure to use as a reference for background matching? "
122 "Ignored if doMatchBackgrounds false. "
123 "If False you must specify the reference temp exposure as the data Id",
127 maskPropagationThresholds = pexConfig.DictField(
130 doc=(
"Threshold (in fractional weight) of rejection at which we propagate a mask plane to "
131 "the coadd; that is, we set the mask bit on the coadd if the fraction the rejected frames "
132 "would have contributed exceeds this value."),
133 default={
"SAT": 0.1},
135 removeMaskPlanes = pexConfig.ListField(dtype=str, default=[
"CROSSTALK",
"NOT_DEBLENDED"],
136 doc=
"Mask planes to remove before coadding")
145 doMaskBrightObjects = pexConfig.Field(dtype=bool, default=
False,
146 doc=
"Set mask and flag bits for bright objects?")
147 brightObjectMaskName = pexConfig.Field(dtype=str, default=
"BRIGHT_OBJECT",
148 doc=
"Name of mask bit used for bright objects")
150 coaddPsf = pexConfig.ConfigField(
151 doc=
"Configuration for CoaddPsf",
152 dtype=measAlg.CoaddPsfConfig,
156 CoaddBaseTask.ConfigClass.setDefaults(self)
160 CoaddBaseTask.ConfigClass.validate(self)
161 if self.makeDirect
and self.makePsfMatched:
162 raise ValueError(
"Currently, assembleCoadd can only make either Direct or PsfMatched Coadds "
163 "at a time. Set either makeDirect or makePsfMatched to False")
165 log.warn(
'doSigmaClip deprecated. To replicate behavior, setting statistic to "MEANCLIP"')
167 if self.
doInterp and self.
statistic not in [
'MEAN',
'MEDIAN',
'MEANCLIP',
'VARIANCE',
'VARIANCECLIP']:
168 raise ValueError(
"Must set doInterp=False for statistic=%s, which does not "
169 "compute and set a non-zero coadd variance estimate." % (self.
statistic))
171 unstackableStats = [
'NOTHING',
'ERROR',
'ORMASK']
172 if not hasattr(afwMath.Property, self.
statistic)
or self.
statistic in unstackableStats:
173 stackableStats = [str(k)
for k
in afwMath.Property.__members__.keys()
174 if str(k)
not in unstackableStats]
175 raise ValueError(
"statistic %s is not allowed. Please choose one of %s."
186 \anchor AssembleCoaddTask_
188 \brief Assemble a coadded image from a set of warps (coadded temporary exposures).
190 \section pipe_tasks_assembleCoadd_Contents Contents
191 - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Purpose
192 - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Initialize
193 - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Run
194 - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Config
195 - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Debug
196 - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Example
198 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Purpose Description
200 \copybrief AssembleCoaddTask_
202 We want to assemble a coadded image from a set of Warps (also called
203 coadded temporary exposures or coaddTempExps.
204 Each input Warp covers a patch on the sky and corresponds to a single run/visit/exposure of the
205 covered patch. We provide the task with a list of Warps (selectDataList) from which it selects
206 Warps that cover the specified patch (pointed at by dataRef).
207 Each Warp that goes into a coadd will typically have an independent photometric zero-point.
208 Therefore, we must scale each Warp to set it to a common photometric zeropoint. By default, each
209 Warp has backgrounds and hence will require config.doMatchBackgrounds=True.
210 When background matching is enabled, the task may be configured to automatically select a reference exposure
211 (config.autoReference=True). If this is not done, we require that the input dataRef provides access to a
212 Warp (dataset type coaddName + 'Coadd' + warpType + 'Warp') which is used as the reference exposure.
213 WarpType may be one of 'direct' or 'psfMatched', and the boolean configs config.makeDirect and
214 config.makePsfMatched set which of the warp types will be coadded.
215 The coadd is computed as a mean with optional outlier rejection.
216 Criteria for outlier rejection are set in \ref AssembleCoaddConfig. Finally, Warps can have bad 'NaN'
217 pixels which received no input from the source calExps. We interpolate over these bad (NaN) pixels.
219 AssembleCoaddTask uses several sub-tasks. These are
221 <DT>\ref ScaleZeroPointTask_ "ScaleZeroPointTask"</DT>
222 <DD> create and use an imageScaler object to scale the photometric zeropoint for each Warp</DD>
223 <DT>\ref MatchBackgroundsTask_ "MatchBackgroundsTask"</DT>
224 <DD> match background in a Warp to a reference exposure (and select the reference exposure if one is
226 <DT>\ref InterpImageTask_ "InterpImageTask"</DT>
227 <DD>interpolate across bad pixels (NaN) in the final coadd</DD>
229 You can retarget these subtasks if you wish.
231 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Initialize Task initialization
232 \copydoc \_\_init\_\_
234 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Run Invoking the Task
237 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Config Configuration parameters
238 See \ref AssembleCoaddConfig_
240 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Debug Debug variables
241 The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a
242 flag \c -d to import \b debug.py from your \c PYTHONPATH; see \ref baseDebug for more about \b debug.py files.
243 AssembleCoaddTask has no debug variables of its own. Some of the subtasks may support debug variables. See
244 the documetation for the subtasks for further information.
246 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Example A complete example of using AssembleCoaddTask
248 AssembleCoaddTask assembles a set of warped images into a coadded image. The AssembleCoaddTask
249 can be invoked by running assembleCoadd.py with the flag '--legacyCoadd'. Usage of assembleCoadd.py expects
250 a data reference to the tract patch and filter to be coadded (specified using
251 '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]') along with a list of
252 Warps to attempt to coadd (specified using
253 '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]'). Only the Warps
254 that cover the specified tract and patch will be coadded. A list of the available optional
255 arguments can be obtained by calling assembleCoadd.py with the --help command line argument:
257 assembleCoadd.py --help
259 To demonstrate usage of the AssembleCoaddTask in the larger context of multi-band processing, we will generate
260 the HSC-I & -R band coadds from HSC engineering test data provided in the ci_hsc package. To begin, assuming
261 that the lsst stack has been already set up, we must set up the obs_subaru and ci_hsc packages.
262 This defines the environment variable $CI_HSC_DIR and points at the location of the package. The raw HSC
263 data live in the $CI_HSC_DIR/raw directory. To begin assembling the coadds, we must first
266 <DD> process the individual ccds in $CI_HSC_RAW to produce calibrated exposures</DD>
268 <DD> create a skymap that covers the area of the sky present in the raw exposures</DD>
269 <DT>makeCoaddTempExp</DT>
270 <DD> warp the individual calibrated exposures to the tangent plane of the coadd</DD>
272 We can perform all of these steps by running
274 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
276 This will produce warped exposures for each visit. To coadd the warped data, we call assembleCoadd.py as
279 assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 --selectId visit=903988 ccd=24\endcode
280 that will process the HSC-I band data. The results are written in $CI_HSC_DIR/DATA/deepCoadd-results/HSC-I
281 You may also choose to run:
283 scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346
284 assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R --selectId visit=903334 ccd=16 --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 --selectId visit=903334 ccd=100 --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 --selectId visit=903338 ccd=18 --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 --selectId visit=903342 ccd=10 --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 --selectId visit=903344 ccd=5 --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 --selectId visit=903346 ccd=6 --selectId visit=903346 ccd=12
286 to generate the coadd for the HSC-R band if you are interested in following multiBand Coadd processing as
287 discussed in \ref pipeTasks_multiBand (but note that normally, one would use the
288 \ref SafeClipAssembleCoaddTask_ "SafeClipAssembleCoaddTask" rather than AssembleCoaddTask to make the coadd.
290 ConfigClass = AssembleCoaddConfig
291 _DefaultName =
"assembleCoadd"
295 \brief Initialize the task. Create the \ref InterpImageTask "interpImage",
296 \ref MatchBackgroundsTask "matchBackgrounds", & \ref ScaleZeroPointTask "scaleZeroPoint" subtasks.
298 CoaddBaseTask.__init__(self, *args, **kwargs)
299 self.makeSubtask(
"interpImage")
300 self.makeSubtask(
"matchBackgrounds")
301 self.makeSubtask(
"scaleZeroPoint")
303 if self.config.doMaskBrightObjects:
304 mask = afwImage.Mask()
307 except pexExceptions.LsstCppException:
308 raise RuntimeError(
"Unable to define mask plane for bright objects; planes used are %s" %
309 mask.getMaskPlaneDict().keys())
312 if self.config.makeDirect:
314 elif self.config.makePsfMatched:
317 raise ValueError(
"Neither makeDirect nor makePsfMatched configs are True")
320 def run(self, dataRef, selectDataList=[]):
322 \brief Assemble a coadd from a set of Warps
324 Coadd a set of Warps. Compute weights to be applied to each Warp and find scalings to
325 match the photometric zeropoint to a reference Warp. Optionally, match backgrounds across
326 Warps if the background has not already been removed. Assemble the Warps using
327 \ref assemble. Interpolate over NaNs and optionally write the coadd to disk. Return the coadded
331 \param[in] dataRef: Data reference defining the patch for coaddition and the reference Warp
332 (if config.autoReference=False). Used to access the following data products:
333 - [in] self.config.coaddName + "Coadd_skyMap"
334 - [in] self.config.coaddName + "Coadd_ + <warpType> + "Warp" (optionally)
335 - [out] self.config.coaddName + "Coadd"
336 \param[in] selectDataList[in]: List of data references to Warps. Data to be coadded will be
337 selected from this list based on overlap with the patch defined by dataRef.
339 \return a pipeBase.Struct with fields:
340 - coaddExposure: coadded exposure
342 skyInfo = self.getSkyInfo(dataRef)
343 calExpRefList = self.selectExposures(dataRef, skyInfo, selectDataList=selectDataList)
344 if len(calExpRefList) == 0:
345 self.log.warn(
"No exposures to coadd")
347 self.log.info(
"Coadding %d exposures", len(calExpRefList))
351 self.log.info(
"Found %d %s", len(inputData.tempExpRefList),
352 self.getTempExpDatasetName(self.
warpType))
353 if len(inputData.tempExpRefList) == 0:
354 self.log.warn(
"No coadd temporary exposures found")
356 if self.config.doMatchBackgrounds:
359 if len(inputData.tempExpRefList) == 0:
360 self.log.warn(
"No valid background models")
363 coaddExp = self.
assemble(skyInfo, inputData.tempExpRefList, inputData.imageScalerList,
364 inputData.weightList,
365 inputData.backgroundInfoList
if self.config.doMatchBackgrounds
else None)
366 if self.config.doMatchBackgrounds:
368 inputData.backgroundInfoList)
370 if self.config.doInterp:
371 self.interpImage.run(coaddExp.getMaskedImage(), planeName=
"NO_DATA")
373 varArray = coaddExp.getMaskedImage().getVariance().getArray()
374 varArray[:] = numpy.where(varArray > 0, varArray, numpy.inf)
376 if self.config.doMaskBrightObjects:
380 if self.config.doWrite:
381 self.log.info(
"Persisting %s" % self.getCoaddDatasetName(self.
warpType))
382 dataRef.put(coaddExp, self.getCoaddDatasetName(self.
warpType))
384 return pipeBase.Struct(coaddExposure=coaddExp)
388 \brief Generate list data references corresponding to warped exposures that lie within the
391 \param[in] patchRef: Data reference for patch
392 \param[in] calExpRefList: List of data references for input calexps
393 \return List of Warp/CoaddTempExp data references
395 butler = patchRef.getButler()
397 self.getTempExpDatasetName(self.
warpType))
399 g, groupData.keys)
for
400 g
in groupData.groups.keys()]
401 return tempExpRefList
405 \brief Construct an image scaler for the background reference frame
407 Each Warp has a different background level. A reference background level must be chosen before
408 coaddition. If config.autoReference=True, \ref backgroundMatching will pick the reference level and
409 this routine is a no-op and None is returned. Otherwise, use the
410 \ref ScaleZeroPointTask_ "scaleZeroPoint" subtask to compute an imageScaler object for the provided
411 reference image and return it.
413 \param[in] dataRef: Data reference for the background reference frame, or None
414 \return image scaler, or None
416 if self.config.autoReference:
420 dataset = self.getTempExpDatasetName(self.
warpType)
421 if not dataRef.datasetExists(dataset):
422 raise RuntimeError(
"Could not find reference exposure %s %s." % (dataset, dataRef.dataId))
424 refExposure = dataRef.get(self.getTempExpDatasetName(self.
warpType), immediate=
True)
425 refImageScaler = self.scaleZeroPoint.computeImageScaler(
426 exposure=refExposure,
429 return refImageScaler
433 \brief Prepare the input warps for coaddition by measuring the weight for each warp and the scaling
434 for the photometric zero point.
436 Each Warp has its own photometric zeropoint and background variance. Before coadding these
437 Warps together, compute a scale factor to normalize the photometric zeropoint and compute the
438 weight for each Warp.
440 \param[in] refList: List of data references to tempExp
442 - tempExprefList: List of data references to tempExp
443 - weightList: List of weightings
444 - imageScalerList: List of image scalers
446 statsCtrl = afwMath.StatisticsControl()
447 statsCtrl.setNumSigmaClip(self.config.sigmaClip)
448 statsCtrl.setNumIter(self.config.clipIter)
449 statsCtrl.setAndMask(self.getBadPixelMask())
450 statsCtrl.setNanSafe(
True)
458 tempExpName = self.getTempExpDatasetName(self.
warpType)
459 for tempExpRef
in refList:
460 if not tempExpRef.datasetExists(tempExpName):
461 self.log.warn(
"Could not find %s %s; skipping it", tempExpName, tempExpRef.dataId)
464 tempExp = tempExpRef.get(tempExpName, immediate=
True)
465 maskedImage = tempExp.getMaskedImage()
466 imageScaler = self.scaleZeroPoint.computeImageScaler(
471 imageScaler.scaleMaskedImage(maskedImage)
472 except Exception
as e:
473 self.log.warn(
"Scaling failed for %s (skipping it): %s", tempExpRef.dataId, e)
475 statObj = afwMath.makeStatistics(maskedImage.getVariance(), maskedImage.getMask(),
476 afwMath.MEANCLIP, statsCtrl)
477 meanVar, meanVarErr = statObj.getResult(afwMath.MEANCLIP)
478 weight = 1.0 / float(meanVar)
479 if not numpy.isfinite(weight):
480 self.log.warn(
"Non-finite weight for %s: skipping", tempExpRef.dataId)
482 self.log.info(
"Weight of %s %s = %0.3f", tempExpName, tempExpRef.dataId, weight)
487 tempExpRefList.append(tempExpRef)
488 weightList.append(weight)
489 imageScalerList.append(imageScaler)
491 return pipeBase.Struct(tempExpRefList=tempExpRefList, weightList=weightList,
492 imageScalerList=imageScalerList)
496 \brief Perform background matching on the prepared inputs
498 Each Warp has a different background level that must be normalized to a reference level
499 before coaddition. If no reference is provided, the background matcher selects one. If the background
500 matching is performed sucessfully, recompute the weight to be applied to the Warp (coaddTempExp) to be
501 consistent with the scaled background.
503 \param[in] inputData: Struct from prepareInputs() with tempExpRefList, weightList, imageScalerList
504 \param[in] refExpDataRef: Data reference for background reference Warp, or None
505 \param[in] refImageScaler: Image scaler for background reference Warp, or None
507 - tempExprefList: List of data references to warped exposures (coaddTempExps)
508 - weightList: List of weightings
509 - imageScalerList: List of image scalers
510 - backgroundInfoList: result from background matching
513 backgroundInfoList = self.matchBackgrounds.run(
514 expRefList=inputData.tempExpRefList,
515 imageScalerList=inputData.imageScalerList,
516 refExpDataRef=refExpDataRef
if not self.config.autoReference
else None,
517 refImageScaler=refImageScaler,
518 expDatasetType=self.getTempExpDatasetName(self.
warpType),
520 except Exception
as e:
521 self.log.fatal(
"Cannot match backgrounds: %s", e)
522 raise pipeBase.TaskError(
"Background matching failed.")
525 newTempExpRefList = []
526 newBackgroundStructList = []
530 for tempExpRef, bgInfo, scaler, weight
in zip(inputData.tempExpRefList, backgroundInfoList,
531 inputData.imageScalerList, inputData.weightList):
532 if not bgInfo.isReference:
535 if (bgInfo.backgroundModel
is None):
536 self.log.info(
"No background offset model available for %s: skipping", tempExpRef.dataId)
539 varianceRatio = bgInfo.matchedMSE / bgInfo.diffImVar
540 except Exception
as e:
541 self.log.info(
"MSE/Var ratio not calculable (%s) for %s: skipping",
542 e, tempExpRef.dataId)
544 if not numpy.isfinite(varianceRatio):
545 self.log.info(
"MSE/Var ratio not finite (%.2f / %.2f) for %s: skipping",
546 bgInfo.matchedMSE, bgInfo.diffImVar, tempExpRef.dataId)
548 elif (varianceRatio > self.config.maxMatchResidualRatio):
549 self.log.info(
"Bad fit. MSE/Var ratio %.2f > %.2f for %s: skipping",
550 varianceRatio, self.config.maxMatchResidualRatio, tempExpRef.dataId)
552 elif (bgInfo.fitRMS > self.config.maxMatchResidualRMS):
553 self.log.info(
"Bad fit. RMS %.2f > %.2f for %s: skipping",
554 bgInfo.fitRMS, self.config.maxMatchResidualRMS, tempExpRef.dataId)
556 newWeightList.append(1 / (1 / weight + bgInfo.fitRMS**2))
557 newTempExpRefList.append(tempExpRef)
558 newBackgroundStructList.append(bgInfo)
559 newScaleList.append(scaler)
561 return pipeBase.Struct(tempExpRefList=newTempExpRefList, weightList=newWeightList,
562 imageScalerList=newScaleList, backgroundInfoList=newBackgroundStructList)
564 def assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList, bgInfoList=None,
565 altMaskList=
None, mask=
None):
567 \anchor AssembleCoaddTask.assemble_
569 \brief Assemble a coadd from input warps
571 Assemble the coadd using the provided list of coaddTempExps. Since the full coadd covers a patch (a
572 large area), the assembly is performed over small areas on the image at a time in order to
573 conserve memory usage. Iterate over subregions within the outer bbox of the patch using
574 \ref assembleSubregion to stack the corresponding subregions from the coaddTempExps with the
575 statistic specified. Set the edge bits the coadd mask based on the weight map.
577 \param[in] skyInfo: Patch geometry information, from getSkyInfo
578 \param[in] tempExpRefList: List of data references to Warps (previously called CoaddTempExps)
579 \param[in] imageScalerList: List of image scalers
580 \param[in] weightList: List of weights
581 \param[in] bgInfoList: List of background data from background matching, or None
582 \param[in] altMaskList: List of alternate masks to use rather than those stored with tempExp, or None
583 \param[in] mask: Mask to ignore when coadding
584 \return coadded exposure
586 tempExpName = self.getTempExpDatasetName(self.
warpType)
587 self.log.info(
"Assembling %s %s", len(tempExpRefList), tempExpName)
589 mask = self.getBadPixelMask()
591 statsCtrl = afwMath.StatisticsControl()
592 statsCtrl.setNumSigmaClip(self.config.sigmaClip)
593 statsCtrl.setNumIter(self.config.clipIter)
594 statsCtrl.setAndMask(mask)
595 statsCtrl.setNanSafe(
True)
596 statsCtrl.setWeighted(
True)
597 statsCtrl.setCalcErrorFromInputVariance(
True)
598 for plane, threshold
in self.config.maskPropagationThresholds.items():
599 bit = afwImage.Mask.getMaskPlane(plane)
600 statsCtrl.setMaskPropagationThreshold(bit, threshold)
602 statsFlags = afwMath.stringToStatisticsProperty(self.config.statistic)
604 if bgInfoList
is None:
605 bgInfoList = [
None]*len(tempExpRefList)
607 if altMaskList
is None:
608 altMaskList = [
None]*len(tempExpRefList)
610 coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
611 coaddExposure.setCalib(self.scaleZeroPoint.getCalib())
612 coaddExposure.getInfo().setCoaddInputs(self.inputRecorder.makeCoaddInputs())
614 coaddMaskedImage = coaddExposure.getMaskedImage()
615 subregionSizeArr = self.config.subregionSize
616 subregionSize = afwGeom.Extent2I(subregionSizeArr[0], subregionSizeArr[1])
617 for subBBox
in _subBBoxIter(skyInfo.bbox, subregionSize):
620 weightList, bgInfoList, altMaskList, statsFlags, statsCtrl)
621 except Exception
as e:
622 self.log.fatal(
"Cannot compute coadd %s: %s", subBBox, e)
624 coaddUtils.setCoaddEdgeBits(coaddMaskedImage.getMask(), coaddMaskedImage.getVariance())
630 \brief Set the metadata for the coadd
632 This basic implementation simply sets the filter from the
635 \param[in] coaddExposure: The target image for the coadd
636 \param[in] tempExpRefList: List of data references to tempExp
637 \param[in] weightList: List of weights
639 assert len(tempExpRefList) == len(weightList),
"Length mismatch"
640 tempExpName = self.getTempExpDatasetName(self.
warpType)
644 tempExpList = [tempExpRef.get(tempExpName +
"_sub",
645 bbox=afwGeom.Box2I(afwGeom.Point2I(0, 0), afwGeom.Extent2I(1, 1)),
646 imageOrigin=
"LOCAL", immediate=
True)
for tempExpRef
in tempExpRefList]
647 numCcds = sum(len(tempExp.getInfo().getCoaddInputs().ccds)
for tempExp
in tempExpList)
649 coaddExposure.setFilter(tempExpList[0].getFilter())
650 coaddInputs = coaddExposure.getInfo().getCoaddInputs()
651 coaddInputs.ccds.reserve(numCcds)
652 coaddInputs.visits.reserve(len(tempExpList))
654 for tempExp, weight
in zip(tempExpList, weightList):
655 self.inputRecorder.addVisitToCoadd(coaddInputs, tempExp, weight)
656 coaddInputs.visits.sort()
662 modelPsfList = [tempExp.getPsf()
for tempExp
in tempExpList]
663 modelPsfWidthList = [modelPsf.computeBBox().getWidth()
for modelPsf
in modelPsfList]
664 psf = modelPsfList[modelPsfWidthList.index(max(modelPsfWidthList))]
666 psf = measAlg.CoaddPsf(coaddInputs.ccds, coaddExposure.getWcs(),
667 self.config.coaddPsf.makeControl())
668 coaddExposure.setPsf(psf)
669 apCorrMap = measAlg.makeCoaddApCorrMap(coaddInputs.ccds, coaddExposure.getBBox(afwImage.PARENT),
670 coaddExposure.getWcs())
671 coaddExposure.getInfo().setApCorrMap(apCorrMap)
673 def assembleSubregion(self, coaddExposure, bbox, tempExpRefList, imageScalerList, weightList,
674 bgInfoList, altMaskList, statsFlags, statsCtrl):
676 \brief Assemble the coadd for a sub-region.
678 For each coaddTempExp, check for (and swap in) an alternative mask if one is passed. If background
679 matching is enabled, add the background and background variance from each coaddTempExp. Remove mask
680 planes listed in config.removeMaskPlanes, Finally, stack the actual exposures using
681 \ref afwMath.statisticsStack "statisticsStack" with the statistic specified
682 by statsFlags. Typically, the statsFlag will be one of afwMath.MEAN for a mean-stack or
683 afwMath.MEANCLIP for outlier rejection using an N-sigma clipped mean where N and iterations
684 are specified by statsCtrl. Assign the stacked subregion back to the coadd.
686 \param[in] coaddExposure: The target image for the coadd
687 \param[in] bbox: Sub-region to coadd
688 \param[in] tempExpRefList: List of data reference to tempExp
689 \param[in] imageScalerList: List of image scalers
690 \param[in] weightList: List of weights
691 \param[in] bgInfoList: List of background data from background matching
692 \param[in] altMaskList: List of alternate masks to use rather than those stored with tempExp, or None
693 \param[in] statsFlags: afwMath.Property object for statistic for coadd
694 \param[in] statsCtrl: Statistics control object for coadd
696 self.log.debug(
"Computing coadd over %s", bbox)
697 tempExpName = self.getTempExpDatasetName(self.
warpType)
698 coaddMaskedImage = coaddExposure.getMaskedImage()
700 for tempExpRef, imageScaler, bgInfo, altMask
in zip(tempExpRefList, imageScalerList, bgInfoList,
702 exposure = tempExpRef.get(tempExpName +
"_sub", bbox=bbox)
703 maskedImage = exposure.getMaskedImage()
706 altMaskSub = altMask.Factory(altMask, bbox, afwImage.PARENT)
707 maskedImage.getMask().swap(altMaskSub)
708 imageScaler.scaleMaskedImage(maskedImage)
710 if self.config.doMatchBackgrounds
and not bgInfo.isReference:
711 backgroundModel = bgInfo.backgroundModel
712 backgroundImage = backgroundModel.getImage()
if \
713 self.matchBackgrounds.config.usePolynomial
else \
714 backgroundModel.getImageF()
715 backgroundImage.setXY0(coaddMaskedImage.getXY0())
716 maskedImage += backgroundImage.Factory(backgroundImage, bbox, afwImage.PARENT,
False)
717 var = maskedImage.getVariance()
718 var += (bgInfo.fitRMS)**2
720 if self.config.removeMaskPlanes:
721 mask = maskedImage.getMask()
722 for maskPlane
in self.config.removeMaskPlanes:
724 mask &= ~mask.getPlaneBitMask(maskPlane)
725 except Exception
as e:
726 self.log.warn(
"Unable to remove mask plane %s: %s", maskPlane, e.message)
728 maskedImageList.append(maskedImage)
730 with self.timer(
"stack"):
731 coaddSubregion = afwMath.statisticsStack(
732 maskedImageList, statsFlags, statsCtrl, weightList)
734 coaddMaskedImage.assign(coaddSubregion, bbox)
738 \brief Add metadata from the background matching to the coadd
740 \param[in] coaddExposure: Coadd
741 \param[in] tempExpRefList: List of data references for temp exps to go into coadd
742 \param[in] backgroundInfoList: List of background info, results from background matching
744 self.log.info(
"Adding exposure information to metadata")
745 metadata = coaddExposure.getMetadata()
746 metadata.addString(
"CTExp_SDQA1_DESCRIPTION",
747 "Background matching: Ratio of matchedMSE / diffImVar")
748 for ind, (tempExpRef, backgroundInfo)
in enumerate(zip(tempExpRefList, backgroundInfoList)):
749 tempExpStr =
'&'.join(
'%s=%s' % (k, v)
for k, v
in tempExpRef.dataId.items())
750 if backgroundInfo.isReference:
751 metadata.addString(
"ReferenceExp_ID", tempExpStr)
753 metadata.addString(
"CTExp_ID_%d" % (ind), tempExpStr)
754 metadata.addDouble(
"CTExp_SDQA1_%d" % (ind),
755 backgroundInfo.matchedMSE/backgroundInfo.diffImVar)
756 metadata.addDouble(
"CTExp_SDQA2_%d" % (ind),
757 backgroundInfo.fitRMS)
760 """Returns None on failure"""
762 return dataRef.get(
"brightObjectMask", immediate=
True)
763 except Exception
as e:
764 self.log.warn(
"Unable to read brightObjectMask for %s: %s", dataRef.dataId, e)
768 """Set the bright object masks
770 exposure: Exposure under consideration
771 dataId: Data identifier dict for patch
772 brightObjectMasks: afwTable of bright objects to mask
777 if brightObjectMasks
is None:
778 self.log.warn(
"Unable to apply bright object mask: none supplied")
780 self.log.info(
"Applying %d bright object masks to %s", len(brightObjectMasks), dataId)
781 md = brightObjectMasks.table.getMetadata()
784 self.log.warn(
"Expected to see %s in metadata", k)
786 if md.get(k) != dataId[k]:
787 self.log.warn(
"Expected to see %s == %s in metadata, saw %s", k, md.get(k), dataId[k])
789 mask = exposure.getMaskedImage().getMask()
790 wcs = exposure.getWcs()
791 plateScale = wcs.pixelScale().asArcseconds()
793 for rec
in brightObjectMasks:
794 center = afwGeom.PointI(wcs.skyToPixel(rec.getCoord()))
795 if rec[
"type"] ==
"box":
796 assert rec[
"angle"] == 0.0, (
"Angle != 0 for mask object %s" % rec[
"id"])
797 width = rec[
"width"].asArcseconds()/plateScale
798 height = rec[
"height"].asArcseconds()/plateScale
800 halfSize = afwGeom.ExtentI(0.5*width, 0.5*height)
801 bbox = afwGeom.Box2I(center - halfSize, center + halfSize)
803 bbox = afwGeom.BoxI(afwGeom.PointI(int(center[0] - 0.5*width), int(center[1] - 0.5*height)),
804 afwGeom.PointI(int(center[0] + 0.5*width), int(center[1] + 0.5*height)))
805 spans = afwGeom.SpanSet(bbox)
806 elif rec[
"type"] ==
"circle":
807 radius = int(rec[
"radius"].asArcseconds()/plateScale)
808 spans = afwGeom.SpanSet.fromShape(radius, offset=center)
810 self.log.warn(
"Unexpected region type %s at %s" % rec[
"type"], center)
815 def _makeArgumentParser(cls):
817 \brief Create an argument parser
819 parser = pipeBase.ArgumentParser(name=cls._DefaultName)
820 parser.add_id_argument(
"--id", cls.ConfigClass().coaddName +
"Coadd_directWarp",
821 help=
"data ID, e.g. --id tract=12345 patch=1,2",
822 ContainerClass=AssembleCoaddDataIdContainer)
823 parser.add_id_argument(
"--selectId",
"calexp", help=
"data ID, e.g. --selectId visit=6789 ccd=0..9",
824 ContainerClass=SelectDataIdContainer)
828 def _subBBoxIter(bbox, subregionSize):
830 \brief Iterate over subregions of a bbox
832 \param[in] bbox: bounding box over which to iterate: afwGeom.Box2I
833 \param[in] subregionSize: size of sub-bboxes
835 \return subBBox: next sub-bounding box of size subregionSize or smaller;
836 each subBBox is contained within bbox, so it may be smaller than subregionSize at the edges of bbox,
837 but it will never be empty
840 raise RuntimeError(
"bbox %s is empty" % (bbox,))
841 if subregionSize[0] < 1
or subregionSize[1] < 1:
842 raise RuntimeError(
"subregionSize %s must be nonzero" % (subregionSize,))
844 for rowShift
in range(0, bbox.getHeight(), subregionSize[1]):
845 for colShift
in range(0, bbox.getWidth(), subregionSize[0]):
846 subBBox = afwGeom.Box2I(bbox.getMin() + afwGeom.Extent2I(colShift, rowShift), subregionSize)
848 if subBBox.isEmpty():
849 raise RuntimeError(
"Bug: empty bbox! bbox=%s, subregionSize=%s, colShift=%s, rowShift=%s" %
850 (bbox, subregionSize, colShift, rowShift))
856 \brief A version of lsst.pipe.base.DataIdContainer specialized for assembleCoadd.
861 \brief Make self.refList from self.idList.
863 Interpret the config.doMatchBackgrounds, config.autoReference,
864 and whether a visit/run supplied.
865 If a visit/run is supplied, config.autoReference is automatically set to False.
866 if config.doMatchBackgrounds == false, then a visit/run will be ignored if accidentally supplied.
869 keysCoadd = namespace.butler.getKeys(datasetType=namespace.config.coaddName +
"Coadd",
871 keysCoaddTempExp = namespace.butler.getKeys(datasetType=namespace.config.coaddName +
872 "Coadd_directWarp", level=self.level)
874 if namespace.config.doMatchBackgrounds:
875 if namespace.config.autoReference:
876 datasetType = namespace.config.coaddName +
"Coadd"
877 validKeys = keysCoadd
879 datasetType = namespace.config.coaddName +
"Coadd_directWarp"
880 validKeys = keysCoaddTempExp
882 datasetType = namespace.config.coaddName +
"Coadd"
883 validKeys = keysCoadd
885 for dataId
in self.idList:
887 for key
in validKeys:
888 if key
not in dataId:
889 raise RuntimeError(
"--id must include " + key)
892 if (key
not in keysCoadd)
and (key
in keysCoaddTempExp):
893 if namespace.config.autoReference:
895 namespace.config.autoReference =
False
896 datasetType = namespace.config.coaddName +
"Coadd_directWarp"
897 print(
"Switching config.autoReference to False; applies only to background Matching.")
900 dataRef = namespace.butler.dataRef(
901 datasetType=datasetType,
904 self.refList.append(dataRef)
909 \brief Function to count the number of pixels with a specific mask in a footprint.
911 Find the intersection of mask & footprint. Count all pixels in the mask that are in the intersection that
912 have bitmask set but do not have ignoreMask set. Return the count.
914 \param[in] mask: mask to define intersection region by.
915 \parma[in] footprint: footprint to define the intersection region by.
916 \param[in] bitmask: specific mask that we wish to count the number of occurances of.
917 \param[in] ignoreMask: pixels to not consider.
918 \return count of number of pixels in footprint with specified mask.
920 bbox = footprint.getBBox()
921 bbox.clip(mask.getBBox(afwImage.PARENT))
922 fp = afwImage.Mask(bbox)
923 subMask = mask.Factory(mask, bbox, afwImage.PARENT)
924 footprint.spans.setMask(fp, bitmask)
925 return numpy.logical_and((subMask.getArray() & fp.getArray()) > 0,
926 (subMask.getArray() & ignoreMask) == 0).sum()
931 \anchor SafeClipAssembleCoaddConfig
933 \brief Configuration parameters for the SafeClipAssembleCoaddTask
935 clipDetection = pexConfig.ConfigurableField(
936 target=SourceDetectionTask,
937 doc=
"Detect sources on difference between unclipped and clipped coadd")
938 minClipFootOverlap = pexConfig.Field(
939 doc=
"Minimum fractional overlap of clipped footprint with visit DETECTED to be clipped",
943 minClipFootOverlapSingle = pexConfig.Field(
944 doc=
"Minimum fractional overlap of clipped footprint with visit DETECTED to be "
945 "clipped when only one visit overlaps",
949 minClipFootOverlapDouble = pexConfig.Field(
950 doc=
"Minimum fractional overlap of clipped footprints with visit DETECTED to be "
951 "clipped when two visits overlap",
955 maxClipFootOverlapDouble = pexConfig.Field(
956 doc=
"Maximum fractional overlap of clipped footprints with visit DETECTED when "
957 "considering two visits",
961 minBigOverlap = pexConfig.Field(
962 doc=
"Minimum number of pixels in footprint to use DETECTED mask from the single visits "
963 "when labeling clipped footprints",
971 AssembleCoaddConfig.setDefaults(self)
972 self.clipDetection.doTempLocalBackground =
False
973 self.clipDetection.reEstimateBackground =
False
974 self.clipDetection.returnOriginalFootprints =
False
975 self.clipDetection.thresholdPolarity =
"both"
976 self.clipDetection.thresholdValue = 2
977 self.clipDetection.nSigmaToGrow = 2
978 self.clipDetection.minPixels = 4
979 self.clipDetection.isotropicGrow =
True
980 self.clipDetection.thresholdType =
"pixel_stdev"
987 log.warn(
"Additional Sigma-clipping not allowed in Safe-clipped Coadds. "
988 "Ignoring doSigmaClip.")
991 raise ValueError(
"Only MEAN statistic allowed for final stacking in SafeClipAssembleCoadd "
992 "(%s chosen). Please set statistic to MEAN."
994 AssembleCoaddTask.ConfigClass.validate(self)
1007 \anchor SafeClipAssembleCoaddTask_
1009 \brief Assemble a coadded image from a set of coadded temporary exposures, being careful to clip & flag areas
1010 with potential artifacts.
1012 \section pipe_tasks_assembleCoadd_Contents Contents
1013 - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Purpose
1014 - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Initialize
1015 - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Run
1016 - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Config
1017 - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Debug
1018 - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Example
1020 \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Purpose Description
1022 \copybrief SafeClipAssembleCoaddTask
1024 Read the documentation for \ref AssembleCoaddTask_ "AssembleCoaddTask" first since
1025 SafeClipAssembleCoaddTask subtasks that task.
1026 In \ref AssembleCoaddTask_ "AssembleCoaddTask", we compute the coadd as an clipped mean (i.e. we clip
1028 The problem with doing this is that when computing the coadd PSF at a given location, individual visit
1029 PSFs from visits with outlier pixels contribute to the coadd PSF and cannot be treated correctly.
1030 In this task, we correct for this behavior by creating a new badMaskPlane 'CLIPPED'.
1031 We populate this plane on the input coaddTempExps and the final coadd where i. difference imaging suggests
1032 that there is an outlier and ii. this outlier appears on only one or two images.
1033 Such regions will not contribute to the final coadd.
1034 Furthermore, any routine to determine the coadd PSF can now be cognizant of clipped regions.
1035 Note that the algorithm implemented by this task is preliminary and works correctly for HSC data.
1036 Parameter modifications and or considerable redesigning of the algorithm is likley required for other
1039 SafeClipAssembleCoaddTask uses a \ref SourceDetectionTask_ "clipDetection" subtask and also sub-classes
1040 \ref AssembleCoaddTask_ "AssembleCoaddTask". You can retarget the
1041 \ref SourceDetectionTask_ "clipDetection" subtask if you wish.
1043 \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Initialize Task initialization
1044 \copydoc \_\_init\_\_
1046 \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Run Invoking the Task
1049 \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Config Configuration parameters
1050 See \ref SafeClipAssembleCoaddConfig
1052 \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Debug Debug variables
1053 The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a
1054 flag \c -d to import \b debug.py from your \c PYTHONPATH; see \ref baseDebug for more about \b debug.py
1056 SafeClipAssembleCoaddTask has no debug variables of its own. The \ref SourceDetectionTask_ "clipDetection"
1057 subtasks may support debug variables. See the documetation for \ref SourceDetectionTask_ "clipDetection"
1058 for further information.
1060 \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Example A complete example of using SafeClipAssembleCoaddTask
1062 SafeClipAssembleCoaddTask assembles a set of warped coaddTempExp images into a coadded image.
1063 The SafeClipAssembleCoaddTask is invoked by running assembleCoadd.py <em>without</em> the flag
1065 Usage of assembleCoadd.py expects a data reference to the tract patch and filter to be coadded
1066 (specified using '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]') along
1067 with a list of coaddTempExps to attempt to coadd (specified using
1068 '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]').
1069 Only the coaddTempExps that cover the specified tract and patch will be coadded.
1070 A list of the available optional arguments can be obtained by calling assembleCoadd.py with the --help
1071 command line argument:
1073 assembleCoadd.py --help
1075 To demonstrate usage of the SafeClipAssembleCoaddTask in the larger context of multi-band processing, we
1076 will generate the HSC-I & -R band coadds from HSC engineering test data provided in the ci_hsc package. To
1077 begin, assuming that the lsst stack has been already set up, we must set up the obs_subaru and ci_hsc
1079 This defines the environment variable $CI_HSC_DIR and points at the location of the package. The raw HSC
1080 data live in the $CI_HSC_DIR/raw directory. To begin assembling the coadds, we must first
1083 <DD> process the individual ccds in $CI_HSC_RAW to produce calibrated exposures</DD>
1085 <DD> create a skymap that covers the area of the sky present in the raw exposures</DD>
1086 <DT>makeCoaddTempExp</DT>
1087 <DD> warp the individual calibrated exposures to the tangent plane of the coadd</DD>
1089 We can perform all of these steps by running
1091 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
1093 This will produce warped coaddTempExps for each visit. To coadd the wraped data, we call assembleCoadd.py
1096 assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 --selectId visit=903988 ccd=24
1098 This will process the HSC-I band data. The results are written in $CI_HSC_DIR/DATA/deepCoadd-results/HSC-I
1099 You may also choose to run:
1101 scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346
1102 assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R --selectId visit=903334 ccd=16 --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 --selectId visit=903334 ccd=100 --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 --selectId visit=903338 ccd=18 --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 --selectId visit=903342 ccd=10 --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 --selectId visit=903344 ccd=5 --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 --selectId visit=903346 ccd=6 --selectId visit=903346 ccd=12
1104 to generate the coadd for the HSC-R band if you are interested in following multiBand Coadd processing as
1105 discussed in \ref pipeTasks_multiBand.
1107 ConfigClass = SafeClipAssembleCoaddConfig
1108 _DefaultName =
"safeClipAssembleCoadd"
1112 \brief Initialize the task and make the \ref SourceDetectionTask_ "clipDetection" subtask.
1114 AssembleCoaddTask.__init__(self, *args, **kwargs)
1115 schema = afwTable.SourceTable.makeMinimalSchema()
1116 self.makeSubtask(
"clipDetection", schema=schema)
1118 def assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList, bgModelList, *args, **kwargs):
1120 \brief Assemble the coadd for a region
1122 Compute the difference of coadds created with and without outlier rejection to identify coadd pixels
1123 that have outlier values in some individual visits. Detect clipped regions on the difference image and
1124 mark these regions on the one or two individual coaddTempExps where they occur if there is significant
1125 overlap between the clipped region and a source.
1126 This leaves us with a set of footprints from the difference image that have been identified as having
1127 occured on just one or two individual visits. However, these footprints were generated from a
1128 difference image. It is conceivable for a large diffuse source to have become broken up into multiple
1129 footprints acrosss the coadd difference in this process.
1130 Determine the clipped region from all overlapping footprints from the detected sources in each visit -
1131 these are big footprints.
1132 Combine the small and big clipped footprints and mark them on a new bad mask plane
1133 Generate the coadd using \ref AssembleCoaddTask.assemble_ "AssembleCoaddTask.assemble" without outlier
1134 removal. Clipped footprints will no longer make it into the coadd because they are marked in the new
1137 N.b. *args and **kwargs are passed but ignored in order to match the call signature expected by the
1140 @param skyInfo: Patch geometry information, from getSkyInfo
1141 @param tempExpRefList: List of data reference to tempExp
1142 @param imageScalerList: List of image scalers
1143 @param weightList: List of weights
1144 @param bgModelList: List of background models from background matching
1145 return coadd exposure
1147 exp = self.
buildDifferenceImage(skyInfo, tempExpRefList, imageScalerList, weightList, bgModelList)
1148 mask = exp.getMaskedImage().getMask()
1149 mask.addMaskPlane(
"CLIPPED")
1151 result = self.
detectClip(exp, tempExpRefList)
1153 self.log.info(
'Found %d clipped objects', len(result.clipFootprints))
1156 maskClipValue = mask.getPlaneBitMask(
"CLIPPED")
1157 maskDetValue = mask.getPlaneBitMask(
"DETECTED") | mask.getPlaneBitMask(
"DETECTED_NEGATIVE")
1158 bigFootprints = self.
detectClipBig(result.tempExpClipList, result.clipFootprints, result.clipIndices,
1159 maskClipValue, maskDetValue)
1162 maskClip = mask.Factory(mask.getBBox(afwImage.PARENT))
1163 afwDet.setMaskFromFootprintList(maskClip, result.clipFootprints, maskClipValue)
1165 maskClipBig = maskClip.Factory(mask.getBBox(afwImage.PARENT))
1166 afwDet.setMaskFromFootprintList(maskClipBig, bigFootprints, maskClipValue)
1167 maskClip |= maskClipBig
1170 badMaskPlanes = self.config.badMaskPlanes[:]
1171 badMaskPlanes.append(
"CLIPPED")
1172 badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes)
1173 coaddExp = AssembleCoaddTask.assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList,
1174 bgModelList, result.tempExpClipList, mask=badPixelMask)
1178 maskExp = coaddExp.getMaskedImage().getMask()
1185 \brief Return an exposure that contains the difference between and unclipped and clipped coadds.
1187 Generate a difference image between clipped and unclipped coadds.
1188 Compute the difference image by subtracting an outlier-clipped coadd from an outlier-unclipped coadd.
1189 Return the difference image.
1191 @param skyInfo: Patch geometry information, from getSkyInfo
1192 @param tempExpRefList: List of data reference to tempExp
1193 @param imageScalerList: List of image scalers
1194 @param weightList: List of weights
1195 @param bgModelList: List of background models from background matching
1196 @return Difference image of unclipped and clipped coadd wrapped in an Exposure
1201 configIntersection = {k: getattr(self.config, k)
1202 for k, v
in self.config.toDict().items()
if (k
in config.keys())}
1203 config.update(**configIntersection)
1206 config.statistic =
'MEAN'
1208 coaddMean = task.assemble(skyInfo, tempExpRefList, imageScalerList, weightList, bgModelList)
1210 config.statistic =
'MEANCLIP'
1212 coaddClip = task.assemble(skyInfo, tempExpRefList, imageScalerList, weightList, bgModelList)
1214 coaddDiff = coaddMean.getMaskedImage().Factory(coaddMean.getMaskedImage())
1215 coaddDiff -= coaddClip.getMaskedImage()
1216 exp = afwImage.ExposureF(coaddDiff)
1217 exp.setPsf(coaddMean.getPsf())
1222 \brief Detect clipped regions on an exposure and set the mask on the individual tempExp masks
1224 Detect footprints in the difference image after smoothing the difference image with a Gaussian kernal.
1225 Identify footprints that overlap with one or two input coaddTempExps by comparing the computed overlap
1226 fraction to thresholds set in the config.
1227 A different threshold is applied depending on the number of overlapping visits (restricted to one or
1229 If the overlap exceeds the thresholds, the footprint is considered "CLIPPED" and is marked as such on
1231 Return a struct with the clipped footprints, the indices of the coaddTempExps that end up overlapping
1232 with the clipped footprints and a list of new masks for the coaddTempExps.
1234 \param[in] exp: Exposure to run detection on
1235 \param[in] tempExpRefList: List of data reference to tempExp
1236 \return struct containing:
1237 - clippedFootprints: list of clipped footprints
1238 - clippedIndices: indices for each clippedFootprint in tempExpRefList
1239 - tempExpClipList: list of new masks for tempExp
1241 mask = exp.getMaskedImage().getMask()
1242 maskClipValue = mask.getPlaneBitMask(
"CLIPPED")
1243 maskDetValue = mask.getPlaneBitMask(
"DETECTED") | mask.getPlaneBitMask(
"DETECTED_NEGATIVE")
1244 fpSet = self.clipDetection.detectFootprints(exp, doSmooth=
True, clearMask=
True)
1246 fpSet.positive.merge(fpSet.negative)
1247 footprints = fpSet.positive
1248 self.log.info(
'Found %d potential clipped objects', len(footprints.getFootprints()))
1249 ignoreMask = self.getBadPixelMask()
1255 tempExpClipList = [tmpExpRef.get(self.getTempExpDatasetName(self.
warpType),
1256 immediate=
True).getMaskedImage().getMask()
for
1257 tmpExpRef
in tempExpRefList]
1259 for footprint
in footprints.getFootprints():
1260 nPixel = footprint.getArea()
1264 for i, tmpExpMask
in enumerate(tempExpClipList):
1268 totPixel = nPixel - ignore
1271 if ignore > overlapDet
or totPixel <= 0.5*nPixel
or overlapDet == 0:
1273 overlap.append(overlapDet/float(totPixel))
1274 maskList.append(tmpExpMask)
1277 overlap = numpy.array(overlap)
1278 if not len(overlap):
1285 if len(overlap) == 1:
1286 if overlap[0] > self.config.minClipFootOverlapSingle:
1291 clipIndex = numpy.where(overlap > self.config.minClipFootOverlap)[0]
1292 if len(clipIndex) == 1:
1294 keepIndex = [clipIndex[0]]
1297 clipIndex = numpy.where(overlap > self.config.minClipFootOverlapDouble)[0]
1298 if len(clipIndex) == 2
and len(overlap) > 3:
1299 clipIndexComp = numpy.where(overlap <= self.config.minClipFootOverlapDouble)[0]
1300 if numpy.max(overlap[clipIndexComp]) <= self.config.maxClipFootOverlapDouble:
1302 keepIndex = clipIndex
1307 for index
in keepIndex:
1308 footprint.spans.setMask(maskList[index], maskClipValue)
1310 clipIndices.append(numpy.array(indexList)[keepIndex])
1311 clipFootprints.append(footprint)
1313 return pipeBase.Struct(clipFootprints=clipFootprints, clipIndices=clipIndices,
1314 tempExpClipList=tempExpClipList)
1316 def detectClipBig(self, tempExpClipList, clipFootprints, clipIndices, maskClipValue, maskDetValue):
1318 \brief Find footprints from individual tempExp footprints for large footprints.
1320 Identify big footprints composed of many sources in the coadd difference that may have originated in a
1321 large diffuse source in the coadd. We do this by indentifying all clipped footprints that overlap
1322 significantly with each source in all the coaddTempExps.
1324 \param[in] tempExpClipList: List of tempExp masks with clipping information
1325 \param[in] clipFootprints: List of clipped footprints
1326 \param[in] clipIndices: List of which entries in tempExpClipList each footprint belongs to
1327 \param[in] maskClipValue: Mask value of clipped pixels
1328 \param[in] maskClipValue: Mask value of detected pixels
1329 \return list of big footprints
1331 bigFootprintsCoadd = []
1332 ignoreMask = self.getBadPixelMask()
1333 for index, tmpExpMask
in enumerate(tempExpClipList):
1336 maskVisitDet = tmpExpMask.Factory(tmpExpMask, tmpExpMask.getBBox(afwImage.PARENT),
1337 afwImage.PARENT,
True)
1338 maskVisitDet &= maskDetValue
1339 visitFootprints = afwDet.FootprintSet(maskVisitDet, afwDet.Threshold(1))
1342 clippedFootprintsVisit = []
1343 for foot, clipIndex
in zip(clipFootprints, clipIndices):
1344 if index
not in clipIndex:
1346 clippedFootprintsVisit.append(foot)
1347 maskVisitClip = maskVisitDet.Factory(maskVisitDet.getBBox(afwImage.PARENT))
1348 afwDet.setMaskFromFootprintList(maskVisitClip, clippedFootprintsVisit, maskClipValue)
1350 bigFootprintsVisit = []
1351 for foot
in visitFootprints.getFootprints():
1352 if foot.getArea() < self.config.minBigOverlap:
1355 if nCount > self.config.minBigOverlap:
1356 bigFootprintsVisit.append(foot)
1357 bigFootprintsCoadd.append(foot)
1360 maskVisitClip.clearAllMaskPlanes()
1361 afwDet.setMaskFromFootprintList(maskVisitClip, bigFootprintsVisit, maskClipValue)
1362 tmpExpMask |= maskVisitClip
1364 return bigFootprintsCoadd
def __init__
Initialize the task and make the clipDetection subtask.
def run
Assemble a coadd from a set of Warps.
def getBackgroundReferenceScaler
Construct an image scaler for the background reference frame.
def assemble
Assemble the coadd for a region.
def addBackgroundMatchingMetadata
Add metadata from the background matching to the coadd.
def detectClipBig
Find footprints from individual tempExp footprints for large footprints.
def __init__
Initialize the task.
def assembleSubregion
Assemble the coadd for a sub-region.
def assembleMetadata
Set the metadata for the coadd.
def backgroundMatching
Perform background matching on the prepared inputs.
Configuration parameters for the SafeClipAssembleCoaddTask.
Assemble a coadded image from a set of warps (coadded temporary exposures).
Assemble a coadded image from a set of coadded temporary exposures, being careful to clip & flag area...
def buildDifferenceImage
Return an exposure that contains the difference between and unclipped and clipped coadds...
def detectClip
Detect clipped regions on an exposure and set the mask on the individual tempExp masks.
def prepareInputs
Prepare the input warps for coaddition by measuring the weight for each warp and the scaling for the ...
def makeDataRefList
Make self.refList from self.idList.
Configuration parameters for the AssembleCoaddTask.
def countMaskFromFootprint
Function to count the number of pixels with a specific mask in a footprint.
def assemble
Assemble a coadd from input warps.
def readBrightObjectMasks
A version of lsst.pipe.base.DataIdContainer specialized for assembleCoadd.
def getTempExpRefList
Generate list data references corresponding to warped exposures that lie within the patch to be coadd...