1 from __future__
import absolute_import, division, print_function
2 from builtins
import zip
3 from builtins
import range
26 import lsst.pex.config
as pexConfig
27 import lsst.pex.exceptions
as pexExceptions
28 import lsst.afw.detection
as afwDetect
29 import lsst.afw.geom
as afwGeom
30 import lsst.afw.image
as afwImage
31 import lsst.afw.math
as afwMath
32 import lsst.afw.table
as afwTable
33 import lsst.afw.detection
as afwDet
34 import lsst.coadd.utils
as coaddUtils
35 import lsst.pipe.base
as pipeBase
36 import lsst.meas.algorithms
as measAlg
37 from .coaddBase
import CoaddBaseTask, SelectDataIdContainer
38 from .interpImage
import InterpImageTask
39 from .matchBackgrounds
import MatchBackgroundsTask
40 from .scaleZeroPoint
import ScaleZeroPointTask
41 from .coaddHelpers
import groupPatchExposures, getGroupDataRef
42 from lsst.meas.algorithms
import SourceDetectionTask
44 __all__ = [
"AssembleCoaddTask",
"SafeClipAssembleCoaddTask"]
49 \anchor AssembleCoaddConfig_
51 \brief Configuration parameters for the \ref AssembleCoaddTask_ "AssembleCoaddTask"
53 subregionSize = pexConfig.ListField(
55 doc=
"Width, height of stack subregion size; "
56 "make small enough that a full stack of images will fit into memory at once.",
60 doSigmaClip = pexConfig.Field(
62 doc=
"Perform sigma clipped outlier rejection? If False then compute a simple mean.",
65 sigmaClip = pexConfig.Field(
67 doc=
"Sigma for outlier rejection; ignored if doSigmaClip false.",
70 clipIter = pexConfig.Field(
72 doc=
"Number of iterations of outlier rejection; ignored if doSigmaClip false.",
75 scaleZeroPoint = pexConfig.ConfigurableField(
76 target=ScaleZeroPointTask,
77 doc=
"Task to adjust the photometric zero point of the coadd temp exposures",
79 doInterp = pexConfig.Field(
80 doc=
"Interpolate over NaN pixels? Also extrapolate, if necessary, but the results are ugly.",
84 interpImage = pexConfig.ConfigurableField(
85 target=InterpImageTask,
86 doc=
"Task to interpolate (and extrapolate) over NaN pixels",
88 matchBackgrounds = pexConfig.ConfigurableField(
89 target=MatchBackgroundsTask,
90 doc=
"Task to match backgrounds",
92 maxMatchResidualRatio = pexConfig.Field(
93 doc=
"Maximum ratio of the mean squared error of the background matching model to the variance "
94 "of the difference in backgrounds",
98 maxMatchResidualRMS = pexConfig.Field(
99 doc=
"Maximum RMS of residuals of the background offset fit in matchBackgrounds.",
103 doWrite = pexConfig.Field(
104 doc=
"Persist coadd?",
108 doMatchBackgrounds = pexConfig.Field(
109 doc=
"Match backgrounds of coadd temp exposures before coadding them? "
110 "If False, the coadd temp expsosures must already have been background subtracted or matched",
114 autoReference = pexConfig.Field(
115 doc=
"Automatically select the coadd temp exposure to use as a reference for background matching? "
116 "Ignored if doMatchBackgrounds false. "
117 "If False you must specify the reference temp exposure as the data Id",
121 maskPropagationThresholds = pexConfig.DictField(
124 doc=(
"Threshold (in fractional weight) of rejection at which we propagate a mask plane to "
125 "the coadd; that is, we set the mask bit on the coadd if the fraction the rejected frames "
126 "would have contributed exceeds this value."),
127 default={
"SAT": 0.1},
129 removeMaskPlanes = pexConfig.ListField(dtype=str, default=[
"CROSSTALK",
"NOT_DEBLENDED"],
130 doc=
"Mask planes to remove before coadding")
139 doMaskBrightObjects = pexConfig.Field(dtype=bool, default=
False,
140 doc=
"Set mask and flag bits for bright objects?")
141 brightObjectMaskName = pexConfig.Field(dtype=str, default=
"BRIGHT_OBJECT",
142 doc=
"Name of mask bit used for bright objects")
144 coaddPsf = pexConfig.ConfigField(
145 doc=
"Configuration for CoaddPsf",
146 dtype=measAlg.CoaddPsfConfig,
150 CoaddBaseTask.ConfigClass.setDefaults(self)
154 CoaddBaseTask.ConfigClass.validate(self)
155 if self.makeDirect
and self.makePsfMatched:
156 raise ValueError(
"Currently, assembleCoadd can only make either Direct or PsfMatched Coadds "
157 "at a time. Set either makeDirect or makePsfMatched to False")
169 \anchor AssembleCoaddTask_
171 \brief Assemble a coadded image from a set of warps (coadded temporary exposures).
173 \section pipe_tasks_assembleCoadd_Contents Contents
174 - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Purpose
175 - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Initialize
176 - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Run
177 - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Config
178 - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Debug
179 - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Example
181 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Purpose Description
183 \copybrief AssembleCoaddTask_
185 We want to assemble a coadded image from a set of Warps (also called
186 coadded temporary exposures or coaddTempExps.
187 Each input Warp covers a patch on the sky and corresponds to a single run/visit/exposure of the
188 covered patch. We provide the task with a list of Warps (selectDataList) from which it selects
189 Warps that cover the specified patch (pointed at by dataRef).
190 Each Warp that goes into a coadd will typically have an independent photometric zero-point.
191 Therefore, we must scale each Warp to set it to a common photometric zeropoint. By default, each
192 Warp has backgrounds and hence will require config.doMatchBackgrounds=True.
193 When background matching is enabled, the task may be configured to automatically select a reference exposure
194 (config.autoReference=True). If this is not done, we require that the input dataRef provides access to a
195 Warp (dataset type coaddName + 'Coadd' + warpType + 'Warp') which is used as the reference exposure.
196 WarpType may be one of 'direct' or 'psfMatched', and the boolean configs config.makeDirect and
197 config.makePsfMatched set which of the warp types will be coadded.
198 The coadd is computed as a mean with optional outlier rejection.
199 Criteria for outlier rejection are set in \ref AssembleCoaddConfig. Finally, Warps can have bad 'NaN'
200 pixels which received no input from the source calExps. We interpolate over these bad (NaN) pixels.
202 AssembleCoaddTask uses several sub-tasks. These are
204 <DT>\ref ScaleZeroPointTask_ "ScaleZeroPointTask"</DT>
205 <DD> create and use an imageScaler object to scale the photometric zeropoint for each Warp</DD>
206 <DT>\ref MatchBackgroundsTask_ "MatchBackgroundsTask"</DT>
207 <DD> match background in a Warp to a reference exposure (and select the reference exposure if one is
209 <DT>\ref InterpImageTask_ "InterpImageTask"</DT>
210 <DD>interpolate across bad pixels (NaN) in the final coadd</DD>
212 You can retarget these subtasks if you wish.
214 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Initialize Task initialization
215 \copydoc \_\_init\_\_
217 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Run Invoking the Task
220 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Config Configuration parameters
221 See \ref AssembleCoaddConfig_
223 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Debug Debug variables
224 The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a
225 flag \c -d to import \b debug.py from your \c PYTHONPATH; see \ref baseDebug for more about \b debug.py files.
226 AssembleCoaddTask has no debug variables of its own. Some of the subtasks may support debug variables. See
227 the documetation for the subtasks for further information.
229 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Example A complete example of using AssembleCoaddTask
231 AssembleCoaddTask assembles a set of warped images into a coadded image. The AssembleCoaddTask
232 can be invoked by running assembleCoadd.py with the flag '--legacyCoadd'. Usage of assembleCoadd.py expects
233 a data reference to the tract patch and filter to be coadded (specified using
234 '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]') along with a list of
235 Warps to attempt to coadd (specified using
236 '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]'). Only the Warps
237 that cover the specified tract and patch will be coadded. A list of the available optional
238 arguments can be obtained by calling assembleCoadd.py with the --help command line argument:
240 assembleCoadd.py --help
242 To demonstrate usage of the AssembleCoaddTask in the larger context of multi-band processing, we will generate
243 the HSC-I & -R band coadds from HSC engineering test data provided in the ci_hsc package. To begin, assuming
244 that the lsst stack has been already set up, we must set up the obs_subaru and ci_hsc packages.
245 This defines the environment variable $CI_HSC_DIR and points at the location of the package. The raw HSC
246 data live in the $CI_HSC_DIR/raw directory. To begin assembling the coadds, we must first
249 <DD> process the individual ccds in $CI_HSC_RAW to produce calibrated exposures</DD>
251 <DD> create a skymap that covers the area of the sky present in the raw exposures</DD>
252 <DT>makeCoaddTempExp</DT>
253 <DD> warp the individual calibrated exposures to the tangent plane of the coadd</DD>
255 We can perform all of these steps by running
257 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
259 This will produce warped exposures for each visit. To coadd the warped data, we call assembleCoadd.py as
262 assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 --selectId visit=903988 ccd=24\endcode
263 that will process the HSC-I band data. The results are written in $CI_HSC_DIR/DATA/deepCoadd-results/HSC-I
264 You may also choose to run:
266 scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346
267 assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R --selectId visit=903334 ccd=16 --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 --selectId visit=903334 ccd=100 --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 --selectId visit=903338 ccd=18 --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 --selectId visit=903342 ccd=10 --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 --selectId visit=903344 ccd=5 --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 --selectId visit=903346 ccd=6 --selectId visit=903346 ccd=12
269 to generate the coadd for the HSC-R band if you are interested in following multiBand Coadd processing as
270 discussed in \ref pipeTasks_multiBand (but note that normally, one would use the
271 \ref SafeClipAssembleCoaddTask_ "SafeClipAssembleCoaddTask" rather than AssembleCoaddTask to make the coadd.
273 ConfigClass = AssembleCoaddConfig
274 _DefaultName =
"assembleCoadd"
278 \brief Initialize the task. Create the \ref InterpImageTask "interpImage",
279 \ref MatchBackgroundsTask "matchBackgrounds", & \ref ScaleZeroPointTask "scaleZeroPoint" subtasks.
281 CoaddBaseTask.__init__(self, *args, **kwargs)
282 self.makeSubtask(
"interpImage")
283 self.makeSubtask(
"matchBackgrounds")
284 self.makeSubtask(
"scaleZeroPoint")
286 if self.config.doMaskBrightObjects:
287 mask = afwImage.Mask()
290 except pexExceptions.LsstCppException:
291 raise RuntimeError(
"Unable to define mask plane for bright objects; planes used are %s" %
292 mask.getMaskPlaneDict().keys())
295 if self.config.makeDirect:
297 elif self.config.makePsfMatched:
300 raise ValueError(
"Neither makeDirect nor makePsfMatched configs are True")
303 def run(self, dataRef, selectDataList=[]):
305 \brief Assemble a coadd from a set of Warps
307 Coadd a set of Warps. Compute weights to be applied to each Warp and find scalings to
308 match the photometric zeropoint to a reference Warp. Optionally, match backgrounds across
309 Warps if the background has not already been removed. Assemble the Warps using
310 \ref assemble. Interpolate over NaNs and optionally write the coadd to disk. Return the coadded
314 \param[in] dataRef: Data reference defining the patch for coaddition and the reference Warp
315 (if config.autoReference=False). Used to access the following data products:
316 - [in] self.config.coaddName + "Coadd_skyMap"
317 - [in] self.config.coaddName + "Coadd_ + <warpType> + "Warp" (optionally)
318 - [out] self.config.coaddName + "Coadd"
319 \param[in] selectDataList[in]: List of data references to Warps. Data to be coadded will be
320 selected from this list based on overlap with the patch defined by dataRef.
322 \return a pipeBase.Struct with fields:
323 - coaddExposure: coadded exposure
325 skyInfo = self.getSkyInfo(dataRef)
326 calExpRefList = self.selectExposures(dataRef, skyInfo, selectDataList=selectDataList)
327 if len(calExpRefList) == 0:
328 self.log.warn(
"No exposures to coadd")
330 self.log.info(
"Coadding %d exposures", len(calExpRefList))
334 self.log.info(
"Found %d %s", len(inputData.tempExpRefList),
335 self.getTempExpDatasetName(self.
warpType))
336 if len(inputData.tempExpRefList) == 0:
337 self.log.warn(
"No coadd temporary exposures found")
339 if self.config.doMatchBackgrounds:
342 if len(inputData.tempExpRefList) == 0:
343 self.log.warn(
"No valid background models")
346 coaddExp = self.
assemble(skyInfo, inputData.tempExpRefList, inputData.imageScalerList,
347 inputData.weightList,
348 inputData.backgroundInfoList
if self.config.doMatchBackgrounds
else None,
349 doClip=self.config.doSigmaClip)
350 if self.config.doMatchBackgrounds:
352 inputData.backgroundInfoList)
354 if self.config.doInterp:
355 self.interpImage.run(coaddExp.getMaskedImage(), planeName=
"NO_DATA")
357 varArray = coaddExp.getMaskedImage().getVariance().getArray()
358 varArray[:] = numpy.where(varArray > 0, varArray, numpy.inf)
360 if self.config.doMaskBrightObjects:
364 if self.config.doWrite:
365 self.log.info(
"Persisting %s" % self.getCoaddDatasetName(self.
warpType))
366 dataRef.put(coaddExp, self.getCoaddDatasetName(self.
warpType))
368 return pipeBase.Struct(coaddExposure=coaddExp)
372 \brief Generate list data references corresponding to warped exposures that lie within the
375 \param[in] patchRef: Data reference for patch
376 \param[in] calExpRefList: List of data references for input calexps
377 \return List of Warp/CoaddTempExp data references
379 butler = patchRef.getButler()
381 self.getTempExpDatasetName(self.
warpType))
383 g, groupData.keys)
for
384 g
in groupData.groups.keys()]
385 return tempExpRefList
389 \brief Construct an image scaler for the background reference frame
391 Each Warp has a different background level. A reference background level must be chosen before
392 coaddition. If config.autoReference=True, \ref backgroundMatching will pick the reference level and
393 this routine is a no-op and None is returned. Otherwise, use the
394 \ref ScaleZeroPointTask_ "scaleZeroPoint" subtask to compute an imageScaler object for the provided
395 reference image and return it.
397 \param[in] dataRef: Data reference for the background reference frame, or None
398 \return image scaler, or None
400 if self.config.autoReference:
404 dataset = self.getTempExpDatasetName(self.
warpType)
405 if not dataRef.datasetExists(dataset):
406 raise RuntimeError(
"Could not find reference exposure %s %s." % (dataset, dataRef.dataId))
408 refExposure = dataRef.get(self.getTempExpDatasetName(self.
warpType), immediate=
True)
409 refImageScaler = self.scaleZeroPoint.computeImageScaler(
410 exposure=refExposure,
413 return refImageScaler
417 \brief Prepare the input warps for coaddition by measuring the weight for each warp and the scaling
418 for the photometric zero point.
420 Each Warp has its own photometric zeropoint and background variance. Before coadding these
421 Warps together, compute a scale factor to normalize the photometric zeropoint and compute the
422 weight for each Warp.
424 \param[in] refList: List of data references to tempExp
426 - tempExprefList: List of data references to tempExp
427 - weightList: List of weightings
428 - imageScalerList: List of image scalers
430 statsCtrl = afwMath.StatisticsControl()
431 statsCtrl.setNumSigmaClip(self.config.sigmaClip)
432 statsCtrl.setNumIter(self.config.clipIter)
433 statsCtrl.setAndMask(self.getBadPixelMask())
434 statsCtrl.setNanSafe(
True)
442 tempExpName = self.getTempExpDatasetName(self.
warpType)
443 for tempExpRef
in refList:
444 if not tempExpRef.datasetExists(tempExpName):
445 self.log.warn(
"Could not find %s %s; skipping it", tempExpName, tempExpRef.dataId)
448 tempExp = tempExpRef.get(tempExpName, immediate=
True)
449 maskedImage = tempExp.getMaskedImage()
450 imageScaler = self.scaleZeroPoint.computeImageScaler(
455 imageScaler.scaleMaskedImage(maskedImage)
456 except Exception
as e:
457 self.log.warn(
"Scaling failed for %s (skipping it): %s", tempExpRef.dataId, e)
459 statObj = afwMath.makeStatistics(maskedImage.getVariance(), maskedImage.getMask(),
460 afwMath.MEANCLIP, statsCtrl)
461 meanVar, meanVarErr = statObj.getResult(afwMath.MEANCLIP)
462 weight = 1.0 / float(meanVar)
463 if not numpy.isfinite(weight):
464 self.log.warn(
"Non-finite weight for %s: skipping", tempExpRef.dataId)
466 self.log.info(
"Weight of %s %s = %0.3f", tempExpName, tempExpRef.dataId, weight)
471 tempExpRefList.append(tempExpRef)
472 weightList.append(weight)
473 imageScalerList.append(imageScaler)
475 return pipeBase.Struct(tempExpRefList=tempExpRefList, weightList=weightList,
476 imageScalerList=imageScalerList)
480 \brief Perform background matching on the prepared inputs
482 Each Warp has a different background level that must be normalized to a reference level
483 before coaddition. If no reference is provided, the background matcher selects one. If the background
484 matching is performed sucessfully, recompute the weight to be applied to the Warp (coaddTempExp) to be
485 consistent with the scaled background.
487 \param[in] inputData: Struct from prepareInputs() with tempExpRefList, weightList, imageScalerList
488 \param[in] refExpDataRef: Data reference for background reference Warp, or None
489 \param[in] refImageScaler: Image scaler for background reference Warp, or None
491 - tempExprefList: List of data references to warped exposures (coaddTempExps)
492 - weightList: List of weightings
493 - imageScalerList: List of image scalers
494 - backgroundInfoList: result from background matching
497 backgroundInfoList = self.matchBackgrounds.run(
498 expRefList=inputData.tempExpRefList,
499 imageScalerList=inputData.imageScalerList,
500 refExpDataRef=refExpDataRef
if not self.config.autoReference
else None,
501 refImageScaler=refImageScaler,
502 expDatasetType=self.getTempExpDatasetName(self.
warpType),
504 except Exception
as e:
505 self.log.fatal(
"Cannot match backgrounds: %s", e)
506 raise pipeBase.TaskError(
"Background matching failed.")
509 newTempExpRefList = []
510 newBackgroundStructList = []
514 for tempExpRef, bgInfo, scaler, weight
in zip(inputData.tempExpRefList, backgroundInfoList,
515 inputData.imageScalerList, inputData.weightList):
516 if not bgInfo.isReference:
519 if (bgInfo.backgroundModel
is None):
520 self.log.info(
"No background offset model available for %s: skipping", tempExpRef.dataId)
523 varianceRatio = bgInfo.matchedMSE / bgInfo.diffImVar
524 except Exception
as e:
525 self.log.info(
"MSE/Var ratio not calculable (%s) for %s: skipping",
526 e, tempExpRef.dataId)
528 if not numpy.isfinite(varianceRatio):
529 self.log.info(
"MSE/Var ratio not finite (%.2f / %.2f) for %s: skipping",
530 bgInfo.matchedMSE, bgInfo.diffImVar, tempExpRef.dataId)
532 elif (varianceRatio > self.config.maxMatchResidualRatio):
533 self.log.info(
"Bad fit. MSE/Var ratio %.2f > %.2f for %s: skipping",
534 varianceRatio, self.config.maxMatchResidualRatio, tempExpRef.dataId)
536 elif (bgInfo.fitRMS > self.config.maxMatchResidualRMS):
537 self.log.info(
"Bad fit. RMS %.2f > %.2f for %s: skipping",
538 bgInfo.fitRMS, self.config.maxMatchResidualRMS, tempExpRef.dataId)
540 newWeightList.append(1 / (1 / weight + bgInfo.fitRMS**2))
541 newTempExpRefList.append(tempExpRef)
542 newBackgroundStructList.append(bgInfo)
543 newScaleList.append(scaler)
545 return pipeBase.Struct(tempExpRefList=newTempExpRefList, weightList=newWeightList,
546 imageScalerList=newScaleList, backgroundInfoList=newBackgroundStructList)
548 def assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList, bgInfoList=None,
549 altMaskList=
None, doClip=
False, mask=
None):
551 \anchor AssembleCoaddTask.assemble_
553 \brief Assemble a coadd from input warps
555 Assemble the coadd using the provided list of coaddTempExps. Since the full coadd covers a patch (a
556 large area), the assembly is performed over small areas on the image at a time in order to
557 conserve memory usage. Iterate over subregions within the outer bbox of the patch using
558 \ref assembleSubregion to mean-stack the corresponding subregions from the coaddTempExps (with outlier
559 rejection if config.doSigmaClip=True). Set the edge bits the the coadd mask based on the weight map.
561 \param[in] skyInfo: Patch geometry information, from getSkyInfo
562 \param[in] tempExpRefList: List of data references to Warps (previously called CoaddTempExps)
563 \param[in] imageScalerList: List of image scalers
564 \param[in] weightList: List of weights
565 \param[in] bgInfoList: List of background data from background matching, or None
566 \param[in] altMaskList: List of alternate masks to use rather than those stored with tempExp, or None
567 \param[in] doClip: Use clipping when codding?
568 \param[in] mask: Mask to ignore when coadding
569 \return coadded exposure
571 tempExpName = self.getTempExpDatasetName(self.
warpType)
572 self.log.info(
"Assembling %s %s", len(tempExpRefList), tempExpName)
574 mask = self.getBadPixelMask()
576 statsCtrl = afwMath.StatisticsControl()
577 statsCtrl.setNumSigmaClip(self.config.sigmaClip)
578 statsCtrl.setNumIter(self.config.clipIter)
579 statsCtrl.setAndMask(mask)
580 statsCtrl.setNanSafe(
True)
581 statsCtrl.setWeighted(
True)
582 statsCtrl.setCalcErrorFromInputVariance(
True)
583 for plane, threshold
in self.config.maskPropagationThresholds.items():
584 bit = afwImage.Mask.getMaskPlane(plane)
585 statsCtrl.setMaskPropagationThreshold(bit, threshold)
588 statsFlags = afwMath.MEANCLIP
590 statsFlags = afwMath.MEAN
592 if bgInfoList
is None:
593 bgInfoList = [
None]*len(tempExpRefList)
595 if altMaskList
is None:
596 altMaskList = [
None]*len(tempExpRefList)
598 coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
599 coaddExposure.setCalib(self.scaleZeroPoint.getCalib())
600 coaddExposure.getInfo().setCoaddInputs(self.inputRecorder.makeCoaddInputs())
602 coaddMaskedImage = coaddExposure.getMaskedImage()
603 subregionSizeArr = self.config.subregionSize
604 subregionSize = afwGeom.Extent2I(subregionSizeArr[0], subregionSizeArr[1])
605 for subBBox
in _subBBoxIter(skyInfo.bbox, subregionSize):
608 weightList, bgInfoList, altMaskList, statsFlags, statsCtrl)
609 except Exception
as e:
610 self.log.fatal(
"Cannot compute coadd %s: %s", subBBox, e)
612 coaddUtils.setCoaddEdgeBits(coaddMaskedImage.getMask(), coaddMaskedImage.getVariance())
618 \brief Set the metadata for the coadd
620 This basic implementation simply sets the filter from the
623 \param[in] coaddExposure: The target image for the coadd
624 \param[in] tempExpRefList: List of data references to tempExp
625 \param[in] weightList: List of weights
627 assert len(tempExpRefList) == len(weightList),
"Length mismatch"
628 tempExpName = self.getTempExpDatasetName(self.
warpType)
632 tempExpList = [tempExpRef.get(tempExpName +
"_sub",
633 bbox=afwGeom.Box2I(afwGeom.Point2I(0, 0), afwGeom.Extent2I(1, 1)),
634 imageOrigin=
"LOCAL", immediate=
True)
for tempExpRef
in tempExpRefList]
635 numCcds = sum(len(tempExp.getInfo().getCoaddInputs().ccds)
for tempExp
in tempExpList)
637 coaddExposure.setFilter(tempExpList[0].getFilter())
638 coaddInputs = coaddExposure.getInfo().getCoaddInputs()
639 coaddInputs.ccds.reserve(numCcds)
640 coaddInputs.visits.reserve(len(tempExpList))
642 for tempExp, weight
in zip(tempExpList, weightList):
643 self.inputRecorder.addVisitToCoadd(coaddInputs, tempExp, weight)
644 coaddInputs.visits.sort()
650 modelPsfList = [tempExp.getPsf()
for tempExp
in tempExpList]
651 modelPsfWidthList = [modelPsf.computeBBox().getWidth()
for modelPsf
in modelPsfList]
652 psf = modelPsfList[modelPsfWidthList.index(max(modelPsfWidthList))]
654 psf = measAlg.CoaddPsf(coaddInputs.ccds, coaddExposure.getWcs(),
655 self.config.coaddPsf.makeControl())
656 coaddExposure.setPsf(psf)
657 apCorrMap = measAlg.makeCoaddApCorrMap(coaddInputs.ccds, coaddExposure.getBBox(afwImage.PARENT),
658 coaddExposure.getWcs())
659 coaddExposure.getInfo().setApCorrMap(apCorrMap)
661 def assembleSubregion(self, coaddExposure, bbox, tempExpRefList, imageScalerList, weightList,
662 bgInfoList, altMaskList, statsFlags, statsCtrl):
664 \brief Assemble the coadd for a sub-region.
666 For each coaddTempExp, check for (and swap in) an alternative mask if one is passed. If background
667 matching is enabled, add the background and background variance from each coaddTempExp. Remove mask
668 planes listed in config.removeMaskPlanes, Finally, mean-stack
669 the actual exposures using \ref afwMath.statisticsStack "statisticsStack" with outlier rejection if
670 config.doSigmaClip=True. Assign the stacked subregion back to the coadd.
672 \param[in] coaddExposure: The target image for the coadd
673 \param[in] bbox: Sub-region to coadd
674 \param[in] tempExpRefList: List of data reference to tempExp
675 \param[in] imageScalerList: List of image scalers
676 \param[in] weightList: List of weights
677 \param[in] bgInfoList: List of background data from background matching
678 \param[in] altMaskList: List of alternate masks to use rather than those stored with tempExp, or None
679 \param[in] statsFlags: Statistic for coadd
680 \param[in] statsCtrl: Statistics control object for coadd
682 self.log.debug(
"Computing coadd over %s", bbox)
683 tempExpName = self.getTempExpDatasetName(self.
warpType)
684 coaddMaskedImage = coaddExposure.getMaskedImage()
686 for tempExpRef, imageScaler, bgInfo, altMask
in zip(tempExpRefList, imageScalerList, bgInfoList,
688 exposure = tempExpRef.get(tempExpName +
"_sub", bbox=bbox)
689 maskedImage = exposure.getMaskedImage()
692 altMaskSub = altMask.Factory(altMask, bbox, afwImage.PARENT)
693 maskedImage.getMask().swap(altMaskSub)
694 imageScaler.scaleMaskedImage(maskedImage)
696 if self.config.doMatchBackgrounds
and not bgInfo.isReference:
697 backgroundModel = bgInfo.backgroundModel
698 backgroundImage = backgroundModel.getImage()
if \
699 self.matchBackgrounds.config.usePolynomial
else \
700 backgroundModel.getImageF()
701 backgroundImage.setXY0(coaddMaskedImage.getXY0())
702 maskedImage += backgroundImage.Factory(backgroundImage, bbox, afwImage.PARENT,
False)
703 var = maskedImage.getVariance()
704 var += (bgInfo.fitRMS)**2
706 if self.config.removeMaskPlanes:
707 mask = maskedImage.getMask()
708 for maskPlane
in self.config.removeMaskPlanes:
710 mask &= ~mask.getPlaneBitMask(maskPlane)
711 except Exception
as e:
712 self.log.warn(
"Unable to remove mask plane %s: %s", maskPlane, e.message)
714 maskedImageList.append(maskedImage)
716 with self.timer(
"stack"):
717 coaddSubregion = afwMath.statisticsStack(
718 maskedImageList, statsFlags, statsCtrl, weightList)
720 coaddMaskedImage.assign(coaddSubregion, bbox)
724 \brief Add metadata from the background matching to the coadd
726 \param[in] coaddExposure: Coadd
727 \param[in] tempExpRefList: List of data references for temp exps to go into coadd
728 \param[in] backgroundInfoList: List of background info, results from background matching
730 self.log.info(
"Adding exposure information to metadata")
731 metadata = coaddExposure.getMetadata()
732 metadata.addString(
"CTExp_SDQA1_DESCRIPTION",
733 "Background matching: Ratio of matchedMSE / diffImVar")
734 for ind, (tempExpRef, backgroundInfo)
in enumerate(zip(tempExpRefList, backgroundInfoList)):
735 tempExpStr =
'&'.join(
'%s=%s' % (k, v)
for k, v
in tempExpRef.dataId.items())
736 if backgroundInfo.isReference:
737 metadata.addString(
"ReferenceExp_ID", tempExpStr)
739 metadata.addString(
"CTExp_ID_%d" % (ind), tempExpStr)
740 metadata.addDouble(
"CTExp_SDQA1_%d" % (ind),
741 backgroundInfo.matchedMSE/backgroundInfo.diffImVar)
742 metadata.addDouble(
"CTExp_SDQA2_%d" % (ind),
743 backgroundInfo.fitRMS)
746 """Returns None on failure"""
748 return dataRef.get(
"brightObjectMask", immediate=
True)
749 except Exception
as e:
750 self.log.warn(
"Unable to read brightObjectMask for %s: %s", dataRef.dataId, e)
754 """Set the bright object masks
756 exposure: Exposure under consideration
757 dataId: Data identifier dict for patch
758 brightObjectMasks: afwTable of bright objects to mask
763 if brightObjectMasks
is None:
764 self.log.warn(
"Unable to apply bright object mask: none supplied")
766 self.log.info(
"Applying %d bright object masks to %s", len(brightObjectMasks), dataId)
767 md = brightObjectMasks.table.getMetadata()
770 self.log.warn(
"Expected to see %s in metadata", k)
772 if md.get(k) != dataId[k]:
773 self.log.warn(
"Expected to see %s == %s in metadata, saw %s", k, md.get(k), dataId[k])
775 mask = exposure.getMaskedImage().getMask()
776 wcs = exposure.getWcs()
777 plateScale = wcs.pixelScale().asArcseconds()
779 for rec
in brightObjectMasks:
780 center = afwGeom.PointI(wcs.skyToPixel(rec.getCoord()))
781 if rec[
"type"] ==
"box":
782 assert rec[
"angle"] == 0.0, (
"Angle != 0 for mask object %s" % rec[
"id"])
783 width = rec[
"width"].asArcseconds()/plateScale
784 height = rec[
"height"].asArcseconds()/plateScale
786 halfSize = afwGeom.ExtentI(0.5*width, 0.5*height)
787 bbox = afwGeom.Box2I(center - halfSize, center + halfSize)
789 bbox = afwGeom.BoxI(afwGeom.PointI(int(center[0] - 0.5*width), int(center[1] - 0.5*height)),
790 afwGeom.PointI(int(center[0] + 0.5*width), int(center[1] + 0.5*height)))
791 spans = afwGeom.SpanSet(bbox)
792 elif rec[
"type"] ==
"circle":
793 radius = int(rec[
"radius"].asArcseconds()/plateScale)
794 spans = afwGeom.SpanSet.fromShape(radius, offset=center)
796 self.log.warn(
"Unexpected region type %s at %s" % rec[
"type"], center)
801 def _makeArgumentParser(cls):
803 \brief Create an argument parser
805 parser = pipeBase.ArgumentParser(name=cls._DefaultName)
806 parser.add_id_argument(
"--id", cls.ConfigClass().coaddName +
"Coadd_directWarp",
807 help=
"data ID, e.g. --id tract=12345 patch=1,2",
808 ContainerClass=AssembleCoaddDataIdContainer)
809 parser.add_id_argument(
"--selectId",
"calexp", help=
"data ID, e.g. --selectId visit=6789 ccd=0..9",
810 ContainerClass=SelectDataIdContainer)
814 def _subBBoxIter(bbox, subregionSize):
816 \brief Iterate over subregions of a bbox
818 \param[in] bbox: bounding box over which to iterate: afwGeom.Box2I
819 \param[in] subregionSize: size of sub-bboxes
821 \return subBBox: next sub-bounding box of size subregionSize or smaller;
822 each subBBox is contained within bbox, so it may be smaller than subregionSize at the edges of bbox,
823 but it will never be empty
826 raise RuntimeError(
"bbox %s is empty" % (bbox,))
827 if subregionSize[0] < 1
or subregionSize[1] < 1:
828 raise RuntimeError(
"subregionSize %s must be nonzero" % (subregionSize,))
830 for rowShift
in range(0, bbox.getHeight(), subregionSize[1]):
831 for colShift
in range(0, bbox.getWidth(), subregionSize[0]):
832 subBBox = afwGeom.Box2I(bbox.getMin() + afwGeom.Extent2I(colShift, rowShift), subregionSize)
834 if subBBox.isEmpty():
835 raise RuntimeError(
"Bug: empty bbox! bbox=%s, subregionSize=%s, colShift=%s, rowShift=%s" %
836 (bbox, subregionSize, colShift, rowShift))
842 \brief A version of lsst.pipe.base.DataIdContainer specialized for assembleCoadd.
847 \brief Make self.refList from self.idList.
849 Interpret the config.doMatchBackgrounds, config.autoReference,
850 and whether a visit/run supplied.
851 If a visit/run is supplied, config.autoReference is automatically set to False.
852 if config.doMatchBackgrounds == false, then a visit/run will be ignored if accidentally supplied.
855 keysCoadd = namespace.butler.getKeys(datasetType=namespace.config.coaddName +
"Coadd",
857 keysCoaddTempExp = namespace.butler.getKeys(datasetType=namespace.config.coaddName +
858 "Coadd_directWarp", level=self.level)
860 if namespace.config.doMatchBackgrounds:
861 if namespace.config.autoReference:
862 datasetType = namespace.config.coaddName +
"Coadd"
863 validKeys = keysCoadd
865 datasetType = namespace.config.coaddName +
"Coadd_directWarp"
866 validKeys = keysCoaddTempExp
868 datasetType = namespace.config.coaddName +
"Coadd"
869 validKeys = keysCoadd
871 for dataId
in self.idList:
873 for key
in validKeys:
874 if key
not in dataId:
875 raise RuntimeError(
"--id must include " + key)
878 if (key
not in keysCoadd)
and (key
in keysCoaddTempExp):
879 if namespace.config.autoReference:
881 namespace.config.autoReference =
False
882 datasetType = namespace.config.coaddName +
"Coadd_directWarp"
883 print(
"Switching config.autoReference to False; applies only to background Matching.")
886 dataRef = namespace.butler.dataRef(
887 datasetType=datasetType,
890 self.refList.append(dataRef)
895 \brief Function to count the number of pixels with a specific mask in a footprint.
897 Find the intersection of mask & footprint. Count all pixels in the mask that are in the intersection that
898 have bitmask set but do not have ignoreMask set. Return the count.
900 \param[in] mask: mask to define intersection region by.
901 \parma[in] footprint: footprint to define the intersection region by.
902 \param[in] bitmask: specific mask that we wish to count the number of occurances of.
903 \param[in] ignoreMask: pixels to not consider.
904 \return count of number of pixels in footprint with specified mask.
906 bbox = footprint.getBBox()
907 bbox.clip(mask.getBBox(afwImage.PARENT))
908 fp = afwImage.Mask(bbox)
909 subMask = mask.Factory(mask, bbox, afwImage.PARENT)
910 footprint.spans.setMask(fp, bitmask)
911 return numpy.logical_and((subMask.getArray() & fp.getArray()) > 0,
912 (subMask.getArray() & ignoreMask) == 0).sum()
917 \anchor SafeClipAssembleCoaddConfig
919 \brief Configuration parameters for the SafeClipAssembleCoaddTask
921 clipDetection = pexConfig.ConfigurableField(
922 target=SourceDetectionTask,
923 doc=
"Detect sources on difference between unclipped and clipped coadd")
924 minClipFootOverlap = pexConfig.Field(
925 doc=
"Minimum fractional overlap of clipped footprint with visit DETECTED to be clipped",
929 minClipFootOverlapSingle = pexConfig.Field(
930 doc=
"Minimum fractional overlap of clipped footprint with visit DETECTED to be "
931 "clipped when only one visit overlaps",
935 minClipFootOverlapDouble = pexConfig.Field(
936 doc=
"Minimum fractional overlap of clipped footprints with visit DETECTED to be "
937 "clipped when two visits overlap",
941 maxClipFootOverlapDouble = pexConfig.Field(
942 doc=
"Maximum fractional overlap of clipped footprints with visit DETECTED when "
943 "considering two visits",
947 minBigOverlap = pexConfig.Field(
948 doc=
"Minimum number of pixels in footprint to use DETECTED mask from the single visits "
949 "when labeling clipped footprints",
957 AssembleCoaddConfig.setDefaults(self)
958 self.clipDetection.doTempLocalBackground =
False
959 self.clipDetection.reEstimateBackground =
False
960 self.clipDetection.returnOriginalFootprints =
False
961 self.clipDetection.thresholdPolarity =
"both"
962 self.clipDetection.thresholdValue = 2
963 self.clipDetection.nSigmaToGrow = 2
964 self.clipDetection.minPixels = 4
965 self.clipDetection.isotropicGrow =
True
966 self.clipDetection.thresholdType =
"pixel_stdev"
980 \anchor SafeClipAssembleCoaddTask_
982 \brief Assemble a coadded image from a set of coadded temporary exposures, being careful to clip & flag areas
983 with potential artifacts.
985 \section pipe_tasks_assembleCoadd_Contents Contents
986 - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Purpose
987 - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Initialize
988 - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Run
989 - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Config
990 - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Debug
991 - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Example
993 \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Purpose Description
995 \copybrief SafeClipAssembleCoaddTask
997 Read the documentation for \ref AssembleCoaddTask_ "AssembleCoaddTask" first since
998 SafeClipAssembleCoaddTask subtasks that task.
999 In \ref AssembleCoaddTask_ "AssembleCoaddTask", we compute the coadd as an clipped mean (i.e. we clip
1001 The problem with doing this is that when computing the coadd PSF at a given location, individual visit
1002 PSFs from visits with outlier pixels contribute to the coadd PSF and cannot be treated correctly.
1003 In this task, we correct for this behavior by creating a new badMaskPlane 'CLIPPED'.
1004 We populate this plane on the input coaddTempExps and the final coadd where i. difference imaging suggests
1005 that there is an outlier and ii. this outlier appears on only one or two images.
1006 Such regions will not contribute to the final coadd.
1007 Furthermore, any routine to determine the coadd PSF can now be cognizant of clipped regions.
1008 Note that the algorithm implemented by this task is preliminary and works correctly for HSC data.
1009 Parameter modifications and or considerable redesigning of the algorithm is likley required for other
1012 SafeClipAssembleCoaddTask uses a \ref SourceDetectionTask_ "clipDetection" subtask and also sub-classes
1013 \ref AssembleCoaddTask_ "AssembleCoaddTask". You can retarget the
1014 \ref SourceDetectionTask_ "clipDetection" subtask if you wish.
1016 \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Initialize Task initialization
1017 \copydoc \_\_init\_\_
1019 \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Run Invoking the Task
1022 \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Config Configuration parameters
1023 See \ref SafeClipAssembleCoaddConfig
1025 \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Debug Debug variables
1026 The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a
1027 flag \c -d to import \b debug.py from your \c PYTHONPATH; see \ref baseDebug for more about \b debug.py
1029 SafeClipAssembleCoaddTask has no debug variables of its own. The \ref SourceDetectionTask_ "clipDetection"
1030 subtasks may support debug variables. See the documetation for \ref SourceDetectionTask_ "clipDetection"
1031 for further information.
1033 \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Example A complete example of using SafeClipAssembleCoaddTask
1035 SafeClipAssembleCoaddTask assembles a set of warped coaddTempExp images into a coadded image.
1036 The SafeClipAssembleCoaddTask is invoked by running assembleCoadd.py <em>without</em> the flag
1038 Usage of assembleCoadd.py expects a data reference to the tract patch and filter to be coadded
1039 (specified using '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]') along
1040 with a list of coaddTempExps to attempt to coadd (specified using
1041 '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]').
1042 Only the coaddTempExps that cover the specified tract and patch will be coadded.
1043 A list of the available optional arguments can be obtained by calling assembleCoadd.py with the --help
1044 command line argument:
1046 assembleCoadd.py --help
1048 To demonstrate usage of the SafeClipAssembleCoaddTask in the larger context of multi-band processing, we
1049 will generate the HSC-I & -R band coadds from HSC engineering test data provided in the ci_hsc package. To
1050 begin, assuming that the lsst stack has been already set up, we must set up the obs_subaru and ci_hsc
1052 This defines the environment variable $CI_HSC_DIR and points at the location of the package. The raw HSC
1053 data live in the $CI_HSC_DIR/raw directory. To begin assembling the coadds, we must first
1056 <DD> process the individual ccds in $CI_HSC_RAW to produce calibrated exposures</DD>
1058 <DD> create a skymap that covers the area of the sky present in the raw exposures</DD>
1059 <DT>makeCoaddTempExp</DT>
1060 <DD> warp the individual calibrated exposures to the tangent plane of the coadd</DD>
1062 We can perform all of these steps by running
1064 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
1066 This will produce warped coaddTempExps for each visit. To coadd the wraped data, we call assembleCoadd.py
1069 assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 --selectId visit=903988 ccd=24
1071 This will process the HSC-I band data. The results are written in $CI_HSC_DIR/DATA/deepCoadd-results/HSC-I
1072 You may also choose to run:
1074 scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346
1075 assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R --selectId visit=903334 ccd=16 --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 --selectId visit=903334 ccd=100 --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 --selectId visit=903338 ccd=18 --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 --selectId visit=903342 ccd=10 --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 --selectId visit=903344 ccd=5 --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 --selectId visit=903346 ccd=6 --selectId visit=903346 ccd=12
1077 to generate the coadd for the HSC-R band if you are interested in following multiBand Coadd processing as
1078 discussed in \ref pipeTasks_multiBand.
1080 ConfigClass = SafeClipAssembleCoaddConfig
1081 _DefaultName =
"safeClipAssembleCoadd"
1085 \brief Initialize the task and make the \ref SourceDetectionTask_ "clipDetection" subtask.
1087 AssembleCoaddTask.__init__(self, *args, **kwargs)
1088 schema = afwTable.SourceTable.makeMinimalSchema()
1089 self.makeSubtask(
"clipDetection", schema=schema)
1091 def assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList, bgModelList, *args, **kwargs):
1093 \brief Assemble the coadd for a region
1095 Compute the difference of coadds created with and without outlier rejection to identify coadd pixels
1096 that have outlier values in some individual visits. Detect clipped regions on the difference image and
1097 mark these regions on the one or two individual coaddTempExps where they occur if there is significant
1098 overlap between the clipped region and a source.
1099 This leaves us with a set of footprints from the difference image that have been identified as having
1100 occured on just one or two individual visits. However, these footprints were generated from a
1101 difference image. It is conceivable for a large diffuse source to have become broken up into multiple
1102 footprints acrosss the coadd difference in this process.
1103 Determine the clipped region from all overlapping footprints from the detected sources in each visit -
1104 these are big footprints.
1105 Combine the small and big clipped footprints and mark them on a new bad mask plane
1106 Generate the coadd using \ref AssembleCoaddTask.assemble_ "AssembleCoaddTask.assemble" without outlier
1107 removal. Clipped footprints will no longer make it into the coadd because they are marked in the new
1110 N.b. *args and **kwargs are passed but ignored in order to match the call signature expected by the
1113 @param skyInfo: Patch geometry information, from getSkyInfo
1114 @param tempExpRefList: List of data reference to tempExp
1115 @param imageScalerList: List of image scalers
1116 @param weightList: List of weights
1117 @param bgModelList: List of background models from background matching
1118 return coadd exposure
1120 exp = self.
buildDifferenceImage(skyInfo, tempExpRefList, imageScalerList, weightList, bgModelList)
1121 mask = exp.getMaskedImage().getMask()
1122 mask.addMaskPlane(
"CLIPPED")
1124 result = self.
detectClip(exp, tempExpRefList)
1126 self.log.info(
'Found %d clipped objects', len(result.clipFootprints))
1129 maskClipValue = mask.getPlaneBitMask(
"CLIPPED")
1130 maskDetValue = mask.getPlaneBitMask(
"DETECTED") | mask.getPlaneBitMask(
"DETECTED_NEGATIVE")
1131 bigFootprints = self.
detectClipBig(result.tempExpClipList, result.clipFootprints, result.clipIndices,
1132 maskClipValue, maskDetValue)
1135 maskClip = mask.Factory(mask.getBBox(afwImage.PARENT))
1136 afwDet.setMaskFromFootprintList(maskClip, result.clipFootprints, maskClipValue)
1138 maskClipBig = maskClip.Factory(mask.getBBox(afwImage.PARENT))
1139 afwDet.setMaskFromFootprintList(maskClipBig, bigFootprints, maskClipValue)
1140 maskClip |= maskClipBig
1143 badMaskPlanes = self.config.badMaskPlanes[:]
1144 badMaskPlanes.append(
"CLIPPED")
1145 badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes)
1146 coaddExp = AssembleCoaddTask.assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList,
1147 bgModelList, result.tempExpClipList,
1153 maskExp = coaddExp.getMaskedImage().getMask()
1160 \brief Return an exposure that contains the difference between and unclipped and clipped coadds.
1162 Generate a difference image between clipped and unclipped coadds.
1163 Compute the difference image by subtracting an outlier-clipped coadd from an outlier-unclipped coadd.
1164 Return the difference image.
1166 @param skyInfo: Patch geometry information, from getSkyInfo
1167 @param tempExpRefList: List of data reference to tempExp
1168 @param imageScalerList: List of image scalers
1169 @param weightList: List of weights
1170 @param bgModelList: List of background models from background matching
1171 @return Difference image of unclipped and clipped coadd wrapped in an Exposure
1174 coaddMean = AssembleCoaddTask.assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList,
1175 bgModelList, doClip=
False)
1178 coaddClip = AssembleCoaddTask.assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList,
1179 bgModelList, doClip=
True)
1181 coaddDiff = coaddMean.getMaskedImage().Factory(coaddMean.getMaskedImage())
1182 coaddDiff -= coaddClip.getMaskedImage()
1183 exp = afwImage.ExposureF(coaddDiff)
1184 exp.setPsf(coaddMean.getPsf())
1189 \brief Detect clipped regions on an exposure and set the mask on the individual tempExp masks
1191 Detect footprints in the difference image after smoothing the difference image with a Gaussian kernal.
1192 Identify footprints that overlap with one or two input coaddTempExps by comparing the computed overlap
1193 fraction to thresholds set in the config.
1194 A different threshold is applied depending on the number of overlapping visits (restricted to one or
1196 If the overlap exceeds the thresholds, the footprint is considered "CLIPPED" and is marked as such on
1198 Return a struct with the clipped footprints, the indices of the coaddTempExps that end up overlapping
1199 with the clipped footprints and a list of new masks for the coaddTempExps.
1201 \param[in] exp: Exposure to run detection on
1202 \param[in] tempExpRefList: List of data reference to tempExp
1203 \return struct containing:
1204 - clippedFootprints: list of clipped footprints
1205 - clippedIndices: indices for each clippedFootprint in tempExpRefList
1206 - tempExpClipList: list of new masks for tempExp
1208 mask = exp.getMaskedImage().getMask()
1209 maskClipValue = mask.getPlaneBitMask(
"CLIPPED")
1210 maskDetValue = mask.getPlaneBitMask(
"DETECTED") | mask.getPlaneBitMask(
"DETECTED_NEGATIVE")
1211 fpSet = self.clipDetection.detectFootprints(exp, doSmooth=
True, clearMask=
True)
1213 fpSet.positive.merge(fpSet.negative)
1214 footprints = fpSet.positive
1215 self.log.info(
'Found %d potential clipped objects', len(footprints.getFootprints()))
1216 ignoreMask = self.getBadPixelMask()
1222 tempExpClipList = [tmpExpRef.get(self.getTempExpDatasetName(self.
warpType),
1223 immediate=
True).getMaskedImage().getMask()
for
1224 tmpExpRef
in tempExpRefList]
1226 for footprint
in footprints.getFootprints():
1227 nPixel = footprint.getArea()
1231 for i, tmpExpMask
in enumerate(tempExpClipList):
1235 totPixel = nPixel - ignore
1238 if ignore > overlapDet
or totPixel <= 0.5*nPixel
or overlapDet == 0:
1240 overlap.append(overlapDet/float(totPixel))
1241 maskList.append(tmpExpMask)
1244 overlap = numpy.array(overlap)
1245 if not len(overlap):
1252 if len(overlap) == 1:
1253 if overlap[0] > self.config.minClipFootOverlapSingle:
1258 clipIndex = numpy.where(overlap > self.config.minClipFootOverlap)[0]
1259 if len(clipIndex) == 1:
1261 keepIndex = [clipIndex[0]]
1264 clipIndex = numpy.where(overlap > self.config.minClipFootOverlapDouble)[0]
1265 if len(clipIndex) == 2
and len(overlap) > 3:
1266 clipIndexComp = numpy.where(overlap <= self.config.minClipFootOverlapDouble)[0]
1267 if numpy.max(overlap[clipIndexComp]) <= self.config.maxClipFootOverlapDouble:
1269 keepIndex = clipIndex
1274 for index
in keepIndex:
1275 footprint.spans.setMask(maskList[index], maskClipValue)
1277 clipIndices.append(numpy.array(indexList)[keepIndex])
1278 clipFootprints.append(footprint)
1280 return pipeBase.Struct(clipFootprints=clipFootprints, clipIndices=clipIndices,
1281 tempExpClipList=tempExpClipList)
1283 def detectClipBig(self, tempExpClipList, clipFootprints, clipIndices, maskClipValue, maskDetValue):
1285 \brief Find footprints from individual tempExp footprints for large footprints.
1287 Identify big footprints composed of many sources in the coadd difference that may have originated in a
1288 large diffuse source in the coadd. We do this by indentifying all clipped footprints that overlap
1289 significantly with each source in all the coaddTempExps.
1291 \param[in] tempExpClipList: List of tempExp masks with clipping information
1292 \param[in] clipFootprints: List of clipped footprints
1293 \param[in] clipIndices: List of which entries in tempExpClipList each footprint belongs to
1294 \param[in] maskClipValue: Mask value of clipped pixels
1295 \param[in] maskClipValue: Mask value of detected pixels
1296 \return list of big footprints
1298 bigFootprintsCoadd = []
1299 ignoreMask = self.getBadPixelMask()
1300 for index, tmpExpMask
in enumerate(tempExpClipList):
1303 maskVisitDet = tmpExpMask.Factory(tmpExpMask, tmpExpMask.getBBox(afwImage.PARENT),
1304 afwImage.PARENT,
True)
1305 maskVisitDet &= maskDetValue
1306 visitFootprints = afwDet.FootprintSet(maskVisitDet, afwDet.Threshold(1))
1309 clippedFootprintsVisit = []
1310 for foot, clipIndex
in zip(clipFootprints, clipIndices):
1311 if index
not in clipIndex:
1313 clippedFootprintsVisit.append(foot)
1314 maskVisitClip = maskVisitDet.Factory(maskVisitDet.getBBox(afwImage.PARENT))
1315 afwDet.setMaskFromFootprintList(maskVisitClip, clippedFootprintsVisit, maskClipValue)
1317 bigFootprintsVisit = []
1318 for foot
in visitFootprints.getFootprints():
1319 if foot.getArea() < self.config.minBigOverlap:
1322 if nCount > self.config.minBigOverlap:
1323 bigFootprintsVisit.append(foot)
1324 bigFootprintsCoadd.append(foot)
1327 maskVisitClip.clearAllMaskPlanes()
1328 afwDet.setMaskFromFootprintList(maskVisitClip, bigFootprintsVisit, maskClipValue)
1329 tmpExpMask |= maskVisitClip
1331 return bigFootprintsCoadd
def __init__
Initialize the task and make the clipDetection subtask.
def run
Assemble a coadd from a set of Warps.
def getBackgroundReferenceScaler
Construct an image scaler for the background reference frame.
def assemble
Assemble the coadd for a region.
def addBackgroundMatchingMetadata
Add metadata from the background matching to the coadd.
def detectClipBig
Find footprints from individual tempExp footprints for large footprints.
def __init__
Initialize the task.
def assembleSubregion
Assemble the coadd for a sub-region.
def assembleMetadata
Set the metadata for the coadd.
def backgroundMatching
Perform background matching on the prepared inputs.
Configuration parameters for the SafeClipAssembleCoaddTask.
Assemble a coadded image from a set of warps (coadded temporary exposures).
Assemble a coadded image from a set of coadded temporary exposures, being careful to clip & flag area...
def buildDifferenceImage
Return an exposure that contains the difference between and unclipped and clipped coadds...
def detectClip
Detect clipped regions on an exposure and set the mask on the individual tempExp masks.
def prepareInputs
Prepare the input warps for coaddition by measuring the weight for each warp and the scaling for the ...
def makeDataRefList
Make self.refList from self.idList.
Configuration parameters for the AssembleCoaddTask.
def countMaskFromFootprint
Function to count the number of pixels with a specific mask in a footprint.
def assemble
Assemble a coadd from input warps.
def readBrightObjectMasks
A version of lsst.pipe.base.DataIdContainer specialized for assembleCoadd.
def getTempExpRefList
Generate list data references corresponding to warped exposures that lie within the patch to be coadd...