1 from __future__
import absolute_import, division, print_function
2 from builtins
import zip
3 from builtins
import range
26 import lsst.pex.config
as pexConfig
27 import lsst.pex.exceptions
as pexExceptions
28 import lsst.afw.detection
as afwDetect
29 import lsst.afw.geom
as afwGeom
30 import lsst.afw.image
as afwImage
31 import lsst.afw.math
as afwMath
32 import lsst.afw.table
as afwTable
33 import lsst.afw.detection
as afwDet
34 import lsst.coadd.utils
as coaddUtils
35 import lsst.pipe.base
as pipeBase
36 import lsst.meas.algorithms
as measAlg
37 from .coaddBase
import CoaddBaseTask, SelectDataIdContainer
38 from .interpImage
import InterpImageTask
39 from .matchBackgrounds
import MatchBackgroundsTask
40 from .scaleZeroPoint
import ScaleZeroPointTask
41 from .coaddHelpers
import groupPatchExposures, getGroupDataRef
42 from lsst.meas.algorithms
import SourceDetectionTask
44 __all__ = [
"AssembleCoaddTask",
"SafeClipAssembleCoaddTask"]
49 \anchor AssembleCoaddConfig_
51 \brief Configuration parameters for the \ref AssembleCoaddTask_ "AssembleCoaddTask"
53 subregionSize = pexConfig.ListField(
55 doc=
"Width, height of stack subregion size; "
56 "make small enough that a full stack of images will fit into memory at once.",
60 doSigmaClip = pexConfig.Field(
62 doc=
"Perform sigma clipped outlier rejection? If False then compute a simple mean.",
65 sigmaClip = pexConfig.Field(
67 doc=
"Sigma for outlier rejection; ignored if doSigmaClip false.",
70 clipIter = pexConfig.Field(
72 doc=
"Number of iterations of outlier rejection; ignored if doSigmaClip false.",
75 scaleZeroPoint = pexConfig.ConfigurableField(
76 target=ScaleZeroPointTask,
77 doc=
"Task to adjust the photometric zero point of the coadd temp exposures",
79 doInterp = pexConfig.Field(
80 doc=
"Interpolate over NaN pixels? Also extrapolate, if necessary, but the results are ugly.",
84 interpImage = pexConfig.ConfigurableField(
85 target=InterpImageTask,
86 doc=
"Task to interpolate (and extrapolate) over NaN pixels",
88 matchBackgrounds = pexConfig.ConfigurableField(
89 target=MatchBackgroundsTask,
90 doc=
"Task to match backgrounds",
92 maxMatchResidualRatio = pexConfig.Field(
93 doc=
"Maximum ratio of the mean squared error of the background matching model to the variance "
94 "of the difference in backgrounds",
98 maxMatchResidualRMS = pexConfig.Field(
99 doc=
"Maximum RMS of residuals of the background offset fit in matchBackgrounds.",
103 doWrite = pexConfig.Field(
104 doc=
"Persist coadd?",
108 doMatchBackgrounds = pexConfig.Field(
109 doc=
"Match backgrounds of coadd temp exposures before coadding them? "
110 "If False, the coadd temp expsosures must already have been background subtracted or matched",
114 autoReference = pexConfig.Field(
115 doc=
"Automatically select the coadd temp exposure to use as a reference for background matching? "
116 "Ignored if doMatchBackgrounds false. "
117 "If False you must specify the reference temp exposure as the data Id",
121 maskPropagationThresholds = pexConfig.DictField(
124 doc=(
"Threshold (in fractional weight) of rejection at which we propagate a mask plane to "
125 "the coadd; that is, we set the mask bit on the coadd if the fraction the rejected frames "
126 "would have contributed exceeds this value."),
127 default={
"SAT": 0.1},
129 removeMaskPlanes = pexConfig.ListField(dtype=str, default=[
"CROSSTALK",
"NOT_DEBLENDED"],
130 doc=
"Mask planes to remove before coadding")
139 doMaskBrightObjects = pexConfig.Field(dtype=bool, default=
False,
140 doc=
"Set mask and flag bits for bright objects?")
141 brightObjectMaskName = pexConfig.Field(dtype=str, default=
"BRIGHT_OBJECT",
142 doc=
"Name of mask bit used for bright objects")
145 CoaddBaseTask.ConfigClass.setDefaults(self)
158 \anchor AssembleCoaddTask_
160 \brief Assemble a coadded image from a set of coadded temporary exposures.
162 \section pipe_tasks_assembleCoadd_Contents Contents
163 - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Purpose
164 - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Initialize
165 - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Run
166 - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Config
167 - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Debug
168 - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Example
170 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Purpose Description
172 \copybrief AssembleCoaddTask_
174 We want to assemble a coadded image from a set of coadded temporary exposures (coaddTempExps).
175 Each input coaddTempExp covers a patch on the sky and corresponds to a single run/visit/exposure of the
176 covered patch. We provide the task with a list of coaddTempExps (selectDataList) from which it selects
177 coaddTempExps that cover the specified patch (pointed at by dataRef).
178 Each coaddTempExp that goes into a coadd will typically have an independent photometric zero-point.
179 Therefore, we must scale each coaddTempExp to set it to a common photometric zeropoint. By default, each
180 coaddTempExp has backgrounds and hence will require config.doMatchBackgrounds=True.
181 When background matching is enabled, the task may be configured to automatically select a reference exposure
182 (config.autoReference=True). If this is not done, we require that the input dataRef provides access to a
183 coaddTempExp (dataset type coaddName + 'Coadd_tempExp') which is used as the reference exposure.
184 The coadd is computed as a mean with optional outlier rejection.
185 Criteria for outlier rejection are set in \ref AssembleCoaddConfig. Finally, coaddTempExps can have bad 'NaN'
186 pixels which received no input from the source calExps. We interpolate over these bad (NaN) pixels.
188 AssembleCoaddTask uses several sub-tasks. These are
190 <DT>\ref ScaleZeroPointTask_ "ScaleZeroPointTask"</DT>
191 <DD> create and use an imageScaler object to scale the photometric zeropoint for each coaddTempExp</DD>
192 <DT>\ref MatchBackgroundsTask_ "MatchBackgroundsTask"</DT>
193 <DD> match background in a coaddTempExp to a reference exposure (and select the reference exposure if one is
195 <DT>\ref InterpImageTask_ "InterpImageTask"</DT>
196 <DD>interpolate across bad pixels (NaN) in the final coadd</DD>
198 You can retarget these subtasks if you wish.
200 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Initialize Task initialization
201 \copydoc \_\_init\_\_
203 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Run Invoking the Task
206 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Config Configuration parameters
207 See \ref AssembleCoaddConfig_
209 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Debug Debug variables
210 The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a
211 flag \c -d to import \b debug.py from your \c PYTHONPATH; see \ref baseDebug for more about \b debug.py files.
212 AssembleCoaddTask has no debug variables of its own. Some of the subtasks may support debug variables. See
213 the documetation for the subtasks for further information.
215 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Example A complete example of using AssembleCoaddTask
217 AssembleCoaddTask assembles a set of warped coaddTempExp images into a coadded image. The AssembleCoaddTask
218 can be invoked by running assembleCoadd.py with the flag '--legacyCoadd'. Usage of assembleCoadd.py expects
219 a data reference to the tract patch and filter to be coadded (specified using
220 '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]') along with a list of
221 coaddTempExps to attempt to coadd (specified using
222 '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]'). Only the coaddTempExps
223 that cover the specified tract and patch will be coadded. A list of the available optional
224 arguments can be obtained by calling assembleCoadd.py with the --help command line argument:
226 assembleCoadd.py --help
228 To demonstrate usage of the AssembleCoaddTask in the larger context of multi-band processing, we will generate
229 the HSC-I & -R band coadds from HSC engineering test data provided in the ci_hsc package. To begin, assuming
230 that the lsst stack has been already set up, we must set up the obs_subaru and ci_hsc packages.
231 This defines the environment variable $CI_HSC_DIR and points at the location of the package. The raw HSC
232 data live in the $CI_HSC_DIR/raw directory. To begin assembling the coadds, we must first
235 <DD> process the individual ccds in $CI_HSC_RAW to produce calibrated exposures</DD>
237 <DD> create a skymap that covers the area of the sky present in the raw exposures</DD>
238 <DT>makeCoaddTempExp</DT>
239 <DD> warp the individual calibrated exposures to the tangent plane of the coadd</DD>
241 We can perform all of these steps by running
243 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
245 This will produce warped coaddTempExps for each visit. To coadd the warped data, we call assembleCoadd.py as
248 assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 --selectId visit=903988 ccd=24\endcode
249 that will process the HSC-I band data. The results are written in $CI_HSC_DIR/DATA/deepCoadd-results/HSC-I
250 You may also choose to run:
252 scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346
253 assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R --selectId visit=903334 ccd=16 --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 --selectId visit=903334 ccd=100 --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 --selectId visit=903338 ccd=18 --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 --selectId visit=903342 ccd=10 --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 --selectId visit=903344 ccd=5 --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 --selectId visit=903346 ccd=6 --selectId visit=903346 ccd=12
255 to generate the coadd for the HSC-R band if you are interested in following multiBand Coadd processing as
256 discussed in \ref pipeTasks_multiBand (but note that normally, one would use the
257 \ref SafeClipAssembleCoaddTask_ "SafeClipAssembleCoaddTask" rather than AssembleCoaddTask to make the coadd.
259 ConfigClass = AssembleCoaddConfig
260 _DefaultName =
"assembleCoadd"
264 \brief Initialize the task. Create the \ref InterpImageTask "interpImage",
265 \ref MatchBackgroundsTask "matchBackgrounds", & \ref ScaleZeroPointTask "scaleZeroPoint" subtasks.
267 CoaddBaseTask.__init__(self, *args, **kwargs)
268 self.makeSubtask(
"interpImage")
269 self.makeSubtask(
"matchBackgrounds")
270 self.makeSubtask(
"scaleZeroPoint")
272 if self.config.doMaskBrightObjects:
273 mask = afwImage.MaskU()
276 except pexExceptions.LsstCppException:
277 raise RuntimeError(
"Unable to define mask plane for bright objects; planes used are %s" %
278 mask.getMaskPlaneDict().keys())
282 def run(self, dataRef, selectDataList=[]):
284 \brief Assemble a coadd from a set of coaddTempExp
286 Coadd a set of coaddTempExps. Compute weights to be applied to each coaddTempExp and find scalings to
287 match the photometric zeropoint to a reference coaddTempExp. Optionally, match backgrounds across
288 coaddTempExps if the background has not already been removed. Assemble the coaddTempExps using
289 \ref assemble. Interpolate over NaNs and optionally write the coadd to disk. Return the coadded
293 \param[in] dataRef: Data reference defining the patch for coaddition and the reference coaddTempExp
294 (if config.autoReference=False). Used to access the following data products:
295 - [in] self.config.coaddName + "Coadd_skyMap"
296 - [in] self.config.coaddName + "Coadd_tempExp" (optionally)
297 - [out] self.config.coaddName + "Coadd"
298 \param[in] selectDataList[in]: List of data references to coaddTempExps. Data to be coadded will be
299 selected from this list based on overlap with the patch defined by dataRef.
301 \return a pipeBase.Struct with fields:
302 - coaddExposure: coadded exposure
304 skyInfo = self.getSkyInfo(dataRef)
305 calExpRefList = self.selectExposures(dataRef, skyInfo, selectDataList=selectDataList)
306 if len(calExpRefList) == 0:
307 self.log.warn(
"No exposures to coadd")
309 self.log.info(
"Coadding %d exposures", len(calExpRefList))
313 self.log.info(
"Found %d %s", len(inputData.tempExpRefList), self.getTempExpDatasetName())
314 if len(inputData.tempExpRefList) == 0:
315 self.log.warn(
"No coadd temporary exposures found")
317 if self.config.doMatchBackgrounds:
320 if len(inputData.tempExpRefList) == 0:
321 self.log.warn(
"No valid background models")
324 coaddExp = self.
assemble(skyInfo, inputData.tempExpRefList, inputData.imageScalerList,
325 inputData.weightList,
326 inputData.backgroundInfoList
if self.config.doMatchBackgrounds
else None,
327 doClip=self.config.doSigmaClip)
328 if self.config.doMatchBackgrounds:
330 inputData.backgroundInfoList)
332 if self.config.doInterp:
333 self.interpImage.run(coaddExp.getMaskedImage(), planeName=
"NO_DATA")
335 varArray = coaddExp.getMaskedImage().getVariance().getArray()
336 varArray[:] = numpy.where(varArray > 0, varArray, numpy.inf)
338 if self.config.doMaskBrightObjects:
342 if self.config.doWrite:
343 self.writeCoaddOutput(dataRef, coaddExp)
345 return pipeBase.Struct(coaddExposure=coaddExp)
349 \brief Generate list of coaddTempExp data references corresponding to exposures that lie within the
352 \param[in] patchRef: Data reference for patch
353 \param[in] calExpRefList: List of data references for input calexps
354 \return List of coaddTempExp data references
356 butler = patchRef.getButler()
358 self.getTempExpDatasetName())
359 tempExpRefList = [
getGroupDataRef(butler, self.getTempExpDatasetName(), g, groupData.keys)
for
360 g
in groupData.groups.keys()]
361 return tempExpRefList
365 \brief Construct an image scaler for the background reference frame
367 Each coaddTempExp has a different background level. A reference background level must be chosen before
368 coaddition. If config.autoReference=True, \ref backgroundMatching will pick the reference level and
369 this routine is a no-op and None is returned. Otherwise, use the
370 \ref ScaleZeroPointTask_ "scaleZeroPoint" subtask to compute an imageScaler object for the provided
371 reference image and return it.
373 \param[in] dataRef: Data reference for the background reference frame, or None
374 \return image scaler, or None
376 if self.config.autoReference:
380 dataset = self.getTempExpDatasetName()
381 if not dataRef.datasetExists(dataset):
382 raise RuntimeError(
"Could not find reference exposure %s %s." % (dataset, dataRef.dataId))
384 refExposure = dataRef.get(self.getTempExpDatasetName(), immediate=
True)
385 refImageScaler = self.scaleZeroPoint.computeImageScaler(
386 exposure=refExposure,
389 return refImageScaler
393 \brief Prepare the input warps for coaddition by measuring the weight for each warp and the scaling
394 for the photometric zero point.
396 Each coaddTempExp has its own photometric zeropoint and background variance. Before coadding these
397 coaddTempExps together, compute a scale factor to normalize the photometric zeropoint and compute the
398 weight for each coaddTempExp.
400 \param[in] refList: List of data references to tempExp
402 - tempExprefList: List of data references to tempExp
403 - weightList: List of weightings
404 - imageScalerList: List of image scalers
406 statsCtrl = afwMath.StatisticsControl()
407 statsCtrl.setNumSigmaClip(self.config.sigmaClip)
408 statsCtrl.setNumIter(self.config.clipIter)
409 statsCtrl.setAndMask(self.getBadPixelMask())
410 statsCtrl.setNanSafe(
True)
418 tempExpName = self.getTempExpDatasetName()
419 for tempExpRef
in refList:
420 if not tempExpRef.datasetExists(tempExpName):
421 self.log.warn(
"Could not find %s %s; skipping it", tempExpName, tempExpRef.dataId)
424 tempExp = tempExpRef.get(tempExpName, immediate=
True)
425 maskedImage = tempExp.getMaskedImage()
426 imageScaler = self.scaleZeroPoint.computeImageScaler(
431 imageScaler.scaleMaskedImage(maskedImage)
432 except Exception
as e:
433 self.log.warn(
"Scaling failed for %s (skipping it): %s", tempExpRef.dataId, e)
435 statObj = afwMath.makeStatistics(maskedImage.getVariance(), maskedImage.getMask(),
436 afwMath.MEANCLIP, statsCtrl)
437 meanVar, meanVarErr = statObj.getResult(afwMath.MEANCLIP)
438 weight = 1.0 / float(meanVar)
439 if not numpy.isfinite(weight):
440 self.log.warn(
"Non-finite weight for %s: skipping", tempExpRef.dataId)
442 self.log.info(
"Weight of %s %s = %0.3f", tempExpName, tempExpRef.dataId, weight)
447 tempExpRefList.append(tempExpRef)
448 weightList.append(weight)
449 imageScalerList.append(imageScaler)
451 return pipeBase.Struct(tempExpRefList=tempExpRefList, weightList=weightList,
452 imageScalerList=imageScalerList)
456 \brief Perform background matching on the prepared inputs
458 Each coaddTempExp has a different background level that must be normalized to a reference level
459 before coaddition. If no reference is provided, the background matcher selects one. If the background
460 matching is performed sucessfully, recompute the weight to be applied to the coaddTempExp to be
461 consistent with the scaled background.
463 \param[in] inputData: Struct from prepareInputs() with tempExpRefList, weightList, imageScalerList
464 \param[in] refExpDataRef: Data reference for background reference tempExp, or None
465 \param[in] refImageScaler: Image scaler for background reference tempExp, or None
467 - tempExprefList: List of data references to tempExp
468 - weightList: List of weightings
469 - imageScalerList: List of image scalers
470 - backgroundInfoList: result from background matching
473 backgroundInfoList = self.matchBackgrounds.run(
474 expRefList=inputData.tempExpRefList,
475 imageScalerList=inputData.imageScalerList,
476 refExpDataRef=refExpDataRef
if not self.config.autoReference
else None,
477 refImageScaler=refImageScaler,
478 expDatasetType=self.getTempExpDatasetName(),
480 except Exception
as e:
481 self.log.fatal(
"Cannot match backgrounds: %s", e)
482 raise pipeBase.TaskError(
"Background matching failed.")
485 newTempExpRefList = []
486 newBackgroundStructList = []
490 for tempExpRef, bgInfo, scaler, weight
in zip(inputData.tempExpRefList, backgroundInfoList,
491 inputData.imageScalerList, inputData.weightList):
492 if not bgInfo.isReference:
495 if (bgInfo.backgroundModel
is None):
496 self.log.info(
"No background offset model available for %s: skipping", tempExpRef.dataId)
499 varianceRatio = bgInfo.matchedMSE / bgInfo.diffImVar
500 except Exception
as e:
501 self.log.info(
"MSE/Var ratio not calculable (%s) for %s: skipping",
502 e, tempExpRef.dataId)
504 if not numpy.isfinite(varianceRatio):
505 self.log.info(
"MSE/Var ratio not finite (%.2f / %.2f) for %s: skipping",
506 bgInfo.matchedMSE, bgInfo.diffImVar, tempExpRef.dataId)
508 elif (varianceRatio > self.config.maxMatchResidualRatio):
509 self.log.info(
"Bad fit. MSE/Var ratio %.2f > %.2f for %s: skipping",
510 varianceRatio, self.config.maxMatchResidualRatio, tempExpRef.dataId)
512 elif (bgInfo.fitRMS > self.config.maxMatchResidualRMS):
513 self.log.info(
"Bad fit. RMS %.2f > %.2f for %s: skipping",
514 bgInfo.fitRMS, self.config.maxMatchResidualRMS, tempExpRef.dataId)
516 newWeightList.append(1 / (1 / weight + bgInfo.fitRMS**2))
517 newTempExpRefList.append(tempExpRef)
518 newBackgroundStructList.append(bgInfo)
519 newScaleList.append(scaler)
521 return pipeBase.Struct(tempExpRefList=newTempExpRefList, weightList=newWeightList,
522 imageScalerList=newScaleList, backgroundInfoList=newBackgroundStructList)
524 def assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList, bgInfoList=None,
525 altMaskList=
None, doClip=
False, mask=
None):
527 \anchor AssembleCoaddTask.assemble_
529 \brief Assemble a coadd from input warps
531 Assemble the coadd using the provided list of coaddTempExps. Since the full coadd covers a patch (a
532 large area), the assembly is performed over small areas on the image at a time in order to
533 conserve memory usage. Iterate over subregions within the outer bbox of the patch using
534 \ref assembleSubregion to mean-stack the corresponding subregions from the coaddTempExps (with outlier
535 rejection if config.doSigmaClip=True). Set the edge bits the the coadd mask based on the weight map.
537 \param[in] skyInfo: Patch geometry information, from getSkyInfo
538 \param[in] tempExpRefList: List of data references to tempExp
539 \param[in] imageScalerList: List of image scalers
540 \param[in] weightList: List of weights
541 \param[in] bgInfoList: List of background data from background matching, or None
542 \param[in] altMaskList: List of alternate masks to use rather than those stored with tempExp, or None
543 \param[in] doClip: Use clipping when codding?
544 \param[in] mask: Mask to ignore when coadding
545 \return coadded exposure
547 tempExpName = self.getTempExpDatasetName()
548 self.log.info(
"Assembling %s %s", len(tempExpRefList), tempExpName)
550 mask = self.getBadPixelMask()
552 statsCtrl = afwMath.StatisticsControl()
553 statsCtrl.setNumSigmaClip(self.config.sigmaClip)
554 statsCtrl.setNumIter(self.config.clipIter)
555 statsCtrl.setAndMask(mask)
556 statsCtrl.setNanSafe(
True)
557 statsCtrl.setWeighted(
True)
558 statsCtrl.setCalcErrorFromInputVariance(
True)
559 for plane, threshold
in self.config.maskPropagationThresholds.items():
560 bit = afwImage.MaskU.getMaskPlane(plane)
561 statsCtrl.setMaskPropagationThreshold(bit, threshold)
564 statsFlags = afwMath.MEANCLIP
566 statsFlags = afwMath.MEAN
568 if bgInfoList
is None:
569 bgInfoList = [
None]*len(tempExpRefList)
571 if altMaskList
is None:
572 altMaskList = [
None]*len(tempExpRefList)
574 coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
575 coaddExposure.setCalib(self.scaleZeroPoint.getCalib())
576 coaddExposure.getInfo().setCoaddInputs(self.inputRecorder.makeCoaddInputs())
578 coaddMaskedImage = coaddExposure.getMaskedImage()
579 subregionSizeArr = self.config.subregionSize
580 subregionSize = afwGeom.Extent2I(subregionSizeArr[0], subregionSizeArr[1])
581 for subBBox
in _subBBoxIter(skyInfo.bbox, subregionSize):
584 weightList, bgInfoList, altMaskList, statsFlags, statsCtrl)
585 except Exception
as e:
586 self.log.fatal(
"Cannot compute coadd %s: %s", subBBox, e)
588 coaddUtils.setCoaddEdgeBits(coaddMaskedImage.getMask(), coaddMaskedImage.getVariance())
594 \brief Set the metadata for the coadd
596 This basic implementation simply sets the filter from the
599 \param[in] coaddExposure: The target image for the coadd
600 \param[in] tempExpRefList: List of data references to tempExp
601 \param[in] weightList: List of weights
603 assert len(tempExpRefList) == len(weightList),
"Length mismatch"
604 tempExpName = self.getTempExpDatasetName()
608 tempExpList = [tempExpRef.get(tempExpName +
"_sub",
609 bbox=afwGeom.Box2I(afwGeom.Point2I(0, 0), afwGeom.Extent2I(1, 1)),
610 imageOrigin=
"LOCAL", immediate=
True)
for tempExpRef
in tempExpRefList]
611 numCcds = sum(len(tempExp.getInfo().getCoaddInputs().ccds)
for tempExp
in tempExpList)
613 coaddExposure.setFilter(tempExpList[0].getFilter())
614 coaddInputs = coaddExposure.getInfo().getCoaddInputs()
615 coaddInputs.ccds.reserve(numCcds)
616 coaddInputs.visits.reserve(len(tempExpList))
618 for tempExp, weight
in zip(tempExpList, weightList):
619 self.inputRecorder.addVisitToCoadd(coaddInputs, tempExp, weight)
620 coaddInputs.visits.sort()
621 if self.config.doPsfMatch:
626 modelPsfList = [tempExp.getPsf()
for tempExp
in tempExpList]
627 modelPsfWidthList = [modelPsf.computeBBox().getWidth()
for modelPsf
in modelPsfList]
628 psf = modelPsfList[modelPsfWidthList.index(max(modelPsfWidthList))]
630 psf = measAlg.CoaddPsf(coaddInputs.ccds, coaddExposure.getWcs())
631 coaddExposure.setPsf(psf)
632 apCorrMap = measAlg.makeCoaddApCorrMap(coaddInputs.ccds, coaddExposure.getBBox(afwImage.PARENT),
633 coaddExposure.getWcs())
634 coaddExposure.getInfo().setApCorrMap(apCorrMap)
636 def assembleSubregion(self, coaddExposure, bbox, tempExpRefList, imageScalerList, weightList,
637 bgInfoList, altMaskList, statsFlags, statsCtrl):
639 \brief Assemble the coadd for a sub-region.
641 For each coaddTempExp, check for (and swap in) an alternative mask if one is passed. If background
642 matching is enabled, add the background and background variance from each coaddTempExp. Remove mask
643 planes listed in config.removeMaskPlanes, Finally, mean-stack
644 the actual exposures using \ref afwMath.statisticsStack "statisticsStack" with outlier rejection if
645 config.doSigmaClip=True. Assign the stacked subregion back to the coadd.
647 \param[in] coaddExposure: The target image for the coadd
648 \param[in] bbox: Sub-region to coadd
649 \param[in] tempExpRefList: List of data reference to tempExp
650 \param[in] imageScalerList: List of image scalers
651 \param[in] weightList: List of weights
652 \param[in] bgInfoList: List of background data from background matching
653 \param[in] altMaskList: List of alternate masks to use rather than those stored with tempExp, or None
654 \param[in] statsFlags: Statistic for coadd
655 \param[in] statsCtrl: Statistics control object for coadd
657 self.log.debug(
"Computing coadd over %s", bbox)
658 tempExpName = self.getTempExpDatasetName()
659 coaddMaskedImage = coaddExposure.getMaskedImage()
661 for tempExpRef, imageScaler, bgInfo, altMask
in zip(tempExpRefList, imageScalerList, bgInfoList,
663 exposure = tempExpRef.get(tempExpName +
"_sub", bbox=bbox)
664 maskedImage = exposure.getMaskedImage()
667 altMaskSub = altMask.Factory(altMask, bbox, afwImage.PARENT)
668 maskedImage.getMask().swap(altMaskSub)
669 imageScaler.scaleMaskedImage(maskedImage)
671 if self.config.doMatchBackgrounds
and not bgInfo.isReference:
672 backgroundModel = bgInfo.backgroundModel
673 backgroundImage = backgroundModel.getImage()
if \
674 self.matchBackgrounds.config.usePolynomial
else \
675 backgroundModel.getImageF()
676 backgroundImage.setXY0(coaddMaskedImage.getXY0())
677 maskedImage += backgroundImage.Factory(backgroundImage, bbox, afwImage.PARENT,
False)
678 var = maskedImage.getVariance()
679 var += (bgInfo.fitRMS)**2
681 if self.config.removeMaskPlanes:
682 mask = maskedImage.getMask()
683 for maskPlane
in self.config.removeMaskPlanes:
685 mask &= ~mask.getPlaneBitMask(maskPlane)
686 except Exception
as e:
687 self.log.warn(
"Unable to remove mask plane %s: %s", maskPlane, e.message)
689 maskedImageList.append(maskedImage)
691 with self.timer(
"stack"):
692 coaddSubregion = afwMath.statisticsStack(
693 maskedImageList, statsFlags, statsCtrl, weightList)
695 coaddMaskedImage.assign(coaddSubregion, bbox)
699 \brief Add metadata from the background matching to the coadd
701 \param[in] coaddExposure: Coadd
702 \param[in] tempExpRefList: List of data references for temp exps to go into coadd
703 \param[in] backgroundInfoList: List of background info, results from background matching
705 self.log.info(
"Adding exposure information to metadata")
706 metadata = coaddExposure.getMetadata()
707 metadata.addString(
"CTExp_SDQA1_DESCRIPTION",
708 "Background matching: Ratio of matchedMSE / diffImVar")
709 for ind, (tempExpRef, backgroundInfo)
in enumerate(zip(tempExpRefList, backgroundInfoList)):
710 tempExpStr =
'&'.join(
'%s=%s' % (k, v)
for k, v
in tempExpRef.dataId.items())
711 if backgroundInfo.isReference:
712 metadata.addString(
"ReferenceExp_ID", tempExpStr)
714 metadata.addString(
"CTExp_ID_%d" % (ind), tempExpStr)
715 metadata.addDouble(
"CTExp_SDQA1_%d" % (ind),
716 backgroundInfo.matchedMSE/backgroundInfo.diffImVar)
717 metadata.addDouble(
"CTExp_SDQA2_%d" % (ind),
718 backgroundInfo.fitRMS)
721 """Returns None on failure"""
723 return dataRef.get(
"brightObjectMask", immediate=
True)
724 except Exception
as e:
725 self.log.warn(
"Unable to read brightObjectMask for %s: %s", dataRef.dataId, e)
729 """Set the bright object masks
731 exposure: Exposure under consideration
732 dataId: Data identifier dict for patch
733 brightObjectMasks: afwTable of bright objects to mask
738 if brightObjectMasks
is None:
739 self.log.warn(
"Unable to apply bright object mask: none supplied")
741 self.log.info(
"Applying %d bright object masks to %s", len(brightObjectMasks), dataId)
742 md = brightObjectMasks.table.getMetadata()
745 self.log.warn(
"Expected to see %s in metadata", k)
747 if md.get(k) != dataId[k]:
748 self.log.warn(
"Expected to see %s == %s in metadata, saw %s", k, md.get(k), dataId[k])
750 mask = exposure.getMaskedImage().getMask()
751 wcs = exposure.getWcs()
752 plateScale = wcs.pixelScale().asArcseconds()
754 for rec
in brightObjectMasks:
755 center = afwGeom.PointI(wcs.skyToPixel(rec.getCoord()))
756 if rec[
"type"] ==
"box":
757 assert rec[
"angle"] == 0.0, (
"Angle != 0 for mask object %s" % rec[
"id"])
758 width = rec[
"width"].asArcseconds()/plateScale
759 height = rec[
"height"].asArcseconds()/plateScale
761 halfSize = afwGeom.ExtentI(0.5*width, 0.5*height)
762 bbox = afwGeom.Box2I(center - halfSize, center + halfSize)
764 bbox = afwGeom.BoxI(afwGeom.PointI(int(center[0] - 0.5*width), int(center[1] - 0.5*height)),
765 afwGeom.PointI(int(center[0] + 0.5*width), int(center[1] + 0.5*height)))
766 foot = afwDetect.Footprint(bbox, exposure.getBBox())
767 elif rec[
"type"] ==
"circle":
768 radius = rec[
"radius"].asArcseconds()/plateScale
769 foot = afwDetect.Footprint(center, radius, exposure.getBBox())
771 self.log.warn(
"Unexpected region type %s at %s" % rec[
"type"], center)
777 def _makeArgumentParser(cls):
779 \brief Create an argument parser
781 parser = pipeBase.ArgumentParser(name=cls._DefaultName)
782 parser.add_id_argument(
"--id", cls.ConfigClass().coaddName +
"Coadd_tempExp",
783 help=
"data ID, e.g. --id tract=12345 patch=1,2",
784 ContainerClass=AssembleCoaddDataIdContainer)
785 parser.add_id_argument(
"--selectId",
"calexp", help=
"data ID, e.g. --selectId visit=6789 ccd=0..9",
786 ContainerClass=SelectDataIdContainer)
790 def _subBBoxIter(bbox, subregionSize):
792 \brief Iterate over subregions of a bbox
794 \param[in] bbox: bounding box over which to iterate: afwGeom.Box2I
795 \param[in] subregionSize: size of sub-bboxes
797 \return subBBox: next sub-bounding box of size subregionSize or smaller;
798 each subBBox is contained within bbox, so it may be smaller than subregionSize at the edges of bbox,
799 but it will never be empty
802 raise RuntimeError(
"bbox %s is empty" % (bbox,))
803 if subregionSize[0] < 1
or subregionSize[1] < 1:
804 raise RuntimeError(
"subregionSize %s must be nonzero" % (subregionSize,))
806 for rowShift
in range(0, bbox.getHeight(), subregionSize[1]):
807 for colShift
in range(0, bbox.getWidth(), subregionSize[0]):
808 subBBox = afwGeom.Box2I(bbox.getMin() + afwGeom.Extent2I(colShift, rowShift), subregionSize)
810 if subBBox.isEmpty():
811 raise RuntimeError(
"Bug: empty bbox! bbox=%s, subregionSize=%s, colShift=%s, rowShift=%s" %
812 (bbox, subregionSize, colShift, rowShift))
818 \brief A version of lsst.pipe.base.DataIdContainer specialized for assembleCoadd.
823 \brief Make self.refList from self.idList.
825 Interpret the config.doMatchBackgrounds, config.autoReference,
826 and whether a visit/run supplied.
827 If a visit/run is supplied, config.autoReference is automatically set to False.
828 if config.doMatchBackgrounds == false, then a visit/run will be ignored if accidentally supplied.
831 keysCoadd = namespace.butler.getKeys(datasetType=namespace.config.coaddName +
"Coadd",
833 keysCoaddTempExp = namespace.butler.getKeys(datasetType=namespace.config.coaddName +
"Coadd_tempExp",
836 if namespace.config.doMatchBackgrounds:
837 if namespace.config.autoReference:
838 datasetType = namespace.config.coaddName +
"Coadd"
839 validKeys = keysCoadd
841 datasetType = namespace.config.coaddName +
"Coadd_tempExp"
842 validKeys = keysCoaddTempExp
844 datasetType = namespace.config.coaddName +
"Coadd"
845 validKeys = keysCoadd
847 for dataId
in self.idList:
849 for key
in validKeys:
850 if key
not in dataId:
851 raise RuntimeError(
"--id must include " + key)
854 if (key
not in keysCoadd)
and (key
in keysCoaddTempExp):
855 if namespace.config.autoReference:
857 namespace.config.autoReference =
False
858 datasetType = namespace.config.coaddName +
"Coadd_tempExp"
859 print(
"Switching config.autoReference to False; applies only to background Matching.")
862 dataRef = namespace.butler.dataRef(
863 datasetType=datasetType,
866 self.refList.append(dataRef)
871 \brief Function to count the number of pixels with a specific mask in a footprint.
873 Find the intersection of mask & footprint. Count all pixels in the mask that are in the intersection that
874 have bitmask set but do not have ignoreMask set. Return the count.
876 \param[in] mask: mask to define intersection region by.
877 \parma[in] footprint: footprint to define the intersection region by.
878 \param[in] bitmask: specific mask that we wish to count the number of occurances of.
879 \param[in] ignoreMask: pixels to not consider.
880 \return count of number of pixels in footprint with specified mask.
882 bbox = footprint.getBBox()
883 bbox.clip(mask.getBBox(afwImage.PARENT))
884 fp = afwImage.MaskU(bbox)
885 subMask = mask.Factory(mask, bbox, afwImage.PARENT)
886 footprint.spans.setMask(fp, bitmask)
887 return numpy.logical_and((subMask.getArray() & fp.getArray()) > 0,
888 (subMask.getArray() & ignoreMask) == 0).sum()
893 \anchor SafeClipAssembleCoaddConfig
895 \brief Configuration parameters for the SafeClipAssembleCoaddTask
897 clipDetection = pexConfig.ConfigurableField(
898 target=SourceDetectionTask,
899 doc=
"Detect sources on difference between unclipped and clipped coadd")
900 minClipFootOverlap = pexConfig.Field(
901 doc=
"Minimum fractional overlap of clipped footprint with visit DETECTED to be clipped",
905 minClipFootOverlapSingle = pexConfig.Field(
906 doc=
"Minimum fractional overlap of clipped footprint with visit DETECTED to be "
907 "clipped when only one visit overlaps",
911 minClipFootOverlapDouble = pexConfig.Field(
912 doc=
"Minimum fractional overlap of clipped footprints with visit DETECTED to be "
913 "clipped when two visits overlap",
917 maxClipFootOverlapDouble = pexConfig.Field(
918 doc=
"Maximum fractional overlap of clipped footprints with visit DETECTED when "
919 "considering two visits",
923 minBigOverlap = pexConfig.Field(
924 doc=
"Minimum number of pixels in footprint to use DETECTED mask from the single visits "
925 "when labeling clipped footprints",
933 AssembleCoaddConfig.setDefaults(self)
934 self.clipDetection.doTempLocalBackground =
False
935 self.clipDetection.reEstimateBackground =
False
936 self.clipDetection.returnOriginalFootprints =
False
937 self.clipDetection.thresholdPolarity =
"both"
938 self.clipDetection.thresholdValue = 2
939 self.clipDetection.nSigmaToGrow = 2
940 self.clipDetection.minPixels = 4
941 self.clipDetection.isotropicGrow =
True
942 self.clipDetection.thresholdType =
"pixel_stdev"
956 \anchor SafeClipAssembleCoaddTask_
958 \brief Assemble a coadded image from a set of coadded temporary exposures, being careful to clip & flag areas
959 with potential artifacts.
961 \section pipe_tasks_assembleCoadd_Contents Contents
962 - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Purpose
963 - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Initialize
964 - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Run
965 - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Config
966 - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Debug
967 - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Example
969 \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Purpose Description
971 \copybrief SafeClipAssembleCoaddTask
973 Read the documentation for \ref AssembleCoaddTask_ "AssembleCoaddTask" first since
974 SafeClipAssembleCoaddTask subtasks that task.
975 In \ref AssembleCoaddTask_ "AssembleCoaddTask", we compute the coadd as an clipped mean (i.e. we clip
977 The problem with doing this is that when computing the coadd PSF at a given location, individual visit
978 PSFs from visits with outlier pixels contribute to the coadd PSF and cannot be treated correctly.
979 In this task, we correct for this behavior by creating a new badMaskPlane 'CLIPPED'.
980 We populate this plane on the input coaddTempExps and the final coadd where i. difference imaging suggests
981 that there is an outlier and ii. this outlier appears on only one or two images.
982 Such regions will not contribute to the final coadd.
983 Furthermore, any routine to determine the coadd PSF can now be cognizant of clipped regions.
984 Note that the algorithm implemented by this task is preliminary and works correctly for HSC data.
985 Parameter modifications and or considerable redesigning of the algorithm is likley required for other
988 SafeClipAssembleCoaddTask uses a \ref SourceDetectionTask_ "clipDetection" subtask and also sub-classes
989 \ref AssembleCoaddTask_ "AssembleCoaddTask". You can retarget the
990 \ref SourceDetectionTask_ "clipDetection" subtask if you wish.
992 \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Initialize Task initialization
993 \copydoc \_\_init\_\_
995 \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Run Invoking the Task
998 \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Config Configuration parameters
999 See \ref SafeClipAssembleCoaddConfig
1001 \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Debug Debug variables
1002 The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a
1003 flag \c -d to import \b debug.py from your \c PYTHONPATH; see \ref baseDebug for more about \b debug.py
1005 SafeClipAssembleCoaddTask has no debug variables of its own. The \ref SourceDetectionTask_ "clipDetection"
1006 subtasks may support debug variables. See the documetation for \ref SourceDetectionTask_ "clipDetection"
1007 for further information.
1009 \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Example A complete example of using SafeClipAssembleCoaddTask
1011 SafeClipAssembleCoaddTask assembles a set of warped coaddTempExp images into a coadded image.
1012 The SafeClipAssembleCoaddTask is invoked by running assembleCoadd.py <em>without</em> the flag
1014 Usage of assembleCoadd.py expects a data reference to the tract patch and filter to be coadded
1015 (specified using '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]') along
1016 with a list of coaddTempExps to attempt to coadd (specified using
1017 '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]').
1018 Only the coaddTempExps that cover the specified tract and patch will be coadded.
1019 A list of the available optional arguments can be obtained by calling assembleCoadd.py with the --help
1020 command line argument:
1022 assembleCoadd.py --help
1024 To demonstrate usage of the SafeClipAssembleCoaddTask in the larger context of multi-band processing, we
1025 will generate the HSC-I & -R band coadds from HSC engineering test data provided in the ci_hsc package. To
1026 begin, assuming that the lsst stack has been already set up, we must set up the obs_subaru and ci_hsc
1028 This defines the environment variable $CI_HSC_DIR and points at the location of the package. The raw HSC
1029 data live in the $CI_HSC_DIR/raw directory. To begin assembling the coadds, we must first
1032 <DD> process the individual ccds in $CI_HSC_RAW to produce calibrated exposures</DD>
1034 <DD> create a skymap that covers the area of the sky present in the raw exposures</DD>
1035 <DT>makeCoaddTempExp</DT>
1036 <DD> warp the individual calibrated exposures to the tangent plane of the coadd</DD>
1038 We can perform all of these steps by running
1040 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
1042 This will produce warped coaddTempExps for each visit. To coadd the wraped data, we call assembleCoadd.py
1045 assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 --selectId visit=903988 ccd=24
1047 This will process the HSC-I band data. The results are written in $CI_HSC_DIR/DATA/deepCoadd-results/HSC-I
1048 You may also choose to run:
1050 scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346
1051 assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R --selectId visit=903334 ccd=16 --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 --selectId visit=903334 ccd=100 --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 --selectId visit=903338 ccd=18 --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 --selectId visit=903342 ccd=10 --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 --selectId visit=903344 ccd=5 --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 --selectId visit=903346 ccd=6 --selectId visit=903346 ccd=12
1053 to generate the coadd for the HSC-R band if you are interested in following multiBand Coadd processing as
1054 discussed in \ref pipeTasks_multiBand.
1056 ConfigClass = SafeClipAssembleCoaddConfig
1057 _DefaultName =
"safeClipAssembleCoadd"
1061 \brief Initialize the task and make the \ref SourceDetectionTask_ "clipDetection" subtask.
1063 AssembleCoaddTask.__init__(self, *args, **kwargs)
1064 schema = afwTable.SourceTable.makeMinimalSchema()
1065 self.makeSubtask(
"clipDetection", schema=schema)
1067 def assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList, bgModelList, *args, **kwargs):
1069 \brief Assemble the coadd for a region
1071 Compute the difference of coadds created with and without outlier rejection to identify coadd pixels
1072 that have outlier values in some individual visits. Detect clipped regions on the difference image and
1073 mark these regions on the one or two individual coaddTempExps where they occur if there is significant
1074 overlap between the clipped region and a source.
1075 This leaves us with a set of footprints from the difference image that have been identified as having
1076 occured on just one or two individual visits. However, these footprints were generated from a
1077 difference image. It is conceivable for a large diffuse source to have become broken up into multiple
1078 footprints acrosss the coadd difference in this process.
1079 Determine the clipped region from all overlapping footprints from the detected sources in each visit -
1080 these are big footprints.
1081 Combine the small and big clipped footprints and mark them on a new bad mask plane
1082 Generate the coadd using \ref AssembleCoaddTask.assemble_ "AssembleCoaddTask.assemble" without outlier
1083 removal. Clipped footprints will no longer make it into the coadd because they are marked in the new
1086 N.b. *args and **kwargs are passed but ignored in order to match the call signature expected by the
1089 @param skyInfo: Patch geometry information, from getSkyInfo
1090 @param tempExpRefList: List of data reference to tempExp
1091 @param imageScalerList: List of image scalers
1092 @param weightList: List of weights
1093 @param bgModelList: List of background models from background matching
1094 return coadd exposure
1096 exp = self.
buildDifferenceImage(skyInfo, tempExpRefList, imageScalerList, weightList, bgModelList)
1097 mask = exp.getMaskedImage().getMask()
1098 mask.addMaskPlane(
"CLIPPED")
1100 result = self.
detectClip(exp, tempExpRefList)
1102 self.log.info(
'Found %d clipped objects', len(result.clipFootprints))
1105 maskClipValue = mask.getPlaneBitMask(
"CLIPPED")
1106 maskDetValue = mask.getPlaneBitMask(
"DETECTED") | mask.getPlaneBitMask(
"DETECTED_NEGATIVE")
1107 bigFootprints = self.
detectClipBig(result.tempExpClipList, result.clipFootprints, result.clipIndices,
1108 maskClipValue, maskDetValue)
1111 maskClip = mask.Factory(mask.getBBox(afwImage.PARENT))
1112 afwDet.setMaskFromFootprintList(maskClip, result.clipFootprints, maskClipValue)
1114 maskClipBig = maskClip.Factory(mask.getBBox(afwImage.PARENT))
1115 afwDet.setMaskFromFootprintList(maskClipBig, bigFootprints, maskClipValue)
1116 maskClip |= maskClipBig
1119 badMaskPlanes = self.config.badMaskPlanes[:]
1120 badMaskPlanes.append(
"CLIPPED")
1121 badPixelMask = afwImage.MaskU.getPlaneBitMask(badMaskPlanes)
1122 coaddExp = AssembleCoaddTask.assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList,
1123 bgModelList, result.tempExpClipList,
1129 maskExp = coaddExp.getMaskedImage().getMask()
1136 \brief Return an exposure that contains the difference between and unclipped and clipped coadds.
1138 Generate a difference image between clipped and unclipped coadds.
1139 Compute the difference image by subtracting an outlier-clipped coadd from an outlier-unclipped coadd.
1140 Return the difference image.
1142 @param skyInfo: Patch geometry information, from getSkyInfo
1143 @param tempExpRefList: List of data reference to tempExp
1144 @param imageScalerList: List of image scalers
1145 @param weightList: List of weights
1146 @param bgModelList: List of background models from background matching
1147 @return Difference image of unclipped and clipped coadd wrapped in an Exposure
1150 coaddMean = AssembleCoaddTask.assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList,
1151 bgModelList, doClip=
False)
1154 coaddClip = AssembleCoaddTask.assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList,
1155 bgModelList, doClip=
True)
1157 coaddDiff = coaddMean.getMaskedImage().Factory(coaddMean.getMaskedImage())
1158 coaddDiff -= coaddClip.getMaskedImage()
1159 exp = afwImage.ExposureF(coaddDiff)
1160 exp.setPsf(coaddMean.getPsf())
1165 \brief Detect clipped regions on an exposure and set the mask on the individual tempExp masks
1167 Detect footprints in the difference image after smoothing the difference image with a Gaussian kernal.
1168 Identify footprints that overlap with one or two input coaddTempExps by comparing the computed overlap
1169 fraction to thresholds set in the config.
1170 A different threshold is applied depending on the number of overlapping visits (restricted to one or
1172 If the overlap exceeds the thresholds, the footprint is considered "CLIPPED" and is marked as such on
1174 Return a struct with the clipped footprints, the indices of the coaddTempExps that end up overlapping
1175 with the clipped footprints and a list of new masks for the coaddTempExps.
1177 \param[in] exp: Exposure to run detection on
1178 \param[in] tempExpRefList: List of data reference to tempExp
1179 \return struct containing:
1180 - clippedFootprints: list of clipped footprints
1181 - clippedIndices: indices for each clippedFootprint in tempExpRefList
1182 - tempExpClipList: list of new masks for tempExp
1184 mask = exp.getMaskedImage().getMask()
1185 maskClipValue = mask.getPlaneBitMask(
"CLIPPED")
1186 maskDetValue = mask.getPlaneBitMask(
"DETECTED") | mask.getPlaneBitMask(
"DETECTED_NEGATIVE")
1187 fpSet = self.clipDetection.detectFootprints(exp, doSmooth=
True, clearMask=
True)
1189 fpSet.positive.merge(fpSet.negative)
1190 footprints = fpSet.positive
1191 self.log.info(
'Found %d potential clipped objects', len(footprints.getFootprints()))
1192 ignoreMask = self.getBadPixelMask()
1198 tempExpClipList = [tmpExpRef.get(self.getTempExpDatasetName(),
1199 immediate=
True).getMaskedImage().getMask()
for
1200 tmpExpRef
in tempExpRefList]
1202 for footprint
in footprints.getFootprints():
1203 nPixel = footprint.getArea()
1207 for i, tmpExpMask
in enumerate(tempExpClipList):
1211 totPixel = nPixel - ignore
1214 if ignore > overlapDet
or totPixel <= 0.5*nPixel
or overlapDet == 0:
1216 overlap.append(overlapDet/float(totPixel))
1217 maskList.append(tmpExpMask)
1220 overlap = numpy.array(overlap)
1221 if not len(overlap):
1228 if len(overlap) == 1:
1229 if overlap[0] > self.config.minClipFootOverlapSingle:
1234 clipIndex = numpy.where(overlap > self.config.minClipFootOverlap)[0]
1235 if len(clipIndex) == 1:
1237 keepIndex = [clipIndex[0]]
1240 clipIndex = numpy.where(overlap > self.config.minClipFootOverlapDouble)[0]
1241 if len(clipIndex) == 2
and len(overlap) > 3:
1242 clipIndexComp = numpy.where(overlap < self.config.minClipFootOverlapDouble)[0]
1243 if numpy.max(overlap[clipIndexComp]) < self.config.maxClipFootOverlapDouble:
1245 keepIndex = clipIndex
1250 for index
in keepIndex:
1251 footprint.spans.setMask(maskList[index], maskClipValue)
1253 clipIndices.append(numpy.array(indexList)[keepIndex])
1254 clipFootprints.append(footprint)
1256 return pipeBase.Struct(clipFootprints=clipFootprints, clipIndices=clipIndices,
1257 tempExpClipList=tempExpClipList)
1259 def detectClipBig(self, tempExpClipList, clipFootprints, clipIndices, maskClipValue, maskDetValue):
1261 \brief Find footprints from individual tempExp footprints for large footprints.
1263 Identify big footprints composed of many sources in the coadd difference that may have originated in a
1264 large diffuse source in the coadd. We do this by indentifying all clipped footprints that overlap
1265 significantly with each source in all the coaddTempExps.
1267 \param[in] tempExpClipList: List of tempExp masks with clipping information
1268 \param[in] clipFootprints: List of clipped footprints
1269 \param[in] clipIndices: List of which entries in tempExpClipList each footprint belongs to
1270 \param[in] maskClipValue: Mask value of clipped pixels
1271 \param[in] maskClipValue: Mask value of detected pixels
1272 \return list of big footprints
1274 bigFootprintsCoadd = []
1275 ignoreMask = self.getBadPixelMask()
1276 for index, tmpExpMask
in enumerate(tempExpClipList):
1279 maskVisitDet = tmpExpMask.Factory(tmpExpMask, tmpExpMask.getBBox(afwImage.PARENT),
1280 afwImage.PARENT,
True)
1281 maskVisitDet &= maskDetValue
1282 visitFootprints = afwDet.FootprintSet(maskVisitDet, afwDet.Threshold(1))
1285 clippedFootprintsVisit = []
1286 for foot, clipIndex
in zip(clipFootprints, clipIndices):
1287 if index
not in clipIndex:
1289 clippedFootprintsVisit.append(foot)
1290 maskVisitClip = maskVisitDet.Factory(maskVisitDet.getBBox(afwImage.PARENT))
1291 afwDet.setMaskFromFootprintList(maskVisitClip, clippedFootprintsVisit, maskClipValue)
1293 bigFootprintsVisit = []
1294 for foot
in visitFootprints.getFootprints():
1295 if foot.getArea() < self.config.minBigOverlap:
1298 if nCount > self.config.minBigOverlap:
1299 bigFootprintsVisit.append(foot)
1300 bigFootprintsCoadd.append(foot)
1303 maskVisitClip.clearAllMaskPlanes()
1304 afwDet.setMaskFromFootprintList(maskVisitClip, bigFootprintsVisit, maskClipValue)
1305 tmpExpMask |= maskVisitClip
1307 return bigFootprintsCoadd
def __init__
Initialize the task and make the clipDetection subtask.
def run
Assemble a coadd from a set of coaddTempExp.
def getBackgroundReferenceScaler
Construct an image scaler for the background reference frame.
def assemble
Assemble the coadd for a region.
def addBackgroundMatchingMetadata
Add metadata from the background matching to the coadd.
def detectClipBig
Find footprints from individual tempExp footprints for large footprints.
def __init__
Initialize the task.
def assembleSubregion
Assemble the coadd for a sub-region.
def assembleMetadata
Set the metadata for the coadd.
def backgroundMatching
Perform background matching on the prepared inputs.
Configuration parameters for the SafeClipAssembleCoaddTask.
Assemble a coadded image from a set of coadded temporary exposures.
Assemble a coadded image from a set of coadded temporary exposures, being careful to clip & flag area...
def buildDifferenceImage
Return an exposure that contains the difference between and unclipped and clipped coadds...
def detectClip
Detect clipped regions on an exposure and set the mask on the individual tempExp masks.
def prepareInputs
Prepare the input warps for coaddition by measuring the weight for each warp and the scaling for the ...
def makeDataRefList
Make self.refList from self.idList.
Configuration parameters for the AssembleCoaddTask.
def countMaskFromFootprint
Function to count the number of pixels with a specific mask in a footprint.
def assemble
Assemble a coadd from input warps.
def readBrightObjectMasks
A version of lsst.pipe.base.DataIdContainer specialized for assembleCoadd.
def getTempExpRefList
Generate list of coaddTempExp data references corresponding to exposures that lie within the patch to...