1 from __future__
import absolute_import, division, print_function
2 from builtins
import zip
3 from builtins
import range
26 import lsst.pex.config
as pexConfig
27 import lsst.pex.exceptions
as pexExceptions
28 import lsst.afw.detection
as afwDetect
29 import lsst.afw.geom
as afwGeom
30 import lsst.afw.image
as afwImage
31 import lsst.afw.math
as afwMath
32 import lsst.afw.table
as afwTable
33 import lsst.afw.detection
as afwDet
34 import lsst.coadd.utils
as coaddUtils
35 import lsst.pipe.base
as pipeBase
36 import lsst.meas.algorithms
as measAlg
37 from .coaddBase
import CoaddBaseTask, SelectDataIdContainer
38 from .interpImage
import InterpImageTask
39 from .matchBackgrounds
import MatchBackgroundsTask
40 from .scaleZeroPoint
import ScaleZeroPointTask
41 from .coaddHelpers
import groupPatchExposures, getGroupDataRef
42 from lsst.meas.algorithms
import SourceDetectionTask
44 __all__ = [
"AssembleCoaddTask",
"SafeClipAssembleCoaddTask"]
49 \anchor AssembleCoaddConfig_
51 \brief Configuration parameters for the \ref AssembleCoaddTask_ "AssembleCoaddTask"
53 subregionSize = pexConfig.ListField(
55 doc=
"Width, height of stack subregion size; "
56 "make small enough that a full stack of images will fit into memory at once.",
60 doSigmaClip = pexConfig.Field(
62 doc=
"Perform sigma clipped outlier rejection? If False then compute a simple mean.",
65 sigmaClip = pexConfig.Field(
67 doc=
"Sigma for outlier rejection; ignored if doSigmaClip false.",
70 clipIter = pexConfig.Field(
72 doc=
"Number of iterations of outlier rejection; ignored if doSigmaClip false.",
75 scaleZeroPoint = pexConfig.ConfigurableField(
76 target=ScaleZeroPointTask,
77 doc=
"Task to adjust the photometric zero point of the coadd temp exposures",
79 doInterp = pexConfig.Field(
80 doc=
"Interpolate over NaN pixels? Also extrapolate, if necessary, but the results are ugly.",
84 interpImage = pexConfig.ConfigurableField(
85 target=InterpImageTask,
86 doc=
"Task to interpolate (and extrapolate) over NaN pixels",
88 matchBackgrounds = pexConfig.ConfigurableField(
89 target=MatchBackgroundsTask,
90 doc=
"Task to match backgrounds",
92 maxMatchResidualRatio = pexConfig.Field(
93 doc=
"Maximum ratio of the mean squared error of the background matching model to the variance "
94 "of the difference in backgrounds",
98 maxMatchResidualRMS = pexConfig.Field(
99 doc=
"Maximum RMS of residuals of the background offset fit in matchBackgrounds.",
103 doWrite = pexConfig.Field(
104 doc=
"Persist coadd?",
108 doMatchBackgrounds = pexConfig.Field(
109 doc=
"Match backgrounds of coadd temp exposures before coadding them? "
110 "If False, the coadd temp expsosures must already have been background subtracted or matched",
114 autoReference = pexConfig.Field(
115 doc=
"Automatically select the coadd temp exposure to use as a reference for background matching? "
116 "Ignored if doMatchBackgrounds false. "
117 "If False you must specify the reference temp exposure as the data Id",
121 maskPropagationThresholds = pexConfig.DictField(
124 doc=(
"Threshold (in fractional weight) of rejection at which we propagate a mask plane to "
125 "the coadd; that is, we set the mask bit on the coadd if the fraction the rejected frames "
126 "would have contributed exceeds this value."),
127 default={
"SAT": 0.1},
129 removeMaskPlanes = pexConfig.ListField(dtype=str, default=[
"CROSSTALK",
"NOT_DEBLENDED"],
130 doc=
"Mask planes to remove before coadding")
139 doMaskBrightObjects = pexConfig.Field(dtype=bool, default=
False,
140 doc=
"Set mask and flag bits for bright objects?")
141 brightObjectMaskName = pexConfig.Field(dtype=str, default=
"BRIGHT_OBJECT",
142 doc=
"Name of mask bit used for bright objects")
145 CoaddBaseTask.ConfigClass.setDefaults(self)
149 CoaddBaseTask.ConfigClass.validate(self)
150 if self.makeDirect
and self.makePsfMatched:
151 raise ValueError(
"Currently, assembleCoadd can only make either Direct or PsfMatched Coadds "
152 "at a time. Set either makeDirect or makePsfMatched to False")
164 \anchor AssembleCoaddTask_
166 \brief Assemble a coadded image from a set of warps (coadded temporary exposures).
168 \section pipe_tasks_assembleCoadd_Contents Contents
169 - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Purpose
170 - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Initialize
171 - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Run
172 - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Config
173 - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Debug
174 - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Example
176 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Purpose Description
178 \copybrief AssembleCoaddTask_
180 We want to assemble a coadded image from a set of Warps (also called
181 coadded temporary exposures or coaddTempExps.
182 Each input Warp covers a patch on the sky and corresponds to a single run/visit/exposure of the
183 covered patch. We provide the task with a list of Warps (selectDataList) from which it selects
184 Warps that cover the specified patch (pointed at by dataRef).
185 Each Warp that goes into a coadd will typically have an independent photometric zero-point.
186 Therefore, we must scale each Warp to set it to a common photometric zeropoint. By default, each
187 Warp has backgrounds and hence will require config.doMatchBackgrounds=True.
188 When background matching is enabled, the task may be configured to automatically select a reference exposure
189 (config.autoReference=True). If this is not done, we require that the input dataRef provides access to a
190 Warp (dataset type coaddName + 'Coadd' + warpType + 'Warp') which is used as the reference exposure.
191 WarpType may be one of 'direct' or 'psfMatched', and the boolean configs config.makeDirect and
192 config.makePsfMatched set which of the warp types will be coadded.
193 The coadd is computed as a mean with optional outlier rejection.
194 Criteria for outlier rejection are set in \ref AssembleCoaddConfig. Finally, Warps can have bad 'NaN'
195 pixels which received no input from the source calExps. We interpolate over these bad (NaN) pixels.
197 AssembleCoaddTask uses several sub-tasks. These are
199 <DT>\ref ScaleZeroPointTask_ "ScaleZeroPointTask"</DT>
200 <DD> create and use an imageScaler object to scale the photometric zeropoint for each Warp</DD>
201 <DT>\ref MatchBackgroundsTask_ "MatchBackgroundsTask"</DT>
202 <DD> match background in a Warp to a reference exposure (and select the reference exposure if one is
204 <DT>\ref InterpImageTask_ "InterpImageTask"</DT>
205 <DD>interpolate across bad pixels (NaN) in the final coadd</DD>
207 You can retarget these subtasks if you wish.
209 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Initialize Task initialization
210 \copydoc \_\_init\_\_
212 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Run Invoking the Task
215 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Config Configuration parameters
216 See \ref AssembleCoaddConfig_
218 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Debug Debug variables
219 The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a
220 flag \c -d to import \b debug.py from your \c PYTHONPATH; see \ref baseDebug for more about \b debug.py files.
221 AssembleCoaddTask has no debug variables of its own. Some of the subtasks may support debug variables. See
222 the documetation for the subtasks for further information.
224 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Example A complete example of using AssembleCoaddTask
226 AssembleCoaddTask assembles a set of warped images into a coadded image. The AssembleCoaddTask
227 can be invoked by running assembleCoadd.py with the flag '--legacyCoadd'. Usage of assembleCoadd.py expects
228 a data reference to the tract patch and filter to be coadded (specified using
229 '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]') along with a list of
230 Warps to attempt to coadd (specified using
231 '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]'). Only the Warps
232 that cover the specified tract and patch will be coadded. A list of the available optional
233 arguments can be obtained by calling assembleCoadd.py with the --help command line argument:
235 assembleCoadd.py --help
237 To demonstrate usage of the AssembleCoaddTask in the larger context of multi-band processing, we will generate
238 the HSC-I & -R band coadds from HSC engineering test data provided in the ci_hsc package. To begin, assuming
239 that the lsst stack has been already set up, we must set up the obs_subaru and ci_hsc packages.
240 This defines the environment variable $CI_HSC_DIR and points at the location of the package. The raw HSC
241 data live in the $CI_HSC_DIR/raw directory. To begin assembling the coadds, we must first
244 <DD> process the individual ccds in $CI_HSC_RAW to produce calibrated exposures</DD>
246 <DD> create a skymap that covers the area of the sky present in the raw exposures</DD>
247 <DT>makeCoaddTempExp</DT>
248 <DD> warp the individual calibrated exposures to the tangent plane of the coadd</DD>
250 We can perform all of these steps by running
252 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
254 This will produce warped exposures for each visit. To coadd the warped data, we call assembleCoadd.py as
257 assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 --selectId visit=903988 ccd=24\endcode
258 that will process the HSC-I band data. The results are written in $CI_HSC_DIR/DATA/deepCoadd-results/HSC-I
259 You may also choose to run:
261 scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346
262 assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R --selectId visit=903334 ccd=16 --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 --selectId visit=903334 ccd=100 --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 --selectId visit=903338 ccd=18 --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 --selectId visit=903342 ccd=10 --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 --selectId visit=903344 ccd=5 --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 --selectId visit=903346 ccd=6 --selectId visit=903346 ccd=12
264 to generate the coadd for the HSC-R band if you are interested in following multiBand Coadd processing as
265 discussed in \ref pipeTasks_multiBand (but note that normally, one would use the
266 \ref SafeClipAssembleCoaddTask_ "SafeClipAssembleCoaddTask" rather than AssembleCoaddTask to make the coadd.
268 ConfigClass = AssembleCoaddConfig
269 _DefaultName =
"assembleCoadd"
273 \brief Initialize the task. Create the \ref InterpImageTask "interpImage",
274 \ref MatchBackgroundsTask "matchBackgrounds", & \ref ScaleZeroPointTask "scaleZeroPoint" subtasks.
276 CoaddBaseTask.__init__(self, *args, **kwargs)
277 self.makeSubtask(
"interpImage")
278 self.makeSubtask(
"matchBackgrounds")
279 self.makeSubtask(
"scaleZeroPoint")
281 if self.config.doMaskBrightObjects:
282 mask = afwImage.MaskU()
285 except pexExceptions.LsstCppException:
286 raise RuntimeError(
"Unable to define mask plane for bright objects; planes used are %s" %
287 mask.getMaskPlaneDict().keys())
290 if self.config.makeDirect:
292 elif self.config.makePsfMatched:
295 raise ValueError(
"Neither makeDirect nor makePsfMatched configs are True")
298 def run(self, dataRef, selectDataList=[]):
300 \brief Assemble a coadd from a set of Warps
302 Coadd a set of Warps. Compute weights to be applied to each Warp and find scalings to
303 match the photometric zeropoint to a reference Warp. Optionally, match backgrounds across
304 Warps if the background has not already been removed. Assemble the Warps using
305 \ref assemble. Interpolate over NaNs and optionally write the coadd to disk. Return the coadded
309 \param[in] dataRef: Data reference defining the patch for coaddition and the reference Warp
310 (if config.autoReference=False). Used to access the following data products:
311 - [in] self.config.coaddName + "Coadd_skyMap"
312 - [in] self.config.coaddName + "Coadd_ + <warpType> + "Warp" (optionally)
313 - [out] self.config.coaddName + "Coadd"
314 \param[in] selectDataList[in]: List of data references to Warps. Data to be coadded will be
315 selected from this list based on overlap with the patch defined by dataRef.
317 \return a pipeBase.Struct with fields:
318 - coaddExposure: coadded exposure
320 skyInfo = self.getSkyInfo(dataRef)
321 calExpRefList = self.selectExposures(dataRef, skyInfo, selectDataList=selectDataList)
322 if len(calExpRefList) == 0:
323 self.log.warn(
"No exposures to coadd")
325 self.log.info(
"Coadding %d exposures", len(calExpRefList))
329 self.log.info(
"Found %d %s", len(inputData.tempExpRefList),
330 self.getTempExpDatasetName(self.
warpType))
331 if len(inputData.tempExpRefList) == 0:
332 self.log.warn(
"No coadd temporary exposures found")
334 if self.config.doMatchBackgrounds:
337 if len(inputData.tempExpRefList) == 0:
338 self.log.warn(
"No valid background models")
341 coaddExp = self.
assemble(skyInfo, inputData.tempExpRefList, inputData.imageScalerList,
342 inputData.weightList,
343 inputData.backgroundInfoList
if self.config.doMatchBackgrounds
else None,
344 doClip=self.config.doSigmaClip)
345 if self.config.doMatchBackgrounds:
347 inputData.backgroundInfoList)
349 if self.config.doInterp:
350 self.interpImage.run(coaddExp.getMaskedImage(), planeName=
"NO_DATA")
352 varArray = coaddExp.getMaskedImage().getVariance().getArray()
353 varArray[:] = numpy.where(varArray > 0, varArray, numpy.inf)
355 if self.config.doMaskBrightObjects:
359 if self.config.doWrite:
360 self.log.info(
"Persisting %s" % self.getCoaddDatasetName(self.
warpType))
361 dataRef.put(coaddExp, self.getCoaddDatasetName(self.
warpType))
363 return pipeBase.Struct(coaddExposure=coaddExp)
367 \brief Generate list data references corresponding to warped exposures that lie within the
370 \param[in] patchRef: Data reference for patch
371 \param[in] calExpRefList: List of data references for input calexps
372 \return List of Warp/CoaddTempExp data references
374 butler = patchRef.getButler()
376 self.getTempExpDatasetName(self.
warpType))
378 g, groupData.keys)
for
379 g
in groupData.groups.keys()]
380 return tempExpRefList
384 \brief Construct an image scaler for the background reference frame
386 Each Warp has a different background level. A reference background level must be chosen before
387 coaddition. If config.autoReference=True, \ref backgroundMatching will pick the reference level and
388 this routine is a no-op and None is returned. Otherwise, use the
389 \ref ScaleZeroPointTask_ "scaleZeroPoint" subtask to compute an imageScaler object for the provided
390 reference image and return it.
392 \param[in] dataRef: Data reference for the background reference frame, or None
393 \return image scaler, or None
395 if self.config.autoReference:
399 dataset = self.getTempExpDatasetName(self.
warpType)
400 if not dataRef.datasetExists(dataset):
401 raise RuntimeError(
"Could not find reference exposure %s %s." % (dataset, dataRef.dataId))
403 refExposure = dataRef.get(self.getTempExpDatasetName(self.
warpType), immediate=
True)
404 refImageScaler = self.scaleZeroPoint.computeImageScaler(
405 exposure=refExposure,
408 return refImageScaler
412 \brief Prepare the input warps for coaddition by measuring the weight for each warp and the scaling
413 for the photometric zero point.
415 Each Warp has its own photometric zeropoint and background variance. Before coadding these
416 Warps together, compute a scale factor to normalize the photometric zeropoint and compute the
417 weight for each Warp.
419 \param[in] refList: List of data references to tempExp
421 - tempExprefList: List of data references to tempExp
422 - weightList: List of weightings
423 - imageScalerList: List of image scalers
425 statsCtrl = afwMath.StatisticsControl()
426 statsCtrl.setNumSigmaClip(self.config.sigmaClip)
427 statsCtrl.setNumIter(self.config.clipIter)
428 statsCtrl.setAndMask(self.getBadPixelMask())
429 statsCtrl.setNanSafe(
True)
437 tempExpName = self.getTempExpDatasetName(self.
warpType)
438 for tempExpRef
in refList:
439 if not tempExpRef.datasetExists(tempExpName):
440 self.log.warn(
"Could not find %s %s; skipping it", tempExpName, tempExpRef.dataId)
443 tempExp = tempExpRef.get(tempExpName, immediate=
True)
444 maskedImage = tempExp.getMaskedImage()
445 imageScaler = self.scaleZeroPoint.computeImageScaler(
450 imageScaler.scaleMaskedImage(maskedImage)
451 except Exception
as e:
452 self.log.warn(
"Scaling failed for %s (skipping it): %s", tempExpRef.dataId, e)
454 statObj = afwMath.makeStatistics(maskedImage.getVariance(), maskedImage.getMask(),
455 afwMath.MEANCLIP, statsCtrl)
456 meanVar, meanVarErr = statObj.getResult(afwMath.MEANCLIP)
457 weight = 1.0 / float(meanVar)
458 if not numpy.isfinite(weight):
459 self.log.warn(
"Non-finite weight for %s: skipping", tempExpRef.dataId)
461 self.log.info(
"Weight of %s %s = %0.3f", tempExpName, tempExpRef.dataId, weight)
466 tempExpRefList.append(tempExpRef)
467 weightList.append(weight)
468 imageScalerList.append(imageScaler)
470 return pipeBase.Struct(tempExpRefList=tempExpRefList, weightList=weightList,
471 imageScalerList=imageScalerList)
475 \brief Perform background matching on the prepared inputs
477 Each Warp has a different background level that must be normalized to a reference level
478 before coaddition. If no reference is provided, the background matcher selects one. If the background
479 matching is performed sucessfully, recompute the weight to be applied to the Warp (coaddTempExp) to be
480 consistent with the scaled background.
482 \param[in] inputData: Struct from prepareInputs() with tempExpRefList, weightList, imageScalerList
483 \param[in] refExpDataRef: Data reference for background reference Warp, or None
484 \param[in] refImageScaler: Image scaler for background reference Warp, or None
486 - tempExprefList: List of data references to warped exposures (coaddTempExps)
487 - weightList: List of weightings
488 - imageScalerList: List of image scalers
489 - backgroundInfoList: result from background matching
492 backgroundInfoList = self.matchBackgrounds.run(
493 expRefList=inputData.tempExpRefList,
494 imageScalerList=inputData.imageScalerList,
495 refExpDataRef=refExpDataRef
if not self.config.autoReference
else None,
496 refImageScaler=refImageScaler,
497 expDatasetType=self.getTempExpDatasetName(self.
warpType),
499 except Exception
as e:
500 self.log.fatal(
"Cannot match backgrounds: %s", e)
501 raise pipeBase.TaskError(
"Background matching failed.")
504 newTempExpRefList = []
505 newBackgroundStructList = []
509 for tempExpRef, bgInfo, scaler, weight
in zip(inputData.tempExpRefList, backgroundInfoList,
510 inputData.imageScalerList, inputData.weightList):
511 if not bgInfo.isReference:
514 if (bgInfo.backgroundModel
is None):
515 self.log.info(
"No background offset model available for %s: skipping", tempExpRef.dataId)
518 varianceRatio = bgInfo.matchedMSE / bgInfo.diffImVar
519 except Exception
as e:
520 self.log.info(
"MSE/Var ratio not calculable (%s) for %s: skipping",
521 e, tempExpRef.dataId)
523 if not numpy.isfinite(varianceRatio):
524 self.log.info(
"MSE/Var ratio not finite (%.2f / %.2f) for %s: skipping",
525 bgInfo.matchedMSE, bgInfo.diffImVar, tempExpRef.dataId)
527 elif (varianceRatio > self.config.maxMatchResidualRatio):
528 self.log.info(
"Bad fit. MSE/Var ratio %.2f > %.2f for %s: skipping",
529 varianceRatio, self.config.maxMatchResidualRatio, tempExpRef.dataId)
531 elif (bgInfo.fitRMS > self.config.maxMatchResidualRMS):
532 self.log.info(
"Bad fit. RMS %.2f > %.2f for %s: skipping",
533 bgInfo.fitRMS, self.config.maxMatchResidualRMS, tempExpRef.dataId)
535 newWeightList.append(1 / (1 / weight + bgInfo.fitRMS**2))
536 newTempExpRefList.append(tempExpRef)
537 newBackgroundStructList.append(bgInfo)
538 newScaleList.append(scaler)
540 return pipeBase.Struct(tempExpRefList=newTempExpRefList, weightList=newWeightList,
541 imageScalerList=newScaleList, backgroundInfoList=newBackgroundStructList)
543 def assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList, bgInfoList=None,
544 altMaskList=
None, doClip=
False, mask=
None):
546 \anchor AssembleCoaddTask.assemble_
548 \brief Assemble a coadd from input warps
550 Assemble the coadd using the provided list of coaddTempExps. Since the full coadd covers a patch (a
551 large area), the assembly is performed over small areas on the image at a time in order to
552 conserve memory usage. Iterate over subregions within the outer bbox of the patch using
553 \ref assembleSubregion to mean-stack the corresponding subregions from the coaddTempExps (with outlier
554 rejection if config.doSigmaClip=True). Set the edge bits the the coadd mask based on the weight map.
556 \param[in] skyInfo: Patch geometry information, from getSkyInfo
557 \param[in] tempExpRefList: List of data references to Warps (previously called CoaddTempExps)
558 \param[in] imageScalerList: List of image scalers
559 \param[in] weightList: List of weights
560 \param[in] bgInfoList: List of background data from background matching, or None
561 \param[in] altMaskList: List of alternate masks to use rather than those stored with tempExp, or None
562 \param[in] doClip: Use clipping when codding?
563 \param[in] mask: Mask to ignore when coadding
564 \return coadded exposure
566 tempExpName = self.getTempExpDatasetName(self.
warpType)
567 self.log.info(
"Assembling %s %s", len(tempExpRefList), tempExpName)
569 mask = self.getBadPixelMask()
571 statsCtrl = afwMath.StatisticsControl()
572 statsCtrl.setNumSigmaClip(self.config.sigmaClip)
573 statsCtrl.setNumIter(self.config.clipIter)
574 statsCtrl.setAndMask(mask)
575 statsCtrl.setNanSafe(
True)
576 statsCtrl.setWeighted(
True)
577 statsCtrl.setCalcErrorFromInputVariance(
True)
578 for plane, threshold
in self.config.maskPropagationThresholds.items():
579 bit = afwImage.MaskU.getMaskPlane(plane)
580 statsCtrl.setMaskPropagationThreshold(bit, threshold)
583 statsFlags = afwMath.MEANCLIP
585 statsFlags = afwMath.MEAN
587 if bgInfoList
is None:
588 bgInfoList = [
None]*len(tempExpRefList)
590 if altMaskList
is None:
591 altMaskList = [
None]*len(tempExpRefList)
593 coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
594 coaddExposure.setCalib(self.scaleZeroPoint.getCalib())
595 coaddExposure.getInfo().setCoaddInputs(self.inputRecorder.makeCoaddInputs())
597 coaddMaskedImage = coaddExposure.getMaskedImage()
598 subregionSizeArr = self.config.subregionSize
599 subregionSize = afwGeom.Extent2I(subregionSizeArr[0], subregionSizeArr[1])
600 for subBBox
in _subBBoxIter(skyInfo.bbox, subregionSize):
603 weightList, bgInfoList, altMaskList, statsFlags, statsCtrl)
604 except Exception
as e:
605 self.log.fatal(
"Cannot compute coadd %s: %s", subBBox, e)
607 coaddUtils.setCoaddEdgeBits(coaddMaskedImage.getMask(), coaddMaskedImage.getVariance())
613 \brief Set the metadata for the coadd
615 This basic implementation simply sets the filter from the
618 \param[in] coaddExposure: The target image for the coadd
619 \param[in] tempExpRefList: List of data references to tempExp
620 \param[in] weightList: List of weights
622 assert len(tempExpRefList) == len(weightList),
"Length mismatch"
623 tempExpName = self.getTempExpDatasetName(self.
warpType)
627 tempExpList = [tempExpRef.get(tempExpName +
"_sub",
628 bbox=afwGeom.Box2I(afwGeom.Point2I(0, 0), afwGeom.Extent2I(1, 1)),
629 imageOrigin=
"LOCAL", immediate=
True)
for tempExpRef
in tempExpRefList]
630 numCcds = sum(len(tempExp.getInfo().getCoaddInputs().ccds)
for tempExp
in tempExpList)
632 coaddExposure.setFilter(tempExpList[0].getFilter())
633 coaddInputs = coaddExposure.getInfo().getCoaddInputs()
634 coaddInputs.ccds.reserve(numCcds)
635 coaddInputs.visits.reserve(len(tempExpList))
637 for tempExp, weight
in zip(tempExpList, weightList):
638 self.inputRecorder.addVisitToCoadd(coaddInputs, tempExp, weight)
639 coaddInputs.visits.sort()
645 modelPsfList = [tempExp.getPsf()
for tempExp
in tempExpList]
646 modelPsfWidthList = [modelPsf.computeBBox().getWidth()
for modelPsf
in modelPsfList]
647 psf = modelPsfList[modelPsfWidthList.index(max(modelPsfWidthList))]
649 psf = measAlg.CoaddPsf(coaddInputs.ccds, coaddExposure.getWcs())
650 coaddExposure.setPsf(psf)
651 apCorrMap = measAlg.makeCoaddApCorrMap(coaddInputs.ccds, coaddExposure.getBBox(afwImage.PARENT),
652 coaddExposure.getWcs())
653 coaddExposure.getInfo().setApCorrMap(apCorrMap)
655 def assembleSubregion(self, coaddExposure, bbox, tempExpRefList, imageScalerList, weightList,
656 bgInfoList, altMaskList, statsFlags, statsCtrl):
658 \brief Assemble the coadd for a sub-region.
660 For each coaddTempExp, check for (and swap in) an alternative mask if one is passed. If background
661 matching is enabled, add the background and background variance from each coaddTempExp. Remove mask
662 planes listed in config.removeMaskPlanes, Finally, mean-stack
663 the actual exposures using \ref afwMath.statisticsStack "statisticsStack" with outlier rejection if
664 config.doSigmaClip=True. Assign the stacked subregion back to the coadd.
666 \param[in] coaddExposure: The target image for the coadd
667 \param[in] bbox: Sub-region to coadd
668 \param[in] tempExpRefList: List of data reference to tempExp
669 \param[in] imageScalerList: List of image scalers
670 \param[in] weightList: List of weights
671 \param[in] bgInfoList: List of background data from background matching
672 \param[in] altMaskList: List of alternate masks to use rather than those stored with tempExp, or None
673 \param[in] statsFlags: Statistic for coadd
674 \param[in] statsCtrl: Statistics control object for coadd
676 self.log.debug(
"Computing coadd over %s", bbox)
677 tempExpName = self.getTempExpDatasetName(self.
warpType)
678 coaddMaskedImage = coaddExposure.getMaskedImage()
680 for tempExpRef, imageScaler, bgInfo, altMask
in zip(tempExpRefList, imageScalerList, bgInfoList,
682 exposure = tempExpRef.get(tempExpName +
"_sub", bbox=bbox)
683 maskedImage = exposure.getMaskedImage()
686 altMaskSub = altMask.Factory(altMask, bbox, afwImage.PARENT)
687 maskedImage.getMask().swap(altMaskSub)
688 imageScaler.scaleMaskedImage(maskedImage)
690 if self.config.doMatchBackgrounds
and not bgInfo.isReference:
691 backgroundModel = bgInfo.backgroundModel
692 backgroundImage = backgroundModel.getImage()
if \
693 self.matchBackgrounds.config.usePolynomial
else \
694 backgroundModel.getImageF()
695 backgroundImage.setXY0(coaddMaskedImage.getXY0())
696 maskedImage += backgroundImage.Factory(backgroundImage, bbox, afwImage.PARENT,
False)
697 var = maskedImage.getVariance()
698 var += (bgInfo.fitRMS)**2
700 if self.config.removeMaskPlanes:
701 mask = maskedImage.getMask()
702 for maskPlane
in self.config.removeMaskPlanes:
704 mask &= ~mask.getPlaneBitMask(maskPlane)
705 except Exception
as e:
706 self.log.warn(
"Unable to remove mask plane %s: %s", maskPlane, e.message)
708 maskedImageList.append(maskedImage)
710 with self.timer(
"stack"):
711 coaddSubregion = afwMath.statisticsStack(
712 maskedImageList, statsFlags, statsCtrl, weightList)
714 coaddMaskedImage.assign(coaddSubregion, bbox)
718 \brief Add metadata from the background matching to the coadd
720 \param[in] coaddExposure: Coadd
721 \param[in] tempExpRefList: List of data references for temp exps to go into coadd
722 \param[in] backgroundInfoList: List of background info, results from background matching
724 self.log.info(
"Adding exposure information to metadata")
725 metadata = coaddExposure.getMetadata()
726 metadata.addString(
"CTExp_SDQA1_DESCRIPTION",
727 "Background matching: Ratio of matchedMSE / diffImVar")
728 for ind, (tempExpRef, backgroundInfo)
in enumerate(zip(tempExpRefList, backgroundInfoList)):
729 tempExpStr =
'&'.join(
'%s=%s' % (k, v)
for k, v
in tempExpRef.dataId.items())
730 if backgroundInfo.isReference:
731 metadata.addString(
"ReferenceExp_ID", tempExpStr)
733 metadata.addString(
"CTExp_ID_%d" % (ind), tempExpStr)
734 metadata.addDouble(
"CTExp_SDQA1_%d" % (ind),
735 backgroundInfo.matchedMSE/backgroundInfo.diffImVar)
736 metadata.addDouble(
"CTExp_SDQA2_%d" % (ind),
737 backgroundInfo.fitRMS)
740 """Returns None on failure"""
742 return dataRef.get(
"brightObjectMask", immediate=
True)
743 except Exception
as e:
744 self.log.warn(
"Unable to read brightObjectMask for %s: %s", dataRef.dataId, e)
748 """Set the bright object masks
750 exposure: Exposure under consideration
751 dataId: Data identifier dict for patch
752 brightObjectMasks: afwTable of bright objects to mask
757 if brightObjectMasks
is None:
758 self.log.warn(
"Unable to apply bright object mask: none supplied")
760 self.log.info(
"Applying %d bright object masks to %s", len(brightObjectMasks), dataId)
761 md = brightObjectMasks.table.getMetadata()
764 self.log.warn(
"Expected to see %s in metadata", k)
766 if md.get(k) != dataId[k]:
767 self.log.warn(
"Expected to see %s == %s in metadata, saw %s", k, md.get(k), dataId[k])
769 mask = exposure.getMaskedImage().getMask()
770 wcs = exposure.getWcs()
771 plateScale = wcs.pixelScale().asArcseconds()
773 for rec
in brightObjectMasks:
774 center = afwGeom.PointI(wcs.skyToPixel(rec.getCoord()))
775 if rec[
"type"] ==
"box":
776 assert rec[
"angle"] == 0.0, (
"Angle != 0 for mask object %s" % rec[
"id"])
777 width = rec[
"width"].asArcseconds()/plateScale
778 height = rec[
"height"].asArcseconds()/plateScale
780 halfSize = afwGeom.ExtentI(0.5*width, 0.5*height)
781 bbox = afwGeom.Box2I(center - halfSize, center + halfSize)
783 bbox = afwGeom.BoxI(afwGeom.PointI(int(center[0] - 0.5*width), int(center[1] - 0.5*height)),
784 afwGeom.PointI(int(center[0] + 0.5*width), int(center[1] + 0.5*height)))
785 spans = afwGeom.SpanSet(bbox)
786 elif rec[
"type"] ==
"circle":
787 radius = int(rec[
"radius"].asArcseconds()/plateScale)
788 spans = afwGeom.SpanSet.fromShape(radius, offset=center)
790 self.log.warn(
"Unexpected region type %s at %s" % rec[
"type"], center)
795 def _makeArgumentParser(cls):
797 \brief Create an argument parser
799 parser = pipeBase.ArgumentParser(name=cls._DefaultName)
800 parser.add_id_argument(
"--id", cls.ConfigClass().coaddName +
"Coadd_directWarp",
801 help=
"data ID, e.g. --id tract=12345 patch=1,2",
802 ContainerClass=AssembleCoaddDataIdContainer)
803 parser.add_id_argument(
"--selectId",
"calexp", help=
"data ID, e.g. --selectId visit=6789 ccd=0..9",
804 ContainerClass=SelectDataIdContainer)
808 def _subBBoxIter(bbox, subregionSize):
810 \brief Iterate over subregions of a bbox
812 \param[in] bbox: bounding box over which to iterate: afwGeom.Box2I
813 \param[in] subregionSize: size of sub-bboxes
815 \return subBBox: next sub-bounding box of size subregionSize or smaller;
816 each subBBox is contained within bbox, so it may be smaller than subregionSize at the edges of bbox,
817 but it will never be empty
820 raise RuntimeError(
"bbox %s is empty" % (bbox,))
821 if subregionSize[0] < 1
or subregionSize[1] < 1:
822 raise RuntimeError(
"subregionSize %s must be nonzero" % (subregionSize,))
824 for rowShift
in range(0, bbox.getHeight(), subregionSize[1]):
825 for colShift
in range(0, bbox.getWidth(), subregionSize[0]):
826 subBBox = afwGeom.Box2I(bbox.getMin() + afwGeom.Extent2I(colShift, rowShift), subregionSize)
828 if subBBox.isEmpty():
829 raise RuntimeError(
"Bug: empty bbox! bbox=%s, subregionSize=%s, colShift=%s, rowShift=%s" %
830 (bbox, subregionSize, colShift, rowShift))
836 \brief A version of lsst.pipe.base.DataIdContainer specialized for assembleCoadd.
841 \brief Make self.refList from self.idList.
843 Interpret the config.doMatchBackgrounds, config.autoReference,
844 and whether a visit/run supplied.
845 If a visit/run is supplied, config.autoReference is automatically set to False.
846 if config.doMatchBackgrounds == false, then a visit/run will be ignored if accidentally supplied.
849 keysCoadd = namespace.butler.getKeys(datasetType=namespace.config.coaddName +
"Coadd",
851 keysCoaddTempExp = namespace.butler.getKeys(datasetType=namespace.config.coaddName +
852 "Coadd_directWarp", level=self.level)
854 if namespace.config.doMatchBackgrounds:
855 if namespace.config.autoReference:
856 datasetType = namespace.config.coaddName +
"Coadd"
857 validKeys = keysCoadd
859 datasetType = namespace.config.coaddName +
"Coadd_directWarp"
860 validKeys = keysCoaddTempExp
862 datasetType = namespace.config.coaddName +
"Coadd"
863 validKeys = keysCoadd
865 for dataId
in self.idList:
867 for key
in validKeys:
868 if key
not in dataId:
869 raise RuntimeError(
"--id must include " + key)
872 if (key
not in keysCoadd)
and (key
in keysCoaddTempExp):
873 if namespace.config.autoReference:
875 namespace.config.autoReference =
False
876 datasetType = namespace.config.coaddName +
"Coadd_directWarp"
877 print(
"Switching config.autoReference to False; applies only to background Matching.")
880 dataRef = namespace.butler.dataRef(
881 datasetType=datasetType,
884 self.refList.append(dataRef)
889 \brief Function to count the number of pixels with a specific mask in a footprint.
891 Find the intersection of mask & footprint. Count all pixels in the mask that are in the intersection that
892 have bitmask set but do not have ignoreMask set. Return the count.
894 \param[in] mask: mask to define intersection region by.
895 \parma[in] footprint: footprint to define the intersection region by.
896 \param[in] bitmask: specific mask that we wish to count the number of occurances of.
897 \param[in] ignoreMask: pixels to not consider.
898 \return count of number of pixels in footprint with specified mask.
900 bbox = footprint.getBBox()
901 bbox.clip(mask.getBBox(afwImage.PARENT))
902 fp = afwImage.MaskU(bbox)
903 subMask = mask.Factory(mask, bbox, afwImage.PARENT)
904 footprint.spans.setMask(fp, bitmask)
905 return numpy.logical_and((subMask.getArray() & fp.getArray()) > 0,
906 (subMask.getArray() & ignoreMask) == 0).sum()
911 \anchor SafeClipAssembleCoaddConfig
913 \brief Configuration parameters for the SafeClipAssembleCoaddTask
915 clipDetection = pexConfig.ConfigurableField(
916 target=SourceDetectionTask,
917 doc=
"Detect sources on difference between unclipped and clipped coadd")
918 minClipFootOverlap = pexConfig.Field(
919 doc=
"Minimum fractional overlap of clipped footprint with visit DETECTED to be clipped",
923 minClipFootOverlapSingle = pexConfig.Field(
924 doc=
"Minimum fractional overlap of clipped footprint with visit DETECTED to be "
925 "clipped when only one visit overlaps",
929 minClipFootOverlapDouble = pexConfig.Field(
930 doc=
"Minimum fractional overlap of clipped footprints with visit DETECTED to be "
931 "clipped when two visits overlap",
935 maxClipFootOverlapDouble = pexConfig.Field(
936 doc=
"Maximum fractional overlap of clipped footprints with visit DETECTED when "
937 "considering two visits",
941 minBigOverlap = pexConfig.Field(
942 doc=
"Minimum number of pixels in footprint to use DETECTED mask from the single visits "
943 "when labeling clipped footprints",
951 AssembleCoaddConfig.setDefaults(self)
952 self.clipDetection.doTempLocalBackground =
False
953 self.clipDetection.reEstimateBackground =
False
954 self.clipDetection.returnOriginalFootprints =
False
955 self.clipDetection.thresholdPolarity =
"both"
956 self.clipDetection.thresholdValue = 2
957 self.clipDetection.nSigmaToGrow = 2
958 self.clipDetection.minPixels = 4
959 self.clipDetection.isotropicGrow =
True
960 self.clipDetection.thresholdType =
"pixel_stdev"
974 \anchor SafeClipAssembleCoaddTask_
976 \brief Assemble a coadded image from a set of coadded temporary exposures, being careful to clip & flag areas
977 with potential artifacts.
979 \section pipe_tasks_assembleCoadd_Contents Contents
980 - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Purpose
981 - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Initialize
982 - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Run
983 - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Config
984 - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Debug
985 - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Example
987 \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Purpose Description
989 \copybrief SafeClipAssembleCoaddTask
991 Read the documentation for \ref AssembleCoaddTask_ "AssembleCoaddTask" first since
992 SafeClipAssembleCoaddTask subtasks that task.
993 In \ref AssembleCoaddTask_ "AssembleCoaddTask", we compute the coadd as an clipped mean (i.e. we clip
995 The problem with doing this is that when computing the coadd PSF at a given location, individual visit
996 PSFs from visits with outlier pixels contribute to the coadd PSF and cannot be treated correctly.
997 In this task, we correct for this behavior by creating a new badMaskPlane 'CLIPPED'.
998 We populate this plane on the input coaddTempExps and the final coadd where i. difference imaging suggests
999 that there is an outlier and ii. this outlier appears on only one or two images.
1000 Such regions will not contribute to the final coadd.
1001 Furthermore, any routine to determine the coadd PSF can now be cognizant of clipped regions.
1002 Note that the algorithm implemented by this task is preliminary and works correctly for HSC data.
1003 Parameter modifications and or considerable redesigning of the algorithm is likley required for other
1006 SafeClipAssembleCoaddTask uses a \ref SourceDetectionTask_ "clipDetection" subtask and also sub-classes
1007 \ref AssembleCoaddTask_ "AssembleCoaddTask". You can retarget the
1008 \ref SourceDetectionTask_ "clipDetection" subtask if you wish.
1010 \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Initialize Task initialization
1011 \copydoc \_\_init\_\_
1013 \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Run Invoking the Task
1016 \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Config Configuration parameters
1017 See \ref SafeClipAssembleCoaddConfig
1019 \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Debug Debug variables
1020 The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a
1021 flag \c -d to import \b debug.py from your \c PYTHONPATH; see \ref baseDebug for more about \b debug.py
1023 SafeClipAssembleCoaddTask has no debug variables of its own. The \ref SourceDetectionTask_ "clipDetection"
1024 subtasks may support debug variables. See the documetation for \ref SourceDetectionTask_ "clipDetection"
1025 for further information.
1027 \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Example A complete example of using SafeClipAssembleCoaddTask
1029 SafeClipAssembleCoaddTask assembles a set of warped coaddTempExp images into a coadded image.
1030 The SafeClipAssembleCoaddTask is invoked by running assembleCoadd.py <em>without</em> the flag
1032 Usage of assembleCoadd.py expects a data reference to the tract patch and filter to be coadded
1033 (specified using '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]') along
1034 with a list of coaddTempExps to attempt to coadd (specified using
1035 '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]').
1036 Only the coaddTempExps that cover the specified tract and patch will be coadded.
1037 A list of the available optional arguments can be obtained by calling assembleCoadd.py with the --help
1038 command line argument:
1040 assembleCoadd.py --help
1042 To demonstrate usage of the SafeClipAssembleCoaddTask in the larger context of multi-band processing, we
1043 will generate the HSC-I & -R band coadds from HSC engineering test data provided in the ci_hsc package. To
1044 begin, assuming that the lsst stack has been already set up, we must set up the obs_subaru and ci_hsc
1046 This defines the environment variable $CI_HSC_DIR and points at the location of the package. The raw HSC
1047 data live in the $CI_HSC_DIR/raw directory. To begin assembling the coadds, we must first
1050 <DD> process the individual ccds in $CI_HSC_RAW to produce calibrated exposures</DD>
1052 <DD> create a skymap that covers the area of the sky present in the raw exposures</DD>
1053 <DT>makeCoaddTempExp</DT>
1054 <DD> warp the individual calibrated exposures to the tangent plane of the coadd</DD>
1056 We can perform all of these steps by running
1058 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988
1060 This will produce warped coaddTempExps for each visit. To coadd the wraped data, we call assembleCoadd.py
1063 assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 --selectId visit=903988 ccd=24
1065 This will process the HSC-I band data. The results are written in $CI_HSC_DIR/DATA/deepCoadd-results/HSC-I
1066 You may also choose to run:
1068 scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346
1069 assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R --selectId visit=903334 ccd=16 --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 --selectId visit=903334 ccd=100 --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 --selectId visit=903338 ccd=18 --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 --selectId visit=903342 ccd=10 --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 --selectId visit=903344 ccd=5 --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 --selectId visit=903346 ccd=6 --selectId visit=903346 ccd=12
1071 to generate the coadd for the HSC-R band if you are interested in following multiBand Coadd processing as
1072 discussed in \ref pipeTasks_multiBand.
1074 ConfigClass = SafeClipAssembleCoaddConfig
1075 _DefaultName =
"safeClipAssembleCoadd"
1079 \brief Initialize the task and make the \ref SourceDetectionTask_ "clipDetection" subtask.
1081 AssembleCoaddTask.__init__(self, *args, **kwargs)
1082 schema = afwTable.SourceTable.makeMinimalSchema()
1083 self.makeSubtask(
"clipDetection", schema=schema)
1085 def assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList, bgModelList, *args, **kwargs):
1087 \brief Assemble the coadd for a region
1089 Compute the difference of coadds created with and without outlier rejection to identify coadd pixels
1090 that have outlier values in some individual visits. Detect clipped regions on the difference image and
1091 mark these regions on the one or two individual coaddTempExps where they occur if there is significant
1092 overlap between the clipped region and a source.
1093 This leaves us with a set of footprints from the difference image that have been identified as having
1094 occured on just one or two individual visits. However, these footprints were generated from a
1095 difference image. It is conceivable for a large diffuse source to have become broken up into multiple
1096 footprints acrosss the coadd difference in this process.
1097 Determine the clipped region from all overlapping footprints from the detected sources in each visit -
1098 these are big footprints.
1099 Combine the small and big clipped footprints and mark them on a new bad mask plane
1100 Generate the coadd using \ref AssembleCoaddTask.assemble_ "AssembleCoaddTask.assemble" without outlier
1101 removal. Clipped footprints will no longer make it into the coadd because they are marked in the new
1104 N.b. *args and **kwargs are passed but ignored in order to match the call signature expected by the
1107 @param skyInfo: Patch geometry information, from getSkyInfo
1108 @param tempExpRefList: List of data reference to tempExp
1109 @param imageScalerList: List of image scalers
1110 @param weightList: List of weights
1111 @param bgModelList: List of background models from background matching
1112 return coadd exposure
1114 exp = self.
buildDifferenceImage(skyInfo, tempExpRefList, imageScalerList, weightList, bgModelList)
1115 mask = exp.getMaskedImage().getMask()
1116 mask.addMaskPlane(
"CLIPPED")
1118 result = self.
detectClip(exp, tempExpRefList)
1120 self.log.info(
'Found %d clipped objects', len(result.clipFootprints))
1123 maskClipValue = mask.getPlaneBitMask(
"CLIPPED")
1124 maskDetValue = mask.getPlaneBitMask(
"DETECTED") | mask.getPlaneBitMask(
"DETECTED_NEGATIVE")
1125 bigFootprints = self.
detectClipBig(result.tempExpClipList, result.clipFootprints, result.clipIndices,
1126 maskClipValue, maskDetValue)
1129 maskClip = mask.Factory(mask.getBBox(afwImage.PARENT))
1130 afwDet.setMaskFromFootprintList(maskClip, result.clipFootprints, maskClipValue)
1132 maskClipBig = maskClip.Factory(mask.getBBox(afwImage.PARENT))
1133 afwDet.setMaskFromFootprintList(maskClipBig, bigFootprints, maskClipValue)
1134 maskClip |= maskClipBig
1137 badMaskPlanes = self.config.badMaskPlanes[:]
1138 badMaskPlanes.append(
"CLIPPED")
1139 badPixelMask = afwImage.MaskU.getPlaneBitMask(badMaskPlanes)
1140 coaddExp = AssembleCoaddTask.assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList,
1141 bgModelList, result.tempExpClipList,
1147 maskExp = coaddExp.getMaskedImage().getMask()
1154 \brief Return an exposure that contains the difference between and unclipped and clipped coadds.
1156 Generate a difference image between clipped and unclipped coadds.
1157 Compute the difference image by subtracting an outlier-clipped coadd from an outlier-unclipped coadd.
1158 Return the difference image.
1160 @param skyInfo: Patch geometry information, from getSkyInfo
1161 @param tempExpRefList: List of data reference to tempExp
1162 @param imageScalerList: List of image scalers
1163 @param weightList: List of weights
1164 @param bgModelList: List of background models from background matching
1165 @return Difference image of unclipped and clipped coadd wrapped in an Exposure
1168 coaddMean = AssembleCoaddTask.assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList,
1169 bgModelList, doClip=
False)
1172 coaddClip = AssembleCoaddTask.assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList,
1173 bgModelList, doClip=
True)
1175 coaddDiff = coaddMean.getMaskedImage().Factory(coaddMean.getMaskedImage())
1176 coaddDiff -= coaddClip.getMaskedImage()
1177 exp = afwImage.ExposureF(coaddDiff)
1178 exp.setPsf(coaddMean.getPsf())
1183 \brief Detect clipped regions on an exposure and set the mask on the individual tempExp masks
1185 Detect footprints in the difference image after smoothing the difference image with a Gaussian kernal.
1186 Identify footprints that overlap with one or two input coaddTempExps by comparing the computed overlap
1187 fraction to thresholds set in the config.
1188 A different threshold is applied depending on the number of overlapping visits (restricted to one or
1190 If the overlap exceeds the thresholds, the footprint is considered "CLIPPED" and is marked as such on
1192 Return a struct with the clipped footprints, the indices of the coaddTempExps that end up overlapping
1193 with the clipped footprints and a list of new masks for the coaddTempExps.
1195 \param[in] exp: Exposure to run detection on
1196 \param[in] tempExpRefList: List of data reference to tempExp
1197 \return struct containing:
1198 - clippedFootprints: list of clipped footprints
1199 - clippedIndices: indices for each clippedFootprint in tempExpRefList
1200 - tempExpClipList: list of new masks for tempExp
1202 mask = exp.getMaskedImage().getMask()
1203 maskClipValue = mask.getPlaneBitMask(
"CLIPPED")
1204 maskDetValue = mask.getPlaneBitMask(
"DETECTED") | mask.getPlaneBitMask(
"DETECTED_NEGATIVE")
1205 fpSet = self.clipDetection.detectFootprints(exp, doSmooth=
True, clearMask=
True)
1207 fpSet.positive.merge(fpSet.negative)
1208 footprints = fpSet.positive
1209 self.log.info(
'Found %d potential clipped objects', len(footprints.getFootprints()))
1210 ignoreMask = self.getBadPixelMask()
1216 tempExpClipList = [tmpExpRef.get(self.getTempExpDatasetName(self.
warpType),
1217 immediate=
True).getMaskedImage().getMask()
for
1218 tmpExpRef
in tempExpRefList]
1220 for footprint
in footprints.getFootprints():
1221 nPixel = footprint.getArea()
1225 for i, tmpExpMask
in enumerate(tempExpClipList):
1229 totPixel = nPixel - ignore
1232 if ignore > overlapDet
or totPixel <= 0.5*nPixel
or overlapDet == 0:
1234 overlap.append(overlapDet/float(totPixel))
1235 maskList.append(tmpExpMask)
1238 overlap = numpy.array(overlap)
1239 if not len(overlap):
1246 if len(overlap) == 1:
1247 if overlap[0] > self.config.minClipFootOverlapSingle:
1252 clipIndex = numpy.where(overlap > self.config.minClipFootOverlap)[0]
1253 if len(clipIndex) == 1:
1255 keepIndex = [clipIndex[0]]
1258 clipIndex = numpy.where(overlap > self.config.minClipFootOverlapDouble)[0]
1259 if len(clipIndex) == 2
and len(overlap) > 3:
1260 clipIndexComp = numpy.where(overlap < self.config.minClipFootOverlapDouble)[0]
1261 if numpy.max(overlap[clipIndexComp]) < self.config.maxClipFootOverlapDouble:
1263 keepIndex = clipIndex
1268 for index
in keepIndex:
1269 footprint.spans.setMask(maskList[index], maskClipValue)
1271 clipIndices.append(numpy.array(indexList)[keepIndex])
1272 clipFootprints.append(footprint)
1274 return pipeBase.Struct(clipFootprints=clipFootprints, clipIndices=clipIndices,
1275 tempExpClipList=tempExpClipList)
1277 def detectClipBig(self, tempExpClipList, clipFootprints, clipIndices, maskClipValue, maskDetValue):
1279 \brief Find footprints from individual tempExp footprints for large footprints.
1281 Identify big footprints composed of many sources in the coadd difference that may have originated in a
1282 large diffuse source in the coadd. We do this by indentifying all clipped footprints that overlap
1283 significantly with each source in all the coaddTempExps.
1285 \param[in] tempExpClipList: List of tempExp masks with clipping information
1286 \param[in] clipFootprints: List of clipped footprints
1287 \param[in] clipIndices: List of which entries in tempExpClipList each footprint belongs to
1288 \param[in] maskClipValue: Mask value of clipped pixels
1289 \param[in] maskClipValue: Mask value of detected pixels
1290 \return list of big footprints
1292 bigFootprintsCoadd = []
1293 ignoreMask = self.getBadPixelMask()
1294 for index, tmpExpMask
in enumerate(tempExpClipList):
1297 maskVisitDet = tmpExpMask.Factory(tmpExpMask, tmpExpMask.getBBox(afwImage.PARENT),
1298 afwImage.PARENT,
True)
1299 maskVisitDet &= maskDetValue
1300 visitFootprints = afwDet.FootprintSet(maskVisitDet, afwDet.Threshold(1))
1303 clippedFootprintsVisit = []
1304 for foot, clipIndex
in zip(clipFootprints, clipIndices):
1305 if index
not in clipIndex:
1307 clippedFootprintsVisit.append(foot)
1308 maskVisitClip = maskVisitDet.Factory(maskVisitDet.getBBox(afwImage.PARENT))
1309 afwDet.setMaskFromFootprintList(maskVisitClip, clippedFootprintsVisit, maskClipValue)
1311 bigFootprintsVisit = []
1312 for foot
in visitFootprints.getFootprints():
1313 if foot.getArea() < self.config.minBigOverlap:
1316 if nCount > self.config.minBigOverlap:
1317 bigFootprintsVisit.append(foot)
1318 bigFootprintsCoadd.append(foot)
1321 maskVisitClip.clearAllMaskPlanes()
1322 afwDet.setMaskFromFootprintList(maskVisitClip, bigFootprintsVisit, maskClipValue)
1323 tmpExpMask |= maskVisitClip
1325 return bigFootprintsCoadd
def __init__
Initialize the task and make the clipDetection subtask.
def run
Assemble a coadd from a set of Warps.
def getBackgroundReferenceScaler
Construct an image scaler for the background reference frame.
def assemble
Assemble the coadd for a region.
def addBackgroundMatchingMetadata
Add metadata from the background matching to the coadd.
def detectClipBig
Find footprints from individual tempExp footprints for large footprints.
def __init__
Initialize the task.
def assembleSubregion
Assemble the coadd for a sub-region.
def assembleMetadata
Set the metadata for the coadd.
def backgroundMatching
Perform background matching on the prepared inputs.
Configuration parameters for the SafeClipAssembleCoaddTask.
Assemble a coadded image from a set of warps (coadded temporary exposures).
Assemble a coadded image from a set of coadded temporary exposures, being careful to clip & flag area...
def buildDifferenceImage
Return an exposure that contains the difference between and unclipped and clipped coadds...
def detectClip
Detect clipped regions on an exposure and set the mask on the individual tempExp masks.
def prepareInputs
Prepare the input warps for coaddition by measuring the weight for each warp and the scaling for the ...
def makeDataRefList
Make self.refList from self.idList.
Configuration parameters for the AssembleCoaddTask.
def countMaskFromFootprint
Function to count the number of pixels with a specific mask in a footprint.
def assemble
Assemble a coadd from input warps.
def readBrightObjectMasks
A version of lsst.pipe.base.DataIdContainer specialized for assembleCoadd.
def getTempExpRefList
Generate list data references corresponding to warped exposures that lie within the patch to be coadd...