1 from __future__
import absolute_import, division, print_function
2 from builtins
import zip
3 from builtins
import range
39 from .coaddBase
import CoaddBaseTask, SelectDataIdContainer
40 from .interpImage
import InterpImageTask
41 from .matchBackgrounds
import MatchBackgroundsTask
42 from .scaleZeroPoint
import ScaleZeroPointTask
43 from .coaddHelpers
import groupPatchExposures, getGroupDataRef
46 __all__ = [
"AssembleCoaddTask",
"SafeClipAssembleCoaddTask",
"CompareWarpAssembleCoaddTask"]
51 \anchor AssembleCoaddConfig_ 53 \brief Configuration parameters for the \ref AssembleCoaddTask_ "AssembleCoaddTask" 55 warpType = pexConfig.Field(
56 doc=
"Warp name: one of 'direct' or 'psfMatched'",
60 subregionSize = pexConfig.ListField(
62 doc=
"Width, height of stack subregion size; " 63 "make small enough that a full stack of images will fit into memory at once.",
67 statistic = pexConfig.Field(
69 doc=
"Main stacking statistic for aggregating over the epochs.",
72 doSigmaClip = pexConfig.Field(
74 doc=
"Perform sigma clipped outlier rejection with MEANCLIP statistic? (DEPRECATED)",
77 sigmaClip = pexConfig.Field(
79 doc=
"Sigma for outlier rejection; ignored if non-clipping statistic selected.",
82 clipIter = pexConfig.Field(
84 doc=
"Number of iterations of outlier rejection; ignored if non-clipping statistic selected.",
87 scaleZeroPoint = pexConfig.ConfigurableField(
88 target=ScaleZeroPointTask,
89 doc=
"Task to adjust the photometric zero point of the coadd temp exposures",
91 doInterp = pexConfig.Field(
92 doc=
"Interpolate over NaN pixels? Also extrapolate, if necessary, but the results are ugly.",
96 interpImage = pexConfig.ConfigurableField(
97 target=InterpImageTask,
98 doc=
"Task to interpolate (and extrapolate) over NaN pixels",
100 matchBackgrounds = pexConfig.ConfigurableField(
101 target=MatchBackgroundsTask,
102 doc=
"Task to match backgrounds",
104 maxMatchResidualRatio = pexConfig.Field(
105 doc=
"Maximum ratio of the mean squared error of the background matching model to the variance " 106 "of the difference in backgrounds",
110 maxMatchResidualRMS = pexConfig.Field(
111 doc=
"Maximum RMS of residuals of the background offset fit in matchBackgrounds.",
115 doWrite = pexConfig.Field(
116 doc=
"Persist coadd?",
120 doNImage = pexConfig.Field(
121 doc=
"Create image of number of contributing exposures for each pixel",
125 doMatchBackgrounds = pexConfig.Field(
126 doc=
"Match backgrounds of coadd temp exposures before coadding them? " 127 "If False, the coadd temp expsosures must already have been background subtracted or matched",
131 autoReference = pexConfig.Field(
132 doc=
"Automatically select the coadd temp exposure to use as a reference for background matching? " 133 "Ignored if doMatchBackgrounds false. " 134 "If False you must specify the reference temp exposure as the data Id",
138 maskPropagationThresholds = pexConfig.DictField(
141 doc=(
"Threshold (in fractional weight) of rejection at which we propagate a mask plane to " 142 "the coadd; that is, we set the mask bit on the coadd if the fraction the rejected frames " 143 "would have contributed exceeds this value."),
144 default={
"SAT": 0.1},
146 removeMaskPlanes = pexConfig.ListField(dtype=str, default=[
"CROSSTALK",
"NOT_DEBLENDED"],
147 doc=
"Mask planes to remove before coadding")
156 doMaskBrightObjects = pexConfig.Field(dtype=bool, default=
False,
157 doc=
"Set mask and flag bits for bright objects?")
158 brightObjectMaskName = pexConfig.Field(dtype=str, default=
"BRIGHT_OBJECT",
159 doc=
"Name of mask bit used for bright objects")
161 coaddPsf = pexConfig.ConfigField(
162 doc=
"Configuration for CoaddPsf",
163 dtype=measAlg.CoaddPsfConfig,
167 CoaddBaseTask.ConfigClass.setDefaults(self)
171 CoaddBaseTask.ConfigClass.validate(self)
175 log.warn(
"Config doPsfMatch deprecated. Setting warpType='psfMatched'")
178 log.warn(
'doSigmaClip deprecated. To replicate behavior, setting statistic to "MEANCLIP"')
180 if self.
doInterp and self.
statistic not in [
'MEAN',
'MEDIAN',
'MEANCLIP',
'VARIANCE',
'VARIANCECLIP']:
181 raise ValueError(
"Must set doInterp=False for statistic=%s, which does not " 182 "compute and set a non-zero coadd variance estimate." % (self.
statistic))
184 unstackableStats = [
'NOTHING',
'ERROR',
'ORMASK']
185 if not hasattr(afwMath.Property, self.
statistic)
or self.
statistic in unstackableStats:
186 stackableStats = [str(k)
for k
in afwMath.Property.__members__.keys()
187 if str(k)
not in unstackableStats]
188 raise ValueError(
"statistic %s is not allowed. Please choose one of %s." 200 \anchor AssembleCoaddTask_ 202 \brief Assemble a coadded image from a set of warps (coadded temporary exposures). 204 \section pipe_tasks_assembleCoadd_Contents Contents 205 - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Purpose 206 - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Initialize 207 - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Run 208 - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Config 209 - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Debug 210 - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Example 212 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Purpose Description 214 \copybrief AssembleCoaddTask_ 216 We want to assemble a coadded image from a set of Warps (also called 217 coadded temporary exposures or coaddTempExps. 218 Each input Warp covers a patch on the sky and corresponds to a single run/visit/exposure of the 219 covered patch. We provide the task with a list of Warps (selectDataList) from which it selects 220 Warps that cover the specified patch (pointed at by dataRef). 221 Each Warp that goes into a coadd will typically have an independent photometric zero-point. 222 Therefore, we must scale each Warp to set it to a common photometric zeropoint. By default, each 223 Warp has backgrounds and hence will require config.doMatchBackgrounds=True. 224 When background matching is enabled, the task may be configured to automatically select a reference exposure 225 (config.autoReference=True). If this is not done, we require that the input dataRef provides access to a 226 Warp (dataset type coaddName + 'Coadd' + warpType + 'Warp') which is used as the reference exposure. 227 WarpType may be one of 'direct' or 'psfMatched', and the boolean configs config.makeDirect and 228 config.makePsfMatched set which of the warp types will be coadded. 229 The coadd is computed as a mean with optional outlier rejection. 230 Criteria for outlier rejection are set in \ref AssembleCoaddConfig. Finally, Warps can have bad 'NaN' 231 pixels which received no input from the source calExps. We interpolate over these bad (NaN) pixels. 233 AssembleCoaddTask uses several sub-tasks. These are 235 <DT>\ref ScaleZeroPointTask_ "ScaleZeroPointTask"</DT> 236 <DD> create and use an imageScaler object to scale the photometric zeropoint for each Warp</DD> 237 <DT>\ref MatchBackgroundsTask_ "MatchBackgroundsTask"</DT> 238 <DD> match background in a Warp to a reference exposure (and select the reference exposure if one is 240 <DT>\ref InterpImageTask_ "InterpImageTask"</DT> 241 <DD>interpolate across bad pixels (NaN) in the final coadd</DD> 243 You can retarget these subtasks if you wish. 245 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Initialize Task initialization 246 \copydoc \_\_init\_\_ 248 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Run Invoking the Task 251 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Config Configuration parameters 252 See \ref AssembleCoaddConfig_ 254 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Debug Debug variables 255 The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a 256 flag \c -d to import \b debug.py from your \c PYTHONPATH; see \ref baseDebug for more about \b debug.py files. 257 AssembleCoaddTask has no debug variables of its own. Some of the subtasks may support debug variables. See 258 the documetation for the subtasks for further information. 260 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Example A complete example of using AssembleCoaddTask 262 AssembleCoaddTask assembles a set of warped images into a coadded image. The AssembleCoaddTask 263 can be invoked by running assembleCoadd.py with the flag '--legacyCoadd'. Usage of assembleCoadd.py expects 264 a data reference to the tract patch and filter to be coadded (specified using 265 '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]') along with a list of 266 Warps to attempt to coadd (specified using 267 '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]'). Only the Warps 268 that cover the specified tract and patch will be coadded. A list of the available optional 269 arguments can be obtained by calling assembleCoadd.py with the --help command line argument: 271 assembleCoadd.py --help 273 To demonstrate usage of the AssembleCoaddTask in the larger context of multi-band processing, we will generate 274 the HSC-I & -R band coadds from HSC engineering test data provided in the ci_hsc package. To begin, assuming 275 that the lsst stack has been already set up, we must set up the obs_subaru and ci_hsc packages. 276 This defines the environment variable $CI_HSC_DIR and points at the location of the package. The raw HSC 277 data live in the $CI_HSC_DIR/raw directory. To begin assembling the coadds, we must first 280 <DD> process the individual ccds in $CI_HSC_RAW to produce calibrated exposures</DD> 282 <DD> create a skymap that covers the area of the sky present in the raw exposures</DD> 283 <DT>makeCoaddTempExp</DT> 284 <DD> warp the individual calibrated exposures to the tangent plane of the coadd</DD> 286 We can perform all of these steps by running 288 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988 290 This will produce warped exposures for each visit. To coadd the warped data, we call assembleCoadd.py as 293 assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \ 294 --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \ 295 --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \ 296 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \ 297 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \ 298 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \ 299 --selectId visit=903988 ccd=24 301 that will process the HSC-I band data. The results are written in 302 `$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I`. 304 You may also choose to run: 306 scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346 307 assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R \ 308 --selectId visit=903334 ccd=16 --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 \ 309 --selectId visit=903334 ccd=100 --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 \ 310 --selectId visit=903338 ccd=18 --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 \ 311 --selectId visit=903342 ccd=10 --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 \ 312 --selectId visit=903344 ccd=5 --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 \ 313 --selectId visit=903346 ccd=6 --selectId visit=903346 ccd=12 315 to generate the coadd for the HSC-R band if you are interested in following multiBand Coadd processing as 316 discussed in \ref pipeTasks_multiBand (but note that normally, one would use the 317 \ref SafeClipAssembleCoaddTask_ "SafeClipAssembleCoaddTask" rather than AssembleCoaddTask to make the coadd. 319 ConfigClass = AssembleCoaddConfig
320 _DefaultName =
"assembleCoadd" 324 \brief Initialize the task. Create the \ref InterpImageTask "interpImage", 325 \ref MatchBackgroundsTask "matchBackgrounds", & \ref ScaleZeroPointTask "scaleZeroPoint" subtasks. 327 CoaddBaseTask.__init__(self, *args, **kwargs)
328 self.makeSubtask(
"interpImage")
329 self.makeSubtask(
"matchBackgrounds")
330 self.makeSubtask(
"scaleZeroPoint")
332 if self.config.doMaskBrightObjects:
333 mask = afwImage.Mask()
336 except pexExceptions.LsstCppException:
337 raise RuntimeError(
"Unable to define mask plane for bright objects; planes used are %s" %
338 mask.getMaskPlaneDict().keys())
344 def run(self, dataRef, selectDataList=[]):
346 \brief Assemble a coadd from a set of Warps 348 Coadd a set of Warps. Compute weights to be applied to each Warp and find scalings to 349 match the photometric zeropoint to a reference Warp. Optionally, match backgrounds across 350 Warps if the background has not already been removed. Assemble the Warps using 351 \ref assemble. Interpolate over NaNs and optionally write the coadd to disk. Return the coadded 355 \param[in] dataRef: Data reference defining the patch for coaddition and the reference Warp 356 (if config.autoReference=False). Used to access the following data products: 357 - [in] self.config.coaddName + "Coadd_skyMap" 358 - [in] self.config.coaddName + "Coadd_ + <warpType> + "Warp" (optionally) 359 - [out] self.config.coaddName + "Coadd" 360 \param[in] selectDataList[in]: List of data references to Warps. Data to be coadded will be 361 selected from this list based on overlap with the patch defined by dataRef. 363 \return a pipeBase.Struct with fields: 364 - coaddExposure: coadded exposure 365 - nImage: exposure count image 368 calExpRefList = self.
selectExposures(dataRef, skyInfo, selectDataList=selectDataList)
369 if len(calExpRefList) == 0:
370 self.log.warn(
"No exposures to coadd")
372 self.log.info(
"Coadding %d exposures", len(calExpRefList))
376 self.log.info(
"Found %d %s", len(inputData.tempExpRefList),
378 if len(inputData.tempExpRefList) == 0:
379 self.log.warn(
"No coadd temporary exposures found")
381 if self.config.doMatchBackgrounds:
384 if len(inputData.tempExpRefList) == 0:
385 self.log.warn(
"No valid background models")
390 retStruct = self.
assemble(skyInfo, inputData.tempExpRefList, inputData.imageScalerList,
391 inputData.weightList,
392 inputData.backgroundInfoList
if self.config.doMatchBackgrounds
else None,
393 supplementaryData=supplementaryData)
395 if self.config.doMatchBackgrounds:
397 inputData.backgroundInfoList)
399 if self.config.doInterp:
400 self.interpImage.
run(retStruct.coaddExposure.getMaskedImage(), planeName=
"NO_DATA")
402 varArray = retStruct.coaddExposure.getMaskedImage().getVariance().getArray()
403 varArray[:] = numpy.where(varArray > 0, varArray, numpy.inf)
405 if self.config.doMaskBrightObjects:
409 if self.config.doWrite:
412 if retStruct.nImage
is not None:
419 \brief Make additional inputs to assemble() specific to subclasses. 421 Available to be implemented by subclasses only if they need the 422 coadd dataRef for performing preliminary processing before 423 assembling the coadd. 429 \brief Generate list data references corresponding to warped exposures that lie within the 432 \param[in] patchRef: Data reference for patch 433 \param[in] calExpRefList: List of data references for input calexps 434 \return List of Warp/CoaddTempExp data references 436 butler = patchRef.getButler()
437 groupData =
groupPatchExposures(patchRef, calExpRefList, self.getCoaddDatasetName(self.warpType),
438 self.getTempExpDatasetName(self.warpType))
439 tempExpRefList = [
getGroupDataRef(butler, self.getTempExpDatasetName(self.warpType),
440 g, groupData.keys)
for 441 g
in groupData.groups.keys()]
442 return tempExpRefList
446 \brief Construct an image scaler for the background reference frame 448 Each Warp has a different background level. A reference background level must be chosen before 449 coaddition. If config.autoReference=True, \ref backgroundMatching will pick the reference level and 450 this routine is a no-op and None is returned. Otherwise, use the 451 \ref ScaleZeroPointTask_ "scaleZeroPoint" subtask to compute an imageScaler object for the provided 452 reference image and return it. 454 \param[in] dataRef: Data reference for the background reference frame, or None 455 \return image scaler, or None 457 if self.config.autoReference:
462 if not dataRef.datasetExists(dataset):
463 raise RuntimeError(
"Could not find reference exposure %s %s." % (dataset, dataRef.dataId))
466 refImageScaler = self.scaleZeroPoint.computeImageScaler(
467 exposure=refExposure,
470 return refImageScaler
474 \brief Prepare the input warps for coaddition by measuring the weight for each warp and the scaling 475 for the photometric zero point. 477 Each Warp has its own photometric zeropoint and background variance. Before coadding these 478 Warps together, compute a scale factor to normalize the photometric zeropoint and compute the 479 weight for each Warp. 481 \param[in] refList: List of data references to tempExp 483 - tempExprefList: List of data references to tempExp 484 - weightList: List of weightings 485 - imageScalerList: List of image scalers 487 statsCtrl = afwMath.StatisticsControl()
488 statsCtrl.setNumSigmaClip(self.config.sigmaClip)
489 statsCtrl.setNumIter(self.config.clipIter)
491 statsCtrl.setNanSafe(
True)
499 for tempExpRef
in refList:
500 if not tempExpRef.datasetExists(tempExpName):
501 self.log.warn(
"Could not find %s %s; skipping it", tempExpName, tempExpRef.dataId)
504 tempExp = tempExpRef.get(tempExpName, immediate=
True)
505 maskedImage = tempExp.getMaskedImage()
506 imageScaler = self.scaleZeroPoint.computeImageScaler(
511 imageScaler.scaleMaskedImage(maskedImage)
512 except Exception
as e:
513 self.log.warn(
"Scaling failed for %s (skipping it): %s", tempExpRef.dataId, e)
515 statObj = afwMath.makeStatistics(maskedImage.getVariance(), maskedImage.getMask(),
516 afwMath.MEANCLIP, statsCtrl)
517 meanVar, meanVarErr = statObj.getResult(afwMath.MEANCLIP)
518 weight = 1.0 / float(meanVar)
519 if not numpy.isfinite(weight):
520 self.log.warn(
"Non-finite weight for %s: skipping", tempExpRef.dataId)
522 self.log.info(
"Weight of %s %s = %0.3f", tempExpName, tempExpRef.dataId, weight)
527 tempExpRefList.append(tempExpRef)
528 weightList.append(weight)
529 imageScalerList.append(imageScaler)
531 return pipeBase.Struct(tempExpRefList=tempExpRefList, weightList=weightList,
532 imageScalerList=imageScalerList)
536 \brief Perform background matching on the prepared inputs 538 Each Warp has a different background level that must be normalized to a reference level 539 before coaddition. If no reference is provided, the background matcher selects one. If the background 540 matching is performed sucessfully, recompute the weight to be applied to the Warp (coaddTempExp) to be 541 consistent with the scaled background. 543 \param[in] inputData: Struct from prepareInputs() with tempExpRefList, weightList, imageScalerList 544 \param[in] refExpDataRef: Data reference for background reference Warp, or None 545 \param[in] refImageScaler: Image scaler for background reference Warp, or None 547 - tempExprefList: List of data references to warped exposures (coaddTempExps) 548 - weightList: List of weightings 549 - imageScalerList: List of image scalers 550 - backgroundInfoList: result from background matching 553 backgroundInfoList = self.matchBackgrounds.
run(
554 expRefList=inputData.tempExpRefList,
555 imageScalerList=inputData.imageScalerList,
556 refExpDataRef=refExpDataRef
if not self.config.autoReference
else None,
557 refImageScaler=refImageScaler,
560 except Exception
as e:
561 self.log.fatal(
"Cannot match backgrounds: %s", e)
562 raise pipeBase.TaskError(
"Background matching failed.")
565 newTempExpRefList = []
566 newBackgroundStructList = []
570 for tempExpRef, bgInfo, scaler, weight
in zip(inputData.tempExpRefList, backgroundInfoList,
571 inputData.imageScalerList, inputData.weightList):
572 if not bgInfo.isReference:
575 if (bgInfo.backgroundModel
is None):
576 self.log.info(
"No background offset model available for %s: skipping", tempExpRef.dataId)
579 varianceRatio = bgInfo.matchedMSE / bgInfo.diffImVar
580 except Exception
as e:
581 self.log.info(
"MSE/Var ratio not calculable (%s) for %s: skipping",
582 e, tempExpRef.dataId)
584 if not numpy.isfinite(varianceRatio):
585 self.log.info(
"MSE/Var ratio not finite (%.2f / %.2f) for %s: skipping",
586 bgInfo.matchedMSE, bgInfo.diffImVar, tempExpRef.dataId)
588 elif (varianceRatio > self.config.maxMatchResidualRatio):
589 self.log.info(
"Bad fit. MSE/Var ratio %.2f > %.2f for %s: skipping",
590 varianceRatio, self.config.maxMatchResidualRatio, tempExpRef.dataId)
592 elif (bgInfo.fitRMS > self.config.maxMatchResidualRMS):
593 self.log.info(
"Bad fit. RMS %.2f > %.2f for %s: skipping",
594 bgInfo.fitRMS, self.config.maxMatchResidualRMS, tempExpRef.dataId)
596 newWeightList.append(1 / (1 / weight + bgInfo.fitRMS**2))
597 newTempExpRefList.append(tempExpRef)
598 newBackgroundStructList.append(bgInfo)
599 newScaleList.append(scaler)
601 return pipeBase.Struct(tempExpRefList=newTempExpRefList, weightList=newWeightList,
602 imageScalerList=newScaleList, backgroundInfoList=newBackgroundStructList)
604 def assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList, bgInfoList=None,
605 altMaskList=None, mask=None, supplementaryData=None):
607 \anchor AssembleCoaddTask.assemble_ 609 \brief Assemble a coadd from input warps 611 Assemble the coadd using the provided list of coaddTempExps. Since the full coadd covers a patch (a 612 large area), the assembly is performed over small areas on the image at a time in order to 613 conserve memory usage. Iterate over subregions within the outer bbox of the patch using 614 \ref assembleSubregion to stack the corresponding subregions from the coaddTempExps with the 615 statistic specified. Set the edge bits the coadd mask based on the weight map. 617 \param[in] skyInfo: Patch geometry information, from getSkyInfo 618 \param[in] tempExpRefList: List of data references to Warps (previously called CoaddTempExps) 619 \param[in] imageScalerList: List of image scalers 620 \param[in] weightList: List of weights 621 \param[in] bgInfoList: List of background data from background matching, or None 622 \param[in] altMaskList: List of alternate masks to use rather than those stored with tempExp, or None 623 \param[in] mask: Mask to ignore when coadding 624 \param[in] supplementaryData: pipeBase.Struct with additional data products needed to assemble coadd. 625 Only used by subclasses that implement makeSupplementaryData and override assemble. 626 \return pipeBase.Struct with coaddExposure, nImage if requested 629 self.log.info(
"Assembling %s %s", len(tempExpRefList), tempExpName)
633 statsCtrl = afwMath.StatisticsControl()
634 statsCtrl.setNumSigmaClip(self.config.sigmaClip)
635 statsCtrl.setNumIter(self.config.clipIter)
636 statsCtrl.setAndMask(mask)
637 statsCtrl.setNanSafe(
True)
638 statsCtrl.setWeighted(
True)
639 statsCtrl.setCalcErrorFromInputVariance(
True)
640 for plane, threshold
in self.config.maskPropagationThresholds.items():
641 bit = afwImage.Mask.getMaskPlane(plane)
642 statsCtrl.setMaskPropagationThreshold(bit, threshold)
644 statsFlags = afwMath.stringToStatisticsProperty(self.config.statistic)
646 if bgInfoList
is None:
647 bgInfoList = [
None]*len(tempExpRefList)
649 if altMaskList
is None:
650 altMaskList = [
None]*len(tempExpRefList)
652 coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
653 coaddExposure.setCalib(self.scaleZeroPoint.getCalib())
654 coaddExposure.getInfo().setCoaddInputs(self.inputRecorder.makeCoaddInputs())
656 coaddMaskedImage = coaddExposure.getMaskedImage()
657 subregionSizeArr = self.config.subregionSize
658 subregionSize = afwGeom.Extent2I(subregionSizeArr[0], subregionSizeArr[1])
660 if self.config.doNImage:
661 nImage = afwImage.ImageU(skyInfo.bbox)
664 for subBBox
in _subBBoxIter(skyInfo.bbox, subregionSize):
667 weightList, bgInfoList, altMaskList, statsFlags, statsCtrl,
669 except Exception
as e:
670 self.log.fatal(
"Cannot compute coadd %s: %s", subBBox, e)
672 coaddUtils.setCoaddEdgeBits(coaddMaskedImage.getMask(), coaddMaskedImage.getVariance())
673 return pipeBase.Struct(coaddExposure=coaddExposure, nImage=nImage)
677 \brief Set the metadata for the coadd 679 This basic implementation simply sets the filter from the 682 \param[in] coaddExposure: The target image for the coadd 683 \param[in] tempExpRefList: List of data references to tempExp 684 \param[in] weightList: List of weights 686 assert len(tempExpRefList) == len(weightList),
"Length mismatch" 691 tempExpList = [tempExpRef.get(tempExpName +
"_sub",
692 bbox=afwGeom.Box2I(afwGeom.Point2I(0, 0), afwGeom.Extent2I(1, 1)),
693 imageOrigin=
"LOCAL", immediate=
True)
for tempExpRef
in tempExpRefList]
694 numCcds = sum(len(tempExp.getInfo().getCoaddInputs().ccds)
for tempExp
in tempExpList)
696 coaddExposure.setFilter(tempExpList[0].getFilter())
697 coaddInputs = coaddExposure.getInfo().getCoaddInputs()
698 coaddInputs.ccds.reserve(numCcds)
699 coaddInputs.visits.reserve(len(tempExpList))
701 for tempExp, weight
in zip(tempExpList, weightList):
702 self.inputRecorder.addVisitToCoadd(coaddInputs, tempExp, weight)
703 coaddInputs.visits.sort()
709 modelPsfList = [tempExp.getPsf()
for tempExp
in tempExpList]
710 modelPsfWidthList = [modelPsf.computeBBox().getWidth()
for modelPsf
in modelPsfList]
711 psf = modelPsfList[modelPsfWidthList.index(max(modelPsfWidthList))]
713 psf = measAlg.CoaddPsf(coaddInputs.ccds, coaddExposure.getWcs(),
714 self.config.coaddPsf.makeControl())
715 coaddExposure.setPsf(psf)
716 apCorrMap = measAlg.makeCoaddApCorrMap(coaddInputs.ccds, coaddExposure.getBBox(afwImage.PARENT),
717 coaddExposure.getWcs())
718 coaddExposure.getInfo().setApCorrMap(apCorrMap)
720 def assembleSubregion(self, coaddExposure, bbox, tempExpRefList, imageScalerList, weightList,
721 bgInfoList, altMaskList, statsFlags, statsCtrl, nImage=None):
723 \brief Assemble the coadd for a sub-region. 725 For each coaddTempExp, check for (and swap in) an alternative mask if one is passed. If background 726 matching is enabled, add the background and background variance from each coaddTempExp. Remove mask 727 planes listed in config.removeMaskPlanes, Finally, stack the actual exposures using 728 \ref afwMath.statisticsStack "statisticsStack" with the statistic specified 729 by statsFlags. Typically, the statsFlag will be one of afwMath.MEAN for a mean-stack or 730 afwMath.MEANCLIP for outlier rejection using an N-sigma clipped mean where N and iterations 731 are specified by statsCtrl. Assign the stacked subregion back to the coadd. 733 \param[in] coaddExposure: The target image for the coadd 734 \param[in] bbox: Sub-region to coadd 735 \param[in] tempExpRefList: List of data reference to tempExp 736 \param[in] imageScalerList: List of image scalers 737 \param[in] weightList: List of weights 738 \param[in] bgInfoList: List of background data from background matching 739 \param[in] altMaskList: List of alternate masks to use rather than those stored with tempExp, or None 740 \param[in] statsFlags: afwMath.Property object for statistic for coadd 741 \param[in] statsCtrl: Statistics control object for coadd 742 \param[in] nImage: optional ImageU keeps track of exposure count for each pixel 744 self.log.debug(
"Computing coadd over %s", bbox)
746 coaddMaskedImage = coaddExposure.getMaskedImage()
748 if nImage
is not None:
749 subNImage = afwImage.ImageU(bbox.getWidth(), bbox.getHeight())
750 for tempExpRef, imageScaler, bgInfo, altMask
in zip(tempExpRefList, imageScalerList, bgInfoList,
752 exposure = tempExpRef.get(tempExpName +
"_sub", bbox=bbox)
753 maskedImage = exposure.getMaskedImage()
755 altMaskSub = altMask.Factory(altMask, bbox, afwImage.PARENT)
756 maskedImage.getMask().swap(altMaskSub)
757 imageScaler.scaleMaskedImage(maskedImage)
759 if self.config.doMatchBackgrounds
and not bgInfo.isReference:
760 backgroundModel = bgInfo.backgroundModel
761 backgroundImage = backgroundModel.getImage()
if \
762 self.matchBackgrounds.config.usePolynomial
else \
763 backgroundModel.getImageF()
764 backgroundImage.setXY0(coaddMaskedImage.getXY0())
765 maskedImage += backgroundImage.Factory(backgroundImage, bbox, afwImage.PARENT,
False)
766 var = maskedImage.getVariance()
767 var += (bgInfo.fitRMS)**2
770 if nImage
is not None:
771 subNImage.getArray()[maskedImage.getMask().getArray() & statsCtrl.getAndMask() == 0] += 1
772 if self.config.removeMaskPlanes:
773 mask = maskedImage.getMask()
774 for maskPlane
in self.config.removeMaskPlanes:
776 mask &= ~mask.getPlaneBitMask(maskPlane)
777 except Exception
as e:
778 self.log.warn(
"Unable to remove mask plane %s: %s", maskPlane, e.message)
780 maskedImageList.append(maskedImage)
782 with self.timer(
"stack"):
783 coaddSubregion = afwMath.statisticsStack(
784 maskedImageList, statsFlags, statsCtrl, weightList)
785 coaddMaskedImage.assign(coaddSubregion, bbox)
786 if nImage
is not None:
787 nImage.assign(subNImage, bbox)
791 \brief Add metadata from the background matching to the coadd 793 \param[in] coaddExposure: Coadd 794 \param[in] tempExpRefList: List of data references for temp exps to go into coadd 795 \param[in] backgroundInfoList: List of background info, results from background matching 797 self.log.info(
"Adding exposure information to metadata")
798 metadata = coaddExposure.getMetadata()
799 metadata.addString(
"CTExp_SDQA1_DESCRIPTION",
800 "Background matching: Ratio of matchedMSE / diffImVar")
801 for ind, (tempExpRef, backgroundInfo)
in enumerate(zip(tempExpRefList, backgroundInfoList)):
802 tempExpStr =
'&'.join(
'%s=%s' % (k, v)
for k, v
in tempExpRef.dataId.items())
803 if backgroundInfo.isReference:
804 metadata.addString(
"ReferenceExp_ID", tempExpStr)
806 metadata.addString(
"CTExp_ID_%d" % (ind), tempExpStr)
807 metadata.addDouble(
"CTExp_SDQA1_%d" % (ind),
808 backgroundInfo.matchedMSE/backgroundInfo.diffImVar)
809 metadata.addDouble(
"CTExp_SDQA2_%d" % (ind),
810 backgroundInfo.fitRMS)
813 """Returns None on failure""" 815 return dataRef.get(
"brightObjectMask", immediate=
True)
816 except Exception
as e:
817 self.log.warn(
"Unable to read brightObjectMask for %s: %s", dataRef.dataId, e)
821 """Set the bright object masks 823 exposure: Exposure under consideration 824 dataId: Data identifier dict for patch 825 brightObjectMasks: afwTable of bright objects to mask 830 if brightObjectMasks
is None:
831 self.log.warn(
"Unable to apply bright object mask: none supplied")
833 self.log.info(
"Applying %d bright object masks to %s", len(brightObjectMasks), dataId)
834 md = brightObjectMasks.table.getMetadata()
837 self.log.warn(
"Expected to see %s in metadata", k)
839 if md.get(k) != dataId[k]:
840 self.log.warn(
"Expected to see %s == %s in metadata, saw %s", k, md.get(k), dataId[k])
842 mask = exposure.getMaskedImage().getMask()
843 wcs = exposure.getWcs()
844 plateScale = wcs.pixelScale().asArcseconds()
846 for rec
in brightObjectMasks:
847 center = afwGeom.PointI(wcs.skyToPixel(rec.getCoord()))
848 if rec[
"type"] ==
"box":
849 assert rec[
"angle"] == 0.0, (
"Angle != 0 for mask object %s" % rec[
"id"])
850 width = rec[
"width"].asArcseconds()/plateScale
851 height = rec[
"height"].asArcseconds()/plateScale
853 halfSize = afwGeom.ExtentI(0.5*width, 0.5*height)
854 bbox = afwGeom.Box2I(center - halfSize, center + halfSize)
856 bbox = afwGeom.BoxI(afwGeom.PointI(int(center[0] - 0.5*width), int(center[1] - 0.5*height)),
857 afwGeom.PointI(int(center[0] + 0.5*width), int(center[1] + 0.5*height)))
858 spans = afwGeom.SpanSet(bbox)
859 elif rec[
"type"] ==
"circle":
860 radius = int(rec[
"radius"].asArcseconds()/plateScale)
861 spans = afwGeom.SpanSet.fromShape(radius, offset=center)
863 self.log.warn(
"Unexpected region type %s at %s" % rec[
"type"], center)
868 def _makeArgumentParser(cls):
870 \brief Create an argument parser 873 parser.add_id_argument(
"--id", cls.
ConfigClass().coaddName +
"Coadd_" +
875 help=
"data ID, e.g. --id tract=12345 patch=1,2",
876 ContainerClass=AssembleCoaddDataIdContainer)
877 parser.add_id_argument(
"--selectId",
"calexp", help=
"data ID, e.g. --selectId visit=6789 ccd=0..9",
878 ContainerClass=SelectDataIdContainer)
882 def _subBBoxIter(bbox, subregionSize):
884 \brief Iterate over subregions of a bbox 886 \param[in] bbox: bounding box over which to iterate: afwGeom.Box2I 887 \param[in] subregionSize: size of sub-bboxes 889 \return subBBox: next sub-bounding box of size subregionSize or smaller; 890 each subBBox is contained within bbox, so it may be smaller than subregionSize at the edges of bbox, 891 but it will never be empty 894 raise RuntimeError(
"bbox %s is empty" % (bbox,))
895 if subregionSize[0] < 1
or subregionSize[1] < 1:
896 raise RuntimeError(
"subregionSize %s must be nonzero" % (subregionSize,))
898 for rowShift
in range(0, bbox.getHeight(), subregionSize[1]):
899 for colShift
in range(0, bbox.getWidth(), subregionSize[0]):
900 subBBox = afwGeom.Box2I(bbox.getMin() + afwGeom.Extent2I(colShift, rowShift), subregionSize)
902 if subBBox.isEmpty():
903 raise RuntimeError(
"Bug: empty bbox! bbox=%s, subregionSize=%s, colShift=%s, rowShift=%s" %
904 (bbox, subregionSize, colShift, rowShift))
910 \brief A version of lsst.pipe.base.DataIdContainer specialized for assembleCoadd. 915 \brief Make self.refList from self.idList. 917 Interpret the config.doMatchBackgrounds, config.autoReference, 918 and whether a visit/run supplied. 919 If a visit/run is supplied, config.autoReference is automatically set to False. 920 if config.doMatchBackgrounds == false, then a visit/run will be ignored if accidentally supplied. 923 keysCoadd = namespace.butler.getKeys(datasetType=namespace.config.coaddName +
"Coadd",
925 keysCoaddTempExp = namespace.butler.getKeys(datasetType=namespace.config.coaddName +
926 "Coadd_directWarp", level=self.level)
928 if namespace.config.doMatchBackgrounds:
929 if namespace.config.autoReference:
930 datasetType = namespace.config.coaddName +
"Coadd" 931 validKeys = keysCoadd
933 datasetType = namespace.config.coaddName +
"Coadd_directWarp" 934 validKeys = keysCoaddTempExp
936 datasetType = namespace.config.coaddName +
"Coadd" 937 validKeys = keysCoadd
939 for dataId
in self.idList:
941 for key
in validKeys:
942 if key
not in dataId:
943 raise RuntimeError(
"--id must include " + key)
946 if (key
not in keysCoadd)
and (key
in keysCoaddTempExp):
947 if namespace.config.autoReference:
949 namespace.config.autoReference =
False 950 datasetType = namespace.config.coaddName +
"Coadd_directWarp" 951 print(
"Switching config.autoReference to False; applies only to background Matching.")
954 dataRef = namespace.butler.dataRef(
955 datasetType=datasetType,
958 self.refList.append(dataRef)
963 \brief Function to count the number of pixels with a specific mask in a footprint. 965 Find the intersection of mask & footprint. Count all pixels in the mask that are in the intersection that 966 have bitmask set but do not have ignoreMask set. Return the count. 968 \param[in] mask: mask to define intersection region by. 969 \parma[in] footprint: footprint to define the intersection region by. 970 \param[in] bitmask: specific mask that we wish to count the number of occurances of. 971 \param[in] ignoreMask: pixels to not consider. 972 \return count of number of pixels in footprint with specified mask. 974 bbox = footprint.getBBox()
975 bbox.clip(mask.getBBox(afwImage.PARENT))
976 fp = afwImage.Mask(bbox)
977 subMask = mask.Factory(mask, bbox, afwImage.PARENT)
978 footprint.spans.setMask(fp, bitmask)
979 return numpy.logical_and((subMask.getArray() & fp.getArray()) > 0,
980 (subMask.getArray() & ignoreMask) == 0).sum()
985 \anchor SafeClipAssembleCoaddConfig 987 \brief Configuration parameters for the SafeClipAssembleCoaddTask 989 clipDetection = pexConfig.ConfigurableField(
990 target=SourceDetectionTask,
991 doc=
"Detect sources on difference between unclipped and clipped coadd")
992 minClipFootOverlap = pexConfig.Field(
993 doc=
"Minimum fractional overlap of clipped footprint with visit DETECTED to be clipped",
997 minClipFootOverlapSingle = pexConfig.Field(
998 doc=
"Minimum fractional overlap of clipped footprint with visit DETECTED to be " 999 "clipped when only one visit overlaps",
1003 minClipFootOverlapDouble = pexConfig.Field(
1004 doc=
"Minimum fractional overlap of clipped footprints with visit DETECTED to be " 1005 "clipped when two visits overlap",
1009 maxClipFootOverlapDouble = pexConfig.Field(
1010 doc=
"Maximum fractional overlap of clipped footprints with visit DETECTED when " 1011 "considering two visits",
1015 minBigOverlap = pexConfig.Field(
1016 doc=
"Minimum number of pixels in footprint to use DETECTED mask from the single visits " 1017 "when labeling clipped footprints",
1025 AssembleCoaddConfig.setDefaults(self)
1041 log.warn(
"Additional Sigma-clipping not allowed in Safe-clipped Coadds. " 1042 "Ignoring doSigmaClip.")
1045 raise ValueError(
"Only MEAN statistic allowed for final stacking in SafeClipAssembleCoadd " 1046 "(%s chosen). Please set statistic to MEAN." 1048 AssembleCoaddTask.ConfigClass.validate(self)
1061 \anchor SafeClipAssembleCoaddTask_ 1063 \brief Assemble a coadded image from a set of coadded temporary exposures, 1064 being careful to clip & flag areas with potential artifacts. 1066 \section pipe_tasks_assembleCoadd_Contents Contents 1067 - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Purpose 1068 - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Initialize 1069 - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Run 1070 - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Config 1071 - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Debug 1072 - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Example 1074 \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Purpose Description 1076 \copybrief SafeClipAssembleCoaddTask 1078 Read the documentation for \ref AssembleCoaddTask_ "AssembleCoaddTask" first since 1079 SafeClipAssembleCoaddTask subtasks that task. 1080 In \ref AssembleCoaddTask_ "AssembleCoaddTask", we compute the coadd as an clipped mean (i.e. we clip 1082 The problem with doing this is that when computing the coadd PSF at a given location, individual visit 1083 PSFs from visits with outlier pixels contribute to the coadd PSF and cannot be treated correctly. 1084 In this task, we correct for this behavior by creating a new badMaskPlane 'CLIPPED'. 1085 We populate this plane on the input coaddTempExps and the final coadd where i. difference imaging suggests 1086 that there is an outlier and ii. this outlier appears on only one or two images. 1087 Such regions will not contribute to the final coadd. 1088 Furthermore, any routine to determine the coadd PSF can now be cognizant of clipped regions. 1089 Note that the algorithm implemented by this task is preliminary and works correctly for HSC data. 1090 Parameter modifications and or considerable redesigning of the algorithm is likley required for other 1093 SafeClipAssembleCoaddTask uses a \ref SourceDetectionTask_ "clipDetection" subtask and also sub-classes 1094 \ref AssembleCoaddTask_ "AssembleCoaddTask". You can retarget the 1095 \ref SourceDetectionTask_ "clipDetection" subtask if you wish. 1097 \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Initialize Task initialization 1098 \copydoc \_\_init\_\_ 1100 \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Run Invoking the Task 1103 \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Config Configuration parameters 1104 See \ref SafeClipAssembleCoaddConfig 1106 \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Debug Debug variables 1107 The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a 1108 flag \c -d to import \b debug.py from your \c PYTHONPATH; see \ref baseDebug for more about \b debug.py 1110 SafeClipAssembleCoaddTask has no debug variables of its own. The \ref SourceDetectionTask_ "clipDetection" 1111 subtasks may support debug variables. See the documetation for \ref SourceDetectionTask_ "clipDetection" 1112 for further information. 1114 \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Example A complete example of using 1115 SafeClipAssembleCoaddTask 1117 SafeClipAssembleCoaddTask assembles a set of warped coaddTempExp images into a coadded image. 1118 The SafeClipAssembleCoaddTask is invoked by running assembleCoadd.py <em>without</em> the flag 1120 Usage of assembleCoadd.py expects a data reference to the tract patch and filter to be coadded 1121 (specified using '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]') along 1122 with a list of coaddTempExps to attempt to coadd (specified using 1123 '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]'). 1124 Only the coaddTempExps that cover the specified tract and patch will be coadded. 1125 A list of the available optional arguments can be obtained by calling assembleCoadd.py with the --help 1126 command line argument: 1128 assembleCoadd.py --help 1130 To demonstrate usage of the SafeClipAssembleCoaddTask in the larger context of multi-band processing, we 1131 will generate the HSC-I & -R band coadds from HSC engineering test data provided in the ci_hsc package. To 1132 begin, assuming that the lsst stack has been already set up, we must set up the obs_subaru and ci_hsc 1134 This defines the environment variable $CI_HSC_DIR and points at the location of the package. The raw HSC 1135 data live in the $CI_HSC_DIR/raw directory. To begin assembling the coadds, we must first 1138 <DD> process the individual ccds in $CI_HSC_RAW to produce calibrated exposures</DD> 1140 <DD> create a skymap that covers the area of the sky present in the raw exposures</DD> 1141 <DT>makeCoaddTempExp</DT> 1142 <DD> warp the individual calibrated exposures to the tangent plane of the coadd</DD> 1144 We can perform all of these steps by running 1146 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988 1148 This will produce warped coaddTempExps for each visit. To coadd the warped data, we call assembleCoadd.py 1151 assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \ 1152 --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \ 1153 --selectId visit=903986 ccd=100--selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \ 1154 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \ 1155 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \ 1156 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \ 1157 --selectId visit=903988 ccd=24 1159 This will process the HSC-I band data. The results are written in 1160 `$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I`. 1162 You may also choose to run: 1164 scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346 1165 assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R --selectId visit=903334 ccd=16 \ 1166 --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 --selectId visit=903334 ccd=100 \ 1167 --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 --selectId visit=903338 ccd=18 \ 1168 --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 --selectId visit=903342 ccd=10 \ 1169 --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 --selectId visit=903344 ccd=5 \ 1170 --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 --selectId visit=903346 ccd=6 \ 1171 --selectId visit=903346 ccd=12 1173 to generate the coadd for the HSC-R band if you are interested in following multiBand Coadd processing as 1174 discussed in \ref pipeTasks_multiBand. 1176 ConfigClass = SafeClipAssembleCoaddConfig
1177 _DefaultName =
"safeClipAssembleCoadd" 1181 \brief Initialize the task and make the \ref SourceDetectionTask_ "clipDetection" subtask. 1183 AssembleCoaddTask.__init__(self, *args, **kwargs)
1184 schema = afwTable.SourceTable.makeMinimalSchema()
1185 self.makeSubtask(
"clipDetection", schema=schema)
1187 def assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList, bgModelList,
1190 \brief Assemble the coadd for a region 1192 Compute the difference of coadds created with and without outlier rejection to identify coadd pixels 1193 that have outlier values in some individual visits. Detect clipped regions on the difference image and 1194 mark these regions on the one or two individual coaddTempExps where they occur if there is significant 1195 overlap between the clipped region and a source. 1196 This leaves us with a set of footprints from the difference image that have been identified as having 1197 occured on just one or two individual visits. However, these footprints were generated from a 1198 difference image. It is conceivable for a large diffuse source to have become broken up into multiple 1199 footprints acrosss the coadd difference in this process. 1200 Determine the clipped region from all overlapping footprints from the detected sources in each visit - 1201 these are big footprints. 1202 Combine the small and big clipped footprints and mark them on a new bad mask plane 1203 Generate the coadd using \ref AssembleCoaddTask.assemble_ "AssembleCoaddTask.assemble" without outlier 1204 removal. Clipped footprints will no longer make it into the coadd because they are marked in the new 1207 N.b. *args and **kwargs are passed but ignored in order to match the call signature expected by the 1210 @param skyInfo: Patch geometry information, from getSkyInfo 1211 @param tempExpRefList: List of data reference to tempExp 1212 @param imageScalerList: List of image scalers 1213 @param weightList: List of weights 1214 @param bgModelList: List of background models from background matching 1215 return pipeBase.Struct with coaddExposure, nImage 1217 exp = self.
buildDifferenceImage(skyInfo, tempExpRefList, imageScalerList, weightList, bgModelList)
1218 mask = exp.getMaskedImage().getMask()
1219 mask.addMaskPlane(
"CLIPPED")
1221 result = self.
detectClip(exp, tempExpRefList)
1223 self.log.info(
'Found %d clipped objects', len(result.clipFootprints))
1226 maskClipValue = mask.getPlaneBitMask(
"CLIPPED")
1227 maskDetValue = mask.getPlaneBitMask(
"DETECTED") | mask.getPlaneBitMask(
"DETECTED_NEGATIVE")
1228 bigFootprints = self.
detectClipBig(result.tempExpClipList, result.clipFootprints, result.clipIndices,
1229 maskClipValue, maskDetValue)
1232 maskClip = mask.Factory(mask.getBBox(afwImage.PARENT))
1233 afwDet.setMaskFromFootprintList(maskClip, result.clipFootprints, maskClipValue)
1235 maskClipBig = maskClip.Factory(mask.getBBox(afwImage.PARENT))
1236 afwDet.setMaskFromFootprintList(maskClipBig, bigFootprints, maskClipValue)
1237 maskClip |= maskClipBig
1240 badMaskPlanes = self.config.badMaskPlanes[:]
1241 badMaskPlanes.append(
"CLIPPED")
1242 badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes)
1243 retStruct = AssembleCoaddTask.assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList,
1244 bgModelList, result.tempExpClipList, mask=badPixelMask)
1248 maskExp = retStruct.coaddExposure.getMaskedImage().getMask()
1255 \brief Return an exposure that contains the difference between and unclipped and clipped coadds. 1257 Generate a difference image between clipped and unclipped coadds. 1258 Compute the difference image by subtracting an outlier-clipped coadd from an outlier-unclipped coadd. 1259 Return the difference image. 1261 @param skyInfo: Patch geometry information, from getSkyInfo 1262 @param tempExpRefList: List of data reference to tempExp 1263 @param imageScalerList: List of image scalers 1264 @param weightList: List of weights 1265 @param bgModelList: List of background models from background matching 1266 @return Difference image of unclipped and clipped coadd wrapped in an Exposure 1271 configIntersection = {k: getattr(self.config, k)
1272 for k, v
in self.config.toDict().items()
if (k
in config.keys())}
1273 config.update(**configIntersection)
1276 config.statistic =
'MEAN' 1278 coaddMean = task.assemble(skyInfo, tempExpRefList, imageScalerList, weightList,
1279 bgModelList).coaddExposure
1281 config.statistic =
'MEANCLIP' 1283 coaddClip = task.assemble(skyInfo, tempExpRefList, imageScalerList, weightList,
1284 bgModelList).coaddExposure
1286 coaddDiff = coaddMean.getMaskedImage().Factory(coaddMean.getMaskedImage())
1287 coaddDiff -= coaddClip.getMaskedImage()
1288 exp = afwImage.ExposureF(coaddDiff)
1289 exp.setPsf(coaddMean.getPsf())
1294 \brief Detect clipped regions on an exposure and set the mask on the individual tempExp masks 1296 Detect footprints in the difference image after smoothing the difference image with a Gaussian kernal. 1297 Identify footprints that overlap with one or two input coaddTempExps by comparing the computed overlap 1298 fraction to thresholds set in the config. 1299 A different threshold is applied depending on the number of overlapping visits (restricted to one or 1301 If the overlap exceeds the thresholds, the footprint is considered "CLIPPED" and is marked as such on 1303 Return a struct with the clipped footprints, the indices of the coaddTempExps that end up overlapping 1304 with the clipped footprints and a list of new masks for the coaddTempExps. 1306 \param[in] exp: Exposure to run detection on 1307 \param[in] tempExpRefList: List of data reference to tempExp 1308 \return struct containing: 1309 - clippedFootprints: list of clipped footprints 1310 - clippedIndices: indices for each clippedFootprint in tempExpRefList 1311 - tempExpClipList: list of new masks for tempExp 1313 mask = exp.getMaskedImage().getMask()
1314 maskClipValue = mask.getPlaneBitMask(
"CLIPPED")
1315 maskDetValue = mask.getPlaneBitMask(
"DETECTED") | mask.getPlaneBitMask(
"DETECTED_NEGATIVE")
1316 fpSet = self.clipDetection.detectFootprints(exp, doSmooth=
True, clearMask=
True)
1318 fpSet.positive.merge(fpSet.negative)
1319 footprints = fpSet.positive
1320 self.log.info(
'Found %d potential clipped objects', len(footprints.getFootprints()))
1328 immediate=
True).getMaskedImage().getMask()
for 1329 tmpExpRef
in tempExpRefList]
1331 for footprint
in footprints.getFootprints():
1332 nPixel = footprint.getArea()
1336 for i, tmpExpMask
in enumerate(tempExpClipList):
1340 totPixel = nPixel - ignore
1343 if ignore > overlapDet
or totPixel <= 0.5*nPixel
or overlapDet == 0:
1345 overlap.append(overlapDet/float(totPixel))
1346 maskList.append(tmpExpMask)
1349 overlap = numpy.array(overlap)
1350 if not len(overlap):
1357 if len(overlap) == 1:
1358 if overlap[0] > self.config.minClipFootOverlapSingle:
1363 clipIndex = numpy.where(overlap > self.config.minClipFootOverlap)[0]
1364 if len(clipIndex) == 1:
1366 keepIndex = [clipIndex[0]]
1369 clipIndex = numpy.where(overlap > self.config.minClipFootOverlapDouble)[0]
1370 if len(clipIndex) == 2
and len(overlap) > 3:
1371 clipIndexComp = numpy.where(overlap <= self.config.minClipFootOverlapDouble)[0]
1372 if numpy.max(overlap[clipIndexComp]) <= self.config.maxClipFootOverlapDouble:
1374 keepIndex = clipIndex
1379 for index
in keepIndex:
1380 footprint.spans.setMask(maskList[index], maskClipValue)
1382 clipIndices.append(numpy.array(indexList)[keepIndex])
1383 clipFootprints.append(footprint)
1385 return pipeBase.Struct(clipFootprints=clipFootprints, clipIndices=clipIndices,
1386 tempExpClipList=tempExpClipList)
1388 def detectClipBig(self, tempExpClipList, clipFootprints, clipIndices, maskClipValue, maskDetValue):
1390 \brief Find footprints from individual tempExp footprints for large footprints. 1392 Identify big footprints composed of many sources in the coadd difference that may have originated in a 1393 large diffuse source in the coadd. We do this by indentifying all clipped footprints that overlap 1394 significantly with each source in all the coaddTempExps. 1396 \param[in] tempExpClipList: List of tempExp masks with clipping information 1397 \param[in] clipFootprints: List of clipped footprints 1398 \param[in] clipIndices: List of which entries in tempExpClipList each footprint belongs to 1399 \param[in] maskClipValue: Mask value of clipped pixels 1400 \param[in] maskClipValue: Mask value of detected pixels 1401 \return list of big footprints 1403 bigFootprintsCoadd = []
1405 for index, tmpExpMask
in enumerate(tempExpClipList):
1408 maskVisitDet = tmpExpMask.Factory(tmpExpMask, tmpExpMask.getBBox(afwImage.PARENT),
1409 afwImage.PARENT,
True)
1410 maskVisitDet &= maskDetValue
1411 visitFootprints = afwDet.FootprintSet(maskVisitDet, afwDet.Threshold(1))
1414 clippedFootprintsVisit = []
1415 for foot, clipIndex
in zip(clipFootprints, clipIndices):
1416 if index
not in clipIndex:
1418 clippedFootprintsVisit.append(foot)
1419 maskVisitClip = maskVisitDet.Factory(maskVisitDet.getBBox(afwImage.PARENT))
1420 afwDet.setMaskFromFootprintList(maskVisitClip, clippedFootprintsVisit, maskClipValue)
1422 bigFootprintsVisit = []
1423 for foot
in visitFootprints.getFootprints():
1424 if foot.getArea() < self.config.minBigOverlap:
1427 if nCount > self.config.minBigOverlap:
1428 bigFootprintsVisit.append(foot)
1429 bigFootprintsCoadd.append(foot)
1432 maskVisitClip.clearAllMaskPlanes()
1433 afwDet.setMaskFromFootprintList(maskVisitClip, bigFootprintsVisit, maskClipValue)
1434 tmpExpMask |= maskVisitClip
1436 return bigFootprintsCoadd
1440 assembleStaticSkyModel = pexConfig.ConfigurableField(
1441 target=AssembleCoaddTask,
1442 doc=
"Task to assemble an artifact-free, PSF-matched Coadd to serve as a" 1443 " naive/first-iteration model of the static sky.",
1445 detect = pexConfig.ConfigurableField(
1446 target=SourceDetectionTask,
1447 doc=
"Detect outlier sources on difference between each psfMatched warp and static sky model" 1449 temporalThreshold = pexConfig.Field(
1450 doc=
"Unitless fraction of number of epochs to classify as an artifact/outlier source versus" 1451 " a source that is intrinsically variable or difficult to subtract cleanly. " 1452 "If outlier region in warp-diff Chi-image is mostly (defined by spatialThreshold) " 1453 "an outlier in less than temporalThreshold * number of epochs, then mask. " 1454 "Otherwise, do not mask.",
1458 spatialThreshold = pexConfig.Field(
1459 doc=
"Unitless fraction of pixels defining how much of the outlier region has to meet the " 1460 "temporal criteria",
1466 AssembleCoaddConfig.setDefaults(self)
1471 self.
detect.doTempLocalBackground =
False 1472 self.
detect.reEstimateBackground =
False 1473 self.
detect.returnOriginalFootprints =
False 1474 self.
detect.thresholdPolarity =
"both" 1475 self.
detect.thresholdValue = 5
1476 self.
detect.nSigmaToGrow = 2
1477 self.
detect.minPixels = 4
1478 self.
detect.isotropicGrow =
True 1479 self.
detect.thresholdType =
"pixel_stdev" 1491 \anchor CompareWarpAssembleCoaddTask_ 1493 \brief Assemble a compareWarp coadded image from a set of warps 1494 by masking artifacts detected by comparing PSF-matched warps 1496 \section pipe_tasks_assembleCoadd_Contents Contents 1497 - \ref pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Purpose 1498 - \ref pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Initialize 1499 - \ref pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Run 1500 - \ref pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Config 1501 - \ref pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Debug 1502 - \ref pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Example 1504 \section pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Purpose Description 1506 \copybrief CompareWarpAssembleCoaddTask 1508 In \ref AssembleCoaddTask_ "AssembleCoaddTask", we compute the coadd as an clipped mean (i.e. we clip 1510 The problem with doing this is that when computing the coadd PSF at a given location, individual visit 1511 PSFs from visits with outlier pixels contribute to the coadd PSF and cannot be treated correctly. 1512 In this task, we correct for this behavior by creating a new badMaskPlane 'CLIPPED' which marks 1513 pixels in the individual warps suspected to contain an artifact. 1514 We populate this plane on the input warps by comparing PSF-matched warps with a PSF-matched median coadd 1515 which serves as a model of the static sky. Any group of pixels that deviates from the PSF-matched 1516 template coadd by more than config.detect.threshold sigma, is an artifact candidate. 1517 The candidates are then filtered to remove variable sources and sources that are difficult to subtract 1518 such as bright stars. 1519 This filter is configured using the config parameters temporalThreshold and spatialThreshold. 1520 The temporalThreshold is the maximum fraction of epochs that the deviation can 1521 appear in and still be considered an artifact. The spatialThreshold is the maximum fraction of pixels in 1522 the footprint of the deviation that appear in other epochs (where other epochs is defined by the 1523 temporalThreshold). If the deviant region meets this criteria of having a significant percentage of pixels 1524 that deviate in only a few epochs, these pixels have the 'CLIPPED' bit set in the mask. 1525 These regions will not contribute to the final coadd. 1526 Furthermore, any routine to determine the coadd PSF can now be cognizant of clipped regions. 1527 Note that the algorithm implemented by this task is preliminary and works correctly for HSC data. 1528 Parameter modifications and or considerable redesigning of the algorithm is likley required for other 1531 CompareWarpAssembleCoaddTask sub-classes 1532 \ref AssembleCoaddTask_ "AssembleCoaddTask" and instantiates \ref AssembleCoaddTask_ "AssembleCoaddTask" 1533 as a subtask to generate the TemplateCoadd (the model of the static sky) 1535 \section pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Initialize Task initialization 1536 \copydoc \_\_init\_\_ 1538 \section pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Run Invoking the Task 1541 \section pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Config Configuration parameters 1542 See \ref CompareWarpAssembleCoaddConfig 1544 \section pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Debug Debug variables 1545 The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a 1546 flag \c -d to import \b debug.py from your \c PYTHONPATH; see \ref baseDebug for more about \b debug.py 1549 This task supports the following debug variables: 1552 <dd> If True then save the Epoch Count Image as a fits file in the `figPath` 1554 <dd> If True then save the new masks with CLIPPED planes as fits files to the `figPath` 1556 <dd> Path to save the debug fits images and figures 1559 For example, put something like: 1562 def DebugInfo(name): 1563 di = lsstDebug.getInfo(name) 1564 if name == "lsst.pipe.tasks.assembleCoadd": 1565 di.saveCountIm = True 1566 di.saveAltMask = True 1567 di.figPath = "/desired/path/to/debugging/output/images" 1569 lsstDebug.Info = DebugInfo 1571 into your `debug.py` file and run `assemebleCoadd.py` with the `--debug` 1573 Some subtasks may have their own debug variables; see individual Task 1576 \section pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Example A complete example of using 1577 CompareWarpAssembleCoaddTask 1579 CompareWarpAssembleCoaddTask assembles a set of warped images into a coadded image. 1580 The CompareWarpAssembleCoaddTask is invoked by running assembleCoadd.py with the flag 1581 '--compareWarpCoadd'. 1582 Usage of assembleCoadd.py expects a data reference to the tract patch and filter to be coadded 1583 (specified using '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]') along 1584 with a list of coaddTempExps to attempt to coadd (specified using 1585 '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]'). 1586 Only the warps that cover the specified tract and patch will be coadded. 1587 A list of the available optional arguments can be obtained by calling assembleCoadd.py with the --help 1588 command line argument: 1590 assembleCoadd.py --help 1592 To demonstrate usage of the CompareWarpAssembleCoaddTask in the larger context of multi-band processing, 1593 we will generate the HSC-I & -R band coadds from HSC engineering test data provided in the ci_hsc package. 1594 To begin, assuming that the lsst stack has been already set up, we must set up the obs_subaru and ci_hsc 1596 This defines the environment variable $CI_HSC_DIR and points at the location of the package. The raw HSC 1597 data live in the $CI_HSC_DIR/raw directory. To begin assembling the coadds, we must first 1600 <DD> process the individual ccds in $CI_HSC_RAW to produce calibrated exposures</DD> 1602 <DD> create a skymap that covers the area of the sky present in the raw exposures</DD> 1603 <DT>makeCoaddTempExp</DT> 1604 <DD> warp the individual calibrated exposures to the tangent plane of the coadd</DD> 1606 We can perform all of these steps by running 1608 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988 1610 This will produce warped coaddTempExps for each visit. To coadd the warped data, we call assembleCoadd.py 1613 assembleCoadd.py --compareWarpCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \ 1614 --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \ 1615 --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \ 1616 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \ 1617 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \ 1618 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \ 1619 --selectId visit=903988 ccd=24 1621 This will process the HSC-I band data. The results are written in 1622 `$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I`. 1624 ConfigClass = CompareWarpAssembleCoaddConfig
1625 _DefaultName =
"compareWarpAssembleCoadd" 1629 \brief Initialize the task and make the \ref AssembleCoadd_ "assembleStaticSkyModel" subtask. 1631 AssembleCoaddTask.__init__(self, *args, **kwargs)
1632 self.makeSubtask(
"assembleStaticSkyModel")
1633 detectionSchema = afwTable.SourceTable.makeMinimalSchema()
1634 self.makeSubtask(
"detect", schema=detectionSchema)
1638 \brief Make inputs specific to Subclass 1640 Generate a templateCoadd to use as a native model of static sky to subtract from warps. 1642 templateCoadd = self.assembleStaticSkyModel.
run(dataRef, selectDataList).coaddExposure
1643 return pipeBase.Struct(templateCoadd=templateCoadd)
1645 def assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList, bgModelList,
1646 supplementaryData, *args, **kwargs):
1648 \brief Assemble the coadd 1650 Requires additional inputs Struct `supplementaryData` to contain a `templateCoadd` that serves 1651 as the model of the static sky. 1653 Find artifacts and apply them to the warps' masks creating a list of alternative masks with a 1654 new "CLIPPED" plane and updated "NO_DATA" plane. 1655 Then pass these alternative masks to the base class's assemble method. 1657 @param skyInfo: Patch geometry information 1658 @param tempExpRefList: List of data references to warps 1659 @param imageScalerList: List of image scalers 1660 @param weightList: List of weights 1661 @param bgModelList: List of background models from background matching 1662 @param supplementaryData: PipeBase.Struct containing a templateCoadd 1664 return pipeBase.Struct with coaddExposure, nImage if requested 1666 templateCoadd = supplementaryData.templateCoadd
1667 spanSetMaskList = self.
findArtifacts(templateCoadd, tempExpRefList, imageScalerList)
1669 badMaskPlanes = self.config.badMaskPlanes[:]
1670 badMaskPlanes.append(
"CLIPPED")
1671 badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes)
1673 retStruct = AssembleCoaddTask.assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList,
1674 bgModelList, maskList, mask=badPixelMask)
1678 mask = retStruct.coaddExposure.maskedImage.mask
1679 for maskClip
in maskList:
1680 maskClip &= mask.getPlaneBitMask(
"CLIPPED")
1687 \brief Find artifacts 1689 Loop through warps twice. The first loop builds a map with the count of how many 1690 epochs each pixel deviates from the templateCoadd by more than config.chiThreshold sigma. 1691 The second loop takes each difference image and filters the artifacts detected 1692 in each using count map to filter out variable sources and sources that are difficult to 1695 @param templateCoadd: Exposure to serve as model of static sky 1696 @param tempExpRefList: List of data references to warps 1697 @param imageScalerList: List of image scalers 1700 self.log.debug(
"Generating Count Image, and mask lists.")
1701 coaddBBox = templateCoadd.getBBox()
1702 slateIm = afwImage.ImageU(coaddBBox)
1703 epochCountImage = afwImage.ImageU(coaddBBox)
1704 spanSetArtifactList = []
1705 spanSetNoDataMaskList = []
1707 maxNumEpochs = int(max(1, self.config.temporalThreshold*len(tempExpRefList)))
1708 for warpRef, imageScaler
in zip(tempExpRefList, imageScalerList):
1710 if warpDiffExp
is not None:
1711 fpSet = self.detect.detectFootprints(warpDiffExp, doSmooth=
False, clearMask=
True)
1712 fpSet.positive.merge(fpSet.negative)
1713 footprints = fpSet.positive
1715 spanSetList = [footprint.spans
for footprint
in footprints.getFootprints()]
1716 for spans
in spanSetList:
1717 spans.setImage(slateIm, 1, doClip=
True)
1718 epochCountImage += slateIm
1724 nans = numpy.where(numpy.isnan(warpDiffExp.maskedImage.image.array), 1, 0)
1725 nansMask = afwImage.makeMaskFromArray(nans.astype(afwImage.MaskPixel))
1726 nansMask.setXY0(warpDiffExp.getXY0())
1730 nansMask = afwImage.MaskX(coaddBBox, 1)
1733 spanSetNoDataMask = afwGeom.SpanSet.fromMask(nansMask).split()
1735 spanSetNoDataMaskList.append(spanSetNoDataMask)
1736 spanSetArtifactList.append(spanSetList)
1740 epochCountImage.writeFits(path)
1742 for i, spanSetList
in enumerate(spanSetArtifactList):
1745 maxNumEpochs=maxNumEpochs)
1746 spanSetArtifactList[i] = filteredSpanSetList
1748 return pipeBase.Struct(artifacts=spanSetArtifactList,
1749 noData=spanSetNoDataMaskList)
1753 \brief Apply artifact span set lists to masks 1755 @param tempExpRefList: List of data references to warps 1756 @param maskSpanSets: Struct containing artifact and noData spanSet lists to apply 1758 return List of alternative masks 1760 Add artifact span set list as "CLIPPED" plane and NaNs to existing "NO_DATA" plane 1762 spanSetMaskList = maskSpanSets.artifacts
1763 spanSetNoDataList = maskSpanSets.noData
1765 for warpRef, artifacts, noData
in zip(tempExpRefList, spanSetMaskList, spanSetNoDataList):
1767 mask = warp.maskedImage.mask
1768 maskClipValue = mask.addMaskPlane(
"CLIPPED")
1769 noDataValue = mask.addMaskPlane(
"NO_DATA")
1770 for artifact
in artifacts:
1771 artifact.clippedTo(mask.getBBox()).setMask(mask, 2**maskClipValue)
1772 for noDataRegion
in noData:
1773 noDataRegion.clippedTo(mask.getBBox()).setMask(mask, 2**noDataValue)
1774 altMaskList.append(mask)
1780 def _filterArtifacts(self, spanSetList, epochCountImage, maxNumEpochs=None):
1781 maskSpanSetList = []
1782 x0, y0 = epochCountImage.getXY0()
1783 for i, span
in enumerate(spanSetList):
1784 y, x = span.indices()
1785 counts = epochCountImage.array[[y1 - y0
for y1
in y], [x1 - x0
for x1
in x]]
1786 idx = numpy.where((counts > 0) & (counts <= maxNumEpochs))
1787 percentBelowThreshold = len(idx[0]) / len(counts)
1788 if percentBelowThreshold > self.config.spatialThreshold:
1789 maskSpanSetList.append(span)
1790 return maskSpanSetList
1792 def _readAndComputeWarpDiff(self, warpRef, imageScaler, templateCoadd):
1795 if not warpRef.datasetExists(warpName):
1796 self.log.warn(
"Could not find %s %s; skipping it", warpName, warpRef.dataId)
1800 imageScaler.scaleMaskedImage(warp.getMaskedImage())
1801 mi = warp.getMaskedImage()
1802 mi -= templateCoadd.getMaskedImage()
1805 def _dataRef2DebugPath(self, prefix, warpRef, coaddLevel=False):
1807 \brief Return a path to which to write debugging output 1809 @param prefix: string, prefix for filename 1810 @param warpRef: Butler dataRef 1811 @param coaddLevel: bool, optional. If True, include only coadd-level keys 1812 (e.g. 'tract', 'patch', 'filter', but no 'visit') 1814 Creates a hyphen-delimited string of dataId values for simple filenames. 1819 keys = warpRef.dataId.keys()
1820 keyList = sorted(keys, reverse=
True)
1822 filename =
"%s-%s.fits" % (prefix,
'-'.join([str(warpRef.dataId[k])
for k
in keyList]))
1823 return os.path.join(directory, filename)
def setBrightObjectMasks(self, exposure, dataId, brightObjectMasks)
def getCoaddDatasetName(self, warpType="direct")
def _dataRef2DebugPath(self, prefix, warpRef, coaddLevel=False)
Return a path to which to write debugging output.
def getGroupDataRef(butler, datasetType, groupTuple, keys)
Base class for coaddition.
def _filterArtifacts(self, spanSetList, epochCountImage, maxNumEpochs=None)
def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList)
Find artifacts.
def assembleMetadata(self, coaddExposure, tempExpRefList, weightList)
Set the metadata for the coadd.
def makeSupplementaryData(self, dataRef, selectDataList)
Make additional inputs to assemble() specific to subclasses.
def makeSupplementaryData(self, dataRef, selectDataList)
Make inputs specific to Subclass.
def assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList, bgInfoList=None, altMaskList=None, mask=None, supplementaryData=None)
Assemble a coadd from input warps.
def backgroundMatching(self, inputData, refExpDataRef=None, refImageScaler=None)
Perform background matching on the prepared inputs.
def getTempExpRefList(self, patchRef, calExpRefList)
Generate list data references corresponding to warped exposures that lie within the patch to be coadd...
def run(self, dataRef, selectDataList=[])
Assemble a coadd from a set of Warps.
def _readAndComputeWarpDiff(self, warpRef, imageScaler, templateCoadd)
def assembleSubregion(self, coaddExposure, bbox, tempExpRefList, imageScalerList, weightList, bgInfoList, altMaskList, statsFlags, statsCtrl, nImage=None)
Assemble the coadd for a sub-region.
def prepareInputs(self, refList)
Prepare the input warps for coaddition by measuring the weight for each warp and the scaling for the ...
def assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList, bgModelList, args, kwargs)
Assemble the coadd for a region.
def getSkyInfo(self, patchRef)
Use getSkyinfo to return the skyMap, tract and patch information, wcs and the outer bbox of the patch...
def getTempExpDatasetName(self, warpType="direct")
def __init__(self, args, kwargs)
Initialize the task and make the assembleStaticSkyModel subtask.
def makeDataRefList(self, namespace)
Make self.refList from self.idList.
def getBadPixelMask(self)
Convenience method to provide the bitmask from the mask plane names.
def detectClip(self, exp, tempExpRefList)
Detect clipped regions on an exposure and set the mask on the individual tempExp masks.
Configuration parameters for the SafeClipAssembleCoaddTask.
def __init__(self, args, kwargs)
Initialize the task.
Assemble a coadded image from a set of warps (coadded temporary exposures).
def buildDifferenceImage(self, skyInfo, tempExpRefList, imageScalerList, weightList, bgModelList)
Return an exposure that contains the difference between and unclipped and clipped coadds...
def computeAltMaskList(self, tempExpRefList, maskSpanSets)
Apply artifact span set lists to masks.
Assemble a coadded image from a set of coadded temporary exposures, being careful to clip & flag area...
def selectExposures(self, patchRef, skyInfo=None, selectDataList=[])
Select exposures to coadd.
def detectClipBig(self, tempExpClipList, clipFootprints, clipIndices, maskClipValue, maskDetValue)
Find footprints from individual tempExp footprints for large footprints.
def readBrightObjectMasks(self, dataRef)
Configuration parameters for the AssembleCoaddTask.
Assemble a compareWarp coadded image from a set of warps by masking artifacts detected by comparing P...
def __init__(self, args, kwargs)
Initialize the task and make the clipDetection subtask.
def getBackgroundReferenceScaler(self, dataRef)
Construct an image scaler for the background reference frame.
def addBackgroundMatchingMetadata(self, coaddExposure, tempExpRefList, backgroundInfoList)
Add metadata from the background matching to the coadd.
A version of lsst.pipe.base.DataIdContainer specialized for assembleCoadd.
def countMaskFromFootprint(mask, footprint, bitmask, ignoreMask)
Function to count the number of pixels with a specific mask in a footprint.
def assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList, bgModelList, supplementaryData, args, kwargs)
Assemble the coadd.
def groupPatchExposures(patchDataRef, calexpDataRefList, coaddDatasetType="deepCoadd", tempExpDatasetType="deepCoadd_directWarp")