1 from __future__
import absolute_import, division, print_function
2 from builtins
import zip
3 from builtins
import range
26 import lsst.pex.config
as pexConfig
27 import lsst.pex.exceptions
as pexExceptions
28 import lsst.afw.geom
as afwGeom
29 import lsst.afw.image
as afwImage
30 import lsst.afw.math
as afwMath
31 import lsst.afw.table
as afwTable
32 import lsst.afw.detection
as afwDet
33 import lsst.coadd.utils
as coaddUtils
34 import lsst.pipe.base
as pipeBase
35 import lsst.meas.algorithms
as measAlg
36 import lsst.log
as log
37 from .coaddBase
import CoaddBaseTask, SelectDataIdContainer
38 from .interpImage
import InterpImageTask
39 from .matchBackgrounds
import MatchBackgroundsTask
40 from .scaleZeroPoint
import ScaleZeroPointTask
41 from .coaddHelpers
import groupPatchExposures, getGroupDataRef
42 from lsst.meas.algorithms
import SourceDetectionTask
44 __all__ = [
"AssembleCoaddTask",
"SafeClipAssembleCoaddTask",
"CompareWarpAssembleCoaddTask"]
49 \anchor AssembleCoaddConfig_ 51 \brief Configuration parameters for the \ref AssembleCoaddTask_ "AssembleCoaddTask" 53 warpType = pexConfig.Field(
54 doc=
"Warp name: one of 'direct' or 'psfMatched'",
58 subregionSize = pexConfig.ListField(
60 doc=
"Width, height of stack subregion size; " 61 "make small enough that a full stack of images will fit into memory at once.",
65 statistic = pexConfig.Field(
67 doc=
"Main stacking statistic for aggregating over the epochs.",
70 doSigmaClip = pexConfig.Field(
72 doc=
"Perform sigma clipped outlier rejection with MEANCLIP statistic? (DEPRECATED)",
75 sigmaClip = pexConfig.Field(
77 doc=
"Sigma for outlier rejection; ignored if non-clipping statistic selected.",
80 clipIter = pexConfig.Field(
82 doc=
"Number of iterations of outlier rejection; ignored if non-clipping statistic selected.",
85 scaleZeroPoint = pexConfig.ConfigurableField(
86 target=ScaleZeroPointTask,
87 doc=
"Task to adjust the photometric zero point of the coadd temp exposures",
89 doInterp = pexConfig.Field(
90 doc=
"Interpolate over NaN pixels? Also extrapolate, if necessary, but the results are ugly.",
94 interpImage = pexConfig.ConfigurableField(
95 target=InterpImageTask,
96 doc=
"Task to interpolate (and extrapolate) over NaN pixels",
98 matchBackgrounds = pexConfig.ConfigurableField(
99 target=MatchBackgroundsTask,
100 doc=
"Task to match backgrounds",
102 maxMatchResidualRatio = pexConfig.Field(
103 doc=
"Maximum ratio of the mean squared error of the background matching model to the variance " 104 "of the difference in backgrounds",
108 maxMatchResidualRMS = pexConfig.Field(
109 doc=
"Maximum RMS of residuals of the background offset fit in matchBackgrounds.",
113 doWrite = pexConfig.Field(
114 doc=
"Persist coadd?",
118 doNImage = pexConfig.Field(
119 doc=
"Create image of number of contributing exposures for each pixel",
123 doMatchBackgrounds = pexConfig.Field(
124 doc=
"Match backgrounds of coadd temp exposures before coadding them? " 125 "If False, the coadd temp expsosures must already have been background subtracted or matched",
129 autoReference = pexConfig.Field(
130 doc=
"Automatically select the coadd temp exposure to use as a reference for background matching? " 131 "Ignored if doMatchBackgrounds false. " 132 "If False you must specify the reference temp exposure as the data Id",
136 maskPropagationThresholds = pexConfig.DictField(
139 doc=(
"Threshold (in fractional weight) of rejection at which we propagate a mask plane to " 140 "the coadd; that is, we set the mask bit on the coadd if the fraction the rejected frames " 141 "would have contributed exceeds this value."),
142 default={
"SAT": 0.1},
144 removeMaskPlanes = pexConfig.ListField(dtype=str, default=[
"CROSSTALK",
"NOT_DEBLENDED"],
145 doc=
"Mask planes to remove before coadding")
154 doMaskBrightObjects = pexConfig.Field(dtype=bool, default=
False,
155 doc=
"Set mask and flag bits for bright objects?")
156 brightObjectMaskName = pexConfig.Field(dtype=str, default=
"BRIGHT_OBJECT",
157 doc=
"Name of mask bit used for bright objects")
159 coaddPsf = pexConfig.ConfigField(
160 doc=
"Configuration for CoaddPsf",
161 dtype=measAlg.CoaddPsfConfig,
165 CoaddBaseTask.ConfigClass.setDefaults(self)
169 CoaddBaseTask.ConfigClass.validate(self)
173 log.warn(
"Config doPsfMatch deprecated. Setting warpType='psfMatched'")
176 log.warn(
'doSigmaClip deprecated. To replicate behavior, setting statistic to "MEANCLIP"')
178 if self.
doInterp and self.
statistic not in [
'MEAN',
'MEDIAN',
'MEANCLIP',
'VARIANCE',
'VARIANCECLIP']:
179 raise ValueError(
"Must set doInterp=False for statistic=%s, which does not " 180 "compute and set a non-zero coadd variance estimate." % (self.
statistic))
182 unstackableStats = [
'NOTHING',
'ERROR',
'ORMASK']
183 if not hasattr(afwMath.Property, self.
statistic)
or self.
statistic in unstackableStats:
184 stackableStats = [str(k)
for k
in afwMath.Property.__members__.keys()
185 if str(k)
not in unstackableStats]
186 raise ValueError(
"statistic %s is not allowed. Please choose one of %s." 198 \anchor AssembleCoaddTask_ 200 \brief Assemble a coadded image from a set of warps (coadded temporary exposures). 202 \section pipe_tasks_assembleCoadd_Contents Contents 203 - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Purpose 204 - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Initialize 205 - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Run 206 - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Config 207 - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Debug 208 - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Example 210 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Purpose Description 212 \copybrief AssembleCoaddTask_ 214 We want to assemble a coadded image from a set of Warps (also called 215 coadded temporary exposures or coaddTempExps. 216 Each input Warp covers a patch on the sky and corresponds to a single run/visit/exposure of the 217 covered patch. We provide the task with a list of Warps (selectDataList) from which it selects 218 Warps that cover the specified patch (pointed at by dataRef). 219 Each Warp that goes into a coadd will typically have an independent photometric zero-point. 220 Therefore, we must scale each Warp to set it to a common photometric zeropoint. By default, each 221 Warp has backgrounds and hence will require config.doMatchBackgrounds=True. 222 When background matching is enabled, the task may be configured to automatically select a reference exposure 223 (config.autoReference=True). If this is not done, we require that the input dataRef provides access to a 224 Warp (dataset type coaddName + 'Coadd' + warpType + 'Warp') which is used as the reference exposure. 225 WarpType may be one of 'direct' or 'psfMatched', and the boolean configs config.makeDirect and 226 config.makePsfMatched set which of the warp types will be coadded. 227 The coadd is computed as a mean with optional outlier rejection. 228 Criteria for outlier rejection are set in \ref AssembleCoaddConfig. Finally, Warps can have bad 'NaN' 229 pixels which received no input from the source calExps. We interpolate over these bad (NaN) pixels. 231 AssembleCoaddTask uses several sub-tasks. These are 233 <DT>\ref ScaleZeroPointTask_ "ScaleZeroPointTask"</DT> 234 <DD> create and use an imageScaler object to scale the photometric zeropoint for each Warp</DD> 235 <DT>\ref MatchBackgroundsTask_ "MatchBackgroundsTask"</DT> 236 <DD> match background in a Warp to a reference exposure (and select the reference exposure if one is 238 <DT>\ref InterpImageTask_ "InterpImageTask"</DT> 239 <DD>interpolate across bad pixels (NaN) in the final coadd</DD> 241 You can retarget these subtasks if you wish. 243 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Initialize Task initialization 244 \copydoc \_\_init\_\_ 246 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Run Invoking the Task 249 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Config Configuration parameters 250 See \ref AssembleCoaddConfig_ 252 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Debug Debug variables 253 The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a 254 flag \c -d to import \b debug.py from your \c PYTHONPATH; see \ref baseDebug for more about \b debug.py files. 255 AssembleCoaddTask has no debug variables of its own. Some of the subtasks may support debug variables. See 256 the documetation for the subtasks for further information. 258 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Example A complete example of using AssembleCoaddTask 260 AssembleCoaddTask assembles a set of warped images into a coadded image. The AssembleCoaddTask 261 can be invoked by running assembleCoadd.py with the flag '--legacyCoadd'. Usage of assembleCoadd.py expects 262 a data reference to the tract patch and filter to be coadded (specified using 263 '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]') along with a list of 264 Warps to attempt to coadd (specified using 265 '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]'). Only the Warps 266 that cover the specified tract and patch will be coadded. A list of the available optional 267 arguments can be obtained by calling assembleCoadd.py with the --help command line argument: 269 assembleCoadd.py --help 271 To demonstrate usage of the AssembleCoaddTask in the larger context of multi-band processing, we will generate 272 the HSC-I & -R band coadds from HSC engineering test data provided in the ci_hsc package. To begin, assuming 273 that the lsst stack has been already set up, we must set up the obs_subaru and ci_hsc packages. 274 This defines the environment variable $CI_HSC_DIR and points at the location of the package. The raw HSC 275 data live in the $CI_HSC_DIR/raw directory. To begin assembling the coadds, we must first 278 <DD> process the individual ccds in $CI_HSC_RAW to produce calibrated exposures</DD> 280 <DD> create a skymap that covers the area of the sky present in the raw exposures</DD> 281 <DT>makeCoaddTempExp</DT> 282 <DD> warp the individual calibrated exposures to the tangent plane of the coadd</DD> 284 We can perform all of these steps by running 286 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988 288 This will produce warped exposures for each visit. To coadd the warped data, we call assembleCoadd.py as 291 assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \ 292 --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \ 293 --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \ 294 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \ 295 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \ 296 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \ 297 --selectId visit=903988 ccd=24 299 that will process the HSC-I band data. The results are written in 300 `$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I`. 302 You may also choose to run: 304 scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346 305 assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R \ 306 --selectId visit=903334 ccd=16 --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 \ 307 --selectId visit=903334 ccd=100 --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 \ 308 --selectId visit=903338 ccd=18 --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 \ 309 --selectId visit=903342 ccd=10 --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 \ 310 --selectId visit=903344 ccd=5 --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 \ 311 --selectId visit=903346 ccd=6 --selectId visit=903346 ccd=12 313 to generate the coadd for the HSC-R band if you are interested in following multiBand Coadd processing as 314 discussed in \ref pipeTasks_multiBand (but note that normally, one would use the 315 \ref SafeClipAssembleCoaddTask_ "SafeClipAssembleCoaddTask" rather than AssembleCoaddTask to make the coadd. 317 ConfigClass = AssembleCoaddConfig
318 _DefaultName =
"assembleCoadd" 322 \brief Initialize the task. Create the \ref InterpImageTask "interpImage", 323 \ref MatchBackgroundsTask "matchBackgrounds", & \ref ScaleZeroPointTask "scaleZeroPoint" subtasks. 325 CoaddBaseTask.__init__(self, *args, **kwargs)
326 self.makeSubtask(
"interpImage")
327 self.makeSubtask(
"matchBackgrounds")
328 self.makeSubtask(
"scaleZeroPoint")
330 if self.config.doMaskBrightObjects:
331 mask = afwImage.Mask()
334 except pexExceptions.LsstCppException:
335 raise RuntimeError(
"Unable to define mask plane for bright objects; planes used are %s" %
336 mask.getMaskPlaneDict().keys())
342 def run(self, dataRef, selectDataList=[]):
344 \brief Assemble a coadd from a set of Warps 346 Coadd a set of Warps. Compute weights to be applied to each Warp and find scalings to 347 match the photometric zeropoint to a reference Warp. Optionally, match backgrounds across 348 Warps if the background has not already been removed. Assemble the Warps using 349 \ref assemble. Interpolate over NaNs and optionally write the coadd to disk. Return the coadded 353 \param[in] dataRef: Data reference defining the patch for coaddition and the reference Warp 354 (if config.autoReference=False). Used to access the following data products: 355 - [in] self.config.coaddName + "Coadd_skyMap" 356 - [in] self.config.coaddName + "Coadd_ + <warpType> + "Warp" (optionally) 357 - [out] self.config.coaddName + "Coadd" 358 \param[in] selectDataList[in]: List of data references to Warps. Data to be coadded will be 359 selected from this list based on overlap with the patch defined by dataRef. 361 \return a pipeBase.Struct with fields: 362 - coaddExposure: coadded exposure 363 - nImage: exposure count image 366 calExpRefList = self.
selectExposures(dataRef, skyInfo, selectDataList=selectDataList)
367 if len(calExpRefList) == 0:
368 self.log.warn(
"No exposures to coadd")
370 self.log.info(
"Coadding %d exposures", len(calExpRefList))
374 self.log.info(
"Found %d %s", len(inputData.tempExpRefList),
376 if len(inputData.tempExpRefList) == 0:
377 self.log.warn(
"No coadd temporary exposures found")
379 if self.config.doMatchBackgrounds:
382 if len(inputData.tempExpRefList) == 0:
383 self.log.warn(
"No valid background models")
388 retStruct = self.
assemble(skyInfo, inputData.tempExpRefList, inputData.imageScalerList,
389 inputData.weightList,
390 inputData.backgroundInfoList
if self.config.doMatchBackgrounds
else None,
391 supplementaryData=supplementaryData)
393 if self.config.doMatchBackgrounds:
395 inputData.backgroundInfoList)
397 if self.config.doInterp:
398 self.interpImage.
run(retStruct.coaddExposure.getMaskedImage(), planeName=
"NO_DATA")
400 varArray = retStruct.coaddExposure.getMaskedImage().getVariance().getArray()
401 varArray[:] = numpy.where(varArray > 0, varArray, numpy.inf)
403 if self.config.doMaskBrightObjects:
407 if self.config.doWrite:
410 if retStruct.nImage
is not None:
417 \brief Make additional inputs to assemble() specific to subclasses. 419 Available to be implemented by subclasses only if they need the 420 coadd dataRef for performing preliminary processing before 421 assembling the coadd. 427 \brief Generate list data references corresponding to warped exposures that lie within the 430 \param[in] patchRef: Data reference for patch 431 \param[in] calExpRefList: List of data references for input calexps 432 \return List of Warp/CoaddTempExp data references 434 butler = patchRef.getButler()
435 groupData =
groupPatchExposures(patchRef, calExpRefList, self.getCoaddDatasetName(self.warpType),
436 self.getTempExpDatasetName(self.warpType))
437 tempExpRefList = [
getGroupDataRef(butler, self.getTempExpDatasetName(self.warpType),
438 g, groupData.keys)
for 439 g
in groupData.groups.keys()]
440 return tempExpRefList
444 \brief Construct an image scaler for the background reference frame 446 Each Warp has a different background level. A reference background level must be chosen before 447 coaddition. If config.autoReference=True, \ref backgroundMatching will pick the reference level and 448 this routine is a no-op and None is returned. Otherwise, use the 449 \ref ScaleZeroPointTask_ "scaleZeroPoint" subtask to compute an imageScaler object for the provided 450 reference image and return it. 452 \param[in] dataRef: Data reference for the background reference frame, or None 453 \return image scaler, or None 455 if self.config.autoReference:
460 if not dataRef.datasetExists(dataset):
461 raise RuntimeError(
"Could not find reference exposure %s %s." % (dataset, dataRef.dataId))
464 refImageScaler = self.scaleZeroPoint.computeImageScaler(
465 exposure=refExposure,
468 return refImageScaler
472 \brief Prepare the input warps for coaddition by measuring the weight for each warp and the scaling 473 for the photometric zero point. 475 Each Warp has its own photometric zeropoint and background variance. Before coadding these 476 Warps together, compute a scale factor to normalize the photometric zeropoint and compute the 477 weight for each Warp. 479 \param[in] refList: List of data references to tempExp 481 - tempExprefList: List of data references to tempExp 482 - weightList: List of weightings 483 - imageScalerList: List of image scalers 485 statsCtrl = afwMath.StatisticsControl()
486 statsCtrl.setNumSigmaClip(self.config.sigmaClip)
487 statsCtrl.setNumIter(self.config.clipIter)
489 statsCtrl.setNanSafe(
True)
497 for tempExpRef
in refList:
498 if not tempExpRef.datasetExists(tempExpName):
499 self.log.warn(
"Could not find %s %s; skipping it", tempExpName, tempExpRef.dataId)
502 tempExp = tempExpRef.get(tempExpName, immediate=
True)
503 maskedImage = tempExp.getMaskedImage()
504 imageScaler = self.scaleZeroPoint.computeImageScaler(
509 imageScaler.scaleMaskedImage(maskedImage)
510 except Exception
as e:
511 self.log.warn(
"Scaling failed for %s (skipping it): %s", tempExpRef.dataId, e)
513 statObj = afwMath.makeStatistics(maskedImage.getVariance(), maskedImage.getMask(),
514 afwMath.MEANCLIP, statsCtrl)
515 meanVar, meanVarErr = statObj.getResult(afwMath.MEANCLIP)
516 weight = 1.0 / float(meanVar)
517 if not numpy.isfinite(weight):
518 self.log.warn(
"Non-finite weight for %s: skipping", tempExpRef.dataId)
520 self.log.info(
"Weight of %s %s = %0.3f", tempExpName, tempExpRef.dataId, weight)
525 tempExpRefList.append(tempExpRef)
526 weightList.append(weight)
527 imageScalerList.append(imageScaler)
529 return pipeBase.Struct(tempExpRefList=tempExpRefList, weightList=weightList,
530 imageScalerList=imageScalerList)
534 \brief Perform background matching on the prepared inputs 536 Each Warp has a different background level that must be normalized to a reference level 537 before coaddition. If no reference is provided, the background matcher selects one. If the background 538 matching is performed sucessfully, recompute the weight to be applied to the Warp (coaddTempExp) to be 539 consistent with the scaled background. 541 \param[in] inputData: Struct from prepareInputs() with tempExpRefList, weightList, imageScalerList 542 \param[in] refExpDataRef: Data reference for background reference Warp, or None 543 \param[in] refImageScaler: Image scaler for background reference Warp, or None 545 - tempExprefList: List of data references to warped exposures (coaddTempExps) 546 - weightList: List of weightings 547 - imageScalerList: List of image scalers 548 - backgroundInfoList: result from background matching 551 backgroundInfoList = self.matchBackgrounds.
run(
552 expRefList=inputData.tempExpRefList,
553 imageScalerList=inputData.imageScalerList,
554 refExpDataRef=refExpDataRef
if not self.config.autoReference
else None,
555 refImageScaler=refImageScaler,
558 except Exception
as e:
559 self.log.fatal(
"Cannot match backgrounds: %s", e)
560 raise pipeBase.TaskError(
"Background matching failed.")
563 newTempExpRefList = []
564 newBackgroundStructList = []
568 for tempExpRef, bgInfo, scaler, weight
in zip(inputData.tempExpRefList, backgroundInfoList,
569 inputData.imageScalerList, inputData.weightList):
570 if not bgInfo.isReference:
573 if (bgInfo.backgroundModel
is None):
574 self.log.info(
"No background offset model available for %s: skipping", tempExpRef.dataId)
577 varianceRatio = bgInfo.matchedMSE / bgInfo.diffImVar
578 except Exception
as e:
579 self.log.info(
"MSE/Var ratio not calculable (%s) for %s: skipping",
580 e, tempExpRef.dataId)
582 if not numpy.isfinite(varianceRatio):
583 self.log.info(
"MSE/Var ratio not finite (%.2f / %.2f) for %s: skipping",
584 bgInfo.matchedMSE, bgInfo.diffImVar, tempExpRef.dataId)
586 elif (varianceRatio > self.config.maxMatchResidualRatio):
587 self.log.info(
"Bad fit. MSE/Var ratio %.2f > %.2f for %s: skipping",
588 varianceRatio, self.config.maxMatchResidualRatio, tempExpRef.dataId)
590 elif (bgInfo.fitRMS > self.config.maxMatchResidualRMS):
591 self.log.info(
"Bad fit. RMS %.2f > %.2f for %s: skipping",
592 bgInfo.fitRMS, self.config.maxMatchResidualRMS, tempExpRef.dataId)
594 newWeightList.append(1 / (1 / weight + bgInfo.fitRMS**2))
595 newTempExpRefList.append(tempExpRef)
596 newBackgroundStructList.append(bgInfo)
597 newScaleList.append(scaler)
599 return pipeBase.Struct(tempExpRefList=newTempExpRefList, weightList=newWeightList,
600 imageScalerList=newScaleList, backgroundInfoList=newBackgroundStructList)
602 def assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList, bgInfoList=None,
603 altMaskList=None, mask=None, supplementaryData=None):
605 \anchor AssembleCoaddTask.assemble_ 607 \brief Assemble a coadd from input warps 609 Assemble the coadd using the provided list of coaddTempExps. Since the full coadd covers a patch (a 610 large area), the assembly is performed over small areas on the image at a time in order to 611 conserve memory usage. Iterate over subregions within the outer bbox of the patch using 612 \ref assembleSubregion to stack the corresponding subregions from the coaddTempExps with the 613 statistic specified. Set the edge bits the coadd mask based on the weight map. 615 \param[in] skyInfo: Patch geometry information, from getSkyInfo 616 \param[in] tempExpRefList: List of data references to Warps (previously called CoaddTempExps) 617 \param[in] imageScalerList: List of image scalers 618 \param[in] weightList: List of weights 619 \param[in] bgInfoList: List of background data from background matching, or None 620 \param[in] altMaskList: List of alternate masks to use rather than those stored with tempExp, or None 621 \param[in] mask: Mask to ignore when coadding 622 \param[in] supplementaryData: pipeBase.Struct with additional data products needed to assemble coadd. 623 Only used by subclasses that implement makeSupplementaryData and override assemble. 624 \return pipeBase.Struct with coaddExposure, nImage if requested 627 self.log.info(
"Assembling %s %s", len(tempExpRefList), tempExpName)
631 statsCtrl = afwMath.StatisticsControl()
632 statsCtrl.setNumSigmaClip(self.config.sigmaClip)
633 statsCtrl.setNumIter(self.config.clipIter)
634 statsCtrl.setAndMask(mask)
635 statsCtrl.setNanSafe(
True)
636 statsCtrl.setWeighted(
True)
637 statsCtrl.setCalcErrorFromInputVariance(
True)
638 for plane, threshold
in self.config.maskPropagationThresholds.items():
639 bit = afwImage.Mask.getMaskPlane(plane)
640 statsCtrl.setMaskPropagationThreshold(bit, threshold)
642 statsFlags = afwMath.stringToStatisticsProperty(self.config.statistic)
644 if bgInfoList
is None:
645 bgInfoList = [
None]*len(tempExpRefList)
647 if altMaskList
is None:
648 altMaskList = [
None]*len(tempExpRefList)
650 coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
651 coaddExposure.setCalib(self.scaleZeroPoint.getCalib())
652 coaddExposure.getInfo().setCoaddInputs(self.inputRecorder.makeCoaddInputs())
654 coaddMaskedImage = coaddExposure.getMaskedImage()
655 subregionSizeArr = self.config.subregionSize
656 subregionSize = afwGeom.Extent2I(subregionSizeArr[0], subregionSizeArr[1])
658 if self.config.doNImage:
659 nImage = afwImage.ImageU(skyInfo.bbox)
662 for subBBox
in _subBBoxIter(skyInfo.bbox, subregionSize):
665 weightList, bgInfoList, altMaskList, statsFlags, statsCtrl,
667 except Exception
as e:
668 self.log.fatal(
"Cannot compute coadd %s: %s", subBBox, e)
670 coaddUtils.setCoaddEdgeBits(coaddMaskedImage.getMask(), coaddMaskedImage.getVariance())
671 return pipeBase.Struct(coaddExposure=coaddExposure, nImage=nImage)
675 \brief Set the metadata for the coadd 677 This basic implementation simply sets the filter from the 680 \param[in] coaddExposure: The target image for the coadd 681 \param[in] tempExpRefList: List of data references to tempExp 682 \param[in] weightList: List of weights 684 assert len(tempExpRefList) == len(weightList),
"Length mismatch" 689 tempExpList = [tempExpRef.get(tempExpName +
"_sub",
690 bbox=afwGeom.Box2I(afwGeom.Point2I(0, 0), afwGeom.Extent2I(1, 1)),
691 imageOrigin=
"LOCAL", immediate=
True)
for tempExpRef
in tempExpRefList]
692 numCcds = sum(len(tempExp.getInfo().getCoaddInputs().ccds)
for tempExp
in tempExpList)
694 coaddExposure.setFilter(tempExpList[0].getFilter())
695 coaddInputs = coaddExposure.getInfo().getCoaddInputs()
696 coaddInputs.ccds.reserve(numCcds)
697 coaddInputs.visits.reserve(len(tempExpList))
699 for tempExp, weight
in zip(tempExpList, weightList):
700 self.inputRecorder.addVisitToCoadd(coaddInputs, tempExp, weight)
701 coaddInputs.visits.sort()
707 modelPsfList = [tempExp.getPsf()
for tempExp
in tempExpList]
708 modelPsfWidthList = [modelPsf.computeBBox().getWidth()
for modelPsf
in modelPsfList]
709 psf = modelPsfList[modelPsfWidthList.index(max(modelPsfWidthList))]
711 psf = measAlg.CoaddPsf(coaddInputs.ccds, coaddExposure.getWcs(),
712 self.config.coaddPsf.makeControl())
713 coaddExposure.setPsf(psf)
714 apCorrMap = measAlg.makeCoaddApCorrMap(coaddInputs.ccds, coaddExposure.getBBox(afwImage.PARENT),
715 coaddExposure.getWcs())
716 coaddExposure.getInfo().setApCorrMap(apCorrMap)
718 def assembleSubregion(self, coaddExposure, bbox, tempExpRefList, imageScalerList, weightList,
719 bgInfoList, altMaskList, statsFlags, statsCtrl, nImage=None):
721 \brief Assemble the coadd for a sub-region. 723 For each coaddTempExp, check for (and swap in) an alternative mask if one is passed. If background 724 matching is enabled, add the background and background variance from each coaddTempExp. Remove mask 725 planes listed in config.removeMaskPlanes, Finally, stack the actual exposures using 726 \ref afwMath.statisticsStack "statisticsStack" with the statistic specified 727 by statsFlags. Typically, the statsFlag will be one of afwMath.MEAN for a mean-stack or 728 afwMath.MEANCLIP for outlier rejection using an N-sigma clipped mean where N and iterations 729 are specified by statsCtrl. Assign the stacked subregion back to the coadd. 731 \param[in] coaddExposure: The target image for the coadd 732 \param[in] bbox: Sub-region to coadd 733 \param[in] tempExpRefList: List of data reference to tempExp 734 \param[in] imageScalerList: List of image scalers 735 \param[in] weightList: List of weights 736 \param[in] bgInfoList: List of background data from background matching 737 \param[in] altMaskList: List of alternate masks to use rather than those stored with tempExp, or None 738 \param[in] statsFlags: afwMath.Property object for statistic for coadd 739 \param[in] statsCtrl: Statistics control object for coadd 740 \param[in] nImage: optional ImageU keeps track of exposure count for each pixel 742 self.log.debug(
"Computing coadd over %s", bbox)
744 coaddMaskedImage = coaddExposure.getMaskedImage()
746 if nImage
is not None:
747 subNImage = afwImage.ImageU(bbox.getWidth(), bbox.getHeight())
748 for tempExpRef, imageScaler, bgInfo, altMask
in zip(tempExpRefList, imageScalerList, bgInfoList,
750 exposure = tempExpRef.get(tempExpName +
"_sub", bbox=bbox)
751 maskedImage = exposure.getMaskedImage()
753 altMaskSub = altMask.Factory(altMask, bbox, afwImage.PARENT)
754 maskedImage.getMask().swap(altMaskSub)
755 imageScaler.scaleMaskedImage(maskedImage)
757 if self.config.doMatchBackgrounds
and not bgInfo.isReference:
758 backgroundModel = bgInfo.backgroundModel
759 backgroundImage = backgroundModel.getImage()
if \
760 self.matchBackgrounds.config.usePolynomial
else \
761 backgroundModel.getImageF()
762 backgroundImage.setXY0(coaddMaskedImage.getXY0())
763 maskedImage += backgroundImage.Factory(backgroundImage, bbox, afwImage.PARENT,
False)
764 var = maskedImage.getVariance()
765 var += (bgInfo.fitRMS)**2
768 if nImage
is not None:
769 subNImage.getArray()[maskedImage.getMask().getArray() & statsCtrl.getAndMask() == 0] += 1
770 if self.config.removeMaskPlanes:
771 mask = maskedImage.getMask()
772 for maskPlane
in self.config.removeMaskPlanes:
774 mask &= ~mask.getPlaneBitMask(maskPlane)
775 except Exception
as e:
776 self.log.warn(
"Unable to remove mask plane %s: %s", maskPlane, e.message)
778 maskedImageList.append(maskedImage)
780 with self.timer(
"stack"):
781 coaddSubregion = afwMath.statisticsStack(
782 maskedImageList, statsFlags, statsCtrl, weightList)
783 coaddMaskedImage.assign(coaddSubregion, bbox)
784 if nImage
is not None:
785 nImage.assign(subNImage, bbox)
789 \brief Add metadata from the background matching to the coadd 791 \param[in] coaddExposure: Coadd 792 \param[in] tempExpRefList: List of data references for temp exps to go into coadd 793 \param[in] backgroundInfoList: List of background info, results from background matching 795 self.log.info(
"Adding exposure information to metadata")
796 metadata = coaddExposure.getMetadata()
797 metadata.addString(
"CTExp_SDQA1_DESCRIPTION",
798 "Background matching: Ratio of matchedMSE / diffImVar")
799 for ind, (tempExpRef, backgroundInfo)
in enumerate(zip(tempExpRefList, backgroundInfoList)):
800 tempExpStr =
'&'.join(
'%s=%s' % (k, v)
for k, v
in tempExpRef.dataId.items())
801 if backgroundInfo.isReference:
802 metadata.addString(
"ReferenceExp_ID", tempExpStr)
804 metadata.addString(
"CTExp_ID_%d" % (ind), tempExpStr)
805 metadata.addDouble(
"CTExp_SDQA1_%d" % (ind),
806 backgroundInfo.matchedMSE/backgroundInfo.diffImVar)
807 metadata.addDouble(
"CTExp_SDQA2_%d" % (ind),
808 backgroundInfo.fitRMS)
811 """Returns None on failure""" 813 return dataRef.get(
"brightObjectMask", immediate=
True)
814 except Exception
as e:
815 self.log.warn(
"Unable to read brightObjectMask for %s: %s", dataRef.dataId, e)
819 """Set the bright object masks 821 exposure: Exposure under consideration 822 dataId: Data identifier dict for patch 823 brightObjectMasks: afwTable of bright objects to mask 828 if brightObjectMasks
is None:
829 self.log.warn(
"Unable to apply bright object mask: none supplied")
831 self.log.info(
"Applying %d bright object masks to %s", len(brightObjectMasks), dataId)
832 md = brightObjectMasks.table.getMetadata()
835 self.log.warn(
"Expected to see %s in metadata", k)
837 if md.get(k) != dataId[k]:
838 self.log.warn(
"Expected to see %s == %s in metadata, saw %s", k, md.get(k), dataId[k])
840 mask = exposure.getMaskedImage().getMask()
841 wcs = exposure.getWcs()
842 plateScale = wcs.pixelScale().asArcseconds()
844 for rec
in brightObjectMasks:
845 center = afwGeom.PointI(wcs.skyToPixel(rec.getCoord()))
846 if rec[
"type"] ==
"box":
847 assert rec[
"angle"] == 0.0, (
"Angle != 0 for mask object %s" % rec[
"id"])
848 width = rec[
"width"].asArcseconds()/plateScale
849 height = rec[
"height"].asArcseconds()/plateScale
851 halfSize = afwGeom.ExtentI(0.5*width, 0.5*height)
852 bbox = afwGeom.Box2I(center - halfSize, center + halfSize)
854 bbox = afwGeom.BoxI(afwGeom.PointI(int(center[0] - 0.5*width), int(center[1] - 0.5*height)),
855 afwGeom.PointI(int(center[0] + 0.5*width), int(center[1] + 0.5*height)))
856 spans = afwGeom.SpanSet(bbox)
857 elif rec[
"type"] ==
"circle":
858 radius = int(rec[
"radius"].asArcseconds()/plateScale)
859 spans = afwGeom.SpanSet.fromShape(radius, offset=center)
861 self.log.warn(
"Unexpected region type %s at %s" % rec[
"type"], center)
866 def _makeArgumentParser(cls):
868 \brief Create an argument parser 871 parser.add_id_argument(
"--id", cls.
ConfigClass().coaddName +
"Coadd_" +
873 help=
"data ID, e.g. --id tract=12345 patch=1,2",
874 ContainerClass=AssembleCoaddDataIdContainer)
875 parser.add_id_argument(
"--selectId",
"calexp", help=
"data ID, e.g. --selectId visit=6789 ccd=0..9",
876 ContainerClass=SelectDataIdContainer)
880 def _subBBoxIter(bbox, subregionSize):
882 \brief Iterate over subregions of a bbox 884 \param[in] bbox: bounding box over which to iterate: afwGeom.Box2I 885 \param[in] subregionSize: size of sub-bboxes 887 \return subBBox: next sub-bounding box of size subregionSize or smaller; 888 each subBBox is contained within bbox, so it may be smaller than subregionSize at the edges of bbox, 889 but it will never be empty 892 raise RuntimeError(
"bbox %s is empty" % (bbox,))
893 if subregionSize[0] < 1
or subregionSize[1] < 1:
894 raise RuntimeError(
"subregionSize %s must be nonzero" % (subregionSize,))
896 for rowShift
in range(0, bbox.getHeight(), subregionSize[1]):
897 for colShift
in range(0, bbox.getWidth(), subregionSize[0]):
898 subBBox = afwGeom.Box2I(bbox.getMin() + afwGeom.Extent2I(colShift, rowShift), subregionSize)
900 if subBBox.isEmpty():
901 raise RuntimeError(
"Bug: empty bbox! bbox=%s, subregionSize=%s, colShift=%s, rowShift=%s" %
902 (bbox, subregionSize, colShift, rowShift))
908 \brief A version of lsst.pipe.base.DataIdContainer specialized for assembleCoadd. 913 \brief Make self.refList from self.idList. 915 Interpret the config.doMatchBackgrounds, config.autoReference, 916 and whether a visit/run supplied. 917 If a visit/run is supplied, config.autoReference is automatically set to False. 918 if config.doMatchBackgrounds == false, then a visit/run will be ignored if accidentally supplied. 921 keysCoadd = namespace.butler.getKeys(datasetType=namespace.config.coaddName +
"Coadd",
923 keysCoaddTempExp = namespace.butler.getKeys(datasetType=namespace.config.coaddName +
924 "Coadd_directWarp", level=self.level)
926 if namespace.config.doMatchBackgrounds:
927 if namespace.config.autoReference:
928 datasetType = namespace.config.coaddName +
"Coadd" 929 validKeys = keysCoadd
931 datasetType = namespace.config.coaddName +
"Coadd_directWarp" 932 validKeys = keysCoaddTempExp
934 datasetType = namespace.config.coaddName +
"Coadd" 935 validKeys = keysCoadd
937 for dataId
in self.idList:
939 for key
in validKeys:
940 if key
not in dataId:
941 raise RuntimeError(
"--id must include " + key)
944 if (key
not in keysCoadd)
and (key
in keysCoaddTempExp):
945 if namespace.config.autoReference:
947 namespace.config.autoReference =
False 948 datasetType = namespace.config.coaddName +
"Coadd_directWarp" 949 print(
"Switching config.autoReference to False; applies only to background Matching.")
952 dataRef = namespace.butler.dataRef(
953 datasetType=datasetType,
956 self.refList.append(dataRef)
961 \brief Function to count the number of pixels with a specific mask in a footprint. 963 Find the intersection of mask & footprint. Count all pixels in the mask that are in the intersection that 964 have bitmask set but do not have ignoreMask set. Return the count. 966 \param[in] mask: mask to define intersection region by. 967 \parma[in] footprint: footprint to define the intersection region by. 968 \param[in] bitmask: specific mask that we wish to count the number of occurances of. 969 \param[in] ignoreMask: pixels to not consider. 970 \return count of number of pixels in footprint with specified mask. 972 bbox = footprint.getBBox()
973 bbox.clip(mask.getBBox(afwImage.PARENT))
974 fp = afwImage.Mask(bbox)
975 subMask = mask.Factory(mask, bbox, afwImage.PARENT)
976 footprint.spans.setMask(fp, bitmask)
977 return numpy.logical_and((subMask.getArray() & fp.getArray()) > 0,
978 (subMask.getArray() & ignoreMask) == 0).sum()
983 \anchor SafeClipAssembleCoaddConfig 985 \brief Configuration parameters for the SafeClipAssembleCoaddTask 987 clipDetection = pexConfig.ConfigurableField(
988 target=SourceDetectionTask,
989 doc=
"Detect sources on difference between unclipped and clipped coadd")
990 minClipFootOverlap = pexConfig.Field(
991 doc=
"Minimum fractional overlap of clipped footprint with visit DETECTED to be clipped",
995 minClipFootOverlapSingle = pexConfig.Field(
996 doc=
"Minimum fractional overlap of clipped footprint with visit DETECTED to be " 997 "clipped when only one visit overlaps",
1001 minClipFootOverlapDouble = pexConfig.Field(
1002 doc=
"Minimum fractional overlap of clipped footprints with visit DETECTED to be " 1003 "clipped when two visits overlap",
1007 maxClipFootOverlapDouble = pexConfig.Field(
1008 doc=
"Maximum fractional overlap of clipped footprints with visit DETECTED when " 1009 "considering two visits",
1013 minBigOverlap = pexConfig.Field(
1014 doc=
"Minimum number of pixels in footprint to use DETECTED mask from the single visits " 1015 "when labeling clipped footprints",
1023 AssembleCoaddConfig.setDefaults(self)
1039 log.warn(
"Additional Sigma-clipping not allowed in Safe-clipped Coadds. " 1040 "Ignoring doSigmaClip.")
1043 raise ValueError(
"Only MEAN statistic allowed for final stacking in SafeClipAssembleCoadd " 1044 "(%s chosen). Please set statistic to MEAN." 1046 AssembleCoaddTask.ConfigClass.validate(self)
1059 \anchor SafeClipAssembleCoaddTask_ 1061 \brief Assemble a coadded image from a set of coadded temporary exposures, 1062 being careful to clip & flag areas with potential artifacts. 1064 \section pipe_tasks_assembleCoadd_Contents Contents 1065 - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Purpose 1066 - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Initialize 1067 - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Run 1068 - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Config 1069 - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Debug 1070 - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Example 1072 \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Purpose Description 1074 \copybrief SafeClipAssembleCoaddTask 1076 Read the documentation for \ref AssembleCoaddTask_ "AssembleCoaddTask" first since 1077 SafeClipAssembleCoaddTask subtasks that task. 1078 In \ref AssembleCoaddTask_ "AssembleCoaddTask", we compute the coadd as an clipped mean (i.e. we clip 1080 The problem with doing this is that when computing the coadd PSF at a given location, individual visit 1081 PSFs from visits with outlier pixels contribute to the coadd PSF and cannot be treated correctly. 1082 In this task, we correct for this behavior by creating a new badMaskPlane 'CLIPPED'. 1083 We populate this plane on the input coaddTempExps and the final coadd where i. difference imaging suggests 1084 that there is an outlier and ii. this outlier appears on only one or two images. 1085 Such regions will not contribute to the final coadd. 1086 Furthermore, any routine to determine the coadd PSF can now be cognizant of clipped regions. 1087 Note that the algorithm implemented by this task is preliminary and works correctly for HSC data. 1088 Parameter modifications and or considerable redesigning of the algorithm is likley required for other 1091 SafeClipAssembleCoaddTask uses a \ref SourceDetectionTask_ "clipDetection" subtask and also sub-classes 1092 \ref AssembleCoaddTask_ "AssembleCoaddTask". You can retarget the 1093 \ref SourceDetectionTask_ "clipDetection" subtask if you wish. 1095 \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Initialize Task initialization 1096 \copydoc \_\_init\_\_ 1098 \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Run Invoking the Task 1101 \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Config Configuration parameters 1102 See \ref SafeClipAssembleCoaddConfig 1104 \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Debug Debug variables 1105 The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a 1106 flag \c -d to import \b debug.py from your \c PYTHONPATH; see \ref baseDebug for more about \b debug.py 1108 SafeClipAssembleCoaddTask has no debug variables of its own. The \ref SourceDetectionTask_ "clipDetection" 1109 subtasks may support debug variables. See the documetation for \ref SourceDetectionTask_ "clipDetection" 1110 for further information. 1112 \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Example A complete example of using 1113 SafeClipAssembleCoaddTask 1115 SafeClipAssembleCoaddTask assembles a set of warped coaddTempExp images into a coadded image. 1116 The SafeClipAssembleCoaddTask is invoked by running assembleCoadd.py <em>without</em> the flag 1118 Usage of assembleCoadd.py expects a data reference to the tract patch and filter to be coadded 1119 (specified using '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]') along 1120 with a list of coaddTempExps to attempt to coadd (specified using 1121 '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]'). 1122 Only the coaddTempExps that cover the specified tract and patch will be coadded. 1123 A list of the available optional arguments can be obtained by calling assembleCoadd.py with the --help 1124 command line argument: 1126 assembleCoadd.py --help 1128 To demonstrate usage of the SafeClipAssembleCoaddTask in the larger context of multi-band processing, we 1129 will generate the HSC-I & -R band coadds from HSC engineering test data provided in the ci_hsc package. To 1130 begin, assuming that the lsst stack has been already set up, we must set up the obs_subaru and ci_hsc 1132 This defines the environment variable $CI_HSC_DIR and points at the location of the package. The raw HSC 1133 data live in the $CI_HSC_DIR/raw directory. To begin assembling the coadds, we must first 1136 <DD> process the individual ccds in $CI_HSC_RAW to produce calibrated exposures</DD> 1138 <DD> create a skymap that covers the area of the sky present in the raw exposures</DD> 1139 <DT>makeCoaddTempExp</DT> 1140 <DD> warp the individual calibrated exposures to the tangent plane of the coadd</DD> 1142 We can perform all of these steps by running 1144 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988 1146 This will produce warped coaddTempExps for each visit. To coadd the warped data, we call assembleCoadd.py 1149 assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \ 1150 --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \ 1151 --selectId visit=903986 ccd=100--selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \ 1152 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \ 1153 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \ 1154 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \ 1155 --selectId visit=903988 ccd=24 1157 This will process the HSC-I band data. The results are written in 1158 `$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I`. 1160 You may also choose to run: 1162 scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346 1163 assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R --selectId visit=903334 ccd=16 \ 1164 --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 --selectId visit=903334 ccd=100 \ 1165 --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 --selectId visit=903338 ccd=18 \ 1166 --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 --selectId visit=903342 ccd=10 \ 1167 --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 --selectId visit=903344 ccd=5 \ 1168 --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 --selectId visit=903346 ccd=6 \ 1169 --selectId visit=903346 ccd=12 1171 to generate the coadd for the HSC-R band if you are interested in following multiBand Coadd processing as 1172 discussed in \ref pipeTasks_multiBand. 1174 ConfigClass = SafeClipAssembleCoaddConfig
1175 _DefaultName =
"safeClipAssembleCoadd" 1179 \brief Initialize the task and make the \ref SourceDetectionTask_ "clipDetection" subtask. 1181 AssembleCoaddTask.__init__(self, *args, **kwargs)
1182 schema = afwTable.SourceTable.makeMinimalSchema()
1183 self.makeSubtask(
"clipDetection", schema=schema)
1185 def assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList, bgModelList,
1188 \brief Assemble the coadd for a region 1190 Compute the difference of coadds created with and without outlier rejection to identify coadd pixels 1191 that have outlier values in some individual visits. Detect clipped regions on the difference image and 1192 mark these regions on the one or two individual coaddTempExps where they occur if there is significant 1193 overlap between the clipped region and a source. 1194 This leaves us with a set of footprints from the difference image that have been identified as having 1195 occured on just one or two individual visits. However, these footprints were generated from a 1196 difference image. It is conceivable for a large diffuse source to have become broken up into multiple 1197 footprints acrosss the coadd difference in this process. 1198 Determine the clipped region from all overlapping footprints from the detected sources in each visit - 1199 these are big footprints. 1200 Combine the small and big clipped footprints and mark them on a new bad mask plane 1201 Generate the coadd using \ref AssembleCoaddTask.assemble_ "AssembleCoaddTask.assemble" without outlier 1202 removal. Clipped footprints will no longer make it into the coadd because they are marked in the new 1205 N.b. *args and **kwargs are passed but ignored in order to match the call signature expected by the 1208 @param skyInfo: Patch geometry information, from getSkyInfo 1209 @param tempExpRefList: List of data reference to tempExp 1210 @param imageScalerList: List of image scalers 1211 @param weightList: List of weights 1212 @param bgModelList: List of background models from background matching 1213 return pipeBase.Struct with coaddExposure, nImage 1215 exp = self.
buildDifferenceImage(skyInfo, tempExpRefList, imageScalerList, weightList, bgModelList)
1216 mask = exp.getMaskedImage().getMask()
1217 mask.addMaskPlane(
"CLIPPED")
1219 result = self.
detectClip(exp, tempExpRefList)
1221 self.log.info(
'Found %d clipped objects', len(result.clipFootprints))
1224 maskClipValue = mask.getPlaneBitMask(
"CLIPPED")
1225 maskDetValue = mask.getPlaneBitMask(
"DETECTED") | mask.getPlaneBitMask(
"DETECTED_NEGATIVE")
1226 bigFootprints = self.
detectClipBig(result.tempExpClipList, result.clipFootprints, result.clipIndices,
1227 maskClipValue, maskDetValue)
1230 maskClip = mask.Factory(mask.getBBox(afwImage.PARENT))
1231 afwDet.setMaskFromFootprintList(maskClip, result.clipFootprints, maskClipValue)
1233 maskClipBig = maskClip.Factory(mask.getBBox(afwImage.PARENT))
1234 afwDet.setMaskFromFootprintList(maskClipBig, bigFootprints, maskClipValue)
1235 maskClip |= maskClipBig
1238 badMaskPlanes = self.config.badMaskPlanes[:]
1239 badMaskPlanes.append(
"CLIPPED")
1240 badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes)
1241 retStruct = AssembleCoaddTask.assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList,
1242 bgModelList, result.tempExpClipList, mask=badPixelMask)
1246 maskExp = retStruct.coaddExposure.getMaskedImage().getMask()
1253 \brief Return an exposure that contains the difference between and unclipped and clipped coadds. 1255 Generate a difference image between clipped and unclipped coadds. 1256 Compute the difference image by subtracting an outlier-clipped coadd from an outlier-unclipped coadd. 1257 Return the difference image. 1259 @param skyInfo: Patch geometry information, from getSkyInfo 1260 @param tempExpRefList: List of data reference to tempExp 1261 @param imageScalerList: List of image scalers 1262 @param weightList: List of weights 1263 @param bgModelList: List of background models from background matching 1264 @return Difference image of unclipped and clipped coadd wrapped in an Exposure 1269 configIntersection = {k: getattr(self.config, k)
1270 for k, v
in self.config.toDict().items()
if (k
in config.keys())}
1271 config.update(**configIntersection)
1274 config.statistic =
'MEAN' 1276 coaddMean = task.assemble(skyInfo, tempExpRefList, imageScalerList, weightList,
1277 bgModelList).coaddExposure
1279 config.statistic =
'MEANCLIP' 1281 coaddClip = task.assemble(skyInfo, tempExpRefList, imageScalerList, weightList,
1282 bgModelList).coaddExposure
1284 coaddDiff = coaddMean.getMaskedImage().Factory(coaddMean.getMaskedImage())
1285 coaddDiff -= coaddClip.getMaskedImage()
1286 exp = afwImage.ExposureF(coaddDiff)
1287 exp.setPsf(coaddMean.getPsf())
1292 \brief Detect clipped regions on an exposure and set the mask on the individual tempExp masks 1294 Detect footprints in the difference image after smoothing the difference image with a Gaussian kernal. 1295 Identify footprints that overlap with one or two input coaddTempExps by comparing the computed overlap 1296 fraction to thresholds set in the config. 1297 A different threshold is applied depending on the number of overlapping visits (restricted to one or 1299 If the overlap exceeds the thresholds, the footprint is considered "CLIPPED" and is marked as such on 1301 Return a struct with the clipped footprints, the indices of the coaddTempExps that end up overlapping 1302 with the clipped footprints and a list of new masks for the coaddTempExps. 1304 \param[in] exp: Exposure to run detection on 1305 \param[in] tempExpRefList: List of data reference to tempExp 1306 \return struct containing: 1307 - clippedFootprints: list of clipped footprints 1308 - clippedIndices: indices for each clippedFootprint in tempExpRefList 1309 - tempExpClipList: list of new masks for tempExp 1311 mask = exp.getMaskedImage().getMask()
1312 maskClipValue = mask.getPlaneBitMask(
"CLIPPED")
1313 maskDetValue = mask.getPlaneBitMask(
"DETECTED") | mask.getPlaneBitMask(
"DETECTED_NEGATIVE")
1314 fpSet = self.clipDetection.detectFootprints(exp, doSmooth=
True, clearMask=
True)
1316 fpSet.positive.merge(fpSet.negative)
1317 footprints = fpSet.positive
1318 self.log.info(
'Found %d potential clipped objects', len(footprints.getFootprints()))
1326 immediate=
True).getMaskedImage().getMask()
for 1327 tmpExpRef
in tempExpRefList]
1329 for footprint
in footprints.getFootprints():
1330 nPixel = footprint.getArea()
1334 for i, tmpExpMask
in enumerate(tempExpClipList):
1338 totPixel = nPixel - ignore
1341 if ignore > overlapDet
or totPixel <= 0.5*nPixel
or overlapDet == 0:
1343 overlap.append(overlapDet/float(totPixel))
1344 maskList.append(tmpExpMask)
1347 overlap = numpy.array(overlap)
1348 if not len(overlap):
1355 if len(overlap) == 1:
1356 if overlap[0] > self.config.minClipFootOverlapSingle:
1361 clipIndex = numpy.where(overlap > self.config.minClipFootOverlap)[0]
1362 if len(clipIndex) == 1:
1364 keepIndex = [clipIndex[0]]
1367 clipIndex = numpy.where(overlap > self.config.minClipFootOverlapDouble)[0]
1368 if len(clipIndex) == 2
and len(overlap) > 3:
1369 clipIndexComp = numpy.where(overlap <= self.config.minClipFootOverlapDouble)[0]
1370 if numpy.max(overlap[clipIndexComp]) <= self.config.maxClipFootOverlapDouble:
1372 keepIndex = clipIndex
1377 for index
in keepIndex:
1378 footprint.spans.setMask(maskList[index], maskClipValue)
1380 clipIndices.append(numpy.array(indexList)[keepIndex])
1381 clipFootprints.append(footprint)
1383 return pipeBase.Struct(clipFootprints=clipFootprints, clipIndices=clipIndices,
1384 tempExpClipList=tempExpClipList)
1386 def detectClipBig(self, tempExpClipList, clipFootprints, clipIndices, maskClipValue, maskDetValue):
1388 \brief Find footprints from individual tempExp footprints for large footprints. 1390 Identify big footprints composed of many sources in the coadd difference that may have originated in a 1391 large diffuse source in the coadd. We do this by indentifying all clipped footprints that overlap 1392 significantly with each source in all the coaddTempExps. 1394 \param[in] tempExpClipList: List of tempExp masks with clipping information 1395 \param[in] clipFootprints: List of clipped footprints 1396 \param[in] clipIndices: List of which entries in tempExpClipList each footprint belongs to 1397 \param[in] maskClipValue: Mask value of clipped pixels 1398 \param[in] maskClipValue: Mask value of detected pixels 1399 \return list of big footprints 1401 bigFootprintsCoadd = []
1403 for index, tmpExpMask
in enumerate(tempExpClipList):
1406 maskVisitDet = tmpExpMask.Factory(tmpExpMask, tmpExpMask.getBBox(afwImage.PARENT),
1407 afwImage.PARENT,
True)
1408 maskVisitDet &= maskDetValue
1409 visitFootprints = afwDet.FootprintSet(maskVisitDet, afwDet.Threshold(1))
1412 clippedFootprintsVisit = []
1413 for foot, clipIndex
in zip(clipFootprints, clipIndices):
1414 if index
not in clipIndex:
1416 clippedFootprintsVisit.append(foot)
1417 maskVisitClip = maskVisitDet.Factory(maskVisitDet.getBBox(afwImage.PARENT))
1418 afwDet.setMaskFromFootprintList(maskVisitClip, clippedFootprintsVisit, maskClipValue)
1420 bigFootprintsVisit = []
1421 for foot
in visitFootprints.getFootprints():
1422 if foot.getArea() < self.config.minBigOverlap:
1425 if nCount > self.config.minBigOverlap:
1426 bigFootprintsVisit.append(foot)
1427 bigFootprintsCoadd.append(foot)
1430 maskVisitClip.clearAllMaskPlanes()
1431 afwDet.setMaskFromFootprintList(maskVisitClip, bigFootprintsVisit, maskClipValue)
1432 tmpExpMask |= maskVisitClip
1434 return bigFootprintsCoadd
1438 assembleStaticSkyModel = pexConfig.ConfigurableField(
1439 target=AssembleCoaddTask,
1440 doc=
"Task to assemble an artifact-free, PSF-matched Coadd to serve as a" 1441 " naive/first-iteration model of the static sky.",
1443 chiThreshold = pexConfig.RangeField(
1444 doc=
"Detection threshold (sigma) for artifacts in warp diff " 1445 "(chi-image of PSF-Matched warp minus the model of the static sky)",
1450 minPixels = pexConfig.Field(
1451 doc=
"Minimum number of pixels in a region (in a warp diff chi-image) " 1452 "above chiThreshold to trigger masking. " 1453 "Detected artifacts with fewer than the specified number of pixels will be ignored.",
1457 doMaskNegative = pexConfig.Field(
1458 doc=
"Also mask outlier regions of flux less than the static sky model?",
1462 temporalThreshold = pexConfig.Field(
1463 doc=
"Unitless fraction of number of epochs to classify as an artifact/outlier source versus" 1464 " a source that is intrinsically variable or difficult to subtract cleanly. " 1465 "If outlier region in warp-diff Chi-image is mostly (defined by spatialThreshold) " 1466 "an outlier in less than temporalThreshold * number of epochs, then mask. " 1467 "Otherwise, do not mask.",
1471 spatialThreshold = pexConfig.Field(
1472 doc=
"Unitless fraction of pixels defining how much of the outlier region has to meet the " 1473 "temporal criteria",
1477 growMaskBy = pexConfig.Field(
1478 doc=
"Number of pixels by which to grow the masks on artifacts.",
1484 AssembleCoaddConfig.setDefaults(self)
1500 \anchor CompareWarpAssembleCoaddTask_ 1502 \brief Assemble a compareWarp coadded image from a set of warps 1503 by masking artifacts detected by comparing PSF-matched warps 1505 \section pipe_tasks_assembleCoadd_Contents Contents 1506 - \ref pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Purpose 1507 - \ref pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Initialize 1508 - \ref pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Run 1509 - \ref pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Config 1510 - \ref pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Debug 1511 - \ref pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Example 1513 \section pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Purpose Description 1515 \copybrief CompareWarpAssembleCoaddTask 1517 In \ref AssembleCoaddTask_ "AssembleCoaddTask", we compute the coadd as an clipped mean (i.e. we clip 1519 The problem with doing this is that when computing the coadd PSF at a given location, individual visit 1520 PSFs from visits with outlier pixels contribute to the coadd PSF and cannot be treated correctly. 1521 In this task, we correct for this behavior by creating a new badMaskPlane 'CLIPPED' which marks 1522 pixels in the individual warps suspected to contain an artifact. 1523 We populate this plane on the input warps by comparing PSF-matched warps with a PSF-matched median coadd 1524 which serves as a model of the static sky. Any group of pixels that deviates from the PSF-matched 1525 median coadd by more than config.chiThreshold sigma, is an artifact candidate. The candidates are then 1526 filtered to remove variable sources and sources that are difficult to subtract such as bright stars. 1527 This filter is configured using the config parameters temporalThreshold and spatialThreshold. 1528 The temporalThreshold is the maximum fraction of epochs that the deviation can 1529 appear in and still be considered an artifact. The spatialThreshold is the maximum fraction of pixels in 1530 the footprint of the deviation that appear in other epochs (where other epochs is defined by the 1531 temporalThreshold). If the deviant region meets this criteria of having a significant percentage of pixels 1532 that deviate in only a few epochs, it is grown by growMaskBy and bits set in the 'CLIPPED' plane. 1533 These regions will not contribute to the final coadd. 1534 Furthermore, any routine to determine the coadd PSF can now be cognizant of clipped regions. 1535 Note that the algorithm implemented by this task is preliminary and works correctly for HSC data. 1536 Parameter modifications and or considerable redesigning of the algorithm is likley required for other 1539 CompareWarpAssembleCoaddTask sub-classes 1540 \ref AssembleCoaddTask_ "AssembleCoaddTask" and instantiates \ref AssembleCoaddTask_ "AssembleCoaddTask" 1541 as a subtask to generate the TemplateCoadd (the model of the static sky) 1543 \section pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Initialize Task initialization 1544 \copydoc \_\_init\_\_ 1546 \section pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Run Invoking the Task 1549 \section pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Config Configuration parameters 1550 See \ref CompareWarpAssembleCoaddConfig 1552 \section pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Debug Debug variables 1553 The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a 1554 flag \c -d to import \b debug.py from your \c PYTHONPATH; see \ref baseDebug for more about \b debug.py 1556 CompareWarpAssembleCoaddTask has no debug variables of its own. 1558 \section pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Example A complete example of using 1559 CompareWarpAssembleCoaddTask 1561 CompareWarpAssembleCoaddTask assembles a set of warped images into a coadded image. 1562 The CompareWarpAssembleCoaddTask is invoked by running assembleCoadd.py with the flag 1563 '--compareWarpCoadd'. 1564 Usage of assembleCoadd.py expects a data reference to the tract patch and filter to be coadded 1565 (specified using '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]') along 1566 with a list of coaddTempExps to attempt to coadd (specified using 1567 '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]'). 1568 Only the warps that cover the specified tract and patch will be coadded. 1569 A list of the available optional arguments can be obtained by calling assembleCoadd.py with the --help 1570 command line argument: 1572 assembleCoadd.py --help 1574 To demonstrate usage of the CompareWarpAssembleCoaddTask in the larger context of multi-band processing, 1575 we will generate the HSC-I & -R band coadds from HSC engineering test data provided in the ci_hsc package. 1576 To begin, assuming that the lsst stack has been already set up, we must set up the obs_subaru and ci_hsc 1578 This defines the environment variable $CI_HSC_DIR and points at the location of the package. The raw HSC 1579 data live in the $CI_HSC_DIR/raw directory. To begin assembling the coadds, we must first 1582 <DD> process the individual ccds in $CI_HSC_RAW to produce calibrated exposures</DD> 1584 <DD> create a skymap that covers the area of the sky present in the raw exposures</DD> 1585 <DT>makeCoaddTempExp</DT> 1586 <DD> warp the individual calibrated exposures to the tangent plane of the coadd</DD> 1588 We can perform all of these steps by running 1590 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988 1592 This will produce warped coaddTempExps for each visit. To coadd the warped data, we call assembleCoadd.py 1595 assembleCoadd.py --compareWarpCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \ 1596 --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \ 1597 --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \ 1598 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \ 1599 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \ 1600 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \ 1601 --selectId visit=903988 ccd=24 1603 This will process the HSC-I band data. The results are written in 1604 `$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I`. 1606 ConfigClass = CompareWarpAssembleCoaddConfig
1607 _DefaultName =
"compareWarpAssembleCoadd" 1611 \brief Initialize the task and make the \ref AssembleCoadd_ "assembleStaticSkyModel" subtask. 1613 AssembleCoaddTask.__init__(self, *args, **kwargs)
1614 self.makeSubtask(
"assembleStaticSkyModel")
1618 \brief Make inputs specific to Subclass 1620 Generate a templateCoadd to use as a native model of static sky to subtract from warps. 1622 templateCoadd = self.assembleStaticSkyModel.
run(dataRef, selectDataList).coaddExposure
1623 return pipeBase.Struct(templateCoadd=templateCoadd)
1625 def assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList, bgModelList,
1626 supplementaryData, *args, **kwargs):
1628 \brief Assemble the coadd 1630 Requires additional inputs Struct `supplementaryData` to contain a `templateCoadd` that serves 1631 as the model of the static sky. 1633 Find artifacts and apply them to the warps' masks creating a list of alternative masks with a 1634 new "CLIPPED" plane and updated "NO_DATA" plane. 1635 Then pass these alternative masks to the base class's assemble method. 1637 @param skyInfo: Patch geometry information 1638 @param tempExpRefList: List of data references to warps 1639 @param imageScalerList: List of image scalers 1640 @param weightList: List of weights 1641 @param bgModelList: List of background models from background matching 1642 @param supplementaryData: PipeBase.Struct containing a templateCoadd 1644 return pipeBase.Struct with coaddExposure, nImage if requested 1646 templateCoadd = supplementaryData.templateCoadd
1647 spanSetMaskList = self.
findArtifacts(templateCoadd, tempExpRefList, imageScalerList)
1649 badMaskPlanes = self.config.badMaskPlanes[:]
1650 badMaskPlanes.append(
"CLIPPED")
1651 badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes)
1653 retStruct = AssembleCoaddTask.assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList,
1654 bgModelList, maskList, mask=badPixelMask)
1659 \brief Find artifacts 1661 Loop through warps twice. The first loop builds a map with the count of how many 1662 epochs each pixel deviates from the templateCoadd by more than config.chiThreshold sigma. 1663 The second loop takes each difference image and filters the artifacts detected 1664 in each using count map to filter out variable sources and sources that are difficult to 1667 @param templateCoadd: Exposure to serve as model of static sky 1668 @param tempExpRefList: List of data references to warps 1669 @param imageScalerList: List of image scalers 1672 self.log.debug(
"Generating Count Image. First loop through warps")
1673 epochCountImage = afwImage.ImageU(templateCoadd.getBBox())
1674 for warpRef, imageScaler
in zip(tempExpRefList, imageScalerList):
1680 epochCountImage += chiOneMap
1682 self.log.debug(
"Generating Mask List. Second loop through warps")
1683 spanSetArtifactList = []
1684 spanSetNoDataMaskList = []
1686 maxNumEpochs = int(max(1, self.config.temporalThreshold*len(tempExpRefList)))
1687 for warpRef, imageScaler
in zip(tempExpRefList, imageScalerList):
1692 outliers = afwImage.makeMaskFromArray(chiOneMap.astype(afwImage.MaskPixel))
1693 outliers.setXY0(mi.getXY0())
1694 spanSetList = afwGeom.SpanSet.fromMask(outliers).split()
1696 chiIm = afwImage.ImageF(templateCoadd.getBBox(), numpy.nan)
1703 nans = numpy.where(numpy.isnan(chiIm.array), 1, 0)
1704 nansMask = afwImage.makeMaskFromArray(nans.astype(afwImage.MaskPixel))
1705 nansMask.setXY0(chiIm.getXY0())
1706 spanSetNoDataMask = afwGeom.SpanSet.fromMask(nansMask).split()
1709 maxNumEpochs=maxNumEpochs)
1710 dilatedSpanSetList = [s.dilated(self.config.growMaskBy, afwGeom.Stencil.CIRCLE)
1711 for s
in filteredSpanSetList]
1712 spanSetArtifactList.append(dilatedSpanSetList)
1713 spanSetNoDataMaskList.append(spanSetNoDataMask)
1714 return pipeBase.Struct(artifacts=spanSetArtifactList,
1715 noData=spanSetNoDataMaskList)
1719 \brief Apply artifact span set lists to masks 1721 @param tempExpRefList: List of data references to warps 1722 @param maskSpanSets: Struct containing artifact and noData spanSet lists to apply 1724 return List of alternative masks 1726 Add artifact span set list as "CLIPPED" plane and NaNs to existing "NO_DATA" plane 1728 spanSetMaskList = maskSpanSets.artifacts
1729 spanSetNoDataList = maskSpanSets.noData
1731 for warpRef, artifacts, noData
in zip(tempExpRefList, spanSetMaskList, spanSetNoDataList):
1733 mask = warp.maskedImage.mask
1734 maskClipValue = mask.addMaskPlane(
"CLIPPED")
1735 noDataValue = mask.addMaskPlane(
"NO_DATA")
1736 for artifact
in artifacts:
1737 artifact.clippedTo(mask.getBBox()).setMask(mask, 2**maskClipValue)
1738 for noDataRegion
in noData:
1739 noDataRegion.clippedTo(mask.getBBox()).setMask(mask, 2**noDataValue)
1740 altMaskList.append(mask)
1743 def _filterArtifacts(self, spanSetList, epochCountImage, maxNumEpochs=None, minPixels=None):
1744 if minPixels
is None:
1745 minPixels = self.config.minPixels
1746 maskSpanSetList = []
1747 x0, y0 = epochCountImage.getXY0()
1748 for i, span
in enumerate(spanSetList):
1749 y, x = span.indices()
1750 if len(y) < minPixels:
1752 counts = epochCountImage.array[[y1 - y0
for y1
in y], [x1 - x0
for x1
in x]]
1753 idx = numpy.where((counts > 0) & (counts <= maxNumEpochs))
1754 percentBelowThreshold = len(idx[0]) / len(counts)
1755 if percentBelowThreshold > self.config.spatialThreshold:
1756 maskSpanSetList.append(span)
1757 return maskSpanSetList
1759 def _snrToBinaryArr(self, arrIn):
1760 with numpy.errstate(invalid=
'ignore'):
1761 arr = numpy.where(arrIn >= self.config.chiThreshold, 1, 0)
1762 if self.config.doMaskNegative:
1763 arr[numpy.where(arrIn <= -self.config.chiThreshold)] = 1
1766 def _snrToBinaryIm(self, im):
1768 return afwImage.ImageU(arr.astype(numpy.uint16), xy0=im.getXY0())
1770 def _makeChiIm(self, maskedImage):
1771 chiIm = maskedImage.image.Factory(1./numpy.sqrt(maskedImage.variance.array))
1772 chiIm *= maskedImage.image
1773 chiIm.setXY0(maskedImage.getXY0())
1776 def _readAndComputeWarpDiff(self, warpRef, imageScaler, templateCoadd):
1779 if not warpRef.datasetExists(warpName):
1780 self.log.warn(
"Could not find %s %s; skipping it", warpName, warpRef.dataId)
1784 imageScaler.scaleMaskedImage(warp.getMaskedImage())
1785 mi = warp.getMaskedImage()
1786 mi -= templateCoadd.getMaskedImage()
def setBrightObjectMasks(self, exposure, dataId, brightObjectMasks)
def getCoaddDatasetName(self, warpType="direct")
def getGroupDataRef(butler, datasetType, groupTuple, keys)
Base class for coaddition.
def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList)
Find artifacts.
def assembleMetadata(self, coaddExposure, tempExpRefList, weightList)
Set the metadata for the coadd.
def makeSupplementaryData(self, dataRef, selectDataList)
Make additional inputs to assemble() specific to subclasses.
def makeSupplementaryData(self, dataRef, selectDataList)
Make inputs specific to Subclass.
def assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList, bgInfoList=None, altMaskList=None, mask=None, supplementaryData=None)
Assemble a coadd from input warps.
def backgroundMatching(self, inputData, refExpDataRef=None, refImageScaler=None)
Perform background matching on the prepared inputs.
def getTempExpRefList(self, patchRef, calExpRefList)
Generate list data references corresponding to warped exposures that lie within the patch to be coadd...
def run(self, dataRef, selectDataList=[])
Assemble a coadd from a set of Warps.
def _readAndComputeWarpDiff(self, warpRef, imageScaler, templateCoadd)
def assembleSubregion(self, coaddExposure, bbox, tempExpRefList, imageScalerList, weightList, bgInfoList, altMaskList, statsFlags, statsCtrl, nImage=None)
Assemble the coadd for a sub-region.
def prepareInputs(self, refList)
Prepare the input warps for coaddition by measuring the weight for each warp and the scaling for the ...
def assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList, bgModelList, args, kwargs)
Assemble the coadd for a region.
def getSkyInfo(self, patchRef)
Use getSkyinfo to return the skyMap, tract and patch information, wcs and the outer bbox of the patch...
def getTempExpDatasetName(self, warpType="direct")
def _makeChiIm(self, maskedImage)
def __init__(self, args, kwargs)
Initialize the task and make the assembleStaticSkyModel subtask.
def makeDataRefList(self, namespace)
Make self.refList from self.idList.
def getBadPixelMask(self)
Convenience method to provide the bitmask from the mask plane names.
def detectClip(self, exp, tempExpRefList)
Detect clipped regions on an exposure and set the mask on the individual tempExp masks.
Configuration parameters for the SafeClipAssembleCoaddTask.
def _snrToBinaryIm(self, im)
def _snrToBinaryArr(self, arrIn)
def __init__(self, args, kwargs)
Initialize the task.
Assemble a coadded image from a set of warps (coadded temporary exposures).
def buildDifferenceImage(self, skyInfo, tempExpRefList, imageScalerList, weightList, bgModelList)
Return an exposure that contains the difference between and unclipped and clipped coadds...
def computeAltMaskList(self, tempExpRefList, maskSpanSets)
Apply artifact span set lists to masks.
Assemble a coadded image from a set of coadded temporary exposures, being careful to clip & flag area...
def selectExposures(self, patchRef, skyInfo=None, selectDataList=[])
Select exposures to coadd.
def detectClipBig(self, tempExpClipList, clipFootprints, clipIndices, maskClipValue, maskDetValue)
Find footprints from individual tempExp footprints for large footprints.
def readBrightObjectMasks(self, dataRef)
Configuration parameters for the AssembleCoaddTask.
Assemble a compareWarp coadded image from a set of warps by masking artifacts detected by comparing P...
def _filterArtifacts(self, spanSetList, epochCountImage, maxNumEpochs=None, minPixels=None)
def __init__(self, args, kwargs)
Initialize the task and make the clipDetection subtask.
def getBackgroundReferenceScaler(self, dataRef)
Construct an image scaler for the background reference frame.
def addBackgroundMatchingMetadata(self, coaddExposure, tempExpRefList, backgroundInfoList)
Add metadata from the background matching to the coadd.
A version of lsst.pipe.base.DataIdContainer specialized for assembleCoadd.
def countMaskFromFootprint(mask, footprint, bitmask, ignoreMask)
Function to count the number of pixels with a specific mask in a footprint.
def assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList, bgModelList, supplementaryData, args, kwargs)
Assemble the coadd.
def groupPatchExposures(patchDataRef, calexpDataRefList, coaddDatasetType="deepCoadd", tempExpDatasetType="deepCoadd_directWarp")