1 from __future__
import absolute_import, division, print_function
2 from builtins
import zip
3 from builtins
import range
39 from .coaddBase
import CoaddBaseTask, SelectDataIdContainer
40 from .interpImage
import InterpImageTask
41 from .scaleZeroPoint
import ScaleZeroPointTask
42 from .coaddHelpers
import groupPatchExposures, getGroupDataRef
43 from .scaleVariance
import ScaleVarianceTask
46 __all__ = [
"AssembleCoaddTask",
"SafeClipAssembleCoaddTask",
"CompareWarpAssembleCoaddTask"]
51 \anchor AssembleCoaddConfig_ 53 \brief Configuration parameters for the \ref AssembleCoaddTask_ "AssembleCoaddTask" 55 warpType = pexConfig.Field(
56 doc=
"Warp name: one of 'direct' or 'psfMatched'",
60 subregionSize = pexConfig.ListField(
62 doc=
"Width, height of stack subregion size; " 63 "make small enough that a full stack of images will fit into memory at once.",
67 statistic = pexConfig.Field(
69 doc=
"Main stacking statistic for aggregating over the epochs.",
72 doSigmaClip = pexConfig.Field(
74 doc=
"Perform sigma clipped outlier rejection with MEANCLIP statistic? (DEPRECATED)",
77 sigmaClip = pexConfig.Field(
79 doc=
"Sigma for outlier rejection; ignored if non-clipping statistic selected.",
82 clipIter = pexConfig.Field(
84 doc=
"Number of iterations of outlier rejection; ignored if non-clipping statistic selected.",
87 calcErrorFromInputVariance = pexConfig.Field(
89 doc=
"Calculate coadd variance from input variance by stacking statistic." 90 "Passed to StatisticsControl.setCalcErrorFromInputVariance()",
93 scaleZeroPoint = pexConfig.ConfigurableField(
94 target=ScaleZeroPointTask,
95 doc=
"Task to adjust the photometric zero point of the coadd temp exposures",
97 doInterp = pexConfig.Field(
98 doc=
"Interpolate over NaN pixels? Also extrapolate, if necessary, but the results are ugly.",
102 interpImage = pexConfig.ConfigurableField(
103 target=InterpImageTask,
104 doc=
"Task to interpolate (and extrapolate) over NaN pixels",
106 doWrite = pexConfig.Field(
107 doc=
"Persist coadd?",
111 doNImage = pexConfig.Field(
112 doc=
"Create image of number of contributing exposures for each pixel",
116 doUsePsfMatchedPolygons = pexConfig.Field(
117 doc=
"Use ValidPolygons from shrunk Psf-Matched Calexps? Should be set to True by CompareWarp only.",
121 maskPropagationThresholds = pexConfig.DictField(
124 doc=(
"Threshold (in fractional weight) of rejection at which we propagate a mask plane to " 125 "the coadd; that is, we set the mask bit on the coadd if the fraction the rejected frames " 126 "would have contributed exceeds this value."),
127 default={
"SAT": 0.1},
129 removeMaskPlanes = pexConfig.ListField(dtype=str, default=[
"NOT_DEBLENDED"],
130 doc=
"Mask planes to remove before coadding")
139 doMaskBrightObjects = pexConfig.Field(dtype=bool, default=
False,
140 doc=
"Set mask and flag bits for bright objects?")
141 brightObjectMaskName = pexConfig.Field(dtype=str, default=
"BRIGHT_OBJECT",
142 doc=
"Name of mask bit used for bright objects")
144 coaddPsf = pexConfig.ConfigField(
145 doc=
"Configuration for CoaddPsf",
146 dtype=measAlg.CoaddPsfConfig,
148 doAttachTransmissionCurve = pexConfig.Field(
149 dtype=bool, default=
False, optional=
False,
150 doc=(
"Attach a piecewise TransmissionCurve for the coadd? " 151 "(requires all input Exposures to have TransmissionCurves).")
155 CoaddBaseTask.ConfigClass.setDefaults(self)
159 CoaddBaseTask.ConfigClass.validate(self)
163 log.warn(
"Config doPsfMatch deprecated. Setting warpType='psfMatched'")
166 log.warn(
'doSigmaClip deprecated. To replicate behavior, setting statistic to "MEANCLIP"')
168 if self.
doInterp and self.
statistic not in [
'MEAN',
'MEDIAN',
'MEANCLIP',
'VARIANCE',
'VARIANCECLIP']:
169 raise ValueError(
"Must set doInterp=False for statistic=%s, which does not " 170 "compute and set a non-zero coadd variance estimate." % (self.
statistic))
172 unstackableStats = [
'NOTHING',
'ERROR',
'ORMASK']
173 if not hasattr(afwMath.Property, self.
statistic)
or self.
statistic in unstackableStats:
174 stackableStats = [str(k)
for k
in afwMath.Property.__members__.keys()
175 if str(k)
not in unstackableStats]
176 raise ValueError(
"statistic %s is not allowed. Please choose one of %s." 188 \anchor AssembleCoaddTask_ 190 \brief Assemble a coadded image from a set of warps (coadded temporary exposures). 192 \section pipe_tasks_assembleCoadd_Contents Contents 193 - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Purpose 194 - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Initialize 195 - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Run 196 - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Config 197 - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Debug 198 - \ref pipe_tasks_assembleCoadd_AssembleCoaddTask_Example 200 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Purpose Description 202 \copybrief AssembleCoaddTask_ 204 We want to assemble a coadded image from a set of Warps (also called 205 coadded temporary exposures or coaddTempExps. 206 Each input Warp covers a patch on the sky and corresponds to a single run/visit/exposure of the 207 covered patch. We provide the task with a list of Warps (selectDataList) from which it selects 208 Warps that cover the specified patch (pointed at by dataRef). 209 Each Warp that goes into a coadd will typically have an independent photometric zero-point. 210 Therefore, we must scale each Warp to set it to a common photometric zeropoint. 211 WarpType may be one of 'direct' or 'psfMatched', and the boolean configs config.makeDirect and 212 config.makePsfMatched set which of the warp types will be coadded. 213 The coadd is computed as a mean with optional outlier rejection. 214 Criteria for outlier rejection are set in \ref AssembleCoaddConfig. Finally, Warps can have bad 'NaN' 215 pixels which received no input from the source calExps. We interpolate over these bad (NaN) pixels. 217 AssembleCoaddTask uses several sub-tasks. These are 219 <DT>\ref ScaleZeroPointTask_ "ScaleZeroPointTask"</DT> 220 <DD> create and use an imageScaler object to scale the photometric zeropoint for each Warp</DD> 221 <DT>\ref InterpImageTask_ "InterpImageTask"</DT> 222 <DD>interpolate across bad pixels (NaN) in the final coadd</DD> 224 You can retarget these subtasks if you wish. 226 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Initialize Task initialization 227 \copydoc \_\_init\_\_ 229 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Run Invoking the Task 232 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Config Configuration parameters 233 See \ref AssembleCoaddConfig_ 235 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Debug Debug variables 236 The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a 237 flag \c -d to import \b debug.py from your \c PYTHONPATH; see \ref baseDebug for more about \b debug.py files. 238 AssembleCoaddTask has no debug variables of its own. Some of the subtasks may support debug variables. See 239 the documetation for the subtasks for further information. 241 \section pipe_tasks_assembleCoadd_AssembleCoaddTask_Example A complete example of using AssembleCoaddTask 243 AssembleCoaddTask assembles a set of warped images into a coadded image. The AssembleCoaddTask 244 can be invoked by running assembleCoadd.py with the flag '--legacyCoadd'. Usage of assembleCoadd.py expects 245 a data reference to the tract patch and filter to be coadded (specified using 246 '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]') along with a list of 247 Warps to attempt to coadd (specified using 248 '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]'). Only the Warps 249 that cover the specified tract and patch will be coadded. A list of the available optional 250 arguments can be obtained by calling assembleCoadd.py with the --help command line argument: 252 assembleCoadd.py --help 254 To demonstrate usage of the AssembleCoaddTask in the larger context of multi-band processing, we will generate 255 the HSC-I & -R band coadds from HSC engineering test data provided in the ci_hsc package. To begin, assuming 256 that the lsst stack has been already set up, we must set up the obs_subaru and ci_hsc packages. 257 This defines the environment variable $CI_HSC_DIR and points at the location of the package. The raw HSC 258 data live in the $CI_HSC_DIR/raw directory. To begin assembling the coadds, we must first 261 <DD> process the individual ccds in $CI_HSC_RAW to produce calibrated exposures</DD> 263 <DD> create a skymap that covers the area of the sky present in the raw exposures</DD> 264 <DT>makeCoaddTempExp</DT> 265 <DD> warp the individual calibrated exposures to the tangent plane of the coadd</DD> 267 We can perform all of these steps by running 269 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988 271 This will produce warped exposures for each visit. To coadd the warped data, we call assembleCoadd.py as 274 assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \ 275 --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \ 276 --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \ 277 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \ 278 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \ 279 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \ 280 --selectId visit=903988 ccd=24 282 that will process the HSC-I band data. The results are written in 283 `$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I`. 285 You may also choose to run: 287 scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346 288 assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R \ 289 --selectId visit=903334 ccd=16 --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 \ 290 --selectId visit=903334 ccd=100 --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 \ 291 --selectId visit=903338 ccd=18 --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 \ 292 --selectId visit=903342 ccd=10 --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 \ 293 --selectId visit=903344 ccd=5 --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 \ 294 --selectId visit=903346 ccd=6 --selectId visit=903346 ccd=12 296 to generate the coadd for the HSC-R band if you are interested in following multiBand Coadd processing as 297 discussed in \ref pipeTasks_multiBand (but note that normally, one would use the 298 \ref SafeClipAssembleCoaddTask_ "SafeClipAssembleCoaddTask" rather than AssembleCoaddTask to make the coadd. 300 ConfigClass = AssembleCoaddConfig
301 _DefaultName =
"assembleCoadd" 305 \brief Initialize the task. Create the \ref InterpImageTask "interpImage", 306 & \ref ScaleZeroPointTask "scaleZeroPoint" subtasks. 308 CoaddBaseTask.__init__(self, *args, **kwargs)
309 self.makeSubtask(
"interpImage")
310 self.makeSubtask(
"scaleZeroPoint")
312 if self.config.doMaskBrightObjects:
313 mask = afwImage.Mask()
316 except pexExceptions.LsstCppException:
317 raise RuntimeError(
"Unable to define mask plane for bright objects; planes used are %s" %
318 mask.getMaskPlaneDict().keys())
324 def run(self, dataRef, selectDataList=[]):
326 \brief Assemble a coadd from a set of Warps 328 Coadd a set of Warps. Compute weights to be applied to each Warp and find scalings to 329 match the photometric zeropoint to a reference Warp. Assemble the Warps using 330 \ref assemble. Interpolate over NaNs and optionally write the coadd to disk. Return the coadded 334 \param[in] dataRef: Data reference defining the patch for coaddition and the reference Warp 335 (if config.autoReference=False). Used to access the following data products: 336 - [in] self.config.coaddName + "Coadd_skyMap" 337 - [in] self.config.coaddName + "Coadd_ + <warpType> + "Warp" (optionally) 338 - [out] self.config.coaddName + "Coadd" 339 \param[in] selectDataList[in]: List of data references to Warps. Data to be coadded will be 340 selected from this list based on overlap with the patch defined by dataRef. 342 \return a pipeBase.Struct with fields: 343 - coaddExposure: coadded exposure 344 - nImage: exposure count image 347 calExpRefList = self.
selectExposures(dataRef, skyInfo, selectDataList=selectDataList)
348 if len(calExpRefList) == 0:
349 self.log.warn(
"No exposures to coadd")
351 self.log.info(
"Coadding %d exposures", len(calExpRefList))
355 self.log.info(
"Found %d %s", len(inputData.tempExpRefList),
357 if len(inputData.tempExpRefList) == 0:
358 self.log.warn(
"No coadd temporary exposures found")
363 retStruct = self.
assemble(skyInfo, inputData.tempExpRefList, inputData.imageScalerList,
364 inputData.weightList, supplementaryData=supplementaryData)
366 if self.config.doInterp:
367 self.interpImage.
run(retStruct.coaddExposure.getMaskedImage(), planeName=
"NO_DATA")
369 varArray = retStruct.coaddExposure.getMaskedImage().getVariance().getArray()
370 with numpy.errstate(invalid=
"ignore"):
371 varArray[:] = numpy.where(varArray > 0, varArray, numpy.inf)
373 if self.config.doMaskBrightObjects:
377 if self.config.doWrite:
380 if self.config.doNImage
and retStruct.nImage
is not None:
387 \brief Make additional inputs to assemble() specific to subclasses. 389 Available to be implemented by subclasses only if they need the 390 coadd dataRef for performing preliminary processing before 391 assembling the coadd. 397 \brief Generate list data references corresponding to warped exposures that lie within the 400 \param[in] patchRef: Data reference for patch 401 \param[in] calExpRefList: List of data references for input calexps 402 \return List of Warp/CoaddTempExp data references 404 butler = patchRef.getButler()
405 groupData =
groupPatchExposures(patchRef, calExpRefList, self.getCoaddDatasetName(self.warpType),
406 self.getTempExpDatasetName(self.warpType))
407 tempExpRefList = [
getGroupDataRef(butler, self.getTempExpDatasetName(self.warpType),
408 g, groupData.keys)
for 409 g
in groupData.groups.keys()]
410 return tempExpRefList
414 \brief Prepare the input warps for coaddition by measuring the weight for each warp and the scaling 415 for the photometric zero point. 417 Each Warp has its own photometric zeropoint and background variance. Before coadding these 418 Warps together, compute a scale factor to normalize the photometric zeropoint and compute the 419 weight for each Warp. 421 \param[in] refList: List of data references to tempExp 423 - tempExprefList: List of data references to tempExp 424 - weightList: List of weightings 425 - imageScalerList: List of image scalers 427 statsCtrl = afwMath.StatisticsControl()
428 statsCtrl.setNumSigmaClip(self.config.sigmaClip)
429 statsCtrl.setNumIter(self.config.clipIter)
431 statsCtrl.setNanSafe(
True)
439 for tempExpRef
in refList:
440 if not tempExpRef.datasetExists(tempExpName):
441 self.log.warn(
"Could not find %s %s; skipping it", tempExpName, tempExpRef.dataId)
444 tempExp = tempExpRef.get(tempExpName, immediate=
True)
445 maskedImage = tempExp.getMaskedImage()
446 imageScaler = self.scaleZeroPoint.computeImageScaler(
451 imageScaler.scaleMaskedImage(maskedImage)
452 except Exception
as e:
453 self.log.warn(
"Scaling failed for %s (skipping it): %s", tempExpRef.dataId, e)
455 statObj = afwMath.makeStatistics(maskedImage.getVariance(), maskedImage.getMask(),
456 afwMath.MEANCLIP, statsCtrl)
457 meanVar, meanVarErr = statObj.getResult(afwMath.MEANCLIP)
458 weight = 1.0 / float(meanVar)
459 if not numpy.isfinite(weight):
460 self.log.warn(
"Non-finite weight for %s: skipping", tempExpRef.dataId)
462 self.log.info(
"Weight of %s %s = %0.3f", tempExpName, tempExpRef.dataId, weight)
467 tempExpRefList.append(tempExpRef)
468 weightList.append(weight)
469 imageScalerList.append(imageScaler)
471 return pipeBase.Struct(tempExpRefList=tempExpRefList, weightList=weightList,
472 imageScalerList=imageScalerList)
474 def assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList,
475 altMaskList=None, mask=None, supplementaryData=None):
477 \anchor AssembleCoaddTask.assemble_ 479 \brief Assemble a coadd from input warps 481 Assemble the coadd using the provided list of coaddTempExps. Since the full coadd covers a patch (a 482 large area), the assembly is performed over small areas on the image at a time in order to 483 conserve memory usage. Iterate over subregions within the outer bbox of the patch using 484 \ref assembleSubregion to stack the corresponding subregions from the coaddTempExps with the 485 statistic specified. Set the edge bits the coadd mask based on the weight map. 487 \param[in] skyInfo: Patch geometry information, from getSkyInfo 488 \param[in] tempExpRefList: List of data references to Warps (previously called CoaddTempExps) 489 \param[in] imageScalerList: List of image scalers 490 \param[in] weightList: List of weights 491 \param[in] altMaskList: List of alternate masks to use rather than those stored with tempExp, or None 492 \param[in] mask: Mask to ignore when coadding 493 \param[in] supplementaryData: pipeBase.Struct with additional data products needed to assemble coadd. 494 Only used by subclasses that implement makeSupplementaryData and override assemble. 495 \return pipeBase.Struct with coaddExposure, nImage if requested 498 self.log.info(
"Assembling %s %s", len(tempExpRefList), tempExpName)
502 statsCtrl = afwMath.StatisticsControl()
503 statsCtrl.setNumSigmaClip(self.config.sigmaClip)
504 statsCtrl.setNumIter(self.config.clipIter)
505 statsCtrl.setAndMask(mask)
506 statsCtrl.setNanSafe(
True)
507 statsCtrl.setWeighted(
True)
508 statsCtrl.setCalcErrorFromInputVariance(self.config.calcErrorFromInputVariance)
509 for plane, threshold
in self.config.maskPropagationThresholds.items():
510 bit = afwImage.Mask.getMaskPlane(plane)
511 statsCtrl.setMaskPropagationThreshold(bit, threshold)
513 statsFlags = afwMath.stringToStatisticsProperty(self.config.statistic)
515 if altMaskList
is None:
516 altMaskList = [
None]*len(tempExpRefList)
518 coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
519 coaddExposure.setCalib(self.scaleZeroPoint.getCalib())
520 coaddExposure.getInfo().setCoaddInputs(self.inputRecorder.makeCoaddInputs())
522 coaddMaskedImage = coaddExposure.getMaskedImage()
523 subregionSizeArr = self.config.subregionSize
524 subregionSize = afwGeom.Extent2I(subregionSizeArr[0], subregionSizeArr[1])
526 if self.config.doNImage:
527 nImage = afwImage.ImageU(skyInfo.bbox)
530 for subBBox
in _subBBoxIter(skyInfo.bbox, subregionSize):
533 weightList, altMaskList, statsFlags, statsCtrl,
535 except Exception
as e:
536 self.log.fatal(
"Cannot compute coadd %s: %s", subBBox, e)
541 coaddUtils.setCoaddEdgeBits(coaddMaskedImage.getMask(), coaddMaskedImage.getVariance())
542 return pipeBase.Struct(coaddExposure=coaddExposure, nImage=nImage)
546 \brief Set the metadata for the coadd 548 This basic implementation simply sets the filter from the 551 \param[in] coaddExposure: The target image for the coadd 552 \param[in] tempExpRefList: List of data references to tempExp 553 \param[in] weightList: List of weights 555 assert len(tempExpRefList) == len(weightList),
"Length mismatch" 560 tempExpList = [tempExpRef.get(tempExpName +
"_sub",
561 bbox=afwGeom.Box2I(afwGeom.Point2I(0, 0), afwGeom.Extent2I(1, 1)),
562 imageOrigin=
"LOCAL", immediate=
True)
for tempExpRef
in tempExpRefList]
563 numCcds = sum(len(tempExp.getInfo().getCoaddInputs().ccds)
for tempExp
in tempExpList)
565 coaddExposure.setFilter(tempExpList[0].getFilter())
566 coaddInputs = coaddExposure.getInfo().getCoaddInputs()
567 coaddInputs.ccds.reserve(numCcds)
568 coaddInputs.visits.reserve(len(tempExpList))
570 for tempExp, weight
in zip(tempExpList, weightList):
571 self.inputRecorder.addVisitToCoadd(coaddInputs, tempExp, weight)
573 if self.config.doUsePsfMatchedPolygons:
576 coaddInputs.visits.sort()
582 modelPsfList = [tempExp.getPsf()
for tempExp
in tempExpList]
583 modelPsfWidthList = [modelPsf.computeBBox().getWidth()
for modelPsf
in modelPsfList]
584 psf = modelPsfList[modelPsfWidthList.index(max(modelPsfWidthList))]
586 psf = measAlg.CoaddPsf(coaddInputs.ccds, coaddExposure.getWcs(),
587 self.config.coaddPsf.makeControl())
588 coaddExposure.setPsf(psf)
589 apCorrMap = measAlg.makeCoaddApCorrMap(coaddInputs.ccds, coaddExposure.getBBox(afwImage.PARENT),
590 coaddExposure.getWcs())
591 coaddExposure.getInfo().setApCorrMap(apCorrMap)
592 if self.config.doAttachTransmissionCurve:
593 transmissionCurve = measAlg.makeCoaddTransmissionCurve(coaddExposure.getWcs(), coaddInputs.ccds)
594 coaddExposure.getInfo().setTransmissionCurve(transmissionCurve)
596 def assembleSubregion(self, coaddExposure, bbox, tempExpRefList, imageScalerList, weightList,
597 altMaskList, statsFlags, statsCtrl, nImage=None):
599 \brief Assemble the coadd for a sub-region. 601 For each coaddTempExp, check for (and swap in) an alternative mask if one is passed. Remove mask 602 planes listed in config.removeMaskPlanes, Finally, stack the actual exposures using 603 \ref afwMath.statisticsStack "statisticsStack" with the statistic specified 604 by statsFlags. Typically, the statsFlag will be one of afwMath.MEAN for a mean-stack or 605 afwMath.MEANCLIP for outlier rejection using an N-sigma clipped mean where N and iterations 606 are specified by statsCtrl. Assign the stacked subregion back to the coadd. 608 \param[in] coaddExposure: The target image for the coadd 609 \param[in] bbox: Sub-region to coadd 610 \param[in] tempExpRefList: List of data reference to tempExp 611 \param[in] imageScalerList: List of image scalers 612 \param[in] weightList: List of weights 613 \param[in] altMaskList: List of alternate masks to use rather than those stored with tempExp, or None 614 Each element is dict with keys = mask plane name to which to add the spans 615 \param[in] statsFlags: afwMath.Property object for statistic for coadd 616 \param[in] statsCtrl: Statistics control object for coadd 617 \param[in] nImage: optional ImageU keeps track of exposure count for each pixel 619 self.log.debug(
"Computing coadd over %s", bbox)
621 coaddExposure.mask.addMaskPlane(
"REJECTED")
622 coaddExposure.mask.addMaskPlane(
"CLIPPED")
623 coaddExposure.mask.addMaskPlane(
"SENSOR_EDGE")
628 edge = afwImage.Mask.getPlaneBitMask(
"EDGE")
629 noData = afwImage.Mask.getPlaneBitMask(
"NO_DATA")
630 clipped = afwImage.Mask.getPlaneBitMask(
"CLIPPED")
631 toReject = statsCtrl.getAndMask() & (~noData) & (~edge) & (~clipped)
632 maskMap = [(toReject, coaddExposure.mask.getPlaneBitMask(
"REJECTED")),
633 (edge, coaddExposure.mask.getPlaneBitMask(
"SENSOR_EDGE")),
636 if nImage
is not None:
637 subNImage = afwImage.ImageU(bbox.getWidth(), bbox.getHeight())
638 for tempExpRef, imageScaler, altMask
in zip(tempExpRefList, imageScalerList, altMaskList):
639 exposure = tempExpRef.get(tempExpName +
"_sub", bbox=bbox)
640 maskedImage = exposure.getMaskedImage()
641 mask = maskedImage.getMask()
642 if altMask
is not None:
644 imageScaler.scaleMaskedImage(maskedImage)
648 if nImage
is not None:
649 subNImage.getArray()[maskedImage.getMask().getArray() & statsCtrl.getAndMask() == 0] += 1
650 if self.config.removeMaskPlanes:
651 mask = maskedImage.getMask()
652 for maskPlane
in self.config.removeMaskPlanes:
654 mask &= ~mask.getPlaneBitMask(maskPlane)
655 except Exception
as e:
656 self.log.warn(
"Unable to remove mask plane %s: %s", maskPlane, e.args[0])
658 maskedImageList.append(maskedImage)
660 with self.timer(
"stack"):
661 coaddSubregion = afwMath.statisticsStack(maskedImageList, statsFlags, statsCtrl, weightList,
664 coaddExposure.maskedImage.assign(coaddSubregion, bbox)
665 if nImage
is not None:
666 nImage.assign(subNImage, bbox)
670 \brief Apply in place alt mask formatted as SpanSets to a mask 672 @param mask: original mask 673 @param altMaskSpans: Dict containing spanSet lists to apply. 674 Each element contains the new mask plane name 675 (e.g. "CLIPPED and/or "NO_DATA") as the key, 676 and list of SpanSets to apply to the mask 678 if self.config.doUsePsfMatchedPolygons:
679 if (
"NO_DATA" in altMaskSpans)
and (
"NO_DATA" in self.config.badMaskPlanes):
684 for spanSet
in altMaskSpans[
'NO_DATA']:
685 spanSet.clippedTo(mask.getBBox()).clearMask(mask, self.
getBadPixelMask())
687 for plane, spanSetList
in altMaskSpans.items():
688 maskClipValue = mask.addMaskPlane(plane)
689 for spanSet
in spanSetList:
690 spanSet.clippedTo(mask.getBBox()).setMask(mask, 2**maskClipValue)
695 \brief Shrink coaddInputs' ccds' ValidPolygons in place 697 @param coaddInputs: original mask 699 Either modify each ccd's validPolygon in place, or if CoaddInputs does not 700 have a validPolygon, create one from its bbox. 702 for ccd
in coaddInputs.ccds:
703 polyOrig = ccd.getValidPolygon()
704 validPolyBBox = polyOrig.getBBox()
if polyOrig
else ccd.getBBox()
705 validPolyBBox.grow(-self.config.matchingKernelSize//2)
707 validPolygon = polyOrig.intersectionSingle(validPolyBBox)
709 validPolygon = afwGeom.polygon.Polygon(afwGeom.Box2D(validPolyBBox))
710 ccd.setValidPolygon(validPolygon)
713 """Returns None on failure""" 715 return dataRef.get(
"brightObjectMask", immediate=
True)
716 except Exception
as e:
717 self.log.warn(
"Unable to read brightObjectMask for %s: %s", dataRef.dataId, e)
721 """Set the bright object masks 723 exposure: Exposure under consideration 724 dataId: Data identifier dict for patch 725 brightObjectMasks: afwTable of bright objects to mask 730 if brightObjectMasks
is None:
731 self.log.warn(
"Unable to apply bright object mask: none supplied")
733 self.log.info(
"Applying %d bright object masks to %s", len(brightObjectMasks), dataId)
734 md = brightObjectMasks.table.getMetadata()
737 self.log.warn(
"Expected to see %s in metadata", k)
739 if md.get(k) != dataId[k]:
740 self.log.warn(
"Expected to see %s == %s in metadata, saw %s", k, md.get(k), dataId[k])
742 mask = exposure.getMaskedImage().getMask()
743 wcs = exposure.getWcs()
744 plateScale = wcs.getPixelScale().asArcseconds()
746 for rec
in brightObjectMasks:
747 center = afwGeom.PointI(wcs.skyToPixel(rec.getCoord()))
748 if rec[
"type"] ==
"box":
749 assert rec[
"angle"] == 0.0, (
"Angle != 0 for mask object %s" % rec[
"id"])
750 width = rec[
"width"].asArcseconds()/plateScale
751 height = rec[
"height"].asArcseconds()/plateScale
753 halfSize = afwGeom.ExtentI(0.5*width, 0.5*height)
754 bbox = afwGeom.Box2I(center - halfSize, center + halfSize)
756 bbox = afwGeom.BoxI(afwGeom.PointI(int(center[0] - 0.5*width), int(center[1] - 0.5*height)),
757 afwGeom.PointI(int(center[0] + 0.5*width), int(center[1] + 0.5*height)))
758 spans = afwGeom.SpanSet(bbox)
759 elif rec[
"type"] ==
"circle":
760 radius = int(rec[
"radius"].asArcseconds()/plateScale)
761 spans = afwGeom.SpanSet.fromShape(radius, offset=center)
763 self.log.warn(
"Unexpected region type %s at %s" % rec[
"type"], center)
768 """Set INEXACT_PSF mask plane 770 If any of the input images isn't represented in the coadd (due to 771 clipped pixels or chip gaps), the `CoaddPsf` will be inexact. Flag 776 mask : `lsst.afw.image.Mask` 777 Coadded exposure's mask, modified in-place. 779 mask.addMaskPlane(
"INEXACT_PSF")
780 inexactPsf = mask.getPlaneBitMask(
"INEXACT_PSF")
781 sensorEdge = mask.getPlaneBitMask(
"SENSOR_EDGE")
782 clipped = mask.getPlaneBitMask(
"CLIPPED")
783 rejected = mask.getPlaneBitMask(
"REJECTED")
784 array = mask.getArray()
785 selected = array & (sensorEdge | clipped | rejected) > 0
786 array[selected] |= inexactPsf
789 def _makeArgumentParser(cls):
791 \brief Create an argument parser 794 parser.add_id_argument(
"--id", cls.
ConfigClass().coaddName +
"Coadd_" +
796 help=
"data ID, e.g. --id tract=12345 patch=1,2",
797 ContainerClass=AssembleCoaddDataIdContainer)
798 parser.add_id_argument(
"--selectId",
"calexp", help=
"data ID, e.g. --selectId visit=6789 ccd=0..9",
799 ContainerClass=SelectDataIdContainer)
803 def _subBBoxIter(bbox, subregionSize):
805 \brief Iterate over subregions of a bbox 807 \param[in] bbox: bounding box over which to iterate: afwGeom.Box2I 808 \param[in] subregionSize: size of sub-bboxes 810 \return subBBox: next sub-bounding box of size subregionSize or smaller; 811 each subBBox is contained within bbox, so it may be smaller than subregionSize at the edges of bbox, 812 but it will never be empty 815 raise RuntimeError(
"bbox %s is empty" % (bbox,))
816 if subregionSize[0] < 1
or subregionSize[1] < 1:
817 raise RuntimeError(
"subregionSize %s must be nonzero" % (subregionSize,))
819 for rowShift
in range(0, bbox.getHeight(), subregionSize[1]):
820 for colShift
in range(0, bbox.getWidth(), subregionSize[0]):
821 subBBox = afwGeom.Box2I(bbox.getMin() + afwGeom.Extent2I(colShift, rowShift), subregionSize)
823 if subBBox.isEmpty():
824 raise RuntimeError(
"Bug: empty bbox! bbox=%s, subregionSize=%s, colShift=%s, rowShift=%s" %
825 (bbox, subregionSize, colShift, rowShift))
831 \brief A version of lsst.pipe.base.DataIdContainer specialized for assembleCoadd. 836 \brief Make self.refList from self.idList. 838 datasetType = namespace.config.coaddName +
"Coadd" 839 keysCoadd = namespace.butler.getKeys(datasetType=datasetType, level=self.level)
841 for dataId
in self.idList:
843 for key
in keysCoadd:
844 if key
not in dataId:
845 raise RuntimeError(
"--id must include " + key)
847 dataRef = namespace.butler.dataRef(
848 datasetType=datasetType,
851 self.refList.append(dataRef)
856 \brief Function to count the number of pixels with a specific mask in a footprint. 858 Find the intersection of mask & footprint. Count all pixels in the mask that are in the intersection that 859 have bitmask set but do not have ignoreMask set. Return the count. 861 \param[in] mask: mask to define intersection region by. 862 \parma[in] footprint: footprint to define the intersection region by. 863 \param[in] bitmask: specific mask that we wish to count the number of occurances of. 864 \param[in] ignoreMask: pixels to not consider. 865 \return count of number of pixels in footprint with specified mask. 867 bbox = footprint.getBBox()
868 bbox.clip(mask.getBBox(afwImage.PARENT))
869 fp = afwImage.Mask(bbox)
870 subMask = mask.Factory(mask, bbox, afwImage.PARENT)
871 footprint.spans.setMask(fp, bitmask)
872 return numpy.logical_and((subMask.getArray() & fp.getArray()) > 0,
873 (subMask.getArray() & ignoreMask) == 0).sum()
878 \anchor SafeClipAssembleCoaddConfig 880 \brief Configuration parameters for the SafeClipAssembleCoaddTask 882 clipDetection = pexConfig.ConfigurableField(
883 target=SourceDetectionTask,
884 doc=
"Detect sources on difference between unclipped and clipped coadd")
885 minClipFootOverlap = pexConfig.Field(
886 doc=
"Minimum fractional overlap of clipped footprint with visit DETECTED to be clipped",
890 minClipFootOverlapSingle = pexConfig.Field(
891 doc=
"Minimum fractional overlap of clipped footprint with visit DETECTED to be " 892 "clipped when only one visit overlaps",
896 minClipFootOverlapDouble = pexConfig.Field(
897 doc=
"Minimum fractional overlap of clipped footprints with visit DETECTED to be " 898 "clipped when two visits overlap",
902 maxClipFootOverlapDouble = pexConfig.Field(
903 doc=
"Maximum fractional overlap of clipped footprints with visit DETECTED when " 904 "considering two visits",
908 minBigOverlap = pexConfig.Field(
909 doc=
"Minimum number of pixels in footprint to use DETECTED mask from the single visits " 910 "when labeling clipped footprints",
918 AssembleCoaddConfig.setDefaults(self)
934 log.warn(
"Additional Sigma-clipping not allowed in Safe-clipped Coadds. " 935 "Ignoring doSigmaClip.")
938 raise ValueError(
"Only MEAN statistic allowed for final stacking in SafeClipAssembleCoadd " 939 "(%s chosen). Please set statistic to MEAN." 941 AssembleCoaddTask.ConfigClass.validate(self)
954 \anchor SafeClipAssembleCoaddTask_ 956 \brief Assemble a coadded image from a set of coadded temporary exposures, 957 being careful to clip & flag areas with potential artifacts. 959 \section pipe_tasks_assembleCoadd_Contents Contents 960 - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Purpose 961 - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Initialize 962 - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Run 963 - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Config 964 - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Debug 965 - \ref pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Example 967 \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Purpose Description 969 \copybrief SafeClipAssembleCoaddTask 971 Read the documentation for \ref AssembleCoaddTask_ "AssembleCoaddTask" first since 972 SafeClipAssembleCoaddTask subtasks that task. 973 In \ref AssembleCoaddTask_ "AssembleCoaddTask", we compute the coadd as an clipped mean (i.e. we clip 975 The problem with doing this is that when computing the coadd PSF at a given location, individual visit 976 PSFs from visits with outlier pixels contribute to the coadd PSF and cannot be treated correctly. 977 In this task, we correct for this behavior by creating a new badMaskPlane 'CLIPPED'. 978 We populate this plane on the input coaddTempExps and the final coadd where i. difference imaging suggests 979 that there is an outlier and ii. this outlier appears on only one or two images. 980 Such regions will not contribute to the final coadd. 981 Furthermore, any routine to determine the coadd PSF can now be cognizant of clipped regions. 982 Note that the algorithm implemented by this task is preliminary and works correctly for HSC data. 983 Parameter modifications and or considerable redesigning of the algorithm is likley required for other 986 SafeClipAssembleCoaddTask uses a \ref SourceDetectionTask_ "clipDetection" subtask and also sub-classes 987 \ref AssembleCoaddTask_ "AssembleCoaddTask". You can retarget the 988 \ref SourceDetectionTask_ "clipDetection" subtask if you wish. 990 \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Initialize Task initialization 991 \copydoc \_\_init\_\_ 993 \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Run Invoking the Task 996 \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Config Configuration parameters 997 See \ref SafeClipAssembleCoaddConfig 999 \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Debug Debug variables 1000 The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a 1001 flag \c -d to import \b debug.py from your \c PYTHONPATH; see \ref baseDebug for more about \b debug.py 1003 SafeClipAssembleCoaddTask has no debug variables of its own. The \ref SourceDetectionTask_ "clipDetection" 1004 subtasks may support debug variables. See the documetation for \ref SourceDetectionTask_ "clipDetection" 1005 for further information. 1007 \section pipe_tasks_assembleCoadd_SafeClipAssembleCoaddTask_Example A complete example of using 1008 SafeClipAssembleCoaddTask 1010 SafeClipAssembleCoaddTask assembles a set of warped coaddTempExp images into a coadded image. 1011 The SafeClipAssembleCoaddTask is invoked by running assembleCoadd.py <em>without</em> the flag 1013 Usage of assembleCoadd.py expects a data reference to the tract patch and filter to be coadded 1014 (specified using '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]') along 1015 with a list of coaddTempExps to attempt to coadd (specified using 1016 '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]'). 1017 Only the coaddTempExps that cover the specified tract and patch will be coadded. 1018 A list of the available optional arguments can be obtained by calling assembleCoadd.py with the --help 1019 command line argument: 1021 assembleCoadd.py --help 1023 To demonstrate usage of the SafeClipAssembleCoaddTask in the larger context of multi-band processing, we 1024 will generate the HSC-I & -R band coadds from HSC engineering test data provided in the ci_hsc package. To 1025 begin, assuming that the lsst stack has been already set up, we must set up the obs_subaru and ci_hsc 1027 This defines the environment variable $CI_HSC_DIR and points at the location of the package. The raw HSC 1028 data live in the $CI_HSC_DIR/raw directory. To begin assembling the coadds, we must first 1031 <DD> process the individual ccds in $CI_HSC_RAW to produce calibrated exposures</DD> 1033 <DD> create a skymap that covers the area of the sky present in the raw exposures</DD> 1034 <DT>makeCoaddTempExp</DT> 1035 <DD> warp the individual calibrated exposures to the tangent plane of the coadd</DD> 1037 We can perform all of these steps by running 1039 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988 1041 This will produce warped coaddTempExps for each visit. To coadd the warped data, we call assembleCoadd.py 1044 assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \ 1045 --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \ 1046 --selectId visit=903986 ccd=100--selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \ 1047 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \ 1048 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \ 1049 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \ 1050 --selectId visit=903988 ccd=24 1052 This will process the HSC-I band data. The results are written in 1053 `$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I`. 1055 You may also choose to run: 1057 scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346 1058 assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R --selectId visit=903334 ccd=16 \ 1059 --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 --selectId visit=903334 ccd=100 \ 1060 --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 --selectId visit=903338 ccd=18 \ 1061 --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 --selectId visit=903342 ccd=10 \ 1062 --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 --selectId visit=903344 ccd=5 \ 1063 --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 --selectId visit=903346 ccd=6 \ 1064 --selectId visit=903346 ccd=12 1066 to generate the coadd for the HSC-R band if you are interested in following multiBand Coadd processing as 1067 discussed in \ref pipeTasks_multiBand. 1069 ConfigClass = SafeClipAssembleCoaddConfig
1070 _DefaultName =
"safeClipAssembleCoadd" 1074 \brief Initialize the task and make the \ref SourceDetectionTask_ "clipDetection" subtask. 1076 AssembleCoaddTask.__init__(self, *args, **kwargs)
1077 schema = afwTable.SourceTable.makeMinimalSchema()
1078 self.makeSubtask(
"clipDetection", schema=schema)
1080 def assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList, *args, **kwargs):
1082 \brief Assemble the coadd for a region 1084 Compute the difference of coadds created with and without outlier rejection to identify coadd pixels 1085 that have outlier values in some individual visits. Detect clipped regions on the difference image and 1086 mark these regions on the one or two individual coaddTempExps where they occur if there is significant 1087 overlap between the clipped region and a source. 1088 This leaves us with a set of footprints from the difference image that have been identified as having 1089 occured on just one or two individual visits. However, these footprints were generated from a 1090 difference image. It is conceivable for a large diffuse source to have become broken up into multiple 1091 footprints acrosss the coadd difference in this process. 1092 Determine the clipped region from all overlapping footprints from the detected sources in each visit - 1093 these are big footprints. 1094 Combine the small and big clipped footprints and mark them on a new bad mask plane 1095 Generate the coadd using \ref AssembleCoaddTask.assemble_ "AssembleCoaddTask.assemble" without outlier 1096 removal. Clipped footprints will no longer make it into the coadd because they are marked in the new 1099 N.b. *args and **kwargs are passed but ignored in order to match the call signature expected by the 1102 @param skyInfo: Patch geometry information, from getSkyInfo 1103 @param tempExpRefList: List of data reference to tempExp 1104 @param imageScalerList: List of image scalers 1105 @param weightList: List of weights 1106 return pipeBase.Struct with coaddExposure, nImage 1109 mask = exp.getMaskedImage().getMask()
1110 mask.addMaskPlane(
"CLIPPED")
1112 result = self.
detectClip(exp, tempExpRefList)
1114 self.log.info(
'Found %d clipped objects', len(result.clipFootprints))
1116 maskClipValue = mask.getPlaneBitMask(
"CLIPPED")
1117 maskDetValue = mask.getPlaneBitMask(
"DETECTED") | mask.getPlaneBitMask(
"DETECTED_NEGATIVE")
1119 bigFootprints = self.
detectClipBig(result.clipSpans, result.clipFootprints, result.clipIndices,
1120 result.detectionFootprints, maskClipValue, maskDetValue,
1123 maskClip = mask.Factory(mask.getBBox(afwImage.PARENT))
1124 afwDet.setMaskFromFootprintList(maskClip, result.clipFootprints, maskClipValue)
1126 maskClipBig = maskClip.Factory(mask.getBBox(afwImage.PARENT))
1127 afwDet.setMaskFromFootprintList(maskClipBig, bigFootprints, maskClipValue)
1128 maskClip |= maskClipBig
1131 badMaskPlanes = self.config.badMaskPlanes[:]
1132 badMaskPlanes.append(
"CLIPPED")
1133 badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes)
1134 return AssembleCoaddTask.assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList,
1135 result.clipSpans, mask=badPixelMask)
1139 \brief Return an exposure that contains the difference between and unclipped and clipped coadds. 1141 Generate a difference image between clipped and unclipped coadds. 1142 Compute the difference image by subtracting an outlier-clipped coadd from an outlier-unclipped coadd. 1143 Return the difference image. 1145 @param skyInfo: Patch geometry information, from getSkyInfo 1146 @param tempExpRefList: List of data reference to tempExp 1147 @param imageScalerList: List of image scalers 1148 @param weightList: List of weights 1149 @return Difference image of unclipped and clipped coadd wrapped in an Exposure 1154 configIntersection = {k: getattr(self.config, k)
1155 for k, v
in self.config.toDict().items()
if (k
in config.keys())}
1156 config.update(**configIntersection)
1159 config.statistic =
'MEAN' 1161 coaddMean = task.assemble(skyInfo, tempExpRefList, imageScalerList, weightList).coaddExposure
1163 config.statistic =
'MEANCLIP' 1165 coaddClip = task.assemble(skyInfo, tempExpRefList, imageScalerList, weightList).coaddExposure
1167 coaddDiff = coaddMean.getMaskedImage().Factory(coaddMean.getMaskedImage())
1168 coaddDiff -= coaddClip.getMaskedImage()
1169 exp = afwImage.ExposureF(coaddDiff)
1170 exp.setPsf(coaddMean.getPsf())
1175 \brief Detect clipped regions on an exposure and set the mask on the individual tempExp masks 1177 Detect footprints in the difference image after smoothing the difference image with a Gaussian kernal. 1178 Identify footprints that overlap with one or two input coaddTempExps by comparing the computed overlap 1179 fraction to thresholds set in the config. 1180 A different threshold is applied depending on the number of overlapping visits (restricted to one or 1182 If the overlap exceeds the thresholds, the footprint is considered "CLIPPED" and is marked as such on 1184 Return a struct with the clipped footprints, the indices of the coaddTempExps that end up overlapping 1185 with the clipped footprints and a list of new masks for the coaddTempExps. 1187 \param[in] exp: Exposure to run detection on 1188 \param[in] tempExpRefList: List of data reference to tempExp 1189 \return struct containing: 1190 - clipFootprints: list of clipped footprints 1191 - clipIndices: indices for each clippedFootprint in tempExpRefList 1192 - clipSpans: List of dictionaries containing spanSet lists to clip. Each element contains the new 1193 maskplane name ("CLIPPED")" as the key and list of SpanSets as value 1194 - detectionFootprints: List of DETECTED/DETECTED_NEGATIVE plane compressed into footprints 1196 mask = exp.getMaskedImage().getMask()
1197 maskDetValue = mask.getPlaneBitMask(
"DETECTED") | mask.getPlaneBitMask(
"DETECTED_NEGATIVE")
1198 fpSet = self.clipDetection.detectFootprints(exp, doSmooth=
True, clearMask=
True)
1200 fpSet.positive.merge(fpSet.negative)
1201 footprints = fpSet.positive
1202 self.log.info(
'Found %d potential clipped objects', len(footprints.getFootprints()))
1207 artifactSpanSets = [{
'CLIPPED': list()}
for _
in tempExpRefList]
1210 visitDetectionFootprints = []
1212 dims = [len(tempExpRefList), len(footprints.getFootprints())]
1213 overlapDetArr = numpy.zeros(dims, dtype=numpy.uint16)
1214 ignoreArr = numpy.zeros(dims, dtype=numpy.uint16)
1217 for i, warpRef
in enumerate(tempExpRefList):
1219 immediate=
True).getMaskedImage().getMask()
1220 maskVisitDet = tmpExpMask.Factory(tmpExpMask, tmpExpMask.getBBox(afwImage.PARENT),
1221 afwImage.PARENT,
True)
1222 maskVisitDet &= maskDetValue
1223 visitFootprints = afwDet.FootprintSet(maskVisitDet, afwDet.Threshold(1))
1224 visitDetectionFootprints.append(visitFootprints)
1226 for j, footprint
in enumerate(footprints.getFootprints()):
1231 for j, footprint
in enumerate(footprints.getFootprints()):
1232 nPixel = footprint.getArea()
1235 for i
in range(len(tempExpRefList)):
1236 ignore = ignoreArr[i, j]
1237 overlapDet = overlapDetArr[i, j]
1238 totPixel = nPixel - ignore
1241 if ignore > overlapDet
or totPixel <= 0.5*nPixel
or overlapDet == 0:
1243 overlap.append(overlapDet/float(totPixel))
1246 overlap = numpy.array(overlap)
1247 if not len(overlap):
1254 if len(overlap) == 1:
1255 if overlap[0] > self.config.minClipFootOverlapSingle:
1260 clipIndex = numpy.where(overlap > self.config.minClipFootOverlap)[0]
1261 if len(clipIndex) == 1:
1263 keepIndex = [clipIndex[0]]
1266 clipIndex = numpy.where(overlap > self.config.minClipFootOverlapDouble)[0]
1267 if len(clipIndex) == 2
and len(overlap) > 3:
1268 clipIndexComp = numpy.where(overlap <= self.config.minClipFootOverlapDouble)[0]
1269 if numpy.max(overlap[clipIndexComp]) <= self.config.maxClipFootOverlapDouble:
1271 keepIndex = clipIndex
1276 for index
in keepIndex:
1277 globalIndex = indexList[index]
1278 artifactSpanSets[globalIndex][
'CLIPPED'].append(footprint.spans)
1280 clipIndices.append(numpy.array(indexList)[keepIndex])
1281 clipFootprints.append(footprint)
1283 return pipeBase.Struct(clipFootprints=clipFootprints, clipIndices=clipIndices,
1284 clipSpans=artifactSpanSets, detectionFootprints=visitDetectionFootprints)
1286 def detectClipBig(self, clipList, clipFootprints, clipIndices, detectionFootprints,
1287 maskClipValue, maskDetValue, coaddBBox):
1289 \brief Return individual warp footprints for large artifacts and append them to clipList in place 1291 Identify big footprints composed of many sources in the coadd difference that may have originated in a 1292 large diffuse source in the coadd. We do this by indentifying all clipped footprints that overlap 1293 significantly with each source in all the coaddTempExps. 1294 \param[in] clipList: List of alt mask SpanSets with clipping information. Modified. 1295 \param[in] clipFootprints: List of clipped footprints 1296 \param[in] clipIndices: List of which entries in tempExpClipList each footprint belongs to 1297 \param[in] maskClipValue: Mask value of clipped pixels 1298 \param[in] maskDetValue: Mask value of detected pixels 1299 \param[in] coaddBBox: BBox of the coadd and warps 1300 \return list of big footprints 1302 bigFootprintsCoadd = []
1304 for index, (clippedSpans, visitFootprints)
in enumerate(zip(clipList, detectionFootprints)):
1305 maskVisitDet = afwImage.MaskX(coaddBBox, 0x0)
1306 for footprint
in visitFootprints.getFootprints():
1307 footprint.spans.setMask(maskVisitDet, maskDetValue)
1310 clippedFootprintsVisit = []
1311 for foot, clipIndex
in zip(clipFootprints, clipIndices):
1312 if index
not in clipIndex:
1314 clippedFootprintsVisit.append(foot)
1315 maskVisitClip = maskVisitDet.Factory(maskVisitDet.getBBox(afwImage.PARENT))
1316 afwDet.setMaskFromFootprintList(maskVisitClip, clippedFootprintsVisit, maskClipValue)
1318 bigFootprintsVisit = []
1319 for foot
in visitFootprints.getFootprints():
1320 if foot.getArea() < self.config.minBigOverlap:
1323 if nCount > self.config.minBigOverlap:
1324 bigFootprintsVisit.append(foot)
1325 bigFootprintsCoadd.append(foot)
1327 for footprint
in bigFootprintsVisit:
1328 clippedSpans[
"CLIPPED"].append(footprint.spans)
1330 return bigFootprintsCoadd
1334 assembleStaticSkyModel = pexConfig.ConfigurableField(
1335 target=AssembleCoaddTask,
1336 doc=
"Task to assemble an artifact-free, PSF-matched Coadd to serve as a" 1337 " naive/first-iteration model of the static sky.",
1339 detect = pexConfig.ConfigurableField(
1340 target=SourceDetectionTask,
1341 doc=
"Detect outlier sources on difference between each psfMatched warp and static sky model" 1343 detectTemplate = pexConfig.ConfigurableField(
1344 target=SourceDetectionTask,
1345 doc=
"Detect sources on static sky model. Only used if doPreserveContainedBySource is True" 1347 maxNumEpochs = pexConfig.Field(
1348 doc=
"Charactistic maximum local number of epochs/visits in which an artifact candidate can appear " 1349 "and still be masked. The effective maxNumEpochs is a broken linear function of local " 1350 "number of epochs (N): min(maxFractionEpochsLow*N, maxNumEpochs + maxFractionEpochsHigh*N). " 1351 "For each footprint detected on the image difference between the psfMatched warp and static sky " 1352 "model, if a significant fraction of pixels (defined by spatialThreshold) are residuals in more " 1353 "than the computed effective maxNumEpochs, the artifact candidate is deemed persistant rather " 1354 "than transient and not masked.",
1358 maxFractionEpochsLow = pexConfig.RangeField(
1359 doc=
"Fraction of local number of epochs (N) to use as effective maxNumEpochs for low N. " 1360 "Effective maxNumEpochs = " 1361 "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)",
1366 maxFractionEpochsHigh = pexConfig.RangeField(
1367 doc=
"Fraction of local number of epochs (N) to use as effective maxNumEpochs for high N. " 1368 "Effective maxNumEpochs = " 1369 "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)",
1374 spatialThreshold = pexConfig.RangeField(
1375 doc=
"Unitless fraction of pixels defining how much of the outlier region has to meet the " 1376 "temporal criteria. If 0, clip all. If 1, clip none.",
1380 inclusiveMin=
True, inclusiveMax=
True 1382 doScaleWarpVariance = pexConfig.Field(
1383 doc=
"Rescale Warp variance plane using empirical noise?",
1387 scaleWarpVariance = pexConfig.ConfigurableField(
1388 target=ScaleVarianceTask,
1389 doc=
"Rescale variance on warps",
1391 doPreserveContainedBySource = pexConfig.Field(
1392 doc=
"Rescue artifacts from clipping that completely lie within a footprint detected" 1393 "on the PsfMatched Template Coadd. Replicates a behavior of SafeClip.",
1397 doPrefilterArtifacts = pexConfig.Field(
1398 doc=
"Ignore artifact candidates that are mostly covered by the bad pixel mask, " 1399 "because they will be excluded anyway. This prevents them from contributing " 1400 "to the outlier epoch count image and potentially being labeled as persistant." 1401 "'Mostly' is defined by the config 'prefilterArtifactsRatio'.",
1405 prefilterArtifactsMaskPlanes = pexConfig.ListField(
1406 doc=
"Prefilter artifact candidates that are mostly covered by these bad mask planes.",
1408 default=(
'NO_DATA',
'BAD',
'SAT',
'SUSPECT'),
1410 prefilterArtifactsRatio = pexConfig.Field(
1411 doc=
"Prefilter artifact candidates with less than this fraction overlapping good pixels",
1417 AssembleCoaddConfig.setDefaults(self)
1433 self.
detect.doTempLocalBackground =
False 1434 self.
detect.reEstimateBackground =
False 1435 self.
detect.returnOriginalFootprints =
False 1436 self.
detect.thresholdPolarity =
"both" 1437 self.
detect.thresholdValue = 5
1438 self.
detect.nSigmaToGrow = 2
1439 self.
detect.minPixels = 4
1440 self.
detect.isotropicGrow =
True 1441 self.
detect.thresholdType =
"pixel_stdev" 1457 \anchor CompareWarpAssembleCoaddTask_ 1459 \brief Assemble a compareWarp coadded image from a set of warps 1460 by masking artifacts detected by comparing PSF-matched warps 1462 \section pipe_tasks_assembleCoadd_Contents Contents 1463 - \ref pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Purpose 1464 - \ref pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Initialize 1465 - \ref pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Run 1466 - \ref pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Config 1467 - \ref pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Debug 1468 - \ref pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Example 1470 \section pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Purpose Description 1472 \copybrief CompareWarpAssembleCoaddTask 1474 In \ref AssembleCoaddTask_ "AssembleCoaddTask", we compute the coadd as an clipped mean (i.e. we clip 1476 The problem with doing this is that when computing the coadd PSF at a given location, individual visit 1477 PSFs from visits with outlier pixels contribute to the coadd PSF and cannot be treated correctly. 1478 In this task, we correct for this behavior by creating a new badMaskPlane 'CLIPPED' which marks 1479 pixels in the individual warps suspected to contain an artifact. 1480 We populate this plane on the input warps by comparing PSF-matched warps with a PSF-matched median coadd 1481 which serves as a model of the static sky. Any group of pixels that deviates from the PSF-matched 1482 template coadd by more than config.detect.threshold sigma, is an artifact candidate. 1483 The candidates are then filtered to remove variable sources and sources that are difficult to subtract 1484 such as bright stars. 1485 This filter is configured using the config parameters temporalThreshold and spatialThreshold. 1486 The temporalThreshold is the maximum fraction of epochs that the deviation can 1487 appear in and still be considered an artifact. The spatialThreshold is the maximum fraction of pixels in 1488 the footprint of the deviation that appear in other epochs (where other epochs is defined by the 1489 temporalThreshold). If the deviant region meets this criteria of having a significant percentage of pixels 1490 that deviate in only a few epochs, these pixels have the 'CLIPPED' bit set in the mask. 1491 These regions will not contribute to the final coadd. 1492 Furthermore, any routine to determine the coadd PSF can now be cognizant of clipped regions. 1493 Note that the algorithm implemented by this task is preliminary and works correctly for HSC data. 1494 Parameter modifications and or considerable redesigning of the algorithm is likley required for other 1497 CompareWarpAssembleCoaddTask sub-classes 1498 \ref AssembleCoaddTask_ "AssembleCoaddTask" and instantiates \ref AssembleCoaddTask_ "AssembleCoaddTask" 1499 as a subtask to generate the TemplateCoadd (the model of the static sky) 1501 \section pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Initialize Task initialization 1502 \copydoc \_\_init\_\_ 1504 \section pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Run Invoking the Task 1507 \section pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Config Configuration parameters 1508 See \ref CompareWarpAssembleCoaddConfig 1510 \section pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Debug Debug variables 1511 The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a 1512 flag \c -d to import \b debug.py from your \c PYTHONPATH; see \ref baseDebug for more about \b debug.py 1515 This task supports the following debug variables: 1518 <dd> If True then save the Epoch Count Image as a fits file in the `figPath` 1520 <dd> Path to save the debug fits images and figures 1523 For example, put something like: 1526 def DebugInfo(name): 1527 di = lsstDebug.getInfo(name) 1528 if name == "lsst.pipe.tasks.assembleCoadd": 1529 di.saveCountIm = True 1530 di.figPath = "/desired/path/to/debugging/output/images" 1532 lsstDebug.Info = DebugInfo 1534 into your `debug.py` file and run `assemebleCoadd.py` with the `--debug` 1536 Some subtasks may have their own debug variables; see individual Task 1539 \section pipe_tasks_assembleCoadd_CompareWarpAssembleCoaddTask_Example A complete example of using 1540 CompareWarpAssembleCoaddTask 1542 CompareWarpAssembleCoaddTask assembles a set of warped images into a coadded image. 1543 The CompareWarpAssembleCoaddTask is invoked by running assembleCoadd.py with the flag 1544 '--compareWarpCoadd'. 1545 Usage of assembleCoadd.py expects a data reference to the tract patch and filter to be coadded 1546 (specified using '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]') along 1547 with a list of coaddTempExps to attempt to coadd (specified using 1548 '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]'). 1549 Only the warps that cover the specified tract and patch will be coadded. 1550 A list of the available optional arguments can be obtained by calling assembleCoadd.py with the --help 1551 command line argument: 1553 assembleCoadd.py --help 1555 To demonstrate usage of the CompareWarpAssembleCoaddTask in the larger context of multi-band processing, 1556 we will generate the HSC-I & -R band coadds from HSC engineering test data provided in the ci_hsc package. 1557 To begin, assuming that the lsst stack has been already set up, we must set up the obs_subaru and ci_hsc 1559 This defines the environment variable $CI_HSC_DIR and points at the location of the package. The raw HSC 1560 data live in the $CI_HSC_DIR/raw directory. To begin assembling the coadds, we must first 1563 <DD> process the individual ccds in $CI_HSC_RAW to produce calibrated exposures</DD> 1565 <DD> create a skymap that covers the area of the sky present in the raw exposures</DD> 1566 <DT>makeCoaddTempExp</DT> 1567 <DD> warp the individual calibrated exposures to the tangent plane of the coadd</DD> 1569 We can perform all of these steps by running 1571 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988 1573 This will produce warped coaddTempExps for each visit. To coadd the warped data, we call assembleCoadd.py 1576 assembleCoadd.py --compareWarpCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \ 1577 --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \ 1578 --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \ 1579 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \ 1580 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \ 1581 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \ 1582 --selectId visit=903988 ccd=24 1584 This will process the HSC-I band data. The results are written in 1585 `$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I`. 1587 ConfigClass = CompareWarpAssembleCoaddConfig
1588 _DefaultName =
"compareWarpAssembleCoadd" 1592 \brief Initialize the task and make the \ref AssembleCoadd_ "assembleStaticSkyModel" subtask. 1594 AssembleCoaddTask.__init__(self, *args, **kwargs)
1595 self.makeSubtask(
"assembleStaticSkyModel")
1596 detectionSchema = afwTable.SourceTable.makeMinimalSchema()
1597 self.makeSubtask(
"detect", schema=detectionSchema)
1598 if self.config.doPreserveContainedBySource:
1599 self.makeSubtask(
"detectTemplate", schema=afwTable.SourceTable.makeMinimalSchema())
1600 if self.config.doScaleWarpVariance:
1601 self.makeSubtask(
"scaleWarpVariance")
1605 \brief Make inputs specific to Subclass 1607 Generate a templateCoadd to use as a native model of static sky to subtract from warps. 1609 templateCoadd = self.assembleStaticSkyModel.
run(dataRef, selectDataList)
1611 if templateCoadd
is None:
1612 warpName = (self.assembleStaticSkyModel.warpType[0].upper() +
1613 self.assembleStaticSkyModel.warpType[1:])
1614 message =
"""No %(warpName)s warps were found to build the template coadd which is 1615 required to run CompareWarpAssembleCoaddTask. To continue assembling this type of coadd, 1616 first either rerun makeCoaddTempExp with config.make%(warpName)s=True or 1617 coaddDriver with config.makeCoadTempExp.make%(warpName)s=True, before assembleCoadd. 1619 Alternatively, to use another algorithm with existing warps, retarget the CoaddDriverConfig to 1620 another algorithm like: 1622 from lsst.pipe.tasks.assembleCoadd import SafeClipAssembleCoaddTask 1623 config.assemble.retarget(SafeClipAssembleCoaddTask) 1624 """ % {
"warpName": warpName}
1625 raise RuntimeError(message)
1627 return pipeBase.Struct(templateCoadd=templateCoadd.coaddExposure)
1629 def assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList,
1630 supplementaryData, *args, **kwargs):
1632 \brief Assemble the coadd 1634 Requires additional inputs Struct `supplementaryData` to contain a `templateCoadd` that serves 1635 as the model of the static sky. 1637 Find artifacts and apply them to the warps' masks creating a list of alternative masks with a 1638 new "CLIPPED" plane and updated "NO_DATA" plane. 1639 Then pass these alternative masks to the base class's assemble method. 1641 @param skyInfo: Patch geometry information 1642 @param tempExpRefList: List of data references to warps 1643 @param imageScalerList: List of image scalers 1644 @param weightList: List of weights 1645 @param supplementaryData: PipeBase.Struct containing a templateCoadd 1647 return pipeBase.Struct with coaddExposure, nImage if requested 1649 templateCoadd = supplementaryData.templateCoadd
1650 spanSetMaskList = self.
findArtifacts(templateCoadd, tempExpRefList, imageScalerList)
1651 badMaskPlanes = self.config.badMaskPlanes[:]
1652 badMaskPlanes.append(
"CLIPPED")
1653 badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes)
1655 result = AssembleCoaddTask.assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList,
1656 spanSetMaskList, mask=badPixelMask)
1660 self.
applyAltEdgeMask(result.coaddExposure.maskedImage.mask, spanSetMaskList)
1665 \brief Propagate alt EDGE mask to SENSOR_EDGE AND INEXACT_PSF planes 1667 @param mask: original mask 1668 @param altMaskList: List of Dicts containing spanSet lists. 1669 Each element contains the new mask plane name 1670 (e.g. "CLIPPED and/or "NO_DATA") as the key, 1671 and list of SpanSets to apply to the mask. 1673 maskValue = mask.getPlaneBitMask([
"SENSOR_EDGE",
"INEXACT_PSF"])
1674 for visitMask
in altMaskList:
1675 if "EDGE" in visitMask:
1676 for spanSet
in visitMask[
'EDGE']:
1677 spanSet.clippedTo(mask.getBBox()).setMask(mask, maskValue)
1681 \brief Find artifacts 1683 Loop through warps twice. The first loop builds a map with the count of how many 1684 epochs each pixel deviates from the templateCoadd by more than config.chiThreshold sigma. 1685 The second loop takes each difference image and filters the artifacts detected 1686 in each using count map to filter out variable sources and sources that are difficult to 1689 @param templateCoadd: Exposure to serve as model of static sky 1690 @param tempExpRefList: List of data references to warps 1691 @param imageScalerList: List of image scalers 1694 self.log.debug(
"Generating Count Image, and mask lists.")
1695 coaddBBox = templateCoadd.getBBox()
1696 slateIm = afwImage.ImageU(coaddBBox)
1697 epochCountImage = afwImage.ImageU(coaddBBox)
1698 nImage = afwImage.ImageU(coaddBBox)
1699 spanSetArtifactList = []
1700 spanSetNoDataMaskList = []
1701 spanSetEdgeList = []
1705 templateCoadd.mask.clearAllMaskPlanes()
1707 if self.config.doPreserveContainedBySource:
1708 templateFootprints = self.detectTemplate.detectFootprints(templateCoadd)
1710 templateFootprints =
None 1712 for warpRef, imageScaler
in zip(tempExpRefList, imageScalerList):
1714 if warpDiffExp
is not None:
1716 nImage.array += (numpy.isfinite(warpDiffExp.image.array) *
1717 ((warpDiffExp.mask.array & badPixelMask) == 0)).astype(numpy.uint16)
1718 fpSet = self.detect.detectFootprints(warpDiffExp, doSmooth=
False, clearMask=
True)
1719 fpSet.positive.merge(fpSet.negative)
1720 footprints = fpSet.positive
1722 spanSetList = [footprint.spans
for footprint
in footprints.getFootprints()]
1725 if self.config.doPrefilterArtifacts:
1727 for spans
in spanSetList:
1728 spans.setImage(slateIm, 1, doClip=
True)
1729 epochCountImage += slateIm
1735 nans = numpy.where(numpy.isnan(warpDiffExp.maskedImage.image.array), 1, 0)
1736 nansMask = afwImage.makeMaskFromArray(nans.astype(afwImage.MaskPixel))
1737 nansMask.setXY0(warpDiffExp.getXY0())
1738 edgeMask = warpDiffExp.mask
1739 spanSetEdgeMask = afwGeom.SpanSet.fromMask(edgeMask,
1740 edgeMask.getPlaneBitMask(
"EDGE")).split()
1744 nansMask = afwImage.MaskX(coaddBBox, 1)
1746 spanSetEdgeMask = []
1748 spanSetNoDataMask = afwGeom.SpanSet.fromMask(nansMask).split()
1750 spanSetNoDataMaskList.append(spanSetNoDataMask)
1751 spanSetArtifactList.append(spanSetList)
1752 spanSetEdgeList.append(spanSetEdgeMask)
1756 epochCountImage.writeFits(path)
1758 for i, spanSetList
in enumerate(spanSetArtifactList):
1760 filteredSpanSetList = self.
filterArtifacts(spanSetList, epochCountImage, nImage,
1762 spanSetArtifactList[i] = filteredSpanSetList
1765 for artifacts, noData, edge
in zip(spanSetArtifactList, spanSetNoDataMaskList, spanSetEdgeList):
1766 altMasks.append({
'CLIPPED': artifacts,
1773 \brief Remove artifact candidates covered by bad mask plane 1775 Any future editing of the candidate list that does not depend on 1776 temporal information should go in this method. 1778 @param spanSetList: List of SpanSets representing artifact candidates 1779 @param exp: Exposure containing mask planes used to prefilter 1781 return List of SpanSets with artifacts 1783 badPixelMask = exp.mask.getPlaneBitMask(self.config.prefilterArtifactsMaskPlanes)
1784 goodArr = (exp.mask.array & badPixelMask) == 0
1785 returnSpanSetList = []
1786 bbox = exp.getBBox()
1787 x0, y0 = exp.getXY0()
1788 for i, span
in enumerate(spanSetList):
1789 y, x = span.clippedTo(bbox).indices()
1790 yIndexLocal = numpy.array(y) - y0
1791 xIndexLocal = numpy.array(x) - x0
1792 goodRatio = numpy.count_nonzero(goodArr[yIndexLocal, xIndexLocal])/span.getArea()
1793 if goodRatio > self.config.prefilterArtifactsRatio:
1794 returnSpanSetList.append(span)
1795 return returnSpanSetList
1797 def filterArtifacts(self, spanSetList, epochCountImage, nImage, footprintsToExclude=None):
1799 \brief Filter artifact candidates 1801 @param spanSetList: List of SpanSets representing artifact candidates 1802 @param epochCountImage: Image of accumulated number of warpDiff detections 1803 @param nImage: Image of the accumulated number of total epochs contributing 1805 return List of SpanSets with artifacts 1808 maskSpanSetList = []
1809 x0, y0 = epochCountImage.getXY0()
1810 for i, span
in enumerate(spanSetList):
1811 y, x = span.indices()
1812 yIdxLocal = [y1 - y0
for y1
in y]
1813 xIdxLocal = [x1 - x0
for x1
in x]
1814 outlierN = epochCountImage.array[yIdxLocal, xIdxLocal]
1815 totalN = nImage.array[yIdxLocal, xIdxLocal]
1818 effMaxNumEpochsHighN = (self.config.maxNumEpochs +
1819 self.config.maxFractionEpochsHigh*numpy.mean(totalN))
1820 effMaxNumEpochsLowN = self.config.maxFractionEpochsLow * numpy.mean(totalN)
1821 effectiveMaxNumEpochs = int(min(effMaxNumEpochsLowN, effMaxNumEpochsHighN))
1822 nPixelsBelowThreshold = numpy.count_nonzero((outlierN > 0) &
1823 (outlierN <= effectiveMaxNumEpochs))
1824 percentBelowThreshold = nPixelsBelowThreshold / len(outlierN)
1825 if percentBelowThreshold > self.config.spatialThreshold:
1826 maskSpanSetList.append(span)
1828 if self.config.doPreserveContainedBySource
and footprintsToExclude
is not None:
1830 filteredMaskSpanSetList = []
1831 for span
in maskSpanSetList:
1833 for footprint
in footprintsToExclude.positive.getFootprints():
1834 if footprint.spans.contains(span):
1838 filteredMaskSpanSetList.append(span)
1839 maskSpanSetList = filteredMaskSpanSetList
1841 return maskSpanSetList
1843 def _readAndComputeWarpDiff(self, warpRef, imageScaler, templateCoadd):
1845 \brief Fetch a warp from the butler and return a warpDiff 1847 @param warpRef: `Butler dataRef` for the warp 1848 @param imageScaler: `scaleZeroPoint.ImageScaler` object 1849 @param templateCoadd: Exposure to be substracted from the scaled warp 1851 return Exposure of the image difference between the warp and template 1856 if not warpRef.datasetExists(warpName):
1857 self.log.warn(
"Could not find %s %s; skipping it", warpName, warpRef.dataId)
1859 warp = warpRef.get(warpName, immediate=
True)
1861 imageScaler.scaleMaskedImage(warp.getMaskedImage())
1862 mi = warp.getMaskedImage()
1863 if self.config.doScaleWarpVariance:
1865 self.scaleWarpVariance.
run(mi)
1866 except Exception
as exc:
1867 self.log.warn(
"Unable to rescale variance of warp (%s); leaving it as-is" % (exc,))
1868 mi -= templateCoadd.getMaskedImage()
1871 def _dataRef2DebugPath(self, prefix, warpRef, coaddLevel=False):
1873 \brief Return a path to which to write debugging output 1875 @param prefix: string, prefix for filename 1876 @param warpRef: Butler dataRef 1877 @param coaddLevel: bool, optional. If True, include only coadd-level keys 1878 (e.g. 'tract', 'patch', 'filter', but no 'visit') 1880 Creates a hyphen-delimited string of dataId values for simple filenames. 1885 keys = warpRef.dataId.keys()
1886 keyList = sorted(keys, reverse=
True)
1888 filename =
"%s-%s.fits" % (prefix,
'-'.join([str(warpRef.dataId[k])
for k
in keyList]))
1889 return os.path.join(directory, filename)
def setBrightObjectMasks(self, exposure, dataId, brightObjectMasks)
def shrinkValidPolygons(self, coaddInputs)
Shrink coaddInputs' ccds' ValidPolygons in place.
def getCoaddDatasetName(self, warpType="direct")
def _dataRef2DebugPath(self, prefix, warpRef, coaddLevel=False)
Return a path to which to write debugging output.
def getGroupDataRef(butler, datasetType, groupTuple, keys)
Base class for coaddition.
def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList)
Find artifacts.
def assembleMetadata(self, coaddExposure, tempExpRefList, weightList)
Set the metadata for the coadd.
def makeSupplementaryData(self, dataRef, selectDataList)
Make additional inputs to assemble() specific to subclasses.
def makeSupplementaryData(self, dataRef, selectDataList)
Make inputs specific to Subclass.
def getTempExpRefList(self, patchRef, calExpRefList)
Generate list data references corresponding to warped exposures that lie within the patch to be coadd...
def assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList, args, kwargs)
Assemble the coadd for a region.
def run(self, dataRef, selectDataList=[])
Assemble a coadd from a set of Warps.
def _readAndComputeWarpDiff(self, warpRef, imageScaler, templateCoadd)
Fetch a warp from the butler and return a warpDiff.
def prepareInputs(self, refList)
Prepare the input warps for coaddition by measuring the weight for each warp and the scaling for the ...
def applyAltMaskPlanes(self, mask, altMaskSpans)
Apply in place alt mask formatted as SpanSets to a mask.
def getSkyInfo(self, patchRef)
Use getSkyinfo to return the skyMap, tract and patch information, wcs and the outer bbox of the patch...
def getTempExpDatasetName(self, warpType="direct")
def __init__(self, args, kwargs)
Initialize the task and make the assembleStaticSkyModel subtask.
def makeDataRefList(self, namespace)
Make self.refList from self.idList.
def getBadPixelMask(self)
Convenience method to provide the bitmask from the mask plane names.
def assembleSubregion(self, coaddExposure, bbox, tempExpRefList, imageScalerList, weightList, altMaskList, statsFlags, statsCtrl, nImage=None)
Assemble the coadd for a sub-region.
def detectClip(self, exp, tempExpRefList)
Detect clipped regions on an exposure and set the mask on the individual tempExp masks.
def setInexactPsf(self, mask)
Configuration parameters for the SafeClipAssembleCoaddTask.
def __init__(self, args, kwargs)
Initialize the task.
Assemble a coadded image from a set of warps (coadded temporary exposures).
def filterArtifacts(self, spanSetList, epochCountImage, nImage, footprintsToExclude=None)
Filter artifact candidates.
Assemble a coadded image from a set of coadded temporary exposures, being careful to clip & flag area...
def buildDifferenceImage(self, skyInfo, tempExpRefList, imageScalerList, weightList)
Return an exposure that contains the difference between and unclipped and clipped coadds...
def selectExposures(self, patchRef, skyInfo=None, selectDataList=[])
Select exposures to coadd.
def applyAltEdgeMask(self, mask, altMaskList)
Propagate alt EDGE mask to SENSOR_EDGE AND INEXACT_PSF planes.
def readBrightObjectMasks(self, dataRef)
Configuration parameters for the AssembleCoaddTask.
Assemble a compareWarp coadded image from a set of warps by masking artifacts detected by comparing P...
def __init__(self, args, kwargs)
Initialize the task and make the clipDetection subtask.
def assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList, supplementaryData, args, kwargs)
Assemble the coadd.
def prefilterArtifacts(self, spanSetList, exp)
Remove artifact candidates covered by bad mask plane.
A version of lsst.pipe.base.DataIdContainer specialized for assembleCoadd.
def countMaskFromFootprint(mask, footprint, bitmask, ignoreMask)
Function to count the number of pixels with a specific mask in a footprint.
def groupPatchExposures(patchDataRef, calexpDataRefList, coaddDatasetType="deepCoadd", tempExpDatasetType="deepCoadd_directWarp")
def assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList, altMaskList=None, mask=None, supplementaryData=None)
Assemble a coadd from input warps.
def detectClipBig(self, clipList, clipFootprints, clipIndices, detectionFootprints, maskClipValue, maskDetValue, coaddBBox)
Return individual warp footprints for large artifacts and append them to clipList in place...