25 import lsst.pex.config
as pexConfig
38 from .coaddBase
import CoaddBaseTask, SelectDataIdContainer, makeSkyInfo
39 from .interpImage
import InterpImageTask
40 from .scaleZeroPoint
import ScaleZeroPointTask
41 from .coaddHelpers
import groupPatchExposures, getGroupDataRef
42 from .scaleVariance
import ScaleVarianceTask
46 __all__ = [
"AssembleCoaddTask",
"AssembleCoaddConfig",
"SafeClipAssembleCoaddTask",
47 "SafeClipAssembleCoaddConfig",
"CompareWarpAssembleCoaddTask",
"CompareWarpAssembleCoaddConfig"]
51 """Configuration parameters for the `AssembleCoaddTask`. 55 The `doMaskBrightObjects` and `brightObjectMaskName` configuration options 56 only set the bitplane config.brightObjectMaskName. To make this useful you 57 *must* also configure the flags.pixel algorithm, for example by adding 61 config.measurement.plugins["base_PixelFlags"].masksFpCenter.append("BRIGHT_OBJECT") 62 config.measurement.plugins["base_PixelFlags"].masksFpAnywhere.append("BRIGHT_OBJECT") 64 to your measureCoaddSources.py and forcedPhotCoadd.py config overrides. 66 warpType = pexConfig.Field(
67 doc=
"Warp name: one of 'direct' or 'psfMatched'",
71 subregionSize = pexConfig.ListField(
73 doc=
"Width, height of stack subregion size; " 74 "make small enough that a full stack of images will fit into memory at once.",
78 statistic = pexConfig.Field(
80 doc=
"Main stacking statistic for aggregating over the epochs.",
83 doSigmaClip = pexConfig.Field(
85 doc=
"Perform sigma clipped outlier rejection with MEANCLIP statistic? (DEPRECATED)",
88 sigmaClip = pexConfig.Field(
90 doc=
"Sigma for outlier rejection; ignored if non-clipping statistic selected.",
93 clipIter = pexConfig.Field(
95 doc=
"Number of iterations of outlier rejection; ignored if non-clipping statistic selected.",
98 calcErrorFromInputVariance = pexConfig.Field(
100 doc=
"Calculate coadd variance from input variance by stacking statistic." 101 "Passed to StatisticsControl.setCalcErrorFromInputVariance()",
104 scaleZeroPoint = pexConfig.ConfigurableField(
105 target=ScaleZeroPointTask,
106 doc=
"Task to adjust the photometric zero point of the coadd temp exposures",
108 doInterp = pexConfig.Field(
109 doc=
"Interpolate over NaN pixels? Also extrapolate, if necessary, but the results are ugly.",
113 interpImage = pexConfig.ConfigurableField(
114 target=InterpImageTask,
115 doc=
"Task to interpolate (and extrapolate) over NaN pixels",
117 doWrite = pexConfig.Field(
118 doc=
"Persist coadd?",
122 doNImage = pexConfig.Field(
123 doc=
"Create image of number of contributing exposures for each pixel",
127 doUsePsfMatchedPolygons = pexConfig.Field(
128 doc=
"Use ValidPolygons from shrunk Psf-Matched Calexps? Should be set to True by CompareWarp only.",
132 maskPropagationThresholds = pexConfig.DictField(
135 doc=(
"Threshold (in fractional weight) of rejection at which we propagate a mask plane to " 136 "the coadd; that is, we set the mask bit on the coadd if the fraction the rejected frames " 137 "would have contributed exceeds this value."),
138 default={
"SAT": 0.1},
140 removeMaskPlanes = pexConfig.ListField(dtype=str, default=[
"NOT_DEBLENDED"],
141 doc=
"Mask planes to remove before coadding")
142 doMaskBrightObjects = pexConfig.Field(dtype=bool, default=
False,
143 doc=
"Set mask and flag bits for bright objects?")
144 brightObjectMaskName = pexConfig.Field(dtype=str, default=
"BRIGHT_OBJECT",
145 doc=
"Name of mask bit used for bright objects")
146 coaddPsf = pexConfig.ConfigField(
147 doc=
"Configuration for CoaddPsf",
148 dtype=measAlg.CoaddPsfConfig,
150 doAttachTransmissionCurve = pexConfig.Field(
151 dtype=bool, default=
False, optional=
False,
152 doc=(
"Attach a piecewise TransmissionCurve for the coadd? " 153 "(requires all input Exposures to have TransmissionCurves).")
155 inputWarps = pipeBase.InputDatasetField(
156 doc=(
"Input list of warps to be assemebled i.e. stacked." 157 "WarpType (e.g. direct, psfMatched) is controlled by we warpType config parameter"),
158 nameTemplate=
"{inputCoaddName}Coadd_{warpType}Warp",
159 storageClass=
"ExposureF",
160 dimensions=(
"tract",
"patch",
"skymap",
"visit",
"instrument"),
163 skyMap = pipeBase.InputDatasetField(
164 doc=
"Input definition of geometry/bbox and projection/wcs for coadded exposures",
165 nameTemplate=
"{inputCoaddName}Coadd_skyMap",
166 storageClass=
"SkyMap",
167 dimensions=(
"skymap", ),
170 brightObjectMask = pipeBase.InputDatasetField(
171 doc=(
"Input Bright Object Mask mask produced with external catalogs to be applied to the mask plane" 173 name=
"brightObjectMask",
174 storageClass=
"ObjectMaskCatalog",
175 dimensions=(
"tract",
"patch",
"skymap",
"abstract_filter"),
178 coaddExposure = pipeBase.OutputDatasetField(
179 doc=
"Output coadded exposure, produced by stacking input warps",
180 nameTemplate=
"{outputCoaddName}Coadd",
181 storageClass=
"ExposureF",
182 dimensions=(
"tract",
"patch",
"skymap",
"abstract_filter"),
185 nImage = pipeBase.OutputDatasetField(
186 doc=
"Output image of number of input images per pixel",
187 nameTemplate=
"{outputCoaddName}Coadd_nImage",
188 storageClass=
"ImageU",
189 dimensions=(
"tract",
"patch",
"skymap",
"abstract_filter"),
193 hasFakes = pexConfig.Field(
196 doc=
"Should be set to True if fake sources have been inserted into the input data." 202 self.formatTemplateNames({
"inputCoaddName":
"deep",
"outputCoaddName":
"deep",
204 self.quantum.dimensions = (
"tract",
"patch",
"abstract_filter",
"skymap")
211 log.warn(
"Config doPsfMatch deprecated. Setting warpType='psfMatched'")
214 log.warn(
'doSigmaClip deprecated. To replicate behavior, setting statistic to "MEANCLIP"')
216 if self.
doInterp and self.
statistic not in [
'MEAN',
'MEDIAN',
'MEANCLIP',
'VARIANCE',
'VARIANCECLIP']:
217 raise ValueError(
"Must set doInterp=False for statistic=%s, which does not " 218 "compute and set a non-zero coadd variance estimate." % (self.
statistic))
220 unstackableStats = [
'NOTHING',
'ERROR',
'ORMASK']
221 if not hasattr(afwMath.Property, self.
statistic)
or self.
statistic in unstackableStats:
222 stackableStats = [str(k)
for k
in afwMath.Property.__members__.keys()
223 if str(k)
not in unstackableStats]
224 raise ValueError(
"statistic %s is not allowed. Please choose one of %s." 229 """Assemble a coadded image from a set of warps (coadded temporary exposures). 231 We want to assemble a coadded image from a set of Warps (also called 232 coadded temporary exposures or ``coaddTempExps``). 233 Each input Warp covers a patch on the sky and corresponds to a single 234 run/visit/exposure of the covered patch. We provide the task with a list 235 of Warps (``selectDataList``) from which it selects Warps that cover the 236 specified patch (pointed at by ``dataRef``). 237 Each Warp that goes into a coadd will typically have an independent 238 photometric zero-point. Therefore, we must scale each Warp to set it to 239 a common photometric zeropoint. WarpType may be one of 'direct' or 240 'psfMatched', and the boolean configs `config.makeDirect` and 241 `config.makePsfMatched` set which of the warp types will be coadded. 242 The coadd is computed as a mean with optional outlier rejection. 243 Criteria for outlier rejection are set in `AssembleCoaddConfig`. 244 Finally, Warps can have bad 'NaN' pixels which received no input from the 245 source calExps. We interpolate over these bad (NaN) pixels. 247 `AssembleCoaddTask` uses several sub-tasks. These are 249 - `ScaleZeroPointTask` 250 - create and use an ``imageScaler`` object to scale the photometric zeropoint for each Warp 252 - interpolate across bad pixels (NaN) in the final coadd 254 You can retarget these subtasks if you wish. 258 The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a 259 flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``; see 260 `baseDebug` for more about ``debug.py`` files. `AssembleCoaddTask` has 261 no debug variables of its own. Some of the subtasks may support debug 262 variables. See the documentation for the subtasks for further information. 266 `AssembleCoaddTask` assembles a set of warped images into a coadded image. 267 The `AssembleCoaddTask` can be invoked by running ``assembleCoadd.py`` 268 with the flag '--legacyCoadd'. Usage of assembleCoadd.py expects two 269 inputs: a data reference to the tract patch and filter to be coadded, and 270 a list of Warps to attempt to coadd. These are specified using ``--id`` and 271 ``--selectId``, respectively: 275 --id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]] 276 --selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]] 278 Only the Warps that cover the specified tract and patch will be coadded. 279 A list of the available optional arguments can be obtained by calling 280 ``assembleCoadd.py`` with the ``--help`` command line argument: 284 assembleCoadd.py --help 286 To demonstrate usage of the `AssembleCoaddTask` in the larger context of 287 multi-band processing, we will generate the HSC-I & -R band coadds from 288 HSC engineering test data provided in the ``ci_hsc`` package. To begin, 289 assuming that the lsst stack has been already set up, we must set up the 290 obs_subaru and ``ci_hsc`` packages. This defines the environment variable 291 ``$CI_HSC_DIR`` and points at the location of the package. The raw HSC 292 data live in the ``$CI_HSC_DIR/raw directory``. To begin assembling the 293 coadds, we must first 296 - process the individual ccds in $CI_HSC_RAW to produce calibrated exposures 298 - create a skymap that covers the area of the sky present in the raw exposures 300 - warp the individual calibrated exposures to the tangent plane of the coadd 302 We can perform all of these steps by running 306 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988 308 This will produce warped exposures for each visit. To coadd the warped 309 data, we call assembleCoadd.py as follows: 313 assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \ 314 --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \ 315 --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \ 316 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \ 317 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \ 318 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \ 319 --selectId visit=903988 ccd=24 321 that will process the HSC-I band data. The results are written in 322 ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``. 324 You may also choose to run: 328 scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346 329 assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R \ 330 --selectId visit=903334 ccd=16 --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 \ 331 --selectId visit=903334 ccd=100 --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 \ 332 --selectId visit=903338 ccd=18 --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 \ 333 --selectId visit=903342 ccd=10 --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 \ 334 --selectId visit=903344 ccd=5 --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 \ 335 --selectId visit=903346 ccd=6 --selectId visit=903346 ccd=12 337 to generate the coadd for the HSC-R band if you are interested in 338 following multiBand Coadd processing as discussed in `pipeTasks_multiBand` 339 (but note that normally, one would use the `SafeClipAssembleCoaddTask` 340 rather than `AssembleCoaddTask` to make the coadd. 342 ConfigClass = AssembleCoaddConfig
343 _DefaultName =
"assembleCoadd" 348 argNames = [
"config",
"name",
"parentTask",
"log"]
349 kwargs.update({k: v
for k, v
in zip(argNames, args)})
350 warnings.warn(
"AssembleCoadd received positional args, and casting them as kwargs: %s. " 351 "PipelineTask will not take positional args" % argNames, FutureWarning)
354 self.makeSubtask(
"interpImage")
355 self.makeSubtask(
"scaleZeroPoint")
357 if self.config.doMaskBrightObjects:
358 mask = afwImage.Mask()
361 except pexExceptions.LsstCppException:
362 raise RuntimeError(
"Unable to define mask plane for bright objects; planes used are %s" %
363 mask.getMaskPlaneDict().keys())
370 """Return output dataset type descriptors 372 Remove output dataset types not produced by the Task 375 if not config.doNImage:
376 outputTypeDict.pop(
"nImage",
None)
377 return outputTypeDict
381 """Return input dataset type descriptors 383 Remove input dataset types not used by the Task 386 if not config.doMaskBrightObjects:
387 inputTypeDict.pop(
"brightObjectMask",
None)
392 return frozenset([
"brightObjectMask"])
395 """Assemble a coadd from a set of Warps. 397 PipelineTask (Gen3) entry point to Coadd a set of Warps. 398 Analogous to `runDataRef`, it prepares all the data products to be 399 passed to `run`, and processes the results before returning to struct 400 of results to be written out. AssembleCoadd cannot fit all Warps in memory. 401 Therefore, its inputs are accessed subregion by subregion 402 by the `lsst.daf.butler.ShimButler` that quacks like a Gen2 403 `lsst.daf.persistence.Butler`. Updates to this method should 404 correspond to an update in `runDataRef` while both entry points 410 Keys are the names of the configs describing input dataset types. 411 Values are input Python-domain data objects (or lists of objects) 412 retrieved from data butler. 413 inputDataIds : `dict` 414 Keys are the names of the configs describing input dataset types. 415 Values are DataIds (or lists of DataIds) that task consumes for 416 corresponding dataset type. 417 outputDataIds : `dict` 418 Keys are the names of the configs describing input dataset types. 419 Values are DataIds (or lists of DataIds) that task is to produce 420 for corresponding dataset type. 421 butler : `lsst.daf.butler.Butler` 422 Gen3 Butler object for fetching additional data products before 427 result : `lsst.pipe.base.Struct` 428 Result struct with components: 430 - ``coaddExposure`` : coadded exposure (``lsst.afw.image.Exposure``) 431 - ``nImage``: N Image (``lsst.afw.image.Image``) 435 skyMap = inputData[
"skyMap"]
436 outputDataId = next(iter(outputDataIds.values()))
438 tractId=outputDataId[
'tract'],
439 patchId=outputDataId[
'patch'])
443 warpRefList = [butlerShim.dataRef(self.config.inputWarps.name, dataId=dataId)
444 for dataId
in inputDataIds[
'inputWarps']]
447 patchRef = butlerShim.dataRef(self.config.coaddExposure.name, dataId=outputDataIds[
'coaddExposure'])
451 self.log.info(
"Found %d %s", len(inputs.tempExpRefList),
453 if len(inputs.tempExpRefList) == 0:
454 self.log.warn(
"No coadd temporary exposures found")
459 retStruct = self.
run(inputData[
'skyInfo'], inputs.tempExpRefList, inputs.imageScalerList,
460 inputs.weightList, supplementaryData=supplementaryData)
466 def runDataRef(self, dataRef, selectDataList=None, warpRefList=None):
467 """Assemble a coadd from a set of Warps. 469 Pipebase.CmdlineTask entry point to Coadd a set of Warps. 470 Compute weights to be applied to each Warp and 471 find scalings to match the photometric zeropoint to a reference Warp. 472 Assemble the Warps using `run`. Interpolate over NaNs and 473 optionally write the coadd to disk. Return the coadded exposure. 477 dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef` 478 Data reference defining the patch for coaddition and the 479 reference Warp (if ``config.autoReference=False``). 480 Used to access the following data products: 481 - ``self.config.coaddName + "Coadd_skyMap"`` 482 - ``self.config.coaddName + "Coadd_ + <warpType> + "Warp"`` (optionally) 483 - ``self.config.coaddName + "Coadd"`` 484 selectDataList : `list` 485 List of data references to Calexps. Data to be coadded will be 486 selected from this list based on overlap with the patch defined 487 by dataRef, grouped by visit, and converted to a list of data 490 List of data references to Warps to be coadded. 491 Note: `warpRefList` is just the new name for `tempExpRefList`. 495 retStruct : `lsst.pipe.base.Struct` 496 Result struct with components: 498 - ``coaddExposure``: coadded exposure (``Exposure``). 499 - ``nImage``: exposure count image (``Image``). 501 if selectDataList
and warpRefList:
502 raise RuntimeError(
"runDataRef received both a selectDataList and warpRefList, " 503 "and which to use is ambiguous. Please pass only one.")
506 if warpRefList
is None:
507 calExpRefList = self.
selectExposures(dataRef, skyInfo, selectDataList=selectDataList)
508 if len(calExpRefList) == 0:
509 self.log.warn(
"No exposures to coadd")
511 self.log.info(
"Coadding %d exposures", len(calExpRefList))
516 self.log.info(
"Found %d %s", len(inputData.tempExpRefList),
518 if len(inputData.tempExpRefList) == 0:
519 self.log.warn(
"No coadd temporary exposures found")
524 retStruct = self.
run(skyInfo, inputData.tempExpRefList, inputData.imageScalerList,
525 inputData.weightList, supplementaryData=supplementaryData)
528 if self.config.doWrite:
533 self.log.info(
"Persisting %s" % coaddDatasetName)
534 dataRef.put(retStruct.coaddExposure, coaddDatasetName)
535 if self.config.doNImage
and retStruct.nImage
is not None:
541 """Interpolate over missing data and mask bright stars. 545 coaddExposure : `lsst.afw.image.Exposure` 546 The coadded exposure to process. 547 dataRef : `lsst.daf.persistence.ButlerDataRef` 548 Butler data reference for supplementary data. 550 if self.config.doInterp:
551 self.interpImage.
run(coaddExposure.getMaskedImage(), planeName=
"NO_DATA")
553 varArray = coaddExposure.variance.array
554 with numpy.errstate(invalid=
"ignore"):
555 varArray[:] = numpy.where(varArray > 0, varArray, numpy.inf)
557 if self.config.doMaskBrightObjects:
562 """Make additional inputs to run() specific to subclasses (Gen2) 564 Duplicates interface of `runDataRef` method 565 Available to be implemented by subclasses only if they need the 566 coadd dataRef for performing preliminary processing before 567 assembling the coadd. 571 dataRef : `lsst.daf.persistence.ButlerDataRef` 572 Butler data reference for supplementary data. 573 selectDataList : `list` 574 List of data references to Warps. 579 """Make additional inputs to run() specific to subclasses (Gen3) 581 Duplicates interface of`adaptArgsAndRun` method. 582 Available to be implemented by subclasses only if they need the 583 coadd dataRef for performing preliminary processing before 584 assembling the coadd. 589 Keys are the names of the configs describing input dataset types. 590 Values are input Python-domain data objects (or lists of objects) 591 retrieved from data butler. 592 inputDataIds : `dict` 593 Keys are the names of the configs describing input dataset types. 594 Values are DataIds (or lists of DataIds) that task consumes for 595 corresponding dataset type. 596 DataIds are guaranteed to match data objects in ``inputData``. 597 outputDataIds : `dict` 598 Keys are the names of the configs describing input dataset types. 599 Values are DataIds (or lists of DataIds) that task is to produce 600 for corresponding dataset type. 601 butler : `lsst.daf.butler.Butler` 602 Gen3 Butler object for fetching additional data products before 607 result : `lsst.pipe.base.Struct` 608 Contains whatever additional data the subclass's `run` method needs 613 """Generate list data references corresponding to warped exposures 614 that lie within the patch to be coadded. 619 Data reference for patch. 620 calExpRefList : `list` 621 List of data references for input calexps. 625 tempExpRefList : `list` 626 List of Warp/CoaddTempExp data references. 628 butler = patchRef.getButler()
629 groupData =
groupPatchExposures(patchRef, calExpRefList, self.getCoaddDatasetName(self.warpType),
630 self.getTempExpDatasetName(self.warpType))
631 tempExpRefList = [
getGroupDataRef(butler, self.getTempExpDatasetName(self.warpType),
632 g, groupData.keys)
for 633 g
in groupData.groups.keys()]
634 return tempExpRefList
637 """Prepare the input warps for coaddition by measuring the weight for 638 each warp and the scaling for the photometric zero point. 640 Each Warp has its own photometric zeropoint and background variance. 641 Before coadding these Warps together, compute a scale factor to 642 normalize the photometric zeropoint and compute the weight for each Warp. 647 List of data references to tempExp 651 result : `lsst.pipe.base.Struct` 652 Result struct with components: 654 - ``tempExprefList``: `list` of data references to tempExp. 655 - ``weightList``: `list` of weightings. 656 - ``imageScalerList``: `list` of image scalers. 658 statsCtrl = afwMath.StatisticsControl()
659 statsCtrl.setNumSigmaClip(self.config.sigmaClip)
660 statsCtrl.setNumIter(self.config.clipIter)
662 statsCtrl.setNanSafe(
True)
670 for tempExpRef
in refList:
671 if not tempExpRef.datasetExists(tempExpName):
672 self.log.warn(
"Could not find %s %s; skipping it", tempExpName, tempExpRef.dataId)
675 tempExp = tempExpRef.get(tempExpName, immediate=
True)
677 if numpy.isnan(tempExp.image.array).all():
679 maskedImage = tempExp.getMaskedImage()
680 imageScaler = self.scaleZeroPoint.computeImageScaler(
685 imageScaler.scaleMaskedImage(maskedImage)
686 except Exception
as e:
687 self.log.warn(
"Scaling failed for %s (skipping it): %s", tempExpRef.dataId, e)
689 statObj = afwMath.makeStatistics(maskedImage.getVariance(), maskedImage.getMask(),
690 afwMath.MEANCLIP, statsCtrl)
691 meanVar, meanVarErr = statObj.getResult(afwMath.MEANCLIP)
692 weight = 1.0 / float(meanVar)
693 if not numpy.isfinite(weight):
694 self.log.warn(
"Non-finite weight for %s: skipping", tempExpRef.dataId)
696 self.log.info(
"Weight of %s %s = %0.3f", tempExpName, tempExpRef.dataId, weight)
701 tempExpRefList.append(tempExpRef)
702 weightList.append(weight)
703 imageScalerList.append(imageScaler)
705 return pipeBase.Struct(tempExpRefList=tempExpRefList, weightList=weightList,
706 imageScalerList=imageScalerList)
709 """Prepare the statistics for coadding images. 713 mask : `int`, optional 714 Bit mask value to exclude from coaddition. 718 stats : `lsst.pipe.base.Struct` 719 Statistics structure with the following fields: 721 - ``statsCtrl``: Statistics control object for coadd 722 (`lsst.afw.math.StatisticsControl`) 723 - ``statsFlags``: Statistic for coadd (`lsst.afw.math.Property`) 727 statsCtrl = afwMath.StatisticsControl()
728 statsCtrl.setNumSigmaClip(self.config.sigmaClip)
729 statsCtrl.setNumIter(self.config.clipIter)
730 statsCtrl.setAndMask(mask)
731 statsCtrl.setNanSafe(
True)
732 statsCtrl.setWeighted(
True)
733 statsCtrl.setCalcErrorFromInputVariance(self.config.calcErrorFromInputVariance)
734 for plane, threshold
in self.config.maskPropagationThresholds.items():
735 bit = afwImage.Mask.getMaskPlane(plane)
736 statsCtrl.setMaskPropagationThreshold(bit, threshold)
737 statsFlags = afwMath.stringToStatisticsProperty(self.config.statistic)
738 return pipeBase.Struct(ctrl=statsCtrl, flags=statsFlags)
740 def run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
741 altMaskList=None, mask=None, supplementaryData=None):
742 """Assemble a coadd from input warps 744 Assemble the coadd using the provided list of coaddTempExps. Since 745 the full coadd covers a patch (a large area), the assembly is 746 performed over small areas on the image at a time in order to 747 conserve memory usage. Iterate over subregions within the outer 748 bbox of the patch using `assembleSubregion` to stack the corresponding 749 subregions from the coaddTempExps with the statistic specified. 750 Set the edge bits the coadd mask based on the weight map. 754 skyInfo : `lsst.pipe.base.Struct` 755 Struct with geometric information about the patch. 756 tempExpRefList : `list` 757 List of data references to Warps (previously called CoaddTempExps). 758 imageScalerList : `list` 759 List of image scalers. 762 altMaskList : `list`, optional 763 List of alternate masks to use rather than those stored with 765 mask : `int`, optional 766 Bit mask value to exclude from coaddition. 767 supplementaryData : lsst.pipe.base.Struct, optional 768 Struct with additional data products needed to assemble coadd. 769 Only used by subclasses that implement `makeSupplementaryData` 774 result : `lsst.pipe.base.Struct` 775 Result struct with components: 777 - ``coaddExposure``: coadded exposure (``lsst.afw.image.Exposure``). 778 - ``nImage``: exposure count image (``lsst.afw.image.Image``). 781 self.log.info(
"Assembling %s %s", len(tempExpRefList), tempExpName)
784 if altMaskList
is None:
785 altMaskList = [
None]*len(tempExpRefList)
787 coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
788 coaddExposure.setPhotoCalib(self.scaleZeroPoint.getPhotoCalib())
789 coaddExposure.getInfo().setCoaddInputs(self.inputRecorder.makeCoaddInputs())
791 coaddMaskedImage = coaddExposure.getMaskedImage()
792 subregionSizeArr = self.config.subregionSize
793 subregionSize =
geom.Extent2I(subregionSizeArr[0], subregionSizeArr[1])
795 if self.config.doNImage:
796 nImage = afwImage.ImageU(skyInfo.bbox)
799 for subBBox
in self.
_subBBoxIter(skyInfo.bbox, subregionSize):
802 weightList, altMaskList, stats.flags, stats.ctrl,
804 except Exception
as e:
805 self.log.fatal(
"Cannot compute coadd %s: %s", subBBox, e)
810 coaddUtils.setCoaddEdgeBits(coaddMaskedImage.getMask(), coaddMaskedImage.getVariance())
811 return pipeBase.Struct(coaddExposure=coaddExposure, nImage=nImage)
814 """Set the metadata for the coadd. 816 This basic implementation sets the filter from the first input. 820 coaddExposure : `lsst.afw.image.Exposure` 821 The target exposure for the coadd. 822 tempExpRefList : `list` 823 List of data references to tempExp. 827 assert len(tempExpRefList) == len(weightList),
"Length mismatch" 832 tempExpList = [tempExpRef.get(tempExpName +
"_sub",
833 bbox=
geom.Box2I(coaddExposure.getBBox().getMin(),
835 for tempExpRef
in tempExpRefList]
836 numCcds = sum(len(tempExp.getInfo().getCoaddInputs().ccds)
for tempExp
in tempExpList)
838 coaddExposure.setFilter(tempExpList[0].getFilter())
839 coaddInputs = coaddExposure.getInfo().getCoaddInputs()
840 coaddInputs.ccds.reserve(numCcds)
841 coaddInputs.visits.reserve(len(tempExpList))
843 for tempExp, weight
in zip(tempExpList, weightList):
844 self.inputRecorder.addVisitToCoadd(coaddInputs, tempExp, weight)
846 if self.config.doUsePsfMatchedPolygons:
849 coaddInputs.visits.sort()
855 modelPsfList = [tempExp.getPsf()
for tempExp
in tempExpList]
856 modelPsfWidthList = [modelPsf.computeBBox().getWidth()
for modelPsf
in modelPsfList]
857 psf = modelPsfList[modelPsfWidthList.index(max(modelPsfWidthList))]
859 psf = measAlg.CoaddPsf(coaddInputs.ccds, coaddExposure.getWcs(),
860 self.config.coaddPsf.makeControl())
861 coaddExposure.setPsf(psf)
862 apCorrMap = measAlg.makeCoaddApCorrMap(coaddInputs.ccds, coaddExposure.getBBox(afwImage.PARENT),
863 coaddExposure.getWcs())
864 coaddExposure.getInfo().setApCorrMap(apCorrMap)
865 if self.config.doAttachTransmissionCurve:
866 transmissionCurve = measAlg.makeCoaddTransmissionCurve(coaddExposure.getWcs(), coaddInputs.ccds)
867 coaddExposure.getInfo().setTransmissionCurve(transmissionCurve)
869 def assembleSubregion(self, coaddExposure, bbox, tempExpRefList, imageScalerList, weightList,
870 altMaskList, statsFlags, statsCtrl, nImage=None):
871 """Assemble the coadd for a sub-region. 873 For each coaddTempExp, check for (and swap in) an alternative mask 874 if one is passed. Remove mask planes listed in 875 `config.removeMaskPlanes`. Finally, stack the actual exposures using 876 `lsst.afw.math.statisticsStack` with the statistic specified by 877 statsFlags. Typically, the statsFlag will be one of lsst.afw.math.MEAN for 878 a mean-stack or `lsst.afw.math.MEANCLIP` for outlier rejection using 879 an N-sigma clipped mean where N and iterations are specified by 880 statsCtrl. Assign the stacked subregion back to the coadd. 884 coaddExposure : `lsst.afw.image.Exposure` 885 The target exposure for the coadd. 886 bbox : `lsst.geom.Box` 888 tempExpRefList : `list` 889 List of data reference to tempExp. 890 imageScalerList : `list` 891 List of image scalers. 895 List of alternate masks to use rather than those stored with 896 tempExp, or None. Each element is dict with keys = mask plane 897 name to which to add the spans. 898 statsFlags : `lsst.afw.math.Property` 899 Property object for statistic for coadd. 900 statsCtrl : `lsst.afw.math.StatisticsControl` 901 Statistics control object for coadd. 902 nImage : `lsst.afw.image.ImageU`, optional 903 Keeps track of exposure count for each pixel. 905 self.log.debug(
"Computing coadd over %s", bbox)
907 coaddExposure.mask.addMaskPlane(
"REJECTED")
908 coaddExposure.mask.addMaskPlane(
"CLIPPED")
909 coaddExposure.mask.addMaskPlane(
"SENSOR_EDGE")
911 clipped = afwImage.Mask.getPlaneBitMask(
"CLIPPED")
913 if nImage
is not None:
914 subNImage = afwImage.ImageU(bbox.getWidth(), bbox.getHeight())
915 for tempExpRef, imageScaler, altMask
in zip(tempExpRefList, imageScalerList, altMaskList):
916 exposure = tempExpRef.get(tempExpName +
"_sub", bbox=bbox)
917 maskedImage = exposure.getMaskedImage()
918 mask = maskedImage.getMask()
919 if altMask
is not None:
921 imageScaler.scaleMaskedImage(maskedImage)
925 if nImage
is not None:
926 subNImage.getArray()[maskedImage.getMask().getArray() & statsCtrl.getAndMask() == 0] += 1
927 if self.config.removeMaskPlanes:
929 maskedImageList.append(maskedImage)
931 with self.timer(
"stack"):
932 coaddSubregion = afwMath.statisticsStack(maskedImageList, statsFlags, statsCtrl, weightList,
935 coaddExposure.maskedImage.assign(coaddSubregion, bbox)
936 if nImage
is not None:
937 nImage.assign(subNImage, bbox)
940 """Unset the mask of an image for mask planes specified in the config. 944 maskedImage : `lsst.afw.image.MaskedImage` 945 The masked image to be modified. 947 mask = maskedImage.getMask()
948 for maskPlane
in self.config.removeMaskPlanes:
950 mask &= ~mask.getPlaneBitMask(maskPlane)
951 except pexExceptions.InvalidParameterError:
952 self.log.debug(
"Unable to remove mask plane %s: no mask plane with that name was found.",
957 """Map certain mask planes of the warps to new planes for the coadd. 959 If a pixel is rejected due to a mask value other than EDGE, NO_DATA, 960 or CLIPPED, set it to REJECTED on the coadd. 961 If a pixel is rejected due to EDGE, set the coadd pixel to SENSOR_EDGE. 962 If a pixel is rejected due to CLIPPED, set the coadd pixel to CLIPPED. 966 statsCtrl : `lsst.afw.math.StatisticsControl` 967 Statistics control object for coadd 971 maskMap : `list` of `tuple` of `int` 972 A list of mappings of mask planes of the warped exposures to 973 mask planes of the coadd. 975 edge = afwImage.Mask.getPlaneBitMask(
"EDGE")
976 noData = afwImage.Mask.getPlaneBitMask(
"NO_DATA")
977 clipped = afwImage.Mask.getPlaneBitMask(
"CLIPPED")
978 toReject = statsCtrl.getAndMask() & (~noData) & (~edge) & (~clipped)
979 maskMap = [(toReject, afwImage.Mask.getPlaneBitMask(
"REJECTED")),
980 (edge, afwImage.Mask.getPlaneBitMask(
"SENSOR_EDGE")),
985 """Apply in place alt mask formatted as SpanSets to a mask. 989 mask : `lsst.afw.image.Mask` 991 altMaskSpans : `dict` 992 SpanSet lists to apply. Each element contains the new mask 993 plane name (e.g. "CLIPPED and/or "NO_DATA") as the key, 994 and list of SpanSets to apply to the mask. 998 mask : `lsst.afw.image.Mask` 1001 if self.config.doUsePsfMatchedPolygons:
1002 if (
"NO_DATA" in altMaskSpans)
and (
"NO_DATA" in self.config.badMaskPlanes):
1007 for spanSet
in altMaskSpans[
'NO_DATA']:
1008 spanSet.clippedTo(mask.getBBox()).clearMask(mask, self.
getBadPixelMask())
1010 for plane, spanSetList
in altMaskSpans.items():
1011 maskClipValue = mask.addMaskPlane(plane)
1012 for spanSet
in spanSetList:
1013 spanSet.clippedTo(mask.getBBox()).setMask(mask, 2**maskClipValue)
1017 """Shrink coaddInputs' ccds' ValidPolygons in place. 1019 Either modify each ccd's validPolygon in place, or if CoaddInputs 1020 does not have a validPolygon, create one from its bbox. 1024 coaddInputs : `lsst.afw.image.coaddInputs` 1028 for ccd
in coaddInputs.ccds:
1029 polyOrig = ccd.getValidPolygon()
1030 validPolyBBox = polyOrig.getBBox()
if polyOrig
else ccd.getBBox()
1031 validPolyBBox.grow(-self.config.matchingKernelSize//2)
1033 validPolygon = polyOrig.intersectionSingle(validPolyBBox)
1035 validPolygon = afwGeom.polygon.Polygon(
geom.Box2D(validPolyBBox))
1036 ccd.setValidPolygon(validPolygon)
1039 """Retrieve the bright object masks. 1041 Returns None on failure. 1045 dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef` 1050 result : `lsst.daf.persistence.butlerSubset.ButlerDataRef` 1051 Bright object mask from the Butler object, or None if it cannot 1055 return dataRef.get(
"brightObjectMask", immediate=
True)
1056 except Exception
as e:
1057 self.log.warn(
"Unable to read brightObjectMask for %s: %s", dataRef.dataId, e)
1061 """Set the bright object masks. 1065 exposure : `lsst.afw.image.Exposure` 1066 Exposure under consideration. 1067 dataId : `lsst.daf.persistence.dataId` 1068 Data identifier dict for patch. 1069 brightObjectMasks : `lsst.afw.table` 1070 Table of bright objects to mask. 1073 if brightObjectMasks
is None:
1074 self.log.warn(
"Unable to apply bright object mask: none supplied")
1076 self.log.info(
"Applying %d bright object masks to %s", len(brightObjectMasks), dataId)
1077 mask = exposure.getMaskedImage().getMask()
1078 wcs = exposure.getWcs()
1079 plateScale = wcs.getPixelScale().asArcseconds()
1081 for rec
in brightObjectMasks:
1082 center =
geom.PointI(wcs.skyToPixel(rec.getCoord()))
1083 if rec[
"type"] ==
"box":
1084 assert rec[
"angle"] == 0.0, (
"Angle != 0 for mask object %s" % rec[
"id"])
1085 width = rec[
"width"].asArcseconds()/plateScale
1086 height = rec[
"height"].asArcseconds()/plateScale
1089 bbox =
geom.Box2I(center - halfSize, center + halfSize)
1092 geom.PointI(int(center[0] + 0.5*width), int(center[1] + 0.5*height)))
1093 spans = afwGeom.SpanSet(bbox)
1094 elif rec[
"type"] ==
"circle":
1095 radius = int(rec[
"radius"].asArcseconds()/plateScale)
1096 spans = afwGeom.SpanSet.fromShape(radius, offset=center)
1098 self.log.warn(
"Unexpected region type %s at %s" % rec[
"type"], center)
1103 """Set INEXACT_PSF mask plane. 1105 If any of the input images isn't represented in the coadd (due to 1106 clipped pixels or chip gaps), the `CoaddPsf` will be inexact. Flag 1111 mask : `lsst.afw.image.Mask` 1112 Coadded exposure's mask, modified in-place. 1114 mask.addMaskPlane(
"INEXACT_PSF")
1115 inexactPsf = mask.getPlaneBitMask(
"INEXACT_PSF")
1116 sensorEdge = mask.getPlaneBitMask(
"SENSOR_EDGE")
1117 clipped = mask.getPlaneBitMask(
"CLIPPED")
1118 rejected = mask.getPlaneBitMask(
"REJECTED")
1119 array = mask.getArray()
1120 selected = array & (sensorEdge | clipped | rejected) > 0
1121 array[selected] |= inexactPsf
1124 def _makeArgumentParser(cls):
1125 """Create an argument parser. 1127 parser = pipeBase.ArgumentParser(name=cls.
_DefaultName)
1128 parser.add_id_argument(
"--id", cls.
ConfigClass().coaddName +
"Coadd_" +
1130 help=
"data ID, e.g. --id tract=12345 patch=1,2",
1131 ContainerClass=AssembleCoaddDataIdContainer)
1132 parser.add_id_argument(
"--selectId",
"calexp", help=
"data ID, e.g. --selectId visit=6789 ccd=0..9",
1133 ContainerClass=SelectDataIdContainer)
1137 def _subBBoxIter(bbox, subregionSize):
1138 """Iterate over subregions of a bbox. 1142 bbox : `lsst.geom.Box2I` 1143 Bounding box over which to iterate. 1144 subregionSize: `lsst.geom.Extent2I` 1149 subBBox : `lsst.geom.Box2I` 1150 Next sub-bounding box of size ``subregionSize`` or smaller; each ``subBBox`` 1151 is contained within ``bbox``, so it may be smaller than ``subregionSize`` at 1152 the edges of ``bbox``, but it will never be empty. 1155 raise RuntimeError(
"bbox %s is empty" % (bbox,))
1156 if subregionSize[0] < 1
or subregionSize[1] < 1:
1157 raise RuntimeError(
"subregionSize %s must be nonzero" % (subregionSize,))
1159 for rowShift
in range(0, bbox.getHeight(), subregionSize[1]):
1160 for colShift
in range(0, bbox.getWidth(), subregionSize[0]):
1163 if subBBox.isEmpty():
1164 raise RuntimeError(
"Bug: empty bbox! bbox=%s, subregionSize=%s, " 1165 "colShift=%s, rowShift=%s" %
1166 (bbox, subregionSize, colShift, rowShift))
1171 """A version of `lsst.pipe.base.DataIdContainer` specialized for assembleCoadd. 1175 """Make self.refList from self.idList. 1180 Results of parsing command-line (with ``butler`` and ``log`` elements). 1182 datasetType = namespace.config.coaddName +
"Coadd" 1183 keysCoadd = namespace.butler.getKeys(datasetType=datasetType, level=self.level)
1185 for dataId
in self.idList:
1187 for key
in keysCoadd:
1188 if key
not in dataId:
1189 raise RuntimeError(
"--id must include " + key)
1191 dataRef = namespace.butler.dataRef(
1192 datasetType=datasetType,
1195 self.refList.append(dataRef)
1199 """Function to count the number of pixels with a specific mask in a 1202 Find the intersection of mask & footprint. Count all pixels in the mask 1203 that are in the intersection that have bitmask set but do not have 1204 ignoreMask set. Return the count. 1208 mask : `lsst.afw.image.Mask` 1209 Mask to define intersection region by. 1210 footprint : `lsst.afw.detection.Footprint` 1211 Footprint to define the intersection region by. 1213 Specific mask that we wish to count the number of occurances of. 1215 Pixels to not consider. 1220 Count of number of pixels in footprint with specified mask. 1222 bbox = footprint.getBBox()
1223 bbox.clip(mask.getBBox(afwImage.PARENT))
1224 fp = afwImage.Mask(bbox)
1225 subMask = mask.Factory(mask, bbox, afwImage.PARENT)
1226 footprint.spans.setMask(fp, bitmask)
1227 return numpy.logical_and((subMask.getArray() & fp.getArray()) > 0,
1228 (subMask.getArray() & ignoreMask) == 0).sum()
1232 """Configuration parameters for the SafeClipAssembleCoaddTask. 1234 clipDetection = pexConfig.ConfigurableField(
1235 target=SourceDetectionTask,
1236 doc=
"Detect sources on difference between unclipped and clipped coadd")
1237 minClipFootOverlap = pexConfig.Field(
1238 doc=
"Minimum fractional overlap of clipped footprint with visit DETECTED to be clipped",
1242 minClipFootOverlapSingle = pexConfig.Field(
1243 doc=
"Minimum fractional overlap of clipped footprint with visit DETECTED to be " 1244 "clipped when only one visit overlaps",
1248 minClipFootOverlapDouble = pexConfig.Field(
1249 doc=
"Minimum fractional overlap of clipped footprints with visit DETECTED to be " 1250 "clipped when two visits overlap",
1254 maxClipFootOverlapDouble = pexConfig.Field(
1255 doc=
"Maximum fractional overlap of clipped footprints with visit DETECTED when " 1256 "considering two visits",
1260 minBigOverlap = pexConfig.Field(
1261 doc=
"Minimum number of pixels in footprint to use DETECTED mask from the single visits " 1262 "when labeling clipped footprints",
1268 """Set default values for clipDetection. 1272 The numeric values for these configuration parameters were 1273 empirically determined, future work may further refine them. 1275 AssembleCoaddConfig.setDefaults(self)
1291 log.warn(
"Additional Sigma-clipping not allowed in Safe-clipped Coadds. " 1292 "Ignoring doSigmaClip.")
1295 raise ValueError(
"Only MEAN statistic allowed for final stacking in SafeClipAssembleCoadd " 1296 "(%s chosen). Please set statistic to MEAN." 1298 AssembleCoaddTask.ConfigClass.validate(self)
1302 """Assemble a coadded image from a set of coadded temporary exposures, 1303 being careful to clip & flag areas with potential artifacts. 1305 In ``AssembleCoaddTask``, we compute the coadd as an clipped mean (i.e., 1306 we clip outliers). The problem with doing this is that when computing the 1307 coadd PSF at a given location, individual visit PSFs from visits with 1308 outlier pixels contribute to the coadd PSF and cannot be treated correctly. 1309 In this task, we correct for this behavior by creating a new 1310 ``badMaskPlane`` 'CLIPPED'. We populate this plane on the input 1311 coaddTempExps and the final coadd where 1313 i. difference imaging suggests that there is an outlier and 1314 ii. this outlier appears on only one or two images. 1316 Such regions will not contribute to the final coadd. Furthermore, any 1317 routine to determine the coadd PSF can now be cognizant of clipped regions. 1318 Note that the algorithm implemented by this task is preliminary and works 1319 correctly for HSC data. Parameter modifications and or considerable 1320 redesigning of the algorithm is likley required for other surveys. 1322 ``SafeClipAssembleCoaddTask`` uses a ``SourceDetectionTask`` 1323 "clipDetection" subtask and also sub-classes ``AssembleCoaddTask``. 1324 You can retarget the ``SourceDetectionTask`` "clipDetection" subtask 1329 The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a 1330 flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``; 1331 see `baseDebug` for more about ``debug.py`` files. 1332 `SafeClipAssembleCoaddTask` has no debug variables of its own. 1333 The ``SourceDetectionTask`` "clipDetection" subtasks may support debug 1334 variables. See the documetation for `SourceDetectionTask` "clipDetection" 1335 for further information. 1339 `SafeClipAssembleCoaddTask` assembles a set of warped ``coaddTempExp`` 1340 images into a coadded image. The `SafeClipAssembleCoaddTask` is invoked by 1341 running assembleCoadd.py *without* the flag '--legacyCoadd'. 1343 Usage of ``assembleCoadd.py`` expects a data reference to the tract patch 1344 and filter to be coadded (specified using 1345 '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]') 1346 along with a list of coaddTempExps to attempt to coadd (specified using 1347 '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]'). 1348 Only the coaddTempExps that cover the specified tract and patch will be 1349 coadded. A list of the available optional arguments can be obtained by 1350 calling assembleCoadd.py with the --help command line argument: 1352 .. code-block:: none 1354 assembleCoadd.py --help 1356 To demonstrate usage of the `SafeClipAssembleCoaddTask` in the larger 1357 context of multi-band processing, we will generate the HSC-I & -R band 1358 coadds from HSC engineering test data provided in the ci_hsc package. 1359 To begin, assuming that the lsst stack has been already set up, we must 1360 set up the obs_subaru and ci_hsc packages. This defines the environment 1361 variable $CI_HSC_DIR and points at the location of the package. The raw 1362 HSC data live in the ``$CI_HSC_DIR/raw`` directory. To begin assembling 1363 the coadds, we must first 1366 process the individual ccds in $CI_HSC_RAW to produce calibrated exposures 1368 create a skymap that covers the area of the sky present in the raw exposures 1369 - ``makeCoaddTempExp`` 1370 warp the individual calibrated exposures to the tangent plane of the coadd</DD> 1372 We can perform all of these steps by running 1374 .. code-block:: none 1376 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988 1378 This will produce warped coaddTempExps for each visit. To coadd the 1379 warped data, we call ``assembleCoadd.py`` as follows: 1381 .. code-block:: none 1383 assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \ 1384 --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \ 1385 --selectId visit=903986 ccd=100--selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \ 1386 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \ 1387 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \ 1388 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \ 1389 --selectId visit=903988 ccd=24 1391 This will process the HSC-I band data. The results are written in 1392 ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``. 1394 You may also choose to run: 1396 .. code-block:: none 1398 scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346 nnn 1399 assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R --selectId visit=903334 ccd=16 \ 1400 --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 --selectId visit=903334 ccd=100 \ 1401 --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 --selectId visit=903338 ccd=18 \ 1402 --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 --selectId visit=903342 ccd=10 \ 1403 --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 --selectId visit=903344 ccd=5 \ 1404 --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 --selectId visit=903346 ccd=6 \ 1405 --selectId visit=903346 ccd=12 1407 to generate the coadd for the HSC-R band if you are interested in following 1408 multiBand Coadd processing as discussed in ``pipeTasks_multiBand``. 1410 ConfigClass = SafeClipAssembleCoaddConfig
1411 _DefaultName =
"safeClipAssembleCoadd" 1414 AssembleCoaddTask.__init__(self, *args, **kwargs)
1415 schema = afwTable.SourceTable.makeMinimalSchema()
1416 self.makeSubtask(
"clipDetection", schema=schema)
1418 def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, *args, **kwargs):
1419 """Assemble the coadd for a region. 1421 Compute the difference of coadds created with and without outlier 1422 rejection to identify coadd pixels that have outlier values in some 1424 Detect clipped regions on the difference image and mark these regions 1425 on the one or two individual coaddTempExps where they occur if there 1426 is significant overlap between the clipped region and a source. This 1427 leaves us with a set of footprints from the difference image that have 1428 been identified as having occured on just one or two individual visits. 1429 However, these footprints were generated from a difference image. It 1430 is conceivable for a large diffuse source to have become broken up 1431 into multiple footprints acrosss the coadd difference in this process. 1432 Determine the clipped region from all overlapping footprints from the 1433 detected sources in each visit - these are big footprints. 1434 Combine the small and big clipped footprints and mark them on a new 1436 Generate the coadd using `AssembleCoaddTask.run` without outlier 1437 removal. Clipped footprints will no longer make it into the coadd 1438 because they are marked in the new bad mask plane. 1442 skyInfo : `lsst.pipe.base.Struct` 1443 Patch geometry information, from getSkyInfo 1444 tempExpRefList : `list` 1445 List of data reference to tempExp 1446 imageScalerList : `list` 1447 List of image scalers 1453 result : `lsst.pipe.base.Struct` 1454 Result struct with components: 1456 - ``coaddExposure``: coadded exposure (``lsst.afw.image.Exposure``). 1457 - ``nImage``: exposure count image (``lsst.afw.image.Image``). 1461 args and kwargs are passed but ignored in order to match the call 1462 signature expected by the parent task. 1465 mask = exp.getMaskedImage().getMask()
1466 mask.addMaskPlane(
"CLIPPED")
1468 result = self.
detectClip(exp, tempExpRefList)
1470 self.log.info(
'Found %d clipped objects', len(result.clipFootprints))
1472 maskClipValue = mask.getPlaneBitMask(
"CLIPPED")
1473 maskDetValue = mask.getPlaneBitMask(
"DETECTED") | mask.getPlaneBitMask(
"DETECTED_NEGATIVE")
1475 bigFootprints = self.
detectClipBig(result.clipSpans, result.clipFootprints, result.clipIndices,
1476 result.detectionFootprints, maskClipValue, maskDetValue,
1479 maskClip = mask.Factory(mask.getBBox(afwImage.PARENT))
1480 afwDet.setMaskFromFootprintList(maskClip, result.clipFootprints, maskClipValue)
1482 maskClipBig = maskClip.Factory(mask.getBBox(afwImage.PARENT))
1483 afwDet.setMaskFromFootprintList(maskClipBig, bigFootprints, maskClipValue)
1484 maskClip |= maskClipBig
1487 badMaskPlanes = self.config.badMaskPlanes[:]
1488 badMaskPlanes.append(
"CLIPPED")
1489 badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes)
1490 return AssembleCoaddTask.run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
1491 result.clipSpans, mask=badPixelMask)
1494 """Return an exposure that contains the difference between unclipped 1497 Generate a difference image between clipped and unclipped coadds. 1498 Compute the difference image by subtracting an outlier-clipped coadd 1499 from an outlier-unclipped coadd. Return the difference image. 1503 skyInfo : `lsst.pipe.base.Struct` 1504 Patch geometry information, from getSkyInfo 1505 tempExpRefList : `list` 1506 List of data reference to tempExp 1507 imageScalerList : `list` 1508 List of image scalers 1514 exp : `lsst.afw.image.Exposure` 1515 Difference image of unclipped and clipped coadd wrapped in an Exposure 1520 configIntersection = {k: getattr(self.config, k)
1521 for k, v
in self.config.toDict().items()
if (k
in config.keys())}
1522 config.update(**configIntersection)
1525 config.statistic =
'MEAN' 1527 coaddMean = task.run(skyInfo, tempExpRefList, imageScalerList, weightList).coaddExposure
1529 config.statistic =
'MEANCLIP' 1531 coaddClip = task.run(skyInfo, tempExpRefList, imageScalerList, weightList).coaddExposure
1533 coaddDiff = coaddMean.getMaskedImage().Factory(coaddMean.getMaskedImage())
1534 coaddDiff -= coaddClip.getMaskedImage()
1535 exp = afwImage.ExposureF(coaddDiff)
1536 exp.setPsf(coaddMean.getPsf())
1540 """Detect clipped regions on an exposure and set the mask on the 1541 individual tempExp masks. 1543 Detect footprints in the difference image after smoothing the 1544 difference image with a Gaussian kernal. Identify footprints that 1545 overlap with one or two input ``coaddTempExps`` by comparing the 1546 computed overlap fraction to thresholds set in the config. A different 1547 threshold is applied depending on the number of overlapping visits 1548 (restricted to one or two). If the overlap exceeds the thresholds, 1549 the footprint is considered "CLIPPED" and is marked as such on the 1550 coaddTempExp. Return a struct with the clipped footprints, the indices 1551 of the ``coaddTempExps`` that end up overlapping with the clipped 1552 footprints, and a list of new masks for the ``coaddTempExps``. 1556 exp : `lsst.afw.image.Exposure` 1557 Exposure to run detection on. 1558 tempExpRefList : `list` 1559 List of data reference to tempExp. 1563 result : `lsst.pipe.base.Struct` 1564 Result struct with components: 1566 - ``clipFootprints``: list of clipped footprints. 1567 - ``clipIndices``: indices for each ``clippedFootprint`` in 1569 - ``clipSpans``: List of dictionaries containing spanSet lists 1570 to clip. Each element contains the new maskplane name 1571 ("CLIPPED") as the key and list of ``SpanSets`` as the value. 1572 - ``detectionFootprints``: List of DETECTED/DETECTED_NEGATIVE plane 1573 compressed into footprints. 1575 mask = exp.getMaskedImage().getMask()
1576 maskDetValue = mask.getPlaneBitMask(
"DETECTED") | mask.getPlaneBitMask(
"DETECTED_NEGATIVE")
1577 fpSet = self.clipDetection.detectFootprints(exp, doSmooth=
True, clearMask=
True)
1579 fpSet.positive.merge(fpSet.negative)
1580 footprints = fpSet.positive
1581 self.log.info(
'Found %d potential clipped objects', len(footprints.getFootprints()))
1586 artifactSpanSets = [{
'CLIPPED': list()}
for _
in tempExpRefList]
1589 visitDetectionFootprints = []
1591 dims = [len(tempExpRefList), len(footprints.getFootprints())]
1592 overlapDetArr = numpy.zeros(dims, dtype=numpy.uint16)
1593 ignoreArr = numpy.zeros(dims, dtype=numpy.uint16)
1596 for i, warpRef
in enumerate(tempExpRefList):
1598 immediate=
True).getMaskedImage().getMask()
1599 maskVisitDet = tmpExpMask.Factory(tmpExpMask, tmpExpMask.getBBox(afwImage.PARENT),
1600 afwImage.PARENT,
True)
1601 maskVisitDet &= maskDetValue
1602 visitFootprints = afwDet.FootprintSet(maskVisitDet, afwDet.Threshold(1))
1603 visitDetectionFootprints.append(visitFootprints)
1605 for j, footprint
in enumerate(footprints.getFootprints()):
1610 for j, footprint
in enumerate(footprints.getFootprints()):
1611 nPixel = footprint.getArea()
1614 for i
in range(len(tempExpRefList)):
1615 ignore = ignoreArr[i, j]
1616 overlapDet = overlapDetArr[i, j]
1617 totPixel = nPixel - ignore
1620 if ignore > overlapDet
or totPixel <= 0.5*nPixel
or overlapDet == 0:
1622 overlap.append(overlapDet/float(totPixel))
1625 overlap = numpy.array(overlap)
1626 if not len(overlap):
1633 if len(overlap) == 1:
1634 if overlap[0] > self.config.minClipFootOverlapSingle:
1639 clipIndex = numpy.where(overlap > self.config.minClipFootOverlap)[0]
1640 if len(clipIndex) == 1:
1642 keepIndex = [clipIndex[0]]
1645 clipIndex = numpy.where(overlap > self.config.minClipFootOverlapDouble)[0]
1646 if len(clipIndex) == 2
and len(overlap) > 3:
1647 clipIndexComp = numpy.where(overlap <= self.config.minClipFootOverlapDouble)[0]
1648 if numpy.max(overlap[clipIndexComp]) <= self.config.maxClipFootOverlapDouble:
1650 keepIndex = clipIndex
1655 for index
in keepIndex:
1656 globalIndex = indexList[index]
1657 artifactSpanSets[globalIndex][
'CLIPPED'].append(footprint.spans)
1659 clipIndices.append(numpy.array(indexList)[keepIndex])
1660 clipFootprints.append(footprint)
1662 return pipeBase.Struct(clipFootprints=clipFootprints, clipIndices=clipIndices,
1663 clipSpans=artifactSpanSets, detectionFootprints=visitDetectionFootprints)
1665 def detectClipBig(self, clipList, clipFootprints, clipIndices, detectionFootprints,
1666 maskClipValue, maskDetValue, coaddBBox):
1667 """Return individual warp footprints for large artifacts and append 1668 them to ``clipList`` in place. 1670 Identify big footprints composed of many sources in the coadd 1671 difference that may have originated in a large diffuse source in the 1672 coadd. We do this by indentifying all clipped footprints that overlap 1673 significantly with each source in all the coaddTempExps. 1678 List of alt mask SpanSets with clipping information. Modified. 1679 clipFootprints : `list` 1680 List of clipped footprints. 1681 clipIndices : `list` 1682 List of which entries in tempExpClipList each footprint belongs to. 1684 Mask value of clipped pixels. 1686 Mask value of detected pixels. 1687 coaddBBox : `lsst.geom.Box` 1688 BBox of the coadd and warps. 1692 bigFootprintsCoadd : `list` 1693 List of big footprints 1695 bigFootprintsCoadd = []
1697 for index, (clippedSpans, visitFootprints)
in enumerate(zip(clipList, detectionFootprints)):
1698 maskVisitDet = afwImage.MaskX(coaddBBox, 0x0)
1699 for footprint
in visitFootprints.getFootprints():
1700 footprint.spans.setMask(maskVisitDet, maskDetValue)
1703 clippedFootprintsVisit = []
1704 for foot, clipIndex
in zip(clipFootprints, clipIndices):
1705 if index
not in clipIndex:
1707 clippedFootprintsVisit.append(foot)
1708 maskVisitClip = maskVisitDet.Factory(maskVisitDet.getBBox(afwImage.PARENT))
1709 afwDet.setMaskFromFootprintList(maskVisitClip, clippedFootprintsVisit, maskClipValue)
1711 bigFootprintsVisit = []
1712 for foot
in visitFootprints.getFootprints():
1713 if foot.getArea() < self.config.minBigOverlap:
1716 if nCount > self.config.minBigOverlap:
1717 bigFootprintsVisit.append(foot)
1718 bigFootprintsCoadd.append(foot)
1720 for footprint
in bigFootprintsVisit:
1721 clippedSpans[
"CLIPPED"].append(footprint.spans)
1723 return bigFootprintsCoadd
1727 assembleStaticSkyModel = pexConfig.ConfigurableField(
1728 target=AssembleCoaddTask,
1729 doc=
"Task to assemble an artifact-free, PSF-matched Coadd to serve as a" 1730 " naive/first-iteration model of the static sky.",
1732 detect = pexConfig.ConfigurableField(
1733 target=SourceDetectionTask,
1734 doc=
"Detect outlier sources on difference between each psfMatched warp and static sky model" 1736 detectTemplate = pexConfig.ConfigurableField(
1737 target=SourceDetectionTask,
1738 doc=
"Detect sources on static sky model. Only used if doPreserveContainedBySource is True" 1740 maxNumEpochs = pexConfig.Field(
1741 doc=
"Charactistic maximum local number of epochs/visits in which an artifact candidate can appear " 1742 "and still be masked. The effective maxNumEpochs is a broken linear function of local " 1743 "number of epochs (N): min(maxFractionEpochsLow*N, maxNumEpochs + maxFractionEpochsHigh*N). " 1744 "For each footprint detected on the image difference between the psfMatched warp and static sky " 1745 "model, if a significant fraction of pixels (defined by spatialThreshold) are residuals in more " 1746 "than the computed effective maxNumEpochs, the artifact candidate is deemed persistant rather " 1747 "than transient and not masked.",
1751 maxFractionEpochsLow = pexConfig.RangeField(
1752 doc=
"Fraction of local number of epochs (N) to use as effective maxNumEpochs for low N. " 1753 "Effective maxNumEpochs = " 1754 "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)",
1759 maxFractionEpochsHigh = pexConfig.RangeField(
1760 doc=
"Fraction of local number of epochs (N) to use as effective maxNumEpochs for high N. " 1761 "Effective maxNumEpochs = " 1762 "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)",
1767 spatialThreshold = pexConfig.RangeField(
1768 doc=
"Unitless fraction of pixels defining how much of the outlier region has to meet the " 1769 "temporal criteria. If 0, clip all. If 1, clip none.",
1773 inclusiveMin=
True, inclusiveMax=
True 1775 doScaleWarpVariance = pexConfig.Field(
1776 doc=
"Rescale Warp variance plane using empirical noise?",
1780 scaleWarpVariance = pexConfig.ConfigurableField(
1781 target=ScaleVarianceTask,
1782 doc=
"Rescale variance on warps",
1784 doPreserveContainedBySource = pexConfig.Field(
1785 doc=
"Rescue artifacts from clipping that completely lie within a footprint detected" 1786 "on the PsfMatched Template Coadd. Replicates a behavior of SafeClip.",
1790 doPrefilterArtifacts = pexConfig.Field(
1791 doc=
"Ignore artifact candidates that are mostly covered by the bad pixel mask, " 1792 "because they will be excluded anyway. This prevents them from contributing " 1793 "to the outlier epoch count image and potentially being labeled as persistant." 1794 "'Mostly' is defined by the config 'prefilterArtifactsRatio'.",
1798 prefilterArtifactsMaskPlanes = pexConfig.ListField(
1799 doc=
"Prefilter artifact candidates that are mostly covered by these bad mask planes.",
1801 default=(
'NO_DATA',
'BAD',
'SAT',
'SUSPECT'),
1803 prefilterArtifactsRatio = pexConfig.Field(
1804 doc=
"Prefilter artifact candidates with less than this fraction overlapping good pixels",
1808 psfMatchedWarps = pipeBase.InputDatasetField(
1809 doc=(
"PSF-Matched Warps are required by CompareWarp regardless of the coadd type requested. " 1810 "Only PSF-Matched Warps make sense for image subtraction. " 1811 "Therefore, they must be in the InputDatasetField and made available to the task."),
1812 nameTemplate=
"{inputCoaddName}Coadd_psfMatchedWarp",
1813 storageClass=
"ExposureF",
1814 dimensions=(
"tract",
"patch",
"skymap",
"visit"),
1819 AssembleCoaddConfig.setDefaults(self)
1835 self.
detect.doTempLocalBackground =
False 1836 self.
detect.reEstimateBackground =
False 1837 self.
detect.returnOriginalFootprints =
False 1838 self.
detect.thresholdPolarity =
"both" 1839 self.
detect.thresholdValue = 5
1840 self.
detect.nSigmaToGrow = 2
1841 self.
detect.minPixels = 4
1842 self.
detect.isotropicGrow =
True 1843 self.
detect.thresholdType =
"pixel_stdev" 1851 """Assemble a compareWarp coadded image from a set of warps 1852 by masking artifacts detected by comparing PSF-matched warps. 1854 In ``AssembleCoaddTask``, we compute the coadd as an clipped mean (i.e., 1855 we clip outliers). The problem with doing this is that when computing the 1856 coadd PSF at a given location, individual visit PSFs from visits with 1857 outlier pixels contribute to the coadd PSF and cannot be treated correctly. 1858 In this task, we correct for this behavior by creating a new badMaskPlane 1859 'CLIPPED' which marks pixels in the individual warps suspected to contain 1860 an artifact. We populate this plane on the input warps by comparing 1861 PSF-matched warps with a PSF-matched median coadd which serves as a 1862 model of the static sky. Any group of pixels that deviates from the 1863 PSF-matched template coadd by more than config.detect.threshold sigma, 1864 is an artifact candidate. The candidates are then filtered to remove 1865 variable sources and sources that are difficult to subtract such as 1866 bright stars. This filter is configured using the config parameters 1867 ``temporalThreshold`` and ``spatialThreshold``. The temporalThreshold is 1868 the maximum fraction of epochs that the deviation can appear in and still 1869 be considered an artifact. The spatialThreshold is the maximum fraction of 1870 pixels in the footprint of the deviation that appear in other epochs 1871 (where other epochs is defined by the temporalThreshold). If the deviant 1872 region meets this criteria of having a significant percentage of pixels 1873 that deviate in only a few epochs, these pixels have the 'CLIPPED' bit 1874 set in the mask. These regions will not contribute to the final coadd. 1875 Furthermore, any routine to determine the coadd PSF can now be cognizant 1876 of clipped regions. Note that the algorithm implemented by this task is 1877 preliminary and works correctly for HSC data. Parameter modifications and 1878 or considerable redesigning of the algorithm is likley required for other 1881 ``CompareWarpAssembleCoaddTask`` sub-classes 1882 ``AssembleCoaddTask`` and instantiates ``AssembleCoaddTask`` 1883 as a subtask to generate the TemplateCoadd (the model of the static sky). 1887 The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a 1888 flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``; see 1889 ``baseDebug`` for more about ``debug.py`` files. 1891 This task supports the following debug variables: 1894 If True then save the Epoch Count Image as a fits file in the `figPath` 1896 Path to save the debug fits images and figures 1898 For example, put something like: 1900 .. code-block:: python 1903 def DebugInfo(name): 1904 di = lsstDebug.getInfo(name) 1905 if name == "lsst.pipe.tasks.assembleCoadd": 1906 di.saveCountIm = True 1907 di.figPath = "/desired/path/to/debugging/output/images" 1909 lsstDebug.Info = DebugInfo 1911 into your ``debug.py`` file and run ``assemebleCoadd.py`` with the 1912 ``--debug`` flag. Some subtasks may have their own debug variables; 1913 see individual Task documentation. 1917 ``CompareWarpAssembleCoaddTask`` assembles a set of warped images into a 1918 coadded image. The ``CompareWarpAssembleCoaddTask`` is invoked by running 1919 ``assembleCoadd.py`` with the flag ``--compareWarpCoadd``. 1920 Usage of ``assembleCoadd.py`` expects a data reference to the tract patch 1921 and filter to be coadded (specified using 1922 '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]') 1923 along with a list of coaddTempExps to attempt to coadd (specified using 1924 '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]'). 1925 Only the warps that cover the specified tract and patch will be coadded. 1926 A list of the available optional arguments can be obtained by calling 1927 ``assembleCoadd.py`` with the ``--help`` command line argument: 1929 .. code-block:: none 1931 assembleCoadd.py --help 1933 To demonstrate usage of the ``CompareWarpAssembleCoaddTask`` in the larger 1934 context of multi-band processing, we will generate the HSC-I & -R band 1935 oadds from HSC engineering test data provided in the ``ci_hsc`` package. 1936 To begin, assuming that the lsst stack has been already set up, we must 1937 set up the ``obs_subaru`` and ``ci_hsc`` packages. 1938 This defines the environment variable ``$CI_HSC_DIR`` and points at the 1939 location of the package. The raw HSC data live in the ``$CI_HSC_DIR/raw`` 1940 directory. To begin assembling the coadds, we must first 1943 process the individual ccds in $CI_HSC_RAW to produce calibrated exposures 1945 create a skymap that covers the area of the sky present in the raw exposures 1947 warp the individual calibrated exposures to the tangent plane of the coadd 1949 We can perform all of these steps by running 1951 .. code-block:: none 1953 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988 1955 This will produce warped ``coaddTempExps`` for each visit. To coadd the 1956 warped data, we call ``assembleCoadd.py`` as follows: 1958 .. code-block:: none 1960 assembleCoadd.py --compareWarpCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \ 1961 --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \ 1962 --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \ 1963 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \ 1964 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \ 1965 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \ 1966 --selectId visit=903988 ccd=24 1968 This will process the HSC-I band data. The results are written in 1969 ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``. 1971 ConfigClass = CompareWarpAssembleCoaddConfig
1972 _DefaultName =
"compareWarpAssembleCoadd" 1975 AssembleCoaddTask.__init__(self, *args, **kwargs)
1976 self.makeSubtask(
"assembleStaticSkyModel")
1977 detectionSchema = afwTable.SourceTable.makeMinimalSchema()
1978 self.makeSubtask(
"detect", schema=detectionSchema)
1979 if self.config.doPreserveContainedBySource:
1980 self.makeSubtask(
"detectTemplate", schema=afwTable.SourceTable.makeMinimalSchema())
1981 if self.config.doScaleWarpVariance:
1982 self.makeSubtask(
"scaleWarpVariance")
1985 """Make inputs specific to Subclass with Gen 3 API 1987 Calls Gen3 `adaptArgsAndRun` instead of the Gen2 specific `runDataRef` 1989 Duplicates interface of`adaptArgsAndRun` method. 1990 Available to be implemented by subclasses only if they need the 1991 coadd dataRef for performing preliminary processing before 1992 assembling the coadd. 1997 Keys are the names of the configs describing input dataset types. 1998 Values are input Python-domain data objects (or lists of objects) 1999 retrieved from data butler. 2000 inputDataIds : `dict` 2001 Keys are the names of the configs describing input dataset types. 2002 Values are DataIds (or lists of DataIds) that task consumes for 2003 corresponding dataset type. 2004 DataIds are guaranteed to match data objects in ``inputData``. 2005 outputDataIds : `dict` 2006 Keys are the names of the configs describing input dataset types. 2007 Values are DataIds (or lists of DataIds) that task is to produce 2008 for corresponding dataset type. 2009 butler : `lsst.daf.butler.Butler` 2010 Gen3 Butler object for fetching additional data products before 2015 result : `lsst.pipe.base.Struct` 2016 Result struct with components: 2018 - ``templateCoadd`` : coadded exposure (``lsst.afw.image.Exposure``) 2019 - ``nImage``: N Image (``lsst.afw.image.Image``) 2023 templateCoadd = self.assembleStaticSkyModel.
adaptArgsAndRun(inputData, inputDataIds,
2024 outputDataIds, butler)
2025 if templateCoadd
is None:
2028 return pipeBase.Struct(templateCoadd=templateCoadd.coaddExposure,
2029 nImage=templateCoadd.nImage)
2032 """Make inputs specific to Subclass. 2034 Generate a templateCoadd to use as a native model of static sky to 2035 subtract from warps. 2039 dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef` 2040 Butler dataRef for supplementary data. 2041 selectDataList : `list` (optional) 2042 Optional List of data references to Calexps. 2043 warpRefList : `list` (optional) 2044 Optional List of data references to Warps. 2048 result : `lsst.pipe.base.Struct` 2049 Result struct with components: 2051 - ``templateCoadd``: coadded exposure (``lsst.afw.image.Exposure``) 2052 - ``nImage``: N Image (``lsst.afw.image.Image``) 2054 templateCoadd = self.assembleStaticSkyModel.
runDataRef(dataRef, selectDataList, warpRefList)
2055 if templateCoadd
is None:
2058 return pipeBase.Struct(templateCoadd=templateCoadd.coaddExposure,
2059 nImage=templateCoadd.nImage)
2061 def _noTemplateMessage(self, warpType):
2062 warpName = (warpType[0].upper() + warpType[1:])
2063 message =
"""No %(warpName)s warps were found to build the template coadd which is 2064 required to run CompareWarpAssembleCoaddTask. To continue assembling this type of coadd, 2065 first either rerun makeCoaddTempExp with config.make%(warpName)s=True or 2066 coaddDriver with config.makeCoadTempExp.make%(warpName)s=True, before assembleCoadd. 2068 Alternatively, to use another algorithm with existing warps, retarget the CoaddDriverConfig to 2069 another algorithm like: 2071 from lsst.pipe.tasks.assembleCoadd import SafeClipAssembleCoaddTask 2072 config.assemble.retarget(SafeClipAssembleCoaddTask) 2073 """ % {
"warpName": warpName}
2076 def run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
2077 supplementaryData, *args, **kwargs):
2078 """Assemble the coadd. 2080 Find artifacts and apply them to the warps' masks creating a list of 2081 alternative masks with a new "CLIPPED" plane and updated "NO_DATA" 2082 plane. Then pass these alternative masks to the base class's `run` 2087 skyInfo : `lsst.pipe.base.Struct` 2088 Patch geometry information. 2089 tempExpRefList : `list` 2090 List of data references to warps. 2091 imageScalerList : `list` 2092 List of image scalers. 2095 supplementaryData : `lsst.pipe.base.Struct` 2096 This Struct must contain a ``templateCoadd`` that serves as the 2097 model of the static sky. 2101 result : `lsst.pipe.base.Struct` 2102 Result struct with components: 2104 - ``coaddExposure``: coadded exposure (``lsst.afw.image.Exposure``). 2105 - ``nImage``: exposure count image (``lsst.afw.image.Image``), if requested. 2107 templateCoadd = supplementaryData.templateCoadd
2108 spanSetMaskList = self.
findArtifacts(templateCoadd, tempExpRefList, imageScalerList)
2109 badMaskPlanes = self.config.badMaskPlanes[:]
2110 badMaskPlanes.append(
"CLIPPED")
2111 badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes)
2113 result = AssembleCoaddTask.run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
2114 spanSetMaskList, mask=badPixelMask)
2118 self.
applyAltEdgeMask(result.coaddExposure.maskedImage.mask, spanSetMaskList)
2122 """Propagate alt EDGE mask to SENSOR_EDGE AND INEXACT_PSF planes. 2126 mask : `lsst.afw.image.Mask` 2128 altMaskList : `list` 2129 List of Dicts containing ``spanSet`` lists. 2130 Each element contains the new mask plane name (e.g. "CLIPPED 2131 and/or "NO_DATA") as the key, and list of ``SpanSets`` to apply to 2134 maskValue = mask.getPlaneBitMask([
"SENSOR_EDGE",
"INEXACT_PSF"])
2135 for visitMask
in altMaskList:
2136 if "EDGE" in visitMask:
2137 for spanSet
in visitMask[
'EDGE']:
2138 spanSet.clippedTo(mask.getBBox()).setMask(mask, maskValue)
2143 Loop through warps twice. The first loop builds a map with the count 2144 of how many epochs each pixel deviates from the templateCoadd by more 2145 than ``config.chiThreshold`` sigma. The second loop takes each 2146 difference image and filters the artifacts detected in each using 2147 count map to filter out variable sources and sources that are 2148 difficult to subtract cleanly. 2152 templateCoadd : `lsst.afw.image.Exposure` 2153 Exposure to serve as model of static sky. 2154 tempExpRefList : `list` 2155 List of data references to warps. 2156 imageScalerList : `list` 2157 List of image scalers. 2162 List of dicts containing information about CLIPPED 2163 (i.e., artifacts), NO_DATA, and EDGE pixels. 2166 self.log.debug(
"Generating Count Image, and mask lists.")
2167 coaddBBox = templateCoadd.getBBox()
2168 slateIm = afwImage.ImageU(coaddBBox)
2169 epochCountImage = afwImage.ImageU(coaddBBox)
2170 nImage = afwImage.ImageU(coaddBBox)
2171 spanSetArtifactList = []
2172 spanSetNoDataMaskList = []
2173 spanSetEdgeList = []
2177 templateCoadd.mask.clearAllMaskPlanes()
2179 if self.config.doPreserveContainedBySource:
2180 templateFootprints = self.detectTemplate.detectFootprints(templateCoadd)
2182 templateFootprints =
None 2184 for warpRef, imageScaler
in zip(tempExpRefList, imageScalerList):
2186 if warpDiffExp
is not None:
2188 nImage.array += (numpy.isfinite(warpDiffExp.image.array) *
2189 ((warpDiffExp.mask.array & badPixelMask) == 0)).astype(numpy.uint16)
2190 fpSet = self.detect.detectFootprints(warpDiffExp, doSmooth=
False, clearMask=
True)
2191 fpSet.positive.merge(fpSet.negative)
2192 footprints = fpSet.positive
2194 spanSetList = [footprint.spans
for footprint
in footprints.getFootprints()]
2197 if self.config.doPrefilterArtifacts:
2199 for spans
in spanSetList:
2200 spans.setImage(slateIm, 1, doClip=
True)
2201 epochCountImage += slateIm
2207 nans = numpy.where(numpy.isnan(warpDiffExp.maskedImage.image.array), 1, 0)
2208 nansMask = afwImage.makeMaskFromArray(nans.astype(afwImage.MaskPixel))
2209 nansMask.setXY0(warpDiffExp.getXY0())
2210 edgeMask = warpDiffExp.mask
2211 spanSetEdgeMask = afwGeom.SpanSet.fromMask(edgeMask,
2212 edgeMask.getPlaneBitMask(
"EDGE")).split()
2216 nansMask = afwImage.MaskX(coaddBBox, 1)
2218 spanSetEdgeMask = []
2220 spanSetNoDataMask = afwGeom.SpanSet.fromMask(nansMask).split()
2222 spanSetNoDataMaskList.append(spanSetNoDataMask)
2223 spanSetArtifactList.append(spanSetList)
2224 spanSetEdgeList.append(spanSetEdgeMask)
2228 epochCountImage.writeFits(path)
2230 for i, spanSetList
in enumerate(spanSetArtifactList):
2232 filteredSpanSetList = self.
filterArtifacts(spanSetList, epochCountImage, nImage,
2234 spanSetArtifactList[i] = filteredSpanSetList
2237 for artifacts, noData, edge
in zip(spanSetArtifactList, spanSetNoDataMaskList, spanSetEdgeList):
2238 altMasks.append({
'CLIPPED': artifacts,
2244 """Remove artifact candidates covered by bad mask plane. 2246 Any future editing of the candidate list that does not depend on 2247 temporal information should go in this method. 2251 spanSetList : `list` 2252 List of SpanSets representing artifact candidates. 2253 exp : `lsst.afw.image.Exposure` 2254 Exposure containing mask planes used to prefilter. 2258 returnSpanSetList : `list` 2259 List of SpanSets with artifacts. 2261 badPixelMask = exp.mask.getPlaneBitMask(self.config.prefilterArtifactsMaskPlanes)
2262 goodArr = (exp.mask.array & badPixelMask) == 0
2263 returnSpanSetList = []
2264 bbox = exp.getBBox()
2265 x0, y0 = exp.getXY0()
2266 for i, span
in enumerate(spanSetList):
2267 y, x = span.clippedTo(bbox).indices()
2268 yIndexLocal = numpy.array(y) - y0
2269 xIndexLocal = numpy.array(x) - x0
2270 goodRatio = numpy.count_nonzero(goodArr[yIndexLocal, xIndexLocal])/span.getArea()
2271 if goodRatio > self.config.prefilterArtifactsRatio:
2272 returnSpanSetList.append(span)
2273 return returnSpanSetList
2275 def filterArtifacts(self, spanSetList, epochCountImage, nImage, footprintsToExclude=None):
2276 """Filter artifact candidates. 2280 spanSetList : `list` 2281 List of SpanSets representing artifact candidates. 2282 epochCountImage : `lsst.afw.image.Image` 2283 Image of accumulated number of warpDiff detections. 2284 nImage : `lsst.afw.image.Image` 2285 Image of the accumulated number of total epochs contributing. 2289 maskSpanSetList : `list` 2290 List of SpanSets with artifacts. 2293 maskSpanSetList = []
2294 x0, y0 = epochCountImage.getXY0()
2295 for i, span
in enumerate(spanSetList):
2296 y, x = span.indices()
2297 yIdxLocal = [y1 - y0
for y1
in y]
2298 xIdxLocal = [x1 - x0
for x1
in x]
2299 outlierN = epochCountImage.array[yIdxLocal, xIdxLocal]
2300 totalN = nImage.array[yIdxLocal, xIdxLocal]
2303 effMaxNumEpochsHighN = (self.config.maxNumEpochs +
2304 self.config.maxFractionEpochsHigh*numpy.mean(totalN))
2305 effMaxNumEpochsLowN = self.config.maxFractionEpochsLow * numpy.mean(totalN)
2306 effectiveMaxNumEpochs = int(min(effMaxNumEpochsLowN, effMaxNumEpochsHighN))
2307 nPixelsBelowThreshold = numpy.count_nonzero((outlierN > 0) &
2308 (outlierN <= effectiveMaxNumEpochs))
2309 percentBelowThreshold = nPixelsBelowThreshold / len(outlierN)
2310 if percentBelowThreshold > self.config.spatialThreshold:
2311 maskSpanSetList.append(span)
2313 if self.config.doPreserveContainedBySource
and footprintsToExclude
is not None:
2315 filteredMaskSpanSetList = []
2316 for span
in maskSpanSetList:
2318 for footprint
in footprintsToExclude.positive.getFootprints():
2319 if footprint.spans.contains(span):
2323 filteredMaskSpanSetList.append(span)
2324 maskSpanSetList = filteredMaskSpanSetList
2326 return maskSpanSetList
2328 def _readAndComputeWarpDiff(self, warpRef, imageScaler, templateCoadd):
2329 """Fetch a warp from the butler and return a warpDiff. 2333 warpRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef` 2334 Butler dataRef for the warp. 2335 imageScaler : `lsst.pipe.tasks.scaleZeroPoint.ImageScaler` 2336 An image scaler object. 2337 templateCoadd : `lsst.afw.image.Exposure` 2338 Exposure to be substracted from the scaled warp. 2342 warp : `lsst.afw.image.Exposure` 2343 Exposure of the image difference between the warp and template. 2348 if not warpRef.datasetExists(warpName):
2349 self.log.warn(
"Could not find %s %s; skipping it", warpName, warpRef.dataId)
2351 warp = warpRef.get(warpName, immediate=
True)
2353 imageScaler.scaleMaskedImage(warp.getMaskedImage())
2354 mi = warp.getMaskedImage()
2355 if self.config.doScaleWarpVariance:
2357 self.scaleWarpVariance.
run(mi)
2358 except Exception
as exc:
2359 self.log.warn(
"Unable to rescale variance of warp (%s); leaving it as-is" % (exc,))
2360 mi -= templateCoadd.getMaskedImage()
2363 def _dataRef2DebugPath(self, prefix, warpRef, coaddLevel=False):
2364 """Return a path to which to write debugging output. 2366 Creates a hyphen-delimited string of dataId values for simple filenames. 2371 Prefix for filename. 2372 warpRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef` 2373 Butler dataRef to make the path from. 2374 coaddLevel : `bool`, optional. 2375 If True, include only coadd-level keys (e.g., 'tract', 'patch', 2376 'filter', but no 'visit'). 2381 Path for debugging output. 2386 keys = warpRef.dataId.keys()
2387 keyList = sorted(keys, reverse=
True)
2389 filename =
"%s-%s.fits" % (prefix,
'-'.join([str(warpRef.dataId[k])
for k
in keyList]))
2390 return os.path.join(directory, filename)
def setBrightObjectMasks(self, exposure, dataId, brightObjectMasks)
def shrinkValidPolygons(self, coaddInputs)
def getCoaddDatasetName(self, warpType="direct")
def _dataRef2DebugPath(self, prefix, warpRef, coaddLevel=False)
def getGroupDataRef(butler, datasetType, groupTuple, keys)
Base class for coaddition.
def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList)
def assembleMetadata(self, coaddExposure, tempExpRefList, weightList)
def getTempExpRefList(self, patchRef, calExpRefList)
def removeMaskPlanes(self, maskedImage)
def makeSkyInfo(skyMap, tractId, patchId)
def _readAndComputeWarpDiff(self, warpRef, imageScaler, templateCoadd)
def prepareInputs(self, refList)
def applyAltMaskPlanes(self, mask, altMaskSpans)
def getSkyInfo(self, patchRef)
Use getSkyinfo to return the skyMap, tract and patch information, wcs and the outer bbox of the patch...
def getTempExpDatasetName(self, warpType="direct")
def __init__(self, args, kwargs)
def makeSupplementaryDataGen3(self, inputData, inputDataIds, outputDataIds, butler)
def prepareStats(self, mask=None)
def makeDataRefList(self, namespace)
def getBadPixelMask(self)
Convenience method to provide the bitmask from the mask plane names.
def assembleSubregion(self, coaddExposure, bbox, tempExpRefList, imageScalerList, weightList, altMaskList, statsFlags, statsCtrl, nImage=None)
def makeSupplementaryData(self, dataRef, selectDataList=None, warpRefList=None)
def detectClip(self, exp, tempExpRefList)
def setInexactPsf(self, mask)
def adaptArgsAndRun(self, inputData, inputDataIds, outputDataIds, butler)
def __init__(self, args, kwargs)
def makeSupplementaryData(self, dataRef, selectDataList=None, warpRefList=None)
def getPrerequisiteDatasetTypes(cls, config)
def filterArtifacts(self, spanSetList, epochCountImage, nImage, footprintsToExclude=None)
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, args, kwargs)
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, supplementaryData, args, kwargs)
def buildDifferenceImage(self, skyInfo, tempExpRefList, imageScalerList, weightList)
def _noTemplateMessage(self, warpType)
def selectExposures(self, patchRef, skyInfo=None, selectDataList=[])
Select exposures to coadd.
def setRejectedMaskMapping(statsCtrl)
def getOutputDatasetTypes(cls, config)
def applyAltEdgeMask(self, mask, altMaskList)
def getInputDatasetTypes(cls, config)
def makeSupplementaryDataGen3(self, inputData, inputDataIds, outputDataIds, butler)
def readBrightObjectMasks(self, dataRef)
def runDataRef(self, dataRef, selectDataList=None, warpRefList=None)
def processResults(self, coaddExposure, dataRef)
def __init__(self, args, kwargs)
def prefilterArtifacts(self, spanSetList, exp)
def _subBBoxIter(bbox, subregionSize)
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, altMaskList=None, mask=None, supplementaryData=None)
def countMaskFromFootprint(mask, footprint, bitmask, ignoreMask)
def groupPatchExposures(patchDataRef, calexpDataRefList, coaddDatasetType="deepCoadd", tempExpDatasetType="deepCoadd_directWarp")
def detectClipBig(self, clipList, clipFootprints, clipIndices, detectionFootprints, maskClipValue, maskDetValue, coaddBBox)