37 from .coaddBase
import CoaddBaseTask, SelectDataIdContainer, makeSkyInfo
38 from .interpImage
import InterpImageTask
39 from .scaleZeroPoint
import ScaleZeroPointTask
40 from .coaddHelpers
import groupPatchExposures, getGroupDataRef
41 from .scaleVariance
import ScaleVarianceTask
45 __all__ = [
"AssembleCoaddTask",
"AssembleCoaddConfig",
"SafeClipAssembleCoaddTask",
46 "SafeClipAssembleCoaddConfig",
"CompareWarpAssembleCoaddTask",
"CompareWarpAssembleCoaddConfig"]
50 """Configuration parameters for the `AssembleCoaddTask`. 54 The `doMaskBrightObjects` and `brightObjectMaskName` configuration options 55 only set the bitplane config.brightObjectMaskName. To make this useful you 56 *must* also configure the flags.pixel algorithm, for example by adding 60 config.measurement.plugins["base_PixelFlags"].masksFpCenter.append("BRIGHT_OBJECT") 61 config.measurement.plugins["base_PixelFlags"].masksFpAnywhere.append("BRIGHT_OBJECT") 63 to your measureCoaddSources.py and forcedPhotCoadd.py config overrides. 65 warpType = pexConfig.Field(
66 doc=
"Warp name: one of 'direct' or 'psfMatched'",
70 subregionSize = pexConfig.ListField(
72 doc=
"Width, height of stack subregion size; " 73 "make small enough that a full stack of images will fit into memory at once.",
77 statistic = pexConfig.Field(
79 doc=
"Main stacking statistic for aggregating over the epochs.",
82 doSigmaClip = pexConfig.Field(
84 doc=
"Perform sigma clipped outlier rejection with MEANCLIP statistic? (DEPRECATED)",
87 sigmaClip = pexConfig.Field(
89 doc=
"Sigma for outlier rejection; ignored if non-clipping statistic selected.",
92 clipIter = pexConfig.Field(
94 doc=
"Number of iterations of outlier rejection; ignored if non-clipping statistic selected.",
97 calcErrorFromInputVariance = pexConfig.Field(
99 doc=
"Calculate coadd variance from input variance by stacking statistic." 100 "Passed to StatisticsControl.setCalcErrorFromInputVariance()",
103 scaleZeroPoint = pexConfig.ConfigurableField(
104 target=ScaleZeroPointTask,
105 doc=
"Task to adjust the photometric zero point of the coadd temp exposures",
107 doInterp = pexConfig.Field(
108 doc=
"Interpolate over NaN pixels? Also extrapolate, if necessary, but the results are ugly.",
112 interpImage = pexConfig.ConfigurableField(
113 target=InterpImageTask,
114 doc=
"Task to interpolate (and extrapolate) over NaN pixels",
116 doWrite = pexConfig.Field(
117 doc=
"Persist coadd?",
121 doNImage = pexConfig.Field(
122 doc=
"Create image of number of contributing exposures for each pixel",
126 doUsePsfMatchedPolygons = pexConfig.Field(
127 doc=
"Use ValidPolygons from shrunk Psf-Matched Calexps? Should be set to True by CompareWarp only.",
131 maskPropagationThresholds = pexConfig.DictField(
134 doc=(
"Threshold (in fractional weight) of rejection at which we propagate a mask plane to " 135 "the coadd; that is, we set the mask bit on the coadd if the fraction the rejected frames " 136 "would have contributed exceeds this value."),
137 default={
"SAT": 0.1},
139 removeMaskPlanes = pexConfig.ListField(dtype=str, default=[
"NOT_DEBLENDED"],
140 doc=
"Mask planes to remove before coadding")
141 doMaskBrightObjects = pexConfig.Field(dtype=bool, default=
False,
142 doc=
"Set mask and flag bits for bright objects?")
143 brightObjectMaskName = pexConfig.Field(dtype=str, default=
"BRIGHT_OBJECT",
144 doc=
"Name of mask bit used for bright objects")
145 coaddPsf = pexConfig.ConfigField(
146 doc=
"Configuration for CoaddPsf",
147 dtype=measAlg.CoaddPsfConfig,
149 doAttachTransmissionCurve = pexConfig.Field(
150 dtype=bool, default=
False, optional=
False,
151 doc=(
"Attach a piecewise TransmissionCurve for the coadd? " 152 "(requires all input Exposures to have TransmissionCurves).")
154 inputWarps = pipeBase.InputDatasetField(
155 doc=(
"Input list of warps to be assemebled i.e. stacked." 156 "WarpType (e.g. direct, psfMatched) is controlled by we warpType config parameter"),
157 nameTemplate=
"{inputCoaddName}Coadd_{warpType}Warp",
158 storageClass=
"ExposureF",
159 dimensions=(
"Tract",
"Patch",
"SkyMap",
"Visit"),
162 skyMap = pipeBase.InputDatasetField(
163 doc=
"Input definition of geometry/bbox and projection/wcs for coadded exposures",
164 nameTemplate=
"{inputCoaddName}Coadd_skyMap",
165 storageClass=
"SkyMap",
166 dimensions=(
"SkyMap", ),
169 brightObjectMask = pipeBase.InputDatasetField(
170 doc=(
"Input Bright Object Mask mask produced with external catalogs to be applied to the mask plane" 171 "BRIGHT_OBJECT. Currently unavailable in Gen3 test repos. TODO: DM-17300"),
172 name=
"brightObjectMask",
174 dimensions=(
"Tract",
"Patch",
"SkyMap",
"AbstractFilter"),
177 coaddExposure = pipeBase.OutputDatasetField(
178 doc=
"Output coadded exposure, produced by stacking input warps",
179 nameTemplate=
"{outputCoaddName}Coadd",
180 storageClass=
"ExposureF",
181 dimensions=(
"Tract",
"Patch",
"SkyMap",
"AbstractFilter"),
184 nImage = pipeBase.OutputDatasetField(
185 doc=
"Output image of number of input images per pixel",
186 nameTemplate=
"{outputCoaddName}Coadd_nImage",
187 storageClass=
"ImageU",
188 dimensions=(
"Tract",
"Patch",
"SkyMap",
"AbstractFilter"),
195 self.formatTemplateNames({
"inputCoaddName":
"deep",
"outputCoaddName":
"deep",
197 self.quantum.dimensions = (
"Tract",
"Patch",
"AbstractFilter",
"SkyMap")
204 log.warn(
"Config doPsfMatch deprecated. Setting warpType='psfMatched'")
207 log.warn(
'doSigmaClip deprecated. To replicate behavior, setting statistic to "MEANCLIP"')
209 if self.
doInterp and self.
statistic not in [
'MEAN',
'MEDIAN',
'MEANCLIP',
'VARIANCE',
'VARIANCECLIP']:
210 raise ValueError(
"Must set doInterp=False for statistic=%s, which does not " 211 "compute and set a non-zero coadd variance estimate." % (self.
statistic))
213 unstackableStats = [
'NOTHING',
'ERROR',
'ORMASK']
214 if not hasattr(afwMath.Property, self.
statistic)
or self.
statistic in unstackableStats:
215 stackableStats = [str(k)
for k
in afwMath.Property.__members__.keys()
216 if str(k)
not in unstackableStats]
217 raise ValueError(
"statistic %s is not allowed. Please choose one of %s." 222 """Assemble a coadded image from a set of warps (coadded temporary exposures). 224 We want to assemble a coadded image from a set of Warps (also called 225 coadded temporary exposures or ``coaddTempExps``). 226 Each input Warp covers a patch on the sky and corresponds to a single 227 run/visit/exposure of the covered patch. We provide the task with a list 228 of Warps (``selectDataList``) from which it selects Warps that cover the 229 specified patch (pointed at by ``dataRef``). 230 Each Warp that goes into a coadd will typically have an independent 231 photometric zero-point. Therefore, we must scale each Warp to set it to 232 a common photometric zeropoint. WarpType may be one of 'direct' or 233 'psfMatched', and the boolean configs `config.makeDirect` and 234 `config.makePsfMatched` set which of the warp types will be coadded. 235 The coadd is computed as a mean with optional outlier rejection. 236 Criteria for outlier rejection are set in `AssembleCoaddConfig`. 237 Finally, Warps can have bad 'NaN' pixels which received no input from the 238 source calExps. We interpolate over these bad (NaN) pixels. 240 `AssembleCoaddTask` uses several sub-tasks. These are 242 - `ScaleZeroPointTask` 243 - create and use an ``imageScaler`` object to scale the photometric zeropoint for each Warp 245 - interpolate across bad pixels (NaN) in the final coadd 247 You can retarget these subtasks if you wish. 251 The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a 252 flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``; see 253 `baseDebug` for more about ``debug.py`` files. `AssembleCoaddTask` has 254 no debug variables of its own. Some of the subtasks may support debug 255 variables. See the documentation for the subtasks for further information. 259 `AssembleCoaddTask` assembles a set of warped images into a coadded image. 260 The `AssembleCoaddTask` can be invoked by running ``assembleCoadd.py`` 261 with the flag '--legacyCoadd'. Usage of assembleCoadd.py expects two 262 inputs: a data reference to the tract patch and filter to be coadded, and 263 a list of Warps to attempt to coadd. These are specified using ``--id`` and 264 ``--selectId``, respectively: 268 --id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]] 269 --selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]] 271 Only the Warps that cover the specified tract and patch will be coadded. 272 A list of the available optional arguments can be obtained by calling 273 ``assembleCoadd.py`` with the ``--help`` command line argument: 277 assembleCoadd.py --help 279 To demonstrate usage of the `AssembleCoaddTask` in the larger context of 280 multi-band processing, we will generate the HSC-I & -R band coadds from 281 HSC engineering test data provided in the ``ci_hsc`` package. To begin, 282 assuming that the lsst stack has been already set up, we must set up the 283 obs_subaru and ``ci_hsc`` packages. This defines the environment variable 284 ``$CI_HSC_DIR`` and points at the location of the package. The raw HSC 285 data live in the ``$CI_HSC_DIR/raw directory``. To begin assembling the 286 coadds, we must first 289 - process the individual ccds in $CI_HSC_RAW to produce calibrated exposures 291 - create a skymap that covers the area of the sky present in the raw exposures 293 - warp the individual calibrated exposures to the tangent plane of the coadd 295 We can perform all of these steps by running 299 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988 301 This will produce warped exposures for each visit. To coadd the warped 302 data, we call assembleCoadd.py as follows: 306 assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \ 307 --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \ 308 --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \ 309 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \ 310 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \ 311 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \ 312 --selectId visit=903988 ccd=24 314 that will process the HSC-I band data. The results are written in 315 ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``. 317 You may also choose to run: 321 scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346 322 assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R \ 323 --selectId visit=903334 ccd=16 --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 \ 324 --selectId visit=903334 ccd=100 --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 \ 325 --selectId visit=903338 ccd=18 --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 \ 326 --selectId visit=903342 ccd=10 --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 \ 327 --selectId visit=903344 ccd=5 --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 \ 328 --selectId visit=903346 ccd=6 --selectId visit=903346 ccd=12 330 to generate the coadd for the HSC-R band if you are interested in 331 following multiBand Coadd processing as discussed in `pipeTasks_multiBand` 332 (but note that normally, one would use the `SafeClipAssembleCoaddTask` 333 rather than `AssembleCoaddTask` to make the coadd. 335 ConfigClass = AssembleCoaddConfig
336 _DefaultName =
"assembleCoadd" 341 argNames = [
"config",
"name",
"parentTask",
"log"]
342 kwargs.update({k: v
for k, v
in zip(argNames, args)})
343 warnings.warn(
"AssembleCoadd received positional args, and casting them as kwargs: %s. " 344 "PipelineTask will not take positional args" % argNames, FutureWarning)
347 self.makeSubtask(
"interpImage")
348 self.makeSubtask(
"scaleZeroPoint")
350 if self.config.doMaskBrightObjects:
351 mask = afwImage.Mask()
354 except pexExceptions.LsstCppException:
355 raise RuntimeError(
"Unable to define mask plane for bright objects; planes used are %s" %
356 mask.getMaskPlaneDict().keys())
363 """Return output dataset type descriptors 365 Remove output dataset types not produced by the Task 368 if not config.doNImage:
369 outputTypeDict.pop(
"nImage",
None)
370 return outputTypeDict
374 """Return input dataset type descriptors 376 Remove input dataset types not used by the Task 379 if not config.doMaskBrightObjects:
380 inputTypeDict.pop(
"brightObjectMask",
None)
382 raise RuntimeError(
"Gen3 Bright object masks unavailable. Set config doMaskBrightObjects=False")
386 """Assemble a coadd from a set of Warps. 388 PipelineTask (Gen3) entry point to Coadd a set of Warps. 389 Analogous to `runDataRef`, it prepares all the data products to be 390 passed to `run`, and processes the results before returning to struct 391 of results to be written out. AssembleCoadd cannot fit all Warps in memory. 392 Therefore, its inputs are accessed subregion by subregion 393 by the `lsst.daf.butler.ShimButler` that quacks like a Gen2 394 `lsst.daf.persistence.Butler`. Updates to this method should 395 correspond to an update in `runDataRef` while both entry points 401 Keys are the names of the configs describing input dataset types. 402 Values are input Python-domain data objects (or lists of objects) 403 retrieved from data butler. 404 inputDataIds : `dict` 405 Keys are the names of the configs describing input dataset types. 406 Values are DataIds (or lists of DataIds) that task consumes for 407 corresponding dataset type. 408 outputDataIds : `dict` 409 Keys are the names of the configs describing input dataset types. 410 Values are DataIds (or lists of DataIds) that task is to produce 411 for corresponding dataset type. 412 butler : `lsst.daf.butler.Butler` 413 Gen3 Butler object for fetching additional data products before 418 result : `lsst.pipe.base.Struct` 419 Result struct with components: 421 - ``coaddExposure`` : coadded exposure (``lsst.afw.image.Exposure``) 422 - ``nImage``: N Image (``lsst.afw.image.Image``) 426 skyMap = inputData[
"skyMap"]
427 outputDataId = next(iter(outputDataIds.values()))
429 tractId=outputDataId[
'tract'],
430 patchId=outputDataId[
'patch'])
434 warpRefList = [butlerShim.dataRef(self.config.inputWarps.name, dataId=dataId)
435 for dataId
in inputDataIds[
'inputWarps']]
438 patchRef = butlerShim.dataRef(self.config.coaddExposure.name, dataId=outputDataIds[
'coaddExposure'])
442 self.log.info(
"Found %d %s", len(inputs.tempExpRefList),
444 if len(inputs.tempExpRefList) == 0:
445 self.log.warn(
"No coadd temporary exposures found")
450 retStruct = self.
run(inputData[
'skyInfo'], inputs.tempExpRefList, inputs.imageScalerList,
451 inputs.weightList, supplementaryData=supplementaryData)
457 def runDataRef(self, dataRef, selectDataList=None, warpRefList=None):
458 """Assemble a coadd from a set of Warps. 460 Pipebase.CmdlineTask entry point to Coadd a set of Warps. 461 Compute weights to be applied to each Warp and 462 find scalings to match the photometric zeropoint to a reference Warp. 463 Assemble the Warps using `run`. Interpolate over NaNs and 464 optionally write the coadd to disk. Return the coadded exposure. 468 dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef` 469 Data reference defining the patch for coaddition and the 470 reference Warp (if ``config.autoReference=False``). 471 Used to access the following data products: 472 - ``self.config.coaddName + "Coadd_skyMap"`` 473 - ``self.config.coaddName + "Coadd_ + <warpType> + "Warp"`` (optionally) 474 - ``self.config.coaddName + "Coadd"`` 475 selectDataList : `list` 476 List of data references to Calexps. Data to be coadded will be 477 selected from this list based on overlap with the patch defined 478 by dataRef, grouped by visit, and converted to a list of data 481 List of data references to Warps to be coadded. 482 Note: `warpRefList` is just the new name for `tempExpRefList`. 486 retStruct : `lsst.pipe.base.Struct` 487 Result struct with components: 489 - ``coaddExposure``: coadded exposure (``Exposure``). 490 - ``nImage``: exposure count image (``Image``). 492 if selectDataList
and warpRefList:
493 raise RuntimeError(
"runDataRef received both a selectDataList and warpRefList, " 494 "and which to use is ambiguous. Please pass only one.")
497 if warpRefList
is None:
498 calExpRefList = self.
selectExposures(dataRef, skyInfo, selectDataList=selectDataList)
499 if len(calExpRefList) == 0:
500 self.log.warn(
"No exposures to coadd")
502 self.log.info(
"Coadding %d exposures", len(calExpRefList))
507 self.log.info(
"Found %d %s", len(inputData.tempExpRefList),
509 if len(inputData.tempExpRefList) == 0:
510 self.log.warn(
"No coadd temporary exposures found")
515 retStruct = self.
run(skyInfo, inputData.tempExpRefList, inputData.imageScalerList,
516 inputData.weightList, supplementaryData=supplementaryData)
519 if self.config.doWrite:
522 if self.config.doNImage
and retStruct.nImage
is not None:
528 """Interpolate over missing data and mask bright stars. 532 coaddExposure : `lsst.afw.image.Exposure` 533 The coadded exposure to process. 534 dataRef : `lsst.daf.persistence.ButlerDataRef` 535 Butler data reference for supplementary data. 537 if self.config.doInterp:
538 self.interpImage.
run(coaddExposure.getMaskedImage(), planeName=
"NO_DATA")
540 varArray = coaddExposure.variance.array
541 with numpy.errstate(invalid=
"ignore"):
542 varArray[:] = numpy.where(varArray > 0, varArray, numpy.inf)
544 if self.config.doMaskBrightObjects:
549 """Make additional inputs to run() specific to subclasses (Gen2) 551 Duplicates interface of `runDataRef` method 552 Available to be implemented by subclasses only if they need the 553 coadd dataRef for performing preliminary processing before 554 assembling the coadd. 558 dataRef : `lsst.daf.persistence.ButlerDataRef` 559 Butler data reference for supplementary data. 560 selectDataList : `list` 561 List of data references to Warps. 566 """Make additional inputs to run() specific to subclasses (Gen3) 568 Duplicates interface of`adaptArgsAndRun` method. 569 Available to be implemented by subclasses only if they need the 570 coadd dataRef for performing preliminary processing before 571 assembling the coadd. 576 Keys are the names of the configs describing input dataset types. 577 Values are input Python-domain data objects (or lists of objects) 578 retrieved from data butler. 579 inputDataIds : `dict` 580 Keys are the names of the configs describing input dataset types. 581 Values are DataIds (or lists of DataIds) that task consumes for 582 corresponding dataset type. 583 DataIds are guaranteed to match data objects in ``inputData``. 584 outputDataIds : `dict` 585 Keys are the names of the configs describing input dataset types. 586 Values are DataIds (or lists of DataIds) that task is to produce 587 for corresponding dataset type. 588 butler : `lsst.daf.butler.Butler` 589 Gen3 Butler object for fetching additional data products before 594 result : `lsst.pipe.base.Struct` 595 Contains whatever additional data the subclass's `run` method needs 600 """Generate list data references corresponding to warped exposures 601 that lie within the patch to be coadded. 606 Data reference for patch. 607 calExpRefList : `list` 608 List of data references for input calexps. 612 tempExpRefList : `list` 613 List of Warp/CoaddTempExp data references. 615 butler = patchRef.getButler()
616 groupData =
groupPatchExposures(patchRef, calExpRefList, self.getCoaddDatasetName(self.warpType),
617 self.getTempExpDatasetName(self.warpType))
618 tempExpRefList = [
getGroupDataRef(butler, self.getTempExpDatasetName(self.warpType),
619 g, groupData.keys)
for 620 g
in groupData.groups.keys()]
621 return tempExpRefList
624 """Prepare the input warps for coaddition by measuring the weight for 625 each warp and the scaling for the photometric zero point. 627 Each Warp has its own photometric zeropoint and background variance. 628 Before coadding these Warps together, compute a scale factor to 629 normalize the photometric zeropoint and compute the weight for each Warp. 634 List of data references to tempExp 638 result : `lsst.pipe.base.Struct` 639 Result struct with components: 641 - ``tempExprefList``: `list` of data references to tempExp. 642 - ``weightList``: `list` of weightings. 643 - ``imageScalerList``: `list` of image scalers. 645 statsCtrl = afwMath.StatisticsControl()
646 statsCtrl.setNumSigmaClip(self.config.sigmaClip)
647 statsCtrl.setNumIter(self.config.clipIter)
649 statsCtrl.setNanSafe(
True)
657 for tempExpRef
in refList:
658 if not tempExpRef.datasetExists(tempExpName):
659 self.log.warn(
"Could not find %s %s; skipping it", tempExpName, tempExpRef.dataId)
662 tempExp = tempExpRef.get(tempExpName, immediate=
True)
663 maskedImage = tempExp.getMaskedImage()
664 imageScaler = self.scaleZeroPoint.computeImageScaler(
669 imageScaler.scaleMaskedImage(maskedImage)
670 except Exception
as e:
671 self.log.warn(
"Scaling failed for %s (skipping it): %s", tempExpRef.dataId, e)
673 statObj = afwMath.makeStatistics(maskedImage.getVariance(), maskedImage.getMask(),
674 afwMath.MEANCLIP, statsCtrl)
675 meanVar, meanVarErr = statObj.getResult(afwMath.MEANCLIP)
676 weight = 1.0 / float(meanVar)
677 if not numpy.isfinite(weight):
678 self.log.warn(
"Non-finite weight for %s: skipping", tempExpRef.dataId)
680 self.log.info(
"Weight of %s %s = %0.3f", tempExpName, tempExpRef.dataId, weight)
685 tempExpRefList.append(tempExpRef)
686 weightList.append(weight)
687 imageScalerList.append(imageScaler)
689 return pipeBase.Struct(tempExpRefList=tempExpRefList, weightList=weightList,
690 imageScalerList=imageScalerList)
693 """Prepare the statistics for coadding images. 697 mask : `int`, optional 698 Bit mask value to exclude from coaddition. 702 stats : `lsst.pipe.base.Struct` 703 Statistics structure with the following fields: 705 - ``statsCtrl``: Statistics control object for coadd 706 (`lsst.afw.math.StatisticsControl`) 707 - ``statsFlags``: Statistic for coadd (`lsst.afw.math.Property`) 711 statsCtrl = afwMath.StatisticsControl()
712 statsCtrl.setNumSigmaClip(self.config.sigmaClip)
713 statsCtrl.setNumIter(self.config.clipIter)
714 statsCtrl.setAndMask(mask)
715 statsCtrl.setNanSafe(
True)
716 statsCtrl.setWeighted(
True)
717 statsCtrl.setCalcErrorFromInputVariance(self.config.calcErrorFromInputVariance)
718 for plane, threshold
in self.config.maskPropagationThresholds.items():
719 bit = afwImage.Mask.getMaskPlane(plane)
720 statsCtrl.setMaskPropagationThreshold(bit, threshold)
721 statsFlags = afwMath.stringToStatisticsProperty(self.config.statistic)
722 return pipeBase.Struct(ctrl=statsCtrl, flags=statsFlags)
724 def run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
725 altMaskList=None, mask=None, supplementaryData=None):
726 """Assemble a coadd from input warps 728 Assemble the coadd using the provided list of coaddTempExps. Since 729 the full coadd covers a patch (a large area), the assembly is 730 performed over small areas on the image at a time in order to 731 conserve memory usage. Iterate over subregions within the outer 732 bbox of the patch using `assembleSubregion` to stack the corresponding 733 subregions from the coaddTempExps with the statistic specified. 734 Set the edge bits the coadd mask based on the weight map. 738 skyInfo : `lsst.pipe.base.Struct` 739 Struct with geometric information about the patch. 740 tempExpRefList : `list` 741 List of data references to Warps (previously called CoaddTempExps). 742 imageScalerList : `list` 743 List of image scalers. 746 altMaskList : `list`, optional 747 List of alternate masks to use rather than those stored with 749 mask : `lsst.afw.image.Mask`, optional 750 Mask to ignore when coadding 751 supplementaryData : lsst.pipe.base.Struct, optional 752 Struct with additional data products needed to assemble coadd. 753 Only used by subclasses that implement `makeSupplementaryData` 758 result : `lsst.pipe.base.Struct` 759 Result struct with components: 761 - ``coaddExposure``: coadded exposure (``lsst.afw.image.Exposure``). 762 - ``nImage``: exposure count image (``lsst.afw.image.Image``). 765 self.log.info(
"Assembling %s %s", len(tempExpRefList), tempExpName)
768 if altMaskList
is None:
769 altMaskList = [
None]*len(tempExpRefList)
771 coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
772 coaddExposure.setCalib(self.scaleZeroPoint.getCalib())
773 coaddExposure.getInfo().setCoaddInputs(self.inputRecorder.makeCoaddInputs())
775 coaddMaskedImage = coaddExposure.getMaskedImage()
776 subregionSizeArr = self.config.subregionSize
777 subregionSize = afwGeom.Extent2I(subregionSizeArr[0], subregionSizeArr[1])
779 if self.config.doNImage:
780 nImage = afwImage.ImageU(skyInfo.bbox)
783 for subBBox
in self.
_subBBoxIter(skyInfo.bbox, subregionSize):
786 weightList, altMaskList, stats.flags, stats.ctrl,
788 except Exception
as e:
789 self.log.fatal(
"Cannot compute coadd %s: %s", subBBox, e)
794 coaddUtils.setCoaddEdgeBits(coaddMaskedImage.getMask(), coaddMaskedImage.getVariance())
795 return pipeBase.Struct(coaddExposure=coaddExposure, nImage=nImage)
798 """Set the metadata for the coadd. 800 This basic implementation sets the filter from the first input. 804 coaddExposure : `lsst.afw.image.Exposure` 805 The target exposure for the coadd. 806 tempExpRefList : `list` 807 List of data references to tempExp. 811 assert len(tempExpRefList) == len(weightList),
"Length mismatch" 816 tempExpList = [tempExpRef.get(tempExpName +
"_sub",
817 bbox=afwGeom.Box2I(coaddExposure.getBBox().getMin(),
818 afwGeom.Extent2I(1, 1)), immediate=
True)
819 for tempExpRef
in tempExpRefList]
820 numCcds = sum(len(tempExp.getInfo().getCoaddInputs().ccds)
for tempExp
in tempExpList)
822 coaddExposure.setFilter(tempExpList[0].getFilter())
823 coaddInputs = coaddExposure.getInfo().getCoaddInputs()
824 coaddInputs.ccds.reserve(numCcds)
825 coaddInputs.visits.reserve(len(tempExpList))
827 for tempExp, weight
in zip(tempExpList, weightList):
828 self.inputRecorder.addVisitToCoadd(coaddInputs, tempExp, weight)
830 if self.config.doUsePsfMatchedPolygons:
833 coaddInputs.visits.sort()
839 modelPsfList = [tempExp.getPsf()
for tempExp
in tempExpList]
840 modelPsfWidthList = [modelPsf.computeBBox().getWidth()
for modelPsf
in modelPsfList]
841 psf = modelPsfList[modelPsfWidthList.index(max(modelPsfWidthList))]
843 psf = measAlg.CoaddPsf(coaddInputs.ccds, coaddExposure.getWcs(),
844 self.config.coaddPsf.makeControl())
845 coaddExposure.setPsf(psf)
846 apCorrMap = measAlg.makeCoaddApCorrMap(coaddInputs.ccds, coaddExposure.getBBox(afwImage.PARENT),
847 coaddExposure.getWcs())
848 coaddExposure.getInfo().setApCorrMap(apCorrMap)
849 if self.config.doAttachTransmissionCurve:
850 transmissionCurve = measAlg.makeCoaddTransmissionCurve(coaddExposure.getWcs(), coaddInputs.ccds)
851 coaddExposure.getInfo().setTransmissionCurve(transmissionCurve)
853 def assembleSubregion(self, coaddExposure, bbox, tempExpRefList, imageScalerList, weightList,
854 altMaskList, statsFlags, statsCtrl, nImage=None):
855 """Assemble the coadd for a sub-region. 857 For each coaddTempExp, check for (and swap in) an alternative mask 858 if one is passed. Remove mask planes listed in 859 `config.removeMaskPlanes`. Finally, stack the actual exposures using 860 `lsst.afw.math.statisticsStack` with the statistic specified by 861 statsFlags. Typically, the statsFlag will be one of lsst.afw.math.MEAN for 862 a mean-stack or `lsst.afw.math.MEANCLIP` for outlier rejection using 863 an N-sigma clipped mean where N and iterations are specified by 864 statsCtrl. Assign the stacked subregion back to the coadd. 868 coaddExposure : `lsst.afw.image.Exposure` 869 The target exposure for the coadd. 870 bbox : `lsst.afw.geom.Box` 872 tempExpRefList : `list` 873 List of data reference to tempExp. 874 imageScalerList : `list` 875 List of image scalers. 879 List of alternate masks to use rather than those stored with 880 tempExp, or None. Each element is dict with keys = mask plane 881 name to which to add the spans. 882 statsFlags : `lsst.afw.math.Property` 883 Property object for statistic for coadd. 884 statsCtrl : `lsst.afw.math.StatisticsControl` 885 Statistics control object for coadd. 886 nImage : `lsst.afw.image.ImageU`, optional 887 Keeps track of exposure count for each pixel. 889 self.log.debug(
"Computing coadd over %s", bbox)
891 coaddExposure.mask.addMaskPlane(
"REJECTED")
892 coaddExposure.mask.addMaskPlane(
"CLIPPED")
893 coaddExposure.mask.addMaskPlane(
"SENSOR_EDGE")
895 clipped = afwImage.Mask.getPlaneBitMask(
"CLIPPED")
897 if nImage
is not None:
898 subNImage = afwImage.ImageU(bbox.getWidth(), bbox.getHeight())
899 for tempExpRef, imageScaler, altMask
in zip(tempExpRefList, imageScalerList, altMaskList):
900 exposure = tempExpRef.get(tempExpName +
"_sub", bbox=bbox)
901 maskedImage = exposure.getMaskedImage()
902 mask = maskedImage.getMask()
903 if altMask
is not None:
905 imageScaler.scaleMaskedImage(maskedImage)
909 if nImage
is not None:
910 subNImage.getArray()[maskedImage.getMask().getArray() & statsCtrl.getAndMask() == 0] += 1
911 if self.config.removeMaskPlanes:
913 maskedImageList.append(maskedImage)
915 with self.timer(
"stack"):
916 coaddSubregion = afwMath.statisticsStack(maskedImageList, statsFlags, statsCtrl, weightList,
919 coaddExposure.maskedImage.assign(coaddSubregion, bbox)
920 if nImage
is not None:
921 nImage.assign(subNImage, bbox)
924 """Unset the mask of an image for mask planes specified in the config. 928 maskedImage : `lsst.afw.image.MaskedImage` 929 The masked image to be modified. 931 mask = maskedImage.getMask()
932 for maskPlane
in self.config.removeMaskPlanes:
934 mask &= ~mask.getPlaneBitMask(maskPlane)
935 except Exception
as e:
936 self.log.warn(
"Unable to remove mask plane %s: %s", maskPlane, e.args[0])
940 """Map certain mask planes of the warps to new planes for the coadd. 942 If a pixel is rejected due to a mask value other than EDGE, NO_DATA, 943 or CLIPPED, set it to REJECTED on the coadd. 944 If a pixel is rejected due to EDGE, set the coadd pixel to SENSOR_EDGE. 945 If a pixel is rejected due to CLIPPED, set the coadd pixel to CLIPPED. 949 statsCtrl : `lsst.afw.math.StatisticsControl` 950 Statistics control object for coadd 954 maskMap : `list` of `tuple` of `int` 955 A list of mappings of mask planes of the warped exposures to 956 mask planes of the coadd. 958 edge = afwImage.Mask.getPlaneBitMask(
"EDGE")
959 noData = afwImage.Mask.getPlaneBitMask(
"NO_DATA")
960 clipped = afwImage.Mask.getPlaneBitMask(
"CLIPPED")
961 toReject = statsCtrl.getAndMask() & (~noData) & (~edge) & (~clipped)
962 maskMap = [(toReject, afwImage.Mask.getPlaneBitMask(
"REJECTED")),
963 (edge, afwImage.Mask.getPlaneBitMask(
"SENSOR_EDGE")),
968 """Apply in place alt mask formatted as SpanSets to a mask. 972 mask : `lsst.afw.image.Mask` 974 altMaskSpans : `dict` 975 SpanSet lists to apply. Each element contains the new mask 976 plane name (e.g. "CLIPPED and/or "NO_DATA") as the key, 977 and list of SpanSets to apply to the mask. 981 mask : `lsst.afw.image.Mask` 984 if self.config.doUsePsfMatchedPolygons:
985 if (
"NO_DATA" in altMaskSpans)
and (
"NO_DATA" in self.config.badMaskPlanes):
990 for spanSet
in altMaskSpans[
'NO_DATA']:
991 spanSet.clippedTo(mask.getBBox()).clearMask(mask, self.
getBadPixelMask())
993 for plane, spanSetList
in altMaskSpans.items():
994 maskClipValue = mask.addMaskPlane(plane)
995 for spanSet
in spanSetList:
996 spanSet.clippedTo(mask.getBBox()).setMask(mask, 2**maskClipValue)
1000 """Shrink coaddInputs' ccds' ValidPolygons in place. 1002 Either modify each ccd's validPolygon in place, or if CoaddInputs 1003 does not have a validPolygon, create one from its bbox. 1007 coaddInputs : `lsst.afw.image.coaddInputs` 1011 for ccd
in coaddInputs.ccds:
1012 polyOrig = ccd.getValidPolygon()
1013 validPolyBBox = polyOrig.getBBox()
if polyOrig
else ccd.getBBox()
1014 validPolyBBox.grow(-self.config.matchingKernelSize//2)
1016 validPolygon = polyOrig.intersectionSingle(validPolyBBox)
1018 validPolygon = afwGeom.polygon.Polygon(afwGeom.Box2D(validPolyBBox))
1019 ccd.setValidPolygon(validPolygon)
1022 """Retrieve the bright object masks. 1024 Returns None on failure. 1028 dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef` 1033 result : `lsst.daf.persistence.butlerSubset.ButlerDataRef` 1034 Bright object mask from the Butler object, or None if it cannot 1038 return dataRef.get(
"brightObjectMask", immediate=
True)
1039 except Exception
as e:
1040 self.log.warn(
"Unable to read brightObjectMask for %s: %s", dataRef.dataId, e)
1044 """Set the bright object masks. 1048 exposure : `lsst.afw.image.Exposure` 1049 Exposure under consideration. 1050 dataId : `lsst.daf.persistence.dataId` 1051 Data identifier dict for patch. 1052 brightObjectMasks : `lsst.afw.table` 1053 Table of bright objects to mask. 1058 if brightObjectMasks
is None:
1059 self.log.warn(
"Unable to apply bright object mask: none supplied")
1061 self.log.info(
"Applying %d bright object masks to %s", len(brightObjectMasks), dataId)
1062 md = brightObjectMasks.table.getMetadata()
1064 if not md.exists(k):
1065 self.log.warn(
"Expected to see %s in metadata", k)
1067 if md.getScalar(k) != dataId[k]:
1068 self.log.warn(
"Expected to see %s == %s in metadata, saw %s", k, md.get(k), dataId[k])
1070 mask = exposure.getMaskedImage().getMask()
1071 wcs = exposure.getWcs()
1072 plateScale = wcs.getPixelScale().asArcseconds()
1074 for rec
in brightObjectMasks:
1075 center = afwGeom.PointI(wcs.skyToPixel(rec.getCoord()))
1076 if rec[
"type"] ==
"box":
1077 assert rec[
"angle"] == 0.0, (
"Angle != 0 for mask object %s" % rec[
"id"])
1078 width = rec[
"width"].asArcseconds()/plateScale
1079 height = rec[
"height"].asArcseconds()/plateScale
1081 halfSize = afwGeom.ExtentI(0.5*width, 0.5*height)
1082 bbox = afwGeom.Box2I(center - halfSize, center + halfSize)
1084 bbox = afwGeom.BoxI(afwGeom.PointI(int(center[0] - 0.5*width), int(center[1] - 0.5*height)),
1085 afwGeom.PointI(int(center[0] + 0.5*width), int(center[1] + 0.5*height)))
1086 spans = afwGeom.SpanSet(bbox)
1087 elif rec[
"type"] ==
"circle":
1088 radius = int(rec[
"radius"].asArcseconds()/plateScale)
1089 spans = afwGeom.SpanSet.fromShape(radius, offset=center)
1091 self.log.warn(
"Unexpected region type %s at %s" % rec[
"type"], center)
1096 """Set INEXACT_PSF mask plane. 1098 If any of the input images isn't represented in the coadd (due to 1099 clipped pixels or chip gaps), the `CoaddPsf` will be inexact. Flag 1104 mask : `lsst.afw.image.Mask` 1105 Coadded exposure's mask, modified in-place. 1107 mask.addMaskPlane(
"INEXACT_PSF")
1108 inexactPsf = mask.getPlaneBitMask(
"INEXACT_PSF")
1109 sensorEdge = mask.getPlaneBitMask(
"SENSOR_EDGE")
1110 clipped = mask.getPlaneBitMask(
"CLIPPED")
1111 rejected = mask.getPlaneBitMask(
"REJECTED")
1112 array = mask.getArray()
1113 selected = array & (sensorEdge | clipped | rejected) > 0
1114 array[selected] |= inexactPsf
1117 def _makeArgumentParser(cls):
1118 """Create an argument parser. 1120 parser = pipeBase.ArgumentParser(name=cls.
_DefaultName)
1121 parser.add_id_argument(
"--id", cls.
ConfigClass().coaddName +
"Coadd_" +
1123 help=
"data ID, e.g. --id tract=12345 patch=1,2",
1124 ContainerClass=AssembleCoaddDataIdContainer)
1125 parser.add_id_argument(
"--selectId",
"calexp", help=
"data ID, e.g. --selectId visit=6789 ccd=0..9",
1126 ContainerClass=SelectDataIdContainer)
1130 def _subBBoxIter(bbox, subregionSize):
1131 """Iterate over subregions of a bbox. 1135 bbox : `lsst.afw.geom.Box2I` 1136 Bounding box over which to iterate. 1137 subregionSize: `lsst.afw.geom.Extent2I` 1142 subBBox : `lsst.afw.geom.Box2I` 1143 Next sub-bounding box of size ``subregionSize`` or smaller; each ``subBBox`` 1144 is contained within ``bbox``, so it may be smaller than ``subregionSize`` at 1145 the edges of ``bbox``, but it will never be empty. 1148 raise RuntimeError(
"bbox %s is empty" % (bbox,))
1149 if subregionSize[0] < 1
or subregionSize[1] < 1:
1150 raise RuntimeError(
"subregionSize %s must be nonzero" % (subregionSize,))
1152 for rowShift
in range(0, bbox.getHeight(), subregionSize[1]):
1153 for colShift
in range(0, bbox.getWidth(), subregionSize[0]):
1154 subBBox = afwGeom.Box2I(bbox.getMin() + afwGeom.Extent2I(colShift, rowShift), subregionSize)
1156 if subBBox.isEmpty():
1157 raise RuntimeError(
"Bug: empty bbox! bbox=%s, subregionSize=%s, " 1158 "colShift=%s, rowShift=%s" %
1159 (bbox, subregionSize, colShift, rowShift))
1164 """A version of `lsst.pipe.base.DataIdContainer` specialized for assembleCoadd. 1168 """Make self.refList from self.idList. 1173 Results of parsing command-line (with ``butler`` and ``log`` elements). 1175 datasetType = namespace.config.coaddName +
"Coadd" 1176 keysCoadd = namespace.butler.getKeys(datasetType=datasetType, level=self.level)
1178 for dataId
in self.idList:
1180 for key
in keysCoadd:
1181 if key
not in dataId:
1182 raise RuntimeError(
"--id must include " + key)
1184 dataRef = namespace.butler.dataRef(
1185 datasetType=datasetType,
1188 self.refList.append(dataRef)
1192 """Function to count the number of pixels with a specific mask in a 1195 Find the intersection of mask & footprint. Count all pixels in the mask 1196 that are in the intersection that have bitmask set but do not have 1197 ignoreMask set. Return the count. 1201 mask : `lsst.afw.image.Mask` 1202 Mask to define intersection region by. 1203 footprint : `lsst.afw.detection.Footprint` 1204 Footprint to define the intersection region by. 1206 Specific mask that we wish to count the number of occurances of. 1208 Pixels to not consider. 1213 Count of number of pixels in footprint with specified mask. 1215 bbox = footprint.getBBox()
1216 bbox.clip(mask.getBBox(afwImage.PARENT))
1217 fp = afwImage.Mask(bbox)
1218 subMask = mask.Factory(mask, bbox, afwImage.PARENT)
1219 footprint.spans.setMask(fp, bitmask)
1220 return numpy.logical_and((subMask.getArray() & fp.getArray()) > 0,
1221 (subMask.getArray() & ignoreMask) == 0).sum()
1225 """Configuration parameters for the SafeClipAssembleCoaddTask. 1227 clipDetection = pexConfig.ConfigurableField(
1228 target=SourceDetectionTask,
1229 doc=
"Detect sources on difference between unclipped and clipped coadd")
1230 minClipFootOverlap = pexConfig.Field(
1231 doc=
"Minimum fractional overlap of clipped footprint with visit DETECTED to be clipped",
1235 minClipFootOverlapSingle = pexConfig.Field(
1236 doc=
"Minimum fractional overlap of clipped footprint with visit DETECTED to be " 1237 "clipped when only one visit overlaps",
1241 minClipFootOverlapDouble = pexConfig.Field(
1242 doc=
"Minimum fractional overlap of clipped footprints with visit DETECTED to be " 1243 "clipped when two visits overlap",
1247 maxClipFootOverlapDouble = pexConfig.Field(
1248 doc=
"Maximum fractional overlap of clipped footprints with visit DETECTED when " 1249 "considering two visits",
1253 minBigOverlap = pexConfig.Field(
1254 doc=
"Minimum number of pixels in footprint to use DETECTED mask from the single visits " 1255 "when labeling clipped footprints",
1261 """Set default values for clipDetection. 1265 The numeric values for these configuration parameters were 1266 empirically determined, future work may further refine them. 1268 AssembleCoaddConfig.setDefaults(self)
1284 log.warn(
"Additional Sigma-clipping not allowed in Safe-clipped Coadds. " 1285 "Ignoring doSigmaClip.")
1288 raise ValueError(
"Only MEAN statistic allowed for final stacking in SafeClipAssembleCoadd " 1289 "(%s chosen). Please set statistic to MEAN." 1291 AssembleCoaddTask.ConfigClass.validate(self)
1295 """Assemble a coadded image from a set of coadded temporary exposures, 1296 being careful to clip & flag areas with potential artifacts. 1298 In ``AssembleCoaddTask``, we compute the coadd as an clipped mean (i.e., 1299 we clip outliers). The problem with doing this is that when computing the 1300 coadd PSF at a given location, individual visit PSFs from visits with 1301 outlier pixels contribute to the coadd PSF and cannot be treated correctly. 1302 In this task, we correct for this behavior by creating a new 1303 ``badMaskPlane`` 'CLIPPED'. We populate this plane on the input 1304 coaddTempExps and the final coadd where 1306 i. difference imaging suggests that there is an outlier and 1307 ii. this outlier appears on only one or two images. 1309 Such regions will not contribute to the final coadd. Furthermore, any 1310 routine to determine the coadd PSF can now be cognizant of clipped regions. 1311 Note that the algorithm implemented by this task is preliminary and works 1312 correctly for HSC data. Parameter modifications and or considerable 1313 redesigning of the algorithm is likley required for other surveys. 1315 ``SafeClipAssembleCoaddTask`` uses a ``SourceDetectionTask`` 1316 "clipDetection" subtask and also sub-classes ``AssembleCoaddTask``. 1317 You can retarget the ``SourceDetectionTask`` "clipDetection" subtask 1322 The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a 1323 flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``; 1324 see `baseDebug` for more about ``debug.py`` files. 1325 `SafeClipAssembleCoaddTask` has no debug variables of its own. 1326 The ``SourceDetectionTask`` "clipDetection" subtasks may support debug 1327 variables. See the documetation for `SourceDetectionTask` "clipDetection" 1328 for further information. 1332 `SafeClipAssembleCoaddTask` assembles a set of warped ``coaddTempExp`` 1333 images into a coadded image. The `SafeClipAssembleCoaddTask` is invoked by 1334 running assembleCoadd.py *without* the flag '--legacyCoadd'. 1336 Usage of ``assembleCoadd.py`` expects a data reference to the tract patch 1337 and filter to be coadded (specified using 1338 '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]') 1339 along with a list of coaddTempExps to attempt to coadd (specified using 1340 '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]'). 1341 Only the coaddTempExps that cover the specified tract and patch will be 1342 coadded. A list of the available optional arguments can be obtained by 1343 calling assembleCoadd.py with the --help command line argument: 1345 .. code-block:: none 1347 assembleCoadd.py --help 1349 To demonstrate usage of the `SafeClipAssembleCoaddTask` in the larger 1350 context of multi-band processing, we will generate the HSC-I & -R band 1351 coadds from HSC engineering test data provided in the ci_hsc package. 1352 To begin, assuming that the lsst stack has been already set up, we must 1353 set up the obs_subaru and ci_hsc packages. This defines the environment 1354 variable $CI_HSC_DIR and points at the location of the package. The raw 1355 HSC data live in the ``$CI_HSC_DIR/raw`` directory. To begin assembling 1356 the coadds, we must first 1359 process the individual ccds in $CI_HSC_RAW to produce calibrated exposures 1361 create a skymap that covers the area of the sky present in the raw exposures 1362 - ``makeCoaddTempExp`` 1363 warp the individual calibrated exposures to the tangent plane of the coadd</DD> 1365 We can perform all of these steps by running 1367 .. code-block:: none 1369 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988 1371 This will produce warped coaddTempExps for each visit. To coadd the 1372 warped data, we call ``assembleCoadd.py`` as follows: 1374 .. code-block:: none 1376 assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \ 1377 --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \ 1378 --selectId visit=903986 ccd=100--selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \ 1379 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \ 1380 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \ 1381 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \ 1382 --selectId visit=903988 ccd=24 1384 This will process the HSC-I band data. The results are written in 1385 ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``. 1387 You may also choose to run: 1389 .. code-block:: none 1391 scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346 nnn 1392 assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R --selectId visit=903334 ccd=16 \ 1393 --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 --selectId visit=903334 ccd=100 \ 1394 --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 --selectId visit=903338 ccd=18 \ 1395 --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 --selectId visit=903342 ccd=10 \ 1396 --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 --selectId visit=903344 ccd=5 \ 1397 --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 --selectId visit=903346 ccd=6 \ 1398 --selectId visit=903346 ccd=12 1400 to generate the coadd for the HSC-R band if you are interested in following 1401 multiBand Coadd processing as discussed in ``pipeTasks_multiBand``. 1403 ConfigClass = SafeClipAssembleCoaddConfig
1404 _DefaultName =
"safeClipAssembleCoadd" 1407 AssembleCoaddTask.__init__(self, *args, **kwargs)
1408 schema = afwTable.SourceTable.makeMinimalSchema()
1409 self.makeSubtask(
"clipDetection", schema=schema)
1411 def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, *args, **kwargs):
1412 """Assemble the coadd for a region. 1414 Compute the difference of coadds created with and without outlier 1415 rejection to identify coadd pixels that have outlier values in some 1417 Detect clipped regions on the difference image and mark these regions 1418 on the one or two individual coaddTempExps where they occur if there 1419 is significant overlap between the clipped region and a source. This 1420 leaves us with a set of footprints from the difference image that have 1421 been identified as having occured on just one or two individual visits. 1422 However, these footprints were generated from a difference image. It 1423 is conceivable for a large diffuse source to have become broken up 1424 into multiple footprints acrosss the coadd difference in this process. 1425 Determine the clipped region from all overlapping footprints from the 1426 detected sources in each visit - these are big footprints. 1427 Combine the small and big clipped footprints and mark them on a new 1429 Generate the coadd using `AssembleCoaddTask.run` without outlier 1430 removal. Clipped footprints will no longer make it into the coadd 1431 because they are marked in the new bad mask plane. 1435 skyInfo : `lsst.pipe.base.Struct` 1436 Patch geometry information, from getSkyInfo 1437 tempExpRefList : `list` 1438 List of data reference to tempExp 1439 imageScalerList : `list` 1440 List of image scalers 1446 result : `lsst.pipe.base.Struct` 1447 Result struct with components: 1449 - ``coaddExposure``: coadded exposure (``lsst.afw.image.Exposure``). 1450 - ``nImage``: exposure count image (``lsst.afw.image.Image``). 1454 args and kwargs are passed but ignored in order to match the call 1455 signature expected by the parent task. 1458 mask = exp.getMaskedImage().getMask()
1459 mask.addMaskPlane(
"CLIPPED")
1461 result = self.
detectClip(exp, tempExpRefList)
1463 self.log.info(
'Found %d clipped objects', len(result.clipFootprints))
1465 maskClipValue = mask.getPlaneBitMask(
"CLIPPED")
1466 maskDetValue = mask.getPlaneBitMask(
"DETECTED") | mask.getPlaneBitMask(
"DETECTED_NEGATIVE")
1468 bigFootprints = self.
detectClipBig(result.clipSpans, result.clipFootprints, result.clipIndices,
1469 result.detectionFootprints, maskClipValue, maskDetValue,
1472 maskClip = mask.Factory(mask.getBBox(afwImage.PARENT))
1473 afwDet.setMaskFromFootprintList(maskClip, result.clipFootprints, maskClipValue)
1475 maskClipBig = maskClip.Factory(mask.getBBox(afwImage.PARENT))
1476 afwDet.setMaskFromFootprintList(maskClipBig, bigFootprints, maskClipValue)
1477 maskClip |= maskClipBig
1480 badMaskPlanes = self.config.badMaskPlanes[:]
1481 badMaskPlanes.append(
"CLIPPED")
1482 badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes)
1483 return AssembleCoaddTask.run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
1484 result.clipSpans, mask=badPixelMask)
1487 """Return an exposure that contains the difference between unclipped 1490 Generate a difference image between clipped and unclipped coadds. 1491 Compute the difference image by subtracting an outlier-clipped coadd 1492 from an outlier-unclipped coadd. Return the difference image. 1496 skyInfo : `lsst.pipe.base.Struct` 1497 Patch geometry information, from getSkyInfo 1498 tempExpRefList : `list` 1499 List of data reference to tempExp 1500 imageScalerList : `list` 1501 List of image scalers 1507 exp : `lsst.afw.image.Exposure` 1508 Difference image of unclipped and clipped coadd wrapped in an Exposure 1513 configIntersection = {k: getattr(self.config, k)
1514 for k, v
in self.config.toDict().items()
if (k
in config.keys())}
1515 config.update(**configIntersection)
1518 config.statistic =
'MEAN' 1520 coaddMean = task.run(skyInfo, tempExpRefList, imageScalerList, weightList).coaddExposure
1522 config.statistic =
'MEANCLIP' 1524 coaddClip = task.run(skyInfo, tempExpRefList, imageScalerList, weightList).coaddExposure
1526 coaddDiff = coaddMean.getMaskedImage().Factory(coaddMean.getMaskedImage())
1527 coaddDiff -= coaddClip.getMaskedImage()
1528 exp = afwImage.ExposureF(coaddDiff)
1529 exp.setPsf(coaddMean.getPsf())
1533 """Detect clipped regions on an exposure and set the mask on the 1534 individual tempExp masks. 1536 Detect footprints in the difference image after smoothing the 1537 difference image with a Gaussian kernal. Identify footprints that 1538 overlap with one or two input ``coaddTempExps`` by comparing the 1539 computed overlap fraction to thresholds set in the config. A different 1540 threshold is applied depending on the number of overlapping visits 1541 (restricted to one or two). If the overlap exceeds the thresholds, 1542 the footprint is considered "CLIPPED" and is marked as such on the 1543 coaddTempExp. Return a struct with the clipped footprints, the indices 1544 of the ``coaddTempExps`` that end up overlapping with the clipped 1545 footprints, and a list of new masks for the ``coaddTempExps``. 1549 exp : `lsst.afw.image.Exposure` 1550 Exposure to run detection on. 1551 tempExpRefList : `list` 1552 List of data reference to tempExp. 1556 result : `lsst.pipe.base.Struct` 1557 Result struct with components: 1559 - ``clipFootprints``: list of clipped footprints. 1560 - ``clipIndices``: indices for each ``clippedFootprint`` in 1562 - ``clipSpans``: List of dictionaries containing spanSet lists 1563 to clip. Each element contains the new maskplane name 1564 ("CLIPPED") as the key and list of ``SpanSets`` as the value. 1565 - ``detectionFootprints``: List of DETECTED/DETECTED_NEGATIVE plane 1566 compressed into footprints. 1568 mask = exp.getMaskedImage().getMask()
1569 maskDetValue = mask.getPlaneBitMask(
"DETECTED") | mask.getPlaneBitMask(
"DETECTED_NEGATIVE")
1570 fpSet = self.clipDetection.detectFootprints(exp, doSmooth=
True, clearMask=
True)
1572 fpSet.positive.merge(fpSet.negative)
1573 footprints = fpSet.positive
1574 self.log.info(
'Found %d potential clipped objects', len(footprints.getFootprints()))
1579 artifactSpanSets = [{
'CLIPPED': list()}
for _
in tempExpRefList]
1582 visitDetectionFootprints = []
1584 dims = [len(tempExpRefList), len(footprints.getFootprints())]
1585 overlapDetArr = numpy.zeros(dims, dtype=numpy.uint16)
1586 ignoreArr = numpy.zeros(dims, dtype=numpy.uint16)
1589 for i, warpRef
in enumerate(tempExpRefList):
1591 immediate=
True).getMaskedImage().getMask()
1592 maskVisitDet = tmpExpMask.Factory(tmpExpMask, tmpExpMask.getBBox(afwImage.PARENT),
1593 afwImage.PARENT,
True)
1594 maskVisitDet &= maskDetValue
1595 visitFootprints = afwDet.FootprintSet(maskVisitDet, afwDet.Threshold(1))
1596 visitDetectionFootprints.append(visitFootprints)
1598 for j, footprint
in enumerate(footprints.getFootprints()):
1603 for j, footprint
in enumerate(footprints.getFootprints()):
1604 nPixel = footprint.getArea()
1607 for i
in range(len(tempExpRefList)):
1608 ignore = ignoreArr[i, j]
1609 overlapDet = overlapDetArr[i, j]
1610 totPixel = nPixel - ignore
1613 if ignore > overlapDet
or totPixel <= 0.5*nPixel
or overlapDet == 0:
1615 overlap.append(overlapDet/float(totPixel))
1618 overlap = numpy.array(overlap)
1619 if not len(overlap):
1626 if len(overlap) == 1:
1627 if overlap[0] > self.config.minClipFootOverlapSingle:
1632 clipIndex = numpy.where(overlap > self.config.minClipFootOverlap)[0]
1633 if len(clipIndex) == 1:
1635 keepIndex = [clipIndex[0]]
1638 clipIndex = numpy.where(overlap > self.config.minClipFootOverlapDouble)[0]
1639 if len(clipIndex) == 2
and len(overlap) > 3:
1640 clipIndexComp = numpy.where(overlap <= self.config.minClipFootOverlapDouble)[0]
1641 if numpy.max(overlap[clipIndexComp]) <= self.config.maxClipFootOverlapDouble:
1643 keepIndex = clipIndex
1648 for index
in keepIndex:
1649 globalIndex = indexList[index]
1650 artifactSpanSets[globalIndex][
'CLIPPED'].append(footprint.spans)
1652 clipIndices.append(numpy.array(indexList)[keepIndex])
1653 clipFootprints.append(footprint)
1655 return pipeBase.Struct(clipFootprints=clipFootprints, clipIndices=clipIndices,
1656 clipSpans=artifactSpanSets, detectionFootprints=visitDetectionFootprints)
1658 def detectClipBig(self, clipList, clipFootprints, clipIndices, detectionFootprints,
1659 maskClipValue, maskDetValue, coaddBBox):
1660 """Return individual warp footprints for large artifacts and append 1661 them to ``clipList`` in place. 1663 Identify big footprints composed of many sources in the coadd 1664 difference that may have originated in a large diffuse source in the 1665 coadd. We do this by indentifying all clipped footprints that overlap 1666 significantly with each source in all the coaddTempExps. 1671 List of alt mask SpanSets with clipping information. Modified. 1672 clipFootprints : `list` 1673 List of clipped footprints. 1674 clipIndices : `list` 1675 List of which entries in tempExpClipList each footprint belongs to. 1677 Mask value of clipped pixels. 1679 Mask value of detected pixels. 1680 coaddBBox : `lsst.afw.geom.Box` 1681 BBox of the coadd and warps. 1685 bigFootprintsCoadd : `list` 1686 List of big footprints 1688 bigFootprintsCoadd = []
1690 for index, (clippedSpans, visitFootprints)
in enumerate(zip(clipList, detectionFootprints)):
1691 maskVisitDet = afwImage.MaskX(coaddBBox, 0x0)
1692 for footprint
in visitFootprints.getFootprints():
1693 footprint.spans.setMask(maskVisitDet, maskDetValue)
1696 clippedFootprintsVisit = []
1697 for foot, clipIndex
in zip(clipFootprints, clipIndices):
1698 if index
not in clipIndex:
1700 clippedFootprintsVisit.append(foot)
1701 maskVisitClip = maskVisitDet.Factory(maskVisitDet.getBBox(afwImage.PARENT))
1702 afwDet.setMaskFromFootprintList(maskVisitClip, clippedFootprintsVisit, maskClipValue)
1704 bigFootprintsVisit = []
1705 for foot
in visitFootprints.getFootprints():
1706 if foot.getArea() < self.config.minBigOverlap:
1709 if nCount > self.config.minBigOverlap:
1710 bigFootprintsVisit.append(foot)
1711 bigFootprintsCoadd.append(foot)
1713 for footprint
in bigFootprintsVisit:
1714 clippedSpans[
"CLIPPED"].append(footprint.spans)
1716 return bigFootprintsCoadd
1720 assembleStaticSkyModel = pexConfig.ConfigurableField(
1721 target=AssembleCoaddTask,
1722 doc=
"Task to assemble an artifact-free, PSF-matched Coadd to serve as a" 1723 " naive/first-iteration model of the static sky.",
1725 detect = pexConfig.ConfigurableField(
1726 target=SourceDetectionTask,
1727 doc=
"Detect outlier sources on difference between each psfMatched warp and static sky model" 1729 detectTemplate = pexConfig.ConfigurableField(
1730 target=SourceDetectionTask,
1731 doc=
"Detect sources on static sky model. Only used if doPreserveContainedBySource is True" 1733 maxNumEpochs = pexConfig.Field(
1734 doc=
"Charactistic maximum local number of epochs/visits in which an artifact candidate can appear " 1735 "and still be masked. The effective maxNumEpochs is a broken linear function of local " 1736 "number of epochs (N): min(maxFractionEpochsLow*N, maxNumEpochs + maxFractionEpochsHigh*N). " 1737 "For each footprint detected on the image difference between the psfMatched warp and static sky " 1738 "model, if a significant fraction of pixels (defined by spatialThreshold) are residuals in more " 1739 "than the computed effective maxNumEpochs, the artifact candidate is deemed persistant rather " 1740 "than transient and not masked.",
1744 maxFractionEpochsLow = pexConfig.RangeField(
1745 doc=
"Fraction of local number of epochs (N) to use as effective maxNumEpochs for low N. " 1746 "Effective maxNumEpochs = " 1747 "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)",
1752 maxFractionEpochsHigh = pexConfig.RangeField(
1753 doc=
"Fraction of local number of epochs (N) to use as effective maxNumEpochs for high N. " 1754 "Effective maxNumEpochs = " 1755 "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)",
1760 spatialThreshold = pexConfig.RangeField(
1761 doc=
"Unitless fraction of pixels defining how much of the outlier region has to meet the " 1762 "temporal criteria. If 0, clip all. If 1, clip none.",
1766 inclusiveMin=
True, inclusiveMax=
True 1768 doScaleWarpVariance = pexConfig.Field(
1769 doc=
"Rescale Warp variance plane using empirical noise?",
1773 scaleWarpVariance = pexConfig.ConfigurableField(
1774 target=ScaleVarianceTask,
1775 doc=
"Rescale variance on warps",
1777 doPreserveContainedBySource = pexConfig.Field(
1778 doc=
"Rescue artifacts from clipping that completely lie within a footprint detected" 1779 "on the PsfMatched Template Coadd. Replicates a behavior of SafeClip.",
1783 doPrefilterArtifacts = pexConfig.Field(
1784 doc=
"Ignore artifact candidates that are mostly covered by the bad pixel mask, " 1785 "because they will be excluded anyway. This prevents them from contributing " 1786 "to the outlier epoch count image and potentially being labeled as persistant." 1787 "'Mostly' is defined by the config 'prefilterArtifactsRatio'.",
1791 prefilterArtifactsMaskPlanes = pexConfig.ListField(
1792 doc=
"Prefilter artifact candidates that are mostly covered by these bad mask planes.",
1794 default=(
'NO_DATA',
'BAD',
'SAT',
'SUSPECT'),
1796 prefilterArtifactsRatio = pexConfig.Field(
1797 doc=
"Prefilter artifact candidates with less than this fraction overlapping good pixels",
1801 psfMatchedWarps = pipeBase.InputDatasetField(
1802 doc=(
"PSF-Matched Warps are required by CompareWarp regardless of the coadd type requested. " 1803 "Only PSF-Matched Warps make sense for image subtraction. " 1804 "Therefore, they must be in the InputDatasetField and made available to the task."),
1805 nameTemplate=
"{inputCoaddName}Coadd_psfMatchedWarp",
1806 storageClass=
"ExposureF",
1807 dimensions=(
"Tract",
"Patch",
"SkyMap",
"Visit"),
1812 AssembleCoaddConfig.setDefaults(self)
1828 self.
detect.doTempLocalBackground =
False 1829 self.
detect.reEstimateBackground =
False 1830 self.
detect.returnOriginalFootprints =
False 1831 self.
detect.thresholdPolarity =
"both" 1832 self.
detect.thresholdValue = 5
1833 self.
detect.nSigmaToGrow = 2
1834 self.
detect.minPixels = 4
1835 self.
detect.isotropicGrow =
True 1836 self.
detect.thresholdType =
"pixel_stdev" 1844 """Assemble a compareWarp coadded image from a set of warps 1845 by masking artifacts detected by comparing PSF-matched warps. 1847 In ``AssembleCoaddTask``, we compute the coadd as an clipped mean (i.e., 1848 we clip outliers). The problem with doing this is that when computing the 1849 coadd PSF at a given location, individual visit PSFs from visits with 1850 outlier pixels contribute to the coadd PSF and cannot be treated correctly. 1851 In this task, we correct for this behavior by creating a new badMaskPlane 1852 'CLIPPED' which marks pixels in the individual warps suspected to contain 1853 an artifact. We populate this plane on the input warps by comparing 1854 PSF-matched warps with a PSF-matched median coadd which serves as a 1855 model of the static sky. Any group of pixels that deviates from the 1856 PSF-matched template coadd by more than config.detect.threshold sigma, 1857 is an artifact candidate. The candidates are then filtered to remove 1858 variable sources and sources that are difficult to subtract such as 1859 bright stars. This filter is configured using the config parameters 1860 ``temporalThreshold`` and ``spatialThreshold``. The temporalThreshold is 1861 the maximum fraction of epochs that the deviation can appear in and still 1862 be considered an artifact. The spatialThreshold is the maximum fraction of 1863 pixels in the footprint of the deviation that appear in other epochs 1864 (where other epochs is defined by the temporalThreshold). If the deviant 1865 region meets this criteria of having a significant percentage of pixels 1866 that deviate in only a few epochs, these pixels have the 'CLIPPED' bit 1867 set in the mask. These regions will not contribute to the final coadd. 1868 Furthermore, any routine to determine the coadd PSF can now be cognizant 1869 of clipped regions. Note that the algorithm implemented by this task is 1870 preliminary and works correctly for HSC data. Parameter modifications and 1871 or considerable redesigning of the algorithm is likley required for other 1874 ``CompareWarpAssembleCoaddTask`` sub-classes 1875 ``AssembleCoaddTask`` and instantiates ``AssembleCoaddTask`` 1876 as a subtask to generate the TemplateCoadd (the model of the static sky). 1880 The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a 1881 flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``; see 1882 ``baseDebug`` for more about ``debug.py`` files. 1884 This task supports the following debug variables: 1887 If True then save the Epoch Count Image as a fits file in the `figPath` 1889 Path to save the debug fits images and figures 1891 For example, put something like: 1893 .. code-block:: python 1896 def DebugInfo(name): 1897 di = lsstDebug.getInfo(name) 1898 if name == "lsst.pipe.tasks.assembleCoadd": 1899 di.saveCountIm = True 1900 di.figPath = "/desired/path/to/debugging/output/images" 1902 lsstDebug.Info = DebugInfo 1904 into your ``debug.py`` file and run ``assemebleCoadd.py`` with the 1905 ``--debug`` flag. Some subtasks may have their own debug variables; 1906 see individual Task documentation. 1910 ``CompareWarpAssembleCoaddTask`` assembles a set of warped images into a 1911 coadded image. The ``CompareWarpAssembleCoaddTask`` is invoked by running 1912 ``assembleCoadd.py`` with the flag ``--compareWarpCoadd``. 1913 Usage of ``assembleCoadd.py`` expects a data reference to the tract patch 1914 and filter to be coadded (specified using 1915 '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]') 1916 along with a list of coaddTempExps to attempt to coadd (specified using 1917 '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]'). 1918 Only the warps that cover the specified tract and patch will be coadded. 1919 A list of the available optional arguments can be obtained by calling 1920 ``assembleCoadd.py`` with the ``--help`` command line argument: 1922 .. code-block:: none 1924 assembleCoadd.py --help 1926 To demonstrate usage of the ``CompareWarpAssembleCoaddTask`` in the larger 1927 context of multi-band processing, we will generate the HSC-I & -R band 1928 oadds from HSC engineering test data provided in the ``ci_hsc`` package. 1929 To begin, assuming that the lsst stack has been already set up, we must 1930 set up the ``obs_subaru`` and ``ci_hsc`` packages. 1931 This defines the environment variable ``$CI_HSC_DIR`` and points at the 1932 location of the package. The raw HSC data live in the ``$CI_HSC_DIR/raw`` 1933 directory. To begin assembling the coadds, we must first 1936 process the individual ccds in $CI_HSC_RAW to produce calibrated exposures 1938 create a skymap that covers the area of the sky present in the raw exposures 1940 warp the individual calibrated exposures to the tangent plane of the coadd 1942 We can perform all of these steps by running 1944 .. code-block:: none 1946 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988 1948 This will produce warped ``coaddTempExps`` for each visit. To coadd the 1949 warped data, we call ``assembleCoadd.py`` as follows: 1951 .. code-block:: none 1953 assembleCoadd.py --compareWarpCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \ 1954 --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \ 1955 --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \ 1956 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \ 1957 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \ 1958 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \ 1959 --selectId visit=903988 ccd=24 1961 This will process the HSC-I band data. The results are written in 1962 ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``. 1964 ConfigClass = CompareWarpAssembleCoaddConfig
1965 _DefaultName =
"compareWarpAssembleCoadd" 1968 AssembleCoaddTask.__init__(self, *args, **kwargs)
1969 self.makeSubtask(
"assembleStaticSkyModel")
1970 detectionSchema = afwTable.SourceTable.makeMinimalSchema()
1971 self.makeSubtask(
"detect", schema=detectionSchema)
1972 if self.config.doPreserveContainedBySource:
1973 self.makeSubtask(
"detectTemplate", schema=afwTable.SourceTable.makeMinimalSchema())
1974 if self.config.doScaleWarpVariance:
1975 self.makeSubtask(
"scaleWarpVariance")
1978 """Make inputs specific to Subclass with Gen 3 API 1980 Calls Gen3 `adaptArgsAndRun` instead of the Gen2 specific `runDataRef` 1982 Duplicates interface of`adaptArgsAndRun` method. 1983 Available to be implemented by subclasses only if they need the 1984 coadd dataRef for performing preliminary processing before 1985 assembling the coadd. 1990 Keys are the names of the configs describing input dataset types. 1991 Values are input Python-domain data objects (or lists of objects) 1992 retrieved from data butler. 1993 inputDataIds : `dict` 1994 Keys are the names of the configs describing input dataset types. 1995 Values are DataIds (or lists of DataIds) that task consumes for 1996 corresponding dataset type. 1997 DataIds are guaranteed to match data objects in ``inputData``. 1998 outputDataIds : `dict` 1999 Keys are the names of the configs describing input dataset types. 2000 Values are DataIds (or lists of DataIds) that task is to produce 2001 for corresponding dataset type. 2002 butler : `lsst.daf.butler.Butler` 2003 Gen3 Butler object for fetching additional data products before 2008 result : `lsst.pipe.base.Struct` 2009 Result struct with components: 2011 - ``templateCoadd`` : coadded exposure (``lsst.afw.image.Exposure``) 2012 - ``nImage``: N Image (``lsst.afw.image.Image``) 2016 templateCoadd = self.assembleStaticSkyModel.
adaptArgsAndRun(inputData, inputDataIds,
2017 outputDataIds, butler)
2018 if templateCoadd
is None:
2021 return pipeBase.Struct(templateCoadd=templateCoadd.coaddExposure,
2022 nImage=templateCoadd.nImage)
2025 """Make inputs specific to Subclass. 2027 Generate a templateCoadd to use as a native model of static sky to 2028 subtract from warps. 2032 dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef` 2033 Butler dataRef for supplementary data. 2034 selectDataList : `list` (optional) 2035 Optional List of data references to Calexps. 2036 warpRefList : `list` (optional) 2037 Optional List of data references to Warps. 2041 result : `lsst.pipe.base.Struct` 2042 Result struct with components: 2044 - ``templateCoadd``: coadded exposure (``lsst.afw.image.Exposure``) 2045 - ``nImage``: N Image (``lsst.afw.image.Image``) 2047 templateCoadd = self.assembleStaticSkyModel.
runDataRef(dataRef, selectDataList, warpRefList)
2048 if templateCoadd
is None:
2051 return pipeBase.Struct(templateCoadd=templateCoadd.coaddExposure,
2052 nImage=templateCoadd.nImage)
2054 def _noTemplateMessage(self, warpType):
2055 warpName = (warpType[0].upper() + warpType[1:])
2056 message =
"""No %(warpName)s warps were found to build the template coadd which is 2057 required to run CompareWarpAssembleCoaddTask. To continue assembling this type of coadd, 2058 first either rerun makeCoaddTempExp with config.make%(warpName)s=True or 2059 coaddDriver with config.makeCoadTempExp.make%(warpName)s=True, before assembleCoadd. 2061 Alternatively, to use another algorithm with existing warps, retarget the CoaddDriverConfig to 2062 another algorithm like: 2064 from lsst.pipe.tasks.assembleCoadd import SafeClipAssembleCoaddTask 2065 config.assemble.retarget(SafeClipAssembleCoaddTask) 2066 """ % {
"warpName": warpName}
2069 def run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
2070 supplementaryData, *args, **kwargs):
2071 """Assemble the coadd. 2073 Find artifacts and apply them to the warps' masks creating a list of 2074 alternative masks with a new "CLIPPED" plane and updated "NO_DATA" 2075 plane. Then pass these alternative masks to the base class's `run` 2080 skyInfo : `lsst.pipe.base.Struct` 2081 Patch geometry information. 2082 tempExpRefList : `list` 2083 List of data references to warps. 2084 imageScalerList : `list` 2085 List of image scalers. 2088 supplementaryData : `lsst.pipe.base.Struct` 2089 This Struct must contain a ``templateCoadd`` that serves as the 2090 model of the static sky. 2094 result : `lsst.pipe.base.Struct` 2095 Result struct with components: 2097 - ``coaddExposure``: coadded exposure (``lsst.afw.image.Exposure``). 2098 - ``nImage``: exposure count image (``lsst.afw.image.Image``), if requested. 2100 templateCoadd = supplementaryData.templateCoadd
2101 spanSetMaskList = self.
findArtifacts(templateCoadd, tempExpRefList, imageScalerList)
2102 badMaskPlanes = self.config.badMaskPlanes[:]
2103 badMaskPlanes.append(
"CLIPPED")
2104 badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes)
2106 result = AssembleCoaddTask.run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
2107 spanSetMaskList, mask=badPixelMask)
2111 self.
applyAltEdgeMask(result.coaddExposure.maskedImage.mask, spanSetMaskList)
2115 """Propagate alt EDGE mask to SENSOR_EDGE AND INEXACT_PSF planes. 2119 mask : `lsst.afw.image.Mask` 2121 altMaskList : `list` 2122 List of Dicts containing ``spanSet`` lists. 2123 Each element contains the new mask plane name (e.g. "CLIPPED 2124 and/or "NO_DATA") as the key, and list of ``SpanSets`` to apply to 2127 maskValue = mask.getPlaneBitMask([
"SENSOR_EDGE",
"INEXACT_PSF"])
2128 for visitMask
in altMaskList:
2129 if "EDGE" in visitMask:
2130 for spanSet
in visitMask[
'EDGE']:
2131 spanSet.clippedTo(mask.getBBox()).setMask(mask, maskValue)
2136 Loop through warps twice. The first loop builds a map with the count 2137 of how many epochs each pixel deviates from the templateCoadd by more 2138 than ``config.chiThreshold`` sigma. The second loop takes each 2139 difference image and filters the artifacts detected in each using 2140 count map to filter out variable sources and sources that are 2141 difficult to subtract cleanly. 2145 templateCoadd : `lsst.afw.image.Exposure` 2146 Exposure to serve as model of static sky. 2147 tempExpRefList : `list` 2148 List of data references to warps. 2149 imageScalerList : `list` 2150 List of image scalers. 2155 List of dicts containing information about CLIPPED 2156 (i.e., artifacts), NO_DATA, and EDGE pixels. 2159 self.log.debug(
"Generating Count Image, and mask lists.")
2160 coaddBBox = templateCoadd.getBBox()
2161 slateIm = afwImage.ImageU(coaddBBox)
2162 epochCountImage = afwImage.ImageU(coaddBBox)
2163 nImage = afwImage.ImageU(coaddBBox)
2164 spanSetArtifactList = []
2165 spanSetNoDataMaskList = []
2166 spanSetEdgeList = []
2170 templateCoadd.mask.clearAllMaskPlanes()
2172 if self.config.doPreserveContainedBySource:
2173 templateFootprints = self.detectTemplate.detectFootprints(templateCoadd)
2175 templateFootprints =
None 2177 for warpRef, imageScaler
in zip(tempExpRefList, imageScalerList):
2179 if warpDiffExp
is not None:
2181 nImage.array += (numpy.isfinite(warpDiffExp.image.array) *
2182 ((warpDiffExp.mask.array & badPixelMask) == 0)).astype(numpy.uint16)
2183 fpSet = self.detect.detectFootprints(warpDiffExp, doSmooth=
False, clearMask=
True)
2184 fpSet.positive.merge(fpSet.negative)
2185 footprints = fpSet.positive
2187 spanSetList = [footprint.spans
for footprint
in footprints.getFootprints()]
2190 if self.config.doPrefilterArtifacts:
2192 for spans
in spanSetList:
2193 spans.setImage(slateIm, 1, doClip=
True)
2194 epochCountImage += slateIm
2200 nans = numpy.where(numpy.isnan(warpDiffExp.maskedImage.image.array), 1, 0)
2201 nansMask = afwImage.makeMaskFromArray(nans.astype(afwImage.MaskPixel))
2202 nansMask.setXY0(warpDiffExp.getXY0())
2203 edgeMask = warpDiffExp.mask
2204 spanSetEdgeMask = afwGeom.SpanSet.fromMask(edgeMask,
2205 edgeMask.getPlaneBitMask(
"EDGE")).split()
2209 nansMask = afwImage.MaskX(coaddBBox, 1)
2211 spanSetEdgeMask = []
2213 spanSetNoDataMask = afwGeom.SpanSet.fromMask(nansMask).split()
2215 spanSetNoDataMaskList.append(spanSetNoDataMask)
2216 spanSetArtifactList.append(spanSetList)
2217 spanSetEdgeList.append(spanSetEdgeMask)
2221 epochCountImage.writeFits(path)
2223 for i, spanSetList
in enumerate(spanSetArtifactList):
2225 filteredSpanSetList = self.
filterArtifacts(spanSetList, epochCountImage, nImage,
2227 spanSetArtifactList[i] = filteredSpanSetList
2230 for artifacts, noData, edge
in zip(spanSetArtifactList, spanSetNoDataMaskList, spanSetEdgeList):
2231 altMasks.append({
'CLIPPED': artifacts,
2237 """Remove artifact candidates covered by bad mask plane. 2239 Any future editing of the candidate list that does not depend on 2240 temporal information should go in this method. 2244 spanSetList : `list` 2245 List of SpanSets representing artifact candidates. 2246 exp : `lsst.afw.image.Exposure` 2247 Exposure containing mask planes used to prefilter. 2251 returnSpanSetList : `list` 2252 List of SpanSets with artifacts. 2254 badPixelMask = exp.mask.getPlaneBitMask(self.config.prefilterArtifactsMaskPlanes)
2255 goodArr = (exp.mask.array & badPixelMask) == 0
2256 returnSpanSetList = []
2257 bbox = exp.getBBox()
2258 x0, y0 = exp.getXY0()
2259 for i, span
in enumerate(spanSetList):
2260 y, x = span.clippedTo(bbox).indices()
2261 yIndexLocal = numpy.array(y) - y0
2262 xIndexLocal = numpy.array(x) - x0
2263 goodRatio = numpy.count_nonzero(goodArr[yIndexLocal, xIndexLocal])/span.getArea()
2264 if goodRatio > self.config.prefilterArtifactsRatio:
2265 returnSpanSetList.append(span)
2266 return returnSpanSetList
2268 def filterArtifacts(self, spanSetList, epochCountImage, nImage, footprintsToExclude=None):
2269 """Filter artifact candidates. 2273 spanSetList : `list` 2274 List of SpanSets representing artifact candidates. 2275 epochCountImage : `lsst.afw.image.Image` 2276 Image of accumulated number of warpDiff detections. 2277 nImage : `lsst.afw.image.Image` 2278 Image of the accumulated number of total epochs contributing. 2282 maskSpanSetList : `list` 2283 List of SpanSets with artifacts. 2286 maskSpanSetList = []
2287 x0, y0 = epochCountImage.getXY0()
2288 for i, span
in enumerate(spanSetList):
2289 y, x = span.indices()
2290 yIdxLocal = [y1 - y0
for y1
in y]
2291 xIdxLocal = [x1 - x0
for x1
in x]
2292 outlierN = epochCountImage.array[yIdxLocal, xIdxLocal]
2293 totalN = nImage.array[yIdxLocal, xIdxLocal]
2296 effMaxNumEpochsHighN = (self.config.maxNumEpochs +
2297 self.config.maxFractionEpochsHigh*numpy.mean(totalN))
2298 effMaxNumEpochsLowN = self.config.maxFractionEpochsLow * numpy.mean(totalN)
2299 effectiveMaxNumEpochs = int(min(effMaxNumEpochsLowN, effMaxNumEpochsHighN))
2300 nPixelsBelowThreshold = numpy.count_nonzero((outlierN > 0) &
2301 (outlierN <= effectiveMaxNumEpochs))
2302 percentBelowThreshold = nPixelsBelowThreshold / len(outlierN)
2303 if percentBelowThreshold > self.config.spatialThreshold:
2304 maskSpanSetList.append(span)
2306 if self.config.doPreserveContainedBySource
and footprintsToExclude
is not None:
2308 filteredMaskSpanSetList = []
2309 for span
in maskSpanSetList:
2311 for footprint
in footprintsToExclude.positive.getFootprints():
2312 if footprint.spans.contains(span):
2316 filteredMaskSpanSetList.append(span)
2317 maskSpanSetList = filteredMaskSpanSetList
2319 return maskSpanSetList
2321 def _readAndComputeWarpDiff(self, warpRef, imageScaler, templateCoadd):
2322 """Fetch a warp from the butler and return a warpDiff. 2326 warpRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef` 2327 Butler dataRef for the warp. 2328 imageScaler : `lsst.pipe.tasks.scaleZeroPoint.ImageScaler` 2329 An image scaler object. 2330 templateCoadd : `lsst.afw.image.Exposure` 2331 Exposure to be substracted from the scaled warp. 2335 warp : `lsst.afw.image.Exposure` 2336 Exposure of the image difference between the warp and template. 2341 if not warpRef.datasetExists(warpName):
2342 self.log.warn(
"Could not find %s %s; skipping it", warpName, warpRef.dataId)
2344 warp = warpRef.get(warpName, immediate=
True)
2346 imageScaler.scaleMaskedImage(warp.getMaskedImage())
2347 mi = warp.getMaskedImage()
2348 if self.config.doScaleWarpVariance:
2350 self.scaleWarpVariance.
run(mi)
2351 except Exception
as exc:
2352 self.log.warn(
"Unable to rescale variance of warp (%s); leaving it as-is" % (exc,))
2353 mi -= templateCoadd.getMaskedImage()
2356 def _dataRef2DebugPath(self, prefix, warpRef, coaddLevel=False):
2357 """Return a path to which to write debugging output. 2359 Creates a hyphen-delimited string of dataId values for simple filenames. 2364 Prefix for filename. 2365 warpRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef` 2366 Butler dataRef to make the path from. 2367 coaddLevel : `bool`, optional. 2368 If True, include only coadd-level keys (e.g., 'tract', 'patch', 2369 'filter', but no 'visit'). 2374 Path for debugging output. 2379 keys = warpRef.dataId.keys()
2380 keyList = sorted(keys, reverse=
True)
2382 filename =
"%s-%s.fits" % (prefix,
'-'.join([str(warpRef.dataId[k])
for k
in keyList]))
2383 return os.path.join(directory, filename)
def setBrightObjectMasks(self, exposure, dataId, brightObjectMasks)
def shrinkValidPolygons(self, coaddInputs)
def getCoaddDatasetName(self, warpType="direct")
def _dataRef2DebugPath(self, prefix, warpRef, coaddLevel=False)
def getGroupDataRef(butler, datasetType, groupTuple, keys)
Base class for coaddition.
def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList)
def assembleMetadata(self, coaddExposure, tempExpRefList, weightList)
def getTempExpRefList(self, patchRef, calExpRefList)
def removeMaskPlanes(self, maskedImage)
def makeSkyInfo(skyMap, tractId, patchId)
def _readAndComputeWarpDiff(self, warpRef, imageScaler, templateCoadd)
def prepareInputs(self, refList)
def applyAltMaskPlanes(self, mask, altMaskSpans)
def getSkyInfo(self, patchRef)
Use getSkyinfo to return the skyMap, tract and patch information, wcs and the outer bbox of the patch...
def getTempExpDatasetName(self, warpType="direct")
def __init__(self, args, kwargs)
def makeSupplementaryDataGen3(self, inputData, inputDataIds, outputDataIds, butler)
def prepareStats(self, mask=None)
def makeDataRefList(self, namespace)
def getBadPixelMask(self)
Convenience method to provide the bitmask from the mask plane names.
def assembleSubregion(self, coaddExposure, bbox, tempExpRefList, imageScalerList, weightList, altMaskList, statsFlags, statsCtrl, nImage=None)
def makeSupplementaryData(self, dataRef, selectDataList=None, warpRefList=None)
def detectClip(self, exp, tempExpRefList)
def setInexactPsf(self, mask)
def adaptArgsAndRun(self, inputData, inputDataIds, outputDataIds, butler)
def __init__(self, args, kwargs)
def makeSupplementaryData(self, dataRef, selectDataList=None, warpRefList=None)
def filterArtifacts(self, spanSetList, epochCountImage, nImage, footprintsToExclude=None)
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, args, kwargs)
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, supplementaryData, args, kwargs)
def buildDifferenceImage(self, skyInfo, tempExpRefList, imageScalerList, weightList)
def _noTemplateMessage(self, warpType)
def selectExposures(self, patchRef, skyInfo=None, selectDataList=[])
Select exposures to coadd.
def setRejectedMaskMapping(statsCtrl)
def getOutputDatasetTypes(cls, config)
def applyAltEdgeMask(self, mask, altMaskList)
def getInputDatasetTypes(cls, config)
def makeSupplementaryDataGen3(self, inputData, inputDataIds, outputDataIds, butler)
def readBrightObjectMasks(self, dataRef)
def runDataRef(self, dataRef, selectDataList=None, warpRefList=None)
def processResults(self, coaddExposure, dataRef)
def __init__(self, args, kwargs)
def prefilterArtifacts(self, spanSetList, exp)
def _subBBoxIter(bbox, subregionSize)
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, altMaskList=None, mask=None, supplementaryData=None)
def countMaskFromFootprint(mask, footprint, bitmask, ignoreMask)
def groupPatchExposures(patchDataRef, calexpDataRefList, coaddDatasetType="deepCoadd", tempExpDatasetType="deepCoadd_directWarp")
def detectClipBig(self, clipList, clipFootprints, clipIndices, detectionFootprints, maskClipValue, maskDetValue, coaddBBox)