26 import lsst.pex.config
as pexConfig
40 from .coaddBase
import CoaddBaseTask, SelectDataIdContainer, makeSkyInfo, makeCoaddSuffix
41 from .interpImage
import InterpImageTask
42 from .scaleZeroPoint
import ScaleZeroPointTask
43 from .coaddHelpers
import groupPatchExposures, getGroupDataRef
44 from .scaleVariance
import ScaleVarianceTask
46 from lsst.daf.butler
import DeferredDatasetHandle
48 __all__ = [
"AssembleCoaddTask",
"AssembleCoaddConnections",
"AssembleCoaddConfig",
49 "SafeClipAssembleCoaddTask",
"SafeClipAssembleCoaddConfig",
50 "CompareWarpAssembleCoaddTask",
"CompareWarpAssembleCoaddConfig"]
54 dimensions=(
"tract",
"patch",
"abstract_filter",
"skymap"),
55 defaultTemplates={
"inputCoaddName":
"deep",
56 "outputCoaddName":
"deep",
60 inputWarps = pipeBase.connectionTypes.Input(
61 doc=(
"Input list of warps to be assemebled i.e. stacked." 62 "WarpType (e.g. direct, psfMatched) is controlled by the warpType config parameter"),
63 name=
"{inputCoaddName}Coadd_{warpType}Warp",
64 storageClass=
"ExposureF",
65 dimensions=(
"tract",
"patch",
"skymap",
"visit",
"instrument"),
69 skyMap = pipeBase.connectionTypes.Input(
70 doc=
"Input definition of geometry/bbox and projection/wcs for coadded exposures",
71 name=
"{inputCoaddName}Coadd_skyMap",
72 storageClass=
"SkyMap",
73 dimensions=(
"skymap", ),
75 brightObjectMask = pipeBase.connectionTypes.PrerequisiteInput(
76 doc=(
"Input Bright Object Mask mask produced with external catalogs to be applied to the mask plane" 78 name=
"brightObjectMask",
79 storageClass=
"ObjectMaskCatalog",
80 dimensions=(
"tract",
"patch",
"skymap",
"abstract_filter"),
82 coaddExposure = pipeBase.connectionTypes.Output(
83 doc=
"Output coadded exposure, produced by stacking input warps",
84 name=
"{fakesType}{outputCoaddName}Coadd{warpTypeSuffix}",
85 storageClass=
"ExposureF",
86 dimensions=(
"tract",
"patch",
"skymap",
"abstract_filter"),
88 nImage = pipeBase.connectionTypes.Output(
89 doc=
"Output image of number of input images per pixel",
90 name=
"{outputCoaddName}Coadd_nImage",
91 storageClass=
"ImageU",
92 dimensions=(
"tract",
"patch",
"skymap",
"abstract_filter"),
95 def __init__(self, *, config=None):
97 config.connections.warpType = config.warpType
101 config.connections.fakesType =
"_fakes" 103 super().__init__(config=config)
105 if not config.doMaskBrightObjects:
106 self.prerequisiteInputs.remove(
"brightObjectMask")
108 if not config.doNImage:
109 self.outputs.remove(
"nImage")
112 class AssembleCoaddConfig(CoaddBaseTask.ConfigClass, pipeBase.PipelineTaskConfig,
113 pipelineConnections=AssembleCoaddConnections):
114 """Configuration parameters for the `AssembleCoaddTask`. 118 The `doMaskBrightObjects` and `brightObjectMaskName` configuration options 119 only set the bitplane config.brightObjectMaskName. To make this useful you 120 *must* also configure the flags.pixel algorithm, for example by adding 124 config.measurement.plugins["base_PixelFlags"].masksFpCenter.append("BRIGHT_OBJECT") 125 config.measurement.plugins["base_PixelFlags"].masksFpAnywhere.append("BRIGHT_OBJECT") 127 to your measureCoaddSources.py and forcedPhotCoadd.py config overrides. 129 warpType = pexConfig.Field(
130 doc=
"Warp name: one of 'direct' or 'psfMatched'",
134 subregionSize = pexConfig.ListField(
136 doc=
"Width, height of stack subregion size; " 137 "make small enough that a full stack of images will fit into memory at once.",
139 default=(2000, 2000),
141 statistic = pexConfig.Field(
143 doc=
"Main stacking statistic for aggregating over the epochs.",
146 doSigmaClip = pexConfig.Field(
148 doc=
"Perform sigma clipped outlier rejection with MEANCLIP statistic? (DEPRECATED)",
151 sigmaClip = pexConfig.Field(
153 doc=
"Sigma for outlier rejection; ignored if non-clipping statistic selected.",
156 clipIter = pexConfig.Field(
158 doc=
"Number of iterations of outlier rejection; ignored if non-clipping statistic selected.",
161 calcErrorFromInputVariance = pexConfig.Field(
163 doc=
"Calculate coadd variance from input variance by stacking statistic." 164 "Passed to StatisticsControl.setCalcErrorFromInputVariance()",
167 scaleZeroPoint = pexConfig.ConfigurableField(
168 target=ScaleZeroPointTask,
169 doc=
"Task to adjust the photometric zero point of the coadd temp exposures",
171 doInterp = pexConfig.Field(
172 doc=
"Interpolate over NaN pixels? Also extrapolate, if necessary, but the results are ugly.",
176 interpImage = pexConfig.ConfigurableField(
177 target=InterpImageTask,
178 doc=
"Task to interpolate (and extrapolate) over NaN pixels",
180 doWrite = pexConfig.Field(
181 doc=
"Persist coadd?",
185 doNImage = pexConfig.Field(
186 doc=
"Create image of number of contributing exposures for each pixel",
190 doUsePsfMatchedPolygons = pexConfig.Field(
191 doc=
"Use ValidPolygons from shrunk Psf-Matched Calexps? Should be set to True by CompareWarp only.",
195 maskPropagationThresholds = pexConfig.DictField(
198 doc=(
"Threshold (in fractional weight) of rejection at which we propagate a mask plane to " 199 "the coadd; that is, we set the mask bit on the coadd if the fraction the rejected frames " 200 "would have contributed exceeds this value."),
201 default={
"SAT": 0.1},
203 removeMaskPlanes = pexConfig.ListField(dtype=str, default=[
"NOT_DEBLENDED"],
204 doc=
"Mask planes to remove before coadding")
205 doMaskBrightObjects = pexConfig.Field(dtype=bool, default=
False,
206 doc=
"Set mask and flag bits for bright objects?")
207 brightObjectMaskName = pexConfig.Field(dtype=str, default=
"BRIGHT_OBJECT",
208 doc=
"Name of mask bit used for bright objects")
209 coaddPsf = pexConfig.ConfigField(
210 doc=
"Configuration for CoaddPsf",
211 dtype=measAlg.CoaddPsfConfig,
213 doAttachTransmissionCurve = pexConfig.Field(
214 dtype=bool, default=
False, optional=
False,
215 doc=(
"Attach a piecewise TransmissionCurve for the coadd? " 216 "(requires all input Exposures to have TransmissionCurves).")
218 hasFakes = pexConfig.Field(
221 doc=
"Should be set to True if fake sources have been inserted into the input data." 224 def setDefaults(self):
225 super().setDefaults()
226 self.badMaskPlanes = [
"NO_DATA",
"BAD",
"SAT",
"EDGE"]
233 log.warn(
"Config doPsfMatch deprecated. Setting warpType='psfMatched'")
234 self.warpType =
'psfMatched' 235 if self.doSigmaClip
and self.statistic !=
"MEANCLIP":
236 log.warn(
'doSigmaClip deprecated. To replicate behavior, setting statistic to "MEANCLIP"')
237 self.statistic =
"MEANCLIP" 238 if self.doInterp
and self.statistic
not in [
'MEAN',
'MEDIAN',
'MEANCLIP',
'VARIANCE',
'VARIANCECLIP']:
239 raise ValueError(
"Must set doInterp=False for statistic=%s, which does not " 240 "compute and set a non-zero coadd variance estimate." % (self.statistic))
242 unstackableStats = [
'NOTHING',
'ERROR',
'ORMASK']
243 if not hasattr(afwMath.Property, self.statistic)
or self.statistic
in unstackableStats:
244 stackableStats = [str(k)
for k
in afwMath.Property.__members__.keys()
245 if str(k)
not in unstackableStats]
246 raise ValueError(
"statistic %s is not allowed. Please choose one of %s." 247 % (self.statistic, stackableStats))
250 class AssembleCoaddTask(
CoaddBaseTask, pipeBase.PipelineTask):
251 """Assemble a coadded image from a set of warps (coadded temporary exposures). 253 We want to assemble a coadded image from a set of Warps (also called 254 coadded temporary exposures or ``coaddTempExps``). 255 Each input Warp covers a patch on the sky and corresponds to a single 256 run/visit/exposure of the covered patch. We provide the task with a list 257 of Warps (``selectDataList``) from which it selects Warps that cover the 258 specified patch (pointed at by ``dataRef``). 259 Each Warp that goes into a coadd will typically have an independent 260 photometric zero-point. Therefore, we must scale each Warp to set it to 261 a common photometric zeropoint. WarpType may be one of 'direct' or 262 'psfMatched', and the boolean configs `config.makeDirect` and 263 `config.makePsfMatched` set which of the warp types will be coadded. 264 The coadd is computed as a mean with optional outlier rejection. 265 Criteria for outlier rejection are set in `AssembleCoaddConfig`. 266 Finally, Warps can have bad 'NaN' pixels which received no input from the 267 source calExps. We interpolate over these bad (NaN) pixels. 269 `AssembleCoaddTask` uses several sub-tasks. These are 271 - `ScaleZeroPointTask` 272 - create and use an ``imageScaler`` object to scale the photometric zeropoint for each Warp 274 - interpolate across bad pixels (NaN) in the final coadd 276 You can retarget these subtasks if you wish. 280 The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a 281 flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``; see 282 `baseDebug` for more about ``debug.py`` files. `AssembleCoaddTask` has 283 no debug variables of its own. Some of the subtasks may support debug 284 variables. See the documentation for the subtasks for further information. 288 `AssembleCoaddTask` assembles a set of warped images into a coadded image. 289 The `AssembleCoaddTask` can be invoked by running ``assembleCoadd.py`` 290 with the flag '--legacyCoadd'. Usage of assembleCoadd.py expects two 291 inputs: a data reference to the tract patch and filter to be coadded, and 292 a list of Warps to attempt to coadd. These are specified using ``--id`` and 293 ``--selectId``, respectively: 297 --id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]] 298 --selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]] 300 Only the Warps that cover the specified tract and patch will be coadded. 301 A list of the available optional arguments can be obtained by calling 302 ``assembleCoadd.py`` with the ``--help`` command line argument: 306 assembleCoadd.py --help 308 To demonstrate usage of the `AssembleCoaddTask` in the larger context of 309 multi-band processing, we will generate the HSC-I & -R band coadds from 310 HSC engineering test data provided in the ``ci_hsc`` package. To begin, 311 assuming that the lsst stack has been already set up, we must set up the 312 obs_subaru and ``ci_hsc`` packages. This defines the environment variable 313 ``$CI_HSC_DIR`` and points at the location of the package. The raw HSC 314 data live in the ``$CI_HSC_DIR/raw directory``. To begin assembling the 315 coadds, we must first 318 - process the individual ccds in $CI_HSC_RAW to produce calibrated exposures 320 - create a skymap that covers the area of the sky present in the raw exposures 322 - warp the individual calibrated exposures to the tangent plane of the coadd 324 We can perform all of these steps by running 328 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988 330 This will produce warped exposures for each visit. To coadd the warped 331 data, we call assembleCoadd.py as follows: 335 assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \ 336 --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \ 337 --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \ 338 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \ 339 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \ 340 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \ 341 --selectId visit=903988 ccd=24 343 that will process the HSC-I band data. The results are written in 344 ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``. 346 You may also choose to run: 350 scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346 351 assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R \ 352 --selectId visit=903334 ccd=16 --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 \ 353 --selectId visit=903334 ccd=100 --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 \ 354 --selectId visit=903338 ccd=18 --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 \ 355 --selectId visit=903342 ccd=10 --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 \ 356 --selectId visit=903344 ccd=5 --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 \ 357 --selectId visit=903346 ccd=6 --selectId visit=903346 ccd=12 359 to generate the coadd for the HSC-R band if you are interested in 360 following multiBand Coadd processing as discussed in `pipeTasks_multiBand` 361 (but note that normally, one would use the `SafeClipAssembleCoaddTask` 362 rather than `AssembleCoaddTask` to make the coadd. 364 ConfigClass = AssembleCoaddConfig
365 _DefaultName =
"assembleCoadd" 367 def __init__(self, *args, **kwargs):
370 argNames = [
"config",
"name",
"parentTask",
"log"]
371 kwargs.update({k: v
for k, v
in zip(argNames, args)})
372 warnings.warn(
"AssembleCoadd received positional args, and casting them as kwargs: %s. " 373 "PipelineTask will not take positional args" % argNames, FutureWarning)
375 super().__init__(**kwargs)
376 self.makeSubtask(
"interpImage")
377 self.makeSubtask(
"scaleZeroPoint")
379 if self.config.doMaskBrightObjects:
380 mask = afwImage.Mask()
382 self.brightObjectBitmask = 1 << mask.addMaskPlane(self.config.brightObjectMaskName)
383 except pexExceptions.LsstCppException:
384 raise RuntimeError(
"Unable to define mask plane for bright objects; planes used are %s" %
385 mask.getMaskPlaneDict().keys())
388 self.warpType = self.config.warpType
390 @utils.inheritDoc(pipeBase.PipelineTask)
391 def runQuantum(self, butlerQC, inputRefs, outputRefs):
396 Assemble a coadd from a set of Warps. 398 PipelineTask (Gen3) entry point to Coadd a set of Warps. 399 Analogous to `runDataRef`, it prepares all the data products to be 400 passed to `run`, and processes the results before returning a struct 401 of results to be written out. AssembleCoadd cannot fit all Warps in memory. 402 Therefore, its inputs are accessed subregion by subregion 403 by the Gen3 `DeferredDatasetHandle` that is analagous to the Gen2 404 `lsst.daf.persistence.ButlerDataRef`. Any updates to this method should 405 correspond to an update in `runDataRef` while both entry points 408 inputData = butlerQC.get(inputRefs)
412 skyMap = inputData[
"skyMap"]
413 outputDataId = butlerQC.quantum.dataId
416 tractId=outputDataId[
'tract'],
417 patchId=outputDataId[
'patch'])
421 warpRefList = inputData[
'inputWarps']
423 inputs = self.prepareInputs(warpRefList)
424 self.log.info(
"Found %d %s", len(inputs.tempExpRefList),
425 self.getTempExpDatasetName(self.warpType))
426 if len(inputs.tempExpRefList) == 0:
427 self.log.warn(
"No coadd temporary exposures found")
430 supplementaryData = self.makeSupplementaryDataGen3(butlerQC, inputRefs, outputRefs)
431 retStruct = self.run(inputData[
'skyInfo'], inputs.tempExpRefList, inputs.imageScalerList,
432 inputs.weightList, supplementaryData=supplementaryData)
434 self.processResults(retStruct.coaddExposure, inputData[
'brightObjectMask'], outputDataId)
436 if self.config.doWrite:
437 butlerQC.put(retStruct, outputRefs)
441 def runDataRef(self, dataRef, selectDataList=None, warpRefList=None):
442 """Assemble a coadd from a set of Warps. 444 Pipebase.CmdlineTask entry point to Coadd a set of Warps. 445 Compute weights to be applied to each Warp and 446 find scalings to match the photometric zeropoint to a reference Warp. 447 Assemble the Warps using `run`. Interpolate over NaNs and 448 optionally write the coadd to disk. Return the coadded exposure. 452 dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef` 453 Data reference defining the patch for coaddition and the 454 reference Warp (if ``config.autoReference=False``). 455 Used to access the following data products: 456 - ``self.config.coaddName + "Coadd_skyMap"`` 457 - ``self.config.coaddName + "Coadd_ + <warpType> + "Warp"`` (optionally) 458 - ``self.config.coaddName + "Coadd"`` 459 selectDataList : `list` 460 List of data references to Calexps. Data to be coadded will be 461 selected from this list based on overlap with the patch defined 462 by dataRef, grouped by visit, and converted to a list of data 465 List of data references to Warps to be coadded. 466 Note: `warpRefList` is just the new name for `tempExpRefList`. 470 retStruct : `lsst.pipe.base.Struct` 471 Result struct with components: 473 - ``coaddExposure``: coadded exposure (``Exposure``). 474 - ``nImage``: exposure count image (``Image``). 476 if selectDataList
and warpRefList:
477 raise RuntimeError(
"runDataRef received both a selectDataList and warpRefList, " 478 "and which to use is ambiguous. Please pass only one.")
480 skyInfo = self.getSkyInfo(dataRef)
481 if warpRefList
is None:
482 calExpRefList = self.selectExposures(dataRef, skyInfo, selectDataList=selectDataList)
483 if len(calExpRefList) == 0:
484 self.log.warn(
"No exposures to coadd")
486 self.log.info(
"Coadding %d exposures", len(calExpRefList))
488 warpRefList = self.getTempExpRefList(dataRef, calExpRefList)
490 inputData = self.prepareInputs(warpRefList)
491 self.log.info(
"Found %d %s", len(inputData.tempExpRefList),
492 self.getTempExpDatasetName(self.warpType))
493 if len(inputData.tempExpRefList) == 0:
494 self.log.warn(
"No coadd temporary exposures found")
497 supplementaryData = self.makeSupplementaryData(dataRef, warpRefList=inputData.tempExpRefList)
499 retStruct = self.run(skyInfo, inputData.tempExpRefList, inputData.imageScalerList,
500 inputData.weightList, supplementaryData=supplementaryData)
502 brightObjects = self.readBrightObjectMasks(dataRef)
if self.config.doMaskBrightObjects
else None 503 self.processResults(retStruct.coaddExposure, brightObjectMasks=brightObjects, dataId=dataRef.dataId)
505 if self.config.doWrite:
506 if self.getCoaddDatasetName(self.warpType) ==
"deepCoadd" and self.config.hasFakes:
507 coaddDatasetName =
"fakes_" + self.getCoaddDatasetName(self.warpType)
509 coaddDatasetName = self.getCoaddDatasetName(self.warpType)
510 self.log.info(
"Persisting %s" % coaddDatasetName)
511 dataRef.put(retStruct.coaddExposure, coaddDatasetName)
512 if self.config.doNImage
and retStruct.nImage
is not None:
513 dataRef.put(retStruct.nImage, self.getCoaddDatasetName(self.warpType) +
'_nImage')
518 """Interpolate over missing data and mask bright stars. 522 coaddExposure : `lsst.afw.image.Exposure` 523 The coadded exposure to process. 524 dataRef : `lsst.daf.persistence.ButlerDataRef` 525 Butler data reference for supplementary data. 527 if self.config.doInterp:
528 self.interpImage.
run(coaddExposure.getMaskedImage(), planeName=
"NO_DATA")
530 varArray = coaddExposure.variance.array
531 with numpy.errstate(invalid=
"ignore"):
532 varArray[:] = numpy.where(varArray > 0, varArray, numpy.inf)
534 if self.config.doMaskBrightObjects:
535 self.setBrightObjectMasks(coaddExposure, brightObjectMasks, dataId)
538 """Make additional inputs to run() specific to subclasses (Gen2) 540 Duplicates interface of `runDataRef` method 541 Available to be implemented by subclasses only if they need the 542 coadd dataRef for performing preliminary processing before 543 assembling the coadd. 547 dataRef : `lsst.daf.persistence.ButlerDataRef` 548 Butler data reference for supplementary data. 549 selectDataList : `list` (optional) 550 Optional List of data references to Calexps. 551 warpRefList : `list` (optional) 552 Optional List of data references to Warps. 554 return pipeBase.Struct()
557 """Make additional inputs to run() specific to subclasses (Gen3) 559 Duplicates interface of `runQuantum` method. 560 Available to be implemented by subclasses only if they need the 561 coadd dataRef for performing preliminary processing before 562 assembling the coadd. 566 butlerQC : `lsst.pipe.base.ButlerQuantumContext` 567 Gen3 Butler object for fetching additional data products before 568 running the Task specialized for quantum being processed 569 inputRefs : `lsst.pipe.base.InputQuantizedConnection` 570 Attributes are the names of the connections describing input dataset types. 571 Values are DatasetRefs that task consumes for corresponding dataset type. 572 DataIds are guaranteed to match data objects in ``inputData``. 573 outputRefs : `lsst.pipe.base.OutputQuantizedConnection` 574 Attributes are the names of the connections describing output dataset types. 575 Values are DatasetRefs that task is to produce 576 for corresponding dataset type. 578 return pipeBase.Struct()
581 """Generate list data references corresponding to warped exposures 582 that lie within the patch to be coadded. 587 Data reference for patch. 588 calExpRefList : `list` 589 List of data references for input calexps. 593 tempExpRefList : `list` 594 List of Warp/CoaddTempExp data references. 596 butler = patchRef.getButler()
597 groupData =
groupPatchExposures(patchRef, calExpRefList, self.getCoaddDatasetName(self.warpType),
598 self.getTempExpDatasetName(self.warpType))
599 tempExpRefList = [
getGroupDataRef(butler, self.getTempExpDatasetName(self.warpType),
600 g, groupData.keys)
for 601 g
in groupData.groups.keys()]
602 return tempExpRefList
605 """Prepare the input warps for coaddition by measuring the weight for 606 each warp and the scaling for the photometric zero point. 608 Each Warp has its own photometric zeropoint and background variance. 609 Before coadding these Warps together, compute a scale factor to 610 normalize the photometric zeropoint and compute the weight for each Warp. 615 List of data references to tempExp 619 result : `lsst.pipe.base.Struct` 620 Result struct with components: 622 - ``tempExprefList``: `list` of data references to tempExp. 623 - ``weightList``: `list` of weightings. 624 - ``imageScalerList``: `list` of image scalers. 626 statsCtrl = afwMath.StatisticsControl()
627 statsCtrl.setNumSigmaClip(self.config.sigmaClip)
628 statsCtrl.setNumIter(self.config.clipIter)
629 statsCtrl.setAndMask(self.getBadPixelMask())
630 statsCtrl.setNanSafe(
True)
637 tempExpName = self.getTempExpDatasetName(self.warpType)
638 for tempExpRef
in refList:
641 if not isinstance(tempExpRef, DeferredDatasetHandle):
642 if not tempExpRef.datasetExists(tempExpName):
643 self.log.warn(
"Could not find %s %s; skipping it", tempExpName, tempExpRef.dataId)
646 tempExp = tempExpRef.get(datasetType=tempExpName, immediate=
True)
648 if numpy.isnan(tempExp.image.array).all():
650 maskedImage = tempExp.getMaskedImage()
651 imageScaler = self.scaleZeroPoint.computeImageScaler(
656 imageScaler.scaleMaskedImage(maskedImage)
657 except Exception
as e:
658 self.log.warn(
"Scaling failed for %s (skipping it): %s", tempExpRef.dataId, e)
660 statObj = afwMath.makeStatistics(maskedImage.getVariance(), maskedImage.getMask(),
661 afwMath.MEANCLIP, statsCtrl)
662 meanVar, meanVarErr = statObj.getResult(afwMath.MEANCLIP)
663 weight = 1.0 / float(meanVar)
664 if not numpy.isfinite(weight):
665 self.log.warn(
"Non-finite weight for %s: skipping", tempExpRef.dataId)
667 self.log.info(
"Weight of %s %s = %0.3f", tempExpName, tempExpRef.dataId, weight)
672 tempExpRefList.append(tempExpRef)
673 weightList.append(weight)
674 imageScalerList.append(imageScaler)
676 return pipeBase.Struct(tempExpRefList=tempExpRefList, weightList=weightList,
677 imageScalerList=imageScalerList)
680 """Prepare the statistics for coadding images. 684 mask : `int`, optional 685 Bit mask value to exclude from coaddition. 689 stats : `lsst.pipe.base.Struct` 690 Statistics structure with the following fields: 692 - ``statsCtrl``: Statistics control object for coadd 693 (`lsst.afw.math.StatisticsControl`) 694 - ``statsFlags``: Statistic for coadd (`lsst.afw.math.Property`) 697 mask = self.getBadPixelMask()
698 statsCtrl = afwMath.StatisticsControl()
699 statsCtrl.setNumSigmaClip(self.config.sigmaClip)
700 statsCtrl.setNumIter(self.config.clipIter)
701 statsCtrl.setAndMask(mask)
702 statsCtrl.setNanSafe(
True)
703 statsCtrl.setWeighted(
True)
704 statsCtrl.setCalcErrorFromInputVariance(self.config.calcErrorFromInputVariance)
705 for plane, threshold
in self.config.maskPropagationThresholds.items():
706 bit = afwImage.Mask.getMaskPlane(plane)
707 statsCtrl.setMaskPropagationThreshold(bit, threshold)
708 statsFlags = afwMath.stringToStatisticsProperty(self.config.statistic)
709 return pipeBase.Struct(ctrl=statsCtrl, flags=statsFlags)
711 def run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
712 altMaskList=None, mask=None, supplementaryData=None):
713 """Assemble a coadd from input warps 715 Assemble the coadd using the provided list of coaddTempExps. Since 716 the full coadd covers a patch (a large area), the assembly is 717 performed over small areas on the image at a time in order to 718 conserve memory usage. Iterate over subregions within the outer 719 bbox of the patch using `assembleSubregion` to stack the corresponding 720 subregions from the coaddTempExps with the statistic specified. 721 Set the edge bits the coadd mask based on the weight map. 725 skyInfo : `lsst.pipe.base.Struct` 726 Struct with geometric information about the patch. 727 tempExpRefList : `list` 728 List of data references to Warps (previously called CoaddTempExps). 729 imageScalerList : `list` 730 List of image scalers. 733 altMaskList : `list`, optional 734 List of alternate masks to use rather than those stored with 736 mask : `int`, optional 737 Bit mask value to exclude from coaddition. 738 supplementaryData : lsst.pipe.base.Struct, optional 739 Struct with additional data products needed to assemble coadd. 740 Only used by subclasses that implement `makeSupplementaryData` 745 result : `lsst.pipe.base.Struct` 746 Result struct with components: 748 - ``coaddExposure``: coadded exposure (``lsst.afw.image.Exposure``). 749 - ``nImage``: exposure count image (``lsst.afw.image.Image``), if requested. 750 - ``warpRefList``: input list of refs to the warps ( 751 ``lsst.daf.butler.DeferredDatasetHandle`` or 752 ``lsst.daf.persistence.ButlerDataRef``) 754 - ``imageScalerList``: input list of image scalers (unmodified) 755 - ``weightList``: input list of weights (unmodified) 757 tempExpName = self.getTempExpDatasetName(self.warpType)
758 self.log.info(
"Assembling %s %s", len(tempExpRefList), tempExpName)
759 stats = self.prepareStats(mask=mask)
761 if altMaskList
is None:
762 altMaskList = [
None]*len(tempExpRefList)
764 coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
765 coaddExposure.setPhotoCalib(self.scaleZeroPoint.getPhotoCalib())
766 coaddExposure.getInfo().setCoaddInputs(self.inputRecorder.makeCoaddInputs())
767 self.assembleMetadata(coaddExposure, tempExpRefList, weightList)
768 coaddMaskedImage = coaddExposure.getMaskedImage()
769 subregionSizeArr = self.config.subregionSize
770 subregionSize =
geom.Extent2I(subregionSizeArr[0], subregionSizeArr[1])
772 if self.config.doNImage:
773 nImage = afwImage.ImageU(skyInfo.bbox)
776 for subBBox
in self._subBBoxIter(skyInfo.bbox, subregionSize):
778 self.assembleSubregion(coaddExposure, subBBox, tempExpRefList, imageScalerList,
779 weightList, altMaskList, stats.flags, stats.ctrl,
781 except Exception
as e:
782 self.log.fatal(
"Cannot compute coadd %s: %s", subBBox, e)
784 self.setInexactPsf(coaddMaskedImage.getMask())
787 coaddUtils.setCoaddEdgeBits(coaddMaskedImage.getMask(), coaddMaskedImage.getVariance())
788 return pipeBase.Struct(coaddExposure=coaddExposure, nImage=nImage,
789 warpRefList=tempExpRefList, imageScalerList=imageScalerList,
790 weightList=weightList)
793 """Set the metadata for the coadd. 795 This basic implementation sets the filter from the first input. 799 coaddExposure : `lsst.afw.image.Exposure` 800 The target exposure for the coadd. 801 tempExpRefList : `list` 802 List of data references to tempExp. 806 assert len(tempExpRefList) == len(weightList),
"Length mismatch" 807 tempExpName = self.getTempExpDatasetName(self.warpType)
813 if isinstance(tempExpRefList[0], DeferredDatasetHandle):
815 tempExpList = [tempExpRef.get(parameters={
'bbox': bbox})
for tempExpRef
in tempExpRefList]
818 tempExpList = [tempExpRef.get(tempExpName +
"_sub", bbox=bbox, immediate=
True)
819 for tempExpRef
in tempExpRefList]
820 numCcds = sum(len(tempExp.getInfo().getCoaddInputs().ccds)
for tempExp
in tempExpList)
822 coaddExposure.setFilter(tempExpList[0].getFilter())
823 coaddInputs = coaddExposure.getInfo().getCoaddInputs()
824 coaddInputs.ccds.reserve(numCcds)
825 coaddInputs.visits.reserve(len(tempExpList))
827 for tempExp, weight
in zip(tempExpList, weightList):
828 self.inputRecorder.addVisitToCoadd(coaddInputs, tempExp, weight)
830 if self.config.doUsePsfMatchedPolygons:
831 self.shrinkValidPolygons(coaddInputs)
833 coaddInputs.visits.sort()
834 if self.warpType ==
"psfMatched":
839 modelPsfList = [tempExp.getPsf()
for tempExp
in tempExpList]
840 modelPsfWidthList = [modelPsf.computeBBox().getWidth()
for modelPsf
in modelPsfList]
841 psf = modelPsfList[modelPsfWidthList.index(max(modelPsfWidthList))]
843 psf = measAlg.CoaddPsf(coaddInputs.ccds, coaddExposure.getWcs(),
844 self.config.coaddPsf.makeControl())
845 coaddExposure.setPsf(psf)
846 apCorrMap = measAlg.makeCoaddApCorrMap(coaddInputs.ccds, coaddExposure.getBBox(afwImage.PARENT),
847 coaddExposure.getWcs())
848 coaddExposure.getInfo().setApCorrMap(apCorrMap)
849 if self.config.doAttachTransmissionCurve:
850 transmissionCurve = measAlg.makeCoaddTransmissionCurve(coaddExposure.getWcs(), coaddInputs.ccds)
851 coaddExposure.getInfo().setTransmissionCurve(transmissionCurve)
853 def assembleSubregion(self, coaddExposure, bbox, tempExpRefList, imageScalerList, weightList,
854 altMaskList, statsFlags, statsCtrl, nImage=None):
855 """Assemble the coadd for a sub-region. 857 For each coaddTempExp, check for (and swap in) an alternative mask 858 if one is passed. Remove mask planes listed in 859 `config.removeMaskPlanes`. Finally, stack the actual exposures using 860 `lsst.afw.math.statisticsStack` with the statistic specified by 861 statsFlags. Typically, the statsFlag will be one of lsst.afw.math.MEAN for 862 a mean-stack or `lsst.afw.math.MEANCLIP` for outlier rejection using 863 an N-sigma clipped mean where N and iterations are specified by 864 statsCtrl. Assign the stacked subregion back to the coadd. 868 coaddExposure : `lsst.afw.image.Exposure` 869 The target exposure for the coadd. 870 bbox : `lsst.geom.Box` 872 tempExpRefList : `list` 873 List of data reference to tempExp. 874 imageScalerList : `list` 875 List of image scalers. 879 List of alternate masks to use rather than those stored with 880 tempExp, or None. Each element is dict with keys = mask plane 881 name to which to add the spans. 882 statsFlags : `lsst.afw.math.Property` 883 Property object for statistic for coadd. 884 statsCtrl : `lsst.afw.math.StatisticsControl` 885 Statistics control object for coadd. 886 nImage : `lsst.afw.image.ImageU`, optional 887 Keeps track of exposure count for each pixel. 889 self.log.debug(
"Computing coadd over %s", bbox)
890 tempExpName = self.getTempExpDatasetName(self.warpType)
891 coaddExposure.mask.addMaskPlane(
"REJECTED")
892 coaddExposure.mask.addMaskPlane(
"CLIPPED")
893 coaddExposure.mask.addMaskPlane(
"SENSOR_EDGE")
894 maskMap = self.setRejectedMaskMapping(statsCtrl)
895 clipped = afwImage.Mask.getPlaneBitMask(
"CLIPPED")
897 if nImage
is not None:
898 subNImage = afwImage.ImageU(bbox.getWidth(), bbox.getHeight())
899 for tempExpRef, imageScaler, altMask
in zip(tempExpRefList, imageScalerList, altMaskList):
901 if isinstance(tempExpRef, DeferredDatasetHandle):
903 exposure = tempExpRef.get(parameters={
'bbox': bbox})
906 exposure = tempExpRef.get(tempExpName +
"_sub", bbox=bbox)
908 maskedImage = exposure.getMaskedImage()
909 mask = maskedImage.getMask()
910 if altMask
is not None:
911 self.applyAltMaskPlanes(mask, altMask)
912 imageScaler.scaleMaskedImage(maskedImage)
916 if nImage
is not None:
917 subNImage.getArray()[maskedImage.getMask().getArray() & statsCtrl.getAndMask() == 0] += 1
918 if self.config.removeMaskPlanes:
919 self.removeMaskPlanes(maskedImage)
920 maskedImageList.append(maskedImage)
922 with self.timer(
"stack"):
923 coaddSubregion = afwMath.statisticsStack(maskedImageList, statsFlags, statsCtrl, weightList,
926 coaddExposure.maskedImage.assign(coaddSubregion, bbox)
927 if nImage
is not None:
928 nImage.assign(subNImage, bbox)
931 """Unset the mask of an image for mask planes specified in the config. 935 maskedImage : `lsst.afw.image.MaskedImage` 936 The masked image to be modified. 938 mask = maskedImage.getMask()
939 for maskPlane
in self.config.removeMaskPlanes:
941 mask &= ~mask.getPlaneBitMask(maskPlane)
942 except pexExceptions.InvalidParameterError:
943 self.log.debug(
"Unable to remove mask plane %s: no mask plane with that name was found.",
947 def setRejectedMaskMapping(statsCtrl):
948 """Map certain mask planes of the warps to new planes for the coadd. 950 If a pixel is rejected due to a mask value other than EDGE, NO_DATA, 951 or CLIPPED, set it to REJECTED on the coadd. 952 If a pixel is rejected due to EDGE, set the coadd pixel to SENSOR_EDGE. 953 If a pixel is rejected due to CLIPPED, set the coadd pixel to CLIPPED. 957 statsCtrl : `lsst.afw.math.StatisticsControl` 958 Statistics control object for coadd 962 maskMap : `list` of `tuple` of `int` 963 A list of mappings of mask planes of the warped exposures to 964 mask planes of the coadd. 966 edge = afwImage.Mask.getPlaneBitMask(
"EDGE")
967 noData = afwImage.Mask.getPlaneBitMask(
"NO_DATA")
968 clipped = afwImage.Mask.getPlaneBitMask(
"CLIPPED")
969 toReject = statsCtrl.getAndMask() & (~noData) & (~edge) & (~clipped)
970 maskMap = [(toReject, afwImage.Mask.getPlaneBitMask(
"REJECTED")),
971 (edge, afwImage.Mask.getPlaneBitMask(
"SENSOR_EDGE")),
976 """Apply in place alt mask formatted as SpanSets to a mask. 980 mask : `lsst.afw.image.Mask` 982 altMaskSpans : `dict` 983 SpanSet lists to apply. Each element contains the new mask 984 plane name (e.g. "CLIPPED and/or "NO_DATA") as the key, 985 and list of SpanSets to apply to the mask. 989 mask : `lsst.afw.image.Mask` 992 if self.config.doUsePsfMatchedPolygons:
993 if (
"NO_DATA" in altMaskSpans)
and (
"NO_DATA" in self.config.badMaskPlanes):
998 for spanSet
in altMaskSpans[
'NO_DATA']:
999 spanSet.clippedTo(mask.getBBox()).clearMask(mask, self.getBadPixelMask())
1001 for plane, spanSetList
in altMaskSpans.items():
1002 maskClipValue = mask.addMaskPlane(plane)
1003 for spanSet
in spanSetList:
1004 spanSet.clippedTo(mask.getBBox()).setMask(mask, 2**maskClipValue)
1008 """Shrink coaddInputs' ccds' ValidPolygons in place. 1010 Either modify each ccd's validPolygon in place, or if CoaddInputs 1011 does not have a validPolygon, create one from its bbox. 1015 coaddInputs : `lsst.afw.image.coaddInputs` 1019 for ccd
in coaddInputs.ccds:
1020 polyOrig = ccd.getValidPolygon()
1021 validPolyBBox = polyOrig.getBBox()
if polyOrig
else ccd.getBBox()
1022 validPolyBBox.grow(-self.config.matchingKernelSize//2)
1024 validPolygon = polyOrig.intersectionSingle(validPolyBBox)
1026 validPolygon = afwGeom.polygon.Polygon(
geom.Box2D(validPolyBBox))
1027 ccd.setValidPolygon(validPolygon)
1030 """Retrieve the bright object masks. 1032 Returns None on failure. 1036 dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef` 1041 result : `lsst.daf.persistence.butlerSubset.ButlerDataRef` 1042 Bright object mask from the Butler object, or None if it cannot 1046 return dataRef.get(datasetType=
"brightObjectMask", immediate=
True)
1047 except Exception
as e:
1048 self.log.warn(
"Unable to read brightObjectMask for %s: %s", dataRef.dataId, e)
1052 """Set the bright object masks. 1056 exposure : `lsst.afw.image.Exposure` 1057 Exposure under consideration. 1058 dataId : `lsst.daf.persistence.dataId` 1059 Data identifier dict for patch. 1060 brightObjectMasks : `lsst.afw.table` 1061 Table of bright objects to mask. 1064 if brightObjectMasks
is None:
1065 self.log.warn(
"Unable to apply bright object mask: none supplied")
1067 self.log.info(
"Applying %d bright object masks to %s", len(brightObjectMasks), dataId)
1068 mask = exposure.getMaskedImage().getMask()
1069 wcs = exposure.getWcs()
1070 plateScale = wcs.getPixelScale().asArcseconds()
1072 for rec
in brightObjectMasks:
1073 center =
geom.PointI(wcs.skyToPixel(rec.getCoord()))
1074 if rec[
"type"] ==
"box":
1075 assert rec[
"angle"] == 0.0, (
"Angle != 0 for mask object %s" % rec[
"id"])
1076 width = rec[
"width"].asArcseconds()/plateScale
1077 height = rec[
"height"].asArcseconds()/plateScale
1080 bbox =
geom.Box2I(center - halfSize, center + halfSize)
1083 geom.PointI(int(center[0] + 0.5*width), int(center[1] + 0.5*height)))
1084 spans = afwGeom.SpanSet(bbox)
1085 elif rec[
"type"] ==
"circle":
1086 radius = int(rec[
"radius"].asArcseconds()/plateScale)
1087 spans = afwGeom.SpanSet.fromShape(radius, offset=center)
1089 self.log.warn(
"Unexpected region type %s at %s" % rec[
"type"], center)
1091 spans.clippedTo(mask.getBBox()).setMask(mask, self.brightObjectBitmask)
1094 """Set INEXACT_PSF mask plane. 1096 If any of the input images isn't represented in the coadd (due to 1097 clipped pixels or chip gaps), the `CoaddPsf` will be inexact. Flag 1102 mask : `lsst.afw.image.Mask` 1103 Coadded exposure's mask, modified in-place. 1105 mask.addMaskPlane(
"INEXACT_PSF")
1106 inexactPsf = mask.getPlaneBitMask(
"INEXACT_PSF")
1107 sensorEdge = mask.getPlaneBitMask(
"SENSOR_EDGE")
1108 clipped = mask.getPlaneBitMask(
"CLIPPED")
1109 rejected = mask.getPlaneBitMask(
"REJECTED")
1110 array = mask.getArray()
1111 selected = array & (sensorEdge | clipped | rejected) > 0
1112 array[selected] |= inexactPsf
1115 def _makeArgumentParser(cls):
1116 """Create an argument parser. 1118 parser = pipeBase.ArgumentParser(name=cls._DefaultName)
1119 parser.add_id_argument(
"--id", cls.ConfigClass().coaddName +
"Coadd_" +
1120 cls.ConfigClass().warpType +
"Warp",
1121 help=
"data ID, e.g. --id tract=12345 patch=1,2",
1122 ContainerClass=AssembleCoaddDataIdContainer)
1123 parser.add_id_argument(
"--selectId",
"calexp", help=
"data ID, e.g. --selectId visit=6789 ccd=0..9",
1124 ContainerClass=SelectDataIdContainer)
1128 def _subBBoxIter(bbox, subregionSize):
1129 """Iterate over subregions of a bbox. 1133 bbox : `lsst.geom.Box2I` 1134 Bounding box over which to iterate. 1135 subregionSize: `lsst.geom.Extent2I` 1140 subBBox : `lsst.geom.Box2I` 1141 Next sub-bounding box of size ``subregionSize`` or smaller; each ``subBBox`` 1142 is contained within ``bbox``, so it may be smaller than ``subregionSize`` at 1143 the edges of ``bbox``, but it will never be empty. 1146 raise RuntimeError(
"bbox %s is empty" % (bbox,))
1147 if subregionSize[0] < 1
or subregionSize[1] < 1:
1148 raise RuntimeError(
"subregionSize %s must be nonzero" % (subregionSize,))
1150 for rowShift
in range(0, bbox.getHeight(), subregionSize[1]):
1151 for colShift
in range(0, bbox.getWidth(), subregionSize[0]):
1154 if subBBox.isEmpty():
1155 raise RuntimeError(
"Bug: empty bbox! bbox=%s, subregionSize=%s, " 1156 "colShift=%s, rowShift=%s" %
1157 (bbox, subregionSize, colShift, rowShift))
1162 """A version of `lsst.pipe.base.DataIdContainer` specialized for assembleCoadd. 1166 """Make self.refList from self.idList. 1171 Results of parsing command-line (with ``butler`` and ``log`` elements). 1173 datasetType = namespace.config.coaddName +
"Coadd" 1174 keysCoadd = namespace.butler.getKeys(datasetType=datasetType, level=self.level)
1176 for dataId
in self.idList:
1178 for key
in keysCoadd:
1179 if key
not in dataId:
1180 raise RuntimeError(
"--id must include " + key)
1182 dataRef = namespace.butler.dataRef(
1183 datasetType=datasetType,
1186 self.refList.append(dataRef)
1190 """Function to count the number of pixels with a specific mask in a 1193 Find the intersection of mask & footprint. Count all pixels in the mask 1194 that are in the intersection that have bitmask set but do not have 1195 ignoreMask set. Return the count. 1199 mask : `lsst.afw.image.Mask` 1200 Mask to define intersection region by. 1201 footprint : `lsst.afw.detection.Footprint` 1202 Footprint to define the intersection region by. 1204 Specific mask that we wish to count the number of occurances of. 1206 Pixels to not consider. 1211 Count of number of pixels in footprint with specified mask. 1213 bbox = footprint.getBBox()
1214 bbox.clip(mask.getBBox(afwImage.PARENT))
1215 fp = afwImage.Mask(bbox)
1216 subMask = mask.Factory(mask, bbox, afwImage.PARENT)
1217 footprint.spans.setMask(fp, bitmask)
1218 return numpy.logical_and((subMask.getArray() & fp.getArray()) > 0,
1219 (subMask.getArray() & ignoreMask) == 0).sum()
1223 """Configuration parameters for the SafeClipAssembleCoaddTask. 1225 clipDetection = pexConfig.ConfigurableField(
1226 target=SourceDetectionTask,
1227 doc=
"Detect sources on difference between unclipped and clipped coadd")
1228 minClipFootOverlap = pexConfig.Field(
1229 doc=
"Minimum fractional overlap of clipped footprint with visit DETECTED to be clipped",
1233 minClipFootOverlapSingle = pexConfig.Field(
1234 doc=
"Minimum fractional overlap of clipped footprint with visit DETECTED to be " 1235 "clipped when only one visit overlaps",
1239 minClipFootOverlapDouble = pexConfig.Field(
1240 doc=
"Minimum fractional overlap of clipped footprints with visit DETECTED to be " 1241 "clipped when two visits overlap",
1245 maxClipFootOverlapDouble = pexConfig.Field(
1246 doc=
"Maximum fractional overlap of clipped footprints with visit DETECTED when " 1247 "considering two visits",
1251 minBigOverlap = pexConfig.Field(
1252 doc=
"Minimum number of pixels in footprint to use DETECTED mask from the single visits " 1253 "when labeling clipped footprints",
1259 """Set default values for clipDetection. 1263 The numeric values for these configuration parameters were 1264 empirically determined, future work may further refine them. 1266 AssembleCoaddConfig.setDefaults(self)
1282 log.warn(
"Additional Sigma-clipping not allowed in Safe-clipped Coadds. " 1283 "Ignoring doSigmaClip.")
1286 raise ValueError(
"Only MEAN statistic allowed for final stacking in SafeClipAssembleCoadd " 1287 "(%s chosen). Please set statistic to MEAN." 1289 AssembleCoaddTask.ConfigClass.validate(self)
1293 """Assemble a coadded image from a set of coadded temporary exposures, 1294 being careful to clip & flag areas with potential artifacts. 1296 In ``AssembleCoaddTask``, we compute the coadd as an clipped mean (i.e., 1297 we clip outliers). The problem with doing this is that when computing the 1298 coadd PSF at a given location, individual visit PSFs from visits with 1299 outlier pixels contribute to the coadd PSF and cannot be treated correctly. 1300 In this task, we correct for this behavior by creating a new 1301 ``badMaskPlane`` 'CLIPPED'. We populate this plane on the input 1302 coaddTempExps and the final coadd where 1304 i. difference imaging suggests that there is an outlier and 1305 ii. this outlier appears on only one or two images. 1307 Such regions will not contribute to the final coadd. Furthermore, any 1308 routine to determine the coadd PSF can now be cognizant of clipped regions. 1309 Note that the algorithm implemented by this task is preliminary and works 1310 correctly for HSC data. Parameter modifications and or considerable 1311 redesigning of the algorithm is likley required for other surveys. 1313 ``SafeClipAssembleCoaddTask`` uses a ``SourceDetectionTask`` 1314 "clipDetection" subtask and also sub-classes ``AssembleCoaddTask``. 1315 You can retarget the ``SourceDetectionTask`` "clipDetection" subtask 1320 The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a 1321 flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``; 1322 see `baseDebug` for more about ``debug.py`` files. 1323 `SafeClipAssembleCoaddTask` has no debug variables of its own. 1324 The ``SourceDetectionTask`` "clipDetection" subtasks may support debug 1325 variables. See the documetation for `SourceDetectionTask` "clipDetection" 1326 for further information. 1330 `SafeClipAssembleCoaddTask` assembles a set of warped ``coaddTempExp`` 1331 images into a coadded image. The `SafeClipAssembleCoaddTask` is invoked by 1332 running assembleCoadd.py *without* the flag '--legacyCoadd'. 1334 Usage of ``assembleCoadd.py`` expects a data reference to the tract patch 1335 and filter to be coadded (specified using 1336 '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]') 1337 along with a list of coaddTempExps to attempt to coadd (specified using 1338 '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]'). 1339 Only the coaddTempExps that cover the specified tract and patch will be 1340 coadded. A list of the available optional arguments can be obtained by 1341 calling assembleCoadd.py with the --help command line argument: 1343 .. code-block:: none 1345 assembleCoadd.py --help 1347 To demonstrate usage of the `SafeClipAssembleCoaddTask` in the larger 1348 context of multi-band processing, we will generate the HSC-I & -R band 1349 coadds from HSC engineering test data provided in the ci_hsc package. 1350 To begin, assuming that the lsst stack has been already set up, we must 1351 set up the obs_subaru and ci_hsc packages. This defines the environment 1352 variable $CI_HSC_DIR and points at the location of the package. The raw 1353 HSC data live in the ``$CI_HSC_DIR/raw`` directory. To begin assembling 1354 the coadds, we must first 1357 process the individual ccds in $CI_HSC_RAW to produce calibrated exposures 1359 create a skymap that covers the area of the sky present in the raw exposures 1360 - ``makeCoaddTempExp`` 1361 warp the individual calibrated exposures to the tangent plane of the coadd</DD> 1363 We can perform all of these steps by running 1365 .. code-block:: none 1367 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988 1369 This will produce warped coaddTempExps for each visit. To coadd the 1370 warped data, we call ``assembleCoadd.py`` as follows: 1372 .. code-block:: none 1374 assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \ 1375 --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \ 1376 --selectId visit=903986 ccd=100--selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \ 1377 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \ 1378 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \ 1379 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \ 1380 --selectId visit=903988 ccd=24 1382 This will process the HSC-I band data. The results are written in 1383 ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``. 1385 You may also choose to run: 1387 .. code-block:: none 1389 scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346 nnn 1390 assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R --selectId visit=903334 ccd=16 \ 1391 --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 --selectId visit=903334 ccd=100 \ 1392 --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 --selectId visit=903338 ccd=18 \ 1393 --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 --selectId visit=903342 ccd=10 \ 1394 --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 --selectId visit=903344 ccd=5 \ 1395 --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 --selectId visit=903346 ccd=6 \ 1396 --selectId visit=903346 ccd=12 1398 to generate the coadd for the HSC-R band if you are interested in following 1399 multiBand Coadd processing as discussed in ``pipeTasks_multiBand``. 1401 ConfigClass = SafeClipAssembleCoaddConfig
1402 _DefaultName =
"safeClipAssembleCoadd" 1405 AssembleCoaddTask.__init__(self, *args, **kwargs)
1406 schema = afwTable.SourceTable.makeMinimalSchema()
1407 self.makeSubtask(
"clipDetection", schema=schema)
1409 @utils.inheritDoc(AssembleCoaddTask)
1410 def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, *args, **kwargs):
1411 """Assemble the coadd for a region. 1413 Compute the difference of coadds created with and without outlier 1414 rejection to identify coadd pixels that have outlier values in some 1416 Detect clipped regions on the difference image and mark these regions 1417 on the one or two individual coaddTempExps where they occur if there 1418 is significant overlap between the clipped region and a source. This 1419 leaves us with a set of footprints from the difference image that have 1420 been identified as having occured on just one or two individual visits. 1421 However, these footprints were generated from a difference image. It 1422 is conceivable for a large diffuse source to have become broken up 1423 into multiple footprints acrosss the coadd difference in this process. 1424 Determine the clipped region from all overlapping footprints from the 1425 detected sources in each visit - these are big footprints. 1426 Combine the small and big clipped footprints and mark them on a new 1428 Generate the coadd using `AssembleCoaddTask.run` without outlier 1429 removal. Clipped footprints will no longer make it into the coadd 1430 because they are marked in the new bad mask plane. 1434 args and kwargs are passed but ignored in order to match the call 1435 signature expected by the parent task. 1438 mask = exp.getMaskedImage().getMask()
1439 mask.addMaskPlane(
"CLIPPED")
1441 result = self.
detectClip(exp, tempExpRefList)
1443 self.log.info(
'Found %d clipped objects', len(result.clipFootprints))
1445 maskClipValue = mask.getPlaneBitMask(
"CLIPPED")
1446 maskDetValue = mask.getPlaneBitMask(
"DETECTED") | mask.getPlaneBitMask(
"DETECTED_NEGATIVE")
1448 bigFootprints = self.
detectClipBig(result.clipSpans, result.clipFootprints, result.clipIndices,
1449 result.detectionFootprints, maskClipValue, maskDetValue,
1452 maskClip = mask.Factory(mask.getBBox(afwImage.PARENT))
1453 afwDet.setMaskFromFootprintList(maskClip, result.clipFootprints, maskClipValue)
1455 maskClipBig = maskClip.Factory(mask.getBBox(afwImage.PARENT))
1456 afwDet.setMaskFromFootprintList(maskClipBig, bigFootprints, maskClipValue)
1457 maskClip |= maskClipBig
1460 badMaskPlanes = self.config.badMaskPlanes[:]
1461 badMaskPlanes.append(
"CLIPPED")
1462 badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes)
1463 return AssembleCoaddTask.run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
1464 result.clipSpans, mask=badPixelMask)
1467 """Return an exposure that contains the difference between unclipped 1470 Generate a difference image between clipped and unclipped coadds. 1471 Compute the difference image by subtracting an outlier-clipped coadd 1472 from an outlier-unclipped coadd. Return the difference image. 1476 skyInfo : `lsst.pipe.base.Struct` 1477 Patch geometry information, from getSkyInfo 1478 tempExpRefList : `list` 1479 List of data reference to tempExp 1480 imageScalerList : `list` 1481 List of image scalers 1487 exp : `lsst.afw.image.Exposure` 1488 Difference image of unclipped and clipped coadd wrapped in an Exposure 1491 config = AssembleCoaddConfig()
1496 configIntersection = {k: getattr(self.config, k)
1497 for k, v
in self.config.toDict().items()
if (k
in config.keys()
and 1498 k !=
"connections")}
1499 config.update(**configIntersection)
1502 config.statistic =
'MEAN' 1503 task = AssembleCoaddTask(config=config)
1504 coaddMean = task.run(skyInfo, tempExpRefList, imageScalerList, weightList).coaddExposure
1506 config.statistic =
'MEANCLIP' 1507 task = AssembleCoaddTask(config=config)
1508 coaddClip = task.run(skyInfo, tempExpRefList, imageScalerList, weightList).coaddExposure
1510 coaddDiff = coaddMean.getMaskedImage().Factory(coaddMean.getMaskedImage())
1511 coaddDiff -= coaddClip.getMaskedImage()
1512 exp = afwImage.ExposureF(coaddDiff)
1513 exp.setPsf(coaddMean.getPsf())
1517 """Detect clipped regions on an exposure and set the mask on the 1518 individual tempExp masks. 1520 Detect footprints in the difference image after smoothing the 1521 difference image with a Gaussian kernal. Identify footprints that 1522 overlap with one or two input ``coaddTempExps`` by comparing the 1523 computed overlap fraction to thresholds set in the config. A different 1524 threshold is applied depending on the number of overlapping visits 1525 (restricted to one or two). If the overlap exceeds the thresholds, 1526 the footprint is considered "CLIPPED" and is marked as such on the 1527 coaddTempExp. Return a struct with the clipped footprints, the indices 1528 of the ``coaddTempExps`` that end up overlapping with the clipped 1529 footprints, and a list of new masks for the ``coaddTempExps``. 1533 exp : `lsst.afw.image.Exposure` 1534 Exposure to run detection on. 1535 tempExpRefList : `list` 1536 List of data reference to tempExp. 1540 result : `lsst.pipe.base.Struct` 1541 Result struct with components: 1543 - ``clipFootprints``: list of clipped footprints. 1544 - ``clipIndices``: indices for each ``clippedFootprint`` in 1546 - ``clipSpans``: List of dictionaries containing spanSet lists 1547 to clip. Each element contains the new maskplane name 1548 ("CLIPPED") as the key and list of ``SpanSets`` as the value. 1549 - ``detectionFootprints``: List of DETECTED/DETECTED_NEGATIVE plane 1550 compressed into footprints. 1552 mask = exp.getMaskedImage().getMask()
1553 maskDetValue = mask.getPlaneBitMask(
"DETECTED") | mask.getPlaneBitMask(
"DETECTED_NEGATIVE")
1554 fpSet = self.clipDetection.detectFootprints(exp, doSmooth=
True, clearMask=
True)
1556 fpSet.positive.merge(fpSet.negative)
1557 footprints = fpSet.positive
1558 self.log.info(
'Found %d potential clipped objects', len(footprints.getFootprints()))
1559 ignoreMask = self.getBadPixelMask()
1563 artifactSpanSets = [{
'CLIPPED': list()}
for _
in tempExpRefList]
1566 visitDetectionFootprints = []
1568 dims = [len(tempExpRefList), len(footprints.getFootprints())]
1569 overlapDetArr = numpy.zeros(dims, dtype=numpy.uint16)
1570 ignoreArr = numpy.zeros(dims, dtype=numpy.uint16)
1573 for i, warpRef
in enumerate(tempExpRefList):
1574 tmpExpMask = warpRef.get(datasetType=self.getTempExpDatasetName(self.warpType),
1575 immediate=
True).getMaskedImage().getMask()
1576 maskVisitDet = tmpExpMask.Factory(tmpExpMask, tmpExpMask.getBBox(afwImage.PARENT),
1577 afwImage.PARENT,
True)
1578 maskVisitDet &= maskDetValue
1579 visitFootprints = afwDet.FootprintSet(maskVisitDet, afwDet.Threshold(1))
1580 visitDetectionFootprints.append(visitFootprints)
1582 for j, footprint
in enumerate(footprints.getFootprints()):
1587 for j, footprint
in enumerate(footprints.getFootprints()):
1588 nPixel = footprint.getArea()
1591 for i
in range(len(tempExpRefList)):
1592 ignore = ignoreArr[i, j]
1593 overlapDet = overlapDetArr[i, j]
1594 totPixel = nPixel - ignore
1597 if ignore > overlapDet
or totPixel <= 0.5*nPixel
or overlapDet == 0:
1599 overlap.append(overlapDet/float(totPixel))
1602 overlap = numpy.array(overlap)
1603 if not len(overlap):
1610 if len(overlap) == 1:
1611 if overlap[0] > self.config.minClipFootOverlapSingle:
1616 clipIndex = numpy.where(overlap > self.config.minClipFootOverlap)[0]
1617 if len(clipIndex) == 1:
1619 keepIndex = [clipIndex[0]]
1622 clipIndex = numpy.where(overlap > self.config.minClipFootOverlapDouble)[0]
1623 if len(clipIndex) == 2
and len(overlap) > 3:
1624 clipIndexComp = numpy.where(overlap <= self.config.minClipFootOverlapDouble)[0]
1625 if numpy.max(overlap[clipIndexComp]) <= self.config.maxClipFootOverlapDouble:
1627 keepIndex = clipIndex
1632 for index
in keepIndex:
1633 globalIndex = indexList[index]
1634 artifactSpanSets[globalIndex][
'CLIPPED'].append(footprint.spans)
1636 clipIndices.append(numpy.array(indexList)[keepIndex])
1637 clipFootprints.append(footprint)
1639 return pipeBase.Struct(clipFootprints=clipFootprints, clipIndices=clipIndices,
1640 clipSpans=artifactSpanSets, detectionFootprints=visitDetectionFootprints)
1642 def detectClipBig(self, clipList, clipFootprints, clipIndices, detectionFootprints,
1643 maskClipValue, maskDetValue, coaddBBox):
1644 """Return individual warp footprints for large artifacts and append 1645 them to ``clipList`` in place. 1647 Identify big footprints composed of many sources in the coadd 1648 difference that may have originated in a large diffuse source in the 1649 coadd. We do this by indentifying all clipped footprints that overlap 1650 significantly with each source in all the coaddTempExps. 1655 List of alt mask SpanSets with clipping information. Modified. 1656 clipFootprints : `list` 1657 List of clipped footprints. 1658 clipIndices : `list` 1659 List of which entries in tempExpClipList each footprint belongs to. 1661 Mask value of clipped pixels. 1663 Mask value of detected pixels. 1664 coaddBBox : `lsst.geom.Box` 1665 BBox of the coadd and warps. 1669 bigFootprintsCoadd : `list` 1670 List of big footprints 1672 bigFootprintsCoadd = []
1673 ignoreMask = self.getBadPixelMask()
1674 for index, (clippedSpans, visitFootprints)
in enumerate(zip(clipList, detectionFootprints)):
1675 maskVisitDet = afwImage.MaskX(coaddBBox, 0x0)
1676 for footprint
in visitFootprints.getFootprints():
1677 footprint.spans.setMask(maskVisitDet, maskDetValue)
1680 clippedFootprintsVisit = []
1681 for foot, clipIndex
in zip(clipFootprints, clipIndices):
1682 if index
not in clipIndex:
1684 clippedFootprintsVisit.append(foot)
1685 maskVisitClip = maskVisitDet.Factory(maskVisitDet.getBBox(afwImage.PARENT))
1686 afwDet.setMaskFromFootprintList(maskVisitClip, clippedFootprintsVisit, maskClipValue)
1688 bigFootprintsVisit = []
1689 for foot
in visitFootprints.getFootprints():
1690 if foot.getArea() < self.config.minBigOverlap:
1693 if nCount > self.config.minBigOverlap:
1694 bigFootprintsVisit.append(foot)
1695 bigFootprintsCoadd.append(foot)
1697 for footprint
in bigFootprintsVisit:
1698 clippedSpans[
"CLIPPED"].append(footprint.spans)
1700 return bigFootprintsCoadd
1704 psfMatchedWarps = pipeBase.connectionTypes.Input(
1705 doc=(
"PSF-Matched Warps are required by CompareWarp regardless of the coadd type requested. " 1706 "Only PSF-Matched Warps make sense for image subtraction. " 1707 "Therefore, they must be an additional declared input."),
1708 name=
"{inputCoaddName}Coadd_psfMatchedWarp",
1709 storageClass=
"ExposureF",
1710 dimensions=(
"tract",
"patch",
"skymap",
"visit"),
1714 templateCoadd = pipeBase.connectionTypes.Output(
1715 doc=(
"Model of the static sky, used to find temporal artifacts. Typically a PSF-Matched, " 1716 "sigma-clipped coadd. Written if and only if assembleStaticSkyModel.doWrite=True"),
1717 name=
"{fakesType}{outputCoaddName}CoaddPsfMatched",
1718 storageClass=
"ExposureF",
1719 dimensions=(
"tract",
"patch",
"skymap",
"abstract_filter"),
1724 if not config.assembleStaticSkyModel.doWrite:
1725 self.outputs.remove(
"templateCoadd")
1730 pipelineConnections=CompareWarpAssembleCoaddConnections):
1731 assembleStaticSkyModel = pexConfig.ConfigurableField(
1732 target=AssembleCoaddTask,
1733 doc=
"Task to assemble an artifact-free, PSF-matched Coadd to serve as a" 1734 " naive/first-iteration model of the static sky.",
1736 detect = pexConfig.ConfigurableField(
1737 target=SourceDetectionTask,
1738 doc=
"Detect outlier sources on difference between each psfMatched warp and static sky model" 1740 detectTemplate = pexConfig.ConfigurableField(
1741 target=SourceDetectionTask,
1742 doc=
"Detect sources on static sky model. Only used if doPreserveContainedBySource is True" 1744 maxNumEpochs = pexConfig.Field(
1745 doc=
"Charactistic maximum local number of epochs/visits in which an artifact candidate can appear " 1746 "and still be masked. The effective maxNumEpochs is a broken linear function of local " 1747 "number of epochs (N): min(maxFractionEpochsLow*N, maxNumEpochs + maxFractionEpochsHigh*N). " 1748 "For each footprint detected on the image difference between the psfMatched warp and static sky " 1749 "model, if a significant fraction of pixels (defined by spatialThreshold) are residuals in more " 1750 "than the computed effective maxNumEpochs, the artifact candidate is deemed persistant rather " 1751 "than transient and not masked.",
1755 maxFractionEpochsLow = pexConfig.RangeField(
1756 doc=
"Fraction of local number of epochs (N) to use as effective maxNumEpochs for low N. " 1757 "Effective maxNumEpochs = " 1758 "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)",
1763 maxFractionEpochsHigh = pexConfig.RangeField(
1764 doc=
"Fraction of local number of epochs (N) to use as effective maxNumEpochs for high N. " 1765 "Effective maxNumEpochs = " 1766 "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)",
1771 spatialThreshold = pexConfig.RangeField(
1772 doc=
"Unitless fraction of pixels defining how much of the outlier region has to meet the " 1773 "temporal criteria. If 0, clip all. If 1, clip none.",
1777 inclusiveMin=
True, inclusiveMax=
True 1779 doScaleWarpVariance = pexConfig.Field(
1780 doc=
"Rescale Warp variance plane using empirical noise?",
1784 scaleWarpVariance = pexConfig.ConfigurableField(
1785 target=ScaleVarianceTask,
1786 doc=
"Rescale variance on warps",
1788 doPreserveContainedBySource = pexConfig.Field(
1789 doc=
"Rescue artifacts from clipping that completely lie within a footprint detected" 1790 "on the PsfMatched Template Coadd. Replicates a behavior of SafeClip.",
1794 doPrefilterArtifacts = pexConfig.Field(
1795 doc=
"Ignore artifact candidates that are mostly covered by the bad pixel mask, " 1796 "because they will be excluded anyway. This prevents them from contributing " 1797 "to the outlier epoch count image and potentially being labeled as persistant." 1798 "'Mostly' is defined by the config 'prefilterArtifactsRatio'.",
1802 prefilterArtifactsMaskPlanes = pexConfig.ListField(
1803 doc=
"Prefilter artifact candidates that are mostly covered by these bad mask planes.",
1805 default=(
'NO_DATA',
'BAD',
'SAT',
'SUSPECT'),
1807 prefilterArtifactsRatio = pexConfig.Field(
1808 doc=
"Prefilter artifact candidates with less than this fraction overlapping good pixels",
1814 AssembleCoaddConfig.setDefaults(self)
1820 if "EDGE" in self.badMaskPlanes:
1821 self.badMaskPlanes.remove(
'EDGE')
1822 self.removeMaskPlanes.append(
'EDGE')
1831 self.
detect.doTempLocalBackground =
False 1832 self.
detect.reEstimateBackground =
False 1833 self.
detect.returnOriginalFootprints =
False 1834 self.
detect.thresholdPolarity =
"both" 1835 self.
detect.thresholdValue = 5
1836 self.
detect.nSigmaToGrow = 2
1837 self.
detect.minPixels = 4
1838 self.
detect.isotropicGrow =
True 1839 self.
detect.thresholdType =
"pixel_stdev" 1848 raise ValueError(
"No dataset type exists for a PSF-Matched Template N Image." 1849 "Please set assembleStaticSkyModel.doNImage=False")
1852 raise ValueError(
"warpType (%s) == assembleStaticSkyModel.warpType (%s) and will compete for " 1853 "the same dataset name. Please set assembleStaticSkyModel.doWrite to False " 1854 "or warpType to 'direct'. assembleStaticSkyModel.warpType should ways be " 1859 """Assemble a compareWarp coadded image from a set of warps 1860 by masking artifacts detected by comparing PSF-matched warps. 1862 In ``AssembleCoaddTask``, we compute the coadd as an clipped mean (i.e., 1863 we clip outliers). The problem with doing this is that when computing the 1864 coadd PSF at a given location, individual visit PSFs from visits with 1865 outlier pixels contribute to the coadd PSF and cannot be treated correctly. 1866 In this task, we correct for this behavior by creating a new badMaskPlane 1867 'CLIPPED' which marks pixels in the individual warps suspected to contain 1868 an artifact. We populate this plane on the input warps by comparing 1869 PSF-matched warps with a PSF-matched median coadd which serves as a 1870 model of the static sky. Any group of pixels that deviates from the 1871 PSF-matched template coadd by more than config.detect.threshold sigma, 1872 is an artifact candidate. The candidates are then filtered to remove 1873 variable sources and sources that are difficult to subtract such as 1874 bright stars. This filter is configured using the config parameters 1875 ``temporalThreshold`` and ``spatialThreshold``. The temporalThreshold is 1876 the maximum fraction of epochs that the deviation can appear in and still 1877 be considered an artifact. The spatialThreshold is the maximum fraction of 1878 pixels in the footprint of the deviation that appear in other epochs 1879 (where other epochs is defined by the temporalThreshold). If the deviant 1880 region meets this criteria of having a significant percentage of pixels 1881 that deviate in only a few epochs, these pixels have the 'CLIPPED' bit 1882 set in the mask. These regions will not contribute to the final coadd. 1883 Furthermore, any routine to determine the coadd PSF can now be cognizant 1884 of clipped regions. Note that the algorithm implemented by this task is 1885 preliminary and works correctly for HSC data. Parameter modifications and 1886 or considerable redesigning of the algorithm is likley required for other 1889 ``CompareWarpAssembleCoaddTask`` sub-classes 1890 ``AssembleCoaddTask`` and instantiates ``AssembleCoaddTask`` 1891 as a subtask to generate the TemplateCoadd (the model of the static sky). 1895 The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a 1896 flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``; see 1897 ``baseDebug`` for more about ``debug.py`` files. 1899 This task supports the following debug variables: 1902 If True then save the Epoch Count Image as a fits file in the `figPath` 1904 Path to save the debug fits images and figures 1906 For example, put something like: 1908 .. code-block:: python 1911 def DebugInfo(name): 1912 di = lsstDebug.getInfo(name) 1913 if name == "lsst.pipe.tasks.assembleCoadd": 1914 di.saveCountIm = True 1915 di.figPath = "/desired/path/to/debugging/output/images" 1917 lsstDebug.Info = DebugInfo 1919 into your ``debug.py`` file and run ``assemebleCoadd.py`` with the 1920 ``--debug`` flag. Some subtasks may have their own debug variables; 1921 see individual Task documentation. 1925 ``CompareWarpAssembleCoaddTask`` assembles a set of warped images into a 1926 coadded image. The ``CompareWarpAssembleCoaddTask`` is invoked by running 1927 ``assembleCoadd.py`` with the flag ``--compareWarpCoadd``. 1928 Usage of ``assembleCoadd.py`` expects a data reference to the tract patch 1929 and filter to be coadded (specified using 1930 '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]') 1931 along with a list of coaddTempExps to attempt to coadd (specified using 1932 '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]'). 1933 Only the warps that cover the specified tract and patch will be coadded. 1934 A list of the available optional arguments can be obtained by calling 1935 ``assembleCoadd.py`` with the ``--help`` command line argument: 1937 .. code-block:: none 1939 assembleCoadd.py --help 1941 To demonstrate usage of the ``CompareWarpAssembleCoaddTask`` in the larger 1942 context of multi-band processing, we will generate the HSC-I & -R band 1943 oadds from HSC engineering test data provided in the ``ci_hsc`` package. 1944 To begin, assuming that the lsst stack has been already set up, we must 1945 set up the ``obs_subaru`` and ``ci_hsc`` packages. 1946 This defines the environment variable ``$CI_HSC_DIR`` and points at the 1947 location of the package. The raw HSC data live in the ``$CI_HSC_DIR/raw`` 1948 directory. To begin assembling the coadds, we must first 1951 process the individual ccds in $CI_HSC_RAW to produce calibrated exposures 1953 create a skymap that covers the area of the sky present in the raw exposures 1955 warp the individual calibrated exposures to the tangent plane of the coadd 1957 We can perform all of these steps by running 1959 .. code-block:: none 1961 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988 1963 This will produce warped ``coaddTempExps`` for each visit. To coadd the 1964 warped data, we call ``assembleCoadd.py`` as follows: 1966 .. code-block:: none 1968 assembleCoadd.py --compareWarpCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \ 1969 --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \ 1970 --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \ 1971 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \ 1972 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \ 1973 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \ 1974 --selectId visit=903988 ccd=24 1976 This will process the HSC-I band data. The results are written in 1977 ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``. 1979 ConfigClass = CompareWarpAssembleCoaddConfig
1980 _DefaultName =
"compareWarpAssembleCoadd" 1983 AssembleCoaddTask.__init__(self, *args, **kwargs)
1984 self.makeSubtask(
"assembleStaticSkyModel")
1985 detectionSchema = afwTable.SourceTable.makeMinimalSchema()
1986 self.makeSubtask(
"detect", schema=detectionSchema)
1987 if self.config.doPreserveContainedBySource:
1988 self.makeSubtask(
"detectTemplate", schema=afwTable.SourceTable.makeMinimalSchema())
1989 if self.config.doScaleWarpVariance:
1990 self.makeSubtask(
"scaleWarpVariance")
1992 @utils.inheritDoc(AssembleCoaddTask)
1995 Generate a templateCoadd to use as a naive model of static sky to 1996 subtract from PSF-Matched warps. 2000 result : `lsst.pipe.base.Struct` 2001 Result struct with components: 2003 - ``templateCoadd`` : coadded exposure (``lsst.afw.image.Exposure``) 2004 - ``nImage`` : N Image (``lsst.afw.image.Image``) 2007 staticSkyModelInputRefs = copy.deepcopy(inputRefs)
2008 staticSkyModelInputRefs.inputWarps = inputRefs.psfMatchedWarps
2012 staticSkyModelOutputRefs = copy.deepcopy(outputRefs)
2013 if self.config.assembleStaticSkyModel.doWrite:
2014 staticSkyModelOutputRefs.coaddExposure = staticSkyModelOutputRefs.templateCoadd
2017 del outputRefs.templateCoadd
2018 del staticSkyModelOutputRefs.templateCoadd
2021 del staticSkyModelOutputRefs.nImage
2023 templateCoadd = self.assembleStaticSkyModel.runQuantum(butlerQC, staticSkyModelInputRefs,
2024 staticSkyModelOutputRefs)
2025 if templateCoadd
is None:
2028 return pipeBase.Struct(templateCoadd=templateCoadd.coaddExposure,
2029 nImage=templateCoadd.nImage,
2030 warpRefList=templateCoadd.warpRefList,
2031 imageScalerList=templateCoadd.imageScalerList,
2032 weightList=templateCoadd.weightList)
2034 @utils.inheritDoc(AssembleCoaddTask)
2037 Generate a templateCoadd to use as a naive model of static sky to 2038 subtract from PSF-Matched warps. 2042 result : `lsst.pipe.base.Struct` 2043 Result struct with components: 2045 - ``templateCoadd``: coadded exposure (``lsst.afw.image.Exposure``) 2046 - ``nImage``: N Image (``lsst.afw.image.Image``) 2048 templateCoadd = self.assembleStaticSkyModel.runDataRef(dataRef, selectDataList, warpRefList)
2049 if templateCoadd
is None:
2052 return pipeBase.Struct(templateCoadd=templateCoadd.coaddExposure,
2053 nImage=templateCoadd.nImage,
2054 warpRefList=templateCoadd.warpRefList,
2055 imageScalerList=templateCoadd.imageScalerList,
2056 weightList=templateCoadd.weightList)
2058 def _noTemplateMessage(self, warpType):
2059 warpName = (warpType[0].upper() + warpType[1:])
2060 message =
"""No %(warpName)s warps were found to build the template coadd which is 2061 required to run CompareWarpAssembleCoaddTask. To continue assembling this type of coadd, 2062 first either rerun makeCoaddTempExp with config.make%(warpName)s=True or 2063 coaddDriver with config.makeCoadTempExp.make%(warpName)s=True, before assembleCoadd. 2065 Alternatively, to use another algorithm with existing warps, retarget the CoaddDriverConfig to 2066 another algorithm like: 2068 from lsst.pipe.tasks.assembleCoadd import SafeClipAssembleCoaddTask 2069 config.assemble.retarget(SafeClipAssembleCoaddTask) 2070 """ % {
"warpName": warpName}
2073 @utils.inheritDoc(AssembleCoaddTask)
2074 def run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
2075 supplementaryData, *args, **kwargs):
2076 """Assemble the coadd. 2078 Find artifacts and apply them to the warps' masks creating a list of 2079 alternative masks with a new "CLIPPED" plane and updated "NO_DATA" 2080 plane. Then pass these alternative masks to the base class's `run` 2083 The input parameters ``supplementaryData`` is a `lsst.pipe.base.Struct` 2084 that must contain a ``templateCoadd`` that serves as the 2085 model of the static sky. 2091 if isinstance(tempExpRefList[0], DeferredDatasetHandle):
2092 dataIds = [ref.datasetRefOrType.dataId
for ref
in tempExpRefList]
2093 psfMatchedDataIds = [ref.datasetRefOrType.dataId
for ref
in supplementaryData.warpRefList]
2094 if dataIds != psfMatchedDataIds:
2095 self.log.info(
"Reordering and or/padding PSF-matched visit input list")
2096 supplementaryData.warpRefList =
reorderAndPadList(supplementaryData.warpRefList,
2097 psfMatchedDataIds, dataIds)
2098 supplementaryData.imageScalerList =
reorderAndPadList(supplementaryData.imageScalerList,
2099 psfMatchedDataIds, dataIds)
2102 spanSetMaskList = self.
findArtifacts(supplementaryData.templateCoadd,
2103 supplementaryData.warpRefList,
2104 supplementaryData.imageScalerList)
2106 badMaskPlanes = self.config.badMaskPlanes[:]
2107 badMaskPlanes.append(
"CLIPPED")
2108 badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes)
2110 result = AssembleCoaddTask.run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
2111 spanSetMaskList, mask=badPixelMask)
2115 self.
applyAltEdgeMask(result.coaddExposure.maskedImage.mask, spanSetMaskList)
2119 """Propagate alt EDGE mask to SENSOR_EDGE AND INEXACT_PSF planes. 2123 mask : `lsst.afw.image.Mask` 2125 altMaskList : `list` 2126 List of Dicts containing ``spanSet`` lists. 2127 Each element contains the new mask plane name (e.g. "CLIPPED 2128 and/or "NO_DATA") as the key, and list of ``SpanSets`` to apply to 2131 maskValue = mask.getPlaneBitMask([
"SENSOR_EDGE",
"INEXACT_PSF"])
2132 for visitMask
in altMaskList:
2133 if "EDGE" in visitMask:
2134 for spanSet
in visitMask[
'EDGE']:
2135 spanSet.clippedTo(mask.getBBox()).setMask(mask, maskValue)
2140 Loop through warps twice. The first loop builds a map with the count 2141 of how many epochs each pixel deviates from the templateCoadd by more 2142 than ``config.chiThreshold`` sigma. The second loop takes each 2143 difference image and filters the artifacts detected in each using 2144 count map to filter out variable sources and sources that are 2145 difficult to subtract cleanly. 2149 templateCoadd : `lsst.afw.image.Exposure` 2150 Exposure to serve as model of static sky. 2151 tempExpRefList : `list` 2152 List of data references to warps. 2153 imageScalerList : `list` 2154 List of image scalers. 2159 List of dicts containing information about CLIPPED 2160 (i.e., artifacts), NO_DATA, and EDGE pixels. 2163 self.log.debug(
"Generating Count Image, and mask lists.")
2164 coaddBBox = templateCoadd.getBBox()
2165 slateIm = afwImage.ImageU(coaddBBox)
2166 epochCountImage = afwImage.ImageU(coaddBBox)
2167 nImage = afwImage.ImageU(coaddBBox)
2168 spanSetArtifactList = []
2169 spanSetNoDataMaskList = []
2170 spanSetEdgeList = []
2171 badPixelMask = self.getBadPixelMask()
2174 templateCoadd.mask.clearAllMaskPlanes()
2176 if self.config.doPreserveContainedBySource:
2177 templateFootprints = self.detectTemplate.detectFootprints(templateCoadd)
2179 templateFootprints =
None 2181 for warpRef, imageScaler
in zip(tempExpRefList, imageScalerList):
2183 if warpDiffExp
is not None:
2185 nImage.array += (numpy.isfinite(warpDiffExp.image.array) *
2186 ((warpDiffExp.mask.array & badPixelMask) == 0)).astype(numpy.uint16)
2187 fpSet = self.detect.detectFootprints(warpDiffExp, doSmooth=
False, clearMask=
True)
2188 fpSet.positive.merge(fpSet.negative)
2189 footprints = fpSet.positive
2191 spanSetList = [footprint.spans
for footprint
in footprints.getFootprints()]
2194 if self.config.doPrefilterArtifacts:
2196 for spans
in spanSetList:
2197 spans.setImage(slateIm, 1, doClip=
True)
2198 epochCountImage += slateIm
2204 nans = numpy.where(numpy.isnan(warpDiffExp.maskedImage.image.array), 1, 0)
2205 nansMask = afwImage.makeMaskFromArray(nans.astype(afwImage.MaskPixel))
2206 nansMask.setXY0(warpDiffExp.getXY0())
2207 edgeMask = warpDiffExp.mask
2208 spanSetEdgeMask = afwGeom.SpanSet.fromMask(edgeMask,
2209 edgeMask.getPlaneBitMask(
"EDGE")).split()
2213 nansMask = afwImage.MaskX(coaddBBox, 1)
2215 spanSetEdgeMask = []
2217 spanSetNoDataMask = afwGeom.SpanSet.fromMask(nansMask).split()
2219 spanSetNoDataMaskList.append(spanSetNoDataMask)
2220 spanSetArtifactList.append(spanSetList)
2221 spanSetEdgeList.append(spanSetEdgeMask)
2225 epochCountImage.writeFits(path)
2227 for i, spanSetList
in enumerate(spanSetArtifactList):
2229 filteredSpanSetList = self.
filterArtifacts(spanSetList, epochCountImage, nImage,
2231 spanSetArtifactList[i] = filteredSpanSetList
2234 for artifacts, noData, edge
in zip(spanSetArtifactList, spanSetNoDataMaskList, spanSetEdgeList):
2235 altMasks.append({
'CLIPPED': artifacts,
2241 """Remove artifact candidates covered by bad mask plane. 2243 Any future editing of the candidate list that does not depend on 2244 temporal information should go in this method. 2248 spanSetList : `list` 2249 List of SpanSets representing artifact candidates. 2250 exp : `lsst.afw.image.Exposure` 2251 Exposure containing mask planes used to prefilter. 2255 returnSpanSetList : `list` 2256 List of SpanSets with artifacts. 2258 badPixelMask = exp.mask.getPlaneBitMask(self.config.prefilterArtifactsMaskPlanes)
2259 goodArr = (exp.mask.array & badPixelMask) == 0
2260 returnSpanSetList = []
2261 bbox = exp.getBBox()
2262 x0, y0 = exp.getXY0()
2263 for i, span
in enumerate(spanSetList):
2264 y, x = span.clippedTo(bbox).indices()
2265 yIndexLocal = numpy.array(y) - y0
2266 xIndexLocal = numpy.array(x) - x0
2267 goodRatio = numpy.count_nonzero(goodArr[yIndexLocal, xIndexLocal])/span.getArea()
2268 if goodRatio > self.config.prefilterArtifactsRatio:
2269 returnSpanSetList.append(span)
2270 return returnSpanSetList
2272 def filterArtifacts(self, spanSetList, epochCountImage, nImage, footprintsToExclude=None):
2273 """Filter artifact candidates. 2277 spanSetList : `list` 2278 List of SpanSets representing artifact candidates. 2279 epochCountImage : `lsst.afw.image.Image` 2280 Image of accumulated number of warpDiff detections. 2281 nImage : `lsst.afw.image.Image` 2282 Image of the accumulated number of total epochs contributing. 2286 maskSpanSetList : `list` 2287 List of SpanSets with artifacts. 2290 maskSpanSetList = []
2291 x0, y0 = epochCountImage.getXY0()
2292 for i, span
in enumerate(spanSetList):
2293 y, x = span.indices()
2294 yIdxLocal = [y1 - y0
for y1
in y]
2295 xIdxLocal = [x1 - x0
for x1
in x]
2296 outlierN = epochCountImage.array[yIdxLocal, xIdxLocal]
2297 totalN = nImage.array[yIdxLocal, xIdxLocal]
2300 effMaxNumEpochsHighN = (self.config.maxNumEpochs +
2301 self.config.maxFractionEpochsHigh*numpy.mean(totalN))
2302 effMaxNumEpochsLowN = self.config.maxFractionEpochsLow * numpy.mean(totalN)
2303 effectiveMaxNumEpochs = int(min(effMaxNumEpochsLowN, effMaxNumEpochsHighN))
2304 nPixelsBelowThreshold = numpy.count_nonzero((outlierN > 0) &
2305 (outlierN <= effectiveMaxNumEpochs))
2306 percentBelowThreshold = nPixelsBelowThreshold / len(outlierN)
2307 if percentBelowThreshold > self.config.spatialThreshold:
2308 maskSpanSetList.append(span)
2310 if self.config.doPreserveContainedBySource
and footprintsToExclude
is not None:
2312 filteredMaskSpanSetList = []
2313 for span
in maskSpanSetList:
2315 for footprint
in footprintsToExclude.positive.getFootprints():
2316 if footprint.spans.contains(span):
2320 filteredMaskSpanSetList.append(span)
2321 maskSpanSetList = filteredMaskSpanSetList
2323 return maskSpanSetList
2325 def _readAndComputeWarpDiff(self, warpRef, imageScaler, templateCoadd):
2326 """Fetch a warp from the butler and return a warpDiff. 2330 warpRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef` 2331 Butler dataRef for the warp. 2332 imageScaler : `lsst.pipe.tasks.scaleZeroPoint.ImageScaler` 2333 An image scaler object. 2334 templateCoadd : `lsst.afw.image.Exposure` 2335 Exposure to be substracted from the scaled warp. 2339 warp : `lsst.afw.image.Exposure` 2340 Exposure of the image difference between the warp and template. 2348 warpName = self.getTempExpDatasetName(
'psfMatched')
2349 if not isinstance(warpRef, DeferredDatasetHandle):
2350 if not warpRef.datasetExists(warpName):
2351 self.log.warn(
"Could not find %s %s; skipping it", warpName, warpRef.dataId)
2353 warp = warpRef.get(datasetType=warpName, immediate=
True)
2355 imageScaler.scaleMaskedImage(warp.getMaskedImage())
2356 mi = warp.getMaskedImage()
2357 if self.config.doScaleWarpVariance:
2359 self.scaleWarpVariance.
run(mi)
2360 except Exception
as exc:
2361 self.log.warn(
"Unable to rescale variance of warp (%s); leaving it as-is" % (exc,))
2362 mi -= templateCoadd.getMaskedImage()
2365 def _dataRef2DebugPath(self, prefix, warpRef, coaddLevel=False):
2366 """Return a path to which to write debugging output. 2368 Creates a hyphen-delimited string of dataId values for simple filenames. 2373 Prefix for filename. 2374 warpRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef` 2375 Butler dataRef to make the path from. 2376 coaddLevel : `bool`, optional. 2377 If True, include only coadd-level keys (e.g., 'tract', 'patch', 2378 'filter', but no 'visit'). 2383 Path for debugging output. 2386 keys = warpRef.getButler().getKeys(self.getCoaddDatasetName(self.warpType))
2388 keys = warpRef.dataId.keys()
2389 keyList = sorted(keys, reverse=
True)
2391 filename =
"%s-%s.fits" % (prefix,
'-'.join([str(warpRef.dataId[k])
for k
in keyList]))
2392 return os.path.join(directory, filename)
2396 """Match the order of one list to another, padding if necessary 2401 List to be reordered and padded 2402 inputKey : Any value that can be compared with outputKey 2403 outputKey : Any value that can be compared with inputKey 2404 padWith : Any value to be inserted where inputKey not in outputKeys 2409 Copy of inputList reordered and padded with `padWith` to match outputList 2414 outputList.append(inputList[inputKey.index(d)])
2416 outputList.append(padWith)
def makeSupplementaryData(self, dataRef, selectDataList=None, warpRefList=None)
def getTempExpRefList(self, patchRef, calExpRefList)
def shrinkValidPolygons(self, coaddInputs)
def setBrightObjectMasks(self, exposure, brightObjectMasks, dataId=None)
def __init__(self, config=None)
def _dataRef2DebugPath(self, prefix, warpRef, coaddLevel=False)
def getGroupDataRef(butler, datasetType, groupTuple, keys)
def processResults(self, coaddExposure, brightObjectMasks=None, dataId=None)
Base class for coaddition.
def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList)
def setInexactPsf(self, mask)
def prepareStats(self, mask=None)
def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs)
def makeSkyInfo(skyMap, tractId, patchId)
def _readAndComputeWarpDiff(self, warpRef, imageScaler, templateCoadd)
def readBrightObjectMasks(self, dataRef)
def applyAltMaskPlanes(self, mask, altMaskSpans)
def makeCoaddSuffix(warpType="direct")
def assembleSubregion(self, coaddExposure, bbox, tempExpRefList, imageScalerList, weightList, altMaskList, statsFlags, statsCtrl, nImage=None)
def prepareInputs(self, refList)
def __init__(self, args, kwargs)
def makeDataRefList(self, namespace)
def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs)
def makeSupplementaryData(self, dataRef, selectDataList=None, warpRefList=None)
def detectClip(self, exp, tempExpRefList)
def filterArtifacts(self, spanSetList, epochCountImage, nImage, footprintsToExclude=None)
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, args, kwargs)
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, supplementaryData, args, kwargs)
def buildDifferenceImage(self, skyInfo, tempExpRefList, imageScalerList, weightList)
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, altMaskList=None, mask=None, supplementaryData=None)
def _noTemplateMessage(self, warpType)
def removeMaskPlanes(self, maskedImage)
def applyAltEdgeMask(self, mask, altMaskList)
def reorderAndPadList(inputList, inputKey, outputKey, padWith=None)
def __init__(self, args, kwargs)
def prefilterArtifacts(self, spanSetList, exp)
def assembleMetadata(self, coaddExposure, tempExpRefList, weightList)
def countMaskFromFootprint(mask, footprint, bitmask, ignoreMask)
def groupPatchExposures(patchDataRef, calexpDataRefList, coaddDatasetType="deepCoadd", tempExpDatasetType="deepCoadd_directWarp")
def detectClipBig(self, clipList, clipFootprints, clipIndices, detectionFootprints, maskClipValue, maskDetValue, coaddBBox)