22__all__ = [
"MakeWarpTask",
"MakeWarpConfig"]
30import lsst.pipe.base
as pipeBase
31import lsst.pipe.base.connectionTypes
as connectionTypes
32import lsst.utils
as utils
34from lsst.daf.butler
import DeferredDatasetHandle
38from lsst.utils.timer
import timeMethod
39from .coaddBase
import CoaddBaseTask, makeSkyInfo, reorderAndPadList
40from .warpAndPsfMatch
import WarpAndPsfMatchTask
41from collections.abc
import Iterable
43log = logging.getLogger(__name__)
47 dimensions=(
"tract",
"patch",
"skymap",
"instrument",
"visit"),
48 defaultTemplates={
"coaddName":
"deep",
49 "skyWcsName":
"gbdesAstrometricFit",
50 "photoCalibName":
"fgcm",
52 calExpList = connectionTypes.Input(
53 doc=
"Input exposures to be resampled and optionally PSF-matched onto a SkyMap projection/patch",
54 name=
"{calexpType}calexp",
55 storageClass=
"ExposureF",
56 dimensions=(
"instrument",
"visit",
"detector"),
60 backgroundList = connectionTypes.Input(
61 doc=
"Input backgrounds to be added back into the calexp if bgSubtracted=False",
62 name=
"calexpBackground",
63 storageClass=
"Background",
64 dimensions=(
"instrument",
"visit",
"detector"),
67 skyCorrList = connectionTypes.Input(
68 doc=
"Input Sky Correction to be subtracted from the calexp if doApplySkyCorr=True",
70 storageClass=
"Background",
71 dimensions=(
"instrument",
"visit",
"detector"),
74 skyMap = connectionTypes.Input(
75 doc=
"Input definition of geometry/bbox and projection/wcs for warped exposures",
76 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
77 storageClass=
"SkyMap",
78 dimensions=(
"skymap",),
80 direct = connectionTypes.Output(
81 doc=(
"Output direct warped exposure (previously called CoaddTempExp), produced by resampling ",
82 "calexps onto the skyMap patch geometry."),
83 name=
"{coaddName}Coadd_directWarp",
84 storageClass=
"ExposureF",
85 dimensions=(
"tract",
"patch",
"skymap",
"visit",
"instrument"),
87 psfMatched = connectionTypes.Output(
88 doc=(
"Output PSF-Matched warped exposure (previously called CoaddTempExp), produced by resampling ",
89 "calexps onto the skyMap patch geometry and PSF-matching to a model PSF."),
90 name=
"{coaddName}Coadd_psfMatchedWarp",
91 storageClass=
"ExposureF",
92 dimensions=(
"tract",
"patch",
"skymap",
"visit",
"instrument"),
94 visitSummary = connectionTypes.Input(
95 doc=
"Input visit-summary catalog with updated calibration objects.",
96 name=
"finalVisitSummary",
97 storageClass=
"ExposureCatalog",
98 dimensions=(
"instrument",
"visit",),
101 def __init__(self, *, config=None):
102 if config.bgSubtracted:
103 del self.backgroundList
104 if not config.doApplySkyCorr:
106 if not config.makeDirect:
108 if not config.makePsfMatched:
112class MakeWarpConfig(pipeBase.PipelineTaskConfig, CoaddBaseTask.ConfigClass,
113 pipelineConnections=MakeWarpConnections):
114 """Config for MakeWarpTask."""
116 warpAndPsfMatch = pexConfig.ConfigurableField(
117 target=WarpAndPsfMatchTask,
118 doc=
"Task to warp and PSF-match calexp",
120 doWrite = pexConfig.Field(
121 doc=
"persist <coaddName>Coadd_<warpType>Warp",
125 bgSubtracted = pexConfig.Field(
126 doc=
"Work with a background subtracted calexp?",
130 coaddPsf = pexConfig.ConfigField(
131 doc=
"Configuration for CoaddPsf",
132 dtype=CoaddPsfConfig,
134 makeDirect = pexConfig.Field(
135 doc=
"Make direct Warp/Coadds",
139 makePsfMatched = pexConfig.Field(
140 doc=
"Make Psf-Matched Warp/Coadd?",
144 useVisitSummaryPsf = pexConfig.Field(
146 "If True, use the PSF model and aperture corrections from the 'visitSummary' connection. "
147 "If False, use the PSF model and aperture corrections from the 'exposure' connection. "
152 doWriteEmptyWarps = pexConfig.Field(
155 doc=
"Write out warps even if they are empty"
157 hasFakes = pexConfig.Field(
158 doc=
"Should be set to True if fake sources have been inserted into the input data.",
162 doApplySkyCorr = pexConfig.Field(
165 doc=
"Apply sky correction?",
167 idGenerator = DetectorVisitIdGeneratorConfig.make_field()
170 CoaddBaseTask.ConfigClass.validate(self)
172 if not self.makePsfMatched
and not self.makeDirect:
173 raise RuntimeError(
"At least one of config.makePsfMatched and config.makeDirect must be True")
176 log.warning(
"Config doPsfMatch deprecated. Setting makePsfMatched=True and makeDirect=False")
177 self.makePsfMatched =
True
178 self.makeDirect =
False
180 def setDefaults(self):
181 CoaddBaseTask.ConfigClass.setDefaults(self)
182 self.warpAndPsfMatch.psfMatch.kernel.active.kernelSize = self.matchingKernelSize
186 """Warp and optionally PSF-Match calexps onto an a common projection.
188 Warp and optionally PSF-Match calexps onto a common projection, by
189 performing the following operations:
190 - Group calexps by visit/run
191 - For each visit, generate a Warp by calling method @ref run.
192 `run` loops over the visit's calexps calling
193 `~lsst.pipe.tasks.warpAndPsfMatch.WarpAndPsfMatchTask` on each visit
196 ConfigClass = MakeWarpConfig
197 _DefaultName =
"makeWarp"
199 def __init__(self, **kwargs):
200 CoaddBaseTask.__init__(self, **kwargs)
201 self.makeSubtask(
"warpAndPsfMatch")
202 if self.config.hasFakes:
203 self.calexpType =
"fakes_calexp"
205 self.calexpType =
"calexp"
207 @utils.inheritDoc(pipeBase.PipelineTask)
208 def runQuantum(self, butlerQC, inputRefs, outputRefs):
212 Obtain the list of input detectors from calExpList. Sort them by
213 detector order (to ensure reproducibility). Then ensure all input
214 lists are in the same sorted detector order.
216 detectorOrder = [ref.datasetRef.dataId[
'detector']
for ref
in inputRefs.calExpList]
218 inputRefs = reorderRefs(inputRefs, detectorOrder, dataIdKey=
'detector')
221 inputs = butlerQC.get(inputRefs)
225 skyMap = inputs.pop(
"skyMap")
226 quantumDataId = butlerQC.quantum.dataId
227 skyInfo = makeSkyInfo(skyMap, tractId=quantumDataId[
'tract'], patchId=quantumDataId[
'patch'])
230 dataIdList = [ref.datasetRef.dataId
for ref
in inputRefs.calExpList]
233 self.config.idGenerator.apply(dataId).catalog_id
234 for dataId
in dataIdList
238 visitSummary = inputs[
"visitSummary"]
241 for dataId
in dataIdList:
242 row = visitSummary.find(dataId[
"detector"])
245 f
"Unexpectedly incomplete visitSummary provided to makeWarp: {dataId} is missing."
247 bboxList.append(row.getBBox())
248 wcsList.append(row.getWcs())
249 inputs[
"bboxList"] = bboxList
250 inputs[
"wcsList"] = wcsList
254 completeIndices = self._prepareCalibratedExposures(**inputs)
255 inputs = self.filterInputs(indices=completeIndices, inputs=inputs)
261 coordList = [skyInfo.wcs.pixelToSky(pos)
for pos
in cornerPosList]
262 goodIndices = self.select.run(**inputs, coordList=coordList, dataIds=dataIdList)
263 inputs = self.filterInputs(indices=goodIndices, inputs=inputs)
266 visitId = dataIdList[0][
"visit"]
268 results = self.run(**inputs,
270 ccdIdList=[ccdIdList[i]
for i
in goodIndices],
271 dataIdList=[dataIdList[i]
for i
in goodIndices],
273 if self.config.makeDirect
and results.exposures[
"direct"]
is not None:
274 butlerQC.put(results.exposures[
"direct"], outputRefs.direct)
275 if self.config.makePsfMatched
and results.exposures[
"psfMatched"]
is not None:
276 butlerQC.put(results.exposures[
"psfMatched"], outputRefs.psfMatched)
279 def run(self, calExpList, ccdIdList, skyInfo, visitId=0, dataIdList=None, **kwargs):
280 """Create a Warp from inputs.
282 We iterate over the multiple calexps in a single exposure to construct
283 the warp (previously called a coaddTempExp) of that exposure to the
284 supplied tract/patch.
286 Pixels that receive no pixels are set to NAN; this is not correct
287 (violates LSST algorithms group policy), but will be fixed up by
288 interpolating after the coaddition.
290 calexpRefList : `list`
291 List of data references for calexps that (may)
292 overlap the patch of interest.
293 skyInfo : `lsst.pipe.base.Struct`
294 Struct from `~lsst.pipe.base.coaddBase.makeSkyInfo()` with
295 geometric information about the patch.
297 Integer identifier for visit, for the table that will
298 produce the CoaddPsf.
302 result : `lsst.pipe.base.Struct`
303 Results as a struct with attributes:
306 A dictionary containing the warps requested:
307 "direct": direct warp if ``config.makeDirect``
308 "psfMatched": PSF-matched warp if ``config.makePsfMatched``
311 warpTypeList = self.getWarpTypeList()
313 totGoodPix = {warpType: 0
for warpType
in warpTypeList}
314 didSetMetadata = {warpType:
False for warpType
in warpTypeList}
315 warps = {warpType: self._prepareEmptyExposure(skyInfo)
for warpType
in warpTypeList}
316 inputRecorder = {warpType: self.inputRecorder.makeCoaddTempExpRecorder(visitId, len(calExpList))
317 for warpType
in warpTypeList}
319 modelPsf = self.config.modelPsf.apply()
if self.config.makePsfMatched
else None
320 if dataIdList
is None:
321 dataIdList = ccdIdList
323 for calExpInd, (calExp, ccdId, dataId)
in enumerate(zip(calExpList, ccdIdList, dataIdList)):
324 self.log.info(
"Processing calexp %d of %d for this Warp: id=%s",
325 calExpInd+1, len(calExpList), dataId)
327 warpedAndMatched = self.warpAndPsfMatch.run(calExp, modelPsf=modelPsf,
328 wcs=skyInfo.wcs, maxBBox=skyInfo.bbox,
329 makeDirect=self.config.makeDirect,
330 makePsfMatched=self.config.makePsfMatched)
331 except Exception
as e:
332 self.log.warning(
"WarpAndPsfMatch failed for calexp %s; skipping it: %s", dataId, e)
335 numGoodPix = {warpType: 0
for warpType
in warpTypeList}
336 for warpType
in warpTypeList:
337 exposure = warpedAndMatched.getDict()[warpType]
340 warp = warps[warpType]
341 if didSetMetadata[warpType]:
342 mimg = exposure.getMaskedImage()
343 mimg *= (warp.getPhotoCalib().getInstFluxAtZeroMagnitude()
344 / exposure.getPhotoCalib().getInstFluxAtZeroMagnitude())
346 numGoodPix[warpType] = coaddUtils.copyGoodPixels(
347 warp.getMaskedImage(), exposure.getMaskedImage(), self.getBadPixelMask())
348 totGoodPix[warpType] += numGoodPix[warpType]
349 self.log.debug(
"Calexp %s has %d good pixels in this patch (%.1f%%) for %s",
350 dataId, numGoodPix[warpType],
351 100.0*numGoodPix[warpType]/skyInfo.bbox.getArea(), warpType)
352 if numGoodPix[warpType] > 0
and not didSetMetadata[warpType]:
353 warp.info.id = exposure.info.id
354 warp.setPhotoCalib(exposure.getPhotoCalib())
355 warp.setFilter(exposure.getFilter())
356 warp.getInfo().setVisitInfo(exposure.getInfo().getVisitInfo())
359 warp.setPsf(exposure.getPsf())
360 didSetMetadata[warpType] =
True
364 inputRecorder[warpType].addCalExp(calExp, ccdId, numGoodPix[warpType])
366 except Exception
as e:
367 self.log.warning(
"Error processing calexp %s; skipping it: %s", dataId, e)
370 for warpType
in warpTypeList:
371 self.log.info(
"%sWarp has %d good pixels (%.1f%%)",
372 warpType, totGoodPix[warpType], 100.0*totGoodPix[warpType]/skyInfo.bbox.getArea())
374 if totGoodPix[warpType] > 0
and didSetMetadata[warpType]:
375 inputRecorder[warpType].finish(warps[warpType], totGoodPix[warpType])
376 if warpType ==
"direct":
377 warps[warpType].setPsf(
378 CoaddPsf(inputRecorder[warpType].coaddInputs.ccds, skyInfo.wcs,
379 self.config.coaddPsf.makeControl()))
381 if not self.config.doWriteEmptyWarps:
383 warps[warpType] =
None
387 result = pipeBase.Struct(exposures=warps)
390 def filterInputs(self, indices, inputs):
391 """Filter task inputs by their indices.
395 indices : `list` [`int`]
396 inputs : `dict` [`list`]
397 A dictionary of input connections to be passed to run.
401 inputs : `dict` [`list`]
402 Task inputs with their lists filtered by indices.
404 for key
in inputs.keys():
406 if isinstance(inputs[key], list):
407 inputs[key] = [inputs[key][ind]
for ind
in indices]
410 def _prepareCalibratedExposures(self, *, visitSummary, calExpList=[], wcsList=None,
411 backgroundList=None, skyCorrList=None, **kwargs):
412 """Calibrate and add backgrounds to input calExpList in place.
416 visitSummary : `lsst.afw.table.ExposureCatalog`
417 Exposure catalog with potentially all calibrations. Attributes set
418 to `None` are ignored.
419 calExpList : `list` [`lsst.afw.image.Exposure` or
420 `lsst.daf.butler.DeferredDatasetHandle`]
421 Sequence of calexps to be modified in place.
422 wcsList : `list` [`lsst.afw.geom.SkyWcs`]
423 The WCSs of the calexps in ``calExpList``. These will be used to
424 determine if the calexp should be used in the warp. The list is
425 dynamically updated with the WCSs from the visitSummary.
426 backgroundList : `list` [`lsst.afw.math.backgroundList`], optional
427 Sequence of backgrounds to be added back in if bgSubtracted=False.
428 skyCorrList : `list` [`lsst.afw.math.backgroundList`], optional
429 Sequence of background corrections to be subtracted if
432 Additional keyword arguments.
436 indices : `list` [`int`]
437 Indices of ``calExpList`` and friends that have valid
440 wcsList = len(calExpList)*[
None]
if wcsList
is None else wcsList
441 backgroundList = len(calExpList)*[
None]
if backgroundList
is None else backgroundList
442 skyCorrList = len(calExpList)*[
None]
if skyCorrList
is None else skyCorrList
444 includeCalibVar = self.config.includeCalibVar
447 for index, (calexp, background, skyCorr)
in enumerate(zip(calExpList,
450 if isinstance(calexp, DeferredDatasetHandle):
451 calexp = calexp.get()
453 if not self.config.bgSubtracted:
454 calexp.maskedImage += background.getImage()
456 detectorId = calexp.info.getDetector().getId()
459 row = visitSummary.find(detectorId)
462 f
"Unexpectedly incomplete visitSummary: detector={detectorId} is missing."
464 if (photoCalib := row.getPhotoCalib())
is not None:
465 calexp.setPhotoCalib(photoCalib)
468 "Detector id %d for visit %d has None for photoCalib in the visitSummary and will "
469 "not be used in the warp", detectorId, row[
"visit"],
472 if (skyWcs := row.getWcs())
is not None:
473 calexp.setWcs(skyWcs)
474 wcsList[index] = skyWcs
477 "Detector id %d for visit %d has None for wcs in the visitSummary and will "
478 "not be used in the warp", detectorId, row[
"visit"],
481 if self.config.useVisitSummaryPsf:
482 if (psf := row.getPsf())
is not None:
486 "Detector id %d for visit %d has None for psf in the visitSummary and will "
487 "not be used in the warp", detectorId, row[
"visit"],
490 if (apCorrMap := row.getApCorrMap())
is not None:
491 calexp.info.setApCorrMap(apCorrMap)
494 "Detector id %d for visit %d has None for apCorrMap in the visitSummary and will "
495 "not be used in the warp", detectorId, row[
"visit"],
499 if calexp.getPsf()
is None:
501 "Detector id %d for visit %d has None for psf for the calexp and will "
502 "not be used in the warp", detectorId, row[
"visit"],
505 if calexp.info.getApCorrMap()
is None:
507 "Detector id %d for visit %d has None for apCorrMap in the calexp and will "
508 "not be used in the warp", detectorId, row[
"visit"],
513 calexp.maskedImage = photoCalib.calibrateImage(calexp.maskedImage,
514 includeScaleUncertainty=includeCalibVar)
515 calexp.maskedImage /= photoCalib.getCalibrationMean()
521 if self.config.doApplySkyCorr:
522 calexp.maskedImage -= skyCorr.getImage()
524 indices.append(index)
525 calExpList[index] = calexp
530 def _prepareEmptyExposure(skyInfo):
531 """Produce an empty exposure for a given patch.
535 skyInfo : `lsst.pipe.base.Struct`
536 Struct from `~lsst.pipe.base.coaddBase.makeSkyInfo()` with
537 geometric information about the patch.
541 exp : `lsst.afw.image.exposure.ExposureF`
542 An empty exposure for a given patch.
544 exp = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
545 exp.getMaskedImage().set(numpy.nan, afwImage.Mask
546 .getPlaneBitMask(
"NO_DATA"), numpy.inf)
549 def getWarpTypeList(self):
550 """Return list of requested warp types per the config.
553 if self.config.makeDirect:
554 warpTypeList.append(
"direct")
555 if self.config.makePsfMatched:
556 warpTypeList.append(
"psfMatched")
560def reorderRefs(inputRefs, outputSortKeyOrder, dataIdKey):
561 """Reorder inputRefs per outputSortKeyOrder.
563 Any inputRefs which are lists will be resorted per specified key e.g.,
564 'detector.' Only iterables will be reordered, and values can be of type
565 `lsst.pipe.base.connections.DeferredDatasetRef` or
566 `lsst.daf.butler.core.datasets.ref.DatasetRef`.
568 Returned lists of refs have the same length as the outputSortKeyOrder.
569 If an outputSortKey not in the inputRef, then it will be padded with None.
570 If an inputRef contains an inputSortKey that is not in the
571 outputSortKeyOrder it will be removed.
575 inputRefs : `lsst.pipe.base.connections.QuantizedConnection`
576 Input references to be reordered and padded.
577 outputSortKeyOrder : `iterable`
578 Iterable of values to be compared with inputRef's dataId[dataIdKey].
580 The data ID key in the dataRefs to compare with the outputSortKeyOrder.
584 inputRefs : `lsst.pipe.base.connections.QuantizedConnection`
585 Quantized Connection with sorted DatasetRef values sorted if iterable.
587 for connectionName, refs
in inputRefs:
588 if isinstance(refs, Iterable):
589 if hasattr(refs[0],
"dataId"):
590 inputSortKeyOrder = [ref.dataId[dataIdKey]
for ref
in refs]
592 inputSortKeyOrder = [ref.datasetRef.dataId[dataIdKey]
for ref
in refs]
593 if inputSortKeyOrder != outputSortKeyOrder:
594 setattr(inputRefs, connectionName,
595 reorderAndPadList(refs, inputSortKeyOrder, outputSortKeyOrder))