22__all__ = [
"MakeWarpTask",
"MakeWarpConfig"]
30import lsst.pipe.base
as pipeBase
31import lsst.pipe.base.connectionTypes
as connectionTypes
32import lsst.utils
as utils
34from lsst.daf.butler
import DeferredDatasetHandle
38from lsst.utils.timer
import timeMethod
39from .coaddBase
import CoaddBaseTask, makeSkyInfo, reorderAndPadList
40from .warpAndPsfMatch
import WarpAndPsfMatchTask
41from collections.abc
import Iterable
43log = logging.getLogger(__name__)
47 dimensions=(
"tract",
"patch",
"skymap",
"instrument",
"visit"),
48 defaultTemplates={
"coaddName":
"deep",
50 calExpList = connectionTypes.Input(
51 doc=
"Input exposures to be resampled and optionally PSF-matched onto a SkyMap projection/patch",
52 name=
"{calexpType}calexp",
53 storageClass=
"ExposureF",
54 dimensions=(
"instrument",
"visit",
"detector"),
58 backgroundList = connectionTypes.Input(
59 doc=
"Input backgrounds to be added back into the calexp if bgSubtracted=False",
60 name=
"calexpBackground",
61 storageClass=
"Background",
62 dimensions=(
"instrument",
"visit",
"detector"),
65 skyCorrList = connectionTypes.Input(
66 doc=
"Input Sky Correction to be subtracted from the calexp if doApplySkyCorr=True",
68 storageClass=
"Background",
69 dimensions=(
"instrument",
"visit",
"detector"),
72 skyMap = connectionTypes.Input(
73 doc=
"Input definition of geometry/bbox and projection/wcs for warped exposures",
74 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
75 storageClass=
"SkyMap",
76 dimensions=(
"skymap",),
78 direct = connectionTypes.Output(
79 doc=(
"Output direct warped exposure (previously called CoaddTempExp), produced by resampling ",
80 "calexps onto the skyMap patch geometry."),
81 name=
"{coaddName}Coadd_directWarp",
82 storageClass=
"ExposureF",
83 dimensions=(
"tract",
"patch",
"skymap",
"visit",
"instrument"),
85 psfMatched = connectionTypes.Output(
86 doc=(
"Output PSF-Matched warped exposure (previously called CoaddTempExp), produced by resampling ",
87 "calexps onto the skyMap patch geometry and PSF-matching to a model PSF."),
88 name=
"{coaddName}Coadd_psfMatchedWarp",
89 storageClass=
"ExposureF",
90 dimensions=(
"tract",
"patch",
"skymap",
"visit",
"instrument"),
92 visitSummary = connectionTypes.Input(
93 doc=
"Input visit-summary catalog with updated calibration objects.",
94 name=
"finalVisitSummary",
95 storageClass=
"ExposureCatalog",
96 dimensions=(
"instrument",
"visit",),
99 def __init__(self, *, config=None):
100 if config.bgSubtracted:
101 del self.backgroundList
102 if not config.doApplySkyCorr:
104 if not config.makeDirect:
106 if not config.makePsfMatched:
110class MakeWarpConfig(pipeBase.PipelineTaskConfig, CoaddBaseTask.ConfigClass,
111 pipelineConnections=MakeWarpConnections):
112 """Config for MakeWarpTask."""
114 warpAndPsfMatch = pexConfig.ConfigurableField(
115 target=WarpAndPsfMatchTask,
116 doc=
"Task to warp and PSF-match calexp",
118 doWrite = pexConfig.Field(
119 doc=
"persist <coaddName>Coadd_<warpType>Warp",
123 bgSubtracted = pexConfig.Field(
124 doc=
"Work with a background subtracted calexp?",
128 coaddPsf = pexConfig.ConfigField(
129 doc=
"Configuration for CoaddPsf",
130 dtype=CoaddPsfConfig,
132 makeDirect = pexConfig.Field(
133 doc=
"Make direct Warp/Coadds",
137 makePsfMatched = pexConfig.Field(
138 doc=
"Make Psf-Matched Warp/Coadd?",
142 modelPsf = GaussianPsfFactory.makeField(doc=
"Model Psf factory")
143 useVisitSummaryPsf = pexConfig.Field(
145 "If True, use the PSF model and aperture corrections from the 'visitSummary' connection. "
146 "If False, use the PSF model and aperture corrections from the 'exposure' connection. "
151 doWriteEmptyWarps = pexConfig.Field(
154 doc=
"Write out warps even if they are empty"
156 hasFakes = pexConfig.Field(
157 doc=
"Should be set to True if fake sources have been inserted into the input data.",
161 doApplySkyCorr = pexConfig.Field(
164 doc=
"Apply sky correction?",
166 idGenerator = DetectorVisitIdGeneratorConfig.make_field()
169 CoaddBaseTask.ConfigClass.validate(self)
171 if not self.makePsfMatched
and not self.makeDirect:
172 raise RuntimeError(
"At least one of config.makePsfMatched and config.makeDirect must be True")
174 def setDefaults(self):
175 CoaddBaseTask.ConfigClass.setDefaults(self)
176 self.warpAndPsfMatch.psfMatch.kernel.active.kernelSize = self.matchingKernelSize
180 """Warp and optionally PSF-Match calexps onto an a common projection.
182 Warp and optionally PSF-Match calexps onto a common projection, by
183 performing the following operations:
184 - Group calexps by visit/run
185 - For each visit, generate a Warp by calling method @ref run.
186 `run` loops over the visit's calexps calling
187 `~lsst.pipe.tasks.warpAndPsfMatch.WarpAndPsfMatchTask` on each visit
190 ConfigClass = MakeWarpConfig
191 _DefaultName =
"makeWarp"
193 def __init__(self, **kwargs):
194 CoaddBaseTask.__init__(self, **kwargs)
195 self.makeSubtask(
"warpAndPsfMatch")
196 if self.config.hasFakes:
197 self.calexpType =
"fakes_calexp"
199 self.calexpType =
"calexp"
201 @utils.inheritDoc(pipeBase.PipelineTask)
202 def runQuantum(self, butlerQC, inputRefs, outputRefs):
206 Obtain the list of input detectors from calExpList. Sort them by
207 detector order (to ensure reproducibility). Then ensure all input
208 lists are in the same sorted detector order.
210 detectorOrder = [ref.datasetRef.dataId[
'detector']
for ref
in inputRefs.calExpList]
212 inputRefs = reorderRefs(inputRefs, detectorOrder, dataIdKey=
'detector')
215 inputs = butlerQC.get(inputRefs)
219 skyMap = inputs.pop(
"skyMap")
220 quantumDataId = butlerQC.quantum.dataId
221 skyInfo = makeSkyInfo(skyMap, tractId=quantumDataId[
'tract'], patchId=quantumDataId[
'patch'])
224 dataIdList = [ref.datasetRef.dataId
for ref
in inputRefs.calExpList]
227 self.config.idGenerator.apply(dataId).catalog_id
228 for dataId
in dataIdList
232 visitSummary = inputs[
"visitSummary"]
235 for dataId
in dataIdList:
236 row = visitSummary.find(dataId[
"detector"])
239 f
"Unexpectedly incomplete visitSummary provided to makeWarp: {dataId} is missing."
241 bboxList.append(row.getBBox())
242 wcsList.append(row.getWcs())
243 inputs[
"bboxList"] = bboxList
244 inputs[
"wcsList"] = wcsList
248 completeIndices = self._prepareCalibratedExposures(**inputs)
249 inputs = self.filterInputs(indices=completeIndices, inputs=inputs)
255 coordList = [skyInfo.wcs.pixelToSky(pos)
for pos
in cornerPosList]
256 goodIndices = self.select.run(**inputs, coordList=coordList, dataIds=dataIdList)
257 inputs = self.filterInputs(indices=goodIndices, inputs=inputs)
260 visitId = dataIdList[0][
"visit"]
262 results = self.run(**inputs,
264 ccdIdList=[ccdIdList[i]
for i
in goodIndices],
265 dataIdList=[dataIdList[i]
for i
in goodIndices],
267 if self.config.makeDirect
and results.exposures[
"direct"]
is not None:
268 butlerQC.put(results.exposures[
"direct"], outputRefs.direct)
269 if self.config.makePsfMatched
and results.exposures[
"psfMatched"]
is not None:
270 butlerQC.put(results.exposures[
"psfMatched"], outputRefs.psfMatched)
273 def run(self, calExpList, ccdIdList, skyInfo, visitId=0, dataIdList=None, **kwargs):
274 """Create a Warp from inputs.
276 We iterate over the multiple calexps in a single exposure to construct
277 the warp (previously called a coaddTempExp) of that exposure to the
278 supplied tract/patch.
280 Pixels that receive no pixels are set to NAN; this is not correct
281 (violates LSST algorithms group policy), but will be fixed up by
282 interpolating after the coaddition.
284 calexpRefList : `list`
285 List of data references for calexps that (may)
286 overlap the patch of interest.
287 skyInfo : `lsst.pipe.base.Struct`
288 Struct from `~lsst.pipe.base.coaddBase.makeSkyInfo()` with
289 geometric information about the patch.
291 Integer identifier for visit, for the table that will
292 produce the CoaddPsf.
296 result : `lsst.pipe.base.Struct`
297 Results as a struct with attributes:
300 A dictionary containing the warps requested:
301 "direct": direct warp if ``config.makeDirect``
302 "psfMatched": PSF-matched warp if ``config.makePsfMatched``
305 warpTypeList = self.getWarpTypeList()
307 totGoodPix = {warpType: 0
for warpType
in warpTypeList}
308 didSetMetadata = {warpType:
False for warpType
in warpTypeList}
309 warps = {warpType: self._prepareEmptyExposure(skyInfo)
for warpType
in warpTypeList}
310 inputRecorder = {warpType: self.inputRecorder.makeCoaddTempExpRecorder(visitId, len(calExpList))
311 for warpType
in warpTypeList}
313 modelPsf = self.config.modelPsf.apply()
if self.config.makePsfMatched
else None
314 if dataIdList
is None:
315 dataIdList = ccdIdList
317 for calExpInd, (calExp, ccdId, dataId)
in enumerate(zip(calExpList, ccdIdList, dataIdList)):
318 self.log.info(
"Processing calexp %d of %d for this Warp: id=%s",
319 calExpInd+1, len(calExpList), dataId)
321 warpedAndMatched = self.warpAndPsfMatch.run(calExp, modelPsf=modelPsf,
322 wcs=skyInfo.wcs, maxBBox=skyInfo.bbox,
323 makeDirect=self.config.makeDirect,
324 makePsfMatched=self.config.makePsfMatched)
325 except Exception
as e:
326 self.log.warning(
"WarpAndPsfMatch failed for calexp %s; skipping it: %s", dataId, e)
329 numGoodPix = {warpType: 0
for warpType
in warpTypeList}
330 for warpType
in warpTypeList:
331 exposure = warpedAndMatched.getDict()[warpType]
334 warp = warps[warpType]
335 if didSetMetadata[warpType]:
336 mimg = exposure.getMaskedImage()
337 mimg *= (warp.getPhotoCalib().getInstFluxAtZeroMagnitude()
338 / exposure.getPhotoCalib().getInstFluxAtZeroMagnitude())
340 numGoodPix[warpType] = coaddUtils.copyGoodPixels(
341 warp.getMaskedImage(), exposure.getMaskedImage(), self.getBadPixelMask())
342 totGoodPix[warpType] += numGoodPix[warpType]
343 self.log.debug(
"Calexp %s has %d good pixels in this patch (%.1f%%) for %s",
344 dataId, numGoodPix[warpType],
345 100.0*numGoodPix[warpType]/skyInfo.bbox.getArea(), warpType)
346 if numGoodPix[warpType] > 0
and not didSetMetadata[warpType]:
347 warp.info.id = exposure.info.id
348 warp.setPhotoCalib(exposure.getPhotoCalib())
349 warp.setFilter(exposure.getFilter())
350 warp.getInfo().setVisitInfo(exposure.getInfo().getVisitInfo())
353 warp.setPsf(exposure.getPsf())
354 didSetMetadata[warpType] =
True
358 inputRecorder[warpType].addCalExp(calExp, ccdId, numGoodPix[warpType])
360 except Exception
as e:
361 self.log.warning(
"Error processing calexp %s; skipping it: %s", dataId, e)
364 for warpType
in warpTypeList:
365 self.log.info(
"%sWarp has %d good pixels (%.1f%%)",
366 warpType, totGoodPix[warpType], 100.0*totGoodPix[warpType]/skyInfo.bbox.getArea())
368 if totGoodPix[warpType] > 0
and didSetMetadata[warpType]:
369 inputRecorder[warpType].finish(warps[warpType], totGoodPix[warpType])
370 if warpType ==
"direct":
371 warps[warpType].setPsf(
372 CoaddPsf(inputRecorder[warpType].coaddInputs.ccds, skyInfo.wcs,
373 self.config.coaddPsf.makeControl()))
375 if not self.config.doWriteEmptyWarps:
377 warps[warpType] =
None
381 result = pipeBase.Struct(exposures=warps)
384 def filterInputs(self, indices, inputs):
385 """Filter task inputs by their indices.
389 indices : `list` [`int`]
390 inputs : `dict` [`list`]
391 A dictionary of input connections to be passed to run.
395 inputs : `dict` [`list`]
396 Task inputs with their lists filtered by indices.
398 for key
in inputs.keys():
400 if isinstance(inputs[key], list):
401 inputs[key] = [inputs[key][ind]
for ind
in indices]
404 def _prepareCalibratedExposures(self, *, visitSummary, calExpList=[], wcsList=None,
405 backgroundList=None, skyCorrList=None, **kwargs):
406 """Calibrate and add backgrounds to input calExpList in place.
410 visitSummary : `lsst.afw.table.ExposureCatalog`
411 Exposure catalog with potentially all calibrations. Attributes set
412 to `None` are ignored.
413 calExpList : `list` [`lsst.afw.image.Exposure` or
414 `lsst.daf.butler.DeferredDatasetHandle`]
415 Sequence of calexps to be modified in place.
416 wcsList : `list` [`lsst.afw.geom.SkyWcs`]
417 The WCSs of the calexps in ``calExpList``. These will be used to
418 determine if the calexp should be used in the warp. The list is
419 dynamically updated with the WCSs from the visitSummary.
420 backgroundList : `list` [`lsst.afw.math.backgroundList`], optional
421 Sequence of backgrounds to be added back in if bgSubtracted=False.
422 skyCorrList : `list` [`lsst.afw.math.backgroundList`], optional
423 Sequence of background corrections to be subtracted if
426 Additional keyword arguments.
430 indices : `list` [`int`]
431 Indices of ``calExpList`` and friends that have valid
434 wcsList = len(calExpList)*[
None]
if wcsList
is None else wcsList
435 backgroundList = len(calExpList)*[
None]
if backgroundList
is None else backgroundList
436 skyCorrList = len(calExpList)*[
None]
if skyCorrList
is None else skyCorrList
438 includeCalibVar = self.config.includeCalibVar
441 for index, (calexp, background, skyCorr)
in enumerate(zip(calExpList,
444 if isinstance(calexp, DeferredDatasetHandle):
445 calexp = calexp.get()
447 if not self.config.bgSubtracted:
448 calexp.maskedImage += background.getImage()
450 detectorId = calexp.info.getDetector().getId()
453 row = visitSummary.find(detectorId)
456 f
"Unexpectedly incomplete visitSummary: detector={detectorId} is missing."
458 if (photoCalib := row.getPhotoCalib())
is not None:
459 calexp.setPhotoCalib(photoCalib)
462 "Detector id %d for visit %d has None for photoCalib in the visitSummary and will "
463 "not be used in the warp", detectorId, row[
"visit"],
466 if (skyWcs := row.getWcs())
is not None:
467 calexp.setWcs(skyWcs)
468 wcsList[index] = skyWcs
471 "Detector id %d for visit %d has None for wcs in the visitSummary and will "
472 "not be used in the warp", detectorId, row[
"visit"],
475 if self.config.useVisitSummaryPsf:
476 if (psf := row.getPsf())
is not None:
480 "Detector id %d for visit %d has None for psf in the visitSummary and will "
481 "not be used in the warp", detectorId, row[
"visit"],
484 if (apCorrMap := row.getApCorrMap())
is not None:
485 calexp.info.setApCorrMap(apCorrMap)
488 "Detector id %d for visit %d has None for apCorrMap in the visitSummary and will "
489 "not be used in the warp", detectorId, row[
"visit"],
493 if calexp.getPsf()
is None:
495 "Detector id %d for visit %d has None for psf for the calexp and will "
496 "not be used in the warp", detectorId, row[
"visit"],
499 if calexp.info.getApCorrMap()
is None:
501 "Detector id %d for visit %d has None for apCorrMap in the calexp and will "
502 "not be used in the warp", detectorId, row[
"visit"],
507 calexp.maskedImage = photoCalib.calibrateImage(calexp.maskedImage,
508 includeScaleUncertainty=includeCalibVar)
509 calexp.maskedImage /= photoCalib.getCalibrationMean()
515 if self.config.doApplySkyCorr:
516 calexp.maskedImage -= skyCorr.getImage()
518 indices.append(index)
519 calExpList[index] = calexp
524 def _prepareEmptyExposure(skyInfo):
525 """Produce an empty exposure for a given patch.
529 skyInfo : `lsst.pipe.base.Struct`
530 Struct from `~lsst.pipe.base.coaddBase.makeSkyInfo()` with
531 geometric information about the patch.
535 exp : `lsst.afw.image.exposure.ExposureF`
536 An empty exposure for a given patch.
538 exp = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
539 exp.getMaskedImage().set(numpy.nan, afwImage.Mask
540 .getPlaneBitMask(
"NO_DATA"), numpy.inf)
543 def getWarpTypeList(self):
544 """Return list of requested warp types per the config.
547 if self.config.makeDirect:
548 warpTypeList.append(
"direct")
549 if self.config.makePsfMatched:
550 warpTypeList.append(
"psfMatched")
554def reorderRefs(inputRefs, outputSortKeyOrder, dataIdKey):
555 """Reorder inputRefs per outputSortKeyOrder.
557 Any inputRefs which are lists will be resorted per specified key e.g.,
558 'detector.' Only iterables will be reordered, and values can be of type
559 `lsst.pipe.base.connections.DeferredDatasetRef` or
560 `lsst.daf.butler.core.datasets.ref.DatasetRef`.
562 Returned lists of refs have the same length as the outputSortKeyOrder.
563 If an outputSortKey not in the inputRef, then it will be padded with None.
564 If an inputRef contains an inputSortKey that is not in the
565 outputSortKeyOrder it will be removed.
569 inputRefs : `lsst.pipe.base.connections.QuantizedConnection`
570 Input references to be reordered and padded.
571 outputSortKeyOrder : `iterable`
572 Iterable of values to be compared with inputRef's dataId[dataIdKey].
574 The data ID key in the dataRefs to compare with the outputSortKeyOrder.
578 inputRefs : `lsst.pipe.base.connections.QuantizedConnection`
579 Quantized Connection with sorted DatasetRef values sorted if iterable.
581 for connectionName, refs
in inputRefs:
582 if isinstance(refs, Iterable):
583 if hasattr(refs[0],
"dataId"):
584 inputSortKeyOrder = [ref.dataId[dataIdKey]
for ref
in refs]
586 inputSortKeyOrder = [ref.datasetRef.dataId[dataIdKey]
for ref
in refs]
587 if inputSortKeyOrder != outputSortKeyOrder:
588 setattr(inputRefs, connectionName,
589 reorderAndPadList(refs, inputSortKeyOrder, outputSortKeyOrder))