Coverage for python/lsst/pipe/tasks/makeWarp.py: 18%
226 statements
« prev ^ index » next coverage.py v7.4.4, created at 2024-04-03 02:24 -0700
« prev ^ index » next coverage.py v7.4.4, created at 2024-04-03 02:24 -0700
1# This file is part of pipe_tasks.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22__all__ = ["MakeWarpTask", "MakeWarpConfig"]
24import logging
25import numpy
27import lsst.pex.config as pexConfig
28import lsst.afw.image as afwImage
29import lsst.coadd.utils as coaddUtils
30import lsst.pipe.base as pipeBase
31import lsst.pipe.base.connectionTypes as connectionTypes
32import lsst.utils as utils
33import lsst.geom
34from lsst.daf.butler import DeferredDatasetHandle
35from lsst.meas.base import DetectorVisitIdGeneratorConfig
36from lsst.meas.algorithms import CoaddPsf, CoaddPsfConfig
37from lsst.skymap import BaseSkyMap
38from lsst.utils.timer import timeMethod
39from .coaddBase import CoaddBaseTask, makeSkyInfo, reorderAndPadList
40from .warpAndPsfMatch import WarpAndPsfMatchTask
41from collections.abc import Iterable
43log = logging.getLogger(__name__)
46class MakeWarpConnections(pipeBase.PipelineTaskConnections,
47 dimensions=("tract", "patch", "skymap", "instrument", "visit"),
48 defaultTemplates={"coaddName": "deep",
49 "skyWcsName": "gbdesAstrometricFit",
50 "photoCalibName": "fgcm",
51 "calexpType": ""}):
52 calExpList = connectionTypes.Input(
53 doc="Input exposures to be resampled and optionally PSF-matched onto a SkyMap projection/patch",
54 name="{calexpType}calexp",
55 storageClass="ExposureF",
56 dimensions=("instrument", "visit", "detector"),
57 multiple=True,
58 deferLoad=True,
59 )
60 backgroundList = connectionTypes.Input(
61 doc="Input backgrounds to be added back into the calexp if bgSubtracted=False",
62 name="calexpBackground",
63 storageClass="Background",
64 dimensions=("instrument", "visit", "detector"),
65 multiple=True,
66 )
67 skyCorrList = connectionTypes.Input(
68 doc="Input Sky Correction to be subtracted from the calexp if doApplySkyCorr=True",
69 name="skyCorr",
70 storageClass="Background",
71 dimensions=("instrument", "visit", "detector"),
72 multiple=True,
73 )
74 skyMap = connectionTypes.Input(
75 doc="Input definition of geometry/bbox and projection/wcs for warped exposures",
76 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
77 storageClass="SkyMap",
78 dimensions=("skymap",),
79 )
80 direct = connectionTypes.Output(
81 doc=("Output direct warped exposure (previously called CoaddTempExp), produced by resampling ",
82 "calexps onto the skyMap patch geometry."),
83 name="{coaddName}Coadd_directWarp",
84 storageClass="ExposureF",
85 dimensions=("tract", "patch", "skymap", "visit", "instrument"),
86 )
87 psfMatched = connectionTypes.Output(
88 doc=("Output PSF-Matched warped exposure (previously called CoaddTempExp), produced by resampling ",
89 "calexps onto the skyMap patch geometry and PSF-matching to a model PSF."),
90 name="{coaddName}Coadd_psfMatchedWarp",
91 storageClass="ExposureF",
92 dimensions=("tract", "patch", "skymap", "visit", "instrument"),
93 )
94 visitSummary = connectionTypes.Input(
95 doc="Input visit-summary catalog with updated calibration objects.",
96 name="finalVisitSummary",
97 storageClass="ExposureCatalog",
98 dimensions=("instrument", "visit",),
99 )
101 def __init__(self, *, config=None):
102 if config.bgSubtracted:
103 del self.backgroundList
104 if not config.doApplySkyCorr:
105 del self.skyCorrList
106 if not config.makeDirect:
107 del self.direct
108 if not config.makePsfMatched:
109 del self.psfMatched
112class MakeWarpConfig(pipeBase.PipelineTaskConfig, CoaddBaseTask.ConfigClass,
113 pipelineConnections=MakeWarpConnections):
114 """Config for MakeWarpTask."""
116 warpAndPsfMatch = pexConfig.ConfigurableField(
117 target=WarpAndPsfMatchTask,
118 doc="Task to warp and PSF-match calexp",
119 )
120 doWrite = pexConfig.Field(
121 doc="persist <coaddName>Coadd_<warpType>Warp",
122 dtype=bool,
123 default=True,
124 )
125 bgSubtracted = pexConfig.Field(
126 doc="Work with a background subtracted calexp?",
127 dtype=bool,
128 default=True,
129 )
130 coaddPsf = pexConfig.ConfigField(
131 doc="Configuration for CoaddPsf",
132 dtype=CoaddPsfConfig,
133 )
134 makeDirect = pexConfig.Field(
135 doc="Make direct Warp/Coadds",
136 dtype=bool,
137 default=True,
138 )
139 makePsfMatched = pexConfig.Field(
140 doc="Make Psf-Matched Warp/Coadd?",
141 dtype=bool,
142 default=False,
143 )
144 useVisitSummaryPsf = pexConfig.Field(
145 doc=(
146 "If True, use the PSF model and aperture corrections from the 'visitSummary' connection. "
147 "If False, use the PSF model and aperture corrections from the 'exposure' connection. "
148 ),
149 dtype=bool,
150 default=True,
151 )
152 doWriteEmptyWarps = pexConfig.Field(
153 dtype=bool,
154 default=False,
155 doc="Write out warps even if they are empty"
156 )
157 hasFakes = pexConfig.Field(
158 doc="Should be set to True if fake sources have been inserted into the input data.",
159 dtype=bool,
160 default=False,
161 )
162 doApplySkyCorr = pexConfig.Field(
163 dtype=bool,
164 default=False,
165 doc="Apply sky correction?",
166 )
167 idGenerator = DetectorVisitIdGeneratorConfig.make_field()
169 def validate(self):
170 CoaddBaseTask.ConfigClass.validate(self)
172 if not self.makePsfMatched and not self.makeDirect:
173 raise RuntimeError("At least one of config.makePsfMatched and config.makeDirect must be True")
174 if self.doPsfMatch: # TODO: Remove this in DM-39841
175 # Backwards compatibility.
176 log.warning("Config doPsfMatch deprecated. Setting makePsfMatched=True and makeDirect=False")
177 self.makePsfMatched = True
178 self.makeDirect = False
180 def setDefaults(self):
181 CoaddBaseTask.ConfigClass.setDefaults(self)
182 self.warpAndPsfMatch.psfMatch.kernel.active.kernelSize = self.matchingKernelSize
185class MakeWarpTask(CoaddBaseTask):
186 """Warp and optionally PSF-Match calexps onto an a common projection.
188 Warp and optionally PSF-Match calexps onto a common projection, by
189 performing the following operations:
190 - Group calexps by visit/run
191 - For each visit, generate a Warp by calling method @ref run.
192 `run` loops over the visit's calexps calling
193 `~lsst.pipe.tasks.warpAndPsfMatch.WarpAndPsfMatchTask` on each visit
195 """
196 ConfigClass = MakeWarpConfig
197 _DefaultName = "makeWarp"
199 def __init__(self, **kwargs):
200 CoaddBaseTask.__init__(self, **kwargs)
201 self.makeSubtask("warpAndPsfMatch")
202 if self.config.hasFakes:
203 self.calexpType = "fakes_calexp"
204 else:
205 self.calexpType = "calexp"
207 @utils.inheritDoc(pipeBase.PipelineTask)
208 def runQuantum(self, butlerQC, inputRefs, outputRefs):
209 # Docstring to be augmented with info from PipelineTask.runQuantum
210 """Notes
211 -----
212 Obtain the list of input detectors from calExpList. Sort them by
213 detector order (to ensure reproducibility). Then ensure all input
214 lists are in the same sorted detector order.
215 """
216 detectorOrder = [ref.datasetRef.dataId['detector'] for ref in inputRefs.calExpList]
217 detectorOrder.sort()
218 inputRefs = reorderRefs(inputRefs, detectorOrder, dataIdKey='detector')
220 # Read in all inputs.
221 inputs = butlerQC.get(inputRefs)
223 # Construct skyInfo expected by `run`. We remove the SkyMap itself
224 # from the dictionary so we can pass it as kwargs later.
225 skyMap = inputs.pop("skyMap")
226 quantumDataId = butlerQC.quantum.dataId
227 skyInfo = makeSkyInfo(skyMap, tractId=quantumDataId['tract'], patchId=quantumDataId['patch'])
229 # Construct list of input DataIds expected by `run`.
230 dataIdList = [ref.datasetRef.dataId for ref in inputRefs.calExpList]
231 # Construct list of packed integer IDs expected by `run`.
232 ccdIdList = [
233 self.config.idGenerator.apply(dataId).catalog_id
234 for dataId in dataIdList
235 ]
237 # Check early that the visitSummary contains everything we need.
238 visitSummary = inputs["visitSummary"]
239 bboxList = []
240 wcsList = []
241 for dataId in dataIdList:
242 row = visitSummary.find(dataId["detector"])
243 if row is None:
244 raise RuntimeError(
245 f"Unexpectedly incomplete visitSummary provided to makeWarp: {dataId} is missing."
246 )
247 bboxList.append(row.getBBox())
248 wcsList.append(row.getWcs())
249 inputs["bboxList"] = bboxList
250 inputs["wcsList"] = wcsList
252 # Do an initial selection on inputs with complete wcs/photoCalib info.
253 # Qualifying calexps will be read in the following call.
254 completeIndices = self._prepareCalibratedExposures(**inputs)
255 inputs = self.filterInputs(indices=completeIndices, inputs=inputs)
257 # Do another selection based on the configured selection task
258 # (using updated WCSs to determine patch overlap if an external
259 # calibration was applied).
260 cornerPosList = lsst.geom.Box2D(skyInfo.bbox).getCorners()
261 coordList = [skyInfo.wcs.pixelToSky(pos) for pos in cornerPosList]
262 goodIndices = self.select.run(**inputs, coordList=coordList, dataIds=dataIdList)
263 inputs = self.filterInputs(indices=goodIndices, inputs=inputs)
265 # Extract integer visitId requested by `run`.
266 visitId = dataIdList[0]["visit"]
268 results = self.run(**inputs,
269 visitId=visitId,
270 ccdIdList=[ccdIdList[i] for i in goodIndices],
271 dataIdList=[dataIdList[i] for i in goodIndices],
272 skyInfo=skyInfo)
273 if self.config.makeDirect and results.exposures["direct"] is not None:
274 butlerQC.put(results.exposures["direct"], outputRefs.direct)
275 if self.config.makePsfMatched and results.exposures["psfMatched"] is not None:
276 butlerQC.put(results.exposures["psfMatched"], outputRefs.psfMatched)
278 @timeMethod
279 def run(self, calExpList, ccdIdList, skyInfo, visitId=0, dataIdList=None, **kwargs):
280 """Create a Warp from inputs.
282 We iterate over the multiple calexps in a single exposure to construct
283 the warp (previously called a coaddTempExp) of that exposure to the
284 supplied tract/patch.
286 Pixels that receive no pixels are set to NAN; this is not correct
287 (violates LSST algorithms group policy), but will be fixed up by
288 interpolating after the coaddition.
290 calexpRefList : `list`
291 List of data references for calexps that (may)
292 overlap the patch of interest.
293 skyInfo : `lsst.pipe.base.Struct`
294 Struct from `~lsst.pipe.base.coaddBase.makeSkyInfo()` with
295 geometric information about the patch.
296 visitId : `int`
297 Integer identifier for visit, for the table that will
298 produce the CoaddPsf.
300 Returns
301 -------
302 result : `lsst.pipe.base.Struct`
303 Results as a struct with attributes:
305 ``exposures``
306 A dictionary containing the warps requested:
307 "direct": direct warp if ``config.makeDirect``
308 "psfMatched": PSF-matched warp if ``config.makePsfMatched``
309 (`dict`).
310 """
311 warpTypeList = self.getWarpTypeList()
313 totGoodPix = {warpType: 0 for warpType in warpTypeList}
314 didSetMetadata = {warpType: False for warpType in warpTypeList}
315 warps = {warpType: self._prepareEmptyExposure(skyInfo) for warpType in warpTypeList}
316 inputRecorder = {warpType: self.inputRecorder.makeCoaddTempExpRecorder(visitId, len(calExpList))
317 for warpType in warpTypeList}
319 modelPsf = self.config.modelPsf.apply() if self.config.makePsfMatched else None
320 if dataIdList is None:
321 dataIdList = ccdIdList
323 for calExpInd, (calExp, ccdId, dataId) in enumerate(zip(calExpList, ccdIdList, dataIdList)):
324 self.log.info("Processing calexp %d of %d for this Warp: id=%s",
325 calExpInd+1, len(calExpList), dataId)
326 try:
327 warpedAndMatched = self.warpAndPsfMatch.run(calExp, modelPsf=modelPsf,
328 wcs=skyInfo.wcs, maxBBox=skyInfo.bbox,
329 makeDirect=self.config.makeDirect,
330 makePsfMatched=self.config.makePsfMatched)
331 except Exception as e:
332 self.log.warning("WarpAndPsfMatch failed for calexp %s; skipping it: %s", dataId, e)
333 continue
334 try:
335 numGoodPix = {warpType: 0 for warpType in warpTypeList}
336 for warpType in warpTypeList:
337 exposure = warpedAndMatched.getDict()[warpType]
338 if exposure is None:
339 continue
340 warp = warps[warpType]
341 if didSetMetadata[warpType]:
342 mimg = exposure.getMaskedImage()
343 mimg *= (warp.getPhotoCalib().getInstFluxAtZeroMagnitude()
344 / exposure.getPhotoCalib().getInstFluxAtZeroMagnitude())
345 del mimg
346 numGoodPix[warpType] = coaddUtils.copyGoodPixels(
347 warp.getMaskedImage(), exposure.getMaskedImage(), self.getBadPixelMask())
348 totGoodPix[warpType] += numGoodPix[warpType]
349 self.log.debug("Calexp %s has %d good pixels in this patch (%.1f%%) for %s",
350 dataId, numGoodPix[warpType],
351 100.0*numGoodPix[warpType]/skyInfo.bbox.getArea(), warpType)
352 if numGoodPix[warpType] > 0 and not didSetMetadata[warpType]:
353 warp.info.id = exposure.info.id
354 warp.setPhotoCalib(exposure.getPhotoCalib())
355 warp.setFilter(exposure.getFilter())
356 warp.getInfo().setVisitInfo(exposure.getInfo().getVisitInfo())
357 # PSF replaced with CoaddPsf after loop if and only if
358 # creating direct warp.
359 warp.setPsf(exposure.getPsf())
360 didSetMetadata[warpType] = True
362 # Need inputRecorder for CoaddApCorrMap for both direct and
363 # PSF-matched.
364 inputRecorder[warpType].addCalExp(calExp, ccdId, numGoodPix[warpType])
366 except Exception as e:
367 self.log.warning("Error processing calexp %s; skipping it: %s", dataId, e)
368 continue
370 for warpType in warpTypeList:
371 self.log.info("%sWarp has %d good pixels (%.1f%%)",
372 warpType, totGoodPix[warpType], 100.0*totGoodPix[warpType]/skyInfo.bbox.getArea())
374 if totGoodPix[warpType] > 0 and didSetMetadata[warpType]:
375 inputRecorder[warpType].finish(warps[warpType], totGoodPix[warpType])
376 if warpType == "direct":
377 warps[warpType].setPsf(
378 CoaddPsf(inputRecorder[warpType].coaddInputs.ccds, skyInfo.wcs,
379 self.config.coaddPsf.makeControl()))
380 else:
381 if not self.config.doWriteEmptyWarps:
382 # No good pixels. Exposure still empty.
383 warps[warpType] = None
384 # NoWorkFound is unnecessary as the downstream tasks will
385 # adjust the quantum accordingly.
387 result = pipeBase.Struct(exposures=warps)
388 return result
390 def filterInputs(self, indices, inputs):
391 """Filter task inputs by their indices.
393 Parameters
394 ----------
395 indices : `list` [`int`]
396 inputs : `dict` [`list`]
397 A dictionary of input connections to be passed to run.
399 Returns
400 -------
401 inputs : `dict` [`list`]
402 Task inputs with their lists filtered by indices.
403 """
404 for key in inputs.keys():
405 # Only down-select on list inputs
406 if isinstance(inputs[key], list):
407 inputs[key] = [inputs[key][ind] for ind in indices]
408 return inputs
410 def _prepareCalibratedExposures(self, *, visitSummary, calExpList=[], wcsList=None,
411 backgroundList=None, skyCorrList=None, **kwargs):
412 """Calibrate and add backgrounds to input calExpList in place.
414 Parameters
415 ----------
416 visitSummary : `lsst.afw.table.ExposureCatalog`
417 Exposure catalog with potentially all calibrations. Attributes set
418 to `None` are ignored.
419 calExpList : `list` [`lsst.afw.image.Exposure` or
420 `lsst.daf.butler.DeferredDatasetHandle`]
421 Sequence of calexps to be modified in place.
422 wcsList : `list` [`lsst.afw.geom.SkyWcs`]
423 The WCSs of the calexps in ``calExpList``. These will be used to
424 determine if the calexp should be used in the warp. The list is
425 dynamically updated with the WCSs from the visitSummary.
426 backgroundList : `list` [`lsst.afw.math.backgroundList`], optional
427 Sequence of backgrounds to be added back in if bgSubtracted=False.
428 skyCorrList : `list` [`lsst.afw.math.backgroundList`], optional
429 Sequence of background corrections to be subtracted if
430 doApplySkyCorr=True.
431 **kwargs
432 Additional keyword arguments.
434 Returns
435 -------
436 indices : `list` [`int`]
437 Indices of ``calExpList`` and friends that have valid
438 photoCalib/skyWcs.
439 """
440 wcsList = len(calExpList)*[None] if wcsList is None else wcsList
441 backgroundList = len(calExpList)*[None] if backgroundList is None else backgroundList
442 skyCorrList = len(calExpList)*[None] if skyCorrList is None else skyCorrList
444 includeCalibVar = self.config.includeCalibVar
446 indices = []
447 for index, (calexp, background, skyCorr) in enumerate(zip(calExpList,
448 backgroundList,
449 skyCorrList)):
450 if isinstance(calexp, DeferredDatasetHandle):
451 calexp = calexp.get()
453 if not self.config.bgSubtracted:
454 calexp.maskedImage += background.getImage()
456 detectorId = calexp.info.getDetector().getId()
458 # Load all calibrations from visitSummary.
459 row = visitSummary.find(detectorId)
460 if row is None:
461 raise RuntimeError(
462 f"Unexpectedly incomplete visitSummary: detector={detectorId} is missing."
463 )
464 if (photoCalib := row.getPhotoCalib()) is not None:
465 calexp.setPhotoCalib(photoCalib)
466 else:
467 self.log.warning(
468 "Detector id %d for visit %d has None for photoCalib in the visitSummary and will "
469 "not be used in the warp", detectorId, row["visit"],
470 )
471 continue
472 if (skyWcs := row.getWcs()) is not None:
473 calexp.setWcs(skyWcs)
474 wcsList[index] = skyWcs
475 else:
476 self.log.warning(
477 "Detector id %d for visit %d has None for wcs in the visitSummary and will "
478 "not be used in the warp", detectorId, row["visit"],
479 )
480 continue
481 if self.config.useVisitSummaryPsf:
482 if (psf := row.getPsf()) is not None:
483 calexp.setPsf(psf)
484 else:
485 self.log.warning(
486 "Detector id %d for visit %d has None for psf in the visitSummary and will "
487 "not be used in the warp", detectorId, row["visit"],
488 )
489 continue
490 if (apCorrMap := row.getApCorrMap()) is not None:
491 calexp.info.setApCorrMap(apCorrMap)
492 else:
493 self.log.warning(
494 "Detector id %d for visit %d has None for apCorrMap in the visitSummary and will "
495 "not be used in the warp", detectorId, row["visit"],
496 )
497 continue
498 else:
499 if calexp.getPsf() is None:
500 self.log.warning(
501 "Detector id %d for visit %d has None for psf for the calexp and will "
502 "not be used in the warp", detectorId, row["visit"],
503 )
504 continue
505 if calexp.info.getApCorrMap() is None:
506 self.log.warning(
507 "Detector id %d for visit %d has None for apCorrMap in the calexp and will "
508 "not be used in the warp", detectorId, row["visit"],
509 )
510 continue
512 # Calibrate the image.
513 calexp.maskedImage = photoCalib.calibrateImage(calexp.maskedImage,
514 includeScaleUncertainty=includeCalibVar)
515 calexp.maskedImage /= photoCalib.getCalibrationMean()
516 # TODO: The images will have a calibration of 1.0 everywhere once
517 # RFC-545 is implemented.
518 # exposure.setCalib(afwImage.Calib(1.0))
520 # Apply skycorr
521 if self.config.doApplySkyCorr:
522 calexp.maskedImage -= skyCorr.getImage()
524 indices.append(index)
525 calExpList[index] = calexp
527 return indices
529 @staticmethod
530 def _prepareEmptyExposure(skyInfo):
531 """Produce an empty exposure for a given patch.
533 Parameters
534 ----------
535 skyInfo : `lsst.pipe.base.Struct`
536 Struct from `~lsst.pipe.base.coaddBase.makeSkyInfo()` with
537 geometric information about the patch.
539 Returns
540 -------
541 exp : `lsst.afw.image.exposure.ExposureF`
542 An empty exposure for a given patch.
543 """
544 exp = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
545 exp.getMaskedImage().set(numpy.nan, afwImage.Mask
546 .getPlaneBitMask("NO_DATA"), numpy.inf)
547 return exp
549 def getWarpTypeList(self):
550 """Return list of requested warp types per the config.
551 """
552 warpTypeList = []
553 if self.config.makeDirect:
554 warpTypeList.append("direct")
555 if self.config.makePsfMatched:
556 warpTypeList.append("psfMatched")
557 return warpTypeList
560def reorderRefs(inputRefs, outputSortKeyOrder, dataIdKey):
561 """Reorder inputRefs per outputSortKeyOrder.
563 Any inputRefs which are lists will be resorted per specified key e.g.,
564 'detector.' Only iterables will be reordered, and values can be of type
565 `lsst.pipe.base.connections.DeferredDatasetRef` or
566 `lsst.daf.butler.core.datasets.ref.DatasetRef`.
568 Returned lists of refs have the same length as the outputSortKeyOrder.
569 If an outputSortKey not in the inputRef, then it will be padded with None.
570 If an inputRef contains an inputSortKey that is not in the
571 outputSortKeyOrder it will be removed.
573 Parameters
574 ----------
575 inputRefs : `lsst.pipe.base.connections.QuantizedConnection`
576 Input references to be reordered and padded.
577 outputSortKeyOrder : `iterable`
578 Iterable of values to be compared with inputRef's dataId[dataIdKey].
579 dataIdKey : `str`
580 The data ID key in the dataRefs to compare with the outputSortKeyOrder.
582 Returns
583 -------
584 inputRefs : `lsst.pipe.base.connections.QuantizedConnection`
585 Quantized Connection with sorted DatasetRef values sorted if iterable.
586 """
587 for connectionName, refs in inputRefs:
588 if isinstance(refs, Iterable):
589 if hasattr(refs[0], "dataId"):
590 inputSortKeyOrder = [ref.dataId[dataIdKey] for ref in refs]
591 else:
592 inputSortKeyOrder = [ref.datasetRef.dataId[dataIdKey] for ref in refs]
593 if inputSortKeyOrder != outputSortKeyOrder:
594 setattr(inputRefs, connectionName,
595 reorderAndPadList(refs, inputSortKeyOrder, outputSortKeyOrder))
596 return inputRefs