lsst.pipe.tasks g06b2ea86fd+734f9505a2
Loading...
Searching...
No Matches
makeWarp.py
Go to the documentation of this file.
1# This file is part of pipe_tasks.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21
22__all__ = ["MakeWarpTask", "MakeWarpConfig"]
23
24import logging
25import numpy
26
27import lsst.pex.config as pexConfig
28import lsst.afw.image as afwImage
29import lsst.coadd.utils as coaddUtils
30import lsst.pipe.base as pipeBase
31import lsst.pipe.base.connectionTypes as connectionTypes
32import lsst.utils as utils
33import lsst.geom
34from lsst.daf.butler import DeferredDatasetHandle
35from lsst.meas.base import DetectorVisitIdGeneratorConfig
36from lsst.meas.algorithms import CoaddPsf, CoaddPsfConfig, GaussianPsfFactory
37from lsst.skymap import BaseSkyMap
38from lsst.utils.timer import timeMethod
39from .coaddBase import CoaddBaseTask, makeSkyInfo, reorderAndPadList
40from .warpAndPsfMatch import WarpAndPsfMatchTask
41from collections.abc import Iterable
42
43log = logging.getLogger(__name__)
44
45
46class MakeWarpConnections(pipeBase.PipelineTaskConnections,
47 dimensions=("tract", "patch", "skymap", "instrument", "visit"),
48 defaultTemplates={"coaddName": "deep",
49 "calexpType": ""}):
50 calExpList = connectionTypes.Input(
51 doc="Input exposures to be resampled and optionally PSF-matched onto a SkyMap projection/patch",
52 name="{calexpType}calexp",
53 storageClass="ExposureF",
54 dimensions=("instrument", "visit", "detector"),
55 multiple=True,
56 deferLoad=True,
57 )
58 backgroundList = connectionTypes.Input(
59 doc="Input backgrounds to be added back into the calexp if bgSubtracted=False",
60 name="calexpBackground",
61 storageClass="Background",
62 dimensions=("instrument", "visit", "detector"),
63 multiple=True,
64 )
65 skyCorrList = connectionTypes.Input(
66 doc="Input Sky Correction to be subtracted from the calexp if doApplySkyCorr=True",
67 name="skyCorr",
68 storageClass="Background",
69 dimensions=("instrument", "visit", "detector"),
70 multiple=True,
71 )
72 skyMap = connectionTypes.Input(
73 doc="Input definition of geometry/bbox and projection/wcs for warped exposures",
74 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
75 storageClass="SkyMap",
76 dimensions=("skymap",),
77 )
78 direct = connectionTypes.Output(
79 doc=("Output direct warped exposure (previously called CoaddTempExp), produced by resampling ",
80 "calexps onto the skyMap patch geometry."),
81 name="{coaddName}Coadd_directWarp",
82 storageClass="ExposureF",
83 dimensions=("tract", "patch", "skymap", "visit", "instrument"),
84 )
85 psfMatched = connectionTypes.Output(
86 doc=("Output PSF-Matched warped exposure (previously called CoaddTempExp), produced by resampling ",
87 "calexps onto the skyMap patch geometry and PSF-matching to a model PSF."),
88 name="{coaddName}Coadd_psfMatchedWarp",
89 storageClass="ExposureF",
90 dimensions=("tract", "patch", "skymap", "visit", "instrument"),
91 )
92 visitSummary = connectionTypes.Input(
93 doc="Input visit-summary catalog with updated calibration objects.",
94 name="finalVisitSummary",
95 storageClass="ExposureCatalog",
96 dimensions=("instrument", "visit",),
97 )
98
99 def __init__(self, *, config=None):
100 if config.bgSubtracted:
101 del self.backgroundList
102 if not config.doApplySkyCorr:
103 del self.skyCorrList
104 if not config.makeDirect:
105 del self.direct
106 if not config.makePsfMatched:
107 del self.psfMatched
108
109
110class MakeWarpConfig(pipeBase.PipelineTaskConfig, CoaddBaseTask.ConfigClass,
111 pipelineConnections=MakeWarpConnections):
112 """Config for MakeWarpTask."""
113
114 warpAndPsfMatch = pexConfig.ConfigurableField(
115 target=WarpAndPsfMatchTask,
116 doc="Task to warp and PSF-match calexp",
117 )
118 doWrite = pexConfig.Field(
119 doc="persist <coaddName>Coadd_<warpType>Warp",
120 dtype=bool,
121 default=True,
122 )
123 bgSubtracted = pexConfig.Field(
124 doc="Work with a background subtracted calexp?",
125 dtype=bool,
126 default=True,
127 )
128 coaddPsf = pexConfig.ConfigField(
129 doc="Configuration for CoaddPsf",
130 dtype=CoaddPsfConfig,
131 )
132 makeDirect = pexConfig.Field(
133 doc="Make direct Warp/Coadds",
134 dtype=bool,
135 default=True,
136 )
137 makePsfMatched = pexConfig.Field(
138 doc="Make Psf-Matched Warp/Coadd?",
139 dtype=bool,
140 default=False,
141 )
142 modelPsf = GaussianPsfFactory.makeField(doc="Model Psf factory")
143 useVisitSummaryPsf = pexConfig.Field(
144 doc=(
145 "If True, use the PSF model and aperture corrections from the 'visitSummary' connection. "
146 "If False, use the PSF model and aperture corrections from the 'exposure' connection. "
147 ),
148 dtype=bool,
149 default=True,
150 )
151 doWriteEmptyWarps = pexConfig.Field(
152 dtype=bool,
153 default=False,
154 doc="Write out warps even if they are empty"
155 )
156 hasFakes = pexConfig.Field(
157 doc="Should be set to True if fake sources have been inserted into the input data.",
158 dtype=bool,
159 default=False,
160 )
161 doApplySkyCorr = pexConfig.Field(
162 dtype=bool,
163 default=False,
164 doc="Apply sky correction?",
165 )
166 idGenerator = DetectorVisitIdGeneratorConfig.make_field()
167
168 def validate(self):
169 CoaddBaseTask.ConfigClass.validate(self)
170
171 if not self.makePsfMatched and not self.makeDirect:
172 raise RuntimeError("At least one of config.makePsfMatched and config.makeDirect must be True")
173
174 def setDefaults(self):
175 CoaddBaseTask.ConfigClass.setDefaults(self)
176 self.warpAndPsfMatch.psfMatch.kernel.active.kernelSize = self.matchingKernelSize
177
178
179class MakeWarpTask(CoaddBaseTask):
180 """Warp and optionally PSF-Match calexps onto an a common projection.
181
182 Warp and optionally PSF-Match calexps onto a common projection, by
183 performing the following operations:
184 - Group calexps by visit/run
185 - For each visit, generate a Warp by calling method @ref run.
186 `run` loops over the visit's calexps calling
187 `~lsst.pipe.tasks.warpAndPsfMatch.WarpAndPsfMatchTask` on each visit
188
189 """
190 ConfigClass = MakeWarpConfig
191 _DefaultName = "makeWarp"
192
193 def __init__(self, **kwargs):
194 CoaddBaseTask.__init__(self, **kwargs)
195 self.makeSubtask("warpAndPsfMatch")
196 if self.config.hasFakes:
197 self.calexpType = "fakes_calexp"
198 else:
199 self.calexpType = "calexp"
200
201 @utils.inheritDoc(pipeBase.PipelineTask)
202 def runQuantum(self, butlerQC, inputRefs, outputRefs):
203 # Docstring to be augmented with info from PipelineTask.runQuantum
204 """Notes
205 -----
206 Obtain the list of input detectors from calExpList. Sort them by
207 detector order (to ensure reproducibility). Then ensure all input
208 lists are in the same sorted detector order.
209 """
210 detectorOrder = [ref.datasetRef.dataId['detector'] for ref in inputRefs.calExpList]
211 detectorOrder.sort()
212 inputRefs = reorderRefs(inputRefs, detectorOrder, dataIdKey='detector')
213
214 # Read in all inputs.
215 inputs = butlerQC.get(inputRefs)
216
217 # Construct skyInfo expected by `run`. We remove the SkyMap itself
218 # from the dictionary so we can pass it as kwargs later.
219 skyMap = inputs.pop("skyMap")
220 quantumDataId = butlerQC.quantum.dataId
221 skyInfo = makeSkyInfo(skyMap, tractId=quantumDataId['tract'], patchId=quantumDataId['patch'])
222
223 # Construct list of input DataIds expected by `run`.
224 dataIdList = [ref.datasetRef.dataId for ref in inputRefs.calExpList]
225 # Construct list of packed integer IDs expected by `run`.
226 ccdIdList = [
227 self.config.idGenerator.apply(dataId).catalog_id
228 for dataId in dataIdList
229 ]
230
231 # Check early that the visitSummary contains everything we need.
232 visitSummary = inputs["visitSummary"]
233 bboxList = []
234 wcsList = []
235 for dataId in dataIdList:
236 row = visitSummary.find(dataId["detector"])
237 if row is None:
238 raise RuntimeError(
239 f"Unexpectedly incomplete visitSummary provided to makeWarp: {dataId} is missing."
240 )
241 bboxList.append(row.getBBox())
242 wcsList.append(row.getWcs())
243 inputs["bboxList"] = bboxList
244 inputs["wcsList"] = wcsList
245
246 # Do an initial selection on inputs with complete wcs/photoCalib info.
247 # Qualifying calexps will be read in the following call.
248 completeIndices = self._prepareCalibratedExposures(**inputs)
249 inputs = self.filterInputs(indices=completeIndices, inputs=inputs)
250
251 # Do another selection based on the configured selection task
252 # (using updated WCSs to determine patch overlap if an external
253 # calibration was applied).
254 cornerPosList = lsst.geom.Box2D(skyInfo.bbox).getCorners()
255 coordList = [skyInfo.wcs.pixelToSky(pos) for pos in cornerPosList]
256 goodIndices = self.select.run(**inputs, coordList=coordList, dataIds=dataIdList)
257 inputs = self.filterInputs(indices=goodIndices, inputs=inputs)
258
259 # Extract integer visitId requested by `run`.
260 visitId = dataIdList[0]["visit"]
261
262 results = self.run(**inputs,
263 visitId=visitId,
264 ccdIdList=[ccdIdList[i] for i in goodIndices],
265 dataIdList=[dataIdList[i] for i in goodIndices],
266 skyInfo=skyInfo)
267 if self.config.makeDirect and results.exposures["direct"] is not None:
268 butlerQC.put(results.exposures["direct"], outputRefs.direct)
269 if self.config.makePsfMatched and results.exposures["psfMatched"] is not None:
270 butlerQC.put(results.exposures["psfMatched"], outputRefs.psfMatched)
271
272 @timeMethod
273 def run(self, calExpList, ccdIdList, skyInfo, visitId=0, dataIdList=None, **kwargs):
274 """Create a Warp from inputs.
275
276 We iterate over the multiple calexps in a single exposure to construct
277 the warp (previously called a coaddTempExp) of that exposure to the
278 supplied tract/patch.
279
280 Pixels that receive no pixels are set to NAN; this is not correct
281 (violates LSST algorithms group policy), but will be fixed up by
282 interpolating after the coaddition.
283
284 calexpRefList : `list`
285 List of data references for calexps that (may)
286 overlap the patch of interest.
287 skyInfo : `lsst.pipe.base.Struct`
288 Struct from `~lsst.pipe.base.coaddBase.makeSkyInfo()` with
289 geometric information about the patch.
290 visitId : `int`
291 Integer identifier for visit, for the table that will
292 produce the CoaddPsf.
293
294 Returns
295 -------
296 result : `lsst.pipe.base.Struct`
297 Results as a struct with attributes:
298
299 ``exposures``
300 A dictionary containing the warps requested:
301 "direct": direct warp if ``config.makeDirect``
302 "psfMatched": PSF-matched warp if ``config.makePsfMatched``
303 (`dict`).
304 """
305 warpTypeList = self.getWarpTypeList()
306
307 totGoodPix = {warpType: 0 for warpType in warpTypeList}
308 didSetMetadata = {warpType: False for warpType in warpTypeList}
309 warps = {warpType: self._prepareEmptyExposure(skyInfo) for warpType in warpTypeList}
310 inputRecorder = {warpType: self.inputRecorder.makeCoaddTempExpRecorder(visitId, len(calExpList))
311 for warpType in warpTypeList}
312
313 modelPsf = self.config.modelPsf.apply() if self.config.makePsfMatched else None
314 if dataIdList is None:
315 dataIdList = ccdIdList
316
317 for calExpInd, (calExp, ccdId, dataId) in enumerate(zip(calExpList, ccdIdList, dataIdList)):
318 self.log.info("Processing calexp %d of %d for this Warp: id=%s",
319 calExpInd+1, len(calExpList), dataId)
320 try:
321 warpedAndMatched = self.warpAndPsfMatch.run(calExp, modelPsf=modelPsf,
322 wcs=skyInfo.wcs, maxBBox=skyInfo.bbox,
323 makeDirect=self.config.makeDirect,
324 makePsfMatched=self.config.makePsfMatched)
325 except Exception as e:
326 self.log.warning("WarpAndPsfMatch failed for calexp %s; skipping it: %s", dataId, e)
327 continue
328 try:
329 numGoodPix = {warpType: 0 for warpType in warpTypeList}
330 for warpType in warpTypeList:
331 exposure = warpedAndMatched.getDict()[warpType]
332 if exposure is None:
333 continue
334 warp = warps[warpType]
335 if didSetMetadata[warpType]:
336 mimg = exposure.getMaskedImage()
337 mimg *= (warp.getPhotoCalib().getInstFluxAtZeroMagnitude()
338 / exposure.getPhotoCalib().getInstFluxAtZeroMagnitude())
339 del mimg
340 numGoodPix[warpType] = coaddUtils.copyGoodPixels(
341 warp.getMaskedImage(), exposure.getMaskedImage(), self.getBadPixelMask())
342 totGoodPix[warpType] += numGoodPix[warpType]
343 self.log.debug("Calexp %s has %d good pixels in this patch (%.1f%%) for %s",
344 dataId, numGoodPix[warpType],
345 100.0*numGoodPix[warpType]/skyInfo.bbox.getArea(), warpType)
346 if numGoodPix[warpType] > 0 and not didSetMetadata[warpType]:
347 warp.info.id = exposure.info.id
348 warp.setPhotoCalib(exposure.getPhotoCalib())
349 warp.setFilter(exposure.getFilter())
350 warp.getInfo().setVisitInfo(exposure.getInfo().getVisitInfo())
351 # PSF replaced with CoaddPsf after loop if and only if
352 # creating direct warp.
353 warp.setPsf(exposure.getPsf())
354 didSetMetadata[warpType] = True
355
356 # Need inputRecorder for CoaddApCorrMap for both direct and
357 # PSF-matched.
358 inputRecorder[warpType].addCalExp(calExp, ccdId, numGoodPix[warpType])
359
360 except Exception as e:
361 self.log.warning("Error processing calexp %s; skipping it: %s", dataId, e)
362 continue
363
364 for warpType in warpTypeList:
365 self.log.info("%sWarp has %d good pixels (%.1f%%)",
366 warpType, totGoodPix[warpType], 100.0*totGoodPix[warpType]/skyInfo.bbox.getArea())
367
368 if totGoodPix[warpType] > 0 and didSetMetadata[warpType]:
369 inputRecorder[warpType].finish(warps[warpType], totGoodPix[warpType])
370 if warpType == "direct":
371 warps[warpType].setPsf(
372 CoaddPsf(inputRecorder[warpType].coaddInputs.ccds, skyInfo.wcs,
373 self.config.coaddPsf.makeControl()))
374 else:
375 if not self.config.doWriteEmptyWarps:
376 # No good pixels. Exposure still empty.
377 warps[warpType] = None
378 # NoWorkFound is unnecessary as the downstream tasks will
379 # adjust the quantum accordingly.
380
381 result = pipeBase.Struct(exposures=warps)
382 return result
383
384 def filterInputs(self, indices, inputs):
385 """Filter task inputs by their indices.
386
387 Parameters
388 ----------
389 indices : `list` [`int`]
390 inputs : `dict` [`list`]
391 A dictionary of input connections to be passed to run.
392
393 Returns
394 -------
395 inputs : `dict` [`list`]
396 Task inputs with their lists filtered by indices.
397 """
398 for key in inputs.keys():
399 # Only down-select on list inputs
400 if isinstance(inputs[key], list):
401 inputs[key] = [inputs[key][ind] for ind in indices]
402 return inputs
403
404 def _prepareCalibratedExposures(self, *, visitSummary, calExpList=[], wcsList=None,
405 backgroundList=None, skyCorrList=None, **kwargs):
406 """Calibrate and add backgrounds to input calExpList in place.
407
408 Parameters
409 ----------
410 visitSummary : `lsst.afw.table.ExposureCatalog`
411 Exposure catalog with potentially all calibrations. Attributes set
412 to `None` are ignored.
413 calExpList : `list` [`lsst.afw.image.Exposure` or
414 `lsst.daf.butler.DeferredDatasetHandle`]
415 Sequence of calexps to be modified in place.
416 wcsList : `list` [`lsst.afw.geom.SkyWcs`]
417 The WCSs of the calexps in ``calExpList``. These will be used to
418 determine if the calexp should be used in the warp. The list is
419 dynamically updated with the WCSs from the visitSummary.
420 backgroundList : `list` [`lsst.afw.math.backgroundList`], optional
421 Sequence of backgrounds to be added back in if bgSubtracted=False.
422 skyCorrList : `list` [`lsst.afw.math.backgroundList`], optional
423 Sequence of background corrections to be subtracted if
424 doApplySkyCorr=True.
425 **kwargs
426 Additional keyword arguments.
427
428 Returns
429 -------
430 indices : `list` [`int`]
431 Indices of ``calExpList`` and friends that have valid
432 photoCalib/skyWcs.
433 """
434 wcsList = len(calExpList)*[None] if wcsList is None else wcsList
435 backgroundList = len(calExpList)*[None] if backgroundList is None else backgroundList
436 skyCorrList = len(calExpList)*[None] if skyCorrList is None else skyCorrList
437
438 includeCalibVar = self.config.includeCalibVar
439
440 indices = []
441 for index, (calexp, background, skyCorr) in enumerate(zip(calExpList,
442 backgroundList,
443 skyCorrList)):
444 if isinstance(calexp, DeferredDatasetHandle):
445 calexp = calexp.get()
446
447 if not self.config.bgSubtracted:
448 calexp.maskedImage += background.getImage()
449
450 detectorId = calexp.info.getDetector().getId()
451
452 # Load all calibrations from visitSummary.
453 row = visitSummary.find(detectorId)
454 if row is None:
455 raise RuntimeError(
456 f"Unexpectedly incomplete visitSummary: detector={detectorId} is missing."
457 )
458 if (photoCalib := row.getPhotoCalib()) is not None:
459 calexp.setPhotoCalib(photoCalib)
460 else:
461 self.log.warning(
462 "Detector id %d for visit %d has None for photoCalib in the visitSummary and will "
463 "not be used in the warp", detectorId, row["visit"],
464 )
465 continue
466 if (skyWcs := row.getWcs()) is not None:
467 calexp.setWcs(skyWcs)
468 wcsList[index] = skyWcs
469 else:
470 self.log.warning(
471 "Detector id %d for visit %d has None for wcs in the visitSummary and will "
472 "not be used in the warp", detectorId, row["visit"],
473 )
474 continue
475 if self.config.useVisitSummaryPsf:
476 if (psf := row.getPsf()) is not None:
477 calexp.setPsf(psf)
478 else:
479 self.log.warning(
480 "Detector id %d for visit %d has None for psf in the visitSummary and will "
481 "not be used in the warp", detectorId, row["visit"],
482 )
483 continue
484 if (apCorrMap := row.getApCorrMap()) is not None:
485 calexp.info.setApCorrMap(apCorrMap)
486 else:
487 self.log.warning(
488 "Detector id %d for visit %d has None for apCorrMap in the visitSummary and will "
489 "not be used in the warp", detectorId, row["visit"],
490 )
491 continue
492 else:
493 if calexp.getPsf() is None:
494 self.log.warning(
495 "Detector id %d for visit %d has None for psf for the calexp and will "
496 "not be used in the warp", detectorId, row["visit"],
497 )
498 continue
499 if calexp.info.getApCorrMap() is None:
500 self.log.warning(
501 "Detector id %d for visit %d has None for apCorrMap in the calexp and will "
502 "not be used in the warp", detectorId, row["visit"],
503 )
504 continue
505
506 # Calibrate the image.
507 calexp.maskedImage = photoCalib.calibrateImage(calexp.maskedImage,
508 includeScaleUncertainty=includeCalibVar)
509 calexp.maskedImage /= photoCalib.getCalibrationMean()
510 # TODO: The images will have a calibration of 1.0 everywhere once
511 # RFC-545 is implemented.
512 # exposure.setCalib(afwImage.Calib(1.0))
513
514 # Apply skycorr
515 if self.config.doApplySkyCorr:
516 calexp.maskedImage -= skyCorr.getImage()
517
518 indices.append(index)
519 calExpList[index] = calexp
520
521 return indices
522
523 @staticmethod
524 def _prepareEmptyExposure(skyInfo):
525 """Produce an empty exposure for a given patch.
526
527 Parameters
528 ----------
529 skyInfo : `lsst.pipe.base.Struct`
530 Struct from `~lsst.pipe.base.coaddBase.makeSkyInfo()` with
531 geometric information about the patch.
532
533 Returns
534 -------
535 exp : `lsst.afw.image.exposure.ExposureF`
536 An empty exposure for a given patch.
537 """
538 exp = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
539 exp.getMaskedImage().set(numpy.nan, afwImage.Mask
540 .getPlaneBitMask("NO_DATA"), numpy.inf)
541 return exp
542
543 def getWarpTypeList(self):
544 """Return list of requested warp types per the config.
545 """
546 warpTypeList = []
547 if self.config.makeDirect:
548 warpTypeList.append("direct")
549 if self.config.makePsfMatched:
550 warpTypeList.append("psfMatched")
551 return warpTypeList
552
553
554def reorderRefs(inputRefs, outputSortKeyOrder, dataIdKey):
555 """Reorder inputRefs per outputSortKeyOrder.
556
557 Any inputRefs which are lists will be resorted per specified key e.g.,
558 'detector.' Only iterables will be reordered, and values can be of type
559 `lsst.pipe.base.connections.DeferredDatasetRef` or
560 `lsst.daf.butler.core.datasets.ref.DatasetRef`.
561
562 Returned lists of refs have the same length as the outputSortKeyOrder.
563 If an outputSortKey not in the inputRef, then it will be padded with None.
564 If an inputRef contains an inputSortKey that is not in the
565 outputSortKeyOrder it will be removed.
566
567 Parameters
568 ----------
569 inputRefs : `lsst.pipe.base.connections.QuantizedConnection`
570 Input references to be reordered and padded.
571 outputSortKeyOrder : `iterable`
572 Iterable of values to be compared with inputRef's dataId[dataIdKey].
573 dataIdKey : `str`
574 The data ID key in the dataRefs to compare with the outputSortKeyOrder.
575
576 Returns
577 -------
578 inputRefs : `lsst.pipe.base.connections.QuantizedConnection`
579 Quantized Connection with sorted DatasetRef values sorted if iterable.
580 """
581 for connectionName, refs in inputRefs:
582 if isinstance(refs, Iterable):
583 if hasattr(refs[0], "dataId"):
584 inputSortKeyOrder = [ref.dataId[dataIdKey] for ref in refs]
585 else:
586 inputSortKeyOrder = [ref.datasetRef.dataId[dataIdKey] for ref in refs]
587 if inputSortKeyOrder != outputSortKeyOrder:
588 setattr(inputRefs, connectionName,
589 reorderAndPadList(refs, inputSortKeyOrder, outputSortKeyOrder))
590 return inputRefs