Coverage for python/lsst/pipe/tasks/makeCoaddTempExp.py: 16%
343 statements
« prev ^ index » next coverage.py v6.4.2, created at 2022-07-17 08:57 +0000
« prev ^ index » next coverage.py v6.4.2, created at 2022-07-17 08:57 +0000
1#
2# LSST Data Management System
3# Copyright 2008, 2009, 2010, 2011, 2012 LSST Corporation.
4#
5# This product includes software developed by the
6# LSST Project (http://www.lsst.org/).
7#
8# This program is free software: you can redistribute it and/or modify
9# it under the terms of the GNU General Public License as published by
10# the Free Software Foundation, either version 3 of the License, or
11# (at your option) any later version.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the LSST License Statement and
19# the GNU General Public License along with this program. If not,
20# see <http://www.lsstcorp.org/LegalNotices/>.
21#
22import numpy
23import logging
25import lsst.pex.config as pexConfig
26import lsst.daf.persistence as dafPersist
27import lsst.afw.image as afwImage
28import lsst.coadd.utils as coaddUtils
29import lsst.pipe.base as pipeBase
30import lsst.pipe.base.connectionTypes as connectionTypes
31import lsst.utils as utils
32import lsst.geom
33from lsst.meas.algorithms import CoaddPsf, CoaddPsfConfig
34from lsst.skymap import BaseSkyMap
35from .coaddBase import CoaddBaseTask, makeSkyInfo, reorderAndPadList
36from .selectImages import PsfWcsSelectImagesTask
37from .warpAndPsfMatch import WarpAndPsfMatchTask
38from .coaddHelpers import groupPatchExposures, getGroupDataRef
39from collections.abc import Iterable
41__all__ = ["MakeCoaddTempExpTask", "MakeWarpTask", "MakeWarpConfig"]
43log = logging.getLogger(__name__.partition(".")[2])
46class MissingExposureError(Exception):
47 """Raised when data cannot be retrieved for an exposure.
48 When processing patches, sometimes one exposure is missing; this lets us
49 distinguish bewteen that case, and other errors.
50 """
51 pass
54class MakeCoaddTempExpConfig(CoaddBaseTask.ConfigClass):
55 """Config for MakeCoaddTempExpTask
56 """
57 warpAndPsfMatch = pexConfig.ConfigurableField(
58 target=WarpAndPsfMatchTask,
59 doc="Task to warp and PSF-match calexp",
60 )
61 doWrite = pexConfig.Field(
62 doc="persist <coaddName>Coadd_<warpType>Warp",
63 dtype=bool,
64 default=True,
65 )
66 bgSubtracted = pexConfig.Field(
67 doc="Work with a background subtracted calexp?",
68 dtype=bool,
69 default=True,
70 )
71 coaddPsf = pexConfig.ConfigField(
72 doc="Configuration for CoaddPsf",
73 dtype=CoaddPsfConfig,
74 )
75 makeDirect = pexConfig.Field(
76 doc="Make direct Warp/Coadds",
77 dtype=bool,
78 default=True,
79 )
80 makePsfMatched = pexConfig.Field(
81 doc="Make Psf-Matched Warp/Coadd?",
82 dtype=bool,
83 default=False,
84 )
86 doWriteEmptyWarps = pexConfig.Field(
87 dtype=bool,
88 default=False,
89 doc="Write out warps even if they are empty"
90 )
92 hasFakes = pexConfig.Field(
93 doc="Should be set to True if fake sources have been inserted into the input data.",
94 dtype=bool,
95 default=False,
96 )
97 doApplySkyCorr = pexConfig.Field(dtype=bool, default=False, doc="Apply sky correction?")
99 def validate(self):
100 CoaddBaseTask.ConfigClass.validate(self)
101 if not self.makePsfMatched and not self.makeDirect:
102 raise RuntimeError("At least one of config.makePsfMatched and config.makeDirect must be True")
103 if self.doPsfMatch:
104 # Backwards compatibility.
105 log.warning("Config doPsfMatch deprecated. Setting makePsfMatched=True and makeDirect=False")
106 self.makePsfMatched = True
107 self.makeDirect = False
109 def setDefaults(self):
110 CoaddBaseTask.ConfigClass.setDefaults(self)
111 self.warpAndPsfMatch.psfMatch.kernel.active.kernelSize = self.matchingKernelSize
112 self.select.retarget(PsfWcsSelectImagesTask)
114## \addtogroup LSST_task_documentation
115## \{
116## \page MakeCoaddTempExpTask
117## \ref MakeCoaddTempExpTask_ "MakeCoaddTempExpTask"
118## \copybrief MakeCoaddTempExpTask
119## \}
122class MakeCoaddTempExpTask(CoaddBaseTask):
123 r"""!Warp and optionally PSF-Match calexps onto an a common projection.
125 @anchor MakeCoaddTempExpTask_
127 @section pipe_tasks_makeCoaddTempExp_Contents Contents
129 - @ref pipe_tasks_makeCoaddTempExp_Purpose
130 - @ref pipe_tasks_makeCoaddTempExp_Initialize
131 - @ref pipe_tasks_makeCoaddTempExp_IO
132 - @ref pipe_tasks_makeCoaddTempExp_Config
133 - @ref pipe_tasks_makeCoaddTempExp_Debug
134 - @ref pipe_tasks_makeCoaddTempExp_Example
136 @section pipe_tasks_makeCoaddTempExp_Purpose Description
138 Warp and optionally PSF-Match calexps onto a common projection, by
139 performing the following operations:
140 - Group calexps by visit/run
141 - For each visit, generate a Warp by calling method @ref makeTempExp.
142 makeTempExp loops over the visit's calexps calling @ref WarpAndPsfMatch
143 on each visit
145 The result is a `directWarp` (and/or optionally a `psfMatchedWarp`).
147 @section pipe_tasks_makeCoaddTempExp_Initialize Task Initialization
149 @copydoc \_\_init\_\_
151 This task has one special keyword argument: passing reuse=True will cause
152 the task to skip the creation of warps that are already present in the
153 output repositories.
155 @section pipe_tasks_makeCoaddTempExp_IO Invoking the Task
157 This task is primarily designed to be run from the command line.
159 The main method is `runDataRef`, which takes a single butler data reference for the patch(es)
160 to process.
162 @copydoc run
164 WarpType identifies the types of convolutions applied to Warps (previously CoaddTempExps).
165 Only two types are available: direct (for regular Warps/Coadds) and psfMatched
166 (for Warps/Coadds with homogenized PSFs). We expect to add a third type, likelihood,
167 for generating likelihood Coadds with Warps that have been correlated with their own PSF.
169 @section pipe_tasks_makeCoaddTempExp_Config Configuration parameters
171 See @ref MakeCoaddTempExpConfig and parameters inherited from
172 @link lsst.pipe.tasks.coaddBase.CoaddBaseConfig CoaddBaseConfig @endlink
174 @subsection pipe_tasks_MakeCoaddTempExp_psfMatching Guide to PSF-Matching Configs
176 To make `psfMatchedWarps`, select `config.makePsfMatched=True`. The subtask
177 @link lsst.ip.diffim.modelPsfMatch.ModelPsfMatchTask ModelPsfMatchTask @endlink
178 is responsible for the PSF-Matching, and its config is accessed via `config.warpAndPsfMatch.psfMatch`.
179 The optimal configuration depends on aspects of dataset: the pixel scale, average PSF FWHM and
180 dimensions of the PSF kernel. These configs include the requested model PSF, the matching kernel size,
181 padding of the science PSF thumbnail and spatial sampling frequency of the PSF.
183 *Config Guidelines*: The user must specify the size of the model PSF to which to match by setting
184 `config.modelPsf.defaultFwhm` in units of pixels. The appropriate values depends on science case.
185 In general, for a set of input images, this config should equal the FWHM of the visit
186 with the worst seeing. The smallest it should be set to is the median FWHM. The defaults
187 of the other config options offer a reasonable starting point.
188 The following list presents the most common problems that arise from a misconfigured
189 @link lsst.ip.diffim.modelPsfMatch.ModelPsfMatchTask ModelPsfMatchTask @endlink
190 and corresponding solutions. All assume the default Alard-Lupton kernel, with configs accessed via
191 ```config.warpAndPsfMatch.psfMatch.kernel['AL']```. Each item in the list is formatted as:
192 Problem: Explanation. *Solution*
194 *Troublshooting PSF-Matching Configuration:*
195 - Matched PSFs look boxy: The matching kernel is too small. _Increase the matching kernel size.
196 For example:_
198 config.warpAndPsfMatch.psfMatch.kernel['AL'].kernelSize=27 # default 21
200 Note that increasing the kernel size also increases runtime.
201 - Matched PSFs look ugly (dipoles, quadropoles, donuts): unable to find good solution
202 for matching kernel. _Provide the matcher with more data by either increasing
203 the spatial sampling by decreasing the spatial cell size,_
205 config.warpAndPsfMatch.psfMatch.kernel['AL'].sizeCellX = 64 # default 128
206 config.warpAndPsfMatch.psfMatch.kernel['AL'].sizeCellY = 64 # default 128
208 _or increasing the padding around the Science PSF, for example:_
210 config.warpAndPsfMatch.psfMatch.autoPadPsfTo=1.6 # default 1.4
212 Increasing `autoPadPsfTo` increases the minimum ratio of input PSF dimensions to the
213 matching kernel dimensions, thus increasing the number of pixels available to fit
214 after convolving the PSF with the matching kernel.
215 Optionally, for debugging the effects of padding, the level of padding may be manually
216 controlled by setting turning off the automatic padding and setting the number
217 of pixels by which to pad the PSF:
219 config.warpAndPsfMatch.psfMatch.doAutoPadPsf = False # default True
220 config.warpAndPsfMatch.psfMatch.padPsfBy = 6 # pixels. default 0
222 - Deconvolution: Matching a large PSF to a smaller PSF produces
223 a telltale noise pattern which looks like ripples or a brain.
224 _Increase the size of the requested model PSF. For example:_
226 config.modelPsf.defaultFwhm = 11 # Gaussian sigma in units of pixels.
228 - High frequency (sometimes checkered) noise: The matching basis functions are too small.
229 _Increase the width of the Gaussian basis functions. For example:_
231 config.warpAndPsfMatch.psfMatch.kernel['AL'].alardSigGauss=[1.5, 3.0, 6.0]
232 # from default [0.7, 1.5, 3.0]
235 @section pipe_tasks_makeCoaddTempExp_Debug Debug variables
237 MakeCoaddTempExpTask has no debug output, but its subtasks do.
239 @section pipe_tasks_makeCoaddTempExp_Example A complete example of using MakeCoaddTempExpTask
241 This example uses the package ci_hsc to show how MakeCoaddTempExp fits
242 into the larger Data Release Processing.
243 Set up by running:
245 setup ci_hsc
246 cd $CI_HSC_DIR
247 # if not built already:
248 python $(which scons) # this will take a while
250 The following assumes that `processCcd.py` and `makeSkyMap.py` have previously been run
251 (e.g. by building `ci_hsc` above) to generate a repository of calexps and an
252 output respository with the desired SkyMap. The command,
254 makeCoaddTempExp.py $CI_HSC_DIR/DATA --rerun ci_hsc \
255 --id patch=5,4 tract=0 filter=HSC-I \
256 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 \
257 --selectId visit=903988 ccd=23 --selectId visit=903988 ccd=24 \
258 --config doApplyExternalPhotoCalib=False doApplyExternalSkyWcs=False \
259 makePsfMatched=True modelPsf.defaultFwhm=11
261 writes a direct and PSF-Matched Warp to
262 - `$CI_HSC_DIR/DATA/rerun/ci_hsc/deepCoadd/HSC-I/0/5,4/warp-HSC-I-0-5,4-903988.fits` and
263 - `$CI_HSC_DIR/DATA/rerun/ci_hsc/deepCoadd/HSC-I/0/5,4/psfMatchedWarp-HSC-I-0-5,4-903988.fits`
264 respectively.
266 @note PSF-Matching in this particular dataset would benefit from adding
267 `--configfile ./matchingConfig.py` to
268 the command line arguments where `matchingConfig.py` is defined by:
270 echo "
271 config.warpAndPsfMatch.psfMatch.kernel['AL'].kernelSize=27
272 config.warpAndPsfMatch.psfMatch.kernel['AL'].alardSigGauss=[1.5, 3.0, 6.0]" > matchingConfig.py
275 Add the option `--help` to see more options.
276 """
277 ConfigClass = MakeCoaddTempExpConfig
278 _DefaultName = "makeCoaddTempExp"
280 def __init__(self, reuse=False, **kwargs):
281 CoaddBaseTask.__init__(self, **kwargs)
282 self.reuse = reuse
283 self.makeSubtask("warpAndPsfMatch")
284 if self.config.hasFakes:
285 self.calexpType = "fakes_calexp"
286 else:
287 self.calexpType = "calexp"
289 @pipeBase.timeMethod
290 def runDataRef(self, patchRef, selectDataList=[]):
291 """!Produce <coaddName>Coadd_<warpType>Warp images by warping and optionally PSF-matching.
293 @param[in] patchRef: data reference for sky map patch. Must include keys "tract", "patch",
294 plus the camera-specific filter key (e.g. "filter" or "band")
295 @return: dataRefList: a list of data references for the new <coaddName>Coadd_directWarps
296 if direct or both warp types are requested and <coaddName>Coadd_psfMatchedWarps if only psfMatched
297 warps are requested.
299 @warning: this task assumes that all exposures in a warp (coaddTempExp) have the same filter.
301 @warning: this task sets the PhotoCalib of the coaddTempExp to the PhotoCalib of the first calexp
302 with any good pixels in the patch. For a mosaic camera the resulting PhotoCalib should be ignored
303 (assembleCoadd should determine zeropoint scaling without referring to it).
304 """
305 skyInfo = self.getSkyInfo(patchRef)
307 # DataRefs to return are of type *_directWarp unless only *_psfMatchedWarp requested
308 if self.config.makePsfMatched and not self.config.makeDirect:
309 primaryWarpDataset = self.getTempExpDatasetName("psfMatched")
310 else:
311 primaryWarpDataset = self.getTempExpDatasetName("direct")
313 calExpRefList = self.selectExposures(patchRef, skyInfo, selectDataList=selectDataList)
315 if len(calExpRefList) == 0:
316 self.log.warning("No exposures to coadd for patch %s", patchRef.dataId)
317 return None
318 self.log.info("Selected %d calexps for patch %s", len(calExpRefList), patchRef.dataId)
319 calExpRefList = [calExpRef for calExpRef in calExpRefList if calExpRef.datasetExists(self.calexpType)]
320 self.log.info("Processing %d existing calexps for patch %s", len(calExpRefList), patchRef.dataId)
322 groupData = groupPatchExposures(patchRef, calExpRefList, self.getCoaddDatasetName(),
323 primaryWarpDataset)
324 self.log.info("Processing %d warp exposures for patch %s", len(groupData.groups), patchRef.dataId)
326 dataRefList = []
327 for i, (tempExpTuple, calexpRefList) in enumerate(groupData.groups.items()):
328 tempExpRef = getGroupDataRef(patchRef.getButler(), primaryWarpDataset,
329 tempExpTuple, groupData.keys)
330 if self.reuse and tempExpRef.datasetExists(datasetType=primaryWarpDataset, write=True):
331 self.log.info("Skipping makeCoaddTempExp for %s; output already exists.", tempExpRef.dataId)
332 dataRefList.append(tempExpRef)
333 continue
334 self.log.info("Processing Warp %d/%d: id=%s", i, len(groupData.groups), tempExpRef.dataId)
336 # TODO: mappers should define a way to go from the "grouping keys" to a numeric ID (#2776).
337 # For now, we try to get a long integer "visit" key, and if we can't, we just use the index
338 # of the visit in the list.
339 try:
340 visitId = int(tempExpRef.dataId["visit"])
341 except (KeyError, ValueError):
342 visitId = i
344 calExpList = []
345 ccdIdList = []
346 dataIdList = []
348 for calExpInd, calExpRef in enumerate(calexpRefList):
349 self.log.info("Reading calexp %s of %s for Warp id=%s", calExpInd+1, len(calexpRefList),
350 calExpRef.dataId)
351 try:
352 ccdId = calExpRef.get("ccdExposureId", immediate=True)
353 except Exception:
354 ccdId = calExpInd
355 try:
356 # We augment the dataRef here with the tract, which is harmless for loading things
357 # like calexps that don't need the tract, and necessary for meas_mosaic outputs,
358 # which do.
359 calExpRef = calExpRef.butlerSubset.butler.dataRef(self.calexpType,
360 dataId=calExpRef.dataId,
361 tract=skyInfo.tractInfo.getId())
362 calExp = self.getCalibratedExposure(calExpRef, bgSubtracted=self.config.bgSubtracted)
363 except Exception as e:
364 self.log.warning("Calexp %s not found; skipping it: %s", calExpRef.dataId, e)
365 continue
367 if self.config.doApplySkyCorr:
368 self.applySkyCorr(calExpRef, calExp)
370 calExpList.append(calExp)
371 ccdIdList.append(ccdId)
372 dataIdList.append(calExpRef.dataId)
374 exps = self.run(calExpList, ccdIdList, skyInfo, visitId, dataIdList).exposures
376 if any(exps.values()):
377 dataRefList.append(tempExpRef)
378 else:
379 self.log.warning("Warp %s could not be created", tempExpRef.dataId)
381 if self.config.doWrite:
382 for (warpType, exposure) in exps.items(): # compatible w/ Py3
383 if exposure is not None:
384 self.log.info("Persisting %s", self.getTempExpDatasetName(warpType))
385 tempExpRef.put(exposure, self.getTempExpDatasetName(warpType))
387 return dataRefList
389 @pipeBase.timeMethod
390 def run(self, calExpList, ccdIdList, skyInfo, visitId=0, dataIdList=None, **kwargs):
391 """Create a Warp from inputs
393 We iterate over the multiple calexps in a single exposure to construct
394 the warp (previously called a coaddTempExp) of that exposure to the
395 supplied tract/patch.
397 Pixels that receive no pixels are set to NAN; this is not correct
398 (violates LSST algorithms group policy), but will be fixed up by
399 interpolating after the coaddition.
401 @param calexpRefList: List of data references for calexps that (may)
402 overlap the patch of interest
403 @param skyInfo: Struct from CoaddBaseTask.getSkyInfo() with geometric
404 information about the patch
405 @param visitId: integer identifier for visit, for the table that will
406 produce the CoaddPsf
407 @return a pipeBase Struct containing:
408 - exposures: a dictionary containing the warps requested:
409 "direct": direct warp if config.makeDirect
410 "psfMatched": PSF-matched warp if config.makePsfMatched
411 """
412 warpTypeList = self.getWarpTypeList()
414 totGoodPix = {warpType: 0 for warpType in warpTypeList}
415 didSetMetadata = {warpType: False for warpType in warpTypeList}
416 coaddTempExps = {warpType: self._prepareEmptyExposure(skyInfo) for warpType in warpTypeList}
417 inputRecorder = {warpType: self.inputRecorder.makeCoaddTempExpRecorder(visitId, len(calExpList))
418 for warpType in warpTypeList}
420 modelPsf = self.config.modelPsf.apply() if self.config.makePsfMatched else None
421 if dataIdList is None:
422 dataIdList = ccdIdList
424 for calExpInd, (calExp, ccdId, dataId) in enumerate(zip(calExpList, ccdIdList, dataIdList)):
425 self.log.info("Processing calexp %d of %d for this Warp: id=%s",
426 calExpInd+1, len(calExpList), dataId)
428 try:
429 warpedAndMatched = self.warpAndPsfMatch.run(calExp, modelPsf=modelPsf,
430 wcs=skyInfo.wcs, maxBBox=skyInfo.bbox,
431 makeDirect=self.config.makeDirect,
432 makePsfMatched=self.config.makePsfMatched)
433 except Exception as e:
434 self.log.warning("WarpAndPsfMatch failed for calexp %s; skipping it: %s", dataId, e)
435 continue
436 try:
437 numGoodPix = {warpType: 0 for warpType in warpTypeList}
438 for warpType in warpTypeList:
439 exposure = warpedAndMatched.getDict()[warpType]
440 if exposure is None:
441 continue
442 coaddTempExp = coaddTempExps[warpType]
443 if didSetMetadata[warpType]:
444 mimg = exposure.getMaskedImage()
445 mimg *= (coaddTempExp.getPhotoCalib().getInstFluxAtZeroMagnitude()
446 / exposure.getPhotoCalib().getInstFluxAtZeroMagnitude())
447 del mimg
448 numGoodPix[warpType] = coaddUtils.copyGoodPixels(
449 coaddTempExp.getMaskedImage(), exposure.getMaskedImage(), self.getBadPixelMask())
450 totGoodPix[warpType] += numGoodPix[warpType]
451 self.log.debug("Calexp %s has %d good pixels in this patch (%.1f%%) for %s",
452 dataId, numGoodPix[warpType],
453 100.0*numGoodPix[warpType]/skyInfo.bbox.getArea(), warpType)
454 if numGoodPix[warpType] > 0 and not didSetMetadata[warpType]:
455 coaddTempExp.setPhotoCalib(exposure.getPhotoCalib())
456 coaddTempExp.setFilterLabel(exposure.getFilterLabel())
457 coaddTempExp.getInfo().setVisitInfo(exposure.getInfo().getVisitInfo())
458 # PSF replaced with CoaddPsf after loop if and only if creating direct warp
459 coaddTempExp.setPsf(exposure.getPsf())
460 didSetMetadata[warpType] = True
462 # Need inputRecorder for CoaddApCorrMap for both direct and PSF-matched
463 inputRecorder[warpType].addCalExp(calExp, ccdId, numGoodPix[warpType])
465 except Exception as e:
466 self.log.warning("Error processing calexp %s; skipping it: %s", dataId, e)
467 continue
469 for warpType in warpTypeList:
470 self.log.info("%sWarp has %d good pixels (%.1f%%)",
471 warpType, totGoodPix[warpType], 100.0*totGoodPix[warpType]/skyInfo.bbox.getArea())
473 if totGoodPix[warpType] > 0 and didSetMetadata[warpType]:
474 inputRecorder[warpType].finish(coaddTempExps[warpType], totGoodPix[warpType])
475 if warpType == "direct":
476 coaddTempExps[warpType].setPsf(
477 CoaddPsf(inputRecorder[warpType].coaddInputs.ccds, skyInfo.wcs,
478 self.config.coaddPsf.makeControl()))
479 else:
480 if not self.config.doWriteEmptyWarps:
481 # No good pixels. Exposure still empty
482 coaddTempExps[warpType] = None
483 # NoWorkFound is unnecessary as the downstream tasks will
484 # adjust the quantum accordingly, and it prevents gen2
485 # MakeCoaddTempExp from continuing to loop over visits.
487 result = pipeBase.Struct(exposures=coaddTempExps)
488 return result
490 def getCalibratedExposure(self, dataRef, bgSubtracted):
491 """Return one calibrated Exposure, possibly with an updated SkyWcs.
493 @param[in] dataRef a sensor-level data reference
494 @param[in] bgSubtracted return calexp with background subtracted? If False get the
495 calexp's background background model and add it to the calexp.
496 @return calibrated exposure
498 @raises MissingExposureError If data for the exposure is not available.
500 If config.doApplyExternalPhotoCalib is `True`, the photometric calibration
501 (`photoCalib`) is taken from `config.externalPhotoCalibName` via the
502 `name_photoCalib` dataset. Otherwise, the photometric calibration is
503 retrieved from the processed exposure. When
504 `config.doApplyExternalSkyWcs` is `True`, the astrometric calibration
505 is taken from `config.externalSkyWcsName` with the `name_wcs` dataset.
506 Otherwise, the astrometric calibration is taken from the processed
507 exposure.
508 """
509 try:
510 exposure = dataRef.get(self.calexpType, immediate=True)
511 except dafPersist.NoResults as e:
512 raise MissingExposureError('Exposure not found: %s ' % str(e)) from e
514 if not bgSubtracted:
515 background = dataRef.get("calexpBackground", immediate=True)
516 mi = exposure.getMaskedImage()
517 mi += background.getImage()
518 del mi
520 if self.config.doApplyExternalPhotoCalib:
521 source = f"{self.config.externalPhotoCalibName}_photoCalib"
522 self.log.debug("Applying external photoCalib to %s from %s", dataRef.dataId, source)
523 photoCalib = dataRef.get(source)
524 exposure.setPhotoCalib(photoCalib)
525 else:
526 photoCalib = exposure.getPhotoCalib()
528 if self.config.doApplyExternalSkyWcs:
529 source = f"{self.config.externalSkyWcsName}_wcs"
530 self.log.debug("Applying external skyWcs to %s from %s", dataRef.dataId, source)
531 skyWcs = dataRef.get(source)
532 exposure.setWcs(skyWcs)
534 exposure.maskedImage = photoCalib.calibrateImage(exposure.maskedImage,
535 includeScaleUncertainty=self.config.includeCalibVar)
536 exposure.maskedImage /= photoCalib.getCalibrationMean()
537 # TODO: The images will have a calibration of 1.0 everywhere once RFC-545 is implemented.
538 # exposure.setCalib(afwImage.Calib(1.0))
539 return exposure
541 @staticmethod
542 def _prepareEmptyExposure(skyInfo):
543 """Produce an empty exposure for a given patch"""
544 exp = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
545 exp.getMaskedImage().set(numpy.nan, afwImage.Mask
546 .getPlaneBitMask("NO_DATA"), numpy.inf)
547 return exp
549 def getWarpTypeList(self):
550 """Return list of requested warp types per the config.
551 """
552 warpTypeList = []
553 if self.config.makeDirect:
554 warpTypeList.append("direct")
555 if self.config.makePsfMatched:
556 warpTypeList.append("psfMatched")
557 return warpTypeList
559 def applySkyCorr(self, dataRef, calexp):
560 """Apply correction to the sky background level
562 Sky corrections can be generated with the 'skyCorrection.py'
563 executable in pipe_drivers. Because the sky model used by that
564 code extends over the entire focal plane, this can produce
565 better sky subtraction.
567 The calexp is updated in-place.
569 Parameters
570 ----------
571 dataRef : `lsst.daf.persistence.ButlerDataRef`
572 Data reference for calexp.
573 calexp : `lsst.afw.image.Exposure` or `lsst.afw.image.MaskedImage`
574 Calibrated exposure.
575 """
576 bg = dataRef.get("skyCorr")
577 self.log.debug("Applying sky correction to %s", dataRef.dataId)
578 if isinstance(calexp, afwImage.Exposure):
579 calexp = calexp.getMaskedImage()
580 calexp -= bg.getImage()
583class MakeWarpConnections(pipeBase.PipelineTaskConnections,
584 dimensions=("tract", "patch", "skymap", "instrument", "visit"),
585 defaultTemplates={"coaddName": "deep",
586 "skyWcsName": "jointcal",
587 "photoCalibName": "fgcm",
588 "calexpType": ""}):
589 calExpList = connectionTypes.Input(
590 doc="Input exposures to be resampled and optionally PSF-matched onto a SkyMap projection/patch",
591 name="{calexpType}calexp",
592 storageClass="ExposureF",
593 dimensions=("instrument", "visit", "detector"),
594 multiple=True,
595 deferLoad=True,
596 )
597 backgroundList = connectionTypes.Input(
598 doc="Input backgrounds to be added back into the calexp if bgSubtracted=False",
599 name="calexpBackground",
600 storageClass="Background",
601 dimensions=("instrument", "visit", "detector"),
602 multiple=True,
603 )
604 skyCorrList = connectionTypes.Input(
605 doc="Input Sky Correction to be subtracted from the calexp if doApplySkyCorr=True",
606 name="skyCorr",
607 storageClass="Background",
608 dimensions=("instrument", "visit", "detector"),
609 multiple=True,
610 )
611 skyMap = connectionTypes.Input(
612 doc="Input definition of geometry/bbox and projection/wcs for warped exposures",
613 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
614 storageClass="SkyMap",
615 dimensions=("skymap",),
616 )
617 externalSkyWcsTractCatalog = connectionTypes.Input(
618 doc=("Per-tract, per-visit wcs calibrations. These catalogs use the detector "
619 "id for the catalog id, sorted on id for fast lookup."),
620 name="{skyWcsName}SkyWcsCatalog",
621 storageClass="ExposureCatalog",
622 dimensions=("instrument", "visit", "tract"),
623 )
624 externalSkyWcsGlobalCatalog = connectionTypes.Input(
625 doc=("Per-visit wcs calibrations computed globally (with no tract information). "
626 "These catalogs use the detector id for the catalog id, sorted on id for "
627 "fast lookup."),
628 name="{skyWcsName}SkyWcsCatalog",
629 storageClass="ExposureCatalog",
630 dimensions=("instrument", "visit"),
631 )
632 externalPhotoCalibTractCatalog = connectionTypes.Input(
633 doc=("Per-tract, per-visit photometric calibrations. These catalogs use the "
634 "detector id for the catalog id, sorted on id for fast lookup."),
635 name="{photoCalibName}PhotoCalibCatalog",
636 storageClass="ExposureCatalog",
637 dimensions=("instrument", "visit", "tract"),
638 )
639 externalPhotoCalibGlobalCatalog = connectionTypes.Input(
640 doc=("Per-visit photometric calibrations computed globally (with no tract "
641 "information). These catalogs use the detector id for the catalog id, "
642 "sorted on id for fast lookup."),
643 name="{photoCalibName}PhotoCalibCatalog",
644 storageClass="ExposureCatalog",
645 dimensions=("instrument", "visit"),
646 )
647 direct = connectionTypes.Output(
648 doc=("Output direct warped exposure (previously called CoaddTempExp), produced by resampling ",
649 "calexps onto the skyMap patch geometry."),
650 name="{coaddName}Coadd_directWarp",
651 storageClass="ExposureF",
652 dimensions=("tract", "patch", "skymap", "visit", "instrument"),
653 )
654 psfMatched = connectionTypes.Output(
655 doc=("Output PSF-Matched warped exposure (previously called CoaddTempExp), produced by resampling ",
656 "calexps onto the skyMap patch geometry and PSF-matching to a model PSF."),
657 name="{coaddName}Coadd_psfMatchedWarp",
658 storageClass="ExposureF",
659 dimensions=("tract", "patch", "skymap", "visit", "instrument"),
660 )
661 # TODO DM-28769, have selectImages subtask indicate which connections they need:
662 wcsList = connectionTypes.Input(
663 doc="WCSs of calexps used by SelectImages subtask to determine if the calexp overlaps the patch",
664 name="{calexpType}calexp.wcs",
665 storageClass="Wcs",
666 dimensions=("instrument", "visit", "detector"),
667 multiple=True,
668 )
669 bboxList = connectionTypes.Input(
670 doc="BBoxes of calexps used by SelectImages subtask to determine if the calexp overlaps the patch",
671 name="{calexpType}calexp.bbox",
672 storageClass="Box2I",
673 dimensions=("instrument", "visit", "detector"),
674 multiple=True,
675 )
676 visitSummary = connectionTypes.Input(
677 doc="Consolidated exposure metadata from ConsolidateVisitSummaryTask",
678 name="{calexpType}visitSummary",
679 storageClass="ExposureCatalog",
680 dimensions=("instrument", "visit",),
681 )
682 srcList = connectionTypes.Input(
683 doc="Source catalogs used by PsfWcsSelectImages subtask to further select on PSF stability",
684 name="src",
685 storageClass="SourceCatalog",
686 dimensions=("instrument", "visit", "detector"),
687 multiple=True,
688 )
690 def __init__(self, *, config=None):
691 super().__init__(config=config)
692 if config.bgSubtracted:
693 self.inputs.remove("backgroundList")
694 if not config.doApplySkyCorr:
695 self.inputs.remove("skyCorrList")
696 if config.doApplyExternalSkyWcs:
697 if config.useGlobalExternalSkyWcs:
698 self.inputs.remove("externalSkyWcsTractCatalog")
699 else:
700 self.inputs.remove("externalSkyWcsGlobalCatalog")
701 else:
702 self.inputs.remove("externalSkyWcsTractCatalog")
703 self.inputs.remove("externalSkyWcsGlobalCatalog")
704 if config.doApplyExternalPhotoCalib:
705 if config.useGlobalExternalPhotoCalib:
706 self.inputs.remove("externalPhotoCalibTractCatalog")
707 else:
708 self.inputs.remove("externalPhotoCalibGlobalCatalog")
709 else:
710 self.inputs.remove("externalPhotoCalibTractCatalog")
711 self.inputs.remove("externalPhotoCalibGlobalCatalog")
712 if not config.makeDirect:
713 self.outputs.remove("direct")
714 if not config.makePsfMatched:
715 self.outputs.remove("psfMatched")
716 # TODO DM-28769: add connection per selectImages connections
717 if config.select.target != lsst.pipe.tasks.selectImages.PsfWcsSelectImagesTask:
718 self.inputs.remove("visitSummary")
719 self.inputs.remove("srcList")
720 elif not config.select.doLegacyStarSelectionComputation:
721 # Remove backwards-compatibility connections.
722 self.inputs.remove("srcList")
725class MakeWarpConfig(pipeBase.PipelineTaskConfig, MakeCoaddTempExpConfig,
726 pipelineConnections=MakeWarpConnections):
728 def validate(self):
729 super().validate()
732class MakeWarpTask(MakeCoaddTempExpTask):
733 """Warp and optionally PSF-Match calexps onto an a common projection
734 """
735 ConfigClass = MakeWarpConfig
736 _DefaultName = "makeWarp"
738 @utils.inheritDoc(pipeBase.PipelineTask)
739 def runQuantum(self, butlerQC, inputRefs, outputRefs):
740 """
741 Notes
742 ----
743 Construct warps for requested warp type for single epoch
745 PipelineTask (Gen3) entry point to warp and optionally PSF-match
746 calexps. This method is analogous to `runDataRef`.
747 """
749 # Ensure all input lists are in same detector order as the calExpList
750 detectorOrder = [ref.datasetRef.dataId['detector'] for ref in inputRefs.calExpList]
751 inputRefs = reorderRefs(inputRefs, detectorOrder, dataIdKey='detector')
753 # Read in all inputs.
754 inputs = butlerQC.get(inputRefs)
756 # Construct skyInfo expected by `run`. We remove the SkyMap itself
757 # from the dictionary so we can pass it as kwargs later.
758 skyMap = inputs.pop("skyMap")
759 quantumDataId = butlerQC.quantum.dataId
760 skyInfo = makeSkyInfo(skyMap, tractId=quantumDataId['tract'], patchId=quantumDataId['patch'])
762 # Construct list of input DataIds expected by `run`
763 dataIdList = [ref.datasetRef.dataId for ref in inputRefs.calExpList]
764 # Construct list of packed integer IDs expected by `run`
765 ccdIdList = [dataId.pack("visit_detector") for dataId in dataIdList]
767 # Run the selector and filter out calexps that were not selected
768 # primarily because they do not overlap the patch
769 cornerPosList = lsst.geom.Box2D(skyInfo.bbox).getCorners()
770 coordList = [skyInfo.wcs.pixelToSky(pos) for pos in cornerPosList]
771 goodIndices = self.select.run(**inputs, coordList=coordList, dataIds=dataIdList)
772 inputs = self.filterInputs(indices=goodIndices, inputs=inputs)
774 # Read from disk only the selected calexps
775 inputs['calExpList'] = [ref.get() for ref in inputs['calExpList']]
777 # Extract integer visitId requested by `run`
778 visits = [dataId['visit'] for dataId in dataIdList]
779 visitId = visits[0]
781 if self.config.doApplyExternalSkyWcs:
782 if self.config.useGlobalExternalSkyWcs:
783 externalSkyWcsCatalog = inputs.pop("externalSkyWcsGlobalCatalog")
784 else:
785 externalSkyWcsCatalog = inputs.pop("externalSkyWcsTractCatalog")
786 else:
787 externalSkyWcsCatalog = None
789 if self.config.doApplyExternalPhotoCalib:
790 if self.config.useGlobalExternalPhotoCalib:
791 externalPhotoCalibCatalog = inputs.pop("externalPhotoCalibGlobalCatalog")
792 else:
793 externalPhotoCalibCatalog = inputs.pop("externalPhotoCalibTractCatalog")
794 else:
795 externalPhotoCalibCatalog = None
797 completeIndices = self.prepareCalibratedExposures(**inputs,
798 externalSkyWcsCatalog=externalSkyWcsCatalog,
799 externalPhotoCalibCatalog=externalPhotoCalibCatalog)
800 # Redo the input selection with inputs with complete wcs/photocalib info.
801 inputs = self.filterInputs(indices=completeIndices, inputs=inputs)
803 results = self.run(**inputs, visitId=visitId,
804 ccdIdList=[ccdIdList[i] for i in goodIndices],
805 dataIdList=[dataIdList[i] for i in goodIndices],
806 skyInfo=skyInfo)
807 if self.config.makeDirect and results.exposures["direct"] is not None:
808 butlerQC.put(results.exposures["direct"], outputRefs.direct)
809 if self.config.makePsfMatched and results.exposures["psfMatched"] is not None:
810 butlerQC.put(results.exposures["psfMatched"], outputRefs.psfMatched)
812 def filterInputs(self, indices, inputs):
813 """Return task inputs with their lists filtered by indices
815 Parameters
816 ----------
817 indices : `list` of integers
818 inputs : `dict` of `list` of input connections to be passed to run
819 """
820 for key in inputs.keys():
821 # Only down-select on list inputs
822 if isinstance(inputs[key], list):
823 inputs[key] = [inputs[key][ind] for ind in indices]
824 return inputs
826 def prepareCalibratedExposures(self, calExpList, backgroundList=None, skyCorrList=None,
827 externalSkyWcsCatalog=None, externalPhotoCalibCatalog=None,
828 **kwargs):
829 """Calibrate and add backgrounds to input calExpList in place
831 Parameters
832 ----------
833 calExpList : `list` of `lsst.afw.image.Exposure`
834 Sequence of calexps to be modified in place
835 backgroundList : `list` of `lsst.afw.math.backgroundList`, optional
836 Sequence of backgrounds to be added back in if bgSubtracted=False
837 skyCorrList : `list` of `lsst.afw.math.backgroundList`, optional
838 Sequence of background corrections to be subtracted if doApplySkyCorr=True
839 externalSkyWcsCatalog : `lsst.afw.table.ExposureCatalog`, optional
840 Exposure catalog with external skyWcs to be applied
841 if config.doApplyExternalSkyWcs=True. Catalog uses the detector id
842 for the catalog id, sorted on id for fast lookup.
843 externalPhotoCalibCatalog : `lsst.afw.table.ExposureCatalog`, optional
844 Exposure catalog with external photoCalib to be applied
845 if config.doApplyExternalPhotoCalib=True. Catalog uses the detector
846 id for the catalog id, sorted on id for fast lookup.
848 Returns
849 -------
850 indices : `list` [`int`]
851 Indices of calExpList and friends that have valid photoCalib/skyWcs
852 """
853 backgroundList = len(calExpList)*[None] if backgroundList is None else backgroundList
854 skyCorrList = len(calExpList)*[None] if skyCorrList is None else skyCorrList
856 includeCalibVar = self.config.includeCalibVar
858 indices = []
859 for index, (calexp, background, skyCorr) in enumerate(zip(calExpList,
860 backgroundList,
861 skyCorrList)):
862 if not self.config.bgSubtracted:
863 calexp.maskedImage += background.getImage()
865 if externalSkyWcsCatalog is not None or externalPhotoCalibCatalog is not None:
866 detectorId = calexp.getInfo().getDetector().getId()
868 # Find the external photoCalib
869 if externalPhotoCalibCatalog is not None:
870 row = externalPhotoCalibCatalog.find(detectorId)
871 if row is None:
872 self.log.warning("Detector id %s not found in externalPhotoCalibCatalog "
873 "and will not be used in the warp.", detectorId)
874 continue
875 photoCalib = row.getPhotoCalib()
876 if photoCalib is None:
877 self.log.warning("Detector id %s has None for photoCalib in externalPhotoCalibCatalog "
878 "and will not be used in the warp.", detectorId)
879 continue
880 calexp.setPhotoCalib(photoCalib)
881 else:
882 photoCalib = calexp.getPhotoCalib()
883 if photoCalib is None:
884 self.log.warning("Detector id %s has None for photoCalib in the calexp "
885 "and will not be used in the warp.", detectorId)
886 continue
888 # Find and apply external skyWcs
889 if externalSkyWcsCatalog is not None:
890 row = externalSkyWcsCatalog.find(detectorId)
891 if row is None:
892 self.log.warning("Detector id %s not found in externalSkyWcsCatalog "
893 "and will not be used in the warp.", detectorId)
894 continue
895 skyWcs = row.getWcs()
896 if skyWcs is None:
897 self.log.warning("Detector id %s has None for skyWcs in externalSkyWcsCatalog "
898 "and will not be used in the warp.", detectorId)
899 continue
900 calexp.setWcs(skyWcs)
901 else:
902 skyWcs = calexp.getWcs()
903 if skyWcs is None:
904 self.log.warning("Detector id %s has None for skyWcs in the calexp "
905 "and will not be used in the warp.", detectorId)
906 continue
908 # Calibrate the image
909 calexp.maskedImage = photoCalib.calibrateImage(calexp.maskedImage,
910 includeScaleUncertainty=includeCalibVar)
911 calexp.maskedImage /= photoCalib.getCalibrationMean()
912 # TODO: The images will have a calibration of 1.0 everywhere once RFC-545 is implemented.
913 # exposure.setCalib(afwImage.Calib(1.0))
915 # Apply skycorr
916 if self.config.doApplySkyCorr:
917 calexp.maskedImage -= skyCorr.getImage()
919 indices.append(index)
921 return indices
924def reorderRefs(inputRefs, outputSortKeyOrder, dataIdKey):
925 """Reorder inputRefs per outputSortKeyOrder
927 Any inputRefs which are lists will be resorted per specified key e.g.,
928 'detector.' Only iterables will be reordered, and values can be of type
929 `lsst.pipe.base.connections.DeferredDatasetRef` or
930 `lsst.daf.butler.core.datasets.ref.DatasetRef`.
931 Returned lists of refs have the same length as the outputSortKeyOrder.
932 If an outputSortKey not in the inputRef, then it will be padded with None.
933 If an inputRef contains an inputSortKey that is not in the
934 outputSortKeyOrder it will be removed.
936 Parameters
937 ----------
938 inputRefs : `lsst.pipe.base.connections.QuantizedConnection`
939 Input references to be reordered and padded.
940 outputSortKeyOrder : iterable
941 Iterable of values to be compared with inputRef's dataId[dataIdKey]
942 dataIdKey : `str`
943 dataIdKey in the dataRefs to compare with the outputSortKeyOrder.
945 Returns:
946 --------
947 inputRefs: `lsst.pipe.base.connections.QuantizedConnection`
948 Quantized Connection with sorted DatasetRef values sorted if iterable.
949 """
950 for connectionName, refs in inputRefs:
951 if isinstance(refs, Iterable):
952 if hasattr(refs[0], "dataId"):
953 inputSortKeyOrder = [ref.dataId[dataIdKey] for ref in refs]
954 else:
955 inputSortKeyOrder = [ref.datasetRef.dataId[dataIdKey] for ref in refs]
956 if inputSortKeyOrder != outputSortKeyOrder:
957 setattr(inputRefs, connectionName,
958 reorderAndPadList(refs, inputSortKeyOrder, outputSortKeyOrder))
959 return inputRefs