Coverage for python/lsst/pipe/tasks/makeCoaddTempExp.py : 15%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1#
2# LSST Data Management System
3# Copyright 2008, 2009, 2010, 2011, 2012 LSST Corporation.
4#
5# This product includes software developed by the
6# LSST Project (http://www.lsst.org/).
7#
8# This program is free software: you can redistribute it and/or modify
9# it under the terms of the GNU General Public License as published by
10# the Free Software Foundation, either version 3 of the License, or
11# (at your option) any later version.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the LSST License Statement and
19# the GNU General Public License along with this program. If not,
20# see <http://www.lsstcorp.org/LegalNotices/>.
21#
22import numpy
24import lsst.pex.config as pexConfig
25import lsst.daf.persistence as dafPersist
26import lsst.afw.image as afwImage
27import lsst.coadd.utils as coaddUtils
28import lsst.pipe.base as pipeBase
29import lsst.pipe.base.connectionTypes as connectionTypes
30import lsst.log as log
31import lsst.utils as utils
32import lsst.geom
33from lsst.meas.algorithms import CoaddPsf, CoaddPsfConfig
34from lsst.skymap import BaseSkyMap
35from .coaddBase import CoaddBaseTask, makeSkyInfo, reorderAndPadList
36from .warpAndPsfMatch import WarpAndPsfMatchTask
37from .coaddHelpers import groupPatchExposures, getGroupDataRef
38from collections.abc import Iterable
40__all__ = ["MakeCoaddTempExpTask", "MakeWarpTask", "MakeWarpConfig"]
43class MissingExposureError(Exception):
44 """Raised when data cannot be retrieved for an exposure.
45 When processing patches, sometimes one exposure is missing; this lets us
46 distinguish bewteen that case, and other errors.
47 """
48 pass
51class MakeCoaddTempExpConfig(CoaddBaseTask.ConfigClass):
52 """Config for MakeCoaddTempExpTask
53 """
54 warpAndPsfMatch = pexConfig.ConfigurableField(
55 target=WarpAndPsfMatchTask,
56 doc="Task to warp and PSF-match calexp",
57 )
58 doWrite = pexConfig.Field(
59 doc="persist <coaddName>Coadd_<warpType>Warp",
60 dtype=bool,
61 default=True,
62 )
63 bgSubtracted = pexConfig.Field(
64 doc="Work with a background subtracted calexp?",
65 dtype=bool,
66 default=True,
67 )
68 coaddPsf = pexConfig.ConfigField(
69 doc="Configuration for CoaddPsf",
70 dtype=CoaddPsfConfig,
71 )
72 makeDirect = pexConfig.Field(
73 doc="Make direct Warp/Coadds",
74 dtype=bool,
75 default=True,
76 )
77 makePsfMatched = pexConfig.Field(
78 doc="Make Psf-Matched Warp/Coadd?",
79 dtype=bool,
80 default=False,
81 )
83 doWriteEmptyWarps = pexConfig.Field(
84 dtype=bool,
85 default=False,
86 doc="Write out warps even if they are empty"
87 )
89 hasFakes = pexConfig.Field(
90 doc="Should be set to True if fake sources have been inserted into the input data.",
91 dtype=bool,
92 default=False,
93 )
94 doApplySkyCorr = pexConfig.Field(dtype=bool, default=False, doc="Apply sky correction?")
96 def validate(self):
97 CoaddBaseTask.ConfigClass.validate(self)
98 if not self.makePsfMatched and not self.makeDirect:
99 raise RuntimeError("At least one of config.makePsfMatched and config.makeDirect must be True")
100 if self.doPsfMatch:
101 # Backwards compatibility.
102 log.warn("Config doPsfMatch deprecated. Setting makePsfMatched=True and makeDirect=False")
103 self.makePsfMatched = True
104 self.makeDirect = False
106 def setDefaults(self):
107 CoaddBaseTask.ConfigClass.setDefaults(self)
108 self.warpAndPsfMatch.psfMatch.kernel.active.kernelSize = self.matchingKernelSize
110## \addtogroup LSST_task_documentation
111## \{
112## \page MakeCoaddTempExpTask
113## \ref MakeCoaddTempExpTask_ "MakeCoaddTempExpTask"
114## \copybrief MakeCoaddTempExpTask
115## \}
118class MakeCoaddTempExpTask(CoaddBaseTask):
119 r"""!Warp and optionally PSF-Match calexps onto an a common projection.
121 @anchor MakeCoaddTempExpTask_
123 @section pipe_tasks_makeCoaddTempExp_Contents Contents
125 - @ref pipe_tasks_makeCoaddTempExp_Purpose
126 - @ref pipe_tasks_makeCoaddTempExp_Initialize
127 - @ref pipe_tasks_makeCoaddTempExp_IO
128 - @ref pipe_tasks_makeCoaddTempExp_Config
129 - @ref pipe_tasks_makeCoaddTempExp_Debug
130 - @ref pipe_tasks_makeCoaddTempExp_Example
132 @section pipe_tasks_makeCoaddTempExp_Purpose Description
134 Warp and optionally PSF-Match calexps onto a common projection, by
135 performing the following operations:
136 - Group calexps by visit/run
137 - For each visit, generate a Warp by calling method @ref makeTempExp.
138 makeTempExp loops over the visit's calexps calling @ref WarpAndPsfMatch
139 on each visit
141 The result is a `directWarp` (and/or optionally a `psfMatchedWarp`).
143 @section pipe_tasks_makeCoaddTempExp_Initialize Task Initialization
145 @copydoc \_\_init\_\_
147 This task has one special keyword argument: passing reuse=True will cause
148 the task to skip the creation of warps that are already present in the
149 output repositories.
151 @section pipe_tasks_makeCoaddTempExp_IO Invoking the Task
153 This task is primarily designed to be run from the command line.
155 The main method is `runDataRef`, which takes a single butler data reference for the patch(es)
156 to process.
158 @copydoc run
160 WarpType identifies the types of convolutions applied to Warps (previously CoaddTempExps).
161 Only two types are available: direct (for regular Warps/Coadds) and psfMatched
162 (for Warps/Coadds with homogenized PSFs). We expect to add a third type, likelihood,
163 for generating likelihood Coadds with Warps that have been correlated with their own PSF.
165 @section pipe_tasks_makeCoaddTempExp_Config Configuration parameters
167 See @ref MakeCoaddTempExpConfig and parameters inherited from
168 @link lsst.pipe.tasks.coaddBase.CoaddBaseConfig CoaddBaseConfig @endlink
170 @subsection pipe_tasks_MakeCoaddTempExp_psfMatching Guide to PSF-Matching Configs
172 To make `psfMatchedWarps`, select `config.makePsfMatched=True`. The subtask
173 @link lsst.ip.diffim.modelPsfMatch.ModelPsfMatchTask ModelPsfMatchTask @endlink
174 is responsible for the PSF-Matching, and its config is accessed via `config.warpAndPsfMatch.psfMatch`.
175 The optimal configuration depends on aspects of dataset: the pixel scale, average PSF FWHM and
176 dimensions of the PSF kernel. These configs include the requested model PSF, the matching kernel size,
177 padding of the science PSF thumbnail and spatial sampling frequency of the PSF.
179 *Config Guidelines*: The user must specify the size of the model PSF to which to match by setting
180 `config.modelPsf.defaultFwhm` in units of pixels. The appropriate values depends on science case.
181 In general, for a set of input images, this config should equal the FWHM of the visit
182 with the worst seeing. The smallest it should be set to is the median FWHM. The defaults
183 of the other config options offer a reasonable starting point.
184 The following list presents the most common problems that arise from a misconfigured
185 @link lsst.ip.diffim.modelPsfMatch.ModelPsfMatchTask ModelPsfMatchTask @endlink
186 and corresponding solutions. All assume the default Alard-Lupton kernel, with configs accessed via
187 ```config.warpAndPsfMatch.psfMatch.kernel['AL']```. Each item in the list is formatted as:
188 Problem: Explanation. *Solution*
190 *Troublshooting PSF-Matching Configuration:*
191 - Matched PSFs look boxy: The matching kernel is too small. _Increase the matching kernel size.
192 For example:_
194 config.warpAndPsfMatch.psfMatch.kernel['AL'].kernelSize=27 # default 21
196 Note that increasing the kernel size also increases runtime.
197 - Matched PSFs look ugly (dipoles, quadropoles, donuts): unable to find good solution
198 for matching kernel. _Provide the matcher with more data by either increasing
199 the spatial sampling by decreasing the spatial cell size,_
201 config.warpAndPsfMatch.psfMatch.kernel['AL'].sizeCellX = 64 # default 128
202 config.warpAndPsfMatch.psfMatch.kernel['AL'].sizeCellY = 64 # default 128
204 _or increasing the padding around the Science PSF, for example:_
206 config.warpAndPsfMatch.psfMatch.autoPadPsfTo=1.6 # default 1.4
208 Increasing `autoPadPsfTo` increases the minimum ratio of input PSF dimensions to the
209 matching kernel dimensions, thus increasing the number of pixels available to fit
210 after convolving the PSF with the matching kernel.
211 Optionally, for debugging the effects of padding, the level of padding may be manually
212 controlled by setting turning off the automatic padding and setting the number
213 of pixels by which to pad the PSF:
215 config.warpAndPsfMatch.psfMatch.doAutoPadPsf = False # default True
216 config.warpAndPsfMatch.psfMatch.padPsfBy = 6 # pixels. default 0
218 - Deconvolution: Matching a large PSF to a smaller PSF produces
219 a telltale noise pattern which looks like ripples or a brain.
220 _Increase the size of the requested model PSF. For example:_
222 config.modelPsf.defaultFwhm = 11 # Gaussian sigma in units of pixels.
224 - High frequency (sometimes checkered) noise: The matching basis functions are too small.
225 _Increase the width of the Gaussian basis functions. For example:_
227 config.warpAndPsfMatch.psfMatch.kernel['AL'].alardSigGauss=[1.5, 3.0, 6.0]
228 # from default [0.7, 1.5, 3.0]
231 @section pipe_tasks_makeCoaddTempExp_Debug Debug variables
233 MakeCoaddTempExpTask has no debug output, but its subtasks do.
235 @section pipe_tasks_makeCoaddTempExp_Example A complete example of using MakeCoaddTempExpTask
237 This example uses the package ci_hsc to show how MakeCoaddTempExp fits
238 into the larger Data Release Processing.
239 Set up by running:
241 setup ci_hsc
242 cd $CI_HSC_DIR
243 # if not built already:
244 python $(which scons) # this will take a while
246 The following assumes that `processCcd.py` and `makeSkyMap.py` have previously been run
247 (e.g. by building `ci_hsc` above) to generate a repository of calexps and an
248 output respository with the desired SkyMap. The command,
250 makeCoaddTempExp.py $CI_HSC_DIR/DATA --rerun ci_hsc \
251 --id patch=5,4 tract=0 filter=HSC-I \
252 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 \
253 --selectId visit=903988 ccd=23 --selectId visit=903988 ccd=24 \
254 --config doApplyExternalPhotoCalib=False doApplyExternalSkyWcs=False \
255 makePsfMatched=True modelPsf.defaultFwhm=11
257 writes a direct and PSF-Matched Warp to
258 - `$CI_HSC_DIR/DATA/rerun/ci_hsc/deepCoadd/HSC-I/0/5,4/warp-HSC-I-0-5,4-903988.fits` and
259 - `$CI_HSC_DIR/DATA/rerun/ci_hsc/deepCoadd/HSC-I/0/5,4/psfMatchedWarp-HSC-I-0-5,4-903988.fits`
260 respectively.
262 @note PSF-Matching in this particular dataset would benefit from adding
263 `--configfile ./matchingConfig.py` to
264 the command line arguments where `matchingConfig.py` is defined by:
266 echo "
267 config.warpAndPsfMatch.psfMatch.kernel['AL'].kernelSize=27
268 config.warpAndPsfMatch.psfMatch.kernel['AL'].alardSigGauss=[1.5, 3.0, 6.0]" > matchingConfig.py
271 Add the option `--help` to see more options.
272 """
273 ConfigClass = MakeCoaddTempExpConfig
274 _DefaultName = "makeCoaddTempExp"
276 def __init__(self, reuse=False, **kwargs):
277 CoaddBaseTask.__init__(self, **kwargs)
278 self.reuse = reuse
279 self.makeSubtask("warpAndPsfMatch")
280 if self.config.hasFakes:
281 self.calexpType = "fakes_calexp"
282 else:
283 self.calexpType = "calexp"
285 @pipeBase.timeMethod
286 def runDataRef(self, patchRef, selectDataList=[]):
287 """!Produce <coaddName>Coadd_<warpType>Warp images by warping and optionally PSF-matching.
289 @param[in] patchRef: data reference for sky map patch. Must include keys "tract", "patch",
290 plus the camera-specific filter key (e.g. "filter" or "band")
291 @return: dataRefList: a list of data references for the new <coaddName>Coadd_directWarps
292 if direct or both warp types are requested and <coaddName>Coadd_psfMatchedWarps if only psfMatched
293 warps are requested.
295 @warning: this task assumes that all exposures in a warp (coaddTempExp) have the same filter.
297 @warning: this task sets the PhotoCalib of the coaddTempExp to the PhotoCalib of the first calexp
298 with any good pixels in the patch. For a mosaic camera the resulting PhotoCalib should be ignored
299 (assembleCoadd should determine zeropoint scaling without referring to it).
300 """
301 skyInfo = self.getSkyInfo(patchRef)
303 # DataRefs to return are of type *_directWarp unless only *_psfMatchedWarp requested
304 if self.config.makePsfMatched and not self.config.makeDirect:
305 primaryWarpDataset = self.getTempExpDatasetName("psfMatched")
306 else:
307 primaryWarpDataset = self.getTempExpDatasetName("direct")
309 calExpRefList = self.selectExposures(patchRef, skyInfo, selectDataList=selectDataList)
311 if len(calExpRefList) == 0:
312 self.log.warn("No exposures to coadd for patch %s", patchRef.dataId)
313 return None
314 self.log.info("Selected %d calexps for patch %s", len(calExpRefList), patchRef.dataId)
315 calExpRefList = [calExpRef for calExpRef in calExpRefList if calExpRef.datasetExists(self.calexpType)]
316 self.log.info("Processing %d existing calexps for patch %s", len(calExpRefList), patchRef.dataId)
318 groupData = groupPatchExposures(patchRef, calExpRefList, self.getCoaddDatasetName(),
319 primaryWarpDataset)
320 self.log.info("Processing %d warp exposures for patch %s", len(groupData.groups), patchRef.dataId)
322 dataRefList = []
323 for i, (tempExpTuple, calexpRefList) in enumerate(groupData.groups.items()):
324 tempExpRef = getGroupDataRef(patchRef.getButler(), primaryWarpDataset,
325 tempExpTuple, groupData.keys)
326 if self.reuse and tempExpRef.datasetExists(datasetType=primaryWarpDataset, write=True):
327 self.log.info("Skipping makeCoaddTempExp for %s; output already exists.", tempExpRef.dataId)
328 dataRefList.append(tempExpRef)
329 continue
330 self.log.info("Processing Warp %d/%d: id=%s", i, len(groupData.groups), tempExpRef.dataId)
332 # TODO: mappers should define a way to go from the "grouping keys" to a numeric ID (#2776).
333 # For now, we try to get a long integer "visit" key, and if we can't, we just use the index
334 # of the visit in the list.
335 try:
336 visitId = int(tempExpRef.dataId["visit"])
337 except (KeyError, ValueError):
338 visitId = i
340 calExpList = []
341 ccdIdList = []
342 dataIdList = []
344 for calExpInd, calExpRef in enumerate(calexpRefList):
345 self.log.info("Reading calexp %s of %s for Warp id=%s", calExpInd+1, len(calexpRefList),
346 calExpRef.dataId)
347 try:
348 ccdId = calExpRef.get("ccdExposureId", immediate=True)
349 except Exception:
350 ccdId = calExpInd
351 try:
352 # We augment the dataRef here with the tract, which is harmless for loading things
353 # like calexps that don't need the tract, and necessary for meas_mosaic outputs,
354 # which do.
355 calExpRef = calExpRef.butlerSubset.butler.dataRef(self.calexpType,
356 dataId=calExpRef.dataId,
357 tract=skyInfo.tractInfo.getId())
358 calExp = self.getCalibratedExposure(calExpRef, bgSubtracted=self.config.bgSubtracted)
359 except Exception as e:
360 self.log.warn("Calexp %s not found; skipping it: %s", calExpRef.dataId, e)
361 continue
363 if self.config.doApplySkyCorr:
364 self.applySkyCorr(calExpRef, calExp)
366 calExpList.append(calExp)
367 ccdIdList.append(ccdId)
368 dataIdList.append(calExpRef.dataId)
370 exps = self.run(calExpList, ccdIdList, skyInfo, visitId, dataIdList).exposures
372 if any(exps.values()):
373 dataRefList.append(tempExpRef)
374 else:
375 self.log.warn("Warp %s could not be created", tempExpRef.dataId)
377 if self.config.doWrite:
378 for (warpType, exposure) in exps.items(): # compatible w/ Py3
379 if exposure is not None:
380 self.log.info("Persisting %s" % self.getTempExpDatasetName(warpType))
381 tempExpRef.put(exposure, self.getTempExpDatasetName(warpType))
383 return dataRefList
385 @pipeBase.timeMethod
386 def run(self, calExpList, ccdIdList, skyInfo, visitId=0, dataIdList=None, **kwargs):
387 """Create a Warp from inputs
389 We iterate over the multiple calexps in a single exposure to construct
390 the warp (previously called a coaddTempExp) of that exposure to the
391 supplied tract/patch.
393 Pixels that receive no pixels are set to NAN; this is not correct
394 (violates LSST algorithms group policy), but will be fixed up by
395 interpolating after the coaddition.
397 @param calexpRefList: List of data references for calexps that (may)
398 overlap the patch of interest
399 @param skyInfo: Struct from CoaddBaseTask.getSkyInfo() with geometric
400 information about the patch
401 @param visitId: integer identifier for visit, for the table that will
402 produce the CoaddPsf
403 @return a pipeBase Struct containing:
404 - exposures: a dictionary containing the warps requested:
405 "direct": direct warp if config.makeDirect
406 "psfMatched": PSF-matched warp if config.makePsfMatched
407 """
408 warpTypeList = self.getWarpTypeList()
410 totGoodPix = {warpType: 0 for warpType in warpTypeList}
411 didSetMetadata = {warpType: False for warpType in warpTypeList}
412 coaddTempExps = {warpType: self._prepareEmptyExposure(skyInfo) for warpType in warpTypeList}
413 inputRecorder = {warpType: self.inputRecorder.makeCoaddTempExpRecorder(visitId, len(calExpList))
414 for warpType in warpTypeList}
416 modelPsf = self.config.modelPsf.apply() if self.config.makePsfMatched else None
417 if dataIdList is None:
418 dataIdList = ccdIdList
420 for calExpInd, (calExp, ccdId, dataId) in enumerate(zip(calExpList, ccdIdList, dataIdList)):
421 self.log.info("Processing calexp %d of %d for this Warp: id=%s",
422 calExpInd+1, len(calExpList), dataId)
424 try:
425 warpedAndMatched = self.warpAndPsfMatch.run(calExp, modelPsf=modelPsf,
426 wcs=skyInfo.wcs, maxBBox=skyInfo.bbox,
427 makeDirect=self.config.makeDirect,
428 makePsfMatched=self.config.makePsfMatched)
429 except Exception as e:
430 self.log.warn("WarpAndPsfMatch failed for calexp %s; skipping it: %s", dataId, e)
431 continue
432 try:
433 numGoodPix = {warpType: 0 for warpType in warpTypeList}
434 for warpType in warpTypeList:
435 exposure = warpedAndMatched.getDict()[warpType]
436 if exposure is None:
437 continue
438 coaddTempExp = coaddTempExps[warpType]
439 if didSetMetadata[warpType]:
440 mimg = exposure.getMaskedImage()
441 mimg *= (coaddTempExp.getPhotoCalib().getInstFluxAtZeroMagnitude()
442 / exposure.getPhotoCalib().getInstFluxAtZeroMagnitude())
443 del mimg
444 numGoodPix[warpType] = coaddUtils.copyGoodPixels(
445 coaddTempExp.getMaskedImage(), exposure.getMaskedImage(), self.getBadPixelMask())
446 totGoodPix[warpType] += numGoodPix[warpType]
447 self.log.debug("Calexp %s has %d good pixels in this patch (%.1f%%) for %s",
448 dataId, numGoodPix[warpType],
449 100.0*numGoodPix[warpType]/skyInfo.bbox.getArea(), warpType)
450 if numGoodPix[warpType] > 0 and not didSetMetadata[warpType]:
451 coaddTempExp.setPhotoCalib(exposure.getPhotoCalib())
452 coaddTempExp.setFilterLabel(exposure.getFilterLabel())
453 coaddTempExp.getInfo().setVisitInfo(exposure.getInfo().getVisitInfo())
454 # PSF replaced with CoaddPsf after loop if and only if creating direct warp
455 coaddTempExp.setPsf(exposure.getPsf())
456 didSetMetadata[warpType] = True
458 # Need inputRecorder for CoaddApCorrMap for both direct and PSF-matched
459 inputRecorder[warpType].addCalExp(calExp, ccdId, numGoodPix[warpType])
461 except Exception as e:
462 self.log.warn("Error processing calexp %s; skipping it: %s", dataId, e)
463 continue
465 for warpType in warpTypeList:
466 self.log.info("%sWarp has %d good pixels (%.1f%%)",
467 warpType, totGoodPix[warpType], 100.0*totGoodPix[warpType]/skyInfo.bbox.getArea())
469 if totGoodPix[warpType] > 0 and didSetMetadata[warpType]:
470 inputRecorder[warpType].finish(coaddTempExps[warpType], totGoodPix[warpType])
471 if warpType == "direct":
472 coaddTempExps[warpType].setPsf(
473 CoaddPsf(inputRecorder[warpType].coaddInputs.ccds, skyInfo.wcs,
474 self.config.coaddPsf.makeControl()))
475 else:
476 if not self.config.doWriteEmptyWarps:
477 # No good pixels. Exposure still empty
478 coaddTempExps[warpType] = None
480 result = pipeBase.Struct(exposures=coaddTempExps)
481 return result
483 def getCalibratedExposure(self, dataRef, bgSubtracted):
484 """Return one calibrated Exposure, possibly with an updated SkyWcs.
486 @param[in] dataRef a sensor-level data reference
487 @param[in] bgSubtracted return calexp with background subtracted? If False get the
488 calexp's background background model and add it to the calexp.
489 @return calibrated exposure
491 @raises MissingExposureError If data for the exposure is not available.
493 If config.doApplyExternalPhotoCalib is `True`, the photometric calibration
494 (`photoCalib`) is taken from `config.externalPhotoCalibName` via the
495 `name_photoCalib` dataset. Otherwise, the photometric calibration is
496 retrieved from the processed exposure. When
497 `config.doApplyExternalSkyWcs` is `True`, the astrometric calibration
498 is taken from `config.externalSkyWcsName` with the `name_wcs` dataset.
499 Otherwise, the astrometric calibration is taken from the processed
500 exposure.
501 """
502 try:
503 exposure = dataRef.get(self.calexpType, immediate=True)
504 except dafPersist.NoResults as e:
505 raise MissingExposureError('Exposure not found: %s ' % str(e)) from e
507 if not bgSubtracted:
508 background = dataRef.get("calexpBackground", immediate=True)
509 mi = exposure.getMaskedImage()
510 mi += background.getImage()
511 del mi
513 if self.config.doApplyExternalPhotoCalib:
514 source = f"{self.config.externalPhotoCalibName}_photoCalib"
515 self.log.debug("Applying external photoCalib to %s from %s", dataRef.dataId, source)
516 photoCalib = dataRef.get(source)
517 exposure.setPhotoCalib(photoCalib)
518 else:
519 photoCalib = exposure.getPhotoCalib()
521 if self.config.doApplyExternalSkyWcs:
522 source = f"{self.config.externalSkyWcsName}_wcs"
523 self.log.debug("Applying external skyWcs to %s from %s", dataRef.dataId, source)
524 skyWcs = dataRef.get(source)
525 exposure.setWcs(skyWcs)
527 exposure.maskedImage = photoCalib.calibrateImage(exposure.maskedImage,
528 includeScaleUncertainty=self.config.includeCalibVar)
529 exposure.maskedImage /= photoCalib.getCalibrationMean()
530 # TODO: The images will have a calibration of 1.0 everywhere once RFC-545 is implemented.
531 # exposure.setCalib(afwImage.Calib(1.0))
532 return exposure
534 @staticmethod
535 def _prepareEmptyExposure(skyInfo):
536 """Produce an empty exposure for a given patch"""
537 exp = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
538 exp.getMaskedImage().set(numpy.nan, afwImage.Mask
539 .getPlaneBitMask("NO_DATA"), numpy.inf)
540 return exp
542 def getWarpTypeList(self):
543 """Return list of requested warp types per the config.
544 """
545 warpTypeList = []
546 if self.config.makeDirect:
547 warpTypeList.append("direct")
548 if self.config.makePsfMatched:
549 warpTypeList.append("psfMatched")
550 return warpTypeList
552 def applySkyCorr(self, dataRef, calexp):
553 """Apply correction to the sky background level
555 Sky corrections can be generated with the 'skyCorrection.py'
556 executable in pipe_drivers. Because the sky model used by that
557 code extends over the entire focal plane, this can produce
558 better sky subtraction.
560 The calexp is updated in-place.
562 Parameters
563 ----------
564 dataRef : `lsst.daf.persistence.ButlerDataRef`
565 Data reference for calexp.
566 calexp : `lsst.afw.image.Exposure` or `lsst.afw.image.MaskedImage`
567 Calibrated exposure.
568 """
569 bg = dataRef.get("skyCorr")
570 self.log.debug("Applying sky correction to %s", dataRef.dataId)
571 if isinstance(calexp, afwImage.Exposure):
572 calexp = calexp.getMaskedImage()
573 calexp -= bg.getImage()
576class MakeWarpConnections(pipeBase.PipelineTaskConnections,
577 dimensions=("tract", "patch", "skymap", "instrument", "visit"),
578 defaultTemplates={"coaddName": "deep",
579 "skyWcsName": "jointcal",
580 "photoCalibName": "fgcm"}):
581 calExpList = connectionTypes.Input(
582 doc="Input exposures to be resampled and optionally PSF-matched onto a SkyMap projection/patch",
583 name="calexp",
584 storageClass="ExposureF",
585 dimensions=("instrument", "visit", "detector"),
586 multiple=True,
587 deferLoad=True,
588 )
589 backgroundList = connectionTypes.Input(
590 doc="Input backgrounds to be added back into the calexp if bgSubtracted=False",
591 name="calexpBackground",
592 storageClass="Background",
593 dimensions=("instrument", "visit", "detector"),
594 multiple=True,
595 )
596 skyCorrList = connectionTypes.Input(
597 doc="Input Sky Correction to be subtracted from the calexp if doApplySkyCorr=True",
598 name="skyCorr",
599 storageClass="Background",
600 dimensions=("instrument", "visit", "detector"),
601 multiple=True,
602 )
603 skyMap = connectionTypes.Input(
604 doc="Input definition of geometry/bbox and projection/wcs for warped exposures",
605 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
606 storageClass="SkyMap",
607 dimensions=("skymap",),
608 )
609 externalSkyWcsTractCatalog = connectionTypes.Input(
610 doc=("Per-tract, per-visit wcs calibrations. These catalogs use the detector "
611 "id for the catalog id, sorted on id for fast lookup."),
612 name="{skyWcsName}SkyWcsCatalog",
613 storageClass="ExposureCatalog",
614 dimensions=("instrument", "visit", "tract"),
615 )
616 externalSkyWcsGlobalCatalog = connectionTypes.Input(
617 doc=("Per-visit wcs calibrations computed globally (with no tract information). "
618 "These catalogs use the detector id for the catalog id, sorted on id for "
619 "fast lookup."),
620 name="{skyWcsName}SkyWcsCatalog",
621 storageClass="ExposureCatalog",
622 dimensions=("instrument", "visit"),
623 )
624 externalPhotoCalibTractCatalog = connectionTypes.Input(
625 doc=("Per-tract, per-visit photometric calibrations. These catalogs use the "
626 "detector id for the catalog id, sorted on id for fast lookup."),
627 name="{photoCalibName}PhotoCalibCatalog",
628 storageClass="ExposureCatalog",
629 dimensions=("instrument", "visit", "tract"),
630 )
631 externalPhotoCalibGlobalCatalog = connectionTypes.Input(
632 doc=("Per-visit photometric calibrations computed globally (with no tract "
633 "information). These catalogs use the detector id for the catalog id, "
634 "sorted on id for fast lookup."),
635 name="{photoCalibName}PhotoCalibCatalog",
636 storageClass="ExposureCatalog",
637 dimensions=("instrument", "visit"),
638 )
639 direct = connectionTypes.Output(
640 doc=("Output direct warped exposure (previously called CoaddTempExp), produced by resampling ",
641 "calexps onto the skyMap patch geometry."),
642 name="{coaddName}Coadd_directWarp",
643 storageClass="ExposureF",
644 dimensions=("tract", "patch", "skymap", "visit", "instrument"),
645 )
646 psfMatched = connectionTypes.Output(
647 doc=("Output PSF-Matched warped exposure (previously called CoaddTempExp), produced by resampling ",
648 "calexps onto the skyMap patch geometry and PSF-matching to a model PSF."),
649 name="{coaddName}Coadd_psfMatchedWarp",
650 storageClass="ExposureF",
651 dimensions=("tract", "patch", "skymap", "visit", "instrument"),
652 )
653 # TODO DM-28769, have selectImages subtask indicate which connections they need:
654 wcsList = connectionTypes.Input(
655 doc="WCSs of calexps used by SelectImages subtask to determine if the calexp overlaps the patch",
656 name="calexp.wcs",
657 storageClass="Wcs",
658 dimensions=("instrument", "visit", "detector"),
659 multiple=True,
660 )
661 bboxList = connectionTypes.Input(
662 doc="BBoxes of calexps used by SelectImages subtask to determine if the calexp overlaps the patch",
663 name="calexp.bbox",
664 storageClass="Box2I",
665 dimensions=("instrument", "visit", "detector"),
666 multiple=True,
667 )
668 srcList = connectionTypes.Input(
669 doc="src catalogs used by PsfWcsSelectImages subtask to further select on PSF stability",
670 name="src",
671 storageClass="SourceCatalog",
672 dimensions=("instrument", "visit", "detector"),
673 multiple=True,
674 )
675 psfList = connectionTypes.Input(
676 doc="PSF models used by BestSeeingWcsSelectImages subtask to futher select on seeing",
677 name="calexp.psf",
678 storageClass="Psf",
679 dimensions=("instrument", "visit", "detector"),
680 multiple=True,
681 )
683 def __init__(self, *, config=None):
684 super().__init__(config=config)
685 if config.bgSubtracted:
686 self.inputs.remove("backgroundList")
687 if not config.doApplySkyCorr:
688 self.inputs.remove("skyCorrList")
689 if config.doApplyExternalSkyWcs:
690 if config.useGlobalExternalSkyWcs:
691 self.inputs.remove("externalSkyWcsTractCatalog")
692 else:
693 self.inputs.remove("externalSkyWcsGlobalCatalog")
694 else:
695 self.inputs.remove("externalSkyWcsTractCatalog")
696 self.inputs.remove("externalSkyWcsGlobalCatalog")
697 if config.doApplyExternalPhotoCalib:
698 if config.useGlobalExternalPhotoCalib:
699 self.inputs.remove("externalPhotoCalibTractCatalog")
700 else:
701 self.inputs.remove("externalPhotoCalibGlobalCatalog")
702 else:
703 self.inputs.remove("externalPhotoCalibTractCatalog")
704 self.inputs.remove("externalPhotoCalibGlobalCatalog")
705 if not config.makeDirect:
706 self.outputs.remove("direct")
707 if not config.makePsfMatched:
708 self.outputs.remove("psfMatched")
709 # TODO DM-28769: add connection per selectImages connections
710 # instead of removing if not PsfWcsSelectImagesTask here:
711 if config.select.target != lsst.pipe.tasks.selectImages.PsfWcsSelectImagesTask:
712 self.inputs.remove("srcList")
713 if config.select.target != lsst.pipe.tasks.selectImages.BestSeeingWcsSelectImagesTask:
714 self.inputs.remove("psfList")
717class MakeWarpConfig(pipeBase.PipelineTaskConfig, MakeCoaddTempExpConfig,
718 pipelineConnections=MakeWarpConnections):
720 def validate(self):
721 super().validate()
724class MakeWarpTask(MakeCoaddTempExpTask):
725 """Warp and optionally PSF-Match calexps onto an a common projection
726 """
727 ConfigClass = MakeWarpConfig
728 _DefaultName = "makeWarp"
730 @utils.inheritDoc(pipeBase.PipelineTask)
731 def runQuantum(self, butlerQC, inputRefs, outputRefs):
732 """
733 Notes
734 ----
735 Construct warps for requested warp type for single epoch
737 PipelineTask (Gen3) entry point to warp and optionally PSF-match
738 calexps. This method is analogous to `runDataRef`.
739 """
741 # Ensure all input lists are in same detector order as the calExpList
742 detectorOrder = [ref.datasetRef.dataId['detector'] for ref in inputRefs.calExpList]
743 inputRefs = reorderRefs(inputRefs, detectorOrder, dataIdKey='detector')
745 # Read in all inputs.
746 inputs = butlerQC.get(inputRefs)
748 # Construct skyInfo expected by `run`. We remove the SkyMap itself
749 # from the dictionary so we can pass it as kwargs later.
750 skyMap = inputs.pop("skyMap")
751 quantumDataId = butlerQC.quantum.dataId
752 skyInfo = makeSkyInfo(skyMap, tractId=quantumDataId['tract'], patchId=quantumDataId['patch'])
754 # Construct list of input DataIds expected by `run`
755 dataIdList = [ref.datasetRef.dataId for ref in inputRefs.calExpList]
756 # Construct list of packed integer IDs expected by `run`
757 ccdIdList = [dataId.pack("visit_detector") for dataId in dataIdList]
759 # Run the selector and filter out calexps that were not selected
760 # primarily because they do not overlap the patch
761 cornerPosList = lsst.geom.Box2D(skyInfo.bbox).getCorners()
762 coordList = [skyInfo.wcs.pixelToSky(pos) for pos in cornerPosList]
763 goodIndices = self.select.run(**inputs, coordList=coordList, dataIds=dataIdList)
764 inputs = self.filterInputs(indices=goodIndices, inputs=inputs)
766 # Read from disk only the selected calexps
767 inputs['calExpList'] = [ref.get() for ref in inputs['calExpList']]
769 # Extract integer visitId requested by `run`
770 visits = [dataId['visit'] for dataId in dataIdList]
771 visitId = visits[0]
773 if self.config.doApplyExternalSkyWcs:
774 if self.config.useGlobalExternalSkyWcs:
775 externalSkyWcsCatalog = inputs.pop("externalSkyWcsGlobalCatalog")
776 else:
777 externalSkyWcsCatalog = inputs.pop("externalSkyWcsTractCatalog")
778 else:
779 externalSkyWcsCatalog = None
781 if self.config.doApplyExternalPhotoCalib:
782 if self.config.useGlobalExternalPhotoCalib:
783 externalPhotoCalibCatalog = inputs.pop("externalPhotoCalibGlobalCatalog")
784 else:
785 externalPhotoCalibCatalog = inputs.pop("externalPhotoCalibTractCatalog")
786 else:
787 externalPhotoCalibCatalog = None
789 self.prepareCalibratedExposures(**inputs, externalSkyWcsCatalog=externalSkyWcsCatalog,
790 externalPhotoCalibCatalog=externalPhotoCalibCatalog)
792 results = self.run(**inputs, visitId=visitId,
793 ccdIdList=[ccdIdList[i] for i in goodIndices],
794 dataIdList=[dataIdList[i] for i in goodIndices],
795 skyInfo=skyInfo)
796 if self.config.makeDirect:
797 butlerQC.put(results.exposures["direct"], outputRefs.direct)
798 if self.config.makePsfMatched:
799 butlerQC.put(results.exposures["psfMatched"], outputRefs.psfMatched)
801 def filterInputs(self, indices, inputs):
802 """Return task inputs with their lists filtered by indices
804 Parameters
805 ----------
806 indices : `list` of integers
807 inputs : `dict` of `list` of input connections to be passed to run
808 """
809 for key in inputs.keys():
810 # Only down-select on list inputs
811 if isinstance(inputs[key], list):
812 inputs[key] = [inputs[key][ind] for ind in indices]
813 return inputs
815 def prepareCalibratedExposures(self, calExpList, backgroundList=None, skyCorrList=None,
816 externalSkyWcsCatalog=None, externalPhotoCalibCatalog=None,
817 **kwargs):
818 """Calibrate and add backgrounds to input calExpList in place
820 Parameters
821 ----------
822 calExpList : `list` of `lsst.afw.image.Exposure`
823 Sequence of calexps to be modified in place
824 backgroundList : `list` of `lsst.afw.math.backgroundList`, optional
825 Sequence of backgrounds to be added back in if bgSubtracted=False
826 skyCorrList : `list` of `lsst.afw.math.backgroundList`, optional
827 Sequence of background corrections to be subtracted if doApplySkyCorr=True
828 externalSkyWcsCatalog : `lsst.afw.table.ExposureCatalog`, optional
829 Exposure catalog with external skyWcs to be applied
830 if config.doApplyExternalSkyWcs=True. Catalog uses the detector id
831 for the catalog id, sorted on id for fast lookup.
832 externalPhotoCalibCatalog : `lsst.afw.table.ExposureCatalog`, optional
833 Exposure catalog with external photoCalib to be applied
834 if config.doApplyExternalPhotoCalib=True. Catalog uses the detector
835 id for the catalog id, sorted on id for fast lookup.
836 """
837 backgroundList = len(calExpList)*[None] if backgroundList is None else backgroundList
838 skyCorrList = len(calExpList)*[None] if skyCorrList is None else skyCorrList
840 includeCalibVar = self.config.includeCalibVar
842 for calexp, background, skyCorr in zip(calExpList, backgroundList, skyCorrList):
843 mi = calexp.maskedImage
844 if not self.config.bgSubtracted:
845 mi += background.getImage()
847 if externalSkyWcsCatalog is not None or externalPhotoCalibCatalog is not None:
848 detectorId = calexp.getInfo().getDetector().getId()
850 # Find the external photoCalib
851 if externalPhotoCalibCatalog is not None:
852 row = externalPhotoCalibCatalog.find(detectorId)
853 if row is None:
854 raise RuntimeError(f"Detector id {detectorId} not found in "
855 f"externalPhotoCalibCatalog.")
856 photoCalib = row.getPhotoCalib()
857 if photoCalib is None:
858 raise RuntimeError(f"Detector id {detectorId} has None for photoCalib "
859 f"in externalPhotoCalibCatalog.")
860 else:
861 photoCalib = calexp.getPhotoCalib()
863 # Find and apply external skyWcs
864 if externalSkyWcsCatalog is not None:
865 row = externalSkyWcsCatalog.find(detectorId)
866 if row is None:
867 raise RuntimeError(f"Detector id {detectorId} not found in externalSkyWcsCatalog.")
868 skyWcs = row.getWcs()
869 if skyWcs is None:
870 raise RuntimeError(f"Detector id {detectorId} has None for WCS "
871 f" in externalSkyWcsCatalog.")
872 calexp.setWcs(skyWcs)
874 # Calibrate the image
875 calexp.maskedImage = photoCalib.calibrateImage(calexp.maskedImage,
876 includeScaleUncertainty=includeCalibVar)
877 calexp.maskedImage /= photoCalib.getCalibrationMean()
878 # TODO: The images will have a calibration of 1.0 everywhere once RFC-545 is implemented.
879 # exposure.setCalib(afwImage.Calib(1.0))
881 # Apply skycorr
882 if self.config.doApplySkyCorr:
883 mi -= skyCorr.getImage()
886def reorderRefs(inputRefs, outputSortKeyOrder, dataIdKey):
887 """Reorder inputRefs per outputSortKeyOrder
889 Any inputRefs which are lists will be resorted per specified key e.g.,
890 'detector.' Only iterables will be reordered, and values can be of type
891 `lsst.pipe.base.connections.DeferredDatasetRef` or
892 `lsst.daf.butler.core.datasets.ref.DatasetRef`.
893 Returned lists of refs have the same length as the outputSortKeyOrder.
894 If an outputSortKey not in the inputRef, then it will be padded with None.
895 If an inputRef contains an inputSortKey that is not in the
896 outputSortKeyOrder it will be removed.
898 Parameters
899 ----------
900 inputRefs : `lsst.pipe.base.connections.QuantizedConnection`
901 Input references to be reordered and padded.
902 outputSortKeyOrder : iterable
903 Iterable of values to be compared with inputRef's dataId[dataIdKey]
904 dataIdKey : `str`
905 dataIdKey in the dataRefs to compare with the outputSortKeyOrder.
907 Returns:
908 --------
909 inputRefs: `lsst.pipe.base.connections.QuantizedConnection`
910 Quantized Connection with sorted DatasetRef values sorted if iterable.
911 """
912 for connectionName, refs in inputRefs:
913 if isinstance(refs, Iterable):
914 if hasattr(refs[0], "dataId"):
915 inputSortKeyOrder = [ref.dataId[dataIdKey] for ref in refs]
916 else:
917 inputSortKeyOrder = [ref.datasetRef.dataId[dataIdKey] for ref in refs]
918 if inputSortKeyOrder != outputSortKeyOrder:
919 setattr(inputRefs, connectionName,
920 reorderAndPadList(refs, inputSortKeyOrder, outputSortKeyOrder))
921 return inputRefs