lsst.pipe.tasks g1952d3b776+66328450fc
makeCoaddTempExp.py
Go to the documentation of this file.
2# LSST Data Management System
3# Copyright 2008, 2009, 2010, 2011, 2012 LSST Corporation.
4#
5# This product includes software developed by the
6# LSST Project (http://www.lsst.org/).
7#
8# This program is free software: you can redistribute it and/or modify
9# it under the terms of the GNU General Public License as published by
10# the Free Software Foundation, either version 3 of the License, or
11# (at your option) any later version.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the LSST License Statement and
19# the GNU General Public License along with this program. If not,
20# see <http://www.lsstcorp.org/LegalNotices/>.
21#
22import numpy
23import logging
24
25import lsst.pex.config as pexConfig
26import lsst.daf.persistence as dafPersist
27import lsst.afw.image as afwImage
28import lsst.coadd.utils as coaddUtils
29import lsst.pipe.base as pipeBase
30import lsst.pipe.base.connectionTypes as connectionTypes
31import lsst.utils as utils
32import lsst.geom
33from lsst.meas.algorithms import CoaddPsf, CoaddPsfConfig
34from lsst.skymap import BaseSkyMap
35from lsst.utils.timer import timeMethod
36from .coaddBase import CoaddBaseTask, makeSkyInfo, reorderAndPadList
37from .selectImages import PsfWcsSelectImagesTask
38from .warpAndPsfMatch import WarpAndPsfMatchTask
39from .coaddHelpers import groupPatchExposures, getGroupDataRef
40from collections.abc import Iterable
41
42__all__ = ["MakeCoaddTempExpTask", "MakeWarpTask", "MakeWarpConfig"]
43
44log = logging.getLogger(__name__)
45
46
47class MissingExposureError(Exception):
48 """Raised when data cannot be retrieved for an exposure.
49 When processing patches, sometimes one exposure is missing; this lets us
50 distinguish bewteen that case, and other errors.
51 """
52 pass
53
54
55class MakeCoaddTempExpConfig(CoaddBaseTask.ConfigClass):
56 """Config for MakeCoaddTempExpTask
57 """
58 warpAndPsfMatch = pexConfig.ConfigurableField(
59 target=WarpAndPsfMatchTask,
60 doc="Task to warp and PSF-match calexp",
61 )
62 doWrite = pexConfig.Field(
63 doc="persist <coaddName>Coadd_<warpType>Warp",
64 dtype=bool,
65 default=True,
66 )
67 bgSubtracted = pexConfig.Field(
68 doc="Work with a background subtracted calexp?",
69 dtype=bool,
70 default=True,
71 )
72 coaddPsf = pexConfig.ConfigField(
73 doc="Configuration for CoaddPsf",
74 dtype=CoaddPsfConfig,
75 )
76 makeDirect = pexConfig.Field(
77 doc="Make direct Warp/Coadds",
78 dtype=bool,
79 default=True,
80 )
81 makePsfMatched = pexConfig.Field(
82 doc="Make Psf-Matched Warp/Coadd?",
83 dtype=bool,
84 default=False,
85 )
86
87 doWriteEmptyWarps = pexConfig.Field(
88 dtype=bool,
89 default=False,
90 doc="Write out warps even if they are empty"
91 )
92
93 hasFakes = pexConfig.Field(
94 doc="Should be set to True if fake sources have been inserted into the input data.",
95 dtype=bool,
96 default=False,
97 )
98 doApplySkyCorr = pexConfig.Field(dtype=bool, default=False, doc="Apply sky correction?")
99
100 doApplyFinalizedPsf = pexConfig.Field(
101 doc="Whether to apply finalized psf models and aperture correction map.",
102 dtype=bool,
103 default=False,
104 )
105
106 def validate(self):
107 CoaddBaseTask.ConfigClass.validate(self)
108 if not self.makePsfMatchedmakePsfMatched and not self.makeDirectmakeDirect:
109 raise RuntimeError("At least one of config.makePsfMatched and config.makeDirect must be True")
110 if self.doPsfMatch:
111 # Backwards compatibility.
112 log.warning("Config doPsfMatch deprecated. Setting makePsfMatched=True and makeDirect=False")
113 self.makePsfMatchedmakePsfMatched = True
114 self.makeDirectmakeDirect = False
115
116 def setDefaults(self):
117 CoaddBaseTask.ConfigClass.setDefaults(self)
118 self.warpAndPsfMatchwarpAndPsfMatch.psfMatch.kernel.active.kernelSize = self.matchingKernelSize
119 self.select.retarget(PsfWcsSelectImagesTask)
120
121
127
128
130 r"""!Warp and optionally PSF-Match calexps onto an a common projection.
131
132 @anchor MakeCoaddTempExpTask_
133
134 @section pipe_tasks_makeCoaddTempExp_Contents Contents
135
136 - @ref pipe_tasks_makeCoaddTempExp_Purpose
137 - @ref pipe_tasks_makeCoaddTempExp_Initialize
138 - @ref pipe_tasks_makeCoaddTempExp_IO
139 - @ref pipe_tasks_makeCoaddTempExp_Config
140 - @ref pipe_tasks_makeCoaddTempExp_Debug
141 - @ref pipe_tasks_makeCoaddTempExp_Example
142
143 @section pipe_tasks_makeCoaddTempExp_Purpose Description
144
145 Warp and optionally PSF-Match calexps onto a common projection, by
146 performing the following operations:
147 - Group calexps by visit/run
148 - For each visit, generate a Warp by calling method @ref makeTempExp.
149 makeTempExp loops over the visit's calexps calling @ref WarpAndPsfMatch
150 on each visit
151
152 The result is a `directWarp` (and/or optionally a `psfMatchedWarp`).
153
154 @section pipe_tasks_makeCoaddTempExp_Initialize Task Initialization
155
156 @copydoc \_\_init\_\_
157
158 This task has one special keyword argument: passing reuse=True will cause
159 the task to skip the creation of warps that are already present in the
160 output repositories.
161
162 @section pipe_tasks_makeCoaddTempExp_IO Invoking the Task
163
164 This task is primarily designed to be run from the command line.
165
166 The main method is `runDataRef`, which takes a single butler data reference for the patch(es)
167 to process.
168
169 @copydoc run
170
171 WarpType identifies the types of convolutions applied to Warps (previously CoaddTempExps).
172 Only two types are available: direct (for regular Warps/Coadds) and psfMatched
173 (for Warps/Coadds with homogenized PSFs). We expect to add a third type, likelihood,
174 for generating likelihood Coadds with Warps that have been correlated with their own PSF.
175
176 @section pipe_tasks_makeCoaddTempExp_Config Configuration parameters
177
178 See @ref MakeCoaddTempExpConfig and parameters inherited from
179 @link lsst.pipe.tasks.coaddBase.CoaddBaseConfig CoaddBaseConfig @endlink
180
181 @subsection pipe_tasks_MakeCoaddTempExp_psfMatching Guide to PSF-Matching Configs
182
183 To make `psfMatchedWarps`, select `config.makePsfMatched=True`. The subtask
184 @link lsst.ip.diffim.modelPsfMatch.ModelPsfMatchTask ModelPsfMatchTask @endlink
185 is responsible for the PSF-Matching, and its config is accessed via `config.warpAndPsfMatch.psfMatch`.
186 The optimal configuration depends on aspects of dataset: the pixel scale, average PSF FWHM and
187 dimensions of the PSF kernel. These configs include the requested model PSF, the matching kernel size,
188 padding of the science PSF thumbnail and spatial sampling frequency of the PSF.
189
190 *Config Guidelines*: The user must specify the size of the model PSF to which to match by setting
191 `config.modelPsf.defaultFwhm` in units of pixels. The appropriate values depends on science case.
192 In general, for a set of input images, this config should equal the FWHM of the visit
193 with the worst seeing. The smallest it should be set to is the median FWHM. The defaults
194 of the other config options offer a reasonable starting point.
195 The following list presents the most common problems that arise from a misconfigured
196 @link lsst.ip.diffim.modelPsfMatch.ModelPsfMatchTask ModelPsfMatchTask @endlink
197 and corresponding solutions. All assume the default Alard-Lupton kernel, with configs accessed via
198 ```config.warpAndPsfMatch.psfMatch.kernel['AL']```. Each item in the list is formatted as:
199 Problem: Explanation. *Solution*
200
201 *Troublshooting PSF-Matching Configuration:*
202 - Matched PSFs look boxy: The matching kernel is too small. _Increase the matching kernel size.
203 For example:_
204
205 config.warpAndPsfMatch.psfMatch.kernel['AL'].kernelSize=27 # default 21
206
207 Note that increasing the kernel size also increases runtime.
208 - Matched PSFs look ugly (dipoles, quadropoles, donuts): unable to find good solution
209 for matching kernel. _Provide the matcher with more data by either increasing
210 the spatial sampling by decreasing the spatial cell size,_
211
212 config.warpAndPsfMatch.psfMatch.kernel['AL'].sizeCellX = 64 # default 128
213 config.warpAndPsfMatch.psfMatch.kernel['AL'].sizeCellY = 64 # default 128
214
215 _or increasing the padding around the Science PSF, for example:_
216
217 config.warpAndPsfMatch.psfMatch.autoPadPsfTo=1.6 # default 1.4
218
219 Increasing `autoPadPsfTo` increases the minimum ratio of input PSF dimensions to the
220 matching kernel dimensions, thus increasing the number of pixels available to fit
221 after convolving the PSF with the matching kernel.
222 Optionally, for debugging the effects of padding, the level of padding may be manually
223 controlled by setting turning off the automatic padding and setting the number
224 of pixels by which to pad the PSF:
225
226 config.warpAndPsfMatch.psfMatch.doAutoPadPsf = False # default True
227 config.warpAndPsfMatch.psfMatch.padPsfBy = 6 # pixels. default 0
228
229 - Deconvolution: Matching a large PSF to a smaller PSF produces
230 a telltale noise pattern which looks like ripples or a brain.
231 _Increase the size of the requested model PSF. For example:_
232
233 config.modelPsf.defaultFwhm = 11 # Gaussian sigma in units of pixels.
234
235 - High frequency (sometimes checkered) noise: The matching basis functions are too small.
236 _Increase the width of the Gaussian basis functions. For example:_
237
238 config.warpAndPsfMatch.psfMatch.kernel['AL'].alardSigGauss=[1.5, 3.0, 6.0]
239 # from default [0.7, 1.5, 3.0]
240
241
242 @section pipe_tasks_makeCoaddTempExp_Debug Debug variables
243
244 MakeCoaddTempExpTask has no debug output, but its subtasks do.
245
246 @section pipe_tasks_makeCoaddTempExp_Example A complete example of using MakeCoaddTempExpTask
247
248 This example uses the package ci_hsc to show how MakeCoaddTempExp fits
249 into the larger Data Release Processing.
250 Set up by running:
251
252 setup ci_hsc
253 cd $CI_HSC_DIR
254 # if not built already:
255 python $(which scons) # this will take a while
256
257 The following assumes that `processCcd.py` and `makeSkyMap.py` have previously been run
258 (e.g. by building `ci_hsc` above) to generate a repository of calexps and an
259 output respository with the desired SkyMap. The command,
260
261 makeCoaddTempExp.py $CI_HSC_DIR/DATA --rerun ci_hsc \
262 --id patch=5,4 tract=0 filter=HSC-I \
263 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 \
264 --selectId visit=903988 ccd=23 --selectId visit=903988 ccd=24 \
265 --config doApplyExternalPhotoCalib=False doApplyExternalSkyWcs=False \
266 makePsfMatched=True modelPsf.defaultFwhm=11
267
268 writes a direct and PSF-Matched Warp to
269 - `$CI_HSC_DIR/DATA/rerun/ci_hsc/deepCoadd/HSC-I/0/5,4/warp-HSC-I-0-5,4-903988.fits` and
270 - `$CI_HSC_DIR/DATA/rerun/ci_hsc/deepCoadd/HSC-I/0/5,4/psfMatchedWarp-HSC-I-0-5,4-903988.fits`
271 respectively.
272
273 @note PSF-Matching in this particular dataset would benefit from adding
274 `--configfile ./matchingConfig.py` to
275 the command line arguments where `matchingConfig.py` is defined by:
276
277 echo "
278 config.warpAndPsfMatch.psfMatch.kernel['AL'].kernelSize=27
279 config.warpAndPsfMatch.psfMatch.kernel['AL'].alardSigGauss=[1.5, 3.0, 6.0]" > matchingConfig.py
280
281
282 Add the option `--help` to see more options.
283 """
284 ConfigClass = MakeCoaddTempExpConfig
285 _DefaultName = "makeCoaddTempExp"
286
287 def __init__(self, reuse=False, **kwargs):
288 CoaddBaseTask.__init__(self, **kwargs)
289 self.reusereuse = reuse
290 self.makeSubtask("warpAndPsfMatch")
291 if self.config.hasFakes:
292 self.calexpTypecalexpType = "fakes_calexp"
293 else:
294 self.calexpTypecalexpType = "calexp"
295
296 @timeMethod
297 def runDataRef(self, patchRef, selectDataList=[]):
298 """!Produce <coaddName>Coadd_<warpType>Warp images by warping and optionally PSF-matching.
299
300 @param[in] patchRef: data reference for sky map patch. Must include keys "tract", "patch",
301 plus the camera-specific filter key (e.g. "filter" or "band")
302 @return: dataRefList: a list of data references for the new <coaddName>Coadd_directWarps
303 if direct or both warp types are requested and <coaddName>Coadd_psfMatchedWarps if only psfMatched
304 warps are requested.
305
306 @warning: this task assumes that all exposures in a warp (coaddTempExp) have the same filter.
307
308 @warning: this task sets the PhotoCalib of the coaddTempExp to the PhotoCalib of the first calexp
309 with any good pixels in the patch. For a mosaic camera the resulting PhotoCalib should be ignored
310 (assembleCoadd should determine zeropoint scaling without referring to it).
311 """
312 skyInfo = self.getSkyInfogetSkyInfo(patchRef)
313
314 # DataRefs to return are of type *_directWarp unless only *_psfMatchedWarp requested
315 if self.config.makePsfMatched and not self.config.makeDirect:
316 primaryWarpDataset = self.getTempExpDatasetNamegetTempExpDatasetName("psfMatched")
317 else:
318 primaryWarpDataset = self.getTempExpDatasetNamegetTempExpDatasetName("direct")
319
320 calExpRefList = self.selectExposuresselectExposures(patchRef, skyInfo, selectDataList=selectDataList)
321
322 if len(calExpRefList) == 0:
323 self.log.warning("No exposures to coadd for patch %s", patchRef.dataId)
324 return None
325 self.log.info("Selected %d calexps for patch %s", len(calExpRefList), patchRef.dataId)
326 calExpRefList = [calExpRef for calExpRef in calExpRefList if calExpRef.datasetExists(self.calexpTypecalexpType)]
327 self.log.info("Processing %d existing calexps for patch %s", len(calExpRefList), patchRef.dataId)
328
329 groupData = groupPatchExposures(patchRef, calExpRefList, self.getCoaddDatasetNamegetCoaddDatasetName(),
330 primaryWarpDataset)
331 self.log.info("Processing %d warp exposures for patch %s", len(groupData.groups), patchRef.dataId)
332
333 dataRefList = []
334 for i, (tempExpTuple, calexpRefList) in enumerate(groupData.groups.items()):
335 tempExpRef = getGroupDataRef(patchRef.getButler(), primaryWarpDataset,
336 tempExpTuple, groupData.keys)
337 if self.reusereuse and tempExpRef.datasetExists(datasetType=primaryWarpDataset, write=True):
338 self.log.info("Skipping makeCoaddTempExp for %s; output already exists.", tempExpRef.dataId)
339 dataRefList.append(tempExpRef)
340 continue
341 self.log.info("Processing Warp %d/%d: id=%s", i, len(groupData.groups), tempExpRef.dataId)
342
343 # TODO: mappers should define a way to go from the "grouping keys" to a numeric ID (#2776).
344 # For now, we try to get a long integer "visit" key, and if we can't, we just use the index
345 # of the visit in the list.
346 try:
347 visitId = int(tempExpRef.dataId["visit"])
348 except (KeyError, ValueError):
349 visitId = i
350
351 calExpList = []
352 ccdIdList = []
353 dataIdList = []
354
355 for calExpInd, calExpRef in enumerate(calexpRefList):
356 self.log.info("Reading calexp %s of %s for Warp id=%s", calExpInd+1, len(calexpRefList),
357 calExpRef.dataId)
358 try:
359 ccdId = calExpRef.get("ccdExposureId", immediate=True)
360 except Exception:
361 ccdId = calExpInd
362 try:
363 # We augment the dataRef here with the tract, which is harmless for loading things
364 # like calexps that don't need the tract, and necessary for meas_mosaic outputs,
365 # which do.
366 calExpRef = calExpRef.butlerSubset.butler.dataRef(self.calexpTypecalexpType,
367 dataId=calExpRef.dataId,
368 tract=skyInfo.tractInfo.getId())
369 calExp = self.getCalibratedExposuregetCalibratedExposure(calExpRef, bgSubtracted=self.config.bgSubtracted)
370 except Exception as e:
371 self.log.warning("Calexp %s not found; skipping it: %s", calExpRef.dataId, e)
372 continue
373
374 if self.config.doApplySkyCorr:
375 self.applySkyCorrapplySkyCorr(calExpRef, calExp)
376
377 calExpList.append(calExp)
378 ccdIdList.append(ccdId)
379 dataIdList.append(calExpRef.dataId)
380
381 exps = self.runrun(calExpList, ccdIdList, skyInfo, visitId, dataIdList).exposures
382
383 if any(exps.values()):
384 dataRefList.append(tempExpRef)
385 else:
386 self.log.warning("Warp %s could not be created", tempExpRef.dataId)
387
388 if self.config.doWrite:
389 for (warpType, exposure) in exps.items(): # compatible w/ Py3
390 if exposure is not None:
391 self.log.info("Persisting %s", self.getTempExpDatasetNamegetTempExpDatasetName(warpType))
392 tempExpRef.put(exposure, self.getTempExpDatasetNamegetTempExpDatasetName(warpType))
393
394 return dataRefList
395
396 @timeMethod
397 def run(self, calExpList, ccdIdList, skyInfo, visitId=0, dataIdList=None, **kwargs):
398 """Create a Warp from inputs
399
400 We iterate over the multiple calexps in a single exposure to construct
401 the warp (previously called a coaddTempExp) of that exposure to the
402 supplied tract/patch.
403
404 Pixels that receive no pixels are set to NAN; this is not correct
405 (violates LSST algorithms group policy), but will be fixed up by
406 interpolating after the coaddition.
407
408 @param calexpRefList: List of data references for calexps that (may)
409 overlap the patch of interest
410 @param skyInfo: Struct from CoaddBaseTask.getSkyInfo() with geometric
411 information about the patch
412 @param visitId: integer identifier for visit, for the table that will
413 produce the CoaddPsf
414 @return a pipeBase Struct containing:
415 - exposures: a dictionary containing the warps requested:
416 "direct": direct warp if config.makeDirect
417 "psfMatched": PSF-matched warp if config.makePsfMatched
418 """
419 warpTypeList = self.getWarpTypeListgetWarpTypeList()
420
421 totGoodPix = {warpType: 0 for warpType in warpTypeList}
422 didSetMetadata = {warpType: False for warpType in warpTypeList}
423 coaddTempExps = {warpType: self._prepareEmptyExposure_prepareEmptyExposure(skyInfo) for warpType in warpTypeList}
424 inputRecorder = {warpType: self.inputRecorder.makeCoaddTempExpRecorder(visitId, len(calExpList))
425 for warpType in warpTypeList}
426
427 modelPsf = self.config.modelPsf.apply() if self.config.makePsfMatched else None
428 if dataIdList is None:
429 dataIdList = ccdIdList
430
431 for calExpInd, (calExp, ccdId, dataId) in enumerate(zip(calExpList, ccdIdList, dataIdList)):
432 self.log.info("Processing calexp %d of %d for this Warp: id=%s",
433 calExpInd+1, len(calExpList), dataId)
434
435 try:
436 warpedAndMatched = self.warpAndPsfMatch.run(calExp, modelPsf=modelPsf,
437 wcs=skyInfo.wcs, maxBBox=skyInfo.bbox,
438 makeDirect=self.config.makeDirect,
439 makePsfMatched=self.config.makePsfMatched)
440 except Exception as e:
441 self.log.warning("WarpAndPsfMatch failed for calexp %s; skipping it: %s", dataId, e)
442 continue
443 try:
444 numGoodPix = {warpType: 0 for warpType in warpTypeList}
445 for warpType in warpTypeList:
446 exposure = warpedAndMatched.getDict()[warpType]
447 if exposure is None:
448 continue
449 coaddTempExp = coaddTempExps[warpType]
450 if didSetMetadata[warpType]:
451 mimg = exposure.getMaskedImage()
452 mimg *= (coaddTempExp.getPhotoCalib().getInstFluxAtZeroMagnitude()
453 / exposure.getPhotoCalib().getInstFluxAtZeroMagnitude())
454 del mimg
455 numGoodPix[warpType] = coaddUtils.copyGoodPixels(
456 coaddTempExp.getMaskedImage(), exposure.getMaskedImage(), self.getBadPixelMaskgetBadPixelMask())
457 totGoodPix[warpType] += numGoodPix[warpType]
458 self.log.debug("Calexp %s has %d good pixels in this patch (%.1f%%) for %s",
459 dataId, numGoodPix[warpType],
460 100.0*numGoodPix[warpType]/skyInfo.bbox.getArea(), warpType)
461 if numGoodPix[warpType] > 0 and not didSetMetadata[warpType]:
462 coaddTempExp.info.id = exposure.info.id
463 coaddTempExp.setPhotoCalib(exposure.getPhotoCalib())
464 coaddTempExp.setFilterLabel(exposure.getFilterLabel())
465 coaddTempExp.getInfo().setVisitInfo(exposure.getInfo().getVisitInfo())
466 # PSF replaced with CoaddPsf after loop if and only if creating direct warp
467 coaddTempExp.setPsf(exposure.getPsf())
468 didSetMetadata[warpType] = True
469
470 # Need inputRecorder for CoaddApCorrMap for both direct and PSF-matched
471 inputRecorder[warpType].addCalExp(calExp, ccdId, numGoodPix[warpType])
472
473 except Exception as e:
474 self.log.warning("Error processing calexp %s; skipping it: %s", dataId, e)
475 continue
476
477 for warpType in warpTypeList:
478 self.log.info("%sWarp has %d good pixels (%.1f%%)",
479 warpType, totGoodPix[warpType], 100.0*totGoodPix[warpType]/skyInfo.bbox.getArea())
480
481 if totGoodPix[warpType] > 0 and didSetMetadata[warpType]:
482 inputRecorder[warpType].finish(coaddTempExps[warpType], totGoodPix[warpType])
483 if warpType == "direct":
484 coaddTempExps[warpType].setPsf(
485 CoaddPsf(inputRecorder[warpType].coaddInputs.ccds, skyInfo.wcs,
486 self.config.coaddPsf.makeControl()))
487 else:
488 if not self.config.doWriteEmptyWarps:
489 # No good pixels. Exposure still empty
490 coaddTempExps[warpType] = None
491 # NoWorkFound is unnecessary as the downstream tasks will
492 # adjust the quantum accordingly, and it prevents gen2
493 # MakeCoaddTempExp from continuing to loop over visits.
494
495 result = pipeBase.Struct(exposures=coaddTempExps)
496 return result
497
498 def getCalibratedExposure(self, dataRef, bgSubtracted):
499 """Return one calibrated Exposure, possibly with an updated SkyWcs.
500
501 @param[in] dataRef a sensor-level data reference
502 @param[in] bgSubtracted return calexp with background subtracted? If False get the
503 calexp's background background model and add it to the calexp.
504 @return calibrated exposure
505
506 @raises MissingExposureError If data for the exposure is not available.
507
508 If config.doApplyExternalPhotoCalib is `True`, the photometric calibration
509 (`photoCalib`) is taken from `config.externalPhotoCalibName` via the
510 `name_photoCalib` dataset. Otherwise, the photometric calibration is
511 retrieved from the processed exposure. When
512 `config.doApplyExternalSkyWcs` is `True`, the astrometric calibration
513 is taken from `config.externalSkyWcsName` with the `name_wcs` dataset.
514 Otherwise, the astrometric calibration is taken from the processed
515 exposure.
516 """
517 try:
518 exposure = dataRef.get(self.calexpTypecalexpType, immediate=True)
519 except dafPersist.NoResults as e:
520 raise MissingExposureError('Exposure not found: %s ' % str(e)) from e
521
522 if not bgSubtracted:
523 background = dataRef.get("calexpBackground", immediate=True)
524 mi = exposure.getMaskedImage()
525 mi += background.getImage()
526 del mi
527
528 if self.config.doApplyExternalPhotoCalib:
529 source = f"{self.config.externalPhotoCalibName}_photoCalib"
530 self.log.debug("Applying external photoCalib to %s from %s", dataRef.dataId, source)
531 photoCalib = dataRef.get(source)
532 exposure.setPhotoCalib(photoCalib)
533 else:
534 photoCalib = exposure.getPhotoCalib()
535
536 if self.config.doApplyExternalSkyWcs:
537 source = f"{self.config.externalSkyWcsName}_wcs"
538 self.log.debug("Applying external skyWcs to %s from %s", dataRef.dataId, source)
539 skyWcs = dataRef.get(source)
540 exposure.setWcs(skyWcs)
541
542 exposure.maskedImage = photoCalib.calibrateImage(exposure.maskedImage,
543 includeScaleUncertainty=self.config.includeCalibVar)
544 exposure.maskedImage /= photoCalib.getCalibrationMean()
545 # TODO: The images will have a calibration of 1.0 everywhere once RFC-545 is implemented.
546 # exposure.setCalib(afwImage.Calib(1.0))
547 return exposure
548
549 @staticmethod
550 def _prepareEmptyExposure(skyInfo):
551 """Produce an empty exposure for a given patch"""
552 exp = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
553 exp.getMaskedImage().set(numpy.nan, afwImage.Mask
554 .getPlaneBitMask("NO_DATA"), numpy.inf)
555 return exp
556
558 """Return list of requested warp types per the config.
559 """
560 warpTypeList = []
561 if self.config.makeDirect:
562 warpTypeList.append("direct")
563 if self.config.makePsfMatched:
564 warpTypeList.append("psfMatched")
565 return warpTypeList
566
567 def applySkyCorr(self, dataRef, calexp):
568 """Apply correction to the sky background level
569
570 Sky corrections can be generated with the 'skyCorrection.py'
571 executable in pipe_drivers. Because the sky model used by that
572 code extends over the entire focal plane, this can produce
573 better sky subtraction.
574
575 The calexp is updated in-place.
576
577 Parameters
578 ----------
579 dataRef : `lsst.daf.persistence.ButlerDataRef`
580 Data reference for calexp.
582 Calibrated exposure.
583 """
584 bg = dataRef.get("skyCorr")
585 self.log.debug("Applying sky correction to %s", dataRef.dataId)
586 if isinstance(calexp, afwImage.Exposure):
587 calexp = calexp.getMaskedImage()
588 calexp -= bg.getImage()
589
590
591class MakeWarpConnections(pipeBase.PipelineTaskConnections,
592 dimensions=("tract", "patch", "skymap", "instrument", "visit"),
593 defaultTemplates={"coaddName": "deep",
594 "skyWcsName": "jointcal",
595 "photoCalibName": "fgcm",
596 "calexpType": ""}):
597 calExpList = connectionTypes.Input(
598 doc="Input exposures to be resampled and optionally PSF-matched onto a SkyMap projection/patch",
599 name="{calexpType}calexp",
600 storageClass="ExposureF",
601 dimensions=("instrument", "visit", "detector"),
602 multiple=True,
603 deferLoad=True,
604 )
605 backgroundList = connectionTypes.Input(
606 doc="Input backgrounds to be added back into the calexp if bgSubtracted=False",
607 name="calexpBackground",
608 storageClass="Background",
609 dimensions=("instrument", "visit", "detector"),
610 multiple=True,
611 )
612 skyCorrList = connectionTypes.Input(
613 doc="Input Sky Correction to be subtracted from the calexp if doApplySkyCorr=True",
614 name="skyCorr",
615 storageClass="Background",
616 dimensions=("instrument", "visit", "detector"),
617 multiple=True,
618 )
619 skyMap = connectionTypes.Input(
620 doc="Input definition of geometry/bbox and projection/wcs for warped exposures",
621 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
622 storageClass="SkyMap",
623 dimensions=("skymap",),
624 )
625 externalSkyWcsTractCatalog = connectionTypes.Input(
626 doc=("Per-tract, per-visit wcs calibrations. These catalogs use the detector "
627 "id for the catalog id, sorted on id for fast lookup."),
628 name="{skyWcsName}SkyWcsCatalog",
629 storageClass="ExposureCatalog",
630 dimensions=("instrument", "visit", "tract"),
631 )
632 externalSkyWcsGlobalCatalog = connectionTypes.Input(
633 doc=("Per-visit wcs calibrations computed globally (with no tract information). "
634 "These catalogs use the detector id for the catalog id, sorted on id for "
635 "fast lookup."),
636 name="{skyWcsName}SkyWcsCatalog",
637 storageClass="ExposureCatalog",
638 dimensions=("instrument", "visit"),
639 )
640 externalPhotoCalibTractCatalog = connectionTypes.Input(
641 doc=("Per-tract, per-visit photometric calibrations. These catalogs use the "
642 "detector id for the catalog id, sorted on id for fast lookup."),
643 name="{photoCalibName}PhotoCalibCatalog",
644 storageClass="ExposureCatalog",
645 dimensions=("instrument", "visit", "tract"),
646 )
647 externalPhotoCalibGlobalCatalog = connectionTypes.Input(
648 doc=("Per-visit photometric calibrations computed globally (with no tract "
649 "information). These catalogs use the detector id for the catalog id, "
650 "sorted on id for fast lookup."),
651 name="{photoCalibName}PhotoCalibCatalog",
652 storageClass="ExposureCatalog",
653 dimensions=("instrument", "visit"),
654 )
655 finalizedPsfApCorrCatalog = connectionTypes.Input(
656 doc=("Per-visit finalized psf models and aperture correction maps. "
657 "These catalogs use the detector id for the catalog id, "
658 "sorted on id for fast lookup."),
659 name="finalized_psf_ap_corr_catalog",
660 storageClass="ExposureCatalog",
661 dimensions=("instrument", "visit"),
662 )
663 direct = connectionTypes.Output(
664 doc=("Output direct warped exposure (previously called CoaddTempExp), produced by resampling ",
665 "calexps onto the skyMap patch geometry."),
666 name="{coaddName}Coadd_directWarp",
667 storageClass="ExposureF",
668 dimensions=("tract", "patch", "skymap", "visit", "instrument"),
669 )
670 psfMatched = connectionTypes.Output(
671 doc=("Output PSF-Matched warped exposure (previously called CoaddTempExp), produced by resampling ",
672 "calexps onto the skyMap patch geometry and PSF-matching to a model PSF."),
673 name="{coaddName}Coadd_psfMatchedWarp",
674 storageClass="ExposureF",
675 dimensions=("tract", "patch", "skymap", "visit", "instrument"),
676 )
677 # TODO DM-28769, have selectImages subtask indicate which connections they need:
678 wcsList = connectionTypes.Input(
679 doc="WCSs of calexps used by SelectImages subtask to determine if the calexp overlaps the patch",
680 name="{calexpType}calexp.wcs",
681 storageClass="Wcs",
682 dimensions=("instrument", "visit", "detector"),
683 multiple=True,
684 )
685 bboxList = connectionTypes.Input(
686 doc="BBoxes of calexps used by SelectImages subtask to determine if the calexp overlaps the patch",
687 name="{calexpType}calexp.bbox",
688 storageClass="Box2I",
689 dimensions=("instrument", "visit", "detector"),
690 multiple=True,
691 )
692 visitSummary = connectionTypes.Input(
693 doc="Consolidated exposure metadata from ConsolidateVisitSummaryTask",
694 name="{calexpType}visitSummary",
695 storageClass="ExposureCatalog",
696 dimensions=("instrument", "visit",),
697 )
698 srcList = connectionTypes.Input(
699 doc="Source catalogs used by PsfWcsSelectImages subtask to further select on PSF stability",
700 name="src",
701 storageClass="SourceCatalog",
702 dimensions=("instrument", "visit", "detector"),
703 multiple=True,
704 )
705
706 def __init__(self, *, config=None):
707 super().__init__(config=config)
708 if config.bgSubtracted:
709 self.inputs.remove("backgroundList")
710 if not config.doApplySkyCorr:
711 self.inputs.remove("skyCorrList")
712 if config.doApplyExternalSkyWcs:
713 if config.useGlobalExternalSkyWcs:
714 self.inputs.remove("externalSkyWcsTractCatalog")
715 else:
716 self.inputs.remove("externalSkyWcsGlobalCatalog")
717 else:
718 self.inputs.remove("externalSkyWcsTractCatalog")
719 self.inputs.remove("externalSkyWcsGlobalCatalog")
720 if config.doApplyExternalPhotoCalib:
721 if config.useGlobalExternalPhotoCalib:
722 self.inputs.remove("externalPhotoCalibTractCatalog")
723 else:
724 self.inputs.remove("externalPhotoCalibGlobalCatalog")
725 else:
726 self.inputs.remove("externalPhotoCalibTractCatalog")
727 self.inputs.remove("externalPhotoCalibGlobalCatalog")
728 if not config.doApplyFinalizedPsf:
729 self.inputs.remove("finalizedPsfApCorrCatalog")
730 if not config.makeDirect:
731 self.outputs.remove("direct")
732 if not config.makePsfMatched:
733 self.outputs.remove("psfMatched")
734 # TODO DM-28769: add connection per selectImages connections
735 if config.select.target != lsst.pipe.tasks.selectImages.PsfWcsSelectImagesTask:
736 self.inputs.remove("visitSummary")
737 self.inputs.remove("srcList")
738 elif not config.select.doLegacyStarSelectionComputation:
739 # Remove backwards-compatibility connections.
740 self.inputs.remove("srcList")
741
742
743class MakeWarpConfig(pipeBase.PipelineTaskConfig, MakeCoaddTempExpConfig,
744 pipelineConnections=MakeWarpConnections):
745
746 def validate(self):
747 super().validate()
748
749
750class MakeWarpTask(MakeCoaddTempExpTask):
751 """Warp and optionally PSF-Match calexps onto an a common projection
752 """
753 ConfigClass = MakeWarpConfig
754 _DefaultName = "makeWarp"
755
756 @utils.inheritDoc(pipeBase.PipelineTask)
757 def runQuantum(self, butlerQC, inputRefs, outputRefs):
758 """
759 Notes
760 ----
761 Construct warps for requested warp type for single epoch
762
763 PipelineTask (Gen3) entry point to warp and optionally PSF-match
764 calexps. This method is analogous to `runDataRef`.
765 """
766 # Obtain the list of input detectors from calExpList. Sort them by
767 # detector order (to ensure reproducibility). Then ensure all input
768 # lists are in the same sorted detector order.
769 detectorOrder = [ref.datasetRef.dataId['detector'] for ref in inputRefs.calExpList]
770 detectorOrder.sort()
771 inputRefs = reorderRefs(inputRefs, detectorOrder, dataIdKey='detector')
772
773 # Read in all inputs.
774 inputs = butlerQC.get(inputRefs)
775
776 # Construct skyInfo expected by `run`. We remove the SkyMap itself
777 # from the dictionary so we can pass it as kwargs later.
778 skyMap = inputs.pop("skyMap")
779 quantumDataId = butlerQC.quantum.dataId
780 skyInfo = makeSkyInfo(skyMap, tractId=quantumDataId['tract'], patchId=quantumDataId['patch'])
781
782 # Construct list of input DataIds expected by `run`
783 dataIdList = [ref.datasetRef.dataId for ref in inputRefs.calExpList]
784 # Construct list of packed integer IDs expected by `run`
785 ccdIdList = [dataId.pack("visit_detector") for dataId in dataIdList]
786
787 # Run the selector and filter out calexps that were not selected
788 # primarily because they do not overlap the patch
789 cornerPosList = lsst.geom.Box2D(skyInfo.bbox).getCorners()
790 coordList = [skyInfo.wcs.pixelToSky(pos) for pos in cornerPosList]
791 goodIndices = self.select.run(**inputs, coordList=coordList, dataIds=dataIdList)
792 inputs = self.filterInputs(indices=goodIndices, inputs=inputs)
793
794 # Read from disk only the selected calexps
795 inputs['calExpList'] = [ref.get() for ref in inputs['calExpList']]
796
797 # Extract integer visitId requested by `run`
798 visits = [dataId['visit'] for dataId in dataIdList]
799 visitId = visits[0]
800
801 if self.config.doApplyExternalSkyWcs:
802 if self.config.useGlobalExternalSkyWcs:
803 externalSkyWcsCatalog = inputs.pop("externalSkyWcsGlobalCatalog")
804 else:
805 externalSkyWcsCatalog = inputs.pop("externalSkyWcsTractCatalog")
806 else:
807 externalSkyWcsCatalog = None
808
809 if self.config.doApplyExternalPhotoCalib:
810 if self.config.useGlobalExternalPhotoCalib:
811 externalPhotoCalibCatalog = inputs.pop("externalPhotoCalibGlobalCatalog")
812 else:
813 externalPhotoCalibCatalog = inputs.pop("externalPhotoCalibTractCatalog")
814 else:
815 externalPhotoCalibCatalog = None
816
817 if self.config.doApplyFinalizedPsf:
818 finalizedPsfApCorrCatalog = inputs.pop("finalizedPsfApCorrCatalog")
819 else:
820 finalizedPsfApCorrCatalog = None
821
822 completeIndices = self.prepareCalibratedExposures(**inputs,
823 externalSkyWcsCatalog=externalSkyWcsCatalog,
824 externalPhotoCalibCatalog=externalPhotoCalibCatalog,
825 finalizedPsfApCorrCatalog=finalizedPsfApCorrCatalog)
826 # Redo the input selection with inputs with complete wcs/photocalib info.
827 inputs = self.filterInputs(indices=completeIndices, inputs=inputs)
828
829 results = self.run(**inputs, visitId=visitId,
830 ccdIdList=[ccdIdList[i] for i in goodIndices],
831 dataIdList=[dataIdList[i] for i in goodIndices],
832 skyInfo=skyInfo)
833 if self.config.makeDirect and results.exposures["direct"] is not None:
834 butlerQC.put(results.exposures["direct"], outputRefs.direct)
835 if self.config.makePsfMatched and results.exposures["psfMatched"] is not None:
836 butlerQC.put(results.exposures["psfMatched"], outputRefs.psfMatched)
837
838 def filterInputs(self, indices, inputs):
839 """Return task inputs with their lists filtered by indices
840
841 Parameters
842 ----------
843 indices : `list` of integers
844 inputs : `dict` of `list` of input connections to be passed to run
845 """
846 for key in inputs.keys():
847 # Only down-select on list inputs
848 if isinstance(inputs[key], list):
849 inputs[key] = [inputs[key][ind] for ind in indices]
850 return inputs
851
852 def prepareCalibratedExposures(self, calExpList, backgroundList=None, skyCorrList=None,
853 externalSkyWcsCatalog=None, externalPhotoCalibCatalog=None,
854 finalizedPsfApCorrCatalog=None,
855 **kwargs):
856 """Calibrate and add backgrounds to input calExpList in place
857
858 Parameters
859 ----------
860 calExpList : `list` of `lsst.afw.image.Exposure`
861 Sequence of calexps to be modified in place
862 backgroundList : `list` of `lsst.afw.math.backgroundList`, optional
863 Sequence of backgrounds to be added back in if bgSubtracted=False
864 skyCorrList : `list` of `lsst.afw.math.backgroundList`, optional
865 Sequence of background corrections to be subtracted if doApplySkyCorr=True
866 externalSkyWcsCatalog : `lsst.afw.table.ExposureCatalog`, optional
867 Exposure catalog with external skyWcs to be applied
868 if config.doApplyExternalSkyWcs=True. Catalog uses the detector id
869 for the catalog id, sorted on id for fast lookup.
870 externalPhotoCalibCatalog : `lsst.afw.table.ExposureCatalog`, optional
871 Exposure catalog with external photoCalib to be applied
872 if config.doApplyExternalPhotoCalib=True. Catalog uses the detector
873 id for the catalog id, sorted on id for fast lookup.
874 finalizedPsfApCorrCatalog : `lsst.afw.table.ExposureCatalog`, optional
875 Exposure catalog with finalized psf models and aperture correction
876 maps to be applied if config.doApplyFinalizedPsf=True. Catalog uses
877 the detector id for the catalog id, sorted on id for fast lookup.
878
879 Returns
880 -------
881 indices : `list` [`int`]
882 Indices of calExpList and friends that have valid photoCalib/skyWcs
883 """
884 backgroundList = len(calExpList)*[None] if backgroundList is None else backgroundList
885 skyCorrList = len(calExpList)*[None] if skyCorrList is None else skyCorrList
886
887 includeCalibVar = self.config.includeCalibVar
888
889 indices = []
890 for index, (calexp, background, skyCorr) in enumerate(zip(calExpList,
891 backgroundList,
892 skyCorrList)):
893 if not self.config.bgSubtracted:
894 calexp.maskedImage += background.getImage()
895
896 detectorId = calexp.getInfo().getDetector().getId()
897
898 # Find the external photoCalib
899 if externalPhotoCalibCatalog is not None:
900 row = externalPhotoCalibCatalog.find(detectorId)
901 if row is None:
902 self.log.warning("Detector id %s not found in externalPhotoCalibCatalog "
903 "and will not be used in the warp.", detectorId)
904 continue
905 photoCalib = row.getPhotoCalib()
906 if photoCalib is None:
907 self.log.warning("Detector id %s has None for photoCalib in externalPhotoCalibCatalog "
908 "and will not be used in the warp.", detectorId)
909 continue
910 calexp.setPhotoCalib(photoCalib)
911 else:
912 photoCalib = calexp.getPhotoCalib()
913 if photoCalib is None:
914 self.log.warning("Detector id %s has None for photoCalib in the calexp "
915 "and will not be used in the warp.", detectorId)
916 continue
917
918 # Find and apply external skyWcs
919 if externalSkyWcsCatalog is not None:
920 row = externalSkyWcsCatalog.find(detectorId)
921 if row is None:
922 self.log.warning("Detector id %s not found in externalSkyWcsCatalog "
923 "and will not be used in the warp.", detectorId)
924 continue
925 skyWcs = row.getWcs()
926 if skyWcs is None:
927 self.log.warning("Detector id %s has None for skyWcs in externalSkyWcsCatalog "
928 "and will not be used in the warp.", detectorId)
929 continue
930 calexp.setWcs(skyWcs)
931 else:
932 skyWcs = calexp.getWcs()
933 if skyWcs is None:
934 self.log.warning("Detector id %s has None for skyWcs in the calexp "
935 "and will not be used in the warp.", detectorId)
936 continue
937
938 # Find and apply finalized psf and aperture correction
939 if finalizedPsfApCorrCatalog is not None:
940 row = finalizedPsfApCorrCatalog.find(detectorId)
941 if row is None:
942 self.log.warning("Detector id %s not found in finalizedPsfApCorrCatalog "
943 "and will not be used in the warp.", detectorId)
944 continue
945 psf = row.getPsf()
946 if psf is None:
947 self.log.warning("Detector id %s has None for psf in finalizedPsfApCorrCatalog "
948 "and will not be used in the warp.", detectorId)
949 continue
950 calexp.setPsf(psf)
951 apCorrMap = row.getApCorrMap()
952 if apCorrMap is None:
953 self.log.warning("Detector id %s has None for ApCorrMap in finalizedPsfApCorrCatalog "
954 "and will not be used in the warp.", detectorId)
955 continue
956 calexp.setApCorrMap(apCorrMap)
957
958 # Calibrate the image
959 calexp.maskedImage = photoCalib.calibrateImage(calexp.maskedImage,
960 includeScaleUncertainty=includeCalibVar)
961 calexp.maskedImage /= photoCalib.getCalibrationMean()
962 # TODO: The images will have a calibration of 1.0 everywhere once RFC-545 is implemented.
963 # exposure.setCalib(afwImage.Calib(1.0))
964
965 # Apply skycorr
966 if self.config.doApplySkyCorr:
967 calexp.maskedImage -= skyCorr.getImage()
968
969 indices.append(index)
970
971 return indices
972
973
974def reorderRefs(inputRefs, outputSortKeyOrder, dataIdKey):
975 """Reorder inputRefs per outputSortKeyOrder
976
977 Any inputRefs which are lists will be resorted per specified key e.g.,
978 'detector.' Only iterables will be reordered, and values can be of type
979 `lsst.pipe.base.connections.DeferredDatasetRef` or
980 `lsst.daf.butler.core.datasets.ref.DatasetRef`.
981 Returned lists of refs have the same length as the outputSortKeyOrder.
982 If an outputSortKey not in the inputRef, then it will be padded with None.
983 If an inputRef contains an inputSortKey that is not in the
984 outputSortKeyOrder it will be removed.
985
986 Parameters
987 ----------
988 inputRefs : `lsst.pipe.base.connections.QuantizedConnection`
989 Input references to be reordered and padded.
990 outputSortKeyOrder : iterable
991 Iterable of values to be compared with inputRef's dataId[dataIdKey]
992 dataIdKey : `str`
993 dataIdKey in the dataRefs to compare with the outputSortKeyOrder.
994
995 Returns:
996 --------
997 inputRefs: `lsst.pipe.base.connections.QuantizedConnection`
998 Quantized Connection with sorted DatasetRef values sorted if iterable.
999 """
1000 for connectionName, refs in inputRefs:
1001 if isinstance(refs, Iterable):
1002 if hasattr(refs[0], "dataId"):
1003 inputSortKeyOrder = [ref.dataId[dataIdKey] for ref in refs]
1004 else:
1005 inputSortKeyOrder = [ref.datasetRef.dataId[dataIdKey] for ref in refs]
1006 if inputSortKeyOrder != outputSortKeyOrder:
1007 setattr(inputRefs, connectionName,
1008 reorderAndPadList(refs, inputSortKeyOrder, outputSortKeyOrder))
1009 return inputRefs
Configuration parameters for CoaddBaseTask.
Definition: coaddBase.py:38
Base class for coaddition.
Definition: coaddBase.py:141
def getTempExpDatasetName(self, warpType="direct")
Definition: coaddBase.py:204
def selectExposures(self, patchRef, skyInfo=None, selectDataList=[])
Select exposures to coadd.
Definition: coaddBase.py:154
def getCoaddDatasetName(self, warpType="direct")
Definition: coaddBase.py:190
def getSkyInfo(self, patchRef)
Use getSkyinfo to return the skyMap, tract and patch information, wcs and the outer bbox of the patch...
Definition: coaddBase.py:174
def getBadPixelMask(self)
Convenience method to provide the bitmask from the mask plane names.
Definition: coaddBase.py:239
Warp and optionally PSF-Match calexps onto an a common projection.
def getCalibratedExposure(self, dataRef, bgSubtracted)
def run(self, calExpList, ccdIdList, skyInfo, visitId=0, dataIdList=None, **kwargs)
def runDataRef(self, patchRef, selectDataList=[])
Produce <coaddName>Coadd_<warpType>Warp images by warping and optionally PSF-matching.
def reorderAndPadList(inputList, inputKeys, outputKeys, padWith=None)
Definition: coaddBase.py:362
def makeSkyInfo(skyMap, tractId, patchId)
Definition: coaddBase.py:289
def getGroupDataRef(butler, datasetType, groupTuple, keys)
Definition: coaddHelpers.py:99
def groupPatchExposures(patchDataRef, calexpDataRefList, coaddDatasetType="deepCoadd", tempExpDatasetType="deepCoadd_directWarp")
Definition: coaddHelpers.py:60