lsst.pipe.tasks v23.0.x-g1e964bd5bd+f2fbba1123
makeCoaddTempExp.py
Go to the documentation of this file.
2# LSST Data Management System
3# Copyright 2008, 2009, 2010, 2011, 2012 LSST Corporation.
4#
5# This product includes software developed by the
6# LSST Project (http://www.lsst.org/).
7#
8# This program is free software: you can redistribute it and/or modify
9# it under the terms of the GNU General Public License as published by
10# the Free Software Foundation, either version 3 of the License, or
11# (at your option) any later version.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the LSST License Statement and
19# the GNU General Public License along with this program. If not,
20# see <http://www.lsstcorp.org/LegalNotices/>.
21#
22import numpy
23import logging
24
25import lsst.pex.config as pexConfig
26import lsst.daf.persistence as dafPersist
27import lsst.afw.image as afwImage
28import lsst.coadd.utils as coaddUtils
29import lsst.pipe.base as pipeBase
30import lsst.pipe.base.connectionTypes as connectionTypes
31import lsst.utils as utils
32import lsst.geom
33from lsst.meas.algorithms import CoaddPsf, CoaddPsfConfig
34from lsst.skymap import BaseSkyMap
35from .coaddBase import CoaddBaseTask, makeSkyInfo, reorderAndPadList
36from .warpAndPsfMatch import WarpAndPsfMatchTask
37from .coaddHelpers import groupPatchExposures, getGroupDataRef
38from collections.abc import Iterable
39
40__all__ = ["MakeCoaddTempExpTask", "MakeWarpTask", "MakeWarpConfig"]
41
42log = logging.getLogger(__name__.partition(".")[2])
43
44
45class MissingExposureError(Exception):
46 """Raised when data cannot be retrieved for an exposure.
47 When processing patches, sometimes one exposure is missing; this lets us
48 distinguish bewteen that case, and other errors.
49 """
50 pass
51
52
53class MakeCoaddTempExpConfig(CoaddBaseTask.ConfigClass):
54 """Config for MakeCoaddTempExpTask
55 """
56 warpAndPsfMatch = pexConfig.ConfigurableField(
57 target=WarpAndPsfMatchTask,
58 doc="Task to warp and PSF-match calexp",
59 )
60 doWrite = pexConfig.Field(
61 doc="persist <coaddName>Coadd_<warpType>Warp",
62 dtype=bool,
63 default=True,
64 )
65 bgSubtracted = pexConfig.Field(
66 doc="Work with a background subtracted calexp?",
67 dtype=bool,
68 default=True,
69 )
70 coaddPsf = pexConfig.ConfigField(
71 doc="Configuration for CoaddPsf",
72 dtype=CoaddPsfConfig,
73 )
74 makeDirect = pexConfig.Field(
75 doc="Make direct Warp/Coadds",
76 dtype=bool,
77 default=True,
78 )
79 makePsfMatched = pexConfig.Field(
80 doc="Make Psf-Matched Warp/Coadd?",
81 dtype=bool,
82 default=False,
83 )
84
85 doWriteEmptyWarps = pexConfig.Field(
86 dtype=bool,
87 default=False,
88 doc="Write out warps even if they are empty"
89 )
90
91 hasFakes = pexConfig.Field(
92 doc="Should be set to True if fake sources have been inserted into the input data.",
93 dtype=bool,
94 default=False,
95 )
96 doApplySkyCorr = pexConfig.Field(dtype=bool, default=False, doc="Apply sky correction?")
97
98 def validate(self):
99 CoaddBaseTask.ConfigClass.validate(self)
100 if not self.makePsfMatchedmakePsfMatched and not self.makeDirectmakeDirect:
101 raise RuntimeError("At least one of config.makePsfMatched and config.makeDirect must be True")
102 if self.doPsfMatch:
103 # Backwards compatibility.
104 log.warning("Config doPsfMatch deprecated. Setting makePsfMatched=True and makeDirect=False")
105 self.makePsfMatchedmakePsfMatched = True
106 self.makeDirectmakeDirect = False
107
108 def setDefaults(self):
109 CoaddBaseTask.ConfigClass.setDefaults(self)
110 self.warpAndPsfMatchwarpAndPsfMatch.psfMatch.kernel.active.kernelSize = self.matchingKernelSize
111
112
118
119
121 r"""!Warp and optionally PSF-Match calexps onto an a common projection.
122
123 @anchor MakeCoaddTempExpTask_
124
125 @section pipe_tasks_makeCoaddTempExp_Contents Contents
126
127 - @ref pipe_tasks_makeCoaddTempExp_Purpose
128 - @ref pipe_tasks_makeCoaddTempExp_Initialize
129 - @ref pipe_tasks_makeCoaddTempExp_IO
130 - @ref pipe_tasks_makeCoaddTempExp_Config
131 - @ref pipe_tasks_makeCoaddTempExp_Debug
132 - @ref pipe_tasks_makeCoaddTempExp_Example
133
134 @section pipe_tasks_makeCoaddTempExp_Purpose Description
135
136 Warp and optionally PSF-Match calexps onto a common projection, by
137 performing the following operations:
138 - Group calexps by visit/run
139 - For each visit, generate a Warp by calling method @ref makeTempExp.
140 makeTempExp loops over the visit's calexps calling @ref WarpAndPsfMatch
141 on each visit
142
143 The result is a `directWarp` (and/or optionally a `psfMatchedWarp`).
144
145 @section pipe_tasks_makeCoaddTempExp_Initialize Task Initialization
146
147 @copydoc \_\_init\_\_
148
149 This task has one special keyword argument: passing reuse=True will cause
150 the task to skip the creation of warps that are already present in the
151 output repositories.
152
153 @section pipe_tasks_makeCoaddTempExp_IO Invoking the Task
154
155 This task is primarily designed to be run from the command line.
156
157 The main method is `runDataRef`, which takes a single butler data reference for the patch(es)
158 to process.
159
160 @copydoc run
161
162 WarpType identifies the types of convolutions applied to Warps (previously CoaddTempExps).
163 Only two types are available: direct (for regular Warps/Coadds) and psfMatched
164 (for Warps/Coadds with homogenized PSFs). We expect to add a third type, likelihood,
165 for generating likelihood Coadds with Warps that have been correlated with their own PSF.
166
167 @section pipe_tasks_makeCoaddTempExp_Config Configuration parameters
168
169 See @ref MakeCoaddTempExpConfig and parameters inherited from
170 @link lsst.pipe.tasks.coaddBase.CoaddBaseConfig CoaddBaseConfig @endlink
171
172 @subsection pipe_tasks_MakeCoaddTempExp_psfMatching Guide to PSF-Matching Configs
173
174 To make `psfMatchedWarps`, select `config.makePsfMatched=True`. The subtask
175 @link lsst.ip.diffim.modelPsfMatch.ModelPsfMatchTask ModelPsfMatchTask @endlink
176 is responsible for the PSF-Matching, and its config is accessed via `config.warpAndPsfMatch.psfMatch`.
177 The optimal configuration depends on aspects of dataset: the pixel scale, average PSF FWHM and
178 dimensions of the PSF kernel. These configs include the requested model PSF, the matching kernel size,
179 padding of the science PSF thumbnail and spatial sampling frequency of the PSF.
180
181 *Config Guidelines*: The user must specify the size of the model PSF to which to match by setting
182 `config.modelPsf.defaultFwhm` in units of pixels. The appropriate values depends on science case.
183 In general, for a set of input images, this config should equal the FWHM of the visit
184 with the worst seeing. The smallest it should be set to is the median FWHM. The defaults
185 of the other config options offer a reasonable starting point.
186 The following list presents the most common problems that arise from a misconfigured
187 @link lsst.ip.diffim.modelPsfMatch.ModelPsfMatchTask ModelPsfMatchTask @endlink
188 and corresponding solutions. All assume the default Alard-Lupton kernel, with configs accessed via
189 ```config.warpAndPsfMatch.psfMatch.kernel['AL']```. Each item in the list is formatted as:
190 Problem: Explanation. *Solution*
191
192 *Troublshooting PSF-Matching Configuration:*
193 - Matched PSFs look boxy: The matching kernel is too small. _Increase the matching kernel size.
194 For example:_
195
196 config.warpAndPsfMatch.psfMatch.kernel['AL'].kernelSize=27 # default 21
197
198 Note that increasing the kernel size also increases runtime.
199 - Matched PSFs look ugly (dipoles, quadropoles, donuts): unable to find good solution
200 for matching kernel. _Provide the matcher with more data by either increasing
201 the spatial sampling by decreasing the spatial cell size,_
202
203 config.warpAndPsfMatch.psfMatch.kernel['AL'].sizeCellX = 64 # default 128
204 config.warpAndPsfMatch.psfMatch.kernel['AL'].sizeCellY = 64 # default 128
205
206 _or increasing the padding around the Science PSF, for example:_
207
208 config.warpAndPsfMatch.psfMatch.autoPadPsfTo=1.6 # default 1.4
209
210 Increasing `autoPadPsfTo` increases the minimum ratio of input PSF dimensions to the
211 matching kernel dimensions, thus increasing the number of pixels available to fit
212 after convolving the PSF with the matching kernel.
213 Optionally, for debugging the effects of padding, the level of padding may be manually
214 controlled by setting turning off the automatic padding and setting the number
215 of pixels by which to pad the PSF:
216
217 config.warpAndPsfMatch.psfMatch.doAutoPadPsf = False # default True
218 config.warpAndPsfMatch.psfMatch.padPsfBy = 6 # pixels. default 0
219
220 - Deconvolution: Matching a large PSF to a smaller PSF produces
221 a telltale noise pattern which looks like ripples or a brain.
222 _Increase the size of the requested model PSF. For example:_
223
224 config.modelPsf.defaultFwhm = 11 # Gaussian sigma in units of pixels.
225
226 - High frequency (sometimes checkered) noise: The matching basis functions are too small.
227 _Increase the width of the Gaussian basis functions. For example:_
228
229 config.warpAndPsfMatch.psfMatch.kernel['AL'].alardSigGauss=[1.5, 3.0, 6.0]
230 # from default [0.7, 1.5, 3.0]
231
232
233 @section pipe_tasks_makeCoaddTempExp_Debug Debug variables
234
235 MakeCoaddTempExpTask has no debug output, but its subtasks do.
236
237 @section pipe_tasks_makeCoaddTempExp_Example A complete example of using MakeCoaddTempExpTask
238
239 This example uses the package ci_hsc to show how MakeCoaddTempExp fits
240 into the larger Data Release Processing.
241 Set up by running:
242
243 setup ci_hsc
244 cd $CI_HSC_DIR
245 # if not built already:
246 python $(which scons) # this will take a while
247
248 The following assumes that `processCcd.py` and `makeSkyMap.py` have previously been run
249 (e.g. by building `ci_hsc` above) to generate a repository of calexps and an
250 output respository with the desired SkyMap. The command,
251
252 makeCoaddTempExp.py $CI_HSC_DIR/DATA --rerun ci_hsc \
253 --id patch=5,4 tract=0 filter=HSC-I \
254 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 \
255 --selectId visit=903988 ccd=23 --selectId visit=903988 ccd=24 \
256 --config doApplyExternalPhotoCalib=False doApplyExternalSkyWcs=False \
257 makePsfMatched=True modelPsf.defaultFwhm=11
258
259 writes a direct and PSF-Matched Warp to
260 - `$CI_HSC_DIR/DATA/rerun/ci_hsc/deepCoadd/HSC-I/0/5,4/warp-HSC-I-0-5,4-903988.fits` and
261 - `$CI_HSC_DIR/DATA/rerun/ci_hsc/deepCoadd/HSC-I/0/5,4/psfMatchedWarp-HSC-I-0-5,4-903988.fits`
262 respectively.
263
264 @note PSF-Matching in this particular dataset would benefit from adding
265 `--configfile ./matchingConfig.py` to
266 the command line arguments where `matchingConfig.py` is defined by:
267
268 echo "
269 config.warpAndPsfMatch.psfMatch.kernel['AL'].kernelSize=27
270 config.warpAndPsfMatch.psfMatch.kernel['AL'].alardSigGauss=[1.5, 3.0, 6.0]" > matchingConfig.py
271
272
273 Add the option `--help` to see more options.
274 """
275 ConfigClass = MakeCoaddTempExpConfig
276 _DefaultName = "makeCoaddTempExp"
277
278 def __init__(self, reuse=False, **kwargs):
279 CoaddBaseTask.__init__(self, **kwargs)
280 self.reusereuse = reuse
281 self.makeSubtask("warpAndPsfMatch")
282 if self.config.hasFakes:
283 self.calexpTypecalexpType = "fakes_calexp"
284 else:
285 self.calexpTypecalexpType = "calexp"
286
287 @pipeBase.timeMethod
288 def runDataRef(self, patchRef, selectDataList=[]):
289 """!Produce <coaddName>Coadd_<warpType>Warp images by warping and optionally PSF-matching.
290
291 @param[in] patchRef: data reference for sky map patch. Must include keys "tract", "patch",
292 plus the camera-specific filter key (e.g. "filter" or "band")
293 @return: dataRefList: a list of data references for the new <coaddName>Coadd_directWarps
294 if direct or both warp types are requested and <coaddName>Coadd_psfMatchedWarps if only psfMatched
295 warps are requested.
296
297 @warning: this task assumes that all exposures in a warp (coaddTempExp) have the same filter.
298
299 @warning: this task sets the PhotoCalib of the coaddTempExp to the PhotoCalib of the first calexp
300 with any good pixels in the patch. For a mosaic camera the resulting PhotoCalib should be ignored
301 (assembleCoadd should determine zeropoint scaling without referring to it).
302 """
303 skyInfo = self.getSkyInfogetSkyInfo(patchRef)
304
305 # DataRefs to return are of type *_directWarp unless only *_psfMatchedWarp requested
306 if self.config.makePsfMatched and not self.config.makeDirect:
307 primaryWarpDataset = self.getTempExpDatasetNamegetTempExpDatasetName("psfMatched")
308 else:
309 primaryWarpDataset = self.getTempExpDatasetNamegetTempExpDatasetName("direct")
310
311 calExpRefList = self.selectExposuresselectExposures(patchRef, skyInfo, selectDataList=selectDataList)
312
313 if len(calExpRefList) == 0:
314 self.log.warning("No exposures to coadd for patch %s", patchRef.dataId)
315 return None
316 self.log.info("Selected %d calexps for patch %s", len(calExpRefList), patchRef.dataId)
317 calExpRefList = [calExpRef for calExpRef in calExpRefList if calExpRef.datasetExists(self.calexpTypecalexpType)]
318 self.log.info("Processing %d existing calexps for patch %s", len(calExpRefList), patchRef.dataId)
319
320 groupData = groupPatchExposures(patchRef, calExpRefList, self.getCoaddDatasetNamegetCoaddDatasetName(),
321 primaryWarpDataset)
322 self.log.info("Processing %d warp exposures for patch %s", len(groupData.groups), patchRef.dataId)
323
324 dataRefList = []
325 for i, (tempExpTuple, calexpRefList) in enumerate(groupData.groups.items()):
326 tempExpRef = getGroupDataRef(patchRef.getButler(), primaryWarpDataset,
327 tempExpTuple, groupData.keys)
328 if self.reusereuse and tempExpRef.datasetExists(datasetType=primaryWarpDataset, write=True):
329 self.log.info("Skipping makeCoaddTempExp for %s; output already exists.", tempExpRef.dataId)
330 dataRefList.append(tempExpRef)
331 continue
332 self.log.info("Processing Warp %d/%d: id=%s", i, len(groupData.groups), tempExpRef.dataId)
333
334 # TODO: mappers should define a way to go from the "grouping keys" to a numeric ID (#2776).
335 # For now, we try to get a long integer "visit" key, and if we can't, we just use the index
336 # of the visit in the list.
337 try:
338 visitId = int(tempExpRef.dataId["visit"])
339 except (KeyError, ValueError):
340 visitId = i
341
342 calExpList = []
343 ccdIdList = []
344 dataIdList = []
345
346 for calExpInd, calExpRef in enumerate(calexpRefList):
347 self.log.info("Reading calexp %s of %s for Warp id=%s", calExpInd+1, len(calexpRefList),
348 calExpRef.dataId)
349 try:
350 ccdId = calExpRef.get("ccdExposureId", immediate=True)
351 except Exception:
352 ccdId = calExpInd
353 try:
354 # We augment the dataRef here with the tract, which is harmless for loading things
355 # like calexps that don't need the tract, and necessary for meas_mosaic outputs,
356 # which do.
357 calExpRef = calExpRef.butlerSubset.butler.dataRef(self.calexpTypecalexpType,
358 dataId=calExpRef.dataId,
359 tract=skyInfo.tractInfo.getId())
360 calExp = self.getCalibratedExposuregetCalibratedExposure(calExpRef, bgSubtracted=self.config.bgSubtracted)
361 except Exception as e:
362 self.log.warning("Calexp %s not found; skipping it: %s", calExpRef.dataId, e)
363 continue
364
365 if self.config.doApplySkyCorr:
366 self.applySkyCorrapplySkyCorr(calExpRef, calExp)
367
368 calExpList.append(calExp)
369 ccdIdList.append(ccdId)
370 dataIdList.append(calExpRef.dataId)
371
372 exps = self.runrun(calExpList, ccdIdList, skyInfo, visitId, dataIdList).exposures
373
374 if any(exps.values()):
375 dataRefList.append(tempExpRef)
376 else:
377 self.log.warning("Warp %s could not be created", tempExpRef.dataId)
378
379 if self.config.doWrite:
380 for (warpType, exposure) in exps.items(): # compatible w/ Py3
381 if exposure is not None:
382 self.log.info("Persisting %s", self.getTempExpDatasetNamegetTempExpDatasetName(warpType))
383 tempExpRef.put(exposure, self.getTempExpDatasetNamegetTempExpDatasetName(warpType))
384
385 return dataRefList
386
387 @pipeBase.timeMethod
388 def run(self, calExpList, ccdIdList, skyInfo, visitId=0, dataIdList=None, **kwargs):
389 """Create a Warp from inputs
390
391 We iterate over the multiple calexps in a single exposure to construct
392 the warp (previously called a coaddTempExp) of that exposure to the
393 supplied tract/patch.
394
395 Pixels that receive no pixels are set to NAN; this is not correct
396 (violates LSST algorithms group policy), but will be fixed up by
397 interpolating after the coaddition.
398
399 @param calexpRefList: List of data references for calexps that (may)
400 overlap the patch of interest
401 @param skyInfo: Struct from CoaddBaseTask.getSkyInfo() with geometric
402 information about the patch
403 @param visitId: integer identifier for visit, for the table that will
404 produce the CoaddPsf
405 @return a pipeBase Struct containing:
406 - exposures: a dictionary containing the warps requested:
407 "direct": direct warp if config.makeDirect
408 "psfMatched": PSF-matched warp if config.makePsfMatched
409 """
410 warpTypeList = self.getWarpTypeListgetWarpTypeList()
411
412 totGoodPix = {warpType: 0 for warpType in warpTypeList}
413 didSetMetadata = {warpType: False for warpType in warpTypeList}
414 coaddTempExps = {warpType: self._prepareEmptyExposure_prepareEmptyExposure(skyInfo) for warpType in warpTypeList}
415 inputRecorder = {warpType: self.inputRecorder.makeCoaddTempExpRecorder(visitId, len(calExpList))
416 for warpType in warpTypeList}
417
418 modelPsf = self.config.modelPsf.apply() if self.config.makePsfMatched else None
419 if dataIdList is None:
420 dataIdList = ccdIdList
421
422 for calExpInd, (calExp, ccdId, dataId) in enumerate(zip(calExpList, ccdIdList, dataIdList)):
423 self.log.info("Processing calexp %d of %d for this Warp: id=%s",
424 calExpInd+1, len(calExpList), dataId)
425
426 try:
427 warpedAndMatched = self.warpAndPsfMatch.run(calExp, modelPsf=modelPsf,
428 wcs=skyInfo.wcs, maxBBox=skyInfo.bbox,
429 makeDirect=self.config.makeDirect,
430 makePsfMatched=self.config.makePsfMatched)
431 except Exception as e:
432 self.log.warning("WarpAndPsfMatch failed for calexp %s; skipping it: %s", dataId, e)
433 continue
434 try:
435 numGoodPix = {warpType: 0 for warpType in warpTypeList}
436 for warpType in warpTypeList:
437 exposure = warpedAndMatched.getDict()[warpType]
438 if exposure is None:
439 continue
440 coaddTempExp = coaddTempExps[warpType]
441 if didSetMetadata[warpType]:
442 mimg = exposure.getMaskedImage()
443 mimg *= (coaddTempExp.getPhotoCalib().getInstFluxAtZeroMagnitude()
444 / exposure.getPhotoCalib().getInstFluxAtZeroMagnitude())
445 del mimg
446 numGoodPix[warpType] = coaddUtils.copyGoodPixels(
447 coaddTempExp.getMaskedImage(), exposure.getMaskedImage(), self.getBadPixelMaskgetBadPixelMask())
448 totGoodPix[warpType] += numGoodPix[warpType]
449 self.log.debug("Calexp %s has %d good pixels in this patch (%.1f%%) for %s",
450 dataId, numGoodPix[warpType],
451 100.0*numGoodPix[warpType]/skyInfo.bbox.getArea(), warpType)
452 if numGoodPix[warpType] > 0 and not didSetMetadata[warpType]:
453 coaddTempExp.setPhotoCalib(exposure.getPhotoCalib())
454 coaddTempExp.setFilterLabel(exposure.getFilterLabel())
455 coaddTempExp.getInfo().setVisitInfo(exposure.getInfo().getVisitInfo())
456 # PSF replaced with CoaddPsf after loop if and only if creating direct warp
457 coaddTempExp.setPsf(exposure.getPsf())
458 didSetMetadata[warpType] = True
459
460 # Need inputRecorder for CoaddApCorrMap for both direct and PSF-matched
461 inputRecorder[warpType].addCalExp(calExp, ccdId, numGoodPix[warpType])
462
463 except Exception as e:
464 self.log.warning("Error processing calexp %s; skipping it: %s", dataId, e)
465 continue
466
467 for warpType in warpTypeList:
468 self.log.info("%sWarp has %d good pixels (%.1f%%)",
469 warpType, totGoodPix[warpType], 100.0*totGoodPix[warpType]/skyInfo.bbox.getArea())
470
471 if totGoodPix[warpType] > 0 and didSetMetadata[warpType]:
472 inputRecorder[warpType].finish(coaddTempExps[warpType], totGoodPix[warpType])
473 if warpType == "direct":
474 coaddTempExps[warpType].setPsf(
475 CoaddPsf(inputRecorder[warpType].coaddInputs.ccds, skyInfo.wcs,
476 self.config.coaddPsf.makeControl()))
477 else:
478 if not self.config.doWriteEmptyWarps:
479 # No good pixels. Exposure still empty
480 coaddTempExps[warpType] = None
481 # NoWorkFound is unnecessary as the downstream tasks will
482 # adjust the quantum accordingly, and it prevents gen2
483 # MakeCoaddTempExp from continuing to loop over visits.
484
485 result = pipeBase.Struct(exposures=coaddTempExps)
486 return result
487
488 def getCalibratedExposure(self, dataRef, bgSubtracted):
489 """Return one calibrated Exposure, possibly with an updated SkyWcs.
490
491 @param[in] dataRef a sensor-level data reference
492 @param[in] bgSubtracted return calexp with background subtracted? If False get the
493 calexp's background background model and add it to the calexp.
494 @return calibrated exposure
495
496 @raises MissingExposureError If data for the exposure is not available.
497
498 If config.doApplyExternalPhotoCalib is `True`, the photometric calibration
499 (`photoCalib`) is taken from `config.externalPhotoCalibName` via the
500 `name_photoCalib` dataset. Otherwise, the photometric calibration is
501 retrieved from the processed exposure. When
502 `config.doApplyExternalSkyWcs` is `True`, the astrometric calibration
503 is taken from `config.externalSkyWcsName` with the `name_wcs` dataset.
504 Otherwise, the astrometric calibration is taken from the processed
505 exposure.
506 """
507 try:
508 exposure = dataRef.get(self.calexpTypecalexpType, immediate=True)
509 except dafPersist.NoResults as e:
510 raise MissingExposureError('Exposure not found: %s ' % str(e)) from e
511
512 if not bgSubtracted:
513 background = dataRef.get("calexpBackground", immediate=True)
514 mi = exposure.getMaskedImage()
515 mi += background.getImage()
516 del mi
517
518 if self.config.doApplyExternalPhotoCalib:
519 source = f"{self.config.externalPhotoCalibName}_photoCalib"
520 self.log.debug("Applying external photoCalib to %s from %s", dataRef.dataId, source)
521 photoCalib = dataRef.get(source)
522 exposure.setPhotoCalib(photoCalib)
523 else:
524 photoCalib = exposure.getPhotoCalib()
525
526 if self.config.doApplyExternalSkyWcs:
527 source = f"{self.config.externalSkyWcsName}_wcs"
528 self.log.debug("Applying external skyWcs to %s from %s", dataRef.dataId, source)
529 skyWcs = dataRef.get(source)
530 exposure.setWcs(skyWcs)
531
532 exposure.maskedImage = photoCalib.calibrateImage(exposure.maskedImage,
533 includeScaleUncertainty=self.config.includeCalibVar)
534 exposure.maskedImage /= photoCalib.getCalibrationMean()
535 # TODO: The images will have a calibration of 1.0 everywhere once RFC-545 is implemented.
536 # exposure.setCalib(afwImage.Calib(1.0))
537 return exposure
538
539 @staticmethod
540 def _prepareEmptyExposure(skyInfo):
541 """Produce an empty exposure for a given patch"""
542 exp = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
543 exp.getMaskedImage().set(numpy.nan, afwImage.Mask
544 .getPlaneBitMask("NO_DATA"), numpy.inf)
545 return exp
546
548 """Return list of requested warp types per the config.
549 """
550 warpTypeList = []
551 if self.config.makeDirect:
552 warpTypeList.append("direct")
553 if self.config.makePsfMatched:
554 warpTypeList.append("psfMatched")
555 return warpTypeList
556
557 def applySkyCorr(self, dataRef, calexp):
558 """Apply correction to the sky background level
559
560 Sky corrections can be generated with the 'skyCorrection.py'
561 executable in pipe_drivers. Because the sky model used by that
562 code extends over the entire focal plane, this can produce
563 better sky subtraction.
564
565 The calexp is updated in-place.
566
567 Parameters
568 ----------
569 dataRef : `lsst.daf.persistence.ButlerDataRef`
570 Data reference for calexp.
572 Calibrated exposure.
573 """
574 bg = dataRef.get("skyCorr")
575 self.log.debug("Applying sky correction to %s", dataRef.dataId)
576 if isinstance(calexp, afwImage.Exposure):
577 calexp = calexp.getMaskedImage()
578 calexp -= bg.getImage()
579
580
581class MakeWarpConnections(pipeBase.PipelineTaskConnections,
582 dimensions=("tract", "patch", "skymap", "instrument", "visit"),
583 defaultTemplates={"coaddName": "deep",
584 "skyWcsName": "jointcal",
585 "photoCalibName": "fgcm",
586 "calexpType": ""}):
587 calExpList = connectionTypes.Input(
588 doc="Input exposures to be resampled and optionally PSF-matched onto a SkyMap projection/patch",
589 name="{calexpType}calexp",
590 storageClass="ExposureF",
591 dimensions=("instrument", "visit", "detector"),
592 multiple=True,
593 deferLoad=True,
594 )
595 backgroundList = connectionTypes.Input(
596 doc="Input backgrounds to be added back into the calexp if bgSubtracted=False",
597 name="calexpBackground",
598 storageClass="Background",
599 dimensions=("instrument", "visit", "detector"),
600 multiple=True,
601 )
602 skyCorrList = connectionTypes.Input(
603 doc="Input Sky Correction to be subtracted from the calexp if doApplySkyCorr=True",
604 name="skyCorr",
605 storageClass="Background",
606 dimensions=("instrument", "visit", "detector"),
607 multiple=True,
608 )
609 skyMap = connectionTypes.Input(
610 doc="Input definition of geometry/bbox and projection/wcs for warped exposures",
611 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
612 storageClass="SkyMap",
613 dimensions=("skymap",),
614 )
615 externalSkyWcsTractCatalog = connectionTypes.Input(
616 doc=("Per-tract, per-visit wcs calibrations. These catalogs use the detector "
617 "id for the catalog id, sorted on id for fast lookup."),
618 name="{skyWcsName}SkyWcsCatalog",
619 storageClass="ExposureCatalog",
620 dimensions=("instrument", "visit", "tract"),
621 )
622 externalSkyWcsGlobalCatalog = connectionTypes.Input(
623 doc=("Per-visit wcs calibrations computed globally (with no tract information). "
624 "These catalogs use the detector id for the catalog id, sorted on id for "
625 "fast lookup."),
626 name="{skyWcsName}SkyWcsCatalog",
627 storageClass="ExposureCatalog",
628 dimensions=("instrument", "visit"),
629 )
630 externalPhotoCalibTractCatalog = connectionTypes.Input(
631 doc=("Per-tract, per-visit photometric calibrations. These catalogs use the "
632 "detector id for the catalog id, sorted on id for fast lookup."),
633 name="{photoCalibName}PhotoCalibCatalog",
634 storageClass="ExposureCatalog",
635 dimensions=("instrument", "visit", "tract"),
636 )
637 externalPhotoCalibGlobalCatalog = connectionTypes.Input(
638 doc=("Per-visit photometric calibrations computed globally (with no tract "
639 "information). These catalogs use the detector id for the catalog id, "
640 "sorted on id for fast lookup."),
641 name="{photoCalibName}PhotoCalibCatalog",
642 storageClass="ExposureCatalog",
643 dimensions=("instrument", "visit"),
644 )
645 direct = connectionTypes.Output(
646 doc=("Output direct warped exposure (previously called CoaddTempExp), produced by resampling ",
647 "calexps onto the skyMap patch geometry."),
648 name="{coaddName}Coadd_directWarp",
649 storageClass="ExposureF",
650 dimensions=("tract", "patch", "skymap", "visit", "instrument"),
651 )
652 psfMatched = connectionTypes.Output(
653 doc=("Output PSF-Matched warped exposure (previously called CoaddTempExp), produced by resampling ",
654 "calexps onto the skyMap patch geometry and PSF-matching to a model PSF."),
655 name="{coaddName}Coadd_psfMatchedWarp",
656 storageClass="ExposureF",
657 dimensions=("tract", "patch", "skymap", "visit", "instrument"),
658 )
659 # TODO DM-28769, have selectImages subtask indicate which connections they need:
660 wcsList = connectionTypes.Input(
661 doc="WCSs of calexps used by SelectImages subtask to determine if the calexp overlaps the patch",
662 name="{calexpType}calexp.wcs",
663 storageClass="Wcs",
664 dimensions=("instrument", "visit", "detector"),
665 multiple=True,
666 )
667 bboxList = connectionTypes.Input(
668 doc="BBoxes of calexps used by SelectImages subtask to determine if the calexp overlaps the patch",
669 name="{calexpType}calexp.bbox",
670 storageClass="Box2I",
671 dimensions=("instrument", "visit", "detector"),
672 multiple=True,
673 )
674 visitSummary = connectionTypes.Input(
675 doc="Consolidated exposure metadata from ConsolidateVisitSummaryTask",
676 name="{calexpType}visitSummary",
677 storageClass="ExposureCatalog",
678 dimensions=("instrument", "visit",),
679 )
680 srcList = connectionTypes.Input(
681 doc="Source catalogs used by PsfWcsSelectImages subtask to further select on PSF stability",
682 name="src",
683 storageClass="SourceCatalog",
684 dimensions=("instrument", "visit", "detector"),
685 multiple=True,
686 )
687
688 def __init__(self, *, config=None):
689 super().__init__(config=config)
690 if config.bgSubtracted:
691 self.inputs.remove("backgroundList")
692 if not config.doApplySkyCorr:
693 self.inputs.remove("skyCorrList")
694 if config.doApplyExternalSkyWcs:
695 if config.useGlobalExternalSkyWcs:
696 self.inputs.remove("externalSkyWcsTractCatalog")
697 else:
698 self.inputs.remove("externalSkyWcsGlobalCatalog")
699 else:
700 self.inputs.remove("externalSkyWcsTractCatalog")
701 self.inputs.remove("externalSkyWcsGlobalCatalog")
702 if config.doApplyExternalPhotoCalib:
703 if config.useGlobalExternalPhotoCalib:
704 self.inputs.remove("externalPhotoCalibTractCatalog")
705 else:
706 self.inputs.remove("externalPhotoCalibGlobalCatalog")
707 else:
708 self.inputs.remove("externalPhotoCalibTractCatalog")
709 self.inputs.remove("externalPhotoCalibGlobalCatalog")
710 if not config.makeDirect:
711 self.outputs.remove("direct")
712 if not config.makePsfMatched:
713 self.outputs.remove("psfMatched")
714 # TODO DM-28769: add connection per selectImages connections
715 if config.select.target != lsst.pipe.tasks.selectImages.PsfWcsSelectImagesTask:
716 self.inputs.remove("visitSummary")
717 self.inputs.remove("srcList")
718 elif not config.select.doLegacyStarSelectionComputation:
719 # Remove backwards-compatibility connections.
720 self.inputs.remove("srcList")
721
722
723class MakeWarpConfig(pipeBase.PipelineTaskConfig, MakeCoaddTempExpConfig,
724 pipelineConnections=MakeWarpConnections):
725
726 def validate(self):
727 super().validate()
728
729
730class MakeWarpTask(MakeCoaddTempExpTask):
731 """Warp and optionally PSF-Match calexps onto an a common projection
732 """
733 ConfigClass = MakeWarpConfig
734 _DefaultName = "makeWarp"
735
736 @utils.inheritDoc(pipeBase.PipelineTask)
737 def runQuantum(self, butlerQC, inputRefs, outputRefs):
738 """
739 Notes
740 ----
741 Construct warps for requested warp type for single epoch
742
743 PipelineTask (Gen3) entry point to warp and optionally PSF-match
744 calexps. This method is analogous to `runDataRef`.
745 """
746
747 # Ensure all input lists are in same detector order as the calExpList
748 detectorOrder = [ref.datasetRef.dataId['detector'] for ref in inputRefs.calExpList]
749 inputRefs = reorderRefs(inputRefs, detectorOrder, dataIdKey='detector')
750
751 # Read in all inputs.
752 inputs = butlerQC.get(inputRefs)
753
754 # Construct skyInfo expected by `run`. We remove the SkyMap itself
755 # from the dictionary so we can pass it as kwargs later.
756 skyMap = inputs.pop("skyMap")
757 quantumDataId = butlerQC.quantum.dataId
758 skyInfo = makeSkyInfo(skyMap, tractId=quantumDataId['tract'], patchId=quantumDataId['patch'])
759
760 # Construct list of input DataIds expected by `run`
761 dataIdList = [ref.datasetRef.dataId for ref in inputRefs.calExpList]
762 # Construct list of packed integer IDs expected by `run`
763 ccdIdList = [dataId.pack("visit_detector") for dataId in dataIdList]
764
765 # Run the selector and filter out calexps that were not selected
766 # primarily because they do not overlap the patch
767 cornerPosList = lsst.geom.Box2D(skyInfo.bbox).getCorners()
768 coordList = [skyInfo.wcs.pixelToSky(pos) for pos in cornerPosList]
769 goodIndices = self.select.run(**inputs, coordList=coordList, dataIds=dataIdList)
770 inputs = self.filterInputs(indices=goodIndices, inputs=inputs)
771
772 # Read from disk only the selected calexps
773 inputs['calExpList'] = [ref.get() for ref in inputs['calExpList']]
774
775 # Extract integer visitId requested by `run`
776 visits = [dataId['visit'] for dataId in dataIdList]
777 visitId = visits[0]
778
779 if self.config.doApplyExternalSkyWcs:
780 if self.config.useGlobalExternalSkyWcs:
781 externalSkyWcsCatalog = inputs.pop("externalSkyWcsGlobalCatalog")
782 else:
783 externalSkyWcsCatalog = inputs.pop("externalSkyWcsTractCatalog")
784 else:
785 externalSkyWcsCatalog = None
786
787 if self.config.doApplyExternalPhotoCalib:
788 if self.config.useGlobalExternalPhotoCalib:
789 externalPhotoCalibCatalog = inputs.pop("externalPhotoCalibGlobalCatalog")
790 else:
791 externalPhotoCalibCatalog = inputs.pop("externalPhotoCalibTractCatalog")
792 else:
793 externalPhotoCalibCatalog = None
794
795 completeIndices = self.prepareCalibratedExposures(**inputs,
796 externalSkyWcsCatalog=externalSkyWcsCatalog,
797 externalPhotoCalibCatalog=externalPhotoCalibCatalog)
798 # Redo the input selection with inputs with complete wcs/photocalib info.
799 inputs = self.filterInputs(indices=completeIndices, inputs=inputs)
800
801 results = self.run(**inputs, visitId=visitId,
802 ccdIdList=[ccdIdList[i] for i in goodIndices],
803 dataIdList=[dataIdList[i] for i in goodIndices],
804 skyInfo=skyInfo)
805 if self.config.makeDirect and results.exposures["direct"] is not None:
806 butlerQC.put(results.exposures["direct"], outputRefs.direct)
807 if self.config.makePsfMatched and results.exposures["psfMatched"] is not None:
808 butlerQC.put(results.exposures["psfMatched"], outputRefs.psfMatched)
809
810 def filterInputs(self, indices, inputs):
811 """Return task inputs with their lists filtered by indices
812
813 Parameters
814 ----------
815 indices : `list` of integers
816 inputs : `dict` of `list` of input connections to be passed to run
817 """
818 for key in inputs.keys():
819 # Only down-select on list inputs
820 if isinstance(inputs[key], list):
821 inputs[key] = [inputs[key][ind] for ind in indices]
822 return inputs
823
824 def prepareCalibratedExposures(self, calExpList, backgroundList=None, skyCorrList=None,
825 externalSkyWcsCatalog=None, externalPhotoCalibCatalog=None,
826 **kwargs):
827 """Calibrate and add backgrounds to input calExpList in place
828
829 Parameters
830 ----------
831 calExpList : `list` of `lsst.afw.image.Exposure`
832 Sequence of calexps to be modified in place
833 backgroundList : `list` of `lsst.afw.math.backgroundList`, optional
834 Sequence of backgrounds to be added back in if bgSubtracted=False
835 skyCorrList : `list` of `lsst.afw.math.backgroundList`, optional
836 Sequence of background corrections to be subtracted if doApplySkyCorr=True
837 externalSkyWcsCatalog : `lsst.afw.table.ExposureCatalog`, optional
838 Exposure catalog with external skyWcs to be applied
839 if config.doApplyExternalSkyWcs=True. Catalog uses the detector id
840 for the catalog id, sorted on id for fast lookup.
841 externalPhotoCalibCatalog : `lsst.afw.table.ExposureCatalog`, optional
842 Exposure catalog with external photoCalib to be applied
843 if config.doApplyExternalPhotoCalib=True. Catalog uses the detector
844 id for the catalog id, sorted on id for fast lookup.
845
846 Returns
847 -------
848 indices : `list` [`int`]
849 Indices of calExpList and friends that have valid photoCalib/skyWcs
850 """
851 backgroundList = len(calExpList)*[None] if backgroundList is None else backgroundList
852 skyCorrList = len(calExpList)*[None] if skyCorrList is None else skyCorrList
853
854 includeCalibVar = self.config.includeCalibVar
855
856 indices = []
857 for index, (calexp, background, skyCorr) in enumerate(zip(calExpList,
858 backgroundList,
859 skyCorrList)):
860 mi = calexp.maskedImage
861 if not self.config.bgSubtracted:
862 mi += background.getImage()
863
864 if externalSkyWcsCatalog is not None or externalPhotoCalibCatalog is not None:
865 detectorId = calexp.getInfo().getDetector().getId()
866
867 # Find the external photoCalib
868 if externalPhotoCalibCatalog is not None:
869 row = externalPhotoCalibCatalog.find(detectorId)
870 if row is None:
871 self.log.warning("Detector id %s not found in externalPhotoCalibCatalog "
872 "and will not be used in the warp.", detectorId)
873 continue
874 photoCalib = row.getPhotoCalib()
875 if photoCalib is None:
876 self.log.warning("Detector id %s has None for photoCalib in externalPhotoCalibCatalog "
877 "and will not be used in the warp.", detectorId)
878 continue
879 calexp.setPhotoCalib(photoCalib)
880 else:
881 photoCalib = calexp.getPhotoCalib()
882 if photoCalib is None:
883 self.log.warning("Detector id %s has None for photoCalib in the calexp "
884 "and will not be used in the warp.", detectorId)
885 continue
886
887 # Find and apply external skyWcs
888 if externalSkyWcsCatalog is not None:
889 row = externalSkyWcsCatalog.find(detectorId)
890 if row is None:
891 self.log.warning("Detector id %s not found in externalSkyWcsCatalog "
892 "and will not be used in the warp.", detectorId)
893 continue
894 skyWcs = row.getWcs()
895 if skyWcs is None:
896 self.log.warning("Detector id %s has None for skyWcs in externalSkyWcsCatalog "
897 "and will not be used in the warp.", detectorId)
898 continue
899 calexp.setWcs(skyWcs)
900 else:
901 skyWcs = calexp.getWcs()
902 if skyWcs is None:
903 self.log.warning("Detector id %s has None for skyWcs in the calexp "
904 "and will not be used in the warp.", detectorId)
905 continue
906
907 # Calibrate the image
908 calexp.maskedImage = photoCalib.calibrateImage(calexp.maskedImage,
909 includeScaleUncertainty=includeCalibVar)
910 calexp.maskedImage /= photoCalib.getCalibrationMean()
911 # TODO: The images will have a calibration of 1.0 everywhere once RFC-545 is implemented.
912 # exposure.setCalib(afwImage.Calib(1.0))
913
914 # Apply skycorr
915 if self.config.doApplySkyCorr:
916 mi -= skyCorr.getImage()
917
918 indices.append(index)
919
920 return indices
921
922
923def reorderRefs(inputRefs, outputSortKeyOrder, dataIdKey):
924 """Reorder inputRefs per outputSortKeyOrder
925
926 Any inputRefs which are lists will be resorted per specified key e.g.,
927 'detector.' Only iterables will be reordered, and values can be of type
928 `lsst.pipe.base.connections.DeferredDatasetRef` or
929 `lsst.daf.butler.core.datasets.ref.DatasetRef`.
930 Returned lists of refs have the same length as the outputSortKeyOrder.
931 If an outputSortKey not in the inputRef, then it will be padded with None.
932 If an inputRef contains an inputSortKey that is not in the
933 outputSortKeyOrder it will be removed.
934
935 Parameters
936 ----------
937 inputRefs : `lsst.pipe.base.connections.QuantizedConnection`
938 Input references to be reordered and padded.
939 outputSortKeyOrder : iterable
940 Iterable of values to be compared with inputRef's dataId[dataIdKey]
941 dataIdKey : `str`
942 dataIdKey in the dataRefs to compare with the outputSortKeyOrder.
943
944 Returns:
945 --------
946 inputRefs: `lsst.pipe.base.connections.QuantizedConnection`
947 Quantized Connection with sorted DatasetRef values sorted if iterable.
948 """
949 for connectionName, refs in inputRefs:
950 if isinstance(refs, Iterable):
951 if hasattr(refs[0], "dataId"):
952 inputSortKeyOrder = [ref.dataId[dataIdKey] for ref in refs]
953 else:
954 inputSortKeyOrder = [ref.datasetRef.dataId[dataIdKey] for ref in refs]
955 if inputSortKeyOrder != outputSortKeyOrder:
956 setattr(inputRefs, connectionName,
957 reorderAndPadList(refs, inputSortKeyOrder, outputSortKeyOrder))
958 return inputRefs
Configuration parameters for CoaddBaseTask.
Definition: coaddBase.py:38
Base class for coaddition.
Definition: coaddBase.py:141
def getTempExpDatasetName(self, warpType="direct")
Definition: coaddBase.py:204
def selectExposures(self, patchRef, skyInfo=None, selectDataList=[])
Select exposures to coadd.
Definition: coaddBase.py:154
def getCoaddDatasetName(self, warpType="direct")
Definition: coaddBase.py:190
def getSkyInfo(self, patchRef)
Use getSkyinfo to return the skyMap, tract and patch information, wcs and the outer bbox of the patch...
Definition: coaddBase.py:174
def getBadPixelMask(self)
Convenience method to provide the bitmask from the mask plane names.
Definition: coaddBase.py:239
Warp and optionally PSF-Match calexps onto an a common projection.
def getCalibratedExposure(self, dataRef, bgSubtracted)
def run(self, calExpList, ccdIdList, skyInfo, visitId=0, dataIdList=None, **kwargs)
def runDataRef(self, patchRef, selectDataList=[])
Produce <coaddName>Coadd_<warpType>Warp images by warping and optionally PSF-matching.
def reorderAndPadList(inputList, inputKeys, outputKeys, padWith=None)
Definition: coaddBase.py:362
def makeSkyInfo(skyMap, tractId, patchId)
Definition: coaddBase.py:289
def getGroupDataRef(butler, datasetType, groupTuple, keys)
Definition: coaddHelpers.py:99
def groupPatchExposures(patchDataRef, calexpDataRefList, coaddDatasetType="deepCoadd", tempExpDatasetType="deepCoadd_directWarp")
Definition: coaddHelpers.py:60