lsst.pipe.tasks gb4d8b8e895+fcaa6a0601
makeCoaddTempExp.py
Go to the documentation of this file.
2# LSST Data Management System
3# Copyright 2008, 2009, 2010, 2011, 2012 LSST Corporation.
4#
5# This product includes software developed by the
6# LSST Project (http://www.lsst.org/).
7#
8# This program is free software: you can redistribute it and/or modify
9# it under the terms of the GNU General Public License as published by
10# the Free Software Foundation, either version 3 of the License, or
11# (at your option) any later version.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the LSST License Statement and
19# the GNU General Public License along with this program. If not,
20# see <http://www.lsstcorp.org/LegalNotices/>.
21#
22import numpy
23import logging
24
25import lsst.pex.config as pexConfig
26import lsst.daf.persistence as dafPersist
27import lsst.afw.image as afwImage
28import lsst.coadd.utils as coaddUtils
29import lsst.pipe.base as pipeBase
30import lsst.pipe.base.connectionTypes as connectionTypes
31import lsst.utils as utils
32import lsst.geom
33from lsst.meas.algorithms import CoaddPsf, CoaddPsfConfig
34from lsst.skymap import BaseSkyMap
35from lsst.utils.timer import timeMethod
36from .coaddBase import CoaddBaseTask, makeSkyInfo, reorderAndPadList
37from .warpAndPsfMatch import WarpAndPsfMatchTask
38from .coaddHelpers import groupPatchExposures, getGroupDataRef
39from collections.abc import Iterable
40
41__all__ = ["MakeCoaddTempExpTask", "MakeWarpTask", "MakeWarpConfig"]
42
43log = logging.getLogger(__name__)
44
45
46class MissingExposureError(Exception):
47 """Raised when data cannot be retrieved for an exposure.
48 When processing patches, sometimes one exposure is missing; this lets us
49 distinguish bewteen that case, and other errors.
50 """
51 pass
52
53
54class MakeCoaddTempExpConfig(CoaddBaseTask.ConfigClass):
55 """Config for MakeCoaddTempExpTask
56 """
57 warpAndPsfMatch = pexConfig.ConfigurableField(
58 target=WarpAndPsfMatchTask,
59 doc="Task to warp and PSF-match calexp",
60 )
61 doWrite = pexConfig.Field(
62 doc="persist <coaddName>Coadd_<warpType>Warp",
63 dtype=bool,
64 default=True,
65 )
66 bgSubtracted = pexConfig.Field(
67 doc="Work with a background subtracted calexp?",
68 dtype=bool,
69 default=True,
70 )
71 coaddPsf = pexConfig.ConfigField(
72 doc="Configuration for CoaddPsf",
73 dtype=CoaddPsfConfig,
74 )
75 makeDirect = pexConfig.Field(
76 doc="Make direct Warp/Coadds",
77 dtype=bool,
78 default=True,
79 )
80 makePsfMatched = pexConfig.Field(
81 doc="Make Psf-Matched Warp/Coadd?",
82 dtype=bool,
83 default=False,
84 )
85
86 doWriteEmptyWarps = pexConfig.Field(
87 dtype=bool,
88 default=False,
89 doc="Write out warps even if they are empty"
90 )
91
92 hasFakes = pexConfig.Field(
93 doc="Should be set to True if fake sources have been inserted into the input data.",
94 dtype=bool,
95 default=False,
96 )
97 doApplySkyCorr = pexConfig.Field(dtype=bool, default=False, doc="Apply sky correction?")
98
99 def validate(self):
100 CoaddBaseTask.ConfigClass.validate(self)
101 if not self.makePsfMatchedmakePsfMatched and not self.makeDirectmakeDirect:
102 raise RuntimeError("At least one of config.makePsfMatched and config.makeDirect must be True")
103 if self.doPsfMatch:
104 # Backwards compatibility.
105 log.warning("Config doPsfMatch deprecated. Setting makePsfMatched=True and makeDirect=False")
106 self.makePsfMatchedmakePsfMatched = True
107 self.makeDirectmakeDirect = False
108
109 def setDefaults(self):
110 CoaddBaseTask.ConfigClass.setDefaults(self)
111 self.warpAndPsfMatchwarpAndPsfMatch.psfMatch.kernel.active.kernelSize = self.matchingKernelSize
112
113
119
120
122 r"""!Warp and optionally PSF-Match calexps onto an a common projection.
123
124 @anchor MakeCoaddTempExpTask_
125
126 @section pipe_tasks_makeCoaddTempExp_Contents Contents
127
128 - @ref pipe_tasks_makeCoaddTempExp_Purpose
129 - @ref pipe_tasks_makeCoaddTempExp_Initialize
130 - @ref pipe_tasks_makeCoaddTempExp_IO
131 - @ref pipe_tasks_makeCoaddTempExp_Config
132 - @ref pipe_tasks_makeCoaddTempExp_Debug
133 - @ref pipe_tasks_makeCoaddTempExp_Example
134
135 @section pipe_tasks_makeCoaddTempExp_Purpose Description
136
137 Warp and optionally PSF-Match calexps onto a common projection, by
138 performing the following operations:
139 - Group calexps by visit/run
140 - For each visit, generate a Warp by calling method @ref makeTempExp.
141 makeTempExp loops over the visit's calexps calling @ref WarpAndPsfMatch
142 on each visit
143
144 The result is a `directWarp` (and/or optionally a `psfMatchedWarp`).
145
146 @section pipe_tasks_makeCoaddTempExp_Initialize Task Initialization
147
148 @copydoc \_\_init\_\_
149
150 This task has one special keyword argument: passing reuse=True will cause
151 the task to skip the creation of warps that are already present in the
152 output repositories.
153
154 @section pipe_tasks_makeCoaddTempExp_IO Invoking the Task
155
156 This task is primarily designed to be run from the command line.
157
158 The main method is `runDataRef`, which takes a single butler data reference for the patch(es)
159 to process.
160
161 @copydoc run
162
163 WarpType identifies the types of convolutions applied to Warps (previously CoaddTempExps).
164 Only two types are available: direct (for regular Warps/Coadds) and psfMatched
165 (for Warps/Coadds with homogenized PSFs). We expect to add a third type, likelihood,
166 for generating likelihood Coadds with Warps that have been correlated with their own PSF.
167
168 @section pipe_tasks_makeCoaddTempExp_Config Configuration parameters
169
170 See @ref MakeCoaddTempExpConfig and parameters inherited from
171 @link lsst.pipe.tasks.coaddBase.CoaddBaseConfig CoaddBaseConfig @endlink
172
173 @subsection pipe_tasks_MakeCoaddTempExp_psfMatching Guide to PSF-Matching Configs
174
175 To make `psfMatchedWarps`, select `config.makePsfMatched=True`. The subtask
176 @link lsst.ip.diffim.modelPsfMatch.ModelPsfMatchTask ModelPsfMatchTask @endlink
177 is responsible for the PSF-Matching, and its config is accessed via `config.warpAndPsfMatch.psfMatch`.
178 The optimal configuration depends on aspects of dataset: the pixel scale, average PSF FWHM and
179 dimensions of the PSF kernel. These configs include the requested model PSF, the matching kernel size,
180 padding of the science PSF thumbnail and spatial sampling frequency of the PSF.
181
182 *Config Guidelines*: The user must specify the size of the model PSF to which to match by setting
183 `config.modelPsf.defaultFwhm` in units of pixels. The appropriate values depends on science case.
184 In general, for a set of input images, this config should equal the FWHM of the visit
185 with the worst seeing. The smallest it should be set to is the median FWHM. The defaults
186 of the other config options offer a reasonable starting point.
187 The following list presents the most common problems that arise from a misconfigured
188 @link lsst.ip.diffim.modelPsfMatch.ModelPsfMatchTask ModelPsfMatchTask @endlink
189 and corresponding solutions. All assume the default Alard-Lupton kernel, with configs accessed via
190 ```config.warpAndPsfMatch.psfMatch.kernel['AL']```. Each item in the list is formatted as:
191 Problem: Explanation. *Solution*
192
193 *Troublshooting PSF-Matching Configuration:*
194 - Matched PSFs look boxy: The matching kernel is too small. _Increase the matching kernel size.
195 For example:_
196
197 config.warpAndPsfMatch.psfMatch.kernel['AL'].kernelSize=27 # default 21
198
199 Note that increasing the kernel size also increases runtime.
200 - Matched PSFs look ugly (dipoles, quadropoles, donuts): unable to find good solution
201 for matching kernel. _Provide the matcher with more data by either increasing
202 the spatial sampling by decreasing the spatial cell size,_
203
204 config.warpAndPsfMatch.psfMatch.kernel['AL'].sizeCellX = 64 # default 128
205 config.warpAndPsfMatch.psfMatch.kernel['AL'].sizeCellY = 64 # default 128
206
207 _or increasing the padding around the Science PSF, for example:_
208
209 config.warpAndPsfMatch.psfMatch.autoPadPsfTo=1.6 # default 1.4
210
211 Increasing `autoPadPsfTo` increases the minimum ratio of input PSF dimensions to the
212 matching kernel dimensions, thus increasing the number of pixels available to fit
213 after convolving the PSF with the matching kernel.
214 Optionally, for debugging the effects of padding, the level of padding may be manually
215 controlled by setting turning off the automatic padding and setting the number
216 of pixels by which to pad the PSF:
217
218 config.warpAndPsfMatch.psfMatch.doAutoPadPsf = False # default True
219 config.warpAndPsfMatch.psfMatch.padPsfBy = 6 # pixels. default 0
220
221 - Deconvolution: Matching a large PSF to a smaller PSF produces
222 a telltale noise pattern which looks like ripples or a brain.
223 _Increase the size of the requested model PSF. For example:_
224
225 config.modelPsf.defaultFwhm = 11 # Gaussian sigma in units of pixels.
226
227 - High frequency (sometimes checkered) noise: The matching basis functions are too small.
228 _Increase the width of the Gaussian basis functions. For example:_
229
230 config.warpAndPsfMatch.psfMatch.kernel['AL'].alardSigGauss=[1.5, 3.0, 6.0]
231 # from default [0.7, 1.5, 3.0]
232
233
234 @section pipe_tasks_makeCoaddTempExp_Debug Debug variables
235
236 MakeCoaddTempExpTask has no debug output, but its subtasks do.
237
238 @section pipe_tasks_makeCoaddTempExp_Example A complete example of using MakeCoaddTempExpTask
239
240 This example uses the package ci_hsc to show how MakeCoaddTempExp fits
241 into the larger Data Release Processing.
242 Set up by running:
243
244 setup ci_hsc
245 cd $CI_HSC_DIR
246 # if not built already:
247 python $(which scons) # this will take a while
248
249 The following assumes that `processCcd.py` and `makeSkyMap.py` have previously been run
250 (e.g. by building `ci_hsc` above) to generate a repository of calexps and an
251 output respository with the desired SkyMap. The command,
252
253 makeCoaddTempExp.py $CI_HSC_DIR/DATA --rerun ci_hsc \
254 --id patch=5,4 tract=0 filter=HSC-I \
255 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 \
256 --selectId visit=903988 ccd=23 --selectId visit=903988 ccd=24 \
257 --config doApplyExternalPhotoCalib=False doApplyExternalSkyWcs=False \
258 makePsfMatched=True modelPsf.defaultFwhm=11
259
260 writes a direct and PSF-Matched Warp to
261 - `$CI_HSC_DIR/DATA/rerun/ci_hsc/deepCoadd/HSC-I/0/5,4/warp-HSC-I-0-5,4-903988.fits` and
262 - `$CI_HSC_DIR/DATA/rerun/ci_hsc/deepCoadd/HSC-I/0/5,4/psfMatchedWarp-HSC-I-0-5,4-903988.fits`
263 respectively.
264
265 @note PSF-Matching in this particular dataset would benefit from adding
266 `--configfile ./matchingConfig.py` to
267 the command line arguments where `matchingConfig.py` is defined by:
268
269 echo "
270 config.warpAndPsfMatch.psfMatch.kernel['AL'].kernelSize=27
271 config.warpAndPsfMatch.psfMatch.kernel['AL'].alardSigGauss=[1.5, 3.0, 6.0]" > matchingConfig.py
272
273
274 Add the option `--help` to see more options.
275 """
276 ConfigClass = MakeCoaddTempExpConfig
277 _DefaultName = "makeCoaddTempExp"
278
279 def __init__(self, reuse=False, **kwargs):
280 CoaddBaseTask.__init__(self, **kwargs)
281 self.reusereuse = reuse
282 self.makeSubtask("warpAndPsfMatch")
283 if self.config.hasFakes:
284 self.calexpTypecalexpType = "fakes_calexp"
285 else:
286 self.calexpTypecalexpType = "calexp"
287
288 @timeMethod
289 def runDataRef(self, patchRef, selectDataList=[]):
290 """!Produce <coaddName>Coadd_<warpType>Warp images by warping and optionally PSF-matching.
291
292 @param[in] patchRef: data reference for sky map patch. Must include keys "tract", "patch",
293 plus the camera-specific filter key (e.g. "filter" or "band")
294 @return: dataRefList: a list of data references for the new <coaddName>Coadd_directWarps
295 if direct or both warp types are requested and <coaddName>Coadd_psfMatchedWarps if only psfMatched
296 warps are requested.
297
298 @warning: this task assumes that all exposures in a warp (coaddTempExp) have the same filter.
299
300 @warning: this task sets the PhotoCalib of the coaddTempExp to the PhotoCalib of the first calexp
301 with any good pixels in the patch. For a mosaic camera the resulting PhotoCalib should be ignored
302 (assembleCoadd should determine zeropoint scaling without referring to it).
303 """
304 skyInfo = self.getSkyInfogetSkyInfo(patchRef)
305
306 # DataRefs to return are of type *_directWarp unless only *_psfMatchedWarp requested
307 if self.config.makePsfMatched and not self.config.makeDirect:
308 primaryWarpDataset = self.getTempExpDatasetNamegetTempExpDatasetName("psfMatched")
309 else:
310 primaryWarpDataset = self.getTempExpDatasetNamegetTempExpDatasetName("direct")
311
312 calExpRefList = self.selectExposuresselectExposures(patchRef, skyInfo, selectDataList=selectDataList)
313
314 if len(calExpRefList) == 0:
315 self.log.warning("No exposures to coadd for patch %s", patchRef.dataId)
316 return None
317 self.log.info("Selected %d calexps for patch %s", len(calExpRefList), patchRef.dataId)
318 calExpRefList = [calExpRef for calExpRef in calExpRefList if calExpRef.datasetExists(self.calexpTypecalexpType)]
319 self.log.info("Processing %d existing calexps for patch %s", len(calExpRefList), patchRef.dataId)
320
321 groupData = groupPatchExposures(patchRef, calExpRefList, self.getCoaddDatasetNamegetCoaddDatasetName(),
322 primaryWarpDataset)
323 self.log.info("Processing %d warp exposures for patch %s", len(groupData.groups), patchRef.dataId)
324
325 dataRefList = []
326 for i, (tempExpTuple, calexpRefList) in enumerate(groupData.groups.items()):
327 tempExpRef = getGroupDataRef(patchRef.getButler(), primaryWarpDataset,
328 tempExpTuple, groupData.keys)
329 if self.reusereuse and tempExpRef.datasetExists(datasetType=primaryWarpDataset, write=True):
330 self.log.info("Skipping makeCoaddTempExp for %s; output already exists.", tempExpRef.dataId)
331 dataRefList.append(tempExpRef)
332 continue
333 self.log.info("Processing Warp %d/%d: id=%s", i, len(groupData.groups), tempExpRef.dataId)
334
335 # TODO: mappers should define a way to go from the "grouping keys" to a numeric ID (#2776).
336 # For now, we try to get a long integer "visit" key, and if we can't, we just use the index
337 # of the visit in the list.
338 try:
339 visitId = int(tempExpRef.dataId["visit"])
340 except (KeyError, ValueError):
341 visitId = i
342
343 calExpList = []
344 ccdIdList = []
345 dataIdList = []
346
347 for calExpInd, calExpRef in enumerate(calexpRefList):
348 self.log.info("Reading calexp %s of %s for Warp id=%s", calExpInd+1, len(calexpRefList),
349 calExpRef.dataId)
350 try:
351 ccdId = calExpRef.get("ccdExposureId", immediate=True)
352 except Exception:
353 ccdId = calExpInd
354 try:
355 # We augment the dataRef here with the tract, which is harmless for loading things
356 # like calexps that don't need the tract, and necessary for meas_mosaic outputs,
357 # which do.
358 calExpRef = calExpRef.butlerSubset.butler.dataRef(self.calexpTypecalexpType,
359 dataId=calExpRef.dataId,
360 tract=skyInfo.tractInfo.getId())
361 calExp = self.getCalibratedExposuregetCalibratedExposure(calExpRef, bgSubtracted=self.config.bgSubtracted)
362 except Exception as e:
363 self.log.warning("Calexp %s not found; skipping it: %s", calExpRef.dataId, e)
364 continue
365
366 if self.config.doApplySkyCorr:
367 self.applySkyCorrapplySkyCorr(calExpRef, calExp)
368
369 calExpList.append(calExp)
370 ccdIdList.append(ccdId)
371 dataIdList.append(calExpRef.dataId)
372
373 exps = self.runrun(calExpList, ccdIdList, skyInfo, visitId, dataIdList).exposures
374
375 if any(exps.values()):
376 dataRefList.append(tempExpRef)
377 else:
378 self.log.warning("Warp %s could not be created", tempExpRef.dataId)
379
380 if self.config.doWrite:
381 for (warpType, exposure) in exps.items(): # compatible w/ Py3
382 if exposure is not None:
383 self.log.info("Persisting %s", self.getTempExpDatasetNamegetTempExpDatasetName(warpType))
384 tempExpRef.put(exposure, self.getTempExpDatasetNamegetTempExpDatasetName(warpType))
385
386 return dataRefList
387
388 @timeMethod
389 def run(self, calExpList, ccdIdList, skyInfo, visitId=0, dataIdList=None, **kwargs):
390 """Create a Warp from inputs
391
392 We iterate over the multiple calexps in a single exposure to construct
393 the warp (previously called a coaddTempExp) of that exposure to the
394 supplied tract/patch.
395
396 Pixels that receive no pixels are set to NAN; this is not correct
397 (violates LSST algorithms group policy), but will be fixed up by
398 interpolating after the coaddition.
399
400 @param calexpRefList: List of data references for calexps that (may)
401 overlap the patch of interest
402 @param skyInfo: Struct from CoaddBaseTask.getSkyInfo() with geometric
403 information about the patch
404 @param visitId: integer identifier for visit, for the table that will
405 produce the CoaddPsf
406 @return a pipeBase Struct containing:
407 - exposures: a dictionary containing the warps requested:
408 "direct": direct warp if config.makeDirect
409 "psfMatched": PSF-matched warp if config.makePsfMatched
410 """
411 warpTypeList = self.getWarpTypeListgetWarpTypeList()
412
413 totGoodPix = {warpType: 0 for warpType in warpTypeList}
414 didSetMetadata = {warpType: False for warpType in warpTypeList}
415 coaddTempExps = {warpType: self._prepareEmptyExposure_prepareEmptyExposure(skyInfo) for warpType in warpTypeList}
416 inputRecorder = {warpType: self.inputRecorder.makeCoaddTempExpRecorder(visitId, len(calExpList))
417 for warpType in warpTypeList}
418
419 modelPsf = self.config.modelPsf.apply() if self.config.makePsfMatched else None
420 if dataIdList is None:
421 dataIdList = ccdIdList
422
423 for calExpInd, (calExp, ccdId, dataId) in enumerate(zip(calExpList, ccdIdList, dataIdList)):
424 self.log.info("Processing calexp %d of %d for this Warp: id=%s",
425 calExpInd+1, len(calExpList), dataId)
426
427 try:
428 warpedAndMatched = self.warpAndPsfMatch.run(calExp, modelPsf=modelPsf,
429 wcs=skyInfo.wcs, maxBBox=skyInfo.bbox,
430 makeDirect=self.config.makeDirect,
431 makePsfMatched=self.config.makePsfMatched)
432 except Exception as e:
433 self.log.warning("WarpAndPsfMatch failed for calexp %s; skipping it: %s", dataId, e)
434 continue
435 try:
436 numGoodPix = {warpType: 0 for warpType in warpTypeList}
437 for warpType in warpTypeList:
438 exposure = warpedAndMatched.getDict()[warpType]
439 if exposure is None:
440 continue
441 coaddTempExp = coaddTempExps[warpType]
442 if didSetMetadata[warpType]:
443 mimg = exposure.getMaskedImage()
444 mimg *= (coaddTempExp.getPhotoCalib().getInstFluxAtZeroMagnitude()
445 / exposure.getPhotoCalib().getInstFluxAtZeroMagnitude())
446 del mimg
447 numGoodPix[warpType] = coaddUtils.copyGoodPixels(
448 coaddTempExp.getMaskedImage(), exposure.getMaskedImage(), self.getBadPixelMaskgetBadPixelMask())
449 totGoodPix[warpType] += numGoodPix[warpType]
450 self.log.debug("Calexp %s has %d good pixels in this patch (%.1f%%) for %s",
451 dataId, numGoodPix[warpType],
452 100.0*numGoodPix[warpType]/skyInfo.bbox.getArea(), warpType)
453 if numGoodPix[warpType] > 0 and not didSetMetadata[warpType]:
454 coaddTempExp.info.id = exposure.info.id
455 coaddTempExp.setPhotoCalib(exposure.getPhotoCalib())
456 coaddTempExp.setFilterLabel(exposure.getFilterLabel())
457 coaddTempExp.getInfo().setVisitInfo(exposure.getInfo().getVisitInfo())
458 # PSF replaced with CoaddPsf after loop if and only if creating direct warp
459 coaddTempExp.setPsf(exposure.getPsf())
460 didSetMetadata[warpType] = True
461
462 # Need inputRecorder for CoaddApCorrMap for both direct and PSF-matched
463 inputRecorder[warpType].addCalExp(calExp, ccdId, numGoodPix[warpType])
464
465 except Exception as e:
466 self.log.warning("Error processing calexp %s; skipping it: %s", dataId, e)
467 continue
468
469 for warpType in warpTypeList:
470 self.log.info("%sWarp has %d good pixels (%.1f%%)",
471 warpType, totGoodPix[warpType], 100.0*totGoodPix[warpType]/skyInfo.bbox.getArea())
472
473 if totGoodPix[warpType] > 0 and didSetMetadata[warpType]:
474 inputRecorder[warpType].finish(coaddTempExps[warpType], totGoodPix[warpType])
475 if warpType == "direct":
476 coaddTempExps[warpType].setPsf(
477 CoaddPsf(inputRecorder[warpType].coaddInputs.ccds, skyInfo.wcs,
478 self.config.coaddPsf.makeControl()))
479 else:
480 if not self.config.doWriteEmptyWarps:
481 # No good pixels. Exposure still empty
482 coaddTempExps[warpType] = None
483 # NoWorkFound is unnecessary as the downstream tasks will
484 # adjust the quantum accordingly, and it prevents gen2
485 # MakeCoaddTempExp from continuing to loop over visits.
486
487 result = pipeBase.Struct(exposures=coaddTempExps)
488 return result
489
490 def getCalibratedExposure(self, dataRef, bgSubtracted):
491 """Return one calibrated Exposure, possibly with an updated SkyWcs.
492
493 @param[in] dataRef a sensor-level data reference
494 @param[in] bgSubtracted return calexp with background subtracted? If False get the
495 calexp's background background model and add it to the calexp.
496 @return calibrated exposure
497
498 @raises MissingExposureError If data for the exposure is not available.
499
500 If config.doApplyExternalPhotoCalib is `True`, the photometric calibration
501 (`photoCalib`) is taken from `config.externalPhotoCalibName` via the
502 `name_photoCalib` dataset. Otherwise, the photometric calibration is
503 retrieved from the processed exposure. When
504 `config.doApplyExternalSkyWcs` is `True`, the astrometric calibration
505 is taken from `config.externalSkyWcsName` with the `name_wcs` dataset.
506 Otherwise, the astrometric calibration is taken from the processed
507 exposure.
508 """
509 try:
510 exposure = dataRef.get(self.calexpTypecalexpType, immediate=True)
511 except dafPersist.NoResults as e:
512 raise MissingExposureError('Exposure not found: %s ' % str(e)) from e
513
514 if not bgSubtracted:
515 background = dataRef.get("calexpBackground", immediate=True)
516 mi = exposure.getMaskedImage()
517 mi += background.getImage()
518 del mi
519
520 if self.config.doApplyExternalPhotoCalib:
521 source = f"{self.config.externalPhotoCalibName}_photoCalib"
522 self.log.debug("Applying external photoCalib to %s from %s", dataRef.dataId, source)
523 photoCalib = dataRef.get(source)
524 exposure.setPhotoCalib(photoCalib)
525 else:
526 photoCalib = exposure.getPhotoCalib()
527
528 if self.config.doApplyExternalSkyWcs:
529 source = f"{self.config.externalSkyWcsName}_wcs"
530 self.log.debug("Applying external skyWcs to %s from %s", dataRef.dataId, source)
531 skyWcs = dataRef.get(source)
532 exposure.setWcs(skyWcs)
533
534 exposure.maskedImage = photoCalib.calibrateImage(exposure.maskedImage,
535 includeScaleUncertainty=self.config.includeCalibVar)
536 exposure.maskedImage /= photoCalib.getCalibrationMean()
537 # TODO: The images will have a calibration of 1.0 everywhere once RFC-545 is implemented.
538 # exposure.setCalib(afwImage.Calib(1.0))
539 return exposure
540
541 @staticmethod
542 def _prepareEmptyExposure(skyInfo):
543 """Produce an empty exposure for a given patch"""
544 exp = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
545 exp.getMaskedImage().set(numpy.nan, afwImage.Mask
546 .getPlaneBitMask("NO_DATA"), numpy.inf)
547 return exp
548
550 """Return list of requested warp types per the config.
551 """
552 warpTypeList = []
553 if self.config.makeDirect:
554 warpTypeList.append("direct")
555 if self.config.makePsfMatched:
556 warpTypeList.append("psfMatched")
557 return warpTypeList
558
559 def applySkyCorr(self, dataRef, calexp):
560 """Apply correction to the sky background level
561
562 Sky corrections can be generated with the 'skyCorrection.py'
563 executable in pipe_drivers. Because the sky model used by that
564 code extends over the entire focal plane, this can produce
565 better sky subtraction.
566
567 The calexp is updated in-place.
568
569 Parameters
570 ----------
571 dataRef : `lsst.daf.persistence.ButlerDataRef`
572 Data reference for calexp.
574 Calibrated exposure.
575 """
576 bg = dataRef.get("skyCorr")
577 self.log.debug("Applying sky correction to %s", dataRef.dataId)
578 if isinstance(calexp, afwImage.Exposure):
579 calexp = calexp.getMaskedImage()
580 calexp -= bg.getImage()
581
582
583class MakeWarpConnections(pipeBase.PipelineTaskConnections,
584 dimensions=("tract", "patch", "skymap", "instrument", "visit"),
585 defaultTemplates={"coaddName": "deep",
586 "skyWcsName": "jointcal",
587 "photoCalibName": "fgcm",
588 "calexpType": ""}):
589 calExpList = connectionTypes.Input(
590 doc="Input exposures to be resampled and optionally PSF-matched onto a SkyMap projection/patch",
591 name="{calexpType}calexp",
592 storageClass="ExposureF",
593 dimensions=("instrument", "visit", "detector"),
594 multiple=True,
595 deferLoad=True,
596 )
597 backgroundList = connectionTypes.Input(
598 doc="Input backgrounds to be added back into the calexp if bgSubtracted=False",
599 name="calexpBackground",
600 storageClass="Background",
601 dimensions=("instrument", "visit", "detector"),
602 multiple=True,
603 )
604 skyCorrList = connectionTypes.Input(
605 doc="Input Sky Correction to be subtracted from the calexp if doApplySkyCorr=True",
606 name="skyCorr",
607 storageClass="Background",
608 dimensions=("instrument", "visit", "detector"),
609 multiple=True,
610 )
611 skyMap = connectionTypes.Input(
612 doc="Input definition of geometry/bbox and projection/wcs for warped exposures",
613 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
614 storageClass="SkyMap",
615 dimensions=("skymap",),
616 )
617 externalSkyWcsTractCatalog = connectionTypes.Input(
618 doc=("Per-tract, per-visit wcs calibrations. These catalogs use the detector "
619 "id for the catalog id, sorted on id for fast lookup."),
620 name="{skyWcsName}SkyWcsCatalog",
621 storageClass="ExposureCatalog",
622 dimensions=("instrument", "visit", "tract"),
623 )
624 externalSkyWcsGlobalCatalog = connectionTypes.Input(
625 doc=("Per-visit wcs calibrations computed globally (with no tract information). "
626 "These catalogs use the detector id for the catalog id, sorted on id for "
627 "fast lookup."),
628 name="{skyWcsName}SkyWcsCatalog",
629 storageClass="ExposureCatalog",
630 dimensions=("instrument", "visit"),
631 )
632 externalPhotoCalibTractCatalog = connectionTypes.Input(
633 doc=("Per-tract, per-visit photometric calibrations. These catalogs use the "
634 "detector id for the catalog id, sorted on id for fast lookup."),
635 name="{photoCalibName}PhotoCalibCatalog",
636 storageClass="ExposureCatalog",
637 dimensions=("instrument", "visit", "tract"),
638 )
639 externalPhotoCalibGlobalCatalog = connectionTypes.Input(
640 doc=("Per-visit photometric calibrations computed globally (with no tract "
641 "information). These catalogs use the detector id for the catalog id, "
642 "sorted on id for fast lookup."),
643 name="{photoCalibName}PhotoCalibCatalog",
644 storageClass="ExposureCatalog",
645 dimensions=("instrument", "visit"),
646 )
647 direct = connectionTypes.Output(
648 doc=("Output direct warped exposure (previously called CoaddTempExp), produced by resampling ",
649 "calexps onto the skyMap patch geometry."),
650 name="{coaddName}Coadd_directWarp",
651 storageClass="ExposureF",
652 dimensions=("tract", "patch", "skymap", "visit", "instrument"),
653 )
654 psfMatched = connectionTypes.Output(
655 doc=("Output PSF-Matched warped exposure (previously called CoaddTempExp), produced by resampling ",
656 "calexps onto the skyMap patch geometry and PSF-matching to a model PSF."),
657 name="{coaddName}Coadd_psfMatchedWarp",
658 storageClass="ExposureF",
659 dimensions=("tract", "patch", "skymap", "visit", "instrument"),
660 )
661 # TODO DM-28769, have selectImages subtask indicate which connections they need:
662 wcsList = connectionTypes.Input(
663 doc="WCSs of calexps used by SelectImages subtask to determine if the calexp overlaps the patch",
664 name="{calexpType}calexp.wcs",
665 storageClass="Wcs",
666 dimensions=("instrument", "visit", "detector"),
667 multiple=True,
668 )
669 bboxList = connectionTypes.Input(
670 doc="BBoxes of calexps used by SelectImages subtask to determine if the calexp overlaps the patch",
671 name="{calexpType}calexp.bbox",
672 storageClass="Box2I",
673 dimensions=("instrument", "visit", "detector"),
674 multiple=True,
675 )
676 visitSummary = connectionTypes.Input(
677 doc="Consolidated exposure metadata from ConsolidateVisitSummaryTask",
678 name="{calexpType}visitSummary",
679 storageClass="ExposureCatalog",
680 dimensions=("instrument", "visit",),
681 )
682 srcList = connectionTypes.Input(
683 doc="Source catalogs used by PsfWcsSelectImages subtask to further select on PSF stability",
684 name="src",
685 storageClass="SourceCatalog",
686 dimensions=("instrument", "visit", "detector"),
687 multiple=True,
688 )
689
690 def __init__(self, *, config=None):
691 super().__init__(config=config)
692 if config.bgSubtracted:
693 self.inputs.remove("backgroundList")
694 if not config.doApplySkyCorr:
695 self.inputs.remove("skyCorrList")
696 if config.doApplyExternalSkyWcs:
697 if config.useGlobalExternalSkyWcs:
698 self.inputs.remove("externalSkyWcsTractCatalog")
699 else:
700 self.inputs.remove("externalSkyWcsGlobalCatalog")
701 else:
702 self.inputs.remove("externalSkyWcsTractCatalog")
703 self.inputs.remove("externalSkyWcsGlobalCatalog")
704 if config.doApplyExternalPhotoCalib:
705 if config.useGlobalExternalPhotoCalib:
706 self.inputs.remove("externalPhotoCalibTractCatalog")
707 else:
708 self.inputs.remove("externalPhotoCalibGlobalCatalog")
709 else:
710 self.inputs.remove("externalPhotoCalibTractCatalog")
711 self.inputs.remove("externalPhotoCalibGlobalCatalog")
712 if not config.makeDirect:
713 self.outputs.remove("direct")
714 if not config.makePsfMatched:
715 self.outputs.remove("psfMatched")
716 # TODO DM-28769: add connection per selectImages connections
717 if config.select.target != lsst.pipe.tasks.selectImages.PsfWcsSelectImagesTask:
718 self.inputs.remove("visitSummary")
719 self.inputs.remove("srcList")
720 elif not config.select.doLegacyStarSelectionComputation:
721 # Remove backwards-compatibility connections.
722 self.inputs.remove("srcList")
723
724
725class MakeWarpConfig(pipeBase.PipelineTaskConfig, MakeCoaddTempExpConfig,
726 pipelineConnections=MakeWarpConnections):
727
728 def validate(self):
729 super().validate()
730
731
732class MakeWarpTask(MakeCoaddTempExpTask):
733 """Warp and optionally PSF-Match calexps onto an a common projection
734 """
735 ConfigClass = MakeWarpConfig
736 _DefaultName = "makeWarp"
737
738 @utils.inheritDoc(pipeBase.PipelineTask)
739 def runQuantum(self, butlerQC, inputRefs, outputRefs):
740 """
741 Notes
742 ----
743 Construct warps for requested warp type for single epoch
744
745 PipelineTask (Gen3) entry point to warp and optionally PSF-match
746 calexps. This method is analogous to `runDataRef`.
747 """
748 # Obtain the list of input detectors from calExpList. Sort them by
749 # detector order (to ensure reproducibility). Then ensure all input
750 # lists are in the same sorted detector order.
751 detectorOrder = [ref.datasetRef.dataId['detector'] for ref in inputRefs.calExpList]
752 detectorOrder.sort()
753 inputRefs = reorderRefs(inputRefs, detectorOrder, dataIdKey='detector')
754
755 # Read in all inputs.
756 inputs = butlerQC.get(inputRefs)
757
758 # Construct skyInfo expected by `run`. We remove the SkyMap itself
759 # from the dictionary so we can pass it as kwargs later.
760 skyMap = inputs.pop("skyMap")
761 quantumDataId = butlerQC.quantum.dataId
762 skyInfo = makeSkyInfo(skyMap, tractId=quantumDataId['tract'], patchId=quantumDataId['patch'])
763
764 # Construct list of input DataIds expected by `run`
765 dataIdList = [ref.datasetRef.dataId for ref in inputRefs.calExpList]
766 # Construct list of packed integer IDs expected by `run`
767 ccdIdList = [dataId.pack("visit_detector") for dataId in dataIdList]
768
769 # Run the selector and filter out calexps that were not selected
770 # primarily because they do not overlap the patch
771 cornerPosList = lsst.geom.Box2D(skyInfo.bbox).getCorners()
772 coordList = [skyInfo.wcs.pixelToSky(pos) for pos in cornerPosList]
773 goodIndices = self.select.run(**inputs, coordList=coordList, dataIds=dataIdList)
774 inputs = self.filterInputs(indices=goodIndices, inputs=inputs)
775
776 # Read from disk only the selected calexps
777 inputs['calExpList'] = [ref.get() for ref in inputs['calExpList']]
778
779 # Extract integer visitId requested by `run`
780 visits = [dataId['visit'] for dataId in dataIdList]
781 visitId = visits[0]
782
783 if self.config.doApplyExternalSkyWcs:
784 if self.config.useGlobalExternalSkyWcs:
785 externalSkyWcsCatalog = inputs.pop("externalSkyWcsGlobalCatalog")
786 else:
787 externalSkyWcsCatalog = inputs.pop("externalSkyWcsTractCatalog")
788 else:
789 externalSkyWcsCatalog = None
790
791 if self.config.doApplyExternalPhotoCalib:
792 if self.config.useGlobalExternalPhotoCalib:
793 externalPhotoCalibCatalog = inputs.pop("externalPhotoCalibGlobalCatalog")
794 else:
795 externalPhotoCalibCatalog = inputs.pop("externalPhotoCalibTractCatalog")
796 else:
797 externalPhotoCalibCatalog = None
798
799 completeIndices = self.prepareCalibratedExposures(**inputs,
800 externalSkyWcsCatalog=externalSkyWcsCatalog,
801 externalPhotoCalibCatalog=externalPhotoCalibCatalog)
802 # Redo the input selection with inputs with complete wcs/photocalib info.
803 inputs = self.filterInputs(indices=completeIndices, inputs=inputs)
804
805 results = self.run(**inputs, visitId=visitId,
806 ccdIdList=[ccdIdList[i] for i in goodIndices],
807 dataIdList=[dataIdList[i] for i in goodIndices],
808 skyInfo=skyInfo)
809 if self.config.makeDirect and results.exposures["direct"] is not None:
810 butlerQC.put(results.exposures["direct"], outputRefs.direct)
811 if self.config.makePsfMatched and results.exposures["psfMatched"] is not None:
812 butlerQC.put(results.exposures["psfMatched"], outputRefs.psfMatched)
813
814 def filterInputs(self, indices, inputs):
815 """Return task inputs with their lists filtered by indices
816
817 Parameters
818 ----------
819 indices : `list` of integers
820 inputs : `dict` of `list` of input connections to be passed to run
821 """
822 for key in inputs.keys():
823 # Only down-select on list inputs
824 if isinstance(inputs[key], list):
825 inputs[key] = [inputs[key][ind] for ind in indices]
826 return inputs
827
828 def prepareCalibratedExposures(self, calExpList, backgroundList=None, skyCorrList=None,
829 externalSkyWcsCatalog=None, externalPhotoCalibCatalog=None,
830 **kwargs):
831 """Calibrate and add backgrounds to input calExpList in place
832
833 Parameters
834 ----------
835 calExpList : `list` of `lsst.afw.image.Exposure`
836 Sequence of calexps to be modified in place
837 backgroundList : `list` of `lsst.afw.math.backgroundList`, optional
838 Sequence of backgrounds to be added back in if bgSubtracted=False
839 skyCorrList : `list` of `lsst.afw.math.backgroundList`, optional
840 Sequence of background corrections to be subtracted if doApplySkyCorr=True
841 externalSkyWcsCatalog : `lsst.afw.table.ExposureCatalog`, optional
842 Exposure catalog with external skyWcs to be applied
843 if config.doApplyExternalSkyWcs=True. Catalog uses the detector id
844 for the catalog id, sorted on id for fast lookup.
845 externalPhotoCalibCatalog : `lsst.afw.table.ExposureCatalog`, optional
846 Exposure catalog with external photoCalib to be applied
847 if config.doApplyExternalPhotoCalib=True. Catalog uses the detector
848 id for the catalog id, sorted on id for fast lookup.
849
850 Returns
851 -------
852 indices : `list` [`int`]
853 Indices of calExpList and friends that have valid photoCalib/skyWcs
854 """
855 backgroundList = len(calExpList)*[None] if backgroundList is None else backgroundList
856 skyCorrList = len(calExpList)*[None] if skyCorrList is None else skyCorrList
857
858 includeCalibVar = self.config.includeCalibVar
859
860 indices = []
861 for index, (calexp, background, skyCorr) in enumerate(zip(calExpList,
862 backgroundList,
863 skyCorrList)):
864 if not self.config.bgSubtracted:
865 calexp.maskedImage += background.getImage()
866
867 if externalSkyWcsCatalog is not None or externalPhotoCalibCatalog is not None:
868 detectorId = calexp.getInfo().getDetector().getId()
869
870 # Find the external photoCalib
871 if externalPhotoCalibCatalog is not None:
872 row = externalPhotoCalibCatalog.find(detectorId)
873 if row is None:
874 self.log.warning("Detector id %s not found in externalPhotoCalibCatalog "
875 "and will not be used in the warp.", detectorId)
876 continue
877 photoCalib = row.getPhotoCalib()
878 if photoCalib is None:
879 self.log.warning("Detector id %s has None for photoCalib in externalPhotoCalibCatalog "
880 "and will not be used in the warp.", detectorId)
881 continue
882 calexp.setPhotoCalib(photoCalib)
883 else:
884 photoCalib = calexp.getPhotoCalib()
885 if photoCalib is None:
886 self.log.warning("Detector id %s has None for photoCalib in the calexp "
887 "and will not be used in the warp.", detectorId)
888 continue
889
890 # Find and apply external skyWcs
891 if externalSkyWcsCatalog is not None:
892 row = externalSkyWcsCatalog.find(detectorId)
893 if row is None:
894 self.log.warning("Detector id %s not found in externalSkyWcsCatalog "
895 "and will not be used in the warp.", detectorId)
896 continue
897 skyWcs = row.getWcs()
898 if skyWcs is None:
899 self.log.warning("Detector id %s has None for skyWcs in externalSkyWcsCatalog "
900 "and will not be used in the warp.", detectorId)
901 continue
902 calexp.setWcs(skyWcs)
903 else:
904 skyWcs = calexp.getWcs()
905 if skyWcs is None:
906 self.log.warning("Detector id %s has None for skyWcs in the calexp "
907 "and will not be used in the warp.", detectorId)
908 continue
909
910 # Calibrate the image
911 calexp.maskedImage = photoCalib.calibrateImage(calexp.maskedImage,
912 includeScaleUncertainty=includeCalibVar)
913 calexp.maskedImage /= photoCalib.getCalibrationMean()
914 # TODO: The images will have a calibration of 1.0 everywhere once RFC-545 is implemented.
915 # exposure.setCalib(afwImage.Calib(1.0))
916
917 # Apply skycorr
918 if self.config.doApplySkyCorr:
919 calexp.maskedImage -= skyCorr.getImage()
920
921 indices.append(index)
922
923 return indices
924
925
926def reorderRefs(inputRefs, outputSortKeyOrder, dataIdKey):
927 """Reorder inputRefs per outputSortKeyOrder
928
929 Any inputRefs which are lists will be resorted per specified key e.g.,
930 'detector.' Only iterables will be reordered, and values can be of type
931 `lsst.pipe.base.connections.DeferredDatasetRef` or
932 `lsst.daf.butler.core.datasets.ref.DatasetRef`.
933 Returned lists of refs have the same length as the outputSortKeyOrder.
934 If an outputSortKey not in the inputRef, then it will be padded with None.
935 If an inputRef contains an inputSortKey that is not in the
936 outputSortKeyOrder it will be removed.
937
938 Parameters
939 ----------
940 inputRefs : `lsst.pipe.base.connections.QuantizedConnection`
941 Input references to be reordered and padded.
942 outputSortKeyOrder : iterable
943 Iterable of values to be compared with inputRef's dataId[dataIdKey]
944 dataIdKey : `str`
945 dataIdKey in the dataRefs to compare with the outputSortKeyOrder.
946
947 Returns:
948 --------
949 inputRefs: `lsst.pipe.base.connections.QuantizedConnection`
950 Quantized Connection with sorted DatasetRef values sorted if iterable.
951 """
952 for connectionName, refs in inputRefs:
953 if isinstance(refs, Iterable):
954 if hasattr(refs[0], "dataId"):
955 inputSortKeyOrder = [ref.dataId[dataIdKey] for ref in refs]
956 else:
957 inputSortKeyOrder = [ref.datasetRef.dataId[dataIdKey] for ref in refs]
958 if inputSortKeyOrder != outputSortKeyOrder:
959 setattr(inputRefs, connectionName,
960 reorderAndPadList(refs, inputSortKeyOrder, outputSortKeyOrder))
961 return inputRefs
Configuration parameters for CoaddBaseTask.
Definition: coaddBase.py:38
Base class for coaddition.
Definition: coaddBase.py:141
def getTempExpDatasetName(self, warpType="direct")
Definition: coaddBase.py:204
def selectExposures(self, patchRef, skyInfo=None, selectDataList=[])
Select exposures to coadd.
Definition: coaddBase.py:154
def getCoaddDatasetName(self, warpType="direct")
Definition: coaddBase.py:190
def getSkyInfo(self, patchRef)
Use getSkyinfo to return the skyMap, tract and patch information, wcs and the outer bbox of the patch...
Definition: coaddBase.py:174
def getBadPixelMask(self)
Convenience method to provide the bitmask from the mask plane names.
Definition: coaddBase.py:239
Warp and optionally PSF-Match calexps onto an a common projection.
def getCalibratedExposure(self, dataRef, bgSubtracted)
def run(self, calExpList, ccdIdList, skyInfo, visitId=0, dataIdList=None, **kwargs)
def runDataRef(self, patchRef, selectDataList=[])
Produce <coaddName>Coadd_<warpType>Warp images by warping and optionally PSF-matching.
def reorderAndPadList(inputList, inputKeys, outputKeys, padWith=None)
Definition: coaddBase.py:362
def makeSkyInfo(skyMap, tractId, patchId)
Definition: coaddBase.py:289
def getGroupDataRef(butler, datasetType, groupTuple, keys)
Definition: coaddHelpers.py:99
def groupPatchExposures(patchDataRef, calexpDataRefList, coaddDatasetType="deepCoadd", tempExpDatasetType="deepCoadd_directWarp")
Definition: coaddHelpers.py:60