lsst.pipe.tasks g12b8063833+2c5ef8e9f2
makeCoaddTempExp.py
Go to the documentation of this file.
2# LSST Data Management System
3# Copyright 2008, 2009, 2010, 2011, 2012 LSST Corporation.
4#
5# This product includes software developed by the
6# LSST Project (http://www.lsst.org/).
7#
8# This program is free software: you can redistribute it and/or modify
9# it under the terms of the GNU General Public License as published by
10# the Free Software Foundation, either version 3 of the License, or
11# (at your option) any later version.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the LSST License Statement and
19# the GNU General Public License along with this program. If not,
20# see <http://www.lsstcorp.org/LegalNotices/>.
21#
22import numpy
23import logging
24
25import lsst.pex.config as pexConfig
26import lsst.daf.persistence as dafPersist
27import lsst.afw.image as afwImage
28import lsst.coadd.utils as coaddUtils
29import lsst.pipe.base as pipeBase
30import lsst.pipe.base.connectionTypes as connectionTypes
31import lsst.utils as utils
32import lsst.geom
33from lsst.meas.algorithms import CoaddPsf, CoaddPsfConfig
34from lsst.skymap import BaseSkyMap
35from lsst.utils.timer import timeMethod
36from .coaddBase import CoaddBaseTask, makeSkyInfo, reorderAndPadList
37from .selectImages import PsfWcsSelectImagesTask
38from .warpAndPsfMatch import WarpAndPsfMatchTask
39from .coaddHelpers import groupPatchExposures, getGroupDataRef
40from collections.abc import Iterable
41
42__all__ = ["MakeCoaddTempExpTask", "MakeWarpTask", "MakeWarpConfig"]
43
44log = logging.getLogger(__name__)
45
46
47class MissingExposureError(Exception):
48 """Raised when data cannot be retrieved for an exposure.
49 When processing patches, sometimes one exposure is missing; this lets us
50 distinguish bewteen that case, and other errors.
51 """
52 pass
53
54
55class MakeCoaddTempExpConfig(CoaddBaseTask.ConfigClass):
56 """Config for MakeCoaddTempExpTask
57 """
58 warpAndPsfMatch = pexConfig.ConfigurableField(
59 target=WarpAndPsfMatchTask,
60 doc="Task to warp and PSF-match calexp",
61 )
62 doWrite = pexConfig.Field(
63 doc="persist <coaddName>Coadd_<warpType>Warp",
64 dtype=bool,
65 default=True,
66 )
67 bgSubtracted = pexConfig.Field(
68 doc="Work with a background subtracted calexp?",
69 dtype=bool,
70 default=True,
71 )
72 coaddPsf = pexConfig.ConfigField(
73 doc="Configuration for CoaddPsf",
74 dtype=CoaddPsfConfig,
75 )
76 makeDirect = pexConfig.Field(
77 doc="Make direct Warp/Coadds",
78 dtype=bool,
79 default=True,
80 )
81 makePsfMatched = pexConfig.Field(
82 doc="Make Psf-Matched Warp/Coadd?",
83 dtype=bool,
84 default=False,
85 )
86
87 doWriteEmptyWarps = pexConfig.Field(
88 dtype=bool,
89 default=False,
90 doc="Write out warps even if they are empty"
91 )
92
93 hasFakes = pexConfig.Field(
94 doc="Should be set to True if fake sources have been inserted into the input data.",
95 dtype=bool,
96 default=False,
97 )
98 doApplySkyCorr = pexConfig.Field(dtype=bool, default=False, doc="Apply sky correction?")
99
100 def validate(self):
101 CoaddBaseTask.ConfigClass.validate(self)
102 if not self.makePsfMatchedmakePsfMatched and not self.makeDirectmakeDirect:
103 raise RuntimeError("At least one of config.makePsfMatched and config.makeDirect must be True")
104 if self.doPsfMatch:
105 # Backwards compatibility.
106 log.warning("Config doPsfMatch deprecated. Setting makePsfMatched=True and makeDirect=False")
107 self.makePsfMatchedmakePsfMatched = True
108 self.makeDirectmakeDirect = False
109
110 def setDefaults(self):
111 CoaddBaseTask.ConfigClass.setDefaults(self)
112 self.warpAndPsfMatchwarpAndPsfMatch.psfMatch.kernel.active.kernelSize = self.matchingKernelSize
113 self.select.retarget(PsfWcsSelectImagesTask)
114
115
121
122
124 r"""!Warp and optionally PSF-Match calexps onto an a common projection.
125
126 @anchor MakeCoaddTempExpTask_
127
128 @section pipe_tasks_makeCoaddTempExp_Contents Contents
129
130 - @ref pipe_tasks_makeCoaddTempExp_Purpose
131 - @ref pipe_tasks_makeCoaddTempExp_Initialize
132 - @ref pipe_tasks_makeCoaddTempExp_IO
133 - @ref pipe_tasks_makeCoaddTempExp_Config
134 - @ref pipe_tasks_makeCoaddTempExp_Debug
135 - @ref pipe_tasks_makeCoaddTempExp_Example
136
137 @section pipe_tasks_makeCoaddTempExp_Purpose Description
138
139 Warp and optionally PSF-Match calexps onto a common projection, by
140 performing the following operations:
141 - Group calexps by visit/run
142 - For each visit, generate a Warp by calling method @ref makeTempExp.
143 makeTempExp loops over the visit's calexps calling @ref WarpAndPsfMatch
144 on each visit
145
146 The result is a `directWarp` (and/or optionally a `psfMatchedWarp`).
147
148 @section pipe_tasks_makeCoaddTempExp_Initialize Task Initialization
149
150 @copydoc \_\_init\_\_
151
152 This task has one special keyword argument: passing reuse=True will cause
153 the task to skip the creation of warps that are already present in the
154 output repositories.
155
156 @section pipe_tasks_makeCoaddTempExp_IO Invoking the Task
157
158 This task is primarily designed to be run from the command line.
159
160 The main method is `runDataRef`, which takes a single butler data reference for the patch(es)
161 to process.
162
163 @copydoc run
164
165 WarpType identifies the types of convolutions applied to Warps (previously CoaddTempExps).
166 Only two types are available: direct (for regular Warps/Coadds) and psfMatched
167 (for Warps/Coadds with homogenized PSFs). We expect to add a third type, likelihood,
168 for generating likelihood Coadds with Warps that have been correlated with their own PSF.
169
170 @section pipe_tasks_makeCoaddTempExp_Config Configuration parameters
171
172 See @ref MakeCoaddTempExpConfig and parameters inherited from
173 @link lsst.pipe.tasks.coaddBase.CoaddBaseConfig CoaddBaseConfig @endlink
174
175 @subsection pipe_tasks_MakeCoaddTempExp_psfMatching Guide to PSF-Matching Configs
176
177 To make `psfMatchedWarps`, select `config.makePsfMatched=True`. The subtask
178 @link lsst.ip.diffim.modelPsfMatch.ModelPsfMatchTask ModelPsfMatchTask @endlink
179 is responsible for the PSF-Matching, and its config is accessed via `config.warpAndPsfMatch.psfMatch`.
180 The optimal configuration depends on aspects of dataset: the pixel scale, average PSF FWHM and
181 dimensions of the PSF kernel. These configs include the requested model PSF, the matching kernel size,
182 padding of the science PSF thumbnail and spatial sampling frequency of the PSF.
183
184 *Config Guidelines*: The user must specify the size of the model PSF to which to match by setting
185 `config.modelPsf.defaultFwhm` in units of pixels. The appropriate values depends on science case.
186 In general, for a set of input images, this config should equal the FWHM of the visit
187 with the worst seeing. The smallest it should be set to is the median FWHM. The defaults
188 of the other config options offer a reasonable starting point.
189 The following list presents the most common problems that arise from a misconfigured
190 @link lsst.ip.diffim.modelPsfMatch.ModelPsfMatchTask ModelPsfMatchTask @endlink
191 and corresponding solutions. All assume the default Alard-Lupton kernel, with configs accessed via
192 ```config.warpAndPsfMatch.psfMatch.kernel['AL']```. Each item in the list is formatted as:
193 Problem: Explanation. *Solution*
194
195 *Troublshooting PSF-Matching Configuration:*
196 - Matched PSFs look boxy: The matching kernel is too small. _Increase the matching kernel size.
197 For example:_
198
199 config.warpAndPsfMatch.psfMatch.kernel['AL'].kernelSize=27 # default 21
200
201 Note that increasing the kernel size also increases runtime.
202 - Matched PSFs look ugly (dipoles, quadropoles, donuts): unable to find good solution
203 for matching kernel. _Provide the matcher with more data by either increasing
204 the spatial sampling by decreasing the spatial cell size,_
205
206 config.warpAndPsfMatch.psfMatch.kernel['AL'].sizeCellX = 64 # default 128
207 config.warpAndPsfMatch.psfMatch.kernel['AL'].sizeCellY = 64 # default 128
208
209 _or increasing the padding around the Science PSF, for example:_
210
211 config.warpAndPsfMatch.psfMatch.autoPadPsfTo=1.6 # default 1.4
212
213 Increasing `autoPadPsfTo` increases the minimum ratio of input PSF dimensions to the
214 matching kernel dimensions, thus increasing the number of pixels available to fit
215 after convolving the PSF with the matching kernel.
216 Optionally, for debugging the effects of padding, the level of padding may be manually
217 controlled by setting turning off the automatic padding and setting the number
218 of pixels by which to pad the PSF:
219
220 config.warpAndPsfMatch.psfMatch.doAutoPadPsf = False # default True
221 config.warpAndPsfMatch.psfMatch.padPsfBy = 6 # pixels. default 0
222
223 - Deconvolution: Matching a large PSF to a smaller PSF produces
224 a telltale noise pattern which looks like ripples or a brain.
225 _Increase the size of the requested model PSF. For example:_
226
227 config.modelPsf.defaultFwhm = 11 # Gaussian sigma in units of pixels.
228
229 - High frequency (sometimes checkered) noise: The matching basis functions are too small.
230 _Increase the width of the Gaussian basis functions. For example:_
231
232 config.warpAndPsfMatch.psfMatch.kernel['AL'].alardSigGauss=[1.5, 3.0, 6.0]
233 # from default [0.7, 1.5, 3.0]
234
235
236 @section pipe_tasks_makeCoaddTempExp_Debug Debug variables
237
238 MakeCoaddTempExpTask has no debug output, but its subtasks do.
239
240 @section pipe_tasks_makeCoaddTempExp_Example A complete example of using MakeCoaddTempExpTask
241
242 This example uses the package ci_hsc to show how MakeCoaddTempExp fits
243 into the larger Data Release Processing.
244 Set up by running:
245
246 setup ci_hsc
247 cd $CI_HSC_DIR
248 # if not built already:
249 python $(which scons) # this will take a while
250
251 The following assumes that `processCcd.py` and `makeSkyMap.py` have previously been run
252 (e.g. by building `ci_hsc` above) to generate a repository of calexps and an
253 output respository with the desired SkyMap. The command,
254
255 makeCoaddTempExp.py $CI_HSC_DIR/DATA --rerun ci_hsc \
256 --id patch=5,4 tract=0 filter=HSC-I \
257 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 \
258 --selectId visit=903988 ccd=23 --selectId visit=903988 ccd=24 \
259 --config doApplyExternalPhotoCalib=False doApplyExternalSkyWcs=False \
260 makePsfMatched=True modelPsf.defaultFwhm=11
261
262 writes a direct and PSF-Matched Warp to
263 - `$CI_HSC_DIR/DATA/rerun/ci_hsc/deepCoadd/HSC-I/0/5,4/warp-HSC-I-0-5,4-903988.fits` and
264 - `$CI_HSC_DIR/DATA/rerun/ci_hsc/deepCoadd/HSC-I/0/5,4/psfMatchedWarp-HSC-I-0-5,4-903988.fits`
265 respectively.
266
267 @note PSF-Matching in this particular dataset would benefit from adding
268 `--configfile ./matchingConfig.py` to
269 the command line arguments where `matchingConfig.py` is defined by:
270
271 echo "
272 config.warpAndPsfMatch.psfMatch.kernel['AL'].kernelSize=27
273 config.warpAndPsfMatch.psfMatch.kernel['AL'].alardSigGauss=[1.5, 3.0, 6.0]" > matchingConfig.py
274
275
276 Add the option `--help` to see more options.
277 """
278 ConfigClass = MakeCoaddTempExpConfig
279 _DefaultName = "makeCoaddTempExp"
280
281 def __init__(self, reuse=False, **kwargs):
282 CoaddBaseTask.__init__(self, **kwargs)
283 self.reusereuse = reuse
284 self.makeSubtask("warpAndPsfMatch")
285 if self.config.hasFakes:
286 self.calexpTypecalexpType = "fakes_calexp"
287 else:
288 self.calexpTypecalexpType = "calexp"
289
290 @timeMethod
291 def runDataRef(self, patchRef, selectDataList=[]):
292 """!Produce <coaddName>Coadd_<warpType>Warp images by warping and optionally PSF-matching.
293
294 @param[in] patchRef: data reference for sky map patch. Must include keys "tract", "patch",
295 plus the camera-specific filter key (e.g. "filter" or "band")
296 @return: dataRefList: a list of data references for the new <coaddName>Coadd_directWarps
297 if direct or both warp types are requested and <coaddName>Coadd_psfMatchedWarps if only psfMatched
298 warps are requested.
299
300 @warning: this task assumes that all exposures in a warp (coaddTempExp) have the same filter.
301
302 @warning: this task sets the PhotoCalib of the coaddTempExp to the PhotoCalib of the first calexp
303 with any good pixels in the patch. For a mosaic camera the resulting PhotoCalib should be ignored
304 (assembleCoadd should determine zeropoint scaling without referring to it).
305 """
306 skyInfo = self.getSkyInfogetSkyInfo(patchRef)
307
308 # DataRefs to return are of type *_directWarp unless only *_psfMatchedWarp requested
309 if self.config.makePsfMatched and not self.config.makeDirect:
310 primaryWarpDataset = self.getTempExpDatasetNamegetTempExpDatasetName("psfMatched")
311 else:
312 primaryWarpDataset = self.getTempExpDatasetNamegetTempExpDatasetName("direct")
313
314 calExpRefList = self.selectExposuresselectExposures(patchRef, skyInfo, selectDataList=selectDataList)
315
316 if len(calExpRefList) == 0:
317 self.log.warning("No exposures to coadd for patch %s", patchRef.dataId)
318 return None
319 self.log.info("Selected %d calexps for patch %s", len(calExpRefList), patchRef.dataId)
320 calExpRefList = [calExpRef for calExpRef in calExpRefList if calExpRef.datasetExists(self.calexpTypecalexpType)]
321 self.log.info("Processing %d existing calexps for patch %s", len(calExpRefList), patchRef.dataId)
322
323 groupData = groupPatchExposures(patchRef, calExpRefList, self.getCoaddDatasetNamegetCoaddDatasetName(),
324 primaryWarpDataset)
325 self.log.info("Processing %d warp exposures for patch %s", len(groupData.groups), patchRef.dataId)
326
327 dataRefList = []
328 for i, (tempExpTuple, calexpRefList) in enumerate(groupData.groups.items()):
329 tempExpRef = getGroupDataRef(patchRef.getButler(), primaryWarpDataset,
330 tempExpTuple, groupData.keys)
331 if self.reusereuse and tempExpRef.datasetExists(datasetType=primaryWarpDataset, write=True):
332 self.log.info("Skipping makeCoaddTempExp for %s; output already exists.", tempExpRef.dataId)
333 dataRefList.append(tempExpRef)
334 continue
335 self.log.info("Processing Warp %d/%d: id=%s", i, len(groupData.groups), tempExpRef.dataId)
336
337 # TODO: mappers should define a way to go from the "grouping keys" to a numeric ID (#2776).
338 # For now, we try to get a long integer "visit" key, and if we can't, we just use the index
339 # of the visit in the list.
340 try:
341 visitId = int(tempExpRef.dataId["visit"])
342 except (KeyError, ValueError):
343 visitId = i
344
345 calExpList = []
346 ccdIdList = []
347 dataIdList = []
348
349 for calExpInd, calExpRef in enumerate(calexpRefList):
350 self.log.info("Reading calexp %s of %s for Warp id=%s", calExpInd+1, len(calexpRefList),
351 calExpRef.dataId)
352 try:
353 ccdId = calExpRef.get("ccdExposureId", immediate=True)
354 except Exception:
355 ccdId = calExpInd
356 try:
357 # We augment the dataRef here with the tract, which is harmless for loading things
358 # like calexps that don't need the tract, and necessary for meas_mosaic outputs,
359 # which do.
360 calExpRef = calExpRef.butlerSubset.butler.dataRef(self.calexpTypecalexpType,
361 dataId=calExpRef.dataId,
362 tract=skyInfo.tractInfo.getId())
363 calExp = self.getCalibratedExposuregetCalibratedExposure(calExpRef, bgSubtracted=self.config.bgSubtracted)
364 except Exception as e:
365 self.log.warning("Calexp %s not found; skipping it: %s", calExpRef.dataId, e)
366 continue
367
368 if self.config.doApplySkyCorr:
369 self.applySkyCorrapplySkyCorr(calExpRef, calExp)
370
371 calExpList.append(calExp)
372 ccdIdList.append(ccdId)
373 dataIdList.append(calExpRef.dataId)
374
375 exps = self.runrun(calExpList, ccdIdList, skyInfo, visitId, dataIdList).exposures
376
377 if any(exps.values()):
378 dataRefList.append(tempExpRef)
379 else:
380 self.log.warning("Warp %s could not be created", tempExpRef.dataId)
381
382 if self.config.doWrite:
383 for (warpType, exposure) in exps.items(): # compatible w/ Py3
384 if exposure is not None:
385 self.log.info("Persisting %s", self.getTempExpDatasetNamegetTempExpDatasetName(warpType))
386 tempExpRef.put(exposure, self.getTempExpDatasetNamegetTempExpDatasetName(warpType))
387
388 return dataRefList
389
390 @timeMethod
391 def run(self, calExpList, ccdIdList, skyInfo, visitId=0, dataIdList=None, **kwargs):
392 """Create a Warp from inputs
393
394 We iterate over the multiple calexps in a single exposure to construct
395 the warp (previously called a coaddTempExp) of that exposure to the
396 supplied tract/patch.
397
398 Pixels that receive no pixels are set to NAN; this is not correct
399 (violates LSST algorithms group policy), but will be fixed up by
400 interpolating after the coaddition.
401
402 @param calexpRefList: List of data references for calexps that (may)
403 overlap the patch of interest
404 @param skyInfo: Struct from CoaddBaseTask.getSkyInfo() with geometric
405 information about the patch
406 @param visitId: integer identifier for visit, for the table that will
407 produce the CoaddPsf
408 @return a pipeBase Struct containing:
409 - exposures: a dictionary containing the warps requested:
410 "direct": direct warp if config.makeDirect
411 "psfMatched": PSF-matched warp if config.makePsfMatched
412 """
413 warpTypeList = self.getWarpTypeListgetWarpTypeList()
414
415 totGoodPix = {warpType: 0 for warpType in warpTypeList}
416 didSetMetadata = {warpType: False for warpType in warpTypeList}
417 coaddTempExps = {warpType: self._prepareEmptyExposure_prepareEmptyExposure(skyInfo) for warpType in warpTypeList}
418 inputRecorder = {warpType: self.inputRecorder.makeCoaddTempExpRecorder(visitId, len(calExpList))
419 for warpType in warpTypeList}
420
421 modelPsf = self.config.modelPsf.apply() if self.config.makePsfMatched else None
422 if dataIdList is None:
423 dataIdList = ccdIdList
424
425 for calExpInd, (calExp, ccdId, dataId) in enumerate(zip(calExpList, ccdIdList, dataIdList)):
426 self.log.info("Processing calexp %d of %d for this Warp: id=%s",
427 calExpInd+1, len(calExpList), dataId)
428
429 try:
430 warpedAndMatched = self.warpAndPsfMatch.run(calExp, modelPsf=modelPsf,
431 wcs=skyInfo.wcs, maxBBox=skyInfo.bbox,
432 makeDirect=self.config.makeDirect,
433 makePsfMatched=self.config.makePsfMatched)
434 except Exception as e:
435 self.log.warning("WarpAndPsfMatch failed for calexp %s; skipping it: %s", dataId, e)
436 continue
437 try:
438 numGoodPix = {warpType: 0 for warpType in warpTypeList}
439 for warpType in warpTypeList:
440 exposure = warpedAndMatched.getDict()[warpType]
441 if exposure is None:
442 continue
443 coaddTempExp = coaddTempExps[warpType]
444 if didSetMetadata[warpType]:
445 mimg = exposure.getMaskedImage()
446 mimg *= (coaddTempExp.getPhotoCalib().getInstFluxAtZeroMagnitude()
447 / exposure.getPhotoCalib().getInstFluxAtZeroMagnitude())
448 del mimg
449 numGoodPix[warpType] = coaddUtils.copyGoodPixels(
450 coaddTempExp.getMaskedImage(), exposure.getMaskedImage(), self.getBadPixelMaskgetBadPixelMask())
451 totGoodPix[warpType] += numGoodPix[warpType]
452 self.log.debug("Calexp %s has %d good pixels in this patch (%.1f%%) for %s",
453 dataId, numGoodPix[warpType],
454 100.0*numGoodPix[warpType]/skyInfo.bbox.getArea(), warpType)
455 if numGoodPix[warpType] > 0 and not didSetMetadata[warpType]:
456 coaddTempExp.info.id = exposure.info.id
457 coaddTempExp.setPhotoCalib(exposure.getPhotoCalib())
458 coaddTempExp.setFilterLabel(exposure.getFilterLabel())
459 coaddTempExp.getInfo().setVisitInfo(exposure.getInfo().getVisitInfo())
460 # PSF replaced with CoaddPsf after loop if and only if creating direct warp
461 coaddTempExp.setPsf(exposure.getPsf())
462 didSetMetadata[warpType] = True
463
464 # Need inputRecorder for CoaddApCorrMap for both direct and PSF-matched
465 inputRecorder[warpType].addCalExp(calExp, ccdId, numGoodPix[warpType])
466
467 except Exception as e:
468 self.log.warning("Error processing calexp %s; skipping it: %s", dataId, e)
469 continue
470
471 for warpType in warpTypeList:
472 self.log.info("%sWarp has %d good pixels (%.1f%%)",
473 warpType, totGoodPix[warpType], 100.0*totGoodPix[warpType]/skyInfo.bbox.getArea())
474
475 if totGoodPix[warpType] > 0 and didSetMetadata[warpType]:
476 inputRecorder[warpType].finish(coaddTempExps[warpType], totGoodPix[warpType])
477 if warpType == "direct":
478 coaddTempExps[warpType].setPsf(
479 CoaddPsf(inputRecorder[warpType].coaddInputs.ccds, skyInfo.wcs,
480 self.config.coaddPsf.makeControl()))
481 else:
482 if not self.config.doWriteEmptyWarps:
483 # No good pixels. Exposure still empty
484 coaddTempExps[warpType] = None
485 # NoWorkFound is unnecessary as the downstream tasks will
486 # adjust the quantum accordingly, and it prevents gen2
487 # MakeCoaddTempExp from continuing to loop over visits.
488
489 result = pipeBase.Struct(exposures=coaddTempExps)
490 return result
491
492 def getCalibratedExposure(self, dataRef, bgSubtracted):
493 """Return one calibrated Exposure, possibly with an updated SkyWcs.
494
495 @param[in] dataRef a sensor-level data reference
496 @param[in] bgSubtracted return calexp with background subtracted? If False get the
497 calexp's background background model and add it to the calexp.
498 @return calibrated exposure
499
500 @raises MissingExposureError If data for the exposure is not available.
501
502 If config.doApplyExternalPhotoCalib is `True`, the photometric calibration
503 (`photoCalib`) is taken from `config.externalPhotoCalibName` via the
504 `name_photoCalib` dataset. Otherwise, the photometric calibration is
505 retrieved from the processed exposure. When
506 `config.doApplyExternalSkyWcs` is `True`, the astrometric calibration
507 is taken from `config.externalSkyWcsName` with the `name_wcs` dataset.
508 Otherwise, the astrometric calibration is taken from the processed
509 exposure.
510 """
511 try:
512 exposure = dataRef.get(self.calexpTypecalexpType, immediate=True)
513 except dafPersist.NoResults as e:
514 raise MissingExposureError('Exposure not found: %s ' % str(e)) from e
515
516 if not bgSubtracted:
517 background = dataRef.get("calexpBackground", immediate=True)
518 mi = exposure.getMaskedImage()
519 mi += background.getImage()
520 del mi
521
522 if self.config.doApplyExternalPhotoCalib:
523 source = f"{self.config.externalPhotoCalibName}_photoCalib"
524 self.log.debug("Applying external photoCalib to %s from %s", dataRef.dataId, source)
525 photoCalib = dataRef.get(source)
526 exposure.setPhotoCalib(photoCalib)
527 else:
528 photoCalib = exposure.getPhotoCalib()
529
530 if self.config.doApplyExternalSkyWcs:
531 source = f"{self.config.externalSkyWcsName}_wcs"
532 self.log.debug("Applying external skyWcs to %s from %s", dataRef.dataId, source)
533 skyWcs = dataRef.get(source)
534 exposure.setWcs(skyWcs)
535
536 exposure.maskedImage = photoCalib.calibrateImage(exposure.maskedImage,
537 includeScaleUncertainty=self.config.includeCalibVar)
538 exposure.maskedImage /= photoCalib.getCalibrationMean()
539 # TODO: The images will have a calibration of 1.0 everywhere once RFC-545 is implemented.
540 # exposure.setCalib(afwImage.Calib(1.0))
541 return exposure
542
543 @staticmethod
544 def _prepareEmptyExposure(skyInfo):
545 """Produce an empty exposure for a given patch"""
546 exp = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
547 exp.getMaskedImage().set(numpy.nan, afwImage.Mask
548 .getPlaneBitMask("NO_DATA"), numpy.inf)
549 return exp
550
552 """Return list of requested warp types per the config.
553 """
554 warpTypeList = []
555 if self.config.makeDirect:
556 warpTypeList.append("direct")
557 if self.config.makePsfMatched:
558 warpTypeList.append("psfMatched")
559 return warpTypeList
560
561 def applySkyCorr(self, dataRef, calexp):
562 """Apply correction to the sky background level
563
564 Sky corrections can be generated with the 'skyCorrection.py'
565 executable in pipe_drivers. Because the sky model used by that
566 code extends over the entire focal plane, this can produce
567 better sky subtraction.
568
569 The calexp is updated in-place.
570
571 Parameters
572 ----------
573 dataRef : `lsst.daf.persistence.ButlerDataRef`
574 Data reference for calexp.
576 Calibrated exposure.
577 """
578 bg = dataRef.get("skyCorr")
579 self.log.debug("Applying sky correction to %s", dataRef.dataId)
580 if isinstance(calexp, afwImage.Exposure):
581 calexp = calexp.getMaskedImage()
582 calexp -= bg.getImage()
583
584
585class MakeWarpConnections(pipeBase.PipelineTaskConnections,
586 dimensions=("tract", "patch", "skymap", "instrument", "visit"),
587 defaultTemplates={"coaddName": "deep",
588 "skyWcsName": "jointcal",
589 "photoCalibName": "fgcm",
590 "calexpType": ""}):
591 calExpList = connectionTypes.Input(
592 doc="Input exposures to be resampled and optionally PSF-matched onto a SkyMap projection/patch",
593 name="{calexpType}calexp",
594 storageClass="ExposureF",
595 dimensions=("instrument", "visit", "detector"),
596 multiple=True,
597 deferLoad=True,
598 )
599 backgroundList = connectionTypes.Input(
600 doc="Input backgrounds to be added back into the calexp if bgSubtracted=False",
601 name="calexpBackground",
602 storageClass="Background",
603 dimensions=("instrument", "visit", "detector"),
604 multiple=True,
605 )
606 skyCorrList = connectionTypes.Input(
607 doc="Input Sky Correction to be subtracted from the calexp if doApplySkyCorr=True",
608 name="skyCorr",
609 storageClass="Background",
610 dimensions=("instrument", "visit", "detector"),
611 multiple=True,
612 )
613 skyMap = connectionTypes.Input(
614 doc="Input definition of geometry/bbox and projection/wcs for warped exposures",
615 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
616 storageClass="SkyMap",
617 dimensions=("skymap",),
618 )
619 externalSkyWcsTractCatalog = connectionTypes.Input(
620 doc=("Per-tract, per-visit wcs calibrations. These catalogs use the detector "
621 "id for the catalog id, sorted on id for fast lookup."),
622 name="{skyWcsName}SkyWcsCatalog",
623 storageClass="ExposureCatalog",
624 dimensions=("instrument", "visit", "tract"),
625 )
626 externalSkyWcsGlobalCatalog = connectionTypes.Input(
627 doc=("Per-visit wcs calibrations computed globally (with no tract information). "
628 "These catalogs use the detector id for the catalog id, sorted on id for "
629 "fast lookup."),
630 name="{skyWcsName}SkyWcsCatalog",
631 storageClass="ExposureCatalog",
632 dimensions=("instrument", "visit"),
633 )
634 externalPhotoCalibTractCatalog = connectionTypes.Input(
635 doc=("Per-tract, per-visit photometric calibrations. These catalogs use the "
636 "detector id for the catalog id, sorted on id for fast lookup."),
637 name="{photoCalibName}PhotoCalibCatalog",
638 storageClass="ExposureCatalog",
639 dimensions=("instrument", "visit", "tract"),
640 )
641 externalPhotoCalibGlobalCatalog = connectionTypes.Input(
642 doc=("Per-visit photometric calibrations computed globally (with no tract "
643 "information). These catalogs use the detector id for the catalog id, "
644 "sorted on id for fast lookup."),
645 name="{photoCalibName}PhotoCalibCatalog",
646 storageClass="ExposureCatalog",
647 dimensions=("instrument", "visit"),
648 )
649 direct = connectionTypes.Output(
650 doc=("Output direct warped exposure (previously called CoaddTempExp), produced by resampling ",
651 "calexps onto the skyMap patch geometry."),
652 name="{coaddName}Coadd_directWarp",
653 storageClass="ExposureF",
654 dimensions=("tract", "patch", "skymap", "visit", "instrument"),
655 )
656 psfMatched = connectionTypes.Output(
657 doc=("Output PSF-Matched warped exposure (previously called CoaddTempExp), produced by resampling ",
658 "calexps onto the skyMap patch geometry and PSF-matching to a model PSF."),
659 name="{coaddName}Coadd_psfMatchedWarp",
660 storageClass="ExposureF",
661 dimensions=("tract", "patch", "skymap", "visit", "instrument"),
662 )
663 # TODO DM-28769, have selectImages subtask indicate which connections they need:
664 wcsList = connectionTypes.Input(
665 doc="WCSs of calexps used by SelectImages subtask to determine if the calexp overlaps the patch",
666 name="{calexpType}calexp.wcs",
667 storageClass="Wcs",
668 dimensions=("instrument", "visit", "detector"),
669 multiple=True,
670 )
671 bboxList = connectionTypes.Input(
672 doc="BBoxes of calexps used by SelectImages subtask to determine if the calexp overlaps the patch",
673 name="{calexpType}calexp.bbox",
674 storageClass="Box2I",
675 dimensions=("instrument", "visit", "detector"),
676 multiple=True,
677 )
678 visitSummary = connectionTypes.Input(
679 doc="Consolidated exposure metadata from ConsolidateVisitSummaryTask",
680 name="{calexpType}visitSummary",
681 storageClass="ExposureCatalog",
682 dimensions=("instrument", "visit",),
683 )
684 srcList = connectionTypes.Input(
685 doc="Source catalogs used by PsfWcsSelectImages subtask to further select on PSF stability",
686 name="src",
687 storageClass="SourceCatalog",
688 dimensions=("instrument", "visit", "detector"),
689 multiple=True,
690 )
691
692 def __init__(self, *, config=None):
693 super().__init__(config=config)
694 if config.bgSubtracted:
695 self.inputs.remove("backgroundList")
696 if not config.doApplySkyCorr:
697 self.inputs.remove("skyCorrList")
698 if config.doApplyExternalSkyWcs:
699 if config.useGlobalExternalSkyWcs:
700 self.inputs.remove("externalSkyWcsTractCatalog")
701 else:
702 self.inputs.remove("externalSkyWcsGlobalCatalog")
703 else:
704 self.inputs.remove("externalSkyWcsTractCatalog")
705 self.inputs.remove("externalSkyWcsGlobalCatalog")
706 if config.doApplyExternalPhotoCalib:
707 if config.useGlobalExternalPhotoCalib:
708 self.inputs.remove("externalPhotoCalibTractCatalog")
709 else:
710 self.inputs.remove("externalPhotoCalibGlobalCatalog")
711 else:
712 self.inputs.remove("externalPhotoCalibTractCatalog")
713 self.inputs.remove("externalPhotoCalibGlobalCatalog")
714 if not config.makeDirect:
715 self.outputs.remove("direct")
716 if not config.makePsfMatched:
717 self.outputs.remove("psfMatched")
718 # TODO DM-28769: add connection per selectImages connections
719 if config.select.target != lsst.pipe.tasks.selectImages.PsfWcsSelectImagesTask:
720 self.inputs.remove("visitSummary")
721 self.inputs.remove("srcList")
722 elif not config.select.doLegacyStarSelectionComputation:
723 # Remove backwards-compatibility connections.
724 self.inputs.remove("srcList")
725
726
727class MakeWarpConfig(pipeBase.PipelineTaskConfig, MakeCoaddTempExpConfig,
728 pipelineConnections=MakeWarpConnections):
729
730 def validate(self):
731 super().validate()
732
733
734class MakeWarpTask(MakeCoaddTempExpTask):
735 """Warp and optionally PSF-Match calexps onto an a common projection
736 """
737 ConfigClass = MakeWarpConfig
738 _DefaultName = "makeWarp"
739
740 @utils.inheritDoc(pipeBase.PipelineTask)
741 def runQuantum(self, butlerQC, inputRefs, outputRefs):
742 """
743 Notes
744 ----
745 Construct warps for requested warp type for single epoch
746
747 PipelineTask (Gen3) entry point to warp and optionally PSF-match
748 calexps. This method is analogous to `runDataRef`.
749 """
750 # Obtain the list of input detectors from calExpList. Sort them by
751 # detector order (to ensure reproducibility). Then ensure all input
752 # lists are in the same sorted detector order.
753 detectorOrder = [ref.datasetRef.dataId['detector'] for ref in inputRefs.calExpList]
754 detectorOrder.sort()
755 inputRefs = reorderRefs(inputRefs, detectorOrder, dataIdKey='detector')
756
757 # Read in all inputs.
758 inputs = butlerQC.get(inputRefs)
759
760 # Construct skyInfo expected by `run`. We remove the SkyMap itself
761 # from the dictionary so we can pass it as kwargs later.
762 skyMap = inputs.pop("skyMap")
763 quantumDataId = butlerQC.quantum.dataId
764 skyInfo = makeSkyInfo(skyMap, tractId=quantumDataId['tract'], patchId=quantumDataId['patch'])
765
766 # Construct list of input DataIds expected by `run`
767 dataIdList = [ref.datasetRef.dataId for ref in inputRefs.calExpList]
768 # Construct list of packed integer IDs expected by `run`
769 ccdIdList = [dataId.pack("visit_detector") for dataId in dataIdList]
770
771 # Run the selector and filter out calexps that were not selected
772 # primarily because they do not overlap the patch
773 cornerPosList = lsst.geom.Box2D(skyInfo.bbox).getCorners()
774 coordList = [skyInfo.wcs.pixelToSky(pos) for pos in cornerPosList]
775 goodIndices = self.select.run(**inputs, coordList=coordList, dataIds=dataIdList)
776 inputs = self.filterInputs(indices=goodIndices, inputs=inputs)
777
778 # Read from disk only the selected calexps
779 inputs['calExpList'] = [ref.get() for ref in inputs['calExpList']]
780
781 # Extract integer visitId requested by `run`
782 visits = [dataId['visit'] for dataId in dataIdList]
783 visitId = visits[0]
784
785 if self.config.doApplyExternalSkyWcs:
786 if self.config.useGlobalExternalSkyWcs:
787 externalSkyWcsCatalog = inputs.pop("externalSkyWcsGlobalCatalog")
788 else:
789 externalSkyWcsCatalog = inputs.pop("externalSkyWcsTractCatalog")
790 else:
791 externalSkyWcsCatalog = None
792
793 if self.config.doApplyExternalPhotoCalib:
794 if self.config.useGlobalExternalPhotoCalib:
795 externalPhotoCalibCatalog = inputs.pop("externalPhotoCalibGlobalCatalog")
796 else:
797 externalPhotoCalibCatalog = inputs.pop("externalPhotoCalibTractCatalog")
798 else:
799 externalPhotoCalibCatalog = None
800
801 completeIndices = self.prepareCalibratedExposures(**inputs,
802 externalSkyWcsCatalog=externalSkyWcsCatalog,
803 externalPhotoCalibCatalog=externalPhotoCalibCatalog)
804 # Redo the input selection with inputs with complete wcs/photocalib info.
805 inputs = self.filterInputs(indices=completeIndices, inputs=inputs)
806
807 results = self.run(**inputs, visitId=visitId,
808 ccdIdList=[ccdIdList[i] for i in goodIndices],
809 dataIdList=[dataIdList[i] for i in goodIndices],
810 skyInfo=skyInfo)
811 if self.config.makeDirect and results.exposures["direct"] is not None:
812 butlerQC.put(results.exposures["direct"], outputRefs.direct)
813 if self.config.makePsfMatched and results.exposures["psfMatched"] is not None:
814 butlerQC.put(results.exposures["psfMatched"], outputRefs.psfMatched)
815
816 def filterInputs(self, indices, inputs):
817 """Return task inputs with their lists filtered by indices
818
819 Parameters
820 ----------
821 indices : `list` of integers
822 inputs : `dict` of `list` of input connections to be passed to run
823 """
824 for key in inputs.keys():
825 # Only down-select on list inputs
826 if isinstance(inputs[key], list):
827 inputs[key] = [inputs[key][ind] for ind in indices]
828 return inputs
829
830 def prepareCalibratedExposures(self, calExpList, backgroundList=None, skyCorrList=None,
831 externalSkyWcsCatalog=None, externalPhotoCalibCatalog=None,
832 **kwargs):
833 """Calibrate and add backgrounds to input calExpList in place
834
835 Parameters
836 ----------
837 calExpList : `list` of `lsst.afw.image.Exposure`
838 Sequence of calexps to be modified in place
839 backgroundList : `list` of `lsst.afw.math.backgroundList`, optional
840 Sequence of backgrounds to be added back in if bgSubtracted=False
841 skyCorrList : `list` of `lsst.afw.math.backgroundList`, optional
842 Sequence of background corrections to be subtracted if doApplySkyCorr=True
843 externalSkyWcsCatalog : `lsst.afw.table.ExposureCatalog`, optional
844 Exposure catalog with external skyWcs to be applied
845 if config.doApplyExternalSkyWcs=True. Catalog uses the detector id
846 for the catalog id, sorted on id for fast lookup.
847 externalPhotoCalibCatalog : `lsst.afw.table.ExposureCatalog`, optional
848 Exposure catalog with external photoCalib to be applied
849 if config.doApplyExternalPhotoCalib=True. Catalog uses the detector
850 id for the catalog id, sorted on id for fast lookup.
851
852 Returns
853 -------
854 indices : `list` [`int`]
855 Indices of calExpList and friends that have valid photoCalib/skyWcs
856 """
857 backgroundList = len(calExpList)*[None] if backgroundList is None else backgroundList
858 skyCorrList = len(calExpList)*[None] if skyCorrList is None else skyCorrList
859
860 includeCalibVar = self.config.includeCalibVar
861
862 indices = []
863 for index, (calexp, background, skyCorr) in enumerate(zip(calExpList,
864 backgroundList,
865 skyCorrList)):
866 if not self.config.bgSubtracted:
867 calexp.maskedImage += background.getImage()
868
869 if externalSkyWcsCatalog is not None or externalPhotoCalibCatalog is not None:
870 detectorId = calexp.getInfo().getDetector().getId()
871
872 # Find the external photoCalib
873 if externalPhotoCalibCatalog is not None:
874 row = externalPhotoCalibCatalog.find(detectorId)
875 if row is None:
876 self.log.warning("Detector id %s not found in externalPhotoCalibCatalog "
877 "and will not be used in the warp.", detectorId)
878 continue
879 photoCalib = row.getPhotoCalib()
880 if photoCalib is None:
881 self.log.warning("Detector id %s has None for photoCalib in externalPhotoCalibCatalog "
882 "and will not be used in the warp.", detectorId)
883 continue
884 calexp.setPhotoCalib(photoCalib)
885 else:
886 photoCalib = calexp.getPhotoCalib()
887 if photoCalib is None:
888 self.log.warning("Detector id %s has None for photoCalib in the calexp "
889 "and will not be used in the warp.", detectorId)
890 continue
891
892 # Find and apply external skyWcs
893 if externalSkyWcsCatalog is not None:
894 row = externalSkyWcsCatalog.find(detectorId)
895 if row is None:
896 self.log.warning("Detector id %s not found in externalSkyWcsCatalog "
897 "and will not be used in the warp.", detectorId)
898 continue
899 skyWcs = row.getWcs()
900 if skyWcs is None:
901 self.log.warning("Detector id %s has None for skyWcs in externalSkyWcsCatalog "
902 "and will not be used in the warp.", detectorId)
903 continue
904 calexp.setWcs(skyWcs)
905 else:
906 skyWcs = calexp.getWcs()
907 if skyWcs is None:
908 self.log.warning("Detector id %s has None for skyWcs in the calexp "
909 "and will not be used in the warp.", detectorId)
910 continue
911
912 # Calibrate the image
913 calexp.maskedImage = photoCalib.calibrateImage(calexp.maskedImage,
914 includeScaleUncertainty=includeCalibVar)
915 calexp.maskedImage /= photoCalib.getCalibrationMean()
916 # TODO: The images will have a calibration of 1.0 everywhere once RFC-545 is implemented.
917 # exposure.setCalib(afwImage.Calib(1.0))
918
919 # Apply skycorr
920 if self.config.doApplySkyCorr:
921 calexp.maskedImage -= skyCorr.getImage()
922
923 indices.append(index)
924
925 return indices
926
927
928def reorderRefs(inputRefs, outputSortKeyOrder, dataIdKey):
929 """Reorder inputRefs per outputSortKeyOrder
930
931 Any inputRefs which are lists will be resorted per specified key e.g.,
932 'detector.' Only iterables will be reordered, and values can be of type
933 `lsst.pipe.base.connections.DeferredDatasetRef` or
934 `lsst.daf.butler.core.datasets.ref.DatasetRef`.
935 Returned lists of refs have the same length as the outputSortKeyOrder.
936 If an outputSortKey not in the inputRef, then it will be padded with None.
937 If an inputRef contains an inputSortKey that is not in the
938 outputSortKeyOrder it will be removed.
939
940 Parameters
941 ----------
942 inputRefs : `lsst.pipe.base.connections.QuantizedConnection`
943 Input references to be reordered and padded.
944 outputSortKeyOrder : iterable
945 Iterable of values to be compared with inputRef's dataId[dataIdKey]
946 dataIdKey : `str`
947 dataIdKey in the dataRefs to compare with the outputSortKeyOrder.
948
949 Returns:
950 --------
951 inputRefs: `lsst.pipe.base.connections.QuantizedConnection`
952 Quantized Connection with sorted DatasetRef values sorted if iterable.
953 """
954 for connectionName, refs in inputRefs:
955 if isinstance(refs, Iterable):
956 if hasattr(refs[0], "dataId"):
957 inputSortKeyOrder = [ref.dataId[dataIdKey] for ref in refs]
958 else:
959 inputSortKeyOrder = [ref.datasetRef.dataId[dataIdKey] for ref in refs]
960 if inputSortKeyOrder != outputSortKeyOrder:
961 setattr(inputRefs, connectionName,
962 reorderAndPadList(refs, inputSortKeyOrder, outputSortKeyOrder))
963 return inputRefs
Configuration parameters for CoaddBaseTask.
Definition: coaddBase.py:38
Base class for coaddition.
Definition: coaddBase.py:141
def getTempExpDatasetName(self, warpType="direct")
Definition: coaddBase.py:204
def selectExposures(self, patchRef, skyInfo=None, selectDataList=[])
Select exposures to coadd.
Definition: coaddBase.py:154
def getCoaddDatasetName(self, warpType="direct")
Definition: coaddBase.py:190
def getSkyInfo(self, patchRef)
Use getSkyinfo to return the skyMap, tract and patch information, wcs and the outer bbox of the patch...
Definition: coaddBase.py:174
def getBadPixelMask(self)
Convenience method to provide the bitmask from the mask plane names.
Definition: coaddBase.py:239
Warp and optionally PSF-Match calexps onto an a common projection.
def getCalibratedExposure(self, dataRef, bgSubtracted)
def run(self, calExpList, ccdIdList, skyInfo, visitId=0, dataIdList=None, **kwargs)
def runDataRef(self, patchRef, selectDataList=[])
Produce <coaddName>Coadd_<warpType>Warp images by warping and optionally PSF-matching.
def reorderAndPadList(inputList, inputKeys, outputKeys, padWith=None)
Definition: coaddBase.py:362
def makeSkyInfo(skyMap, tractId, patchId)
Definition: coaddBase.py:289
def getGroupDataRef(butler, datasetType, groupTuple, keys)
Definition: coaddHelpers.py:99
def groupPatchExposures(patchDataRef, calexpDataRefList, coaddDatasetType="deepCoadd", tempExpDatasetType="deepCoadd_directWarp")
Definition: coaddHelpers.py:60