Coverage for python/lsst/pipe/tasks/makeCoaddTempExp.py : 65%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1#
2# LSST Data Management System
3# Copyright 2008, 2009, 2010, 2011, 2012 LSST Corporation.
4#
5# This product includes software developed by the
6# LSST Project (http://www.lsst.org/).
7#
8# This program is free software: you can redistribute it and/or modify
9# it under the terms of the GNU General Public License as published by
10# the Free Software Foundation, either version 3 of the License, or
11# (at your option) any later version.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the LSST License Statement and
19# the GNU General Public License along with this program. If not,
20# see <http://www.lsstcorp.org/LegalNotices/>.
21#
22import numpy
24import lsst.pex.config as pexConfig
25import lsst.daf.persistence as dafPersist
26import lsst.afw.image as afwImage
27import lsst.coadd.utils as coaddUtils
28import lsst.pipe.base as pipeBase
29import lsst.pipe.base.connectionTypes as cT
30import lsst.log as log
31import lsst.utils as utils
32from lsst.meas.algorithms import CoaddPsf, CoaddPsfConfig
33from .coaddBase import CoaddBaseTask, makeSkyInfo
34from .warpAndPsfMatch import WarpAndPsfMatchTask
35from .coaddHelpers import groupPatchExposures, getGroupDataRef
37__all__ = ["MakeCoaddTempExpTask", "MakeWarpTask", "MakeWarpConfig"]
40class MissingExposureError(Exception):
41 """Raised when data cannot be retrieved for an exposure.
42 When processing patches, sometimes one exposure is missing; this lets us
43 distinguish bewteen that case, and other errors.
44 """
45 pass
48class MakeCoaddTempExpConfig(CoaddBaseTask.ConfigClass):
49 """Config for MakeCoaddTempExpTask
50 """
51 warpAndPsfMatch = pexConfig.ConfigurableField(
52 target=WarpAndPsfMatchTask,
53 doc="Task to warp and PSF-match calexp",
54 )
55 doWrite = pexConfig.Field(
56 doc="persist <coaddName>Coadd_<warpType>Warp",
57 dtype=bool,
58 default=True,
59 )
60 bgSubtracted = pexConfig.Field(
61 doc="Work with a background subtracted calexp?",
62 dtype=bool,
63 default=True,
64 )
65 coaddPsf = pexConfig.ConfigField(
66 doc="Configuration for CoaddPsf",
67 dtype=CoaddPsfConfig,
68 )
69 makeDirect = pexConfig.Field(
70 doc="Make direct Warp/Coadds",
71 dtype=bool,
72 default=True,
73 )
74 makePsfMatched = pexConfig.Field(
75 doc="Make Psf-Matched Warp/Coadd?",
76 dtype=bool,
77 default=False,
78 )
80 doWriteEmptyWarps = pexConfig.Field(
81 dtype=bool,
82 default=False,
83 doc="Write out warps even if they are empty"
84 )
86 hasFakes = pexConfig.Field(
87 doc="Should be set to True if fake sources have been inserted into the input data.",
88 dtype=bool,
89 default=False,
90 )
91 doApplySkyCorr = pexConfig.Field(dtype=bool, default=False, doc="Apply sky correction?")
93 def validate(self):
94 CoaddBaseTask.ConfigClass.validate(self)
95 if not self.makePsfMatched and not self.makeDirect:
96 raise RuntimeError("At least one of config.makePsfMatched and config.makeDirect must be True")
97 if self.doPsfMatch:
98 # Backwards compatibility.
99 log.warn("Config doPsfMatch deprecated. Setting makePsfMatched=True and makeDirect=False")
100 self.makePsfMatched = True
101 self.makeDirect = False
103 def setDefaults(self):
104 CoaddBaseTask.ConfigClass.setDefaults(self)
105 self.warpAndPsfMatch.psfMatch.kernel.active.kernelSize = self.matchingKernelSize
107## \addtogroup LSST_task_documentation
108## \{
109## \page MakeCoaddTempExpTask
110## \ref MakeCoaddTempExpTask_ "MakeCoaddTempExpTask"
111## \copybrief MakeCoaddTempExpTask
112## \}
115class MakeCoaddTempExpTask(CoaddBaseTask):
116 r"""!Warp and optionally PSF-Match calexps onto an a common projection.
118 @anchor MakeCoaddTempExpTask_
120 @section pipe_tasks_makeCoaddTempExp_Contents Contents
122 - @ref pipe_tasks_makeCoaddTempExp_Purpose
123 - @ref pipe_tasks_makeCoaddTempExp_Initialize
124 - @ref pipe_tasks_makeCoaddTempExp_IO
125 - @ref pipe_tasks_makeCoaddTempExp_Config
126 - @ref pipe_tasks_makeCoaddTempExp_Debug
127 - @ref pipe_tasks_makeCoaddTempExp_Example
129 @section pipe_tasks_makeCoaddTempExp_Purpose Description
131 Warp and optionally PSF-Match calexps onto a common projection, by
132 performing the following operations:
133 - Group calexps by visit/run
134 - For each visit, generate a Warp by calling method @ref makeTempExp.
135 makeTempExp loops over the visit's calexps calling @ref WarpAndPsfMatch
136 on each visit
138 The result is a `directWarp` (and/or optionally a `psfMatchedWarp`).
140 @section pipe_tasks_makeCoaddTempExp_Initialize Task Initialization
142 @copydoc \_\_init\_\_
144 This task has one special keyword argument: passing reuse=True will cause
145 the task to skip the creation of warps that are already present in the
146 output repositories.
148 @section pipe_tasks_makeCoaddTempExp_IO Invoking the Task
150 This task is primarily designed to be run from the command line.
152 The main method is `runDataRef`, which takes a single butler data reference for the patch(es)
153 to process.
155 @copydoc run
157 WarpType identifies the types of convolutions applied to Warps (previously CoaddTempExps).
158 Only two types are available: direct (for regular Warps/Coadds) and psfMatched
159 (for Warps/Coadds with homogenized PSFs). We expect to add a third type, likelihood,
160 for generating likelihood Coadds with Warps that have been correlated with their own PSF.
162 @section pipe_tasks_makeCoaddTempExp_Config Configuration parameters
164 See @ref MakeCoaddTempExpConfig and parameters inherited from
165 @link lsst.pipe.tasks.coaddBase.CoaddBaseConfig CoaddBaseConfig @endlink
167 @subsection pipe_tasks_MakeCoaddTempExp_psfMatching Guide to PSF-Matching Configs
169 To make `psfMatchedWarps`, select `config.makePsfMatched=True`. The subtask
170 @link lsst.ip.diffim.modelPsfMatch.ModelPsfMatchTask ModelPsfMatchTask @endlink
171 is responsible for the PSF-Matching, and its config is accessed via `config.warpAndPsfMatch.psfMatch`.
172 The optimal configuration depends on aspects of dataset: the pixel scale, average PSF FWHM and
173 dimensions of the PSF kernel. These configs include the requested model PSF, the matching kernel size,
174 padding of the science PSF thumbnail and spatial sampling frequency of the PSF.
176 *Config Guidelines*: The user must specify the size of the model PSF to which to match by setting
177 `config.modelPsf.defaultFwhm` in units of pixels. The appropriate values depends on science case.
178 In general, for a set of input images, this config should equal the FWHM of the visit
179 with the worst seeing. The smallest it should be set to is the median FWHM. The defaults
180 of the other config options offer a reasonable starting point.
181 The following list presents the most common problems that arise from a misconfigured
182 @link lsst.ip.diffim.modelPsfMatch.ModelPsfMatchTask ModelPsfMatchTask @endlink
183 and corresponding solutions. All assume the default Alard-Lupton kernel, with configs accessed via
184 ```config.warpAndPsfMatch.psfMatch.kernel['AL']```. Each item in the list is formatted as:
185 Problem: Explanation. *Solution*
187 *Troublshooting PSF-Matching Configuration:*
188 - Matched PSFs look boxy: The matching kernel is too small. _Increase the matching kernel size.
189 For example:_
191 config.warpAndPsfMatch.psfMatch.kernel['AL'].kernelSize=27 # default 21
193 Note that increasing the kernel size also increases runtime.
194 - Matched PSFs look ugly (dipoles, quadropoles, donuts): unable to find good solution
195 for matching kernel. _Provide the matcher with more data by either increasing
196 the spatial sampling by decreasing the spatial cell size,_
198 config.warpAndPsfMatch.psfMatch.kernel['AL'].sizeCellX = 64 # default 128
199 config.warpAndPsfMatch.psfMatch.kernel['AL'].sizeCellY = 64 # default 128
201 _or increasing the padding around the Science PSF, for example:_
203 config.warpAndPsfMatch.psfMatch.autoPadPsfTo=1.6 # default 1.4
205 Increasing `autoPadPsfTo` increases the minimum ratio of input PSF dimensions to the
206 matching kernel dimensions, thus increasing the number of pixels available to fit
207 after convolving the PSF with the matching kernel.
208 Optionally, for debugging the effects of padding, the level of padding may be manually
209 controlled by setting turning off the automatic padding and setting the number
210 of pixels by which to pad the PSF:
212 config.warpAndPsfMatch.psfMatch.doAutoPadPsf = False # default True
213 config.warpAndPsfMatch.psfMatch.padPsfBy = 6 # pixels. default 0
215 - Deconvolution: Matching a large PSF to a smaller PSF produces
216 a telltale noise pattern which looks like ripples or a brain.
217 _Increase the size of the requested model PSF. For example:_
219 config.modelPsf.defaultFwhm = 11 # Gaussian sigma in units of pixels.
221 - High frequency (sometimes checkered) noise: The matching basis functions are too small.
222 _Increase the width of the Gaussian basis functions. For example:_
224 config.warpAndPsfMatch.psfMatch.kernel['AL'].alardSigGauss=[1.5, 3.0, 6.0]
225 # from default [0.7, 1.5, 3.0]
228 @section pipe_tasks_makeCoaddTempExp_Debug Debug variables
230 MakeCoaddTempExpTask has no debug output, but its subtasks do.
232 @section pipe_tasks_makeCoaddTempExp_Example A complete example of using MakeCoaddTempExpTask
234 This example uses the package ci_hsc to show how MakeCoaddTempExp fits
235 into the larger Data Release Processing.
236 Set up by running:
238 setup ci_hsc
239 cd $CI_HSC_DIR
240 # if not built already:
241 python $(which scons) # this will take a while
243 The following assumes that `processCcd.py` and `makeSkyMap.py` have previously been run
244 (e.g. by building `ci_hsc` above) to generate a repository of calexps and an
245 output respository with the desired SkyMap. The command,
247 makeCoaddTempExp.py $CI_HSC_DIR/DATA --rerun ci_hsc \
248 --id patch=5,4 tract=0 filter=HSC-I \
249 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 \
250 --selectId visit=903988 ccd=23 --selectId visit=903988 ccd=24 \
251 --config doApplyExternalPhotoCalib=False doApplyExternalSkyWcs=False \
252 makePsfMatched=True modelPsf.defaultFwhm=11
254 writes a direct and PSF-Matched Warp to
255 - `$CI_HSC_DIR/DATA/rerun/ci_hsc/deepCoadd/HSC-I/0/5,4/warp-HSC-I-0-5,4-903988.fits` and
256 - `$CI_HSC_DIR/DATA/rerun/ci_hsc/deepCoadd/HSC-I/0/5,4/psfMatchedWarp-HSC-I-0-5,4-903988.fits`
257 respectively.
259 @note PSF-Matching in this particular dataset would benefit from adding
260 `--configfile ./matchingConfig.py` to
261 the command line arguments where `matchingConfig.py` is defined by:
263 echo "
264 config.warpAndPsfMatch.psfMatch.kernel['AL'].kernelSize=27
265 config.warpAndPsfMatch.psfMatch.kernel['AL'].alardSigGauss=[1.5, 3.0, 6.0]" > matchingConfig.py
268 Add the option `--help` to see more options.
269 """
270 ConfigClass = MakeCoaddTempExpConfig
271 _DefaultName = "makeCoaddTempExp"
273 def __init__(self, reuse=False, **kwargs):
274 CoaddBaseTask.__init__(self, **kwargs)
275 self.reuse = reuse
276 self.makeSubtask("warpAndPsfMatch")
277 if self.config.hasFakes: 277 ↛ 278line 277 didn't jump to line 278, because the condition on line 277 was never true
278 self.calexpType = "fakes_calexp"
279 else:
280 self.calexpType = "calexp"
282 @pipeBase.timeMethod
283 def runDataRef(self, patchRef, selectDataList=[]):
284 """!Produce <coaddName>Coadd_<warpType>Warp images by warping and optionally PSF-matching.
286 @param[in] patchRef: data reference for sky map patch. Must include keys "tract", "patch",
287 plus the camera-specific filter key (e.g. "filter" or "band")
288 @return: dataRefList: a list of data references for the new <coaddName>Coadd_directWarps
289 if direct or both warp types are requested and <coaddName>Coadd_psfMatchedWarps if only psfMatched
290 warps are requested.
292 @warning: this task assumes that all exposures in a warp (coaddTempExp) have the same filter.
294 @warning: this task sets the PhotoCalib of the coaddTempExp to the PhotoCalib of the first calexp
295 with any good pixels in the patch. For a mosaic camera the resulting PhotoCalib should be ignored
296 (assembleCoadd should determine zeropoint scaling without referring to it).
297 """
298 skyInfo = self.getSkyInfo(patchRef)
300 # DataRefs to return are of type *_directWarp unless only *_psfMatchedWarp requested
301 if self.config.makePsfMatched and not self.config.makeDirect: 301 ↛ 302line 301 didn't jump to line 302, because the condition on line 301 was never true
302 primaryWarpDataset = self.getTempExpDatasetName("psfMatched")
303 else:
304 primaryWarpDataset = self.getTempExpDatasetName("direct")
306 calExpRefList = self.selectExposures(patchRef, skyInfo, selectDataList=selectDataList)
308 if len(calExpRefList) == 0: 308 ↛ 309line 308 didn't jump to line 309, because the condition on line 308 was never true
309 self.log.warn("No exposures to coadd for patch %s", patchRef.dataId)
310 return None
311 self.log.info("Selected %d calexps for patch %s", len(calExpRefList), patchRef.dataId)
312 calExpRefList = [calExpRef for calExpRef in calExpRefList if calExpRef.datasetExists(self.calexpType)]
313 self.log.info("Processing %d existing calexps for patch %s", len(calExpRefList), patchRef.dataId)
315 groupData = groupPatchExposures(patchRef, calExpRefList, self.getCoaddDatasetName(),
316 primaryWarpDataset)
317 self.log.info("Processing %d warp exposures for patch %s", len(groupData.groups), patchRef.dataId)
319 dataRefList = []
320 for i, (tempExpTuple, calexpRefList) in enumerate(groupData.groups.items()):
321 tempExpRef = getGroupDataRef(patchRef.getButler(), primaryWarpDataset,
322 tempExpTuple, groupData.keys)
323 if self.reuse and tempExpRef.datasetExists(datasetType=primaryWarpDataset, write=True): 323 ↛ 324line 323 didn't jump to line 324, because the condition on line 323 was never true
324 self.log.info("Skipping makeCoaddTempExp for %s; output already exists.", tempExpRef.dataId)
325 dataRefList.append(tempExpRef)
326 continue
327 self.log.info("Processing Warp %d/%d: id=%s", i, len(groupData.groups), tempExpRef.dataId)
329 # TODO: mappers should define a way to go from the "grouping keys" to a numeric ID (#2776).
330 # For now, we try to get a long integer "visit" key, and if we can't, we just use the index
331 # of the visit in the list.
332 try:
333 visitId = int(tempExpRef.dataId["visit"])
334 except (KeyError, ValueError):
335 visitId = i
337 calExpList = []
338 ccdIdList = []
339 dataIdList = []
341 for calExpInd, calExpRef in enumerate(calexpRefList):
342 self.log.info("Reading calexp %s of %s for Warp id=%s", calExpInd+1, len(calexpRefList),
343 calExpRef.dataId)
344 try:
345 ccdId = calExpRef.get("ccdExposureId", immediate=True)
346 except Exception:
347 ccdId = calExpInd
348 try:
349 # We augment the dataRef here with the tract, which is harmless for loading things
350 # like calexps that don't need the tract, and necessary for meas_mosaic outputs,
351 # which do.
352 calExpRef = calExpRef.butlerSubset.butler.dataRef(self.calexpType,
353 dataId=calExpRef.dataId,
354 tract=skyInfo.tractInfo.getId())
355 calExp = self.getCalibratedExposure(calExpRef, bgSubtracted=self.config.bgSubtracted)
356 except Exception as e:
357 self.log.warn("Calexp %s not found; skipping it: %s", calExpRef.dataId, e)
358 continue
360 if self.config.doApplySkyCorr: 360 ↛ 361line 360 didn't jump to line 361, because the condition on line 360 was never true
361 self.applySkyCorr(calExpRef, calExp)
363 calExpList.append(calExp)
364 ccdIdList.append(ccdId)
365 dataIdList.append(calExpRef.dataId)
367 exps = self.run(calExpList, ccdIdList, skyInfo, visitId, dataIdList).exposures
369 if any(exps.values()):
370 dataRefList.append(tempExpRef)
371 else:
372 self.log.warn("Warp %s could not be created", tempExpRef.dataId)
374 if self.config.doWrite: 374 ↛ 320line 374 didn't jump to line 320, because the condition on line 374 was never false
375 for (warpType, exposure) in exps.items(): # compatible w/ Py3
376 if exposure is not None:
377 self.log.info("Persisting %s" % self.getTempExpDatasetName(warpType))
378 tempExpRef.put(exposure, self.getTempExpDatasetName(warpType))
380 return dataRefList
382 @pipeBase.timeMethod
383 def run(self, calExpList, ccdIdList, skyInfo, visitId=0, dataIdList=None):
384 """Create a Warp from inputs
386 We iterate over the multiple calexps in a single exposure to construct
387 the warp (previously called a coaddTempExp) of that exposure to the
388 supplied tract/patch.
390 Pixels that receive no pixels are set to NAN; this is not correct
391 (violates LSST algorithms group policy), but will be fixed up by
392 interpolating after the coaddition.
394 @param calexpRefList: List of data references for calexps that (may)
395 overlap the patch of interest
396 @param skyInfo: Struct from CoaddBaseTask.getSkyInfo() with geometric
397 information about the patch
398 @param visitId: integer identifier for visit, for the table that will
399 produce the CoaddPsf
400 @return a pipeBase Struct containing:
401 - exposures: a dictionary containing the warps requested:
402 "direct": direct warp if config.makeDirect
403 "psfMatched": PSF-matched warp if config.makePsfMatched
404 """
405 warpTypeList = self.getWarpTypeList()
407 totGoodPix = {warpType: 0 for warpType in warpTypeList}
408 didSetMetadata = {warpType: False for warpType in warpTypeList}
409 coaddTempExps = {warpType: self._prepareEmptyExposure(skyInfo) for warpType in warpTypeList}
410 inputRecorder = {warpType: self.inputRecorder.makeCoaddTempExpRecorder(visitId, len(calExpList))
411 for warpType in warpTypeList}
413 modelPsf = self.config.modelPsf.apply() if self.config.makePsfMatched else None
414 if dataIdList is None: 414 ↛ 415line 414 didn't jump to line 415, because the condition on line 414 was never true
415 dataIdList = ccdIdList
417 for calExpInd, (calExp, ccdId, dataId) in enumerate(zip(calExpList, ccdIdList, dataIdList)):
418 self.log.info("Processing calexp %d of %d for this Warp: id=%s",
419 calExpInd+1, len(calExpList), dataId)
421 try:
422 warpedAndMatched = self.warpAndPsfMatch.run(calExp, modelPsf=modelPsf,
423 wcs=skyInfo.wcs, maxBBox=skyInfo.bbox,
424 makeDirect=self.config.makeDirect,
425 makePsfMatched=self.config.makePsfMatched)
426 except Exception as e:
427 self.log.warn("WarpAndPsfMatch failed for calexp %s; skipping it: %s", dataId, e)
428 continue
429 try:
430 numGoodPix = {warpType: 0 for warpType in warpTypeList}
431 for warpType in warpTypeList:
432 exposure = warpedAndMatched.getDict()[warpType]
433 if exposure is None:
434 continue
435 coaddTempExp = coaddTempExps[warpType]
436 if didSetMetadata[warpType]:
437 mimg = exposure.getMaskedImage()
438 mimg *= (coaddTempExp.getPhotoCalib().getInstFluxAtZeroMagnitude()
439 / exposure.getPhotoCalib().getInstFluxAtZeroMagnitude())
440 del mimg
441 numGoodPix[warpType] = coaddUtils.copyGoodPixels(
442 coaddTempExp.getMaskedImage(), exposure.getMaskedImage(), self.getBadPixelMask())
443 totGoodPix[warpType] += numGoodPix[warpType]
444 self.log.debug("Calexp %s has %d good pixels in this patch (%.1f%%) for %s",
445 dataId, numGoodPix[warpType],
446 100.0*numGoodPix[warpType]/skyInfo.bbox.getArea(), warpType)
447 if numGoodPix[warpType] > 0 and not didSetMetadata[warpType]:
448 coaddTempExp.setPhotoCalib(exposure.getPhotoCalib())
449 coaddTempExp.setFilter(exposure.getFilter())
450 coaddTempExp.getInfo().setVisitInfo(exposure.getInfo().getVisitInfo())
451 # PSF replaced with CoaddPsf after loop if and only if creating direct warp
452 coaddTempExp.setPsf(exposure.getPsf())
453 didSetMetadata[warpType] = True
455 # Need inputRecorder for CoaddApCorrMap for both direct and PSF-matched
456 inputRecorder[warpType].addCalExp(calExp, ccdId, numGoodPix[warpType])
458 except Exception as e:
459 self.log.warn("Error processing calexp %s; skipping it: %s", dataId, e)
460 continue
462 for warpType in warpTypeList:
463 self.log.info("%sWarp has %d good pixels (%.1f%%)",
464 warpType, totGoodPix[warpType], 100.0*totGoodPix[warpType]/skyInfo.bbox.getArea())
466 if totGoodPix[warpType] > 0 and didSetMetadata[warpType]:
467 inputRecorder[warpType].finish(coaddTempExps[warpType], totGoodPix[warpType])
468 if warpType == "direct":
469 coaddTempExps[warpType].setPsf(
470 CoaddPsf(inputRecorder[warpType].coaddInputs.ccds, skyInfo.wcs,
471 self.config.coaddPsf.makeControl()))
472 else:
473 if not self.config.doWriteEmptyWarps: 473 ↛ 462line 473 didn't jump to line 462, because the condition on line 473 was never false
474 # No good pixels. Exposure still empty
475 coaddTempExps[warpType] = None
477 result = pipeBase.Struct(exposures=coaddTempExps)
478 return result
480 def getCalibratedExposure(self, dataRef, bgSubtracted):
481 """Return one calibrated Exposure, possibly with an updated SkyWcs.
483 @param[in] dataRef a sensor-level data reference
484 @param[in] bgSubtracted return calexp with background subtracted? If False get the
485 calexp's background background model and add it to the calexp.
486 @return calibrated exposure
488 @raises MissingExposureError If data for the exposure is not available.
490 If config.doApplyExternalPhotoCalib is `True`, the photometric calibration
491 (`photoCalib`) is taken from `config.externalPhotoCalibName` via the
492 `name_photoCalib` dataset. Otherwise, the photometric calibration is
493 retrieved from the processed exposure. When
494 `config.doApplyExternalSkyWcs` is `True`, the astrometric calibration
495 is taken from `config.externalSkyWcsName` with the `name_wcs` dataset.
496 Otherwise, the astrometric calibration is taken from the processed
497 exposure.
498 """
499 try:
500 exposure = dataRef.get(self.calexpType, immediate=True)
501 except dafPersist.NoResults as e:
502 raise MissingExposureError('Exposure not found: %s ' % str(e)) from e
504 if not bgSubtracted: 504 ↛ 505line 504 didn't jump to line 505, because the condition on line 504 was never true
505 background = dataRef.get("calexpBackground", immediate=True)
506 mi = exposure.getMaskedImage()
507 mi += background.getImage()
508 del mi
510 if self.config.doApplyExternalPhotoCalib:
511 source = f"{self.config.externalPhotoCalibName}_photoCalib"
512 self.log.debug("Applying external photoCalib to %s from %s", dataRef.dataId, source)
513 photoCalib = dataRef.get(source)
514 exposure.setPhotoCalib(photoCalib)
515 else:
516 photoCalib = exposure.getPhotoCalib()
518 if self.config.doApplyExternalSkyWcs:
519 source = f"{self.config.externalSkyWcsName}_wcs"
520 self.log.debug("Applying external skyWcs to %s from %s", dataRef.dataId, source)
521 skyWcs = dataRef.get(source)
522 exposure.setWcs(skyWcs)
524 exposure.maskedImage = photoCalib.calibrateImage(exposure.maskedImage,
525 includeScaleUncertainty=self.config.includeCalibVar)
526 exposure.maskedImage /= photoCalib.getCalibrationMean()
527 # TODO: The images will have a calibration of 1.0 everywhere once RFC-545 is implemented.
528 # exposure.setCalib(afwImage.Calib(1.0))
529 return exposure
531 @staticmethod
532 def _prepareEmptyExposure(skyInfo):
533 """Produce an empty exposure for a given patch"""
534 exp = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
535 exp.getMaskedImage().set(numpy.nan, afwImage.Mask
536 .getPlaneBitMask("NO_DATA"), numpy.inf)
537 return exp
539 def getWarpTypeList(self):
540 """Return list of requested warp types per the config.
541 """
542 warpTypeList = []
543 if self.config.makeDirect: 543 ↛ 545line 543 didn't jump to line 545, because the condition on line 543 was never false
544 warpTypeList.append("direct")
545 if self.config.makePsfMatched: 545 ↛ 547line 545 didn't jump to line 547, because the condition on line 545 was never false
546 warpTypeList.append("psfMatched")
547 return warpTypeList
549 def applySkyCorr(self, dataRef, calexp):
550 """Apply correction to the sky background level
552 Sky corrections can be generated with the 'skyCorrection.py'
553 executable in pipe_drivers. Because the sky model used by that
554 code extends over the entire focal plane, this can produce
555 better sky subtraction.
557 The calexp is updated in-place.
559 Parameters
560 ----------
561 dataRef : `lsst.daf.persistence.ButlerDataRef`
562 Data reference for calexp.
563 calexp : `lsst.afw.image.Exposure` or `lsst.afw.image.MaskedImage`
564 Calibrated exposure.
565 """
566 bg = dataRef.get("skyCorr")
567 self.log.debug("Applying sky correction to %s", dataRef.dataId)
568 if isinstance(calexp, afwImage.Exposure):
569 calexp = calexp.getMaskedImage()
570 calexp -= bg.getImage()
573class MakeWarpConnections(pipeBase.PipelineTaskConnections,
574 dimensions=("tract", "patch", "skymap", "instrument", "visit"),
575 defaultTemplates={"coaddName": "deep"}):
576 calExpList = cT.Input(
577 doc="Input exposures to be resampled and optionally PSF-matched onto a SkyMap projection/patch",
578 name="calexp",
579 storageClass="ExposureF",
580 dimensions=("instrument", "visit", "detector"),
581 multiple=True,
582 )
583 backgroundList = cT.Input(
584 doc="Input backgrounds to be added back into the calexp if bgSubtracted=False",
585 name="calexpBackground",
586 storageClass="Background",
587 dimensions=("instrument", "visit", "detector"),
588 multiple=True,
589 )
590 skyCorrList = cT.Input(
591 doc="Input Sky Correction to be subtracted from the calexp if doApplySkyCorr=True",
592 name="skyCorr",
593 storageClass="Background",
594 dimensions=("instrument", "visit", "detector"),
595 multiple=True,
596 )
597 skyMap = cT.Input(
598 doc="Input definition of geometry/bbox and projection/wcs for warped exposures",
599 name="{coaddName}Coadd_skyMap",
600 storageClass="SkyMap",
601 dimensions=("skymap",),
602 )
603 direct = cT.Output(
604 doc=("Output direct warped exposure (previously called CoaddTempExp), produced by resampling ",
605 "calexps onto the skyMap patch geometry."),
606 name="{coaddName}Coadd_directWarp",
607 storageClass="ExposureF",
608 dimensions=("tract", "patch", "skymap", "visit", "instrument"),
609 )
610 psfMatched = cT.Output(
611 doc=("Output PSF-Matched warped exposure (previously called CoaddTempExp), produced by resampling ",
612 "calexps onto the skyMap patch geometry and PSF-matching to a model PSF."),
613 name="{coaddName}Coadd_psfMatchedWarp",
614 storageClass="ExposureF",
615 dimensions=("tract", "patch", "skymap", "visit", "instrument"),
616 )
618 def __init__(self, *, config=None):
619 super().__init__(config=config)
620 if config.bgSubtracted:
621 self.inputs.remove("backgroundList")
622 if not config.doApplySkyCorr:
623 self.inputs.remove("skyCorrList")
624 if not config.makeDirect:
625 self.outputs.remove("direct")
626 if not config.makePsfMatched:
627 self.outputs.remove("psfMatched")
630class MakeWarpConfig(pipeBase.PipelineTaskConfig, MakeCoaddTempExpConfig,
631 pipelineConnections=MakeWarpConnections):
633 def validate(self):
634 super().validate()
635 # TODO: Remove this constraint after DM-17062
636 if self.doApplyExternalPhotoCalib:
637 raise RuntimeError("Gen3 MakeWarpTask cannot apply external PhotoCalib results. "
638 "Please set doApplyExternalPhotoCalib=False.")
639 if self.doApplyExternalSkyWcs:
640 raise RuntimeError("Gen3 MakeWarpTask cannot apply external SkyWcs results. "
641 "Please set doApplyExternalSkyWcs=False.")
644class MakeWarpTask(MakeCoaddTempExpTask, pipeBase.PipelineTask):
645 """Warp and optionally PSF-Match calexps onto an a common projection
647 First Draft of a Gen3 compatible MakeWarpTask which
648 currently does not handle doApplyExternalPhotoCalib=True or
649 doApplyExternalSkyWcs=True.
650 """
651 ConfigClass = MakeWarpConfig
652 _DefaultName = "makeWarp"
654 @utils.inheritDoc(pipeBase.PipelineTask)
655 def runQuantum(self, butlerQC, inputRefs, outputRefs):
656 """
657 Notes
658 ----
659 Construct warps for requested warp type for single epoch
661 PipelineTask (Gen3) entry point to warp and optionally PSF-match
662 calexps. This method is analogous to `runDataRef`.
663 """
664 # Read in all inputs.
665 inputs = butlerQC.get(inputRefs)
667 # Construct skyInfo expected by `run`. We remove the SkyMap itself
668 # from the dictionary so we can pass it as kwargs later.
669 skyMap = inputs.pop("skyMap")
670 quantumDataId = butlerQC.quantum.dataId
671 skyInfo = makeSkyInfo(skyMap, tractId=quantumDataId['tract'], patchId=quantumDataId['patch'])
673 # Construct list of input DataIds expected by `run`
674 dataIdList = [ref.dataId for ref in inputRefs.calExpList]
676 # Construct list of packed integer IDs expected by `run`
677 ccdIdList = [dataId.pack("visit_detector") for dataId in dataIdList]
679 # Extract integer visitId requested by `run`
680 visits = [dataId['visit'] for dataId in dataIdList]
681 assert(all(visits[0] == visit for visit in visits))
682 visitId = visits[0]
684 self.prepareCalibratedExposures(**inputs)
685 results = self.run(**inputs, visitId=visitId, ccdIdList=ccdIdList, dataIdList=dataIdList,
686 skyInfo=skyInfo)
687 if self.config.makeDirect:
688 butlerQC.put(results.exposures["direct"], outputRefs.direct)
689 if self.config.makePsfMatched:
690 butlerQC.put(results.exposures["psfMatched"], outputRefs.psfMatched)
692 def prepareCalibratedExposures(self, calExpList, backgroundList=None, skyCorrList=None):
693 """Calibrate and add backgrounds to input calExpList in place
695 TODO DM-17062: apply jointcal/meas_mosaic here
697 Parameters
698 ----------
699 calExpList : `list` of `lsst.afw.image.Exposure`
700 Sequence of calexps to be modified in place
701 backgroundList : `list` of `lsst.afw.math.backgroundList`
702 Sequence of backgrounds to be added back in if bgSubtracted=False
703 skyCorrList : `list` of `lsst.afw.math.backgroundList`
704 Sequence of background corrections to be subtracted if doApplySkyCorr=True
705 """
706 backgroundList = len(calExpList)*[None] if backgroundList is None else backgroundList
707 skyCorrList = len(calExpList)*[None] if skyCorrList is None else skyCorrList
708 for calexp, background, skyCorr in zip(calExpList, backgroundList, skyCorrList):
709 mi = calexp.maskedImage
710 if not self.config.bgSubtracted:
711 mi += background.getImage()
712 if self.config.doApplySkyCorr:
713 mi -= skyCorr.getImage()