Coverage for python/lsst/pipe/tasks/makeCoaddTempExp.py : 15%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1#
2# LSST Data Management System
3# Copyright 2008, 2009, 2010, 2011, 2012 LSST Corporation.
4#
5# This product includes software developed by the
6# LSST Project (http://www.lsst.org/).
7#
8# This program is free software: you can redistribute it and/or modify
9# it under the terms of the GNU General Public License as published by
10# the Free Software Foundation, either version 3 of the License, or
11# (at your option) any later version.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the LSST License Statement and
19# the GNU General Public License along with this program. If not,
20# see <http://www.lsstcorp.org/LegalNotices/>.
21#
22import numpy
24import lsst.pex.config as pexConfig
25import lsst.daf.persistence as dafPersist
26import lsst.afw.image as afwImage
27import lsst.coadd.utils as coaddUtils
28import lsst.pipe.base as pipeBase
29import lsst.pipe.base.connectionTypes as connectionTypes
30import lsst.log as log
31import lsst.utils as utils
32import lsst.geom
33from lsst.meas.algorithms import CoaddPsf, CoaddPsfConfig
34from lsst.skymap import BaseSkyMap
35from .coaddBase import CoaddBaseTask, makeSkyInfo
36from .warpAndPsfMatch import WarpAndPsfMatchTask
37from .coaddHelpers import groupPatchExposures, getGroupDataRef
39__all__ = ["MakeCoaddTempExpTask", "MakeWarpTask", "MakeWarpConfig"]
42class MissingExposureError(Exception):
43 """Raised when data cannot be retrieved for an exposure.
44 When processing patches, sometimes one exposure is missing; this lets us
45 distinguish bewteen that case, and other errors.
46 """
47 pass
50class MakeCoaddTempExpConfig(CoaddBaseTask.ConfigClass):
51 """Config for MakeCoaddTempExpTask
52 """
53 warpAndPsfMatch = pexConfig.ConfigurableField(
54 target=WarpAndPsfMatchTask,
55 doc="Task to warp and PSF-match calexp",
56 )
57 doWrite = pexConfig.Field(
58 doc="persist <coaddName>Coadd_<warpType>Warp",
59 dtype=bool,
60 default=True,
61 )
62 bgSubtracted = pexConfig.Field(
63 doc="Work with a background subtracted calexp?",
64 dtype=bool,
65 default=True,
66 )
67 coaddPsf = pexConfig.ConfigField(
68 doc="Configuration for CoaddPsf",
69 dtype=CoaddPsfConfig,
70 )
71 makeDirect = pexConfig.Field(
72 doc="Make direct Warp/Coadds",
73 dtype=bool,
74 default=True,
75 )
76 makePsfMatched = pexConfig.Field(
77 doc="Make Psf-Matched Warp/Coadd?",
78 dtype=bool,
79 default=False,
80 )
82 doWriteEmptyWarps = pexConfig.Field(
83 dtype=bool,
84 default=False,
85 doc="Write out warps even if they are empty"
86 )
88 hasFakes = pexConfig.Field(
89 doc="Should be set to True if fake sources have been inserted into the input data.",
90 dtype=bool,
91 default=False,
92 )
93 doApplySkyCorr = pexConfig.Field(dtype=bool, default=False, doc="Apply sky correction?")
95 def validate(self):
96 CoaddBaseTask.ConfigClass.validate(self)
97 if not self.makePsfMatched and not self.makeDirect:
98 raise RuntimeError("At least one of config.makePsfMatched and config.makeDirect must be True")
99 if self.doPsfMatch:
100 # Backwards compatibility.
101 log.warn("Config doPsfMatch deprecated. Setting makePsfMatched=True and makeDirect=False")
102 self.makePsfMatched = True
103 self.makeDirect = False
105 def setDefaults(self):
106 CoaddBaseTask.ConfigClass.setDefaults(self)
107 self.warpAndPsfMatch.psfMatch.kernel.active.kernelSize = self.matchingKernelSize
109## \addtogroup LSST_task_documentation
110## \{
111## \page MakeCoaddTempExpTask
112## \ref MakeCoaddTempExpTask_ "MakeCoaddTempExpTask"
113## \copybrief MakeCoaddTempExpTask
114## \}
117class MakeCoaddTempExpTask(CoaddBaseTask):
118 r"""!Warp and optionally PSF-Match calexps onto an a common projection.
120 @anchor MakeCoaddTempExpTask_
122 @section pipe_tasks_makeCoaddTempExp_Contents Contents
124 - @ref pipe_tasks_makeCoaddTempExp_Purpose
125 - @ref pipe_tasks_makeCoaddTempExp_Initialize
126 - @ref pipe_tasks_makeCoaddTempExp_IO
127 - @ref pipe_tasks_makeCoaddTempExp_Config
128 - @ref pipe_tasks_makeCoaddTempExp_Debug
129 - @ref pipe_tasks_makeCoaddTempExp_Example
131 @section pipe_tasks_makeCoaddTempExp_Purpose Description
133 Warp and optionally PSF-Match calexps onto a common projection, by
134 performing the following operations:
135 - Group calexps by visit/run
136 - For each visit, generate a Warp by calling method @ref makeTempExp.
137 makeTempExp loops over the visit's calexps calling @ref WarpAndPsfMatch
138 on each visit
140 The result is a `directWarp` (and/or optionally a `psfMatchedWarp`).
142 @section pipe_tasks_makeCoaddTempExp_Initialize Task Initialization
144 @copydoc \_\_init\_\_
146 This task has one special keyword argument: passing reuse=True will cause
147 the task to skip the creation of warps that are already present in the
148 output repositories.
150 @section pipe_tasks_makeCoaddTempExp_IO Invoking the Task
152 This task is primarily designed to be run from the command line.
154 The main method is `runDataRef`, which takes a single butler data reference for the patch(es)
155 to process.
157 @copydoc run
159 WarpType identifies the types of convolutions applied to Warps (previously CoaddTempExps).
160 Only two types are available: direct (for regular Warps/Coadds) and psfMatched
161 (for Warps/Coadds with homogenized PSFs). We expect to add a third type, likelihood,
162 for generating likelihood Coadds with Warps that have been correlated with their own PSF.
164 @section pipe_tasks_makeCoaddTempExp_Config Configuration parameters
166 See @ref MakeCoaddTempExpConfig and parameters inherited from
167 @link lsst.pipe.tasks.coaddBase.CoaddBaseConfig CoaddBaseConfig @endlink
169 @subsection pipe_tasks_MakeCoaddTempExp_psfMatching Guide to PSF-Matching Configs
171 To make `psfMatchedWarps`, select `config.makePsfMatched=True`. The subtask
172 @link lsst.ip.diffim.modelPsfMatch.ModelPsfMatchTask ModelPsfMatchTask @endlink
173 is responsible for the PSF-Matching, and its config is accessed via `config.warpAndPsfMatch.psfMatch`.
174 The optimal configuration depends on aspects of dataset: the pixel scale, average PSF FWHM and
175 dimensions of the PSF kernel. These configs include the requested model PSF, the matching kernel size,
176 padding of the science PSF thumbnail and spatial sampling frequency of the PSF.
178 *Config Guidelines*: The user must specify the size of the model PSF to which to match by setting
179 `config.modelPsf.defaultFwhm` in units of pixels. The appropriate values depends on science case.
180 In general, for a set of input images, this config should equal the FWHM of the visit
181 with the worst seeing. The smallest it should be set to is the median FWHM. The defaults
182 of the other config options offer a reasonable starting point.
183 The following list presents the most common problems that arise from a misconfigured
184 @link lsst.ip.diffim.modelPsfMatch.ModelPsfMatchTask ModelPsfMatchTask @endlink
185 and corresponding solutions. All assume the default Alard-Lupton kernel, with configs accessed via
186 ```config.warpAndPsfMatch.psfMatch.kernel['AL']```. Each item in the list is formatted as:
187 Problem: Explanation. *Solution*
189 *Troublshooting PSF-Matching Configuration:*
190 - Matched PSFs look boxy: The matching kernel is too small. _Increase the matching kernel size.
191 For example:_
193 config.warpAndPsfMatch.psfMatch.kernel['AL'].kernelSize=27 # default 21
195 Note that increasing the kernel size also increases runtime.
196 - Matched PSFs look ugly (dipoles, quadropoles, donuts): unable to find good solution
197 for matching kernel. _Provide the matcher with more data by either increasing
198 the spatial sampling by decreasing the spatial cell size,_
200 config.warpAndPsfMatch.psfMatch.kernel['AL'].sizeCellX = 64 # default 128
201 config.warpAndPsfMatch.psfMatch.kernel['AL'].sizeCellY = 64 # default 128
203 _or increasing the padding around the Science PSF, for example:_
205 config.warpAndPsfMatch.psfMatch.autoPadPsfTo=1.6 # default 1.4
207 Increasing `autoPadPsfTo` increases the minimum ratio of input PSF dimensions to the
208 matching kernel dimensions, thus increasing the number of pixels available to fit
209 after convolving the PSF with the matching kernel.
210 Optionally, for debugging the effects of padding, the level of padding may be manually
211 controlled by setting turning off the automatic padding and setting the number
212 of pixels by which to pad the PSF:
214 config.warpAndPsfMatch.psfMatch.doAutoPadPsf = False # default True
215 config.warpAndPsfMatch.psfMatch.padPsfBy = 6 # pixels. default 0
217 - Deconvolution: Matching a large PSF to a smaller PSF produces
218 a telltale noise pattern which looks like ripples or a brain.
219 _Increase the size of the requested model PSF. For example:_
221 config.modelPsf.defaultFwhm = 11 # Gaussian sigma in units of pixels.
223 - High frequency (sometimes checkered) noise: The matching basis functions are too small.
224 _Increase the width of the Gaussian basis functions. For example:_
226 config.warpAndPsfMatch.psfMatch.kernel['AL'].alardSigGauss=[1.5, 3.0, 6.0]
227 # from default [0.7, 1.5, 3.0]
230 @section pipe_tasks_makeCoaddTempExp_Debug Debug variables
232 MakeCoaddTempExpTask has no debug output, but its subtasks do.
234 @section pipe_tasks_makeCoaddTempExp_Example A complete example of using MakeCoaddTempExpTask
236 This example uses the package ci_hsc to show how MakeCoaddTempExp fits
237 into the larger Data Release Processing.
238 Set up by running:
240 setup ci_hsc
241 cd $CI_HSC_DIR
242 # if not built already:
243 python $(which scons) # this will take a while
245 The following assumes that `processCcd.py` and `makeSkyMap.py` have previously been run
246 (e.g. by building `ci_hsc` above) to generate a repository of calexps and an
247 output respository with the desired SkyMap. The command,
249 makeCoaddTempExp.py $CI_HSC_DIR/DATA --rerun ci_hsc \
250 --id patch=5,4 tract=0 filter=HSC-I \
251 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 \
252 --selectId visit=903988 ccd=23 --selectId visit=903988 ccd=24 \
253 --config doApplyExternalPhotoCalib=False doApplyExternalSkyWcs=False \
254 makePsfMatched=True modelPsf.defaultFwhm=11
256 writes a direct and PSF-Matched Warp to
257 - `$CI_HSC_DIR/DATA/rerun/ci_hsc/deepCoadd/HSC-I/0/5,4/warp-HSC-I-0-5,4-903988.fits` and
258 - `$CI_HSC_DIR/DATA/rerun/ci_hsc/deepCoadd/HSC-I/0/5,4/psfMatchedWarp-HSC-I-0-5,4-903988.fits`
259 respectively.
261 @note PSF-Matching in this particular dataset would benefit from adding
262 `--configfile ./matchingConfig.py` to
263 the command line arguments where `matchingConfig.py` is defined by:
265 echo "
266 config.warpAndPsfMatch.psfMatch.kernel['AL'].kernelSize=27
267 config.warpAndPsfMatch.psfMatch.kernel['AL'].alardSigGauss=[1.5, 3.0, 6.0]" > matchingConfig.py
270 Add the option `--help` to see more options.
271 """
272 ConfigClass = MakeCoaddTempExpConfig
273 _DefaultName = "makeCoaddTempExp"
275 def __init__(self, reuse=False, **kwargs):
276 CoaddBaseTask.__init__(self, **kwargs)
277 self.reuse = reuse
278 self.makeSubtask("warpAndPsfMatch")
279 if self.config.hasFakes:
280 self.calexpType = "fakes_calexp"
281 else:
282 self.calexpType = "calexp"
284 @pipeBase.timeMethod
285 def runDataRef(self, patchRef, selectDataList=[]):
286 """!Produce <coaddName>Coadd_<warpType>Warp images by warping and optionally PSF-matching.
288 @param[in] patchRef: data reference for sky map patch. Must include keys "tract", "patch",
289 plus the camera-specific filter key (e.g. "filter" or "band")
290 @return: dataRefList: a list of data references for the new <coaddName>Coadd_directWarps
291 if direct or both warp types are requested and <coaddName>Coadd_psfMatchedWarps if only psfMatched
292 warps are requested.
294 @warning: this task assumes that all exposures in a warp (coaddTempExp) have the same filter.
296 @warning: this task sets the PhotoCalib of the coaddTempExp to the PhotoCalib of the first calexp
297 with any good pixels in the patch. For a mosaic camera the resulting PhotoCalib should be ignored
298 (assembleCoadd should determine zeropoint scaling without referring to it).
299 """
300 skyInfo = self.getSkyInfo(patchRef)
302 # DataRefs to return are of type *_directWarp unless only *_psfMatchedWarp requested
303 if self.config.makePsfMatched and not self.config.makeDirect:
304 primaryWarpDataset = self.getTempExpDatasetName("psfMatched")
305 else:
306 primaryWarpDataset = self.getTempExpDatasetName("direct")
308 calExpRefList = self.selectExposures(patchRef, skyInfo, selectDataList=selectDataList)
310 if len(calExpRefList) == 0:
311 self.log.warn("No exposures to coadd for patch %s", patchRef.dataId)
312 return None
313 self.log.info("Selected %d calexps for patch %s", len(calExpRefList), patchRef.dataId)
314 calExpRefList = [calExpRef for calExpRef in calExpRefList if calExpRef.datasetExists(self.calexpType)]
315 self.log.info("Processing %d existing calexps for patch %s", len(calExpRefList), patchRef.dataId)
317 groupData = groupPatchExposures(patchRef, calExpRefList, self.getCoaddDatasetName(),
318 primaryWarpDataset)
319 self.log.info("Processing %d warp exposures for patch %s", len(groupData.groups), patchRef.dataId)
321 dataRefList = []
322 for i, (tempExpTuple, calexpRefList) in enumerate(groupData.groups.items()):
323 tempExpRef = getGroupDataRef(patchRef.getButler(), primaryWarpDataset,
324 tempExpTuple, groupData.keys)
325 if self.reuse and tempExpRef.datasetExists(datasetType=primaryWarpDataset, write=True):
326 self.log.info("Skipping makeCoaddTempExp for %s; output already exists.", tempExpRef.dataId)
327 dataRefList.append(tempExpRef)
328 continue
329 self.log.info("Processing Warp %d/%d: id=%s", i, len(groupData.groups), tempExpRef.dataId)
331 # TODO: mappers should define a way to go from the "grouping keys" to a numeric ID (#2776).
332 # For now, we try to get a long integer "visit" key, and if we can't, we just use the index
333 # of the visit in the list.
334 try:
335 visitId = int(tempExpRef.dataId["visit"])
336 except (KeyError, ValueError):
337 visitId = i
339 calExpList = []
340 ccdIdList = []
341 dataIdList = []
343 for calExpInd, calExpRef in enumerate(calexpRefList):
344 self.log.info("Reading calexp %s of %s for Warp id=%s", calExpInd+1, len(calexpRefList),
345 calExpRef.dataId)
346 try:
347 ccdId = calExpRef.get("ccdExposureId", immediate=True)
348 except Exception:
349 ccdId = calExpInd
350 try:
351 # We augment the dataRef here with the tract, which is harmless for loading things
352 # like calexps that don't need the tract, and necessary for meas_mosaic outputs,
353 # which do.
354 calExpRef = calExpRef.butlerSubset.butler.dataRef(self.calexpType,
355 dataId=calExpRef.dataId,
356 tract=skyInfo.tractInfo.getId())
357 calExp = self.getCalibratedExposure(calExpRef, bgSubtracted=self.config.bgSubtracted)
358 except Exception as e:
359 self.log.warn("Calexp %s not found; skipping it: %s", calExpRef.dataId, e)
360 continue
362 if self.config.doApplySkyCorr:
363 self.applySkyCorr(calExpRef, calExp)
365 calExpList.append(calExp)
366 ccdIdList.append(ccdId)
367 dataIdList.append(calExpRef.dataId)
369 exps = self.run(calExpList, ccdIdList, skyInfo, visitId, dataIdList).exposures
371 if any(exps.values()):
372 dataRefList.append(tempExpRef)
373 else:
374 self.log.warn("Warp %s could not be created", tempExpRef.dataId)
376 if self.config.doWrite:
377 for (warpType, exposure) in exps.items(): # compatible w/ Py3
378 if exposure is not None:
379 self.log.info("Persisting %s" % self.getTempExpDatasetName(warpType))
380 tempExpRef.put(exposure, self.getTempExpDatasetName(warpType))
382 return dataRefList
384 @pipeBase.timeMethod
385 def run(self, calExpList, ccdIdList, skyInfo, visitId=0, dataIdList=None, **kwargs):
386 """Create a Warp from inputs
388 We iterate over the multiple calexps in a single exposure to construct
389 the warp (previously called a coaddTempExp) of that exposure to the
390 supplied tract/patch.
392 Pixels that receive no pixels are set to NAN; this is not correct
393 (violates LSST algorithms group policy), but will be fixed up by
394 interpolating after the coaddition.
396 @param calexpRefList: List of data references for calexps that (may)
397 overlap the patch of interest
398 @param skyInfo: Struct from CoaddBaseTask.getSkyInfo() with geometric
399 information about the patch
400 @param visitId: integer identifier for visit, for the table that will
401 produce the CoaddPsf
402 @return a pipeBase Struct containing:
403 - exposures: a dictionary containing the warps requested:
404 "direct": direct warp if config.makeDirect
405 "psfMatched": PSF-matched warp if config.makePsfMatched
406 """
407 warpTypeList = self.getWarpTypeList()
409 totGoodPix = {warpType: 0 for warpType in warpTypeList}
410 didSetMetadata = {warpType: False for warpType in warpTypeList}
411 coaddTempExps = {warpType: self._prepareEmptyExposure(skyInfo) for warpType in warpTypeList}
412 inputRecorder = {warpType: self.inputRecorder.makeCoaddTempExpRecorder(visitId, len(calExpList))
413 for warpType in warpTypeList}
415 modelPsf = self.config.modelPsf.apply() if self.config.makePsfMatched else None
416 if dataIdList is None:
417 dataIdList = ccdIdList
419 for calExpInd, (calExp, ccdId, dataId) in enumerate(zip(calExpList, ccdIdList, dataIdList)):
420 self.log.info("Processing calexp %d of %d for this Warp: id=%s",
421 calExpInd+1, len(calExpList), dataId)
423 try:
424 warpedAndMatched = self.warpAndPsfMatch.run(calExp, modelPsf=modelPsf,
425 wcs=skyInfo.wcs, maxBBox=skyInfo.bbox,
426 makeDirect=self.config.makeDirect,
427 makePsfMatched=self.config.makePsfMatched)
428 except Exception as e:
429 self.log.warn("WarpAndPsfMatch failed for calexp %s; skipping it: %s", dataId, e)
430 continue
431 try:
432 numGoodPix = {warpType: 0 for warpType in warpTypeList}
433 for warpType in warpTypeList:
434 exposure = warpedAndMatched.getDict()[warpType]
435 if exposure is None:
436 continue
437 coaddTempExp = coaddTempExps[warpType]
438 if didSetMetadata[warpType]:
439 mimg = exposure.getMaskedImage()
440 mimg *= (coaddTempExp.getPhotoCalib().getInstFluxAtZeroMagnitude()
441 / exposure.getPhotoCalib().getInstFluxAtZeroMagnitude())
442 del mimg
443 numGoodPix[warpType] = coaddUtils.copyGoodPixels(
444 coaddTempExp.getMaskedImage(), exposure.getMaskedImage(), self.getBadPixelMask())
445 totGoodPix[warpType] += numGoodPix[warpType]
446 self.log.debug("Calexp %s has %d good pixels in this patch (%.1f%%) for %s",
447 dataId, numGoodPix[warpType],
448 100.0*numGoodPix[warpType]/skyInfo.bbox.getArea(), warpType)
449 if numGoodPix[warpType] > 0 and not didSetMetadata[warpType]:
450 coaddTempExp.setPhotoCalib(exposure.getPhotoCalib())
451 coaddTempExp.setFilterLabel(exposure.getFilterLabel())
452 coaddTempExp.getInfo().setVisitInfo(exposure.getInfo().getVisitInfo())
453 # PSF replaced with CoaddPsf after loop if and only if creating direct warp
454 coaddTempExp.setPsf(exposure.getPsf())
455 didSetMetadata[warpType] = True
457 # Need inputRecorder for CoaddApCorrMap for both direct and PSF-matched
458 inputRecorder[warpType].addCalExp(calExp, ccdId, numGoodPix[warpType])
460 except Exception as e:
461 self.log.warn("Error processing calexp %s; skipping it: %s", dataId, e)
462 continue
464 for warpType in warpTypeList:
465 self.log.info("%sWarp has %d good pixels (%.1f%%)",
466 warpType, totGoodPix[warpType], 100.0*totGoodPix[warpType]/skyInfo.bbox.getArea())
468 if totGoodPix[warpType] > 0 and didSetMetadata[warpType]:
469 inputRecorder[warpType].finish(coaddTempExps[warpType], totGoodPix[warpType])
470 if warpType == "direct":
471 coaddTempExps[warpType].setPsf(
472 CoaddPsf(inputRecorder[warpType].coaddInputs.ccds, skyInfo.wcs,
473 self.config.coaddPsf.makeControl()))
474 else:
475 if not self.config.doWriteEmptyWarps:
476 # No good pixels. Exposure still empty
477 coaddTempExps[warpType] = None
479 result = pipeBase.Struct(exposures=coaddTempExps)
480 return result
482 def getCalibratedExposure(self, dataRef, bgSubtracted):
483 """Return one calibrated Exposure, possibly with an updated SkyWcs.
485 @param[in] dataRef a sensor-level data reference
486 @param[in] bgSubtracted return calexp with background subtracted? If False get the
487 calexp's background background model and add it to the calexp.
488 @return calibrated exposure
490 @raises MissingExposureError If data for the exposure is not available.
492 If config.doApplyExternalPhotoCalib is `True`, the photometric calibration
493 (`photoCalib`) is taken from `config.externalPhotoCalibName` via the
494 `name_photoCalib` dataset. Otherwise, the photometric calibration is
495 retrieved from the processed exposure. When
496 `config.doApplyExternalSkyWcs` is `True`, the astrometric calibration
497 is taken from `config.externalSkyWcsName` with the `name_wcs` dataset.
498 Otherwise, the astrometric calibration is taken from the processed
499 exposure.
500 """
501 try:
502 exposure = dataRef.get(self.calexpType, immediate=True)
503 except dafPersist.NoResults as e:
504 raise MissingExposureError('Exposure not found: %s ' % str(e)) from e
506 if not bgSubtracted:
507 background = dataRef.get("calexpBackground", immediate=True)
508 mi = exposure.getMaskedImage()
509 mi += background.getImage()
510 del mi
512 if self.config.doApplyExternalPhotoCalib:
513 source = f"{self.config.externalPhotoCalibName}_photoCalib"
514 self.log.debug("Applying external photoCalib to %s from %s", dataRef.dataId, source)
515 photoCalib = dataRef.get(source)
516 exposure.setPhotoCalib(photoCalib)
517 else:
518 photoCalib = exposure.getPhotoCalib()
520 if self.config.doApplyExternalSkyWcs:
521 source = f"{self.config.externalSkyWcsName}_wcs"
522 self.log.debug("Applying external skyWcs to %s from %s", dataRef.dataId, source)
523 skyWcs = dataRef.get(source)
524 exposure.setWcs(skyWcs)
526 exposure.maskedImage = photoCalib.calibrateImage(exposure.maskedImage,
527 includeScaleUncertainty=self.config.includeCalibVar)
528 exposure.maskedImage /= photoCalib.getCalibrationMean()
529 # TODO: The images will have a calibration of 1.0 everywhere once RFC-545 is implemented.
530 # exposure.setCalib(afwImage.Calib(1.0))
531 return exposure
533 @staticmethod
534 def _prepareEmptyExposure(skyInfo):
535 """Produce an empty exposure for a given patch"""
536 exp = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
537 exp.getMaskedImage().set(numpy.nan, afwImage.Mask
538 .getPlaneBitMask("NO_DATA"), numpy.inf)
539 return exp
541 def getWarpTypeList(self):
542 """Return list of requested warp types per the config.
543 """
544 warpTypeList = []
545 if self.config.makeDirect:
546 warpTypeList.append("direct")
547 if self.config.makePsfMatched:
548 warpTypeList.append("psfMatched")
549 return warpTypeList
551 def applySkyCorr(self, dataRef, calexp):
552 """Apply correction to the sky background level
554 Sky corrections can be generated with the 'skyCorrection.py'
555 executable in pipe_drivers. Because the sky model used by that
556 code extends over the entire focal plane, this can produce
557 better sky subtraction.
559 The calexp is updated in-place.
561 Parameters
562 ----------
563 dataRef : `lsst.daf.persistence.ButlerDataRef`
564 Data reference for calexp.
565 calexp : `lsst.afw.image.Exposure` or `lsst.afw.image.MaskedImage`
566 Calibrated exposure.
567 """
568 bg = dataRef.get("skyCorr")
569 self.log.debug("Applying sky correction to %s", dataRef.dataId)
570 if isinstance(calexp, afwImage.Exposure):
571 calexp = calexp.getMaskedImage()
572 calexp -= bg.getImage()
575class MakeWarpConnections(pipeBase.PipelineTaskConnections,
576 dimensions=("tract", "patch", "skymap", "instrument", "visit"),
577 defaultTemplates={"coaddName": "deep",
578 "skyWcsName": "jointcal",
579 "photoCalibName": "fgcmcal"}):
580 calExpList = connectionTypes.Input(
581 doc="Input exposures to be resampled and optionally PSF-matched onto a SkyMap projection/patch",
582 name="calexp",
583 storageClass="ExposureF",
584 dimensions=("instrument", "visit", "detector"),
585 multiple=True,
586 deferLoad=True,
587 )
588 backgroundList = connectionTypes.Input(
589 doc="Input backgrounds to be added back into the calexp if bgSubtracted=False",
590 name="calexpBackground",
591 storageClass="Background",
592 dimensions=("instrument", "visit", "detector"),
593 multiple=True,
594 )
595 skyCorrList = connectionTypes.Input(
596 doc="Input Sky Correction to be subtracted from the calexp if doApplySkyCorr=True",
597 name="skyCorr",
598 storageClass="Background",
599 dimensions=("instrument", "visit", "detector"),
600 multiple=True,
601 )
602 skyMap = connectionTypes.Input(
603 doc="Input definition of geometry/bbox and projection/wcs for warped exposures",
604 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
605 storageClass="SkyMap",
606 dimensions=("skymap",),
607 )
608 externalSkyWcsTractCatalog = connectionTypes.Input(
609 doc=("Per-tract, per-visit wcs calibrations. These catalogs use the detector "
610 "id for the catalog id, sorted on id for fast lookup."),
611 name="{skyWcsName}SkyWcsCatalog",
612 storageClass="ExposureCatalog",
613 dimensions=("instrument", "visit", "tract"),
614 )
615 externalSkyWcsGlobalCatalog = connectionTypes.Input(
616 doc=("Per-visit wcs calibrations computed globally (with no tract information). "
617 "These catalogs use the detector id for the catalog id, sorted on id for "
618 "fast lookup."),
619 name="{skyWcsName}SkyWcsCatalog",
620 storageClass="ExposureCatalog",
621 dimensions=("instrument", "visit"),
622 )
623 externalPhotoCalibTractCatalog = connectionTypes.Input(
624 doc=("Per-tract, per-visit photometric calibrations. These catalogs use the "
625 "detector id for the catalog id, sorted on id for fast lookup."),
626 name="{photoCalibName}PhotoCalibCatalog",
627 storageClass="ExposureCatalog",
628 dimensions=("instrument", "visit", "tract"),
629 )
630 externalPhotoCalibGlobalCatalog = connectionTypes.Input(
631 doc=("Per-visit photometric calibrations computed globally (with no tract "
632 "information). These catalogs use the detector id for the catalog id, "
633 "sorted on id for fast lookup."),
634 name="{photoCalibName}PhotoCalibCatalog",
635 storageClass="ExposureCatalog",
636 dimensions=("instrument", "visit"),
637 )
638 direct = connectionTypes.Output(
639 doc=("Output direct warped exposure (previously called CoaddTempExp), produced by resampling ",
640 "calexps onto the skyMap patch geometry."),
641 name="{coaddName}Coadd_directWarp",
642 storageClass="ExposureF",
643 dimensions=("tract", "patch", "skymap", "visit", "instrument"),
644 )
645 psfMatched = connectionTypes.Output(
646 doc=("Output PSF-Matched warped exposure (previously called CoaddTempExp), produced by resampling ",
647 "calexps onto the skyMap patch geometry and PSF-matching to a model PSF."),
648 name="{coaddName}Coadd_psfMatchedWarp",
649 storageClass="ExposureF",
650 dimensions=("tract", "patch", "skymap", "visit", "instrument"),
651 )
652 # TODO DM-28769, have selectImages subtask indicate which connections they need:
653 wcsList = connectionTypes.Input(
654 doc="WCSs of calexps used by SelectImages subtask to determine if the calexp overlaps the patch",
655 name="calexp.wcs",
656 storageClass="Wcs",
657 dimensions=("instrument", "visit", "detector"),
658 multiple=True,
659 )
660 bboxList = connectionTypes.Input(
661 doc="BBoxes of calexps used by SelectImages subtask to determine if the calexp overlaps the patch",
662 name="calexp.bbox",
663 storageClass="Box2I",
664 dimensions=("instrument", "visit", "detector"),
665 multiple=True,
666 )
667 srcList = connectionTypes.Input(
668 doc="src catalogs used by PsfWcsSelectImages subtask to further select on PSF stability",
669 name="src",
670 storageClass="SourceCatalog",
671 dimensions=("instrument", "visit", "detector"),
672 multiple=True,
673 )
674 psfList = connectionTypes.Input(
675 doc="PSF models used by BestSeeingWcsSelectImages subtask to futher select on seeing",
676 name="calexp.psf",
677 storageClass="Psf",
678 dimensions=("instrument", "visit", "detector"),
679 multiple=True,
680 )
682 def __init__(self, *, config=None):
683 super().__init__(config=config)
684 if config.bgSubtracted:
685 self.inputs.remove("backgroundList")
686 if not config.doApplySkyCorr:
687 self.inputs.remove("skyCorrList")
688 if config.doApplyExternalSkyWcs:
689 if config.useGlobalExternalSkyWcs:
690 self.inputs.remove("externalSkyWcsTractCatalog")
691 else:
692 self.inputs.remove("externalSkyWcsGlobalCatalog")
693 else:
694 self.inputs.remove("externalSkyWcsTractCatalog")
695 self.inputs.remove("externalSkyWcsGlobalCatalog")
696 if config.doApplyExternalPhotoCalib:
697 if config.useGlobalExternalPhotoCalib:
698 self.inputs.remove("externalPhotoCalibTractCatalog")
699 else:
700 self.inputs.remove("externalPhotoCalibGlobalCatalog")
701 else:
702 self.inputs.remove("externalPhotoCalibTractCatalog")
703 self.inputs.remove("externalPhotoCalibGlobalCatalog")
704 if not config.makeDirect:
705 self.outputs.remove("direct")
706 if not config.makePsfMatched:
707 self.outputs.remove("psfMatched")
708 # TODO DM-28769: add connection per selectImages connections
709 # instead of removing if not PsfWcsSelectImagesTask here:
710 if config.select.target != lsst.pipe.tasks.selectImages.PsfWcsSelectImagesTask:
711 self.inputs.remove("srcList")
712 if config.select.target != lsst.pipe.tasks.selectImages.BestSeeingWcsSelectImagesTask:
713 self.inputs.remove("psfList")
716class MakeWarpConfig(pipeBase.PipelineTaskConfig, MakeCoaddTempExpConfig,
717 pipelineConnections=MakeWarpConnections):
719 def validate(self):
720 super().validate()
723class MakeWarpTask(MakeCoaddTempExpTask):
724 """Warp and optionally PSF-Match calexps onto an a common projection
725 """
726 ConfigClass = MakeWarpConfig
727 _DefaultName = "makeWarp"
729 @utils.inheritDoc(pipeBase.PipelineTask)
730 def runQuantum(self, butlerQC, inputRefs, outputRefs):
731 """
732 Notes
733 ----
734 Construct warps for requested warp type for single epoch
736 PipelineTask (Gen3) entry point to warp and optionally PSF-match
737 calexps. This method is analogous to `runDataRef`.
738 """
739 # Read in all inputs.
740 inputs = butlerQC.get(inputRefs)
742 # Construct skyInfo expected by `run`. We remove the SkyMap itself
743 # from the dictionary so we can pass it as kwargs later.
744 skyMap = inputs.pop("skyMap")
745 quantumDataId = butlerQC.quantum.dataId
746 skyInfo = makeSkyInfo(skyMap, tractId=quantumDataId['tract'], patchId=quantumDataId['patch'])
748 # Construct list of input DataIds expected by `run`
749 dataIdList = [ref.datasetRef.dataId for ref in inputRefs.calExpList]
750 # Construct list of packed integer IDs expected by `run`
751 ccdIdList = [dataId.pack("visit_detector") for dataId in dataIdList]
753 # Run the selector and filter out calexps that were not selected
754 # primarily because they do not overlap the patch
755 cornerPosList = lsst.geom.Box2D(skyInfo.bbox).getCorners()
756 coordList = [skyInfo.wcs.pixelToSky(pos) for pos in cornerPosList]
757 goodIndices = self.select.run(**inputs, coordList=coordList, dataIds=dataIdList)
758 inputs = self.filterInputs(indices=goodIndices, inputs=inputs)
760 # Read from disk only the selected calexps
761 inputs['calExpList'] = [ref.get() for ref in inputs['calExpList']]
763 # Extract integer visitId requested by `run`
764 visits = [dataId['visit'] for dataId in dataIdList]
765 visitId = visits[0]
767 if self.config.doApplyExternalSkyWcs:
768 if self.config.useGlobalExternalSkyWcs:
769 externalSkyWcsCatalog = inputs.pop("externalSkyWcsGlobalCatalog")
770 else:
771 externalSkyWcsCatalog = inputs.pop("externalSkyWcsTractCatalog")
772 else:
773 externalSkyWcsCatalog = None
775 if self.config.doApplyExternalPhotoCalib:
776 if self.config.useGlobalExternalPhotoCalib:
777 externalPhotoCalibCatalog = inputs.pop("externalPhotoCalibGlobalCatalog")
778 else:
779 externalPhotoCalibCatalog = inputs.pop("externalPhotoCalibTractCatalog")
780 else:
781 externalPhotoCalibCatalog = None
783 self.prepareCalibratedExposures(**inputs, externalSkyWcsCatalog=externalSkyWcsCatalog,
784 externalPhotoCalibCatalog=externalPhotoCalibCatalog)
786 results = self.run(**inputs, visitId=visitId,
787 ccdIdList=[ccdIdList[i] for i in goodIndices],
788 dataIdList=[dataIdList[i] for i in goodIndices],
789 skyInfo=skyInfo)
790 if self.config.makeDirect:
791 butlerQC.put(results.exposures["direct"], outputRefs.direct)
792 if self.config.makePsfMatched:
793 butlerQC.put(results.exposures["psfMatched"], outputRefs.psfMatched)
795 def filterInputs(self, indices, inputs):
796 """Return task inputs with their lists filtered by indices
798 Parameters
799 ----------
800 indices : `list` of integers
801 inputs : `dict` of `list` of input connections to be passed to run
802 """
803 for key in inputs.keys():
804 # Only down-select on list inputs
805 if isinstance(inputs[key], list):
806 inputs[key] = [inputs[key][ind] for ind in indices]
807 return inputs
809 def prepareCalibratedExposures(self, calExpList, backgroundList=None, skyCorrList=None,
810 externalSkyWcsCatalog=None, externalPhotoCalibCatalog=None,
811 **kwargs):
812 """Calibrate and add backgrounds to input calExpList in place
814 Parameters
815 ----------
816 calExpList : `list` of `lsst.afw.image.Exposure`
817 Sequence of calexps to be modified in place
818 backgroundList : `list` of `lsst.afw.math.backgroundList`, optional
819 Sequence of backgrounds to be added back in if bgSubtracted=False
820 skyCorrList : `list` of `lsst.afw.math.backgroundList`, optional
821 Sequence of background corrections to be subtracted if doApplySkyCorr=True
822 externalSkyWcsCatalog : `lsst.afw.table.ExposureCatalog`, optional
823 Exposure catalog with external skyWcs to be applied
824 if config.doApplyExternalSkyWcs=True. Catalog uses the detector id
825 for the catalog id, sorted on id for fast lookup.
826 externalPhotoCalibCatalog : `lsst.afw.table.ExposureCatalog`, optional
827 Exposure catalog with external photoCalib to be applied
828 if config.doApplyExternalPhotoCalib=True. Catalog uses the detector
829 id for the catalog id, sorted on id for fast lookup.
830 """
831 backgroundList = len(calExpList)*[None] if backgroundList is None else backgroundList
832 skyCorrList = len(calExpList)*[None] if skyCorrList is None else skyCorrList
834 includeCalibVar = self.config.includeCalibVar
836 for calexp, background, skyCorr in zip(calExpList, backgroundList, skyCorrList):
837 mi = calexp.maskedImage
838 if not self.config.bgSubtracted:
839 mi += background.getImage()
841 if externalSkyWcsCatalog is not None or externalPhotoCalibCatalog is not None:
842 detectorId = calexp.getInfo().getDetector().getId()
844 # Find the external photoCalib
845 if externalPhotoCalibCatalog is not None:
846 row = externalPhotoCalibCatalog.find(detectorId)
847 if row is None:
848 raise RuntimeError(f"Detector id {detectorId} not found in "
849 f"externalPhotoCalibCatalog.")
850 photoCalib = row.getPhotoCalib()
851 if photoCalib is None:
852 raise RuntimeError(f"Detector id {detectorId} has None for photoCalib "
853 f"in externalPhotoCalibCatalog.")
854 else:
855 photoCalib = calexp.getPhotoCalib()
857 # Find and apply external skyWcs
858 if externalSkyWcsCatalog is not None:
859 row = externalSkyWcsCatalog.find(detectorId)
860 if row is None:
861 raise RuntimeError(f"Detector id {detectorId} not found in externalSkyWcsCatalog.")
862 skyWcs = row.getWcs()
863 if skyWcs is None:
864 raise RuntimeError(f"Detector id {detectorId} has None for WCS "
865 f" in externalSkyWcsCatalog.")
866 calexp.setWcs(skyWcs)
868 # Calibrate the image
869 calexp.maskedImage = photoCalib.calibrateImage(calexp.maskedImage,
870 includeScaleUncertainty=includeCalibVar)
871 calexp.maskedImage /= photoCalib.getCalibrationMean()
872 # TODO: The images will have a calibration of 1.0 everywhere once RFC-545 is implemented.
873 # exposure.setCalib(afwImage.Calib(1.0))
875 # Apply skycorr
876 if self.config.doApplySkyCorr:
877 mi -= skyCorr.getImage()