23 from __future__
import absolute_import, division, print_function
26 import lsst.pex.config
as pexConfig
27 import lsst.afw.image
as afwImage
28 import lsst.coadd.utils
as coaddUtils
29 import lsst.pipe.base
as pipeBase
30 from lsst.meas.algorithms
import CoaddPsf
31 from .coaddBase
import CoaddBaseTask
32 from .warpAndPsfMatch
import WarpAndPsfMatchTask
33 from .coaddHelpers
import groupPatchExposures, getGroupDataRef
35 __all__ = [
"MakeCoaddTempExpTask"]
39 """Config for MakeCoaddTempExpTask
41 warpAndPsfMatch = pexConfig.ConfigurableField(
42 target=WarpAndPsfMatchTask,
43 doc=
"Task to warp and PSF-match calexp",
45 doWrite = pexConfig.Field(
46 doc=
"persist <coaddName>Coadd_<warpType>Warp",
50 doOverwrite = pexConfig.Field(
51 doc=
"overwrite <coaddName>Coadd_<warpType>Warp; If False, continue if the file exists on disk",
55 bgSubtracted = pexConfig.Field(
56 doc=
"Work with a background subtracted calexp?",
70 """!Warp and optionally PSF-Match calexps onto an a common projection.
72 @anchor MakeCoaddTempExpTask_
74 @section pipe_tasks_makeCoaddTempExp_Contents Contents
76 - @ref pipe_tasks_makeCoaddTempExp_Purpose
77 - @ref pipe_tasks_makeCoaddTempExp_Initialize
78 - @ref pipe_tasks_makeCoaddTempExp_IO
79 - @ref pipe_tasks_makeCoaddTempExp_Config
80 - @ref pipe_tasks_makeCoaddTempExp_Debug
81 - @ref pipe_tasks_makeCoaddTempExp_Example
83 @section pipe_tasks_makeCoaddTempExp_Purpose Description
85 Warp and optionally PSF-Match calexps onto a common projection, by
86 performing the following operations:
87 - Group calexps by visit/run
88 - For each visit, generate a Warp by calling method @ref makeTempExp.
89 makeTempExp loops over the visit's calexps calling @ref WarpAndPsfMatch
92 The result is a `directWarp` (and/or optionally a `psfMatchedWarp`).
94 @section pipe_tasks_makeCoaddTempExp_Initialize Task Initialization
98 This task has no special keyword arguments.
100 @section pipe_tasks_makeCoaddTempExp_IO Invoking the Task
102 This task is primarily designed to be run from the command line.
104 The main method is `run`, which takes a single butler data reference for the patch(es)
109 WarpType identifies the types of convolutions applied to Warps (previously CoaddTempExps).
110 Only two types are available: direct (for regular Warps/Coadds) and psfMatched
111 (for Warps/Coadds with homogenized PSFs). We expect to add a third type, likelihood,
112 for generating likelihood Coadds with Warps that have been correlated with their own PSF.
114 @section pipe_tasks_makeCoaddTempExp_Config Configuration parameters
116 See @ref MakeCoaddTempExpConfig and parameters inherited from
117 @link lsst.pipe.tasks.coaddBase.CoaddBaseConfig CoaddBaseConfig @endlink
119 @subsection pipe_tasks_MakeCoaddTempExp_psfMatching Guide to PSF-Matching Configs
121 To make `psfMatchedWarps`, select `config.makePsfMatched=True`. The subtask
122 @link lsst.ip.diffim.modelPsfMatch.ModelPsfMatchTask ModelPsfMatchTask @endlink
123 is responsible for the PSF-Matching, and its config is accessed via `config.warpAndPsfMatch.psfMatch`.
124 The optimal configuration depends on aspects of dataset: the pixel scale, average PSF FWHM and
125 dimensions of the PSF kernel. These configs include the requested model PSF, the matching kernel size,
126 padding of the science PSF thumbnail and spatial sampling frequency of the PSF.
128 *Config Guidelines*: The user must specify the size of the model PSF to which to match by setting
129 `config.modelPsf.defaultFwhm` in units of pixels. The appropriate values depends on science case.
130 In general, for a set of input images, this config should equal the FWHM of the visit
131 with the worst seeing. The smallest it should be set to is the median FWHM. The defaults
132 of the other config options offer a reasonable starting point.
133 The following list presents the most common problems that arise from a misconfigured
134 @link lsst.ip.diffim.modelPsfMatch.ModelPsfMatchTask ModelPsfMatchTask @endlink
135 and corresponding solutions. All assume the default Alard-Lupton kernel, with configs accessed via
136 ```config.warpAndPsfMatch.psfMatch.kernel['AL']```. Each item in the list is formatted as:
137 Problem: Explanation. *Solution*
139 *Troublshooting PSF-Matching Configuration:*
140 - Matched PSFs look boxy: The matching kernel is too small. _Increase the matching kernel size.
143 config.warpAndPsfMatch.psfMatch.kernel['AL'].kernelSize=27 # default 21
145 Note that increasing the kernel size also increases runtime.
146 - Matched PSFs look ugly (dipoles, quadropoles, donuts): unable to find good solution
147 for matching kernel. _Provide the matcher with more data by either increasing
148 the spatial sampling by decreasing the spatial cell size,_
150 config.warpAndPsfMatch.psfMatch.kernel['AL'].sizeCellX = 64 # default 128
151 config.warpAndPsfMatch.psfMatch.kernel['AL'].sizeCellY = 64 # default 128
153 _or increasing the padding around the Science PSF, for example:_
155 config.warpAndPsfMatch.psfMatch.autoPadPsfTo=1.6 # default 1.4
157 Increasing `autoPadPsfTo` increases the minimum ratio of input PSF dimensions to the
158 matching kernel dimensions, thus increasing the number of pixels available to fit
159 after convolving the PSF with the matching kernel.
160 Optionally, for debugging the effects of padding, the level of padding may be manually
161 controlled by setting turning off the automatic padding and setting the number
162 of pixels by which to pad the PSF:
164 config.warpAndPsfMatch.psfMatch.doAutoPadPsf = False # default True
165 config.warpAndPsfMatch.psfMatch.padPsfBy = 6 # pixels. default 0
167 - Deconvolution: Matching a large PSF to a smaller PSF produces
168 a telltale noise pattern which looks like ripples or a brain.
169 _Increase the size of the requested model PSF. For example:_
171 config.modelPsf.defaultFwhm = 11 # Gaussian sigma in units of pixels.
173 - High frequency (sometimes checkered) noise: The matching basis functions are too small.
174 _Increase the width of the Gaussian basis functions. For example:_
176 config.warpAndPsfMatch.psfMatch.kernel['AL'].alardSigGauss=[1.5, 3.0, 6.0]
177 # from default [0.7, 1.5, 3.0]
180 @section pipe_tasks_makeCoaddTempExp_Debug Debug variables
182 MakeCoaddTempExpTask has no debug output, but its subtasks do.
184 @section pipe_tasks_makeCoaddTempExp_Example A complete example of using MakeCoaddTempExpTask
186 This example uses the package ci_hsc to show how MakeCoaddTempExp fits
187 into the larger Data Release Processing.
192 # if not built already:
193 python $(which scons) # this will take a while
195 The following assumes that `processCcd.py` and `makeSkyMap.py` have previously been run
196 (e.g. by building `ci_hsc` above) to generate a repository of calexps and an
197 output respository with the desired SkyMap. The command,
199 makeCoaddTempExp.py $CI_HSC_DIR/DATA --rerun ci_hsc \
200 --id patch=5,4 tract=0 filter=HSC-I \
201 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 \
202 --selectId visit=903988 ccd=23 --selectId visit=903988 ccd=24 \
203 --config doApplyUberCal=False makePsfMatched=True modelPsf.defaultFwhm=11
205 writes a direct and PSF-Matched Warp to
206 - `$CI_HSC_DIR/DATA/rerun/ci_hsc/deepCoadd/HSC-I/0/5,4/warp-HSC-I-0-5,4-903988.fits` and
207 - `$CI_HSC_DIR/DATA/rerun/ci_hsc/deepCoadd/HSC-I/0/5,4/psfMatchedWarp-HSC-I-0-5,4-903988.fits`
210 @note PSF-Matching in this particular dataset would benefit from adding
211 `--configfile ./matchingConfig.py` to
212 the command line arguments where `matchingConfig.py` is defined by:
215 config.warpAndPsfMatch.psfMatch.kernel['AL'].kernelSize=27
216 config.warpAndPsfMatch.psfMatch.kernel['AL'].alardSigGauss=[1.5, 3.0, 6.0]" > matchingConfig.py
219 Add the option `--help` to see more options.
221 ConfigClass = MakeCoaddTempExpConfig
222 _DefaultName =
"makeCoaddTempExp"
225 CoaddBaseTask.__init__(self, *args, **kwargs)
226 self.makeSubtask(
"warpAndPsfMatch")
229 def run(self, patchRef, selectDataList=[]):
230 """!Produce <coaddName>Coadd_<warpType>Warp images by warping and optionally PSF-matching.
232 @param[in] patchRef: data reference for sky map patch. Must include keys "tract", "patch",
233 plus the camera-specific filter key (e.g. "filter" or "band")
234 @return: dataRefList: a list of data references for the new <coaddName>Coadd_directWarps
235 if direct or both warp types are requested and <coaddName>Coadd_psfMatchedWarps if only psfMatched
238 @warning: this task assumes that all exposures in a warp (coaddTempExp) have the same filter.
240 @warning: this task sets the Calib of the coaddTempExp to the Calib of the first calexp
241 with any good pixels in the patch. For a mosaic camera the resulting Calib should be ignored
242 (assembleCoadd should determine zeropoint scaling without referring to it).
244 skyInfo = self.getSkyInfo(patchRef)
247 if self.config.makePsfMatched
and not self.config.makeDirect:
248 primaryWarpDataset = self.getTempExpDatasetName(
"psfMatched")
250 primaryWarpDataset = self.getTempExpDatasetName(
"direct")
252 calExpRefList = self.selectExposures(patchRef, skyInfo, selectDataList=selectDataList)
253 if len(calExpRefList) == 0:
254 self.log.warn(
"No exposures to coadd for patch %s", patchRef.dataId)
256 self.log.info(
"Selected %d calexps for patch %s", len(calExpRefList), patchRef.dataId)
257 calExpRefList = [calExpRef
for calExpRef
in calExpRefList
if calExpRef.datasetExists(
"calexp")]
258 self.log.info(
"Processing %d existing calexps for patch %s", len(calExpRefList), patchRef.dataId)
262 self.log.info(
"Processing %d warp exposures for patch %s", len(groupData.groups), patchRef.dataId)
265 for i, (tempExpTuple, calexpRefList)
in enumerate(groupData.groups.items()):
267 tempExpTuple, groupData.keys)
268 if not self.config.doOverwrite
and tempExpRef.datasetExists(datasetType=primaryWarpDataset):
269 self.log.info(
"Warp %s exists; skipping", tempExpRef.dataId)
270 dataRefList.append(tempExpRef)
272 self.log.info(
"Processing Warp %d/%d: id=%s", i, len(groupData.groups), tempExpRef.dataId)
278 visitId = int(tempExpRef.dataId[
"visit"])
279 except (KeyError, ValueError):
282 exps = self.
createTempExp(calexpRefList, skyInfo, visitId).exposures
284 if any(exps.values()):
285 dataRefList.append(tempExpRef)
287 self.log.warn(
"Warp %s could not be created", tempExpRef.dataId)
289 if self.config.doWrite:
290 for (warpType, exposure)
in exps.items():
291 if exposure
is not None:
292 self.log.info(
"Persisting %s" % self.getTempExpDatasetName(warpType))
293 tempExpRef.put(exposure, self.getTempExpDatasetName(warpType))
298 """Create a Warp from inputs
300 We iterate over the multiple calexps in a single exposure to construct
301 the warp (previously called a coaddTempExp) of that exposure to the
302 supplied tract/patch.
304 Pixels that receive no pixels are set to NAN; this is not correct
305 (violates LSST algorithms group policy), but will be fixed up by
306 interpolating after the coaddition.
308 @param calexpRefList: List of data references for calexps that (may)
309 overlap the patch of interest
310 @param skyInfo: Struct from CoaddBaseTask.getSkyInfo() with geometric
311 information about the patch
312 @param visitId: integer identifier for visit, for the table that will
314 @return a pipeBase Struct containing:
315 - exposures: a dictionary containing the warps requested:
316 "direct": direct warp if config.makeDirect
317 "psfMatched": PSF-matched warp if config.makePsfMatched
319 warpTypeList = self.getWarpTypeList()
321 totGoodPix = {warpType: 0
for warpType
in warpTypeList}
322 didSetMetadata = {warpType:
False for warpType
in warpTypeList}
324 inputRecorder = {warpType: self.inputRecorder.makeCoaddTempExpRecorder(visitId, len(calexpRefList))
325 for warpType
in warpTypeList}
327 modelPsf = self.config.modelPsf.apply()
if self.config.makePsfMatched
else None
328 for calExpInd, calExpRef
in enumerate(calexpRefList):
329 self.log.info(
"Processing calexp %d of %d for this Warp: id=%s",
330 calExpInd+1, len(calexpRefList), calExpRef.dataId)
332 ccdId = calExpRef.get(
"ccdExposureId", immediate=
True)
339 calExpRef = calExpRef.butlerSubset.butler.dataRef(
"calexp", dataId=calExpRef.dataId,
340 tract=skyInfo.tractInfo.getId())
341 calExp = self.getCalExp(calExpRef, bgSubtracted=self.config.bgSubtracted)
342 except Exception
as e:
343 self.log.warn(
"Calexp %s not found; skipping it: %s", calExpRef.dataId, e)
346 warpedAndMatched = self.warpAndPsfMatch.run(calExp, modelPsf=modelPsf,
347 wcs=skyInfo.wcs, maxBBox=skyInfo.bbox,
348 makeDirect=self.config.makeDirect,
349 makePsfMatched=self.config.makePsfMatched)
350 except Exception
as e:
351 self.log.warn(
"WarpAndPsfMatch failed for calexp %s; skipping it: %s", calExpRef.dataId, e)
354 numGoodPix = {warpType: 0
for warpType
in warpTypeList}
355 for warpType
in warpTypeList:
356 exposure = warpedAndMatched.getDict()[warpType]
359 coaddTempExp = coaddTempExps[warpType]
360 if didSetMetadata[warpType]:
361 mimg = exposure.getMaskedImage()
362 mimg *= (coaddTempExp.getCalib().getFluxMag0()[0] /
363 exposure.getCalib().getFluxMag0()[0])
365 numGoodPix[warpType] = coaddUtils.copyGoodPixels(
366 coaddTempExp.getMaskedImage(), exposure.getMaskedImage(), self.getBadPixelMask())
367 totGoodPix[warpType] += numGoodPix[warpType]
368 self.log.debug(
"Calexp %s has %d good pixels in this patch (%.1f%%) for %s",
369 calExpRef.dataId, numGoodPix[warpType],
370 100.0*numGoodPix[warpType]/skyInfo.bbox.getArea(), warpType)
371 if numGoodPix[warpType] > 0
and not didSetMetadata[warpType]:
372 coaddTempExp.setCalib(exposure.getCalib())
373 coaddTempExp.setFilter(exposure.getFilter())
375 coaddTempExp.setPsf(exposure.getPsf())
376 didSetMetadata[warpType] =
True
379 inputRecorder[warpType].addCalExp(calExp, ccdId, numGoodPix[warpType])
381 except Exception
as e:
382 self.log.warn(
"Error processing calexp %s; skipping it: %s", calExpRef.dataId, e)
385 for warpType
in warpTypeList:
386 self.log.info(
"%sWarp has %d good pixels (%.1f%%)",
387 warpType, totGoodPix[warpType], 100.0*totGoodPix[warpType]/skyInfo.bbox.getArea())
389 if totGoodPix[warpType] > 0
and didSetMetadata[warpType]:
390 inputRecorder[warpType].finish(coaddTempExps[warpType], totGoodPix[warpType])
391 if warpType ==
"direct":
392 coaddTempExps[warpType].setPsf(
393 CoaddPsf(inputRecorder[warpType].coaddInputs.ccds, skyInfo.wcs))
396 coaddTempExps[warpType] =
None
398 result = pipeBase.Struct(exposures=coaddTempExps)
401 def _prepareEmptyExposure(cls, skyInfo):
402 """Produce an empty exposure for a given patch"""
403 exp = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
404 exp.getMaskedImage().set(numpy.nan, afwImage.MaskU.getPlaneBitMask(
"NO_DATA"), numpy.inf)
def _prepareEmptyExposure
def run
Produce <coaddName>Coadd_<warpType>Warp images by warping and optionally PSF-matching.
Warp and optionally PSF-Match calexps onto an a common projection.