23 from __future__
import absolute_import, division, print_function
26 import lsst.pex.config
as pexConfig
27 import lsst.afw.image
as afwImage
28 import lsst.coadd.utils
as coaddUtils
29 import lsst.pipe.base
as pipeBase
30 from lsst.meas.algorithms
import CoaddPsf, CoaddPsfConfig
31 from .coaddBase
import CoaddBaseTask
32 from .warpAndPsfMatch
import WarpAndPsfMatchTask
33 from .coaddHelpers
import groupPatchExposures, getGroupDataRef
35 __all__ = [
"MakeCoaddTempExpTask"]
39 """Config for MakeCoaddTempExpTask 41 warpAndPsfMatch = pexConfig.ConfigurableField(
42 target=WarpAndPsfMatchTask,
43 doc=
"Task to warp and PSF-match calexp",
45 doWrite = pexConfig.Field(
46 doc=
"persist <coaddName>Coadd_<warpType>Warp",
50 doOverwrite = pexConfig.Field(
51 doc=
"overwrite <coaddName>Coadd_<warpType>Warp; If False, continue if the file exists on disk",
55 bgSubtracted = pexConfig.Field(
56 doc=
"Work with a background subtracted calexp?",
60 coaddPsf = pexConfig.ConfigField(
61 doc=
"Configuration for CoaddPsf",
74 """!Warp and optionally PSF-Match calexps onto an a common projection. 76 @anchor MakeCoaddTempExpTask_ 78 @section pipe_tasks_makeCoaddTempExp_Contents Contents 80 - @ref pipe_tasks_makeCoaddTempExp_Purpose 81 - @ref pipe_tasks_makeCoaddTempExp_Initialize 82 - @ref pipe_tasks_makeCoaddTempExp_IO 83 - @ref pipe_tasks_makeCoaddTempExp_Config 84 - @ref pipe_tasks_makeCoaddTempExp_Debug 85 - @ref pipe_tasks_makeCoaddTempExp_Example 87 @section pipe_tasks_makeCoaddTempExp_Purpose Description 89 Warp and optionally PSF-Match calexps onto a common projection, by 90 performing the following operations: 91 - Group calexps by visit/run 92 - For each visit, generate a Warp by calling method @ref makeTempExp. 93 makeTempExp loops over the visit's calexps calling @ref WarpAndPsfMatch 96 The result is a `directWarp` (and/or optionally a `psfMatchedWarp`). 98 @section pipe_tasks_makeCoaddTempExp_Initialize Task Initialization 100 @copydoc \_\_init\_\_ 102 This task has no special keyword arguments. 104 @section pipe_tasks_makeCoaddTempExp_IO Invoking the Task 106 This task is primarily designed to be run from the command line. 108 The main method is `run`, which takes a single butler data reference for the patch(es) 113 WarpType identifies the types of convolutions applied to Warps (previously CoaddTempExps). 114 Only two types are available: direct (for regular Warps/Coadds) and psfMatched 115 (for Warps/Coadds with homogenized PSFs). We expect to add a third type, likelihood, 116 for generating likelihood Coadds with Warps that have been correlated with their own PSF. 118 @section pipe_tasks_makeCoaddTempExp_Config Configuration parameters 120 See @ref MakeCoaddTempExpConfig and parameters inherited from 121 @link lsst.pipe.tasks.coaddBase.CoaddBaseConfig CoaddBaseConfig @endlink 123 @subsection pipe_tasks_MakeCoaddTempExp_psfMatching Guide to PSF-Matching Configs 125 To make `psfMatchedWarps`, select `config.makePsfMatched=True`. The subtask 126 @link lsst.ip.diffim.modelPsfMatch.ModelPsfMatchTask ModelPsfMatchTask @endlink 127 is responsible for the PSF-Matching, and its config is accessed via `config.warpAndPsfMatch.psfMatch`. 128 The optimal configuration depends on aspects of dataset: the pixel scale, average PSF FWHM and 129 dimensions of the PSF kernel. These configs include the requested model PSF, the matching kernel size, 130 padding of the science PSF thumbnail and spatial sampling frequency of the PSF. 132 *Config Guidelines*: The user must specify the size of the model PSF to which to match by setting 133 `config.modelPsf.defaultFwhm` in units of pixels. The appropriate values depends on science case. 134 In general, for a set of input images, this config should equal the FWHM of the visit 135 with the worst seeing. The smallest it should be set to is the median FWHM. The defaults 136 of the other config options offer a reasonable starting point. 137 The following list presents the most common problems that arise from a misconfigured 138 @link lsst.ip.diffim.modelPsfMatch.ModelPsfMatchTask ModelPsfMatchTask @endlink 139 and corresponding solutions. All assume the default Alard-Lupton kernel, with configs accessed via 140 ```config.warpAndPsfMatch.psfMatch.kernel['AL']```. Each item in the list is formatted as: 141 Problem: Explanation. *Solution* 143 *Troublshooting PSF-Matching Configuration:* 144 - Matched PSFs look boxy: The matching kernel is too small. _Increase the matching kernel size. 147 config.warpAndPsfMatch.psfMatch.kernel['AL'].kernelSize=27 # default 21 149 Note that increasing the kernel size also increases runtime. 150 - Matched PSFs look ugly (dipoles, quadropoles, donuts): unable to find good solution 151 for matching kernel. _Provide the matcher with more data by either increasing 152 the spatial sampling by decreasing the spatial cell size,_ 154 config.warpAndPsfMatch.psfMatch.kernel['AL'].sizeCellX = 64 # default 128 155 config.warpAndPsfMatch.psfMatch.kernel['AL'].sizeCellY = 64 # default 128 157 _or increasing the padding around the Science PSF, for example:_ 159 config.warpAndPsfMatch.psfMatch.autoPadPsfTo=1.6 # default 1.4 161 Increasing `autoPadPsfTo` increases the minimum ratio of input PSF dimensions to the 162 matching kernel dimensions, thus increasing the number of pixels available to fit 163 after convolving the PSF with the matching kernel. 164 Optionally, for debugging the effects of padding, the level of padding may be manually 165 controlled by setting turning off the automatic padding and setting the number 166 of pixels by which to pad the PSF: 168 config.warpAndPsfMatch.psfMatch.doAutoPadPsf = False # default True 169 config.warpAndPsfMatch.psfMatch.padPsfBy = 6 # pixels. default 0 171 - Deconvolution: Matching a large PSF to a smaller PSF produces 172 a telltale noise pattern which looks like ripples or a brain. 173 _Increase the size of the requested model PSF. For example:_ 175 config.modelPsf.defaultFwhm = 11 # Gaussian sigma in units of pixels. 177 - High frequency (sometimes checkered) noise: The matching basis functions are too small. 178 _Increase the width of the Gaussian basis functions. For example:_ 180 config.warpAndPsfMatch.psfMatch.kernel['AL'].alardSigGauss=[1.5, 3.0, 6.0] 181 # from default [0.7, 1.5, 3.0] 184 @section pipe_tasks_makeCoaddTempExp_Debug Debug variables 186 MakeCoaddTempExpTask has no debug output, but its subtasks do. 188 @section pipe_tasks_makeCoaddTempExp_Example A complete example of using MakeCoaddTempExpTask 190 This example uses the package ci_hsc to show how MakeCoaddTempExp fits 191 into the larger Data Release Processing. 196 # if not built already: 197 python $(which scons) # this will take a while 199 The following assumes that `processCcd.py` and `makeSkyMap.py` have previously been run 200 (e.g. by building `ci_hsc` above) to generate a repository of calexps and an 201 output respository with the desired SkyMap. The command, 203 makeCoaddTempExp.py $CI_HSC_DIR/DATA --rerun ci_hsc \ 204 --id patch=5,4 tract=0 filter=HSC-I \ 205 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 \ 206 --selectId visit=903988 ccd=23 --selectId visit=903988 ccd=24 \ 207 --config doApplyUberCal=False makePsfMatched=True modelPsf.defaultFwhm=11 209 writes a direct and PSF-Matched Warp to 210 - `$CI_HSC_DIR/DATA/rerun/ci_hsc/deepCoadd/HSC-I/0/5,4/warp-HSC-I-0-5,4-903988.fits` and 211 - `$CI_HSC_DIR/DATA/rerun/ci_hsc/deepCoadd/HSC-I/0/5,4/psfMatchedWarp-HSC-I-0-5,4-903988.fits` 214 @note PSF-Matching in this particular dataset would benefit from adding 215 `--configfile ./matchingConfig.py` to 216 the command line arguments where `matchingConfig.py` is defined by: 219 config.warpAndPsfMatch.psfMatch.kernel['AL'].kernelSize=27 220 config.warpAndPsfMatch.psfMatch.kernel['AL'].alardSigGauss=[1.5, 3.0, 6.0]" > matchingConfig.py 223 Add the option `--help` to see more options. 225 ConfigClass = MakeCoaddTempExpConfig
226 _DefaultName =
"makeCoaddTempExp" 229 CoaddBaseTask.__init__(self, *args, **kwargs)
230 self.makeSubtask(
"warpAndPsfMatch")
233 def run(self, patchRef, selectDataList=[]):
234 """!Produce <coaddName>Coadd_<warpType>Warp images by warping and optionally PSF-matching. 236 @param[in] patchRef: data reference for sky map patch. Must include keys "tract", "patch", 237 plus the camera-specific filter key (e.g. "filter" or "band") 238 @return: dataRefList: a list of data references for the new <coaddName>Coadd_directWarps 239 if direct or both warp types are requested and <coaddName>Coadd_psfMatchedWarps if only psfMatched 242 @warning: this task assumes that all exposures in a warp (coaddTempExp) have the same filter. 244 @warning: this task sets the Calib of the coaddTempExp to the Calib of the first calexp 245 with any good pixels in the patch. For a mosaic camera the resulting Calib should be ignored 246 (assembleCoadd should determine zeropoint scaling without referring to it). 251 if self.config.makePsfMatched
and not self.config.makeDirect:
256 calExpRefList = self.
selectExposures(patchRef, skyInfo, selectDataList=selectDataList)
257 if len(calExpRefList) == 0:
258 self.log.warn(
"No exposures to coadd for patch %s", patchRef.dataId)
260 self.log.info(
"Selected %d calexps for patch %s", len(calExpRefList), patchRef.dataId)
261 calExpRefList = [calExpRef
for calExpRef
in calExpRefList
if calExpRef.datasetExists(
"calexp")]
262 self.log.info(
"Processing %d existing calexps for patch %s", len(calExpRefList), patchRef.dataId)
266 self.log.info(
"Processing %d warp exposures for patch %s", len(groupData.groups), patchRef.dataId)
269 for i, (tempExpTuple, calexpRefList)
in enumerate(groupData.groups.items()):
271 tempExpTuple, groupData.keys)
272 if not self.config.doOverwrite
and tempExpRef.datasetExists(datasetType=primaryWarpDataset):
273 self.log.info(
"Warp %s exists; skipping", tempExpRef.dataId)
274 dataRefList.append(tempExpRef)
276 self.log.info(
"Processing Warp %d/%d: id=%s", i, len(groupData.groups), tempExpRef.dataId)
282 visitId = int(tempExpRef.dataId[
"visit"])
283 except (KeyError, ValueError):
286 exps = self.
createTempExp(calexpRefList, skyInfo, visitId).exposures
288 if any(exps.values()):
289 dataRefList.append(tempExpRef)
291 self.log.warn(
"Warp %s could not be created", tempExpRef.dataId)
293 if self.config.doWrite:
294 for (warpType, exposure)
in exps.items():
295 if exposure
is not None:
302 """Create a Warp from inputs 304 We iterate over the multiple calexps in a single exposure to construct 305 the warp (previously called a coaddTempExp) of that exposure to the 306 supplied tract/patch. 308 Pixels that receive no pixels are set to NAN; this is not correct 309 (violates LSST algorithms group policy), but will be fixed up by 310 interpolating after the coaddition. 312 @param calexpRefList: List of data references for calexps that (may) 313 overlap the patch of interest 314 @param skyInfo: Struct from CoaddBaseTask.getSkyInfo() with geometric 315 information about the patch 316 @param visitId: integer identifier for visit, for the table that will 318 @return a pipeBase Struct containing: 319 - exposures: a dictionary containing the warps requested: 320 "direct": direct warp if config.makeDirect 321 "psfMatched": PSF-matched warp if config.makePsfMatched 325 totGoodPix = {warpType: 0
for warpType
in warpTypeList}
326 didSetMetadata = {warpType:
False for warpType
in warpTypeList}
328 inputRecorder = {warpType: self.inputRecorder.makeCoaddTempExpRecorder(visitId, len(calexpRefList))
329 for warpType
in warpTypeList}
331 modelPsf = self.config.modelPsf.apply()
if self.config.makePsfMatched
else None 332 for calExpInd, calExpRef
in enumerate(calexpRefList):
333 self.log.info(
"Processing calexp %d of %d for this Warp: id=%s",
334 calExpInd+1, len(calexpRefList), calExpRef.dataId)
336 ccdId = calExpRef.get(
"ccdExposureId", immediate=
True)
343 calExpRef = calExpRef.butlerSubset.butler.dataRef(
"calexp", dataId=calExpRef.dataId,
344 tract=skyInfo.tractInfo.getId())
345 calExp = self.
getCalExp(calExpRef, bgSubtracted=self.config.bgSubtracted)
346 except Exception
as e:
347 self.log.warn(
"Calexp %s not found; skipping it: %s", calExpRef.dataId, e)
350 warpedAndMatched = self.warpAndPsfMatch.
run(calExp, modelPsf=modelPsf,
351 wcs=skyInfo.wcs, maxBBox=skyInfo.bbox,
352 makeDirect=self.config.makeDirect,
353 makePsfMatched=self.config.makePsfMatched)
354 except Exception
as e:
355 self.log.warn(
"WarpAndPsfMatch failed for calexp %s; skipping it: %s", calExpRef.dataId, e)
358 numGoodPix = {warpType: 0
for warpType
in warpTypeList}
359 for warpType
in warpTypeList:
360 exposure = warpedAndMatched.getDict()[warpType]
363 coaddTempExp = coaddTempExps[warpType]
364 if didSetMetadata[warpType]:
365 mimg = exposure.getMaskedImage()
366 mimg *= (coaddTempExp.getCalib().getFluxMag0()[0] /
367 exposure.getCalib().getFluxMag0()[0])
369 numGoodPix[warpType] = coaddUtils.copyGoodPixels(
370 coaddTempExp.getMaskedImage(), exposure.getMaskedImage(), self.
getBadPixelMask())
371 totGoodPix[warpType] += numGoodPix[warpType]
372 self.log.debug(
"Calexp %s has %d good pixels in this patch (%.1f%%) for %s",
373 calExpRef.dataId, numGoodPix[warpType],
374 100.0*numGoodPix[warpType]/skyInfo.bbox.getArea(), warpType)
375 if numGoodPix[warpType] > 0
and not didSetMetadata[warpType]:
376 coaddTempExp.setCalib(exposure.getCalib())
377 coaddTempExp.setFilter(exposure.getFilter())
379 coaddTempExp.setPsf(exposure.getPsf())
380 didSetMetadata[warpType] =
True 383 inputRecorder[warpType].addCalExp(calExp, ccdId, numGoodPix[warpType])
385 except Exception
as e:
386 self.log.warn(
"Error processing calexp %s; skipping it: %s", calExpRef.dataId, e)
389 for warpType
in warpTypeList:
390 self.log.info(
"%sWarp has %d good pixels (%.1f%%)",
391 warpType, totGoodPix[warpType], 100.0*totGoodPix[warpType]/skyInfo.bbox.getArea())
393 if totGoodPix[warpType] > 0
and didSetMetadata[warpType]:
394 inputRecorder[warpType].finish(coaddTempExps[warpType], totGoodPix[warpType])
395 if warpType ==
"direct":
396 coaddTempExps[warpType].setPsf(
397 CoaddPsf(inputRecorder[warpType].coaddInputs.ccds, skyInfo.wcs,
398 self.config.coaddPsf.makeControl()))
401 coaddTempExps[warpType] =
None 403 result = pipeBase.Struct(exposures=coaddTempExps)
406 def _prepareEmptyExposure(cls, skyInfo):
407 """Produce an empty exposure for a given patch""" 408 exp = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
409 exp.getMaskedImage().set(numpy.nan, afwImage.Mask\
410 .getPlaneBitMask(
"NO_DATA"), numpy.inf)
def getCoaddDatasetName(self, warpType="direct")
def getGroupDataRef(butler, datasetType, groupTuple, keys)
Base class for coaddition.
def __init__(self, args, kwargs)
Warp and optionally PSF-Match calexps onto an a common projection.
def getSkyInfo(self, patchRef)
Use getSkyinfo to return the skyMap, tract and patch information, wcs and the outer bbox of the patch...
def getTempExpDatasetName(self, warpType="direct")
def createTempExp(self, calexpRefList, skyInfo, visitId=0)
def getBadPixelMask(self)
Convenience method to provide the bitmask from the mask plane names.
def _prepareEmptyExposure(cls, skyInfo)
def selectExposures(self, patchRef, skyInfo=None, selectDataList=[])
Select exposures to coadd.
def run(self, patchRef, selectDataList=[])
Produce <coaddName>Coadd_<warpType>Warp images by warping and optionally PSF-matching.
def getCalExp(self, dataRef, bgSubtracted)
Return one "calexp" calibrated exposure.
def getWarpTypeList(self)
def groupPatchExposures(patchDataRef, calexpDataRefList, coaddDatasetType="deepCoadd", tempExpDatasetType="deepCoadd_directWarp")