23 from __future__
import absolute_import, division, print_function
32 from .coaddBase
import CoaddBaseTask
33 from .warpAndPsfMatch
import WarpAndPsfMatchTask
34 from .coaddHelpers
import groupPatchExposures, getGroupDataRef
36 __all__ = [
"MakeCoaddTempExpTask"]
40 """Config for MakeCoaddTempExpTask 42 warpAndPsfMatch = pexConfig.ConfigurableField(
43 target=WarpAndPsfMatchTask,
44 doc=
"Task to warp and PSF-match calexp",
46 doWrite = pexConfig.Field(
47 doc=
"persist <coaddName>Coadd_<warpType>Warp",
51 bgSubtracted = pexConfig.Field(
52 doc=
"Work with a background subtracted calexp?",
56 coaddPsf = pexConfig.ConfigField(
57 doc=
"Configuration for CoaddPsf",
60 makeDirect = pexConfig.Field(
61 doc=
"Make direct Warp/Coadds",
65 makePsfMatched = pexConfig.Field(
66 doc=
"Make Psf-Matched Warp/Coadd?",
72 CoaddBaseTask.ConfigClass.validate(self)
74 raise RuntimeError(
"At least one of config.makePsfMatched and config.makeDirect must be True")
77 log.warn(
"Config doPsfMatch deprecated. Setting makePsfMatched=True and makeDirect=False")
91 """!Warp and optionally PSF-Match calexps onto an a common projection. 93 @anchor MakeCoaddTempExpTask_ 95 @section pipe_tasks_makeCoaddTempExp_Contents Contents 97 - @ref pipe_tasks_makeCoaddTempExp_Purpose 98 - @ref pipe_tasks_makeCoaddTempExp_Initialize 99 - @ref pipe_tasks_makeCoaddTempExp_IO 100 - @ref pipe_tasks_makeCoaddTempExp_Config 101 - @ref pipe_tasks_makeCoaddTempExp_Debug 102 - @ref pipe_tasks_makeCoaddTempExp_Example 104 @section pipe_tasks_makeCoaddTempExp_Purpose Description 106 Warp and optionally PSF-Match calexps onto a common projection, by 107 performing the following operations: 108 - Group calexps by visit/run 109 - For each visit, generate a Warp by calling method @ref makeTempExp. 110 makeTempExp loops over the visit's calexps calling @ref WarpAndPsfMatch 113 The result is a `directWarp` (and/or optionally a `psfMatchedWarp`). 115 @section pipe_tasks_makeCoaddTempExp_Initialize Task Initialization 117 @copydoc \_\_init\_\_ 119 This task has one special keyword argument: passing reuse=True will cause 120 the task to skip the creation of warps that are already present in the 123 @section pipe_tasks_makeCoaddTempExp_IO Invoking the Task 125 This task is primarily designed to be run from the command line. 127 The main method is `run`, which takes a single butler data reference for the patch(es) 132 WarpType identifies the types of convolutions applied to Warps (previously CoaddTempExps). 133 Only two types are available: direct (for regular Warps/Coadds) and psfMatched 134 (for Warps/Coadds with homogenized PSFs). We expect to add a third type, likelihood, 135 for generating likelihood Coadds with Warps that have been correlated with their own PSF. 137 @section pipe_tasks_makeCoaddTempExp_Config Configuration parameters 139 See @ref MakeCoaddTempExpConfig and parameters inherited from 140 @link lsst.pipe.tasks.coaddBase.CoaddBaseConfig CoaddBaseConfig @endlink 142 @subsection pipe_tasks_MakeCoaddTempExp_psfMatching Guide to PSF-Matching Configs 144 To make `psfMatchedWarps`, select `config.makePsfMatched=True`. The subtask 145 @link lsst.ip.diffim.modelPsfMatch.ModelPsfMatchTask ModelPsfMatchTask @endlink 146 is responsible for the PSF-Matching, and its config is accessed via `config.warpAndPsfMatch.psfMatch`. 147 The optimal configuration depends on aspects of dataset: the pixel scale, average PSF FWHM and 148 dimensions of the PSF kernel. These configs include the requested model PSF, the matching kernel size, 149 padding of the science PSF thumbnail and spatial sampling frequency of the PSF. 151 *Config Guidelines*: The user must specify the size of the model PSF to which to match by setting 152 `config.modelPsf.defaultFwhm` in units of pixels. The appropriate values depends on science case. 153 In general, for a set of input images, this config should equal the FWHM of the visit 154 with the worst seeing. The smallest it should be set to is the median FWHM. The defaults 155 of the other config options offer a reasonable starting point. 156 The following list presents the most common problems that arise from a misconfigured 157 @link lsst.ip.diffim.modelPsfMatch.ModelPsfMatchTask ModelPsfMatchTask @endlink 158 and corresponding solutions. All assume the default Alard-Lupton kernel, with configs accessed via 159 ```config.warpAndPsfMatch.psfMatch.kernel['AL']```. Each item in the list is formatted as: 160 Problem: Explanation. *Solution* 162 *Troublshooting PSF-Matching Configuration:* 163 - Matched PSFs look boxy: The matching kernel is too small. _Increase the matching kernel size. 166 config.warpAndPsfMatch.psfMatch.kernel['AL'].kernelSize=27 # default 21 168 Note that increasing the kernel size also increases runtime. 169 - Matched PSFs look ugly (dipoles, quadropoles, donuts): unable to find good solution 170 for matching kernel. _Provide the matcher with more data by either increasing 171 the spatial sampling by decreasing the spatial cell size,_ 173 config.warpAndPsfMatch.psfMatch.kernel['AL'].sizeCellX = 64 # default 128 174 config.warpAndPsfMatch.psfMatch.kernel['AL'].sizeCellY = 64 # default 128 176 _or increasing the padding around the Science PSF, for example:_ 178 config.warpAndPsfMatch.psfMatch.autoPadPsfTo=1.6 # default 1.4 180 Increasing `autoPadPsfTo` increases the minimum ratio of input PSF dimensions to the 181 matching kernel dimensions, thus increasing the number of pixels available to fit 182 after convolving the PSF with the matching kernel. 183 Optionally, for debugging the effects of padding, the level of padding may be manually 184 controlled by setting turning off the automatic padding and setting the number 185 of pixels by which to pad the PSF: 187 config.warpAndPsfMatch.psfMatch.doAutoPadPsf = False # default True 188 config.warpAndPsfMatch.psfMatch.padPsfBy = 6 # pixels. default 0 190 - Deconvolution: Matching a large PSF to a smaller PSF produces 191 a telltale noise pattern which looks like ripples or a brain. 192 _Increase the size of the requested model PSF. For example:_ 194 config.modelPsf.defaultFwhm = 11 # Gaussian sigma in units of pixels. 196 - High frequency (sometimes checkered) noise: The matching basis functions are too small. 197 _Increase the width of the Gaussian basis functions. For example:_ 199 config.warpAndPsfMatch.psfMatch.kernel['AL'].alardSigGauss=[1.5, 3.0, 6.0] 200 # from default [0.7, 1.5, 3.0] 203 @section pipe_tasks_makeCoaddTempExp_Debug Debug variables 205 MakeCoaddTempExpTask has no debug output, but its subtasks do. 207 @section pipe_tasks_makeCoaddTempExp_Example A complete example of using MakeCoaddTempExpTask 209 This example uses the package ci_hsc to show how MakeCoaddTempExp fits 210 into the larger Data Release Processing. 215 # if not built already: 216 python $(which scons) # this will take a while 218 The following assumes that `processCcd.py` and `makeSkyMap.py` have previously been run 219 (e.g. by building `ci_hsc` above) to generate a repository of calexps and an 220 output respository with the desired SkyMap. The command, 222 makeCoaddTempExp.py $CI_HSC_DIR/DATA --rerun ci_hsc \ 223 --id patch=5,4 tract=0 filter=HSC-I \ 224 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 \ 225 --selectId visit=903988 ccd=23 --selectId visit=903988 ccd=24 \ 226 --config doApplyUberCal=False makePsfMatched=True modelPsf.defaultFwhm=11 228 writes a direct and PSF-Matched Warp to 229 - `$CI_HSC_DIR/DATA/rerun/ci_hsc/deepCoadd/HSC-I/0/5,4/warp-HSC-I-0-5,4-903988.fits` and 230 - `$CI_HSC_DIR/DATA/rerun/ci_hsc/deepCoadd/HSC-I/0/5,4/psfMatchedWarp-HSC-I-0-5,4-903988.fits` 233 @note PSF-Matching in this particular dataset would benefit from adding 234 `--configfile ./matchingConfig.py` to 235 the command line arguments where `matchingConfig.py` is defined by: 238 config.warpAndPsfMatch.psfMatch.kernel['AL'].kernelSize=27 239 config.warpAndPsfMatch.psfMatch.kernel['AL'].alardSigGauss=[1.5, 3.0, 6.0]" > matchingConfig.py 242 Add the option `--help` to see more options. 244 ConfigClass = MakeCoaddTempExpConfig
245 _DefaultName =
"makeCoaddTempExp" 248 CoaddBaseTask.__init__(self, **kwargs)
250 self.makeSubtask(
"warpAndPsfMatch")
253 def run(self, patchRef, selectDataList=[]):
254 """!Produce <coaddName>Coadd_<warpType>Warp images by warping and optionally PSF-matching. 256 @param[in] patchRef: data reference for sky map patch. Must include keys "tract", "patch", 257 plus the camera-specific filter key (e.g. "filter" or "band") 258 @return: dataRefList: a list of data references for the new <coaddName>Coadd_directWarps 259 if direct or both warp types are requested and <coaddName>Coadd_psfMatchedWarps if only psfMatched 262 @warning: this task assumes that all exposures in a warp (coaddTempExp) have the same filter. 264 @warning: this task sets the Calib of the coaddTempExp to the Calib of the first calexp 265 with any good pixels in the patch. For a mosaic camera the resulting Calib should be ignored 266 (assembleCoadd should determine zeropoint scaling without referring to it). 271 if self.config.makePsfMatched
and not self.config.makeDirect:
276 calExpRefList = self.
selectExposures(patchRef, skyInfo, selectDataList=selectDataList)
277 if len(calExpRefList) == 0:
278 self.log.warn(
"No exposures to coadd for patch %s", patchRef.dataId)
280 self.log.info(
"Selected %d calexps for patch %s", len(calExpRefList), patchRef.dataId)
281 calExpRefList = [calExpRef
for calExpRef
in calExpRefList
if calExpRef.datasetExists(
"calexp")]
282 self.log.info(
"Processing %d existing calexps for patch %s", len(calExpRefList), patchRef.dataId)
286 self.log.info(
"Processing %d warp exposures for patch %s", len(groupData.groups), patchRef.dataId)
289 for i, (tempExpTuple, calexpRefList)
in enumerate(groupData.groups.items()):
291 tempExpTuple, groupData.keys)
292 if self.
reuse and tempExpRef.datasetExists(datasetType=primaryWarpDataset, write=
True):
293 self.log.info(
"Skipping makeCoaddTempExp for %s; output already exists.", tempExpRef.dataId)
294 dataRefList.append(tempExpRef)
296 self.log.info(
"Processing Warp %d/%d: id=%s", i, len(groupData.groups), tempExpRef.dataId)
302 visitId = int(tempExpRef.dataId[
"visit"])
303 except (KeyError, ValueError):
306 exps = self.
createTempExp(calexpRefList, skyInfo, visitId).exposures
308 if any(exps.values()):
309 dataRefList.append(tempExpRef)
311 self.log.warn(
"Warp %s could not be created", tempExpRef.dataId)
313 if self.config.doWrite:
314 for (warpType, exposure)
in exps.items():
315 if exposure
is not None:
322 """Create a Warp from inputs 324 We iterate over the multiple calexps in a single exposure to construct 325 the warp (previously called a coaddTempExp) of that exposure to the 326 supplied tract/patch. 328 Pixels that receive no pixels are set to NAN; this is not correct 329 (violates LSST algorithms group policy), but will be fixed up by 330 interpolating after the coaddition. 332 @param calexpRefList: List of data references for calexps that (may) 333 overlap the patch of interest 334 @param skyInfo: Struct from CoaddBaseTask.getSkyInfo() with geometric 335 information about the patch 336 @param visitId: integer identifier for visit, for the table that will 338 @return a pipeBase Struct containing: 339 - exposures: a dictionary containing the warps requested: 340 "direct": direct warp if config.makeDirect 341 "psfMatched": PSF-matched warp if config.makePsfMatched 345 totGoodPix = {warpType: 0
for warpType
in warpTypeList}
346 didSetMetadata = {warpType:
False for warpType
in warpTypeList}
348 inputRecorder = {warpType: self.inputRecorder.makeCoaddTempExpRecorder(visitId, len(calexpRefList))
349 for warpType
in warpTypeList}
351 modelPsf = self.config.modelPsf.apply()
if self.config.makePsfMatched
else None 352 for calExpInd, calExpRef
in enumerate(calexpRefList):
353 self.log.info(
"Processing calexp %d of %d for this Warp: id=%s",
354 calExpInd+1, len(calexpRefList), calExpRef.dataId)
356 ccdId = calExpRef.get(
"ccdExposureId", immediate=
True)
363 calExpRef = calExpRef.butlerSubset.butler.dataRef(
"calexp", dataId=calExpRef.dataId,
364 tract=skyInfo.tractInfo.getId())
365 calExp = self.
getCalExp(calExpRef, bgSubtracted=self.config.bgSubtracted)
366 except Exception
as e:
367 self.log.warn(
"Calexp %s not found; skipping it: %s", calExpRef.dataId, e)
370 warpedAndMatched = self.warpAndPsfMatch.
run(calExp, modelPsf=modelPsf,
371 wcs=skyInfo.wcs, maxBBox=skyInfo.bbox,
372 makeDirect=self.config.makeDirect,
373 makePsfMatched=self.config.makePsfMatched)
374 except Exception
as e:
375 self.log.warn(
"WarpAndPsfMatch failed for calexp %s; skipping it: %s", calExpRef.dataId, e)
378 numGoodPix = {warpType: 0
for warpType
in warpTypeList}
379 for warpType
in warpTypeList:
380 exposure = warpedAndMatched.getDict()[warpType]
383 coaddTempExp = coaddTempExps[warpType]
384 if didSetMetadata[warpType]:
385 mimg = exposure.getMaskedImage()
386 mimg *= (coaddTempExp.getCalib().getFluxMag0()[0] /
387 exposure.getCalib().getFluxMag0()[0])
389 numGoodPix[warpType] = coaddUtils.copyGoodPixels(
390 coaddTempExp.getMaskedImage(), exposure.getMaskedImage(), self.
getBadPixelMask())
391 totGoodPix[warpType] += numGoodPix[warpType]
392 self.log.debug(
"Calexp %s has %d good pixels in this patch (%.1f%%) for %s",
393 calExpRef.dataId, numGoodPix[warpType],
394 100.0*numGoodPix[warpType]/skyInfo.bbox.getArea(), warpType)
395 if numGoodPix[warpType] > 0
and not didSetMetadata[warpType]:
396 coaddTempExp.setCalib(exposure.getCalib())
397 coaddTempExp.setFilter(exposure.getFilter())
399 coaddTempExp.setPsf(exposure.getPsf())
400 didSetMetadata[warpType] =
True 403 inputRecorder[warpType].addCalExp(calExp, ccdId, numGoodPix[warpType])
405 except Exception
as e:
406 self.log.warn(
"Error processing calexp %s; skipping it: %s", calExpRef.dataId, e)
409 for warpType
in warpTypeList:
410 self.log.info(
"%sWarp has %d good pixels (%.1f%%)",
411 warpType, totGoodPix[warpType], 100.0*totGoodPix[warpType]/skyInfo.bbox.getArea())
413 if totGoodPix[warpType] > 0
and didSetMetadata[warpType]:
414 inputRecorder[warpType].finish(coaddTempExps[warpType], totGoodPix[warpType])
415 if warpType ==
"direct":
416 coaddTempExps[warpType].setPsf(
417 CoaddPsf(inputRecorder[warpType].coaddInputs.ccds, skyInfo.wcs,
418 self.config.coaddPsf.makeControl()))
421 coaddTempExps[warpType] =
None 423 result = pipeBase.Struct(exposures=coaddTempExps)
426 def _prepareEmptyExposure(cls, skyInfo):
427 """Produce an empty exposure for a given patch""" 428 exp = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
429 exp.getMaskedImage().set(numpy.nan, afwImage.Mask\
430 .getPlaneBitMask(
"NO_DATA"), numpy.inf)
434 """Return list of requested warp types per the config. 437 if self.config.makeDirect:
438 warpTypeList.append(
"direct")
439 if self.config.makePsfMatched:
440 warpTypeList.append(
"psfMatched")
def getCoaddDatasetName(self, warpType="direct")
def getGroupDataRef(butler, datasetType, groupTuple, keys)
Base class for coaddition.
def __init__(self, reuse=False, kwargs)
Warp and optionally PSF-Match calexps onto an a common projection.
def getSkyInfo(self, patchRef)
Use getSkyinfo to return the skyMap, tract and patch information, wcs and the outer bbox of the patch...
def getTempExpDatasetName(self, warpType="direct")
def createTempExp(self, calexpRefList, skyInfo, visitId=0)
def getBadPixelMask(self)
Convenience method to provide the bitmask from the mask plane names.
def _prepareEmptyExposure(cls, skyInfo)
def selectExposures(self, patchRef, skyInfo=None, selectDataList=[])
Select exposures to coadd.
def run(self, patchRef, selectDataList=[])
Produce <coaddName>Coadd_<warpType>Warp images by warping and optionally PSF-matching.
def getCalExp(self, dataRef, bgSubtracted)
Return one "calexp" calibrated exposure.
def getWarpTypeList(self)
def groupPatchExposures(patchDataRef, calexpDataRefList, coaddDatasetType="deepCoadd", tempExpDatasetType="deepCoadd_directWarp")