23 from __future__
import absolute_import, division, print_function
32 from .coaddBase
import CoaddBaseTask
33 from .warpAndPsfMatch
import WarpAndPsfMatchTask
34 from .coaddHelpers
import groupPatchExposures, getGroupDataRef
36 __all__ = [
"MakeCoaddTempExpTask"]
40 """Config for MakeCoaddTempExpTask 42 warpAndPsfMatch = pexConfig.ConfigurableField(
43 target=WarpAndPsfMatchTask,
44 doc=
"Task to warp and PSF-match calexp",
46 doWrite = pexConfig.Field(
47 doc=
"persist <coaddName>Coadd_<warpType>Warp",
51 bgSubtracted = pexConfig.Field(
52 doc=
"Work with a background subtracted calexp?",
56 coaddPsf = pexConfig.ConfigField(
57 doc=
"Configuration for CoaddPsf",
60 makeDirect = pexConfig.Field(
61 doc=
"Make direct Warp/Coadds",
65 makePsfMatched = pexConfig.Field(
66 doc=
"Make Psf-Matched Warp/Coadd?",
70 doApplySkyCorr = pexConfig.Field(dtype=bool, default=
False, doc=
"Apply sky correction?")
73 CoaddBaseTask.ConfigClass.validate(self)
75 raise RuntimeError(
"At least one of config.makePsfMatched and config.makeDirect must be True")
78 log.warn(
"Config doPsfMatch deprecated. Setting makePsfMatched=True and makeDirect=False")
83 CoaddBaseTask.ConfigClass.setDefaults(self)
84 self.
warpAndPsfMatch.psfMatch.kernel.active.kernelSize = self.matchingKernelSize
95 """!Warp and optionally PSF-Match calexps onto an a common projection. 97 @anchor MakeCoaddTempExpTask_ 99 @section pipe_tasks_makeCoaddTempExp_Contents Contents 101 - @ref pipe_tasks_makeCoaddTempExp_Purpose 102 - @ref pipe_tasks_makeCoaddTempExp_Initialize 103 - @ref pipe_tasks_makeCoaddTempExp_IO 104 - @ref pipe_tasks_makeCoaddTempExp_Config 105 - @ref pipe_tasks_makeCoaddTempExp_Debug 106 - @ref pipe_tasks_makeCoaddTempExp_Example 108 @section pipe_tasks_makeCoaddTempExp_Purpose Description 110 Warp and optionally PSF-Match calexps onto a common projection, by 111 performing the following operations: 112 - Group calexps by visit/run 113 - For each visit, generate a Warp by calling method @ref makeTempExp. 114 makeTempExp loops over the visit's calexps calling @ref WarpAndPsfMatch 117 The result is a `directWarp` (and/or optionally a `psfMatchedWarp`). 119 @section pipe_tasks_makeCoaddTempExp_Initialize Task Initialization 121 @copydoc \_\_init\_\_ 123 This task has one special keyword argument: passing reuse=True will cause 124 the task to skip the creation of warps that are already present in the 127 @section pipe_tasks_makeCoaddTempExp_IO Invoking the Task 129 This task is primarily designed to be run from the command line. 131 The main method is `run`, which takes a single butler data reference for the patch(es) 136 WarpType identifies the types of convolutions applied to Warps (previously CoaddTempExps). 137 Only two types are available: direct (for regular Warps/Coadds) and psfMatched 138 (for Warps/Coadds with homogenized PSFs). We expect to add a third type, likelihood, 139 for generating likelihood Coadds with Warps that have been correlated with their own PSF. 141 @section pipe_tasks_makeCoaddTempExp_Config Configuration parameters 143 See @ref MakeCoaddTempExpConfig and parameters inherited from 144 @link lsst.pipe.tasks.coaddBase.CoaddBaseConfig CoaddBaseConfig @endlink 146 @subsection pipe_tasks_MakeCoaddTempExp_psfMatching Guide to PSF-Matching Configs 148 To make `psfMatchedWarps`, select `config.makePsfMatched=True`. The subtask 149 @link lsst.ip.diffim.modelPsfMatch.ModelPsfMatchTask ModelPsfMatchTask @endlink 150 is responsible for the PSF-Matching, and its config is accessed via `config.warpAndPsfMatch.psfMatch`. 151 The optimal configuration depends on aspects of dataset: the pixel scale, average PSF FWHM and 152 dimensions of the PSF kernel. These configs include the requested model PSF, the matching kernel size, 153 padding of the science PSF thumbnail and spatial sampling frequency of the PSF. 155 *Config Guidelines*: The user must specify the size of the model PSF to which to match by setting 156 `config.modelPsf.defaultFwhm` in units of pixels. The appropriate values depends on science case. 157 In general, for a set of input images, this config should equal the FWHM of the visit 158 with the worst seeing. The smallest it should be set to is the median FWHM. The defaults 159 of the other config options offer a reasonable starting point. 160 The following list presents the most common problems that arise from a misconfigured 161 @link lsst.ip.diffim.modelPsfMatch.ModelPsfMatchTask ModelPsfMatchTask @endlink 162 and corresponding solutions. All assume the default Alard-Lupton kernel, with configs accessed via 163 ```config.warpAndPsfMatch.psfMatch.kernel['AL']```. Each item in the list is formatted as: 164 Problem: Explanation. *Solution* 166 *Troublshooting PSF-Matching Configuration:* 167 - Matched PSFs look boxy: The matching kernel is too small. _Increase the matching kernel size. 170 config.warpAndPsfMatch.psfMatch.kernel['AL'].kernelSize=27 # default 21 172 Note that increasing the kernel size also increases runtime. 173 - Matched PSFs look ugly (dipoles, quadropoles, donuts): unable to find good solution 174 for matching kernel. _Provide the matcher with more data by either increasing 175 the spatial sampling by decreasing the spatial cell size,_ 177 config.warpAndPsfMatch.psfMatch.kernel['AL'].sizeCellX = 64 # default 128 178 config.warpAndPsfMatch.psfMatch.kernel['AL'].sizeCellY = 64 # default 128 180 _or increasing the padding around the Science PSF, for example:_ 182 config.warpAndPsfMatch.psfMatch.autoPadPsfTo=1.6 # default 1.4 184 Increasing `autoPadPsfTo` increases the minimum ratio of input PSF dimensions to the 185 matching kernel dimensions, thus increasing the number of pixels available to fit 186 after convolving the PSF with the matching kernel. 187 Optionally, for debugging the effects of padding, the level of padding may be manually 188 controlled by setting turning off the automatic padding and setting the number 189 of pixels by which to pad the PSF: 191 config.warpAndPsfMatch.psfMatch.doAutoPadPsf = False # default True 192 config.warpAndPsfMatch.psfMatch.padPsfBy = 6 # pixels. default 0 194 - Deconvolution: Matching a large PSF to a smaller PSF produces 195 a telltale noise pattern which looks like ripples or a brain. 196 _Increase the size of the requested model PSF. For example:_ 198 config.modelPsf.defaultFwhm = 11 # Gaussian sigma in units of pixels. 200 - High frequency (sometimes checkered) noise: The matching basis functions are too small. 201 _Increase the width of the Gaussian basis functions. For example:_ 203 config.warpAndPsfMatch.psfMatch.kernel['AL'].alardSigGauss=[1.5, 3.0, 6.0] 204 # from default [0.7, 1.5, 3.0] 207 @section pipe_tasks_makeCoaddTempExp_Debug Debug variables 209 MakeCoaddTempExpTask has no debug output, but its subtasks do. 211 @section pipe_tasks_makeCoaddTempExp_Example A complete example of using MakeCoaddTempExpTask 213 This example uses the package ci_hsc to show how MakeCoaddTempExp fits 214 into the larger Data Release Processing. 219 # if not built already: 220 python $(which scons) # this will take a while 222 The following assumes that `processCcd.py` and `makeSkyMap.py` have previously been run 223 (e.g. by building `ci_hsc` above) to generate a repository of calexps and an 224 output respository with the desired SkyMap. The command, 226 makeCoaddTempExp.py $CI_HSC_DIR/DATA --rerun ci_hsc \ 227 --id patch=5,4 tract=0 filter=HSC-I \ 228 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 \ 229 --selectId visit=903988 ccd=23 --selectId visit=903988 ccd=24 \ 230 --config doApplyUberCal=False makePsfMatched=True modelPsf.defaultFwhm=11 232 writes a direct and PSF-Matched Warp to 233 - `$CI_HSC_DIR/DATA/rerun/ci_hsc/deepCoadd/HSC-I/0/5,4/warp-HSC-I-0-5,4-903988.fits` and 234 - `$CI_HSC_DIR/DATA/rerun/ci_hsc/deepCoadd/HSC-I/0/5,4/psfMatchedWarp-HSC-I-0-5,4-903988.fits` 237 @note PSF-Matching in this particular dataset would benefit from adding 238 `--configfile ./matchingConfig.py` to 239 the command line arguments where `matchingConfig.py` is defined by: 242 config.warpAndPsfMatch.psfMatch.kernel['AL'].kernelSize=27 243 config.warpAndPsfMatch.psfMatch.kernel['AL'].alardSigGauss=[1.5, 3.0, 6.0]" > matchingConfig.py 246 Add the option `--help` to see more options. 248 ConfigClass = MakeCoaddTempExpConfig
249 _DefaultName =
"makeCoaddTempExp" 252 CoaddBaseTask.__init__(self, **kwargs)
254 self.makeSubtask(
"warpAndPsfMatch")
257 def run(self, patchRef, selectDataList=[]):
258 """!Produce <coaddName>Coadd_<warpType>Warp images by warping and optionally PSF-matching. 260 @param[in] patchRef: data reference for sky map patch. Must include keys "tract", "patch", 261 plus the camera-specific filter key (e.g. "filter" or "band") 262 @return: dataRefList: a list of data references for the new <coaddName>Coadd_directWarps 263 if direct or both warp types are requested and <coaddName>Coadd_psfMatchedWarps if only psfMatched 266 @warning: this task assumes that all exposures in a warp (coaddTempExp) have the same filter. 268 @warning: this task sets the Calib of the coaddTempExp to the Calib of the first calexp 269 with any good pixels in the patch. For a mosaic camera the resulting Calib should be ignored 270 (assembleCoadd should determine zeropoint scaling without referring to it). 275 if self.config.makePsfMatched
and not self.config.makeDirect:
280 calExpRefList = self.
selectExposures(patchRef, skyInfo, selectDataList=selectDataList)
281 if len(calExpRefList) == 0:
282 self.log.warn(
"No exposures to coadd for patch %s", patchRef.dataId)
284 self.log.info(
"Selected %d calexps for patch %s", len(calExpRefList), patchRef.dataId)
285 calExpRefList = [calExpRef
for calExpRef
in calExpRefList
if calExpRef.datasetExists(
"calexp")]
286 self.log.info(
"Processing %d existing calexps for patch %s", len(calExpRefList), patchRef.dataId)
290 self.log.info(
"Processing %d warp exposures for patch %s", len(groupData.groups), patchRef.dataId)
293 for i, (tempExpTuple, calexpRefList)
in enumerate(groupData.groups.items()):
295 tempExpTuple, groupData.keys)
296 if self.
reuse and tempExpRef.datasetExists(datasetType=primaryWarpDataset, write=
True):
297 self.log.info(
"Skipping makeCoaddTempExp for %s; output already exists.", tempExpRef.dataId)
298 dataRefList.append(tempExpRef)
300 self.log.info(
"Processing Warp %d/%d: id=%s", i, len(groupData.groups), tempExpRef.dataId)
306 visitId = int(tempExpRef.dataId[
"visit"])
307 except (KeyError, ValueError):
310 exps = self.
createTempExp(calexpRefList, skyInfo, visitId).exposures
312 if any(exps.values()):
313 dataRefList.append(tempExpRef)
315 self.log.warn(
"Warp %s could not be created", tempExpRef.dataId)
317 if self.config.doWrite:
318 for (warpType, exposure)
in exps.items():
319 if exposure
is not None:
326 """Create a Warp from inputs 328 We iterate over the multiple calexps in a single exposure to construct 329 the warp (previously called a coaddTempExp) of that exposure to the 330 supplied tract/patch. 332 Pixels that receive no pixels are set to NAN; this is not correct 333 (violates LSST algorithms group policy), but will be fixed up by 334 interpolating after the coaddition. 336 @param calexpRefList: List of data references for calexps that (may) 337 overlap the patch of interest 338 @param skyInfo: Struct from CoaddBaseTask.getSkyInfo() with geometric 339 information about the patch 340 @param visitId: integer identifier for visit, for the table that will 342 @return a pipeBase Struct containing: 343 - exposures: a dictionary containing the warps requested: 344 "direct": direct warp if config.makeDirect 345 "psfMatched": PSF-matched warp if config.makePsfMatched 349 totGoodPix = {warpType: 0
for warpType
in warpTypeList}
350 didSetMetadata = {warpType:
False for warpType
in warpTypeList}
352 inputRecorder = {warpType: self.inputRecorder.makeCoaddTempExpRecorder(visitId, len(calexpRefList))
353 for warpType
in warpTypeList}
355 modelPsf = self.config.modelPsf.apply()
if self.config.makePsfMatched
else None 356 for calExpInd, calExpRef
in enumerate(calexpRefList):
357 self.log.info(
"Processing calexp %d of %d for this Warp: id=%s",
358 calExpInd+1, len(calexpRefList), calExpRef.dataId)
360 ccdId = calExpRef.get(
"ccdExposureId", immediate=
True)
367 calExpRef = calExpRef.butlerSubset.butler.dataRef(
"calexp", dataId=calExpRef.dataId,
368 tract=skyInfo.tractInfo.getId())
369 calExp = self.
getCalExp(calExpRef, bgSubtracted=self.config.bgSubtracted)
370 except Exception
as e:
371 self.log.warn(
"Calexp %s not found; skipping it: %s", calExpRef.dataId, e)
374 if self.config.doApplySkyCorr:
378 warpedAndMatched = self.warpAndPsfMatch.
run(calExp, modelPsf=modelPsf,
379 wcs=skyInfo.wcs, maxBBox=skyInfo.bbox,
380 makeDirect=self.config.makeDirect,
381 makePsfMatched=self.config.makePsfMatched)
382 except Exception
as e:
383 self.log.warn(
"WarpAndPsfMatch failed for calexp %s; skipping it: %s", calExpRef.dataId, e)
386 numGoodPix = {warpType: 0
for warpType
in warpTypeList}
387 for warpType
in warpTypeList:
388 exposure = warpedAndMatched.getDict()[warpType]
391 coaddTempExp = coaddTempExps[warpType]
392 if didSetMetadata[warpType]:
393 mimg = exposure.getMaskedImage()
394 mimg *= (coaddTempExp.getCalib().getFluxMag0()[0] /
395 exposure.getCalib().getFluxMag0()[0])
397 numGoodPix[warpType] = coaddUtils.copyGoodPixels(
398 coaddTempExp.getMaskedImage(), exposure.getMaskedImage(), self.
getBadPixelMask())
399 totGoodPix[warpType] += numGoodPix[warpType]
400 self.log.debug(
"Calexp %s has %d good pixels in this patch (%.1f%%) for %s",
401 calExpRef.dataId, numGoodPix[warpType],
402 100.0*numGoodPix[warpType]/skyInfo.bbox.getArea(), warpType)
403 if numGoodPix[warpType] > 0
and not didSetMetadata[warpType]:
404 coaddTempExp.setCalib(exposure.getCalib())
405 coaddTempExp.setFilter(exposure.getFilter())
406 coaddTempExp.getInfo().setVisitInfo(exposure.getInfo().getVisitInfo())
408 coaddTempExp.setPsf(exposure.getPsf())
409 didSetMetadata[warpType] =
True 412 inputRecorder[warpType].addCalExp(calExp, ccdId, numGoodPix[warpType])
414 except Exception
as e:
415 self.log.warn(
"Error processing calexp %s; skipping it: %s", calExpRef.dataId, e)
418 for warpType
in warpTypeList:
419 self.log.info(
"%sWarp has %d good pixels (%.1f%%)",
420 warpType, totGoodPix[warpType], 100.0*totGoodPix[warpType]/skyInfo.bbox.getArea())
422 if totGoodPix[warpType] > 0
and didSetMetadata[warpType]:
423 inputRecorder[warpType].finish(coaddTempExps[warpType], totGoodPix[warpType])
424 if warpType ==
"direct":
425 coaddTempExps[warpType].setPsf(
426 CoaddPsf(inputRecorder[warpType].coaddInputs.ccds, skyInfo.wcs,
427 self.config.coaddPsf.makeControl()))
430 coaddTempExps[warpType] =
None 432 result = pipeBase.Struct(exposures=coaddTempExps)
436 def _prepareEmptyExposure(skyInfo):
437 """Produce an empty exposure for a given patch""" 438 exp = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
439 exp.getMaskedImage().set(numpy.nan, afwImage.Mask
440 .getPlaneBitMask(
"NO_DATA"), numpy.inf)
444 """Return list of requested warp types per the config. 447 if self.config.makeDirect:
448 warpTypeList.append(
"direct")
449 if self.config.makePsfMatched:
450 warpTypeList.append(
"psfMatched")
454 """Apply correction to the sky background level 456 Sky corrections can be generated with the 'skyCorrection.py' 457 executable in pipe_drivers. Because the sky model used by that 458 code extends over the entire focal plane, this can produce 459 better sky subtraction. 461 The calexp is updated in-place. 465 dataRef : `lsst.daf.persistence.ButlerDataRef` 466 Data reference for calexp. 467 calexp : `lsst.afw.image.Exposure` or `lsst.afw.image.MaskedImage` 470 bg = dataRef.get(
"skyCorr")
471 if isinstance(calexp, afwImage.Exposure):
472 calexp = calexp.getMaskedImage()
473 calexp -= bg.getImage()
def getCoaddDatasetName(self, warpType="direct")
def getGroupDataRef(butler, datasetType, groupTuple, keys)
Base class for coaddition.
def __init__(self, reuse=False, kwargs)
def _prepareEmptyExposure(skyInfo)
Warp and optionally PSF-Match calexps onto an a common projection.
def getSkyInfo(self, patchRef)
Use getSkyinfo to return the skyMap, tract and patch information, wcs and the outer bbox of the patch...
def getTempExpDatasetName(self, warpType="direct")
def createTempExp(self, calexpRefList, skyInfo, visitId=0)
def getBadPixelMask(self)
Convenience method to provide the bitmask from the mask plane names.
def selectExposures(self, patchRef, skyInfo=None, selectDataList=[])
Select exposures to coadd.
def run(self, patchRef, selectDataList=[])
Produce <coaddName>Coadd_<warpType>Warp images by warping and optionally PSF-matching.
def getCalExp(self, dataRef, bgSubtracted)
Return one "calexp" calibrated exposure.
def getWarpTypeList(self)
def groupPatchExposures(patchDataRef, calexpDataRefList, coaddDatasetType="deepCoadd", tempExpDatasetType="deepCoadd_directWarp")
def applySkyCorr(self, dataRef, calexp)