23 from __future__
import absolute_import, division, print_function
32 from .coaddBase
import CoaddBaseTask
33 from .warpAndPsfMatch
import WarpAndPsfMatchTask
34 from .coaddHelpers
import groupPatchExposures, getGroupDataRef
36 __all__ = [
"MakeCoaddTempExpTask"]
40 """Config for MakeCoaddTempExpTask 42 warpAndPsfMatch = pexConfig.ConfigurableField(
43 target=WarpAndPsfMatchTask,
44 doc=
"Task to warp and PSF-match calexp",
46 doWrite = pexConfig.Field(
47 doc=
"persist <coaddName>Coadd_<warpType>Warp",
51 bgSubtracted = pexConfig.Field(
52 doc=
"Work with a background subtracted calexp?",
56 coaddPsf = pexConfig.ConfigField(
57 doc=
"Configuration for CoaddPsf",
60 makeDirect = pexConfig.Field(
61 doc=
"Make direct Warp/Coadds",
65 makePsfMatched = pexConfig.Field(
66 doc=
"Make Psf-Matched Warp/Coadd?",
70 doApplySkyCorr = pexConfig.Field(dtype=bool, default=
False, doc=
"Apply sky correction?")
73 CoaddBaseTask.ConfigClass.validate(self)
75 raise RuntimeError(
"At least one of config.makePsfMatched and config.makeDirect must be True")
78 log.warn(
"Config doPsfMatch deprecated. Setting makePsfMatched=True and makeDirect=False")
92 """!Warp and optionally PSF-Match calexps onto an a common projection. 94 @anchor MakeCoaddTempExpTask_ 96 @section pipe_tasks_makeCoaddTempExp_Contents Contents 98 - @ref pipe_tasks_makeCoaddTempExp_Purpose 99 - @ref pipe_tasks_makeCoaddTempExp_Initialize 100 - @ref pipe_tasks_makeCoaddTempExp_IO 101 - @ref pipe_tasks_makeCoaddTempExp_Config 102 - @ref pipe_tasks_makeCoaddTempExp_Debug 103 - @ref pipe_tasks_makeCoaddTempExp_Example 105 @section pipe_tasks_makeCoaddTempExp_Purpose Description 107 Warp and optionally PSF-Match calexps onto a common projection, by 108 performing the following operations: 109 - Group calexps by visit/run 110 - For each visit, generate a Warp by calling method @ref makeTempExp. 111 makeTempExp loops over the visit's calexps calling @ref WarpAndPsfMatch 114 The result is a `directWarp` (and/or optionally a `psfMatchedWarp`). 116 @section pipe_tasks_makeCoaddTempExp_Initialize Task Initialization 118 @copydoc \_\_init\_\_ 120 This task has one special keyword argument: passing reuse=True will cause 121 the task to skip the creation of warps that are already present in the 124 @section pipe_tasks_makeCoaddTempExp_IO Invoking the Task 126 This task is primarily designed to be run from the command line. 128 The main method is `run`, which takes a single butler data reference for the patch(es) 133 WarpType identifies the types of convolutions applied to Warps (previously CoaddTempExps). 134 Only two types are available: direct (for regular Warps/Coadds) and psfMatched 135 (for Warps/Coadds with homogenized PSFs). We expect to add a third type, likelihood, 136 for generating likelihood Coadds with Warps that have been correlated with their own PSF. 138 @section pipe_tasks_makeCoaddTempExp_Config Configuration parameters 140 See @ref MakeCoaddTempExpConfig and parameters inherited from 141 @link lsst.pipe.tasks.coaddBase.CoaddBaseConfig CoaddBaseConfig @endlink 143 @subsection pipe_tasks_MakeCoaddTempExp_psfMatching Guide to PSF-Matching Configs 145 To make `psfMatchedWarps`, select `config.makePsfMatched=True`. The subtask 146 @link lsst.ip.diffim.modelPsfMatch.ModelPsfMatchTask ModelPsfMatchTask @endlink 147 is responsible for the PSF-Matching, and its config is accessed via `config.warpAndPsfMatch.psfMatch`. 148 The optimal configuration depends on aspects of dataset: the pixel scale, average PSF FWHM and 149 dimensions of the PSF kernel. These configs include the requested model PSF, the matching kernel size, 150 padding of the science PSF thumbnail and spatial sampling frequency of the PSF. 152 *Config Guidelines*: The user must specify the size of the model PSF to which to match by setting 153 `config.modelPsf.defaultFwhm` in units of pixels. The appropriate values depends on science case. 154 In general, for a set of input images, this config should equal the FWHM of the visit 155 with the worst seeing. The smallest it should be set to is the median FWHM. The defaults 156 of the other config options offer a reasonable starting point. 157 The following list presents the most common problems that arise from a misconfigured 158 @link lsst.ip.diffim.modelPsfMatch.ModelPsfMatchTask ModelPsfMatchTask @endlink 159 and corresponding solutions. All assume the default Alard-Lupton kernel, with configs accessed via 160 ```config.warpAndPsfMatch.psfMatch.kernel['AL']```. Each item in the list is formatted as: 161 Problem: Explanation. *Solution* 163 *Troublshooting PSF-Matching Configuration:* 164 - Matched PSFs look boxy: The matching kernel is too small. _Increase the matching kernel size. 167 config.warpAndPsfMatch.psfMatch.kernel['AL'].kernelSize=27 # default 21 169 Note that increasing the kernel size also increases runtime. 170 - Matched PSFs look ugly (dipoles, quadropoles, donuts): unable to find good solution 171 for matching kernel. _Provide the matcher with more data by either increasing 172 the spatial sampling by decreasing the spatial cell size,_ 174 config.warpAndPsfMatch.psfMatch.kernel['AL'].sizeCellX = 64 # default 128 175 config.warpAndPsfMatch.psfMatch.kernel['AL'].sizeCellY = 64 # default 128 177 _or increasing the padding around the Science PSF, for example:_ 179 config.warpAndPsfMatch.psfMatch.autoPadPsfTo=1.6 # default 1.4 181 Increasing `autoPadPsfTo` increases the minimum ratio of input PSF dimensions to the 182 matching kernel dimensions, thus increasing the number of pixels available to fit 183 after convolving the PSF with the matching kernel. 184 Optionally, for debugging the effects of padding, the level of padding may be manually 185 controlled by setting turning off the automatic padding and setting the number 186 of pixels by which to pad the PSF: 188 config.warpAndPsfMatch.psfMatch.doAutoPadPsf = False # default True 189 config.warpAndPsfMatch.psfMatch.padPsfBy = 6 # pixels. default 0 191 - Deconvolution: Matching a large PSF to a smaller PSF produces 192 a telltale noise pattern which looks like ripples or a brain. 193 _Increase the size of the requested model PSF. For example:_ 195 config.modelPsf.defaultFwhm = 11 # Gaussian sigma in units of pixels. 197 - High frequency (sometimes checkered) noise: The matching basis functions are too small. 198 _Increase the width of the Gaussian basis functions. For example:_ 200 config.warpAndPsfMatch.psfMatch.kernel['AL'].alardSigGauss=[1.5, 3.0, 6.0] 201 # from default [0.7, 1.5, 3.0] 204 @section pipe_tasks_makeCoaddTempExp_Debug Debug variables 206 MakeCoaddTempExpTask has no debug output, but its subtasks do. 208 @section pipe_tasks_makeCoaddTempExp_Example A complete example of using MakeCoaddTempExpTask 210 This example uses the package ci_hsc to show how MakeCoaddTempExp fits 211 into the larger Data Release Processing. 216 # if not built already: 217 python $(which scons) # this will take a while 219 The following assumes that `processCcd.py` and `makeSkyMap.py` have previously been run 220 (e.g. by building `ci_hsc` above) to generate a repository of calexps and an 221 output respository with the desired SkyMap. The command, 223 makeCoaddTempExp.py $CI_HSC_DIR/DATA --rerun ci_hsc \ 224 --id patch=5,4 tract=0 filter=HSC-I \ 225 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 \ 226 --selectId visit=903988 ccd=23 --selectId visit=903988 ccd=24 \ 227 --config doApplyUberCal=False makePsfMatched=True modelPsf.defaultFwhm=11 229 writes a direct and PSF-Matched Warp to 230 - `$CI_HSC_DIR/DATA/rerun/ci_hsc/deepCoadd/HSC-I/0/5,4/warp-HSC-I-0-5,4-903988.fits` and 231 - `$CI_HSC_DIR/DATA/rerun/ci_hsc/deepCoadd/HSC-I/0/5,4/psfMatchedWarp-HSC-I-0-5,4-903988.fits` 234 @note PSF-Matching in this particular dataset would benefit from adding 235 `--configfile ./matchingConfig.py` to 236 the command line arguments where `matchingConfig.py` is defined by: 239 config.warpAndPsfMatch.psfMatch.kernel['AL'].kernelSize=27 240 config.warpAndPsfMatch.psfMatch.kernel['AL'].alardSigGauss=[1.5, 3.0, 6.0]" > matchingConfig.py 243 Add the option `--help` to see more options. 245 ConfigClass = MakeCoaddTempExpConfig
246 _DefaultName =
"makeCoaddTempExp" 249 CoaddBaseTask.__init__(self, **kwargs)
251 self.makeSubtask(
"warpAndPsfMatch")
254 def run(self, patchRef, selectDataList=[]):
255 """!Produce <coaddName>Coadd_<warpType>Warp images by warping and optionally PSF-matching. 257 @param[in] patchRef: data reference for sky map patch. Must include keys "tract", "patch", 258 plus the camera-specific filter key (e.g. "filter" or "band") 259 @return: dataRefList: a list of data references for the new <coaddName>Coadd_directWarps 260 if direct or both warp types are requested and <coaddName>Coadd_psfMatchedWarps if only psfMatched 263 @warning: this task assumes that all exposures in a warp (coaddTempExp) have the same filter. 265 @warning: this task sets the Calib of the coaddTempExp to the Calib of the first calexp 266 with any good pixels in the patch. For a mosaic camera the resulting Calib should be ignored 267 (assembleCoadd should determine zeropoint scaling without referring to it). 272 if self.config.makePsfMatched
and not self.config.makeDirect:
277 calExpRefList = self.
selectExposures(patchRef, skyInfo, selectDataList=selectDataList)
278 if len(calExpRefList) == 0:
279 self.log.warn(
"No exposures to coadd for patch %s", patchRef.dataId)
281 self.log.info(
"Selected %d calexps for patch %s", len(calExpRefList), patchRef.dataId)
282 calExpRefList = [calExpRef
for calExpRef
in calExpRefList
if calExpRef.datasetExists(
"calexp")]
283 self.log.info(
"Processing %d existing calexps for patch %s", len(calExpRefList), patchRef.dataId)
287 self.log.info(
"Processing %d warp exposures for patch %s", len(groupData.groups), patchRef.dataId)
290 for i, (tempExpTuple, calexpRefList)
in enumerate(groupData.groups.items()):
292 tempExpTuple, groupData.keys)
293 if self.
reuse and tempExpRef.datasetExists(datasetType=primaryWarpDataset, write=
True):
294 self.log.info(
"Skipping makeCoaddTempExp for %s; output already exists.", tempExpRef.dataId)
295 dataRefList.append(tempExpRef)
297 self.log.info(
"Processing Warp %d/%d: id=%s", i, len(groupData.groups), tempExpRef.dataId)
303 visitId = int(tempExpRef.dataId[
"visit"])
304 except (KeyError, ValueError):
307 exps = self.
createTempExp(calexpRefList, skyInfo, visitId).exposures
309 if any(exps.values()):
310 dataRefList.append(tempExpRef)
312 self.log.warn(
"Warp %s could not be created", tempExpRef.dataId)
314 if self.config.doWrite:
315 for (warpType, exposure)
in exps.items():
316 if exposure
is not None:
323 """Create a Warp from inputs 325 We iterate over the multiple calexps in a single exposure to construct 326 the warp (previously called a coaddTempExp) of that exposure to the 327 supplied tract/patch. 329 Pixels that receive no pixels are set to NAN; this is not correct 330 (violates LSST algorithms group policy), but will be fixed up by 331 interpolating after the coaddition. 333 @param calexpRefList: List of data references for calexps that (may) 334 overlap the patch of interest 335 @param skyInfo: Struct from CoaddBaseTask.getSkyInfo() with geometric 336 information about the patch 337 @param visitId: integer identifier for visit, for the table that will 339 @return a pipeBase Struct containing: 340 - exposures: a dictionary containing the warps requested: 341 "direct": direct warp if config.makeDirect 342 "psfMatched": PSF-matched warp if config.makePsfMatched 346 totGoodPix = {warpType: 0
for warpType
in warpTypeList}
347 didSetMetadata = {warpType:
False for warpType
in warpTypeList}
349 inputRecorder = {warpType: self.inputRecorder.makeCoaddTempExpRecorder(visitId, len(calexpRefList))
350 for warpType
in warpTypeList}
352 modelPsf = self.config.modelPsf.apply()
if self.config.makePsfMatched
else None 353 for calExpInd, calExpRef
in enumerate(calexpRefList):
354 self.log.info(
"Processing calexp %d of %d for this Warp: id=%s",
355 calExpInd+1, len(calexpRefList), calExpRef.dataId)
357 ccdId = calExpRef.get(
"ccdExposureId", immediate=
True)
364 calExpRef = calExpRef.butlerSubset.butler.dataRef(
"calexp", dataId=calExpRef.dataId,
365 tract=skyInfo.tractInfo.getId())
366 calExp = self.
getCalExp(calExpRef, bgSubtracted=self.config.bgSubtracted)
367 except Exception
as e:
368 self.log.warn(
"Calexp %s not found; skipping it: %s", calExpRef.dataId, e)
371 if self.config.doApplySkyCorr:
375 warpedAndMatched = self.warpAndPsfMatch.
run(calExp, modelPsf=modelPsf,
376 wcs=skyInfo.wcs, maxBBox=skyInfo.bbox,
377 makeDirect=self.config.makeDirect,
378 makePsfMatched=self.config.makePsfMatched)
379 except Exception
as e:
380 self.log.warn(
"WarpAndPsfMatch failed for calexp %s; skipping it: %s", calExpRef.dataId, e)
383 numGoodPix = {warpType: 0
for warpType
in warpTypeList}
384 for warpType
in warpTypeList:
385 exposure = warpedAndMatched.getDict()[warpType]
388 coaddTempExp = coaddTempExps[warpType]
389 if didSetMetadata[warpType]:
390 mimg = exposure.getMaskedImage()
391 mimg *= (coaddTempExp.getCalib().getFluxMag0()[0] /
392 exposure.getCalib().getFluxMag0()[0])
394 numGoodPix[warpType] = coaddUtils.copyGoodPixels(
395 coaddTempExp.getMaskedImage(), exposure.getMaskedImage(), self.
getBadPixelMask())
396 totGoodPix[warpType] += numGoodPix[warpType]
397 self.log.debug(
"Calexp %s has %d good pixels in this patch (%.1f%%) for %s",
398 calExpRef.dataId, numGoodPix[warpType],
399 100.0*numGoodPix[warpType]/skyInfo.bbox.getArea(), warpType)
400 if numGoodPix[warpType] > 0
and not didSetMetadata[warpType]:
401 coaddTempExp.setCalib(exposure.getCalib())
402 coaddTempExp.setFilter(exposure.getFilter())
404 coaddTempExp.setPsf(exposure.getPsf())
405 didSetMetadata[warpType] =
True 408 inputRecorder[warpType].addCalExp(calExp, ccdId, numGoodPix[warpType])
410 except Exception
as e:
411 self.log.warn(
"Error processing calexp %s; skipping it: %s", calExpRef.dataId, e)
414 for warpType
in warpTypeList:
415 self.log.info(
"%sWarp has %d good pixels (%.1f%%)",
416 warpType, totGoodPix[warpType], 100.0*totGoodPix[warpType]/skyInfo.bbox.getArea())
418 if totGoodPix[warpType] > 0
and didSetMetadata[warpType]:
419 inputRecorder[warpType].finish(coaddTempExps[warpType], totGoodPix[warpType])
420 if warpType ==
"direct":
421 coaddTempExps[warpType].setPsf(
422 CoaddPsf(inputRecorder[warpType].coaddInputs.ccds, skyInfo.wcs,
423 self.config.coaddPsf.makeControl()))
426 coaddTempExps[warpType] =
None 428 result = pipeBase.Struct(exposures=coaddTempExps)
431 def _prepareEmptyExposure(cls, skyInfo):
432 """Produce an empty exposure for a given patch""" 433 exp = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
434 exp.getMaskedImage().set(numpy.nan, afwImage.Mask\
435 .getPlaneBitMask(
"NO_DATA"), numpy.inf)
439 """Return list of requested warp types per the config. 442 if self.config.makeDirect:
443 warpTypeList.append(
"direct")
444 if self.config.makePsfMatched:
445 warpTypeList.append(
"psfMatched")
449 """Apply correction to the sky background level 451 Sky corrections can be generated with the 'skyCorrection.py' 452 executable in pipe_drivers. Because the sky model used by that 453 code extends over the entire focal plane, this can produce 454 better sky subtraction. 456 The calexp is updated in-place. 460 dataRef : `lsst.daf.persistence.ButlerDataRef` 461 Data reference for calexp. 462 calexp : `lsst.afw.image.Exposure` or `lsst.afw.image.MaskedImage` 465 bg = dataRef.get(
"skyCorr")
466 if isinstance(calexp, afwImage.Exposure):
467 calexp = calexp.getMaskedImage()
468 calexp -= bg.getImage()
def getCoaddDatasetName(self, warpType="direct")
def getGroupDataRef(butler, datasetType, groupTuple, keys)
Base class for coaddition.
def __init__(self, reuse=False, kwargs)
Warp and optionally PSF-Match calexps onto an a common projection.
def getSkyInfo(self, patchRef)
Use getSkyinfo to return the skyMap, tract and patch information, wcs and the outer bbox of the patch...
def getTempExpDatasetName(self, warpType="direct")
def createTempExp(self, calexpRefList, skyInfo, visitId=0)
def getBadPixelMask(self)
Convenience method to provide the bitmask from the mask plane names.
def _prepareEmptyExposure(cls, skyInfo)
def selectExposures(self, patchRef, skyInfo=None, selectDataList=[])
Select exposures to coadd.
def run(self, patchRef, selectDataList=[])
Produce <coaddName>Coadd_<warpType>Warp images by warping and optionally PSF-matching.
def getCalExp(self, dataRef, bgSubtracted)
Return one "calexp" calibrated exposure.
def getWarpTypeList(self)
def groupPatchExposures(patchDataRef, calexpDataRefList, coaddDatasetType="deepCoadd", tempExpDatasetType="deepCoadd_directWarp")
def applySkyCorr(self, dataRef, calexp)