30 from .coaddBase
import CoaddBaseTask
31 from .warpAndPsfMatch
import WarpAndPsfMatchTask
32 from .coaddHelpers
import groupPatchExposures, getGroupDataRef
34 __all__ = [
"MakeCoaddTempExpTask"]
38 """Config for MakeCoaddTempExpTask 40 warpAndPsfMatch = pexConfig.ConfigurableField(
41 target=WarpAndPsfMatchTask,
42 doc=
"Task to warp and PSF-match calexp",
44 doWrite = pexConfig.Field(
45 doc=
"persist <coaddName>Coadd_<warpType>Warp",
49 bgSubtracted = pexConfig.Field(
50 doc=
"Work with a background subtracted calexp?",
54 coaddPsf = pexConfig.ConfigField(
55 doc=
"Configuration for CoaddPsf",
58 makeDirect = pexConfig.Field(
59 doc=
"Make direct Warp/Coadds",
63 makePsfMatched = pexConfig.Field(
64 doc=
"Make Psf-Matched Warp/Coadd?",
68 doApplySkyCorr = pexConfig.Field(dtype=bool, default=
False, doc=
"Apply sky correction?")
71 CoaddBaseTask.ConfigClass.validate(self)
73 raise RuntimeError(
"At least one of config.makePsfMatched and config.makeDirect must be True")
76 log.warn(
"Config doPsfMatch deprecated. Setting makePsfMatched=True and makeDirect=False")
81 CoaddBaseTask.ConfigClass.setDefaults(self)
82 self.
warpAndPsfMatch.psfMatch.kernel.active.kernelSize = self.matchingKernelSize
93 """!Warp and optionally PSF-Match calexps onto an a common projection. 95 @anchor MakeCoaddTempExpTask_ 97 @section pipe_tasks_makeCoaddTempExp_Contents Contents 99 - @ref pipe_tasks_makeCoaddTempExp_Purpose 100 - @ref pipe_tasks_makeCoaddTempExp_Initialize 101 - @ref pipe_tasks_makeCoaddTempExp_IO 102 - @ref pipe_tasks_makeCoaddTempExp_Config 103 - @ref pipe_tasks_makeCoaddTempExp_Debug 104 - @ref pipe_tasks_makeCoaddTempExp_Example 106 @section pipe_tasks_makeCoaddTempExp_Purpose Description 108 Warp and optionally PSF-Match calexps onto a common projection, by 109 performing the following operations: 110 - Group calexps by visit/run 111 - For each visit, generate a Warp by calling method @ref makeTempExp. 112 makeTempExp loops over the visit's calexps calling @ref WarpAndPsfMatch 115 The result is a `directWarp` (and/or optionally a `psfMatchedWarp`). 117 @section pipe_tasks_makeCoaddTempExp_Initialize Task Initialization 119 @copydoc \_\_init\_\_ 121 This task has one special keyword argument: passing reuse=True will cause 122 the task to skip the creation of warps that are already present in the 125 @section pipe_tasks_makeCoaddTempExp_IO Invoking the Task 127 This task is primarily designed to be run from the command line. 129 The main method is `runDataRef`, which takes a single butler data reference for the patch(es) 134 WarpType identifies the types of convolutions applied to Warps (previously CoaddTempExps). 135 Only two types are available: direct (for regular Warps/Coadds) and psfMatched 136 (for Warps/Coadds with homogenized PSFs). We expect to add a third type, likelihood, 137 for generating likelihood Coadds with Warps that have been correlated with their own PSF. 139 @section pipe_tasks_makeCoaddTempExp_Config Configuration parameters 141 See @ref MakeCoaddTempExpConfig and parameters inherited from 142 @link lsst.pipe.tasks.coaddBase.CoaddBaseConfig CoaddBaseConfig @endlink 144 @subsection pipe_tasks_MakeCoaddTempExp_psfMatching Guide to PSF-Matching Configs 146 To make `psfMatchedWarps`, select `config.makePsfMatched=True`. The subtask 147 @link lsst.ip.diffim.modelPsfMatch.ModelPsfMatchTask ModelPsfMatchTask @endlink 148 is responsible for the PSF-Matching, and its config is accessed via `config.warpAndPsfMatch.psfMatch`. 149 The optimal configuration depends on aspects of dataset: the pixel scale, average PSF FWHM and 150 dimensions of the PSF kernel. These configs include the requested model PSF, the matching kernel size, 151 padding of the science PSF thumbnail and spatial sampling frequency of the PSF. 153 *Config Guidelines*: The user must specify the size of the model PSF to which to match by setting 154 `config.modelPsf.defaultFwhm` in units of pixels. The appropriate values depends on science case. 155 In general, for a set of input images, this config should equal the FWHM of the visit 156 with the worst seeing. The smallest it should be set to is the median FWHM. The defaults 157 of the other config options offer a reasonable starting point. 158 The following list presents the most common problems that arise from a misconfigured 159 @link lsst.ip.diffim.modelPsfMatch.ModelPsfMatchTask ModelPsfMatchTask @endlink 160 and corresponding solutions. All assume the default Alard-Lupton kernel, with configs accessed via 161 ```config.warpAndPsfMatch.psfMatch.kernel['AL']```. Each item in the list is formatted as: 162 Problem: Explanation. *Solution* 164 *Troublshooting PSF-Matching Configuration:* 165 - Matched PSFs look boxy: The matching kernel is too small. _Increase the matching kernel size. 168 config.warpAndPsfMatch.psfMatch.kernel['AL'].kernelSize=27 # default 21 170 Note that increasing the kernel size also increases runtime. 171 - Matched PSFs look ugly (dipoles, quadropoles, donuts): unable to find good solution 172 for matching kernel. _Provide the matcher with more data by either increasing 173 the spatial sampling by decreasing the spatial cell size,_ 175 config.warpAndPsfMatch.psfMatch.kernel['AL'].sizeCellX = 64 # default 128 176 config.warpAndPsfMatch.psfMatch.kernel['AL'].sizeCellY = 64 # default 128 178 _or increasing the padding around the Science PSF, for example:_ 180 config.warpAndPsfMatch.psfMatch.autoPadPsfTo=1.6 # default 1.4 182 Increasing `autoPadPsfTo` increases the minimum ratio of input PSF dimensions to the 183 matching kernel dimensions, thus increasing the number of pixels available to fit 184 after convolving the PSF with the matching kernel. 185 Optionally, for debugging the effects of padding, the level of padding may be manually 186 controlled by setting turning off the automatic padding and setting the number 187 of pixels by which to pad the PSF: 189 config.warpAndPsfMatch.psfMatch.doAutoPadPsf = False # default True 190 config.warpAndPsfMatch.psfMatch.padPsfBy = 6 # pixels. default 0 192 - Deconvolution: Matching a large PSF to a smaller PSF produces 193 a telltale noise pattern which looks like ripples or a brain. 194 _Increase the size of the requested model PSF. For example:_ 196 config.modelPsf.defaultFwhm = 11 # Gaussian sigma in units of pixels. 198 - High frequency (sometimes checkered) noise: The matching basis functions are too small. 199 _Increase the width of the Gaussian basis functions. For example:_ 201 config.warpAndPsfMatch.psfMatch.kernel['AL'].alardSigGauss=[1.5, 3.0, 6.0] 202 # from default [0.7, 1.5, 3.0] 205 @section pipe_tasks_makeCoaddTempExp_Debug Debug variables 207 MakeCoaddTempExpTask has no debug output, but its subtasks do. 209 @section pipe_tasks_makeCoaddTempExp_Example A complete example of using MakeCoaddTempExpTask 211 This example uses the package ci_hsc to show how MakeCoaddTempExp fits 212 into the larger Data Release Processing. 217 # if not built already: 218 python $(which scons) # this will take a while 220 The following assumes that `processCcd.py` and `makeSkyMap.py` have previously been run 221 (e.g. by building `ci_hsc` above) to generate a repository of calexps and an 222 output respository with the desired SkyMap. The command, 224 makeCoaddTempExp.py $CI_HSC_DIR/DATA --rerun ci_hsc \ 225 --id patch=5,4 tract=0 filter=HSC-I \ 226 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 \ 227 --selectId visit=903988 ccd=23 --selectId visit=903988 ccd=24 \ 228 --config doApplyUberCal=False makePsfMatched=True modelPsf.defaultFwhm=11 230 writes a direct and PSF-Matched Warp to 231 - `$CI_HSC_DIR/DATA/rerun/ci_hsc/deepCoadd/HSC-I/0/5,4/warp-HSC-I-0-5,4-903988.fits` and 232 - `$CI_HSC_DIR/DATA/rerun/ci_hsc/deepCoadd/HSC-I/0/5,4/psfMatchedWarp-HSC-I-0-5,4-903988.fits` 235 @note PSF-Matching in this particular dataset would benefit from adding 236 `--configfile ./matchingConfig.py` to 237 the command line arguments where `matchingConfig.py` is defined by: 240 config.warpAndPsfMatch.psfMatch.kernel['AL'].kernelSize=27 241 config.warpAndPsfMatch.psfMatch.kernel['AL'].alardSigGauss=[1.5, 3.0, 6.0]" > matchingConfig.py 244 Add the option `--help` to see more options. 246 ConfigClass = MakeCoaddTempExpConfig
247 _DefaultName =
"makeCoaddTempExp" 250 CoaddBaseTask.__init__(self, **kwargs)
252 self.makeSubtask(
"warpAndPsfMatch")
256 """!Produce <coaddName>Coadd_<warpType>Warp images by warping and optionally PSF-matching. 258 @param[in] patchRef: data reference for sky map patch. Must include keys "tract", "patch", 259 plus the camera-specific filter key (e.g. "filter" or "band") 260 @return: dataRefList: a list of data references for the new <coaddName>Coadd_directWarps 261 if direct or both warp types are requested and <coaddName>Coadd_psfMatchedWarps if only psfMatched 264 @warning: this task assumes that all exposures in a warp (coaddTempExp) have the same filter. 266 @warning: this task sets the Calib of the coaddTempExp to the Calib of the first calexp 267 with any good pixels in the patch. For a mosaic camera the resulting Calib should be ignored 268 (assembleCoadd should determine zeropoint scaling without referring to it). 273 if self.config.makePsfMatched
and not self.config.makeDirect:
278 calExpRefList = self.
selectExposures(patchRef, skyInfo, selectDataList=selectDataList)
279 if len(calExpRefList) == 0:
280 self.log.warn(
"No exposures to coadd for patch %s", patchRef.dataId)
282 self.log.info(
"Selected %d calexps for patch %s", len(calExpRefList), patchRef.dataId)
283 calExpRefList = [calExpRef
for calExpRef
in calExpRefList
if calExpRef.datasetExists(
"calexp")]
284 self.log.info(
"Processing %d existing calexps for patch %s", len(calExpRefList), patchRef.dataId)
288 self.log.info(
"Processing %d warp exposures for patch %s", len(groupData.groups), patchRef.dataId)
291 for i, (tempExpTuple, calexpRefList)
in enumerate(groupData.groups.items()):
293 tempExpTuple, groupData.keys)
294 if self.
reuse and tempExpRef.datasetExists(datasetType=primaryWarpDataset, write=
True):
295 self.log.info(
"Skipping makeCoaddTempExp for %s; output already exists.", tempExpRef.dataId)
296 dataRefList.append(tempExpRef)
298 self.log.info(
"Processing Warp %d/%d: id=%s", i, len(groupData.groups), tempExpRef.dataId)
304 visitId = int(tempExpRef.dataId[
"visit"])
305 except (KeyError, ValueError):
308 exps = self.
run(calexpRefList, skyInfo, visitId).exposures
310 if any(exps.values()):
311 dataRefList.append(tempExpRef)
313 self.log.warn(
"Warp %s could not be created", tempExpRef.dataId)
315 if self.config.doWrite:
316 for (warpType, exposure)
in exps.items():
317 if exposure
is not None:
323 def run(self, calexpRefList, skyInfo, visitId=0):
324 """Create a Warp from inputs 326 We iterate over the multiple calexps in a single exposure to construct 327 the warp (previously called a coaddTempExp) of that exposure to the 328 supplied tract/patch. 330 Pixels that receive no pixels are set to NAN; this is not correct 331 (violates LSST algorithms group policy), but will be fixed up by 332 interpolating after the coaddition. 334 @param calexpRefList: List of data references for calexps that (may) 335 overlap the patch of interest 336 @param skyInfo: Struct from CoaddBaseTask.getSkyInfo() with geometric 337 information about the patch 338 @param visitId: integer identifier for visit, for the table that will 340 @return a pipeBase Struct containing: 341 - exposures: a dictionary containing the warps requested: 342 "direct": direct warp if config.makeDirect 343 "psfMatched": PSF-matched warp if config.makePsfMatched 347 totGoodPix = {warpType: 0
for warpType
in warpTypeList}
348 didSetMetadata = {warpType:
False for warpType
in warpTypeList}
350 inputRecorder = {warpType: self.inputRecorder.makeCoaddTempExpRecorder(visitId, len(calexpRefList))
351 for warpType
in warpTypeList}
353 modelPsf = self.config.modelPsf.apply()
if self.config.makePsfMatched
else None 354 for calExpInd, calExpRef
in enumerate(calexpRefList):
355 self.log.info(
"Processing calexp %d of %d for this Warp: id=%s",
356 calExpInd+1, len(calexpRefList), calExpRef.dataId)
358 ccdId = calExpRef.get(
"ccdExposureId", immediate=
True)
365 calExpRef = calExpRef.butlerSubset.butler.dataRef(
"calexp", dataId=calExpRef.dataId,
366 tract=skyInfo.tractInfo.getId())
367 calExp = self.
getCalExp(calExpRef, bgSubtracted=self.config.bgSubtracted)
368 except Exception
as e:
369 self.log.warn(
"Calexp %s not found; skipping it: %s", calExpRef.dataId, e)
372 if self.config.doApplySkyCorr:
376 warpedAndMatched = self.warpAndPsfMatch.
run(calExp, modelPsf=modelPsf,
377 wcs=skyInfo.wcs, maxBBox=skyInfo.bbox,
378 makeDirect=self.config.makeDirect,
379 makePsfMatched=self.config.makePsfMatched)
380 except Exception
as e:
381 self.log.warn(
"WarpAndPsfMatch failed for calexp %s; skipping it: %s", calExpRef.dataId, e)
384 numGoodPix = {warpType: 0
for warpType
in warpTypeList}
385 for warpType
in warpTypeList:
386 exposure = warpedAndMatched.getDict()[warpType]
389 coaddTempExp = coaddTempExps[warpType]
390 if didSetMetadata[warpType]:
391 mimg = exposure.getMaskedImage()
392 mimg *= (coaddTempExp.getCalib().getFluxMag0()[0] /
393 exposure.getCalib().getFluxMag0()[0])
395 numGoodPix[warpType] = coaddUtils.copyGoodPixels(
396 coaddTempExp.getMaskedImage(), exposure.getMaskedImage(), self.
getBadPixelMask())
397 totGoodPix[warpType] += numGoodPix[warpType]
398 self.log.debug(
"Calexp %s has %d good pixels in this patch (%.1f%%) for %s",
399 calExpRef.dataId, numGoodPix[warpType],
400 100.0*numGoodPix[warpType]/skyInfo.bbox.getArea(), warpType)
401 if numGoodPix[warpType] > 0
and not didSetMetadata[warpType]:
402 coaddTempExp.setCalib(exposure.getCalib())
403 coaddTempExp.setFilter(exposure.getFilter())
404 coaddTempExp.getInfo().setVisitInfo(exposure.getInfo().getVisitInfo())
406 coaddTempExp.setPsf(exposure.getPsf())
407 didSetMetadata[warpType] =
True 410 inputRecorder[warpType].addCalExp(calExp, ccdId, numGoodPix[warpType])
412 except Exception
as e:
413 self.log.warn(
"Error processing calexp %s; skipping it: %s", calExpRef.dataId, e)
416 for warpType
in warpTypeList:
417 self.log.info(
"%sWarp has %d good pixels (%.1f%%)",
418 warpType, totGoodPix[warpType], 100.0*totGoodPix[warpType]/skyInfo.bbox.getArea())
420 if totGoodPix[warpType] > 0
and didSetMetadata[warpType]:
421 inputRecorder[warpType].finish(coaddTempExps[warpType], totGoodPix[warpType])
422 if warpType ==
"direct":
423 coaddTempExps[warpType].setPsf(
424 CoaddPsf(inputRecorder[warpType].coaddInputs.ccds, skyInfo.wcs,
425 self.config.coaddPsf.makeControl()))
428 coaddTempExps[warpType] =
None 430 result = pipeBase.Struct(exposures=coaddTempExps)
434 def _prepareEmptyExposure(skyInfo):
435 """Produce an empty exposure for a given patch""" 436 exp = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
437 exp.getMaskedImage().set(numpy.nan, afwImage.Mask
438 .getPlaneBitMask(
"NO_DATA"), numpy.inf)
442 """Return list of requested warp types per the config. 445 if self.config.makeDirect:
446 warpTypeList.append(
"direct")
447 if self.config.makePsfMatched:
448 warpTypeList.append(
"psfMatched")
452 """Apply correction to the sky background level 454 Sky corrections can be generated with the 'skyCorrection.py' 455 executable in pipe_drivers. Because the sky model used by that 456 code extends over the entire focal plane, this can produce 457 better sky subtraction. 459 The calexp is updated in-place. 463 dataRef : `lsst.daf.persistence.ButlerDataRef` 464 Data reference for calexp. 465 calexp : `lsst.afw.image.Exposure` or `lsst.afw.image.MaskedImage` 468 bg = dataRef.get(
"skyCorr")
469 if isinstance(calexp, afwImage.Exposure):
470 calexp = calexp.getMaskedImage()
471 calexp -= bg.getImage()
def getCoaddDatasetName(self, warpType="direct")
def getGroupDataRef(butler, datasetType, groupTuple, keys)
Base class for coaddition.
def __init__(self, reuse=False, kwargs)
def _prepareEmptyExposure(skyInfo)
Warp and optionally PSF-Match calexps onto an a common projection.
def getSkyInfo(self, patchRef)
Use getSkyinfo to return the skyMap, tract and patch information, wcs and the outer bbox of the patch...
def getTempExpDatasetName(self, warpType="direct")
def getBadPixelMask(self)
Convenience method to provide the bitmask from the mask plane names.
def run(self, calexpRefList, skyInfo, visitId=0)
def selectExposures(self, patchRef, skyInfo=None, selectDataList=[])
Select exposures to coadd.
def getCalExp(self, dataRef, bgSubtracted)
Return one "calexp" calibrated exposure.
def runDataRef(self, patchRef, selectDataList=[])
Produce <coaddName>Coadd_<warpType>Warp images by warping and optionally PSF-matching.
def getWarpTypeList(self)
def groupPatchExposures(patchDataRef, calexpDataRefList, coaddDatasetType="deepCoadd", tempExpDatasetType="deepCoadd_directWarp")
def applySkyCorr(self, dataRef, calexp)