31 from .coaddBase
import CoaddBaseTask
32 from .warpAndPsfMatch
import WarpAndPsfMatchTask
33 from .coaddHelpers
import groupPatchExposures, getGroupDataRef
35 __all__ = [
"MakeCoaddTempExpTask"]
39 """Raised when data cannot be retrieved for an exposure. 40 When processing patches, sometimes one exposure is missing; this lets us 41 distinguish bewteen that case, and other errors. 47 """Config for MakeCoaddTempExpTask 49 warpAndPsfMatch = pexConfig.ConfigurableField(
50 target=WarpAndPsfMatchTask,
51 doc=
"Task to warp and PSF-match calexp",
53 doWrite = pexConfig.Field(
54 doc=
"persist <coaddName>Coadd_<warpType>Warp",
58 bgSubtracted = pexConfig.Field(
59 doc=
"Work with a background subtracted calexp?",
63 coaddPsf = pexConfig.ConfigField(
64 doc=
"Configuration for CoaddPsf",
67 makeDirect = pexConfig.Field(
68 doc=
"Make direct Warp/Coadds",
72 makePsfMatched = pexConfig.Field(
73 doc=
"Make Psf-Matched Warp/Coadd?",
77 doApplySkyCorr = pexConfig.Field(dtype=bool, default=
False, doc=
"Apply sky correction?")
80 CoaddBaseTask.ConfigClass.validate(self)
82 raise RuntimeError(
"At least one of config.makePsfMatched and config.makeDirect must be True")
85 log.warn(
"Config doPsfMatch deprecated. Setting makePsfMatched=True and makeDirect=False")
90 CoaddBaseTask.ConfigClass.setDefaults(self)
91 self.
warpAndPsfMatch.psfMatch.kernel.active.kernelSize = self.matchingKernelSize
102 r"""!Warp and optionally PSF-Match calexps onto an a common projection. 104 @anchor MakeCoaddTempExpTask_ 106 @section pipe_tasks_makeCoaddTempExp_Contents Contents 108 - @ref pipe_tasks_makeCoaddTempExp_Purpose 109 - @ref pipe_tasks_makeCoaddTempExp_Initialize 110 - @ref pipe_tasks_makeCoaddTempExp_IO 111 - @ref pipe_tasks_makeCoaddTempExp_Config 112 - @ref pipe_tasks_makeCoaddTempExp_Debug 113 - @ref pipe_tasks_makeCoaddTempExp_Example 115 @section pipe_tasks_makeCoaddTempExp_Purpose Description 117 Warp and optionally PSF-Match calexps onto a common projection, by 118 performing the following operations: 119 - Group calexps by visit/run 120 - For each visit, generate a Warp by calling method @ref makeTempExp. 121 makeTempExp loops over the visit's calexps calling @ref WarpAndPsfMatch 124 The result is a `directWarp` (and/or optionally a `psfMatchedWarp`). 126 @section pipe_tasks_makeCoaddTempExp_Initialize Task Initialization 128 @copydoc \_\_init\_\_ 130 This task has one special keyword argument: passing reuse=True will cause 131 the task to skip the creation of warps that are already present in the 134 @section pipe_tasks_makeCoaddTempExp_IO Invoking the Task 136 This task is primarily designed to be run from the command line. 138 The main method is `runDataRef`, which takes a single butler data reference for the patch(es) 143 WarpType identifies the types of convolutions applied to Warps (previously CoaddTempExps). 144 Only two types are available: direct (for regular Warps/Coadds) and psfMatched 145 (for Warps/Coadds with homogenized PSFs). We expect to add a third type, likelihood, 146 for generating likelihood Coadds with Warps that have been correlated with their own PSF. 148 @section pipe_tasks_makeCoaddTempExp_Config Configuration parameters 150 See @ref MakeCoaddTempExpConfig and parameters inherited from 151 @link lsst.pipe.tasks.coaddBase.CoaddBaseConfig CoaddBaseConfig @endlink 153 @subsection pipe_tasks_MakeCoaddTempExp_psfMatching Guide to PSF-Matching Configs 155 To make `psfMatchedWarps`, select `config.makePsfMatched=True`. The subtask 156 @link lsst.ip.diffim.modelPsfMatch.ModelPsfMatchTask ModelPsfMatchTask @endlink 157 is responsible for the PSF-Matching, and its config is accessed via `config.warpAndPsfMatch.psfMatch`. 158 The optimal configuration depends on aspects of dataset: the pixel scale, average PSF FWHM and 159 dimensions of the PSF kernel. These configs include the requested model PSF, the matching kernel size, 160 padding of the science PSF thumbnail and spatial sampling frequency of the PSF. 162 *Config Guidelines*: The user must specify the size of the model PSF to which to match by setting 163 `config.modelPsf.defaultFwhm` in units of pixels. The appropriate values depends on science case. 164 In general, for a set of input images, this config should equal the FWHM of the visit 165 with the worst seeing. The smallest it should be set to is the median FWHM. The defaults 166 of the other config options offer a reasonable starting point. 167 The following list presents the most common problems that arise from a misconfigured 168 @link lsst.ip.diffim.modelPsfMatch.ModelPsfMatchTask ModelPsfMatchTask @endlink 169 and corresponding solutions. All assume the default Alard-Lupton kernel, with configs accessed via 170 ```config.warpAndPsfMatch.psfMatch.kernel['AL']```. Each item in the list is formatted as: 171 Problem: Explanation. *Solution* 173 *Troublshooting PSF-Matching Configuration:* 174 - Matched PSFs look boxy: The matching kernel is too small. _Increase the matching kernel size. 177 config.warpAndPsfMatch.psfMatch.kernel['AL'].kernelSize=27 # default 21 179 Note that increasing the kernel size also increases runtime. 180 - Matched PSFs look ugly (dipoles, quadropoles, donuts): unable to find good solution 181 for matching kernel. _Provide the matcher with more data by either increasing 182 the spatial sampling by decreasing the spatial cell size,_ 184 config.warpAndPsfMatch.psfMatch.kernel['AL'].sizeCellX = 64 # default 128 185 config.warpAndPsfMatch.psfMatch.kernel['AL'].sizeCellY = 64 # default 128 187 _or increasing the padding around the Science PSF, for example:_ 189 config.warpAndPsfMatch.psfMatch.autoPadPsfTo=1.6 # default 1.4 191 Increasing `autoPadPsfTo` increases the minimum ratio of input PSF dimensions to the 192 matching kernel dimensions, thus increasing the number of pixels available to fit 193 after convolving the PSF with the matching kernel. 194 Optionally, for debugging the effects of padding, the level of padding may be manually 195 controlled by setting turning off the automatic padding and setting the number 196 of pixels by which to pad the PSF: 198 config.warpAndPsfMatch.psfMatch.doAutoPadPsf = False # default True 199 config.warpAndPsfMatch.psfMatch.padPsfBy = 6 # pixels. default 0 201 - Deconvolution: Matching a large PSF to a smaller PSF produces 202 a telltale noise pattern which looks like ripples or a brain. 203 _Increase the size of the requested model PSF. For example:_ 205 config.modelPsf.defaultFwhm = 11 # Gaussian sigma in units of pixels. 207 - High frequency (sometimes checkered) noise: The matching basis functions are too small. 208 _Increase the width of the Gaussian basis functions. For example:_ 210 config.warpAndPsfMatch.psfMatch.kernel['AL'].alardSigGauss=[1.5, 3.0, 6.0] 211 # from default [0.7, 1.5, 3.0] 214 @section pipe_tasks_makeCoaddTempExp_Debug Debug variables 216 MakeCoaddTempExpTask has no debug output, but its subtasks do. 218 @section pipe_tasks_makeCoaddTempExp_Example A complete example of using MakeCoaddTempExpTask 220 This example uses the package ci_hsc to show how MakeCoaddTempExp fits 221 into the larger Data Release Processing. 226 # if not built already: 227 python $(which scons) # this will take a while 229 The following assumes that `processCcd.py` and `makeSkyMap.py` have previously been run 230 (e.g. by building `ci_hsc` above) to generate a repository of calexps and an 231 output respository with the desired SkyMap. The command, 233 makeCoaddTempExp.py $CI_HSC_DIR/DATA --rerun ci_hsc \ 234 --id patch=5,4 tract=0 filter=HSC-I \ 235 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 \ 236 --selectId visit=903988 ccd=23 --selectId visit=903988 ccd=24 \ 237 --config doApplyUberCal=False makePsfMatched=True modelPsf.defaultFwhm=11 239 writes a direct and PSF-Matched Warp to 240 - `$CI_HSC_DIR/DATA/rerun/ci_hsc/deepCoadd/HSC-I/0/5,4/warp-HSC-I-0-5,4-903988.fits` and 241 - `$CI_HSC_DIR/DATA/rerun/ci_hsc/deepCoadd/HSC-I/0/5,4/psfMatchedWarp-HSC-I-0-5,4-903988.fits` 244 @note PSF-Matching in this particular dataset would benefit from adding 245 `--configfile ./matchingConfig.py` to 246 the command line arguments where `matchingConfig.py` is defined by: 249 config.warpAndPsfMatch.psfMatch.kernel['AL'].kernelSize=27 250 config.warpAndPsfMatch.psfMatch.kernel['AL'].alardSigGauss=[1.5, 3.0, 6.0]" > matchingConfig.py 253 Add the option `--help` to see more options. 255 ConfigClass = MakeCoaddTempExpConfig
256 _DefaultName =
"makeCoaddTempExp" 259 CoaddBaseTask.__init__(self, **kwargs)
261 self.makeSubtask(
"warpAndPsfMatch")
265 """!Produce <coaddName>Coadd_<warpType>Warp images by warping and optionally PSF-matching. 267 @param[in] patchRef: data reference for sky map patch. Must include keys "tract", "patch", 268 plus the camera-specific filter key (e.g. "filter" or "band") 269 @return: dataRefList: a list of data references for the new <coaddName>Coadd_directWarps 270 if direct or both warp types are requested and <coaddName>Coadd_psfMatchedWarps if only psfMatched 273 @warning: this task assumes that all exposures in a warp (coaddTempExp) have the same filter. 275 @warning: this task sets the Calib of the coaddTempExp to the Calib of the first calexp 276 with any good pixels in the patch. For a mosaic camera the resulting Calib should be ignored 277 (assembleCoadd should determine zeropoint scaling without referring to it). 282 if self.config.makePsfMatched
and not self.config.makeDirect:
287 calExpRefList = self.
selectExposures(patchRef, skyInfo, selectDataList=selectDataList)
288 if len(calExpRefList) == 0:
289 self.log.warn(
"No exposures to coadd for patch %s", patchRef.dataId)
291 self.log.info(
"Selected %d calexps for patch %s", len(calExpRefList), patchRef.dataId)
292 calExpRefList = [calExpRef
for calExpRef
in calExpRefList
if calExpRef.datasetExists(
"calexp")]
293 self.log.info(
"Processing %d existing calexps for patch %s", len(calExpRefList), patchRef.dataId)
297 self.log.info(
"Processing %d warp exposures for patch %s", len(groupData.groups), patchRef.dataId)
300 for i, (tempExpTuple, calexpRefList)
in enumerate(groupData.groups.items()):
302 tempExpTuple, groupData.keys)
303 if self.
reuse and tempExpRef.datasetExists(datasetType=primaryWarpDataset, write=
True):
304 self.log.info(
"Skipping makeCoaddTempExp for %s; output already exists.", tempExpRef.dataId)
305 dataRefList.append(tempExpRef)
307 self.log.info(
"Processing Warp %d/%d: id=%s", i, len(groupData.groups), tempExpRef.dataId)
313 visitId = int(tempExpRef.dataId[
"visit"])
314 except (KeyError, ValueError):
317 exps = self.
run(calexpRefList, skyInfo, visitId).exposures
319 if any(exps.values()):
320 dataRefList.append(tempExpRef)
322 self.log.warn(
"Warp %s could not be created", tempExpRef.dataId)
324 if self.config.doWrite:
325 for (warpType, exposure)
in exps.items():
326 if exposure
is not None:
332 def run(self, calexpRefList, skyInfo, visitId=0):
333 """Create a Warp from inputs 335 We iterate over the multiple calexps in a single exposure to construct 336 the warp (previously called a coaddTempExp) of that exposure to the 337 supplied tract/patch. 339 Pixels that receive no pixels are set to NAN; this is not correct 340 (violates LSST algorithms group policy), but will be fixed up by 341 interpolating after the coaddition. 343 @param calexpRefList: List of data references for calexps that (may) 344 overlap the patch of interest 345 @param skyInfo: Struct from CoaddBaseTask.getSkyInfo() with geometric 346 information about the patch 347 @param visitId: integer identifier for visit, for the table that will 349 @return a pipeBase Struct containing: 350 - exposures: a dictionary containing the warps requested: 351 "direct": direct warp if config.makeDirect 352 "psfMatched": PSF-matched warp if config.makePsfMatched 356 totGoodPix = {warpType: 0
for warpType
in warpTypeList}
357 didSetMetadata = {warpType:
False for warpType
in warpTypeList}
359 inputRecorder = {warpType: self.inputRecorder.makeCoaddTempExpRecorder(visitId, len(calexpRefList))
360 for warpType
in warpTypeList}
362 modelPsf = self.config.modelPsf.apply()
if self.config.makePsfMatched
else None 363 for calExpInd, calExpRef
in enumerate(calexpRefList):
364 self.log.info(
"Processing calexp %d of %d for this Warp: id=%s",
365 calExpInd+1, len(calexpRefList), calExpRef.dataId)
367 ccdId = calExpRef.get(
"ccdExposureId", immediate=
True)
374 calExpRef = calExpRef.butlerSubset.butler.dataRef(
"calexp", dataId=calExpRef.dataId,
375 tract=skyInfo.tractInfo.getId())
377 except Exception
as e:
378 self.log.warn(
"Calexp %s not found; skipping it: %s", calExpRef.dataId, e)
381 if self.config.doApplySkyCorr:
385 warpedAndMatched = self.warpAndPsfMatch.
run(calExp, modelPsf=modelPsf,
386 wcs=skyInfo.wcs, maxBBox=skyInfo.bbox,
387 makeDirect=self.config.makeDirect,
388 makePsfMatched=self.config.makePsfMatched)
389 except Exception
as e:
390 self.log.warn(
"WarpAndPsfMatch failed for calexp %s; skipping it: %s", calExpRef.dataId, e)
393 numGoodPix = {warpType: 0
for warpType
in warpTypeList}
394 for warpType
in warpTypeList:
395 exposure = warpedAndMatched.getDict()[warpType]
398 coaddTempExp = coaddTempExps[warpType]
399 if didSetMetadata[warpType]:
400 mimg = exposure.getMaskedImage()
401 mimg *= (coaddTempExp.getCalib().getFluxMag0()[0] /
402 exposure.getCalib().getFluxMag0()[0])
404 numGoodPix[warpType] = coaddUtils.copyGoodPixels(
405 coaddTempExp.getMaskedImage(), exposure.getMaskedImage(), self.
getBadPixelMask())
406 totGoodPix[warpType] += numGoodPix[warpType]
407 self.log.debug(
"Calexp %s has %d good pixels in this patch (%.1f%%) for %s",
408 calExpRef.dataId, numGoodPix[warpType],
409 100.0*numGoodPix[warpType]/skyInfo.bbox.getArea(), warpType)
410 if numGoodPix[warpType] > 0
and not didSetMetadata[warpType]:
411 coaddTempExp.setCalib(exposure.getCalib())
412 coaddTempExp.setFilter(exposure.getFilter())
413 coaddTempExp.getInfo().setVisitInfo(exposure.getInfo().getVisitInfo())
415 coaddTempExp.setPsf(exposure.getPsf())
416 didSetMetadata[warpType] =
True 419 inputRecorder[warpType].addCalExp(calExp, ccdId, numGoodPix[warpType])
421 except Exception
as e:
422 self.log.warn(
"Error processing calexp %s; skipping it: %s", calExpRef.dataId, e)
425 for warpType
in warpTypeList:
426 self.log.info(
"%sWarp has %d good pixels (%.1f%%)",
427 warpType, totGoodPix[warpType], 100.0*totGoodPix[warpType]/skyInfo.bbox.getArea())
429 if totGoodPix[warpType] > 0
and didSetMetadata[warpType]:
430 inputRecorder[warpType].finish(coaddTempExps[warpType], totGoodPix[warpType])
431 if warpType ==
"direct":
432 coaddTempExps[warpType].setPsf(
433 CoaddPsf(inputRecorder[warpType].coaddInputs.ccds, skyInfo.wcs,
434 self.config.coaddPsf.makeControl()))
437 coaddTempExps[warpType] =
None 439 result = pipeBase.Struct(exposures=coaddTempExps)
443 """Return one calibrated Exposure, possibly with an updated SkyWcs. 445 @param[in] dataRef a sensor-level data reference 446 @param[in] bgSubtracted return calexp with background subtracted? If False get the 447 calexp's background background model and add it to the calexp. 448 @return calibrated exposure 450 @raises MissingExposureError If data for the exposure is not available. 452 If config.doApplyUberCal, the exposure will be photometrically 453 calibrated via the `jointcal_photoCalib` dataset and have its SkyWcs 454 updated to the `jointcal_wcs`, otherwise it will be calibrated via the 455 Exposure's own Calib and have the original SkyWcs. 458 exposure = dataRef.get(
"calexp", immediate=
True)
459 except dafPersist.NoResults
as e:
463 background = dataRef.get(
"calexpBackground", immediate=
True)
464 mi = exposure.getMaskedImage()
465 mi += background.getImage()
468 if self.config.doApplyUberCal:
469 if self.config.useMeasMosaic:
470 from lsst.meas.mosaic
import applyMosaicResultsExposure
473 fluxMag0Err = exposure.getCalib().getFluxMag0()[1]
475 applyMosaicResultsExposure(dataRef, calexp=exposure)
476 except dafPersist.NoResults
as e:
478 fluxMag0 = exposure.getCalib().getFluxMag0()[0]
479 photoCalib = afwImage.PhotoCalib(1.0/fluxMag0,
480 fluxMag0Err/fluxMag0**2,
483 photoCalib = dataRef.get(
"jointcal_photoCalib")
484 skyWcs = dataRef.get(
"jointcal_wcs")
485 exposure.setWcs(skyWcs)
487 fluxMag0 = exposure.getCalib().getFluxMag0()
488 photoCalib = afwImage.PhotoCalib(1.0/fluxMag0[0],
489 fluxMag0[1]/fluxMag0[0]**2,
492 exposure.maskedImage = photoCalib.calibrateImage(exposure.maskedImage)
493 exposure.maskedImage /= photoCalib.getCalibrationMean()
494 exposure.setCalib(afwImage.Calib(1/photoCalib.getCalibrationMean()))
500 def _prepareEmptyExposure(skyInfo):
501 """Produce an empty exposure for a given patch""" 502 exp = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
503 exp.getMaskedImage().set(numpy.nan, afwImage.Mask
504 .getPlaneBitMask(
"NO_DATA"), numpy.inf)
508 """Return list of requested warp types per the config. 511 if self.config.makeDirect:
512 warpTypeList.append(
"direct")
513 if self.config.makePsfMatched:
514 warpTypeList.append(
"psfMatched")
518 """Apply correction to the sky background level 520 Sky corrections can be generated with the 'skyCorrection.py' 521 executable in pipe_drivers. Because the sky model used by that 522 code extends over the entire focal plane, this can produce 523 better sky subtraction. 525 The calexp is updated in-place. 529 dataRef : `lsst.daf.persistence.ButlerDataRef` 530 Data reference for calexp. 531 calexp : `lsst.afw.image.Exposure` or `lsst.afw.image.MaskedImage` 534 bg = dataRef.get(
"skyCorr")
535 if isinstance(calexp, afwImage.Exposure):
536 calexp = calexp.getMaskedImage()
537 calexp -= bg.getImage()
def getCoaddDatasetName(self, warpType="direct")
def getGroupDataRef(butler, datasetType, groupTuple, keys)
Base class for coaddition.
def __init__(self, reuse=False, kwargs)
def _prepareEmptyExposure(skyInfo)
Warp and optionally PSF-Match calexps onto an a common projection.
def getSkyInfo(self, patchRef)
Use getSkyinfo to return the skyMap, tract and patch information, wcs and the outer bbox of the patch...
def getTempExpDatasetName(self, warpType="direct")
def getBadPixelMask(self)
Convenience method to provide the bitmask from the mask plane names.
def run(self, calexpRefList, skyInfo, visitId=0)
def getCalibratedExposure(self, dataRef, bgSubtracted)
def selectExposures(self, patchRef, skyInfo=None, selectDataList=[])
Select exposures to coadd.
def runDataRef(self, patchRef, selectDataList=[])
Produce <coaddName>Coadd_<warpType>Warp images by warping and optionally PSF-matching.
def getWarpTypeList(self)
def groupPatchExposures(patchDataRef, calexpDataRefList, coaddDatasetType="deepCoadd", tempExpDatasetType="deepCoadd_directWarp")
def applySkyCorr(self, dataRef, calexp)