29 import lsst.pipe.base.connectionTypes
as connectionTypes
35 from .coaddBase
import CoaddBaseTask, makeSkyInfo, reorderAndPadList
36 from .warpAndPsfMatch
import WarpAndPsfMatchTask
37 from .coaddHelpers
import groupPatchExposures, getGroupDataRef
38 from collections.abc
import Iterable
40 __all__ = [
"MakeCoaddTempExpTask",
"MakeWarpTask",
"MakeWarpConfig"]
44 """Raised when data cannot be retrieved for an exposure.
45 When processing patches, sometimes one exposure is missing; this lets us
46 distinguish bewteen that case, and other errors.
52 """Config for MakeCoaddTempExpTask
54 warpAndPsfMatch = pexConfig.ConfigurableField(
55 target=WarpAndPsfMatchTask,
56 doc=
"Task to warp and PSF-match calexp",
58 doWrite = pexConfig.Field(
59 doc=
"persist <coaddName>Coadd_<warpType>Warp",
63 bgSubtracted = pexConfig.Field(
64 doc=
"Work with a background subtracted calexp?",
68 coaddPsf = pexConfig.ConfigField(
69 doc=
"Configuration for CoaddPsf",
72 makeDirect = pexConfig.Field(
73 doc=
"Make direct Warp/Coadds",
77 makePsfMatched = pexConfig.Field(
78 doc=
"Make Psf-Matched Warp/Coadd?",
83 doWriteEmptyWarps = pexConfig.Field(
86 doc=
"Write out warps even if they are empty"
89 hasFakes = pexConfig.Field(
90 doc=
"Should be set to True if fake sources have been inserted into the input data.",
94 doApplySkyCorr = pexConfig.Field(dtype=bool, default=
False, doc=
"Apply sky correction?")
97 CoaddBaseTask.ConfigClass.validate(self)
99 raise RuntimeError(
"At least one of config.makePsfMatched and config.makeDirect must be True")
102 log.warning(
"Config doPsfMatch deprecated. Setting makePsfMatched=True and makeDirect=False")
107 CoaddBaseTask.ConfigClass.setDefaults(self)
108 self.
warpAndPsfMatchwarpAndPsfMatch.psfMatch.kernel.active.kernelSize = self.matchingKernelSize
119 r"""!Warp and optionally PSF-Match calexps onto an a common projection.
121 @anchor MakeCoaddTempExpTask_
123 @section pipe_tasks_makeCoaddTempExp_Contents Contents
125 - @ref pipe_tasks_makeCoaddTempExp_Purpose
126 - @ref pipe_tasks_makeCoaddTempExp_Initialize
127 - @ref pipe_tasks_makeCoaddTempExp_IO
128 - @ref pipe_tasks_makeCoaddTempExp_Config
129 - @ref pipe_tasks_makeCoaddTempExp_Debug
130 - @ref pipe_tasks_makeCoaddTempExp_Example
132 @section pipe_tasks_makeCoaddTempExp_Purpose Description
134 Warp and optionally PSF-Match calexps onto a common projection, by
135 performing the following operations:
136 - Group calexps by visit/run
137 - For each visit, generate a Warp by calling method @ref makeTempExp.
138 makeTempExp loops over the visit's calexps calling @ref WarpAndPsfMatch
141 The result is a `directWarp` (and/or optionally a `psfMatchedWarp`).
143 @section pipe_tasks_makeCoaddTempExp_Initialize Task Initialization
145 @copydoc \_\_init\_\_
147 This task has one special keyword argument: passing reuse=True will cause
148 the task to skip the creation of warps that are already present in the
151 @section pipe_tasks_makeCoaddTempExp_IO Invoking the Task
153 This task is primarily designed to be run from the command line.
155 The main method is `runDataRef`, which takes a single butler data reference for the patch(es)
160 WarpType identifies the types of convolutions applied to Warps (previously CoaddTempExps).
161 Only two types are available: direct (for regular Warps/Coadds) and psfMatched
162 (for Warps/Coadds with homogenized PSFs). We expect to add a third type, likelihood,
163 for generating likelihood Coadds with Warps that have been correlated with their own PSF.
165 @section pipe_tasks_makeCoaddTempExp_Config Configuration parameters
167 See @ref MakeCoaddTempExpConfig and parameters inherited from
168 @link lsst.pipe.tasks.coaddBase.CoaddBaseConfig CoaddBaseConfig @endlink
170 @subsection pipe_tasks_MakeCoaddTempExp_psfMatching Guide to PSF-Matching Configs
172 To make `psfMatchedWarps`, select `config.makePsfMatched=True`. The subtask
173 @link lsst.ip.diffim.modelPsfMatch.ModelPsfMatchTask ModelPsfMatchTask @endlink
174 is responsible for the PSF-Matching, and its config is accessed via `config.warpAndPsfMatch.psfMatch`.
175 The optimal configuration depends on aspects of dataset: the pixel scale, average PSF FWHM and
176 dimensions of the PSF kernel. These configs include the requested model PSF, the matching kernel size,
177 padding of the science PSF thumbnail and spatial sampling frequency of the PSF.
179 *Config Guidelines*: The user must specify the size of the model PSF to which to match by setting
180 `config.modelPsf.defaultFwhm` in units of pixels. The appropriate values depends on science case.
181 In general, for a set of input images, this config should equal the FWHM of the visit
182 with the worst seeing. The smallest it should be set to is the median FWHM. The defaults
183 of the other config options offer a reasonable starting point.
184 The following list presents the most common problems that arise from a misconfigured
185 @link lsst.ip.diffim.modelPsfMatch.ModelPsfMatchTask ModelPsfMatchTask @endlink
186 and corresponding solutions. All assume the default Alard-Lupton kernel, with configs accessed via
187 ```config.warpAndPsfMatch.psfMatch.kernel['AL']```. Each item in the list is formatted as:
188 Problem: Explanation. *Solution*
190 *Troublshooting PSF-Matching Configuration:*
191 - Matched PSFs look boxy: The matching kernel is too small. _Increase the matching kernel size.
194 config.warpAndPsfMatch.psfMatch.kernel['AL'].kernelSize=27 # default 21
196 Note that increasing the kernel size also increases runtime.
197 - Matched PSFs look ugly (dipoles, quadropoles, donuts): unable to find good solution
198 for matching kernel. _Provide the matcher with more data by either increasing
199 the spatial sampling by decreasing the spatial cell size,_
201 config.warpAndPsfMatch.psfMatch.kernel['AL'].sizeCellX = 64 # default 128
202 config.warpAndPsfMatch.psfMatch.kernel['AL'].sizeCellY = 64 # default 128
204 _or increasing the padding around the Science PSF, for example:_
206 config.warpAndPsfMatch.psfMatch.autoPadPsfTo=1.6 # default 1.4
208 Increasing `autoPadPsfTo` increases the minimum ratio of input PSF dimensions to the
209 matching kernel dimensions, thus increasing the number of pixels available to fit
210 after convolving the PSF with the matching kernel.
211 Optionally, for debugging the effects of padding, the level of padding may be manually
212 controlled by setting turning off the automatic padding and setting the number
213 of pixels by which to pad the PSF:
215 config.warpAndPsfMatch.psfMatch.doAutoPadPsf = False # default True
216 config.warpAndPsfMatch.psfMatch.padPsfBy = 6 # pixels. default 0
218 - Deconvolution: Matching a large PSF to a smaller PSF produces
219 a telltale noise pattern which looks like ripples or a brain.
220 _Increase the size of the requested model PSF. For example:_
222 config.modelPsf.defaultFwhm = 11 # Gaussian sigma in units of pixels.
224 - High frequency (sometimes checkered) noise: The matching basis functions are too small.
225 _Increase the width of the Gaussian basis functions. For example:_
227 config.warpAndPsfMatch.psfMatch.kernel['AL'].alardSigGauss=[1.5, 3.0, 6.0]
228 # from default [0.7, 1.5, 3.0]
231 @section pipe_tasks_makeCoaddTempExp_Debug Debug variables
233 MakeCoaddTempExpTask has no debug output, but its subtasks do.
235 @section pipe_tasks_makeCoaddTempExp_Example A complete example of using MakeCoaddTempExpTask
237 This example uses the package ci_hsc to show how MakeCoaddTempExp fits
238 into the larger Data Release Processing.
243 # if not built already:
244 python $(which scons) # this will take a while
246 The following assumes that `processCcd.py` and `makeSkyMap.py` have previously been run
247 (e.g. by building `ci_hsc` above) to generate a repository of calexps and an
248 output respository with the desired SkyMap. The command,
250 makeCoaddTempExp.py $CI_HSC_DIR/DATA --rerun ci_hsc \
251 --id patch=5,4 tract=0 filter=HSC-I \
252 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 \
253 --selectId visit=903988 ccd=23 --selectId visit=903988 ccd=24 \
254 --config doApplyExternalPhotoCalib=False doApplyExternalSkyWcs=False \
255 makePsfMatched=True modelPsf.defaultFwhm=11
257 writes a direct and PSF-Matched Warp to
258 - `$CI_HSC_DIR/DATA/rerun/ci_hsc/deepCoadd/HSC-I/0/5,4/warp-HSC-I-0-5,4-903988.fits` and
259 - `$CI_HSC_DIR/DATA/rerun/ci_hsc/deepCoadd/HSC-I/0/5,4/psfMatchedWarp-HSC-I-0-5,4-903988.fits`
262 @note PSF-Matching in this particular dataset would benefit from adding
263 `--configfile ./matchingConfig.py` to
264 the command line arguments where `matchingConfig.py` is defined by:
267 config.warpAndPsfMatch.psfMatch.kernel['AL'].kernelSize=27
268 config.warpAndPsfMatch.psfMatch.kernel['AL'].alardSigGauss=[1.5, 3.0, 6.0]" > matchingConfig.py
271 Add the option `--help` to see more options.
273 ConfigClass = MakeCoaddTempExpConfig
274 _DefaultName =
"makeCoaddTempExp"
277 CoaddBaseTask.__init__(self, **kwargs)
279 self.makeSubtask(
"warpAndPsfMatch")
280 if self.config.hasFakes:
287 """!Produce <coaddName>Coadd_<warpType>Warp images by warping and optionally PSF-matching.
289 @param[in] patchRef: data reference for sky map patch. Must include keys "tract", "patch",
290 plus the camera-specific filter key (e.g. "filter" or "band")
291 @return: dataRefList: a list of data references for the new <coaddName>Coadd_directWarps
292 if direct or both warp types are requested and <coaddName>Coadd_psfMatchedWarps if only psfMatched
295 @warning: this task assumes that all exposures in a warp (coaddTempExp) have the same filter.
297 @warning: this task sets the PhotoCalib of the coaddTempExp to the PhotoCalib of the first calexp
298 with any good pixels in the patch. For a mosaic camera the resulting PhotoCalib should be ignored
299 (assembleCoadd should determine zeropoint scaling without referring to it).
304 if self.config.makePsfMatched
and not self.config.makeDirect:
309 calExpRefList = self.
selectExposuresselectExposures(patchRef, skyInfo, selectDataList=selectDataList)
311 if len(calExpRefList) == 0:
312 self.log.warning(
"No exposures to coadd for patch %s", patchRef.dataId)
314 self.log.info(
"Selected %d calexps for patch %s", len(calExpRefList), patchRef.dataId)
315 calExpRefList = [calExpRef
for calExpRef
in calExpRefList
if calExpRef.datasetExists(self.
calexpTypecalexpType)]
316 self.log.info(
"Processing %d existing calexps for patch %s", len(calExpRefList), patchRef.dataId)
320 self.log.info(
"Processing %d warp exposures for patch %s", len(groupData.groups), patchRef.dataId)
323 for i, (tempExpTuple, calexpRefList)
in enumerate(groupData.groups.items()):
325 tempExpTuple, groupData.keys)
326 if self.
reusereuse
and tempExpRef.datasetExists(datasetType=primaryWarpDataset, write=
True):
327 self.log.info(
"Skipping makeCoaddTempExp for %s; output already exists.", tempExpRef.dataId)
328 dataRefList.append(tempExpRef)
330 self.log.info(
"Processing Warp %d/%d: id=%s", i, len(groupData.groups), tempExpRef.dataId)
336 visitId = int(tempExpRef.dataId[
"visit"])
337 except (KeyError, ValueError):
344 for calExpInd, calExpRef
in enumerate(calexpRefList):
345 self.log.info(
"Reading calexp %s of %s for Warp id=%s", calExpInd+1, len(calexpRefList),
348 ccdId = calExpRef.get(
"ccdExposureId", immediate=
True)
355 calExpRef = calExpRef.butlerSubset.butler.dataRef(self.
calexpTypecalexpType,
356 dataId=calExpRef.dataId,
357 tract=skyInfo.tractInfo.getId())
358 calExp = self.
getCalibratedExposuregetCalibratedExposure(calExpRef, bgSubtracted=self.config.bgSubtracted)
359 except Exception
as e:
360 self.log.warning(
"Calexp %s not found; skipping it: %s", calExpRef.dataId, e)
363 if self.config.doApplySkyCorr:
366 calExpList.append(calExp)
367 ccdIdList.append(ccdId)
368 dataIdList.append(calExpRef.dataId)
370 exps = self.
runrun(calExpList, ccdIdList, skyInfo, visitId, dataIdList).exposures
372 if any(exps.values()):
373 dataRefList.append(tempExpRef)
375 self.log.warning(
"Warp %s could not be created", tempExpRef.dataId)
377 if self.config.doWrite:
378 for (warpType, exposure)
in exps.items():
379 if exposure
is not None:
386 def run(self, calExpList, ccdIdList, skyInfo, visitId=0, dataIdList=None, **kwargs):
387 """Create a Warp from inputs
389 We iterate over the multiple calexps in a single exposure to construct
390 the warp (previously called a coaddTempExp) of that exposure to the
391 supplied tract/patch.
393 Pixels that receive no pixels are set to NAN; this is not correct
394 (violates LSST algorithms group policy), but will be fixed up by
395 interpolating after the coaddition.
397 @param calexpRefList: List of data references for calexps that (may)
398 overlap the patch of interest
399 @param skyInfo: Struct from CoaddBaseTask.getSkyInfo() with geometric
400 information about the patch
401 @param visitId: integer identifier for visit, for the table that will
403 @return a pipeBase Struct containing:
404 - exposures: a dictionary containing the warps requested:
405 "direct": direct warp if config.makeDirect
406 "psfMatched": PSF-matched warp if config.makePsfMatched
410 totGoodPix = {warpType: 0
for warpType
in warpTypeList}
411 didSetMetadata = {warpType:
False for warpType
in warpTypeList}
412 coaddTempExps = {warpType: self.
_prepareEmptyExposure_prepareEmptyExposure(skyInfo)
for warpType
in warpTypeList}
413 inputRecorder = {warpType: self.inputRecorder.makeCoaddTempExpRecorder(visitId, len(calExpList))
414 for warpType
in warpTypeList}
416 modelPsf = self.config.modelPsf.apply()
if self.config.makePsfMatched
else None
417 if dataIdList
is None:
418 dataIdList = ccdIdList
420 for calExpInd, (calExp, ccdId, dataId)
in enumerate(zip(calExpList, ccdIdList, dataIdList)):
421 self.log.info(
"Processing calexp %d of %d for this Warp: id=%s",
422 calExpInd+1, len(calExpList), dataId)
425 warpedAndMatched = self.warpAndPsfMatch.
run(calExp, modelPsf=modelPsf,
426 wcs=skyInfo.wcs, maxBBox=skyInfo.bbox,
427 makeDirect=self.config.makeDirect,
428 makePsfMatched=self.config.makePsfMatched)
429 except Exception
as e:
430 self.log.warning(
"WarpAndPsfMatch failed for calexp %s; skipping it: %s", dataId, e)
433 numGoodPix = {warpType: 0
for warpType
in warpTypeList}
434 for warpType
in warpTypeList:
435 exposure = warpedAndMatched.getDict()[warpType]
438 coaddTempExp = coaddTempExps[warpType]
439 if didSetMetadata[warpType]:
440 mimg = exposure.getMaskedImage()
441 mimg *= (coaddTempExp.getPhotoCalib().getInstFluxAtZeroMagnitude()
442 / exposure.getPhotoCalib().getInstFluxAtZeroMagnitude())
444 numGoodPix[warpType] = coaddUtils.copyGoodPixels(
445 coaddTempExp.getMaskedImage(), exposure.getMaskedImage(), self.
getBadPixelMaskgetBadPixelMask())
446 totGoodPix[warpType] += numGoodPix[warpType]
447 self.log.debug(
"Calexp %s has %d good pixels in this patch (%.1f%%) for %s",
448 dataId, numGoodPix[warpType],
449 100.0*numGoodPix[warpType]/skyInfo.bbox.getArea(), warpType)
450 if numGoodPix[warpType] > 0
and not didSetMetadata[warpType]:
451 coaddTempExp.setPhotoCalib(exposure.getPhotoCalib())
452 coaddTempExp.setFilterLabel(exposure.getFilterLabel())
453 coaddTempExp.getInfo().setVisitInfo(exposure.getInfo().getVisitInfo())
455 coaddTempExp.setPsf(exposure.getPsf())
456 didSetMetadata[warpType] =
True
459 inputRecorder[warpType].addCalExp(calExp, ccdId, numGoodPix[warpType])
461 except Exception
as e:
462 self.log.warning(
"Error processing calexp %s; skipping it: %s", dataId, e)
465 for warpType
in warpTypeList:
466 self.log.info(
"%sWarp has %d good pixels (%.1f%%)",
467 warpType, totGoodPix[warpType], 100.0*totGoodPix[warpType]/skyInfo.bbox.getArea())
469 if totGoodPix[warpType] > 0
and didSetMetadata[warpType]:
470 inputRecorder[warpType].finish(coaddTempExps[warpType], totGoodPix[warpType])
471 if warpType ==
"direct":
472 coaddTempExps[warpType].setPsf(
473 CoaddPsf(inputRecorder[warpType].coaddInputs.ccds, skyInfo.wcs,
474 self.config.coaddPsf.makeControl()))
476 if not self.config.doWriteEmptyWarps:
478 coaddTempExps[warpType] =
None
483 result = pipeBase.Struct(exposures=coaddTempExps)
487 """Return one calibrated Exposure, possibly with an updated SkyWcs.
489 @param[in] dataRef a sensor-level data reference
490 @param[in] bgSubtracted return calexp with background subtracted? If False get the
491 calexp's background background model and add it to the calexp.
492 @return calibrated exposure
494 @raises MissingExposureError If data for the exposure is not available.
496 If config.doApplyExternalPhotoCalib is `True`, the photometric calibration
497 (`photoCalib`) is taken from `config.externalPhotoCalibName` via the
498 `name_photoCalib` dataset. Otherwise, the photometric calibration is
499 retrieved from the processed exposure. When
500 `config.doApplyExternalSkyWcs` is `True`, the astrometric calibration
501 is taken from `config.externalSkyWcsName` with the `name_wcs` dataset.
502 Otherwise, the astrometric calibration is taken from the processed
506 exposure = dataRef.get(self.
calexpTypecalexpType, immediate=
True)
507 except dafPersist.NoResults
as e:
511 background = dataRef.get(
"calexpBackground", immediate=
True)
512 mi = exposure.getMaskedImage()
513 mi += background.getImage()
516 if self.config.doApplyExternalPhotoCalib:
517 source = f
"{self.config.externalPhotoCalibName}_photoCalib"
518 self.log.debug(
"Applying external photoCalib to %s from %s", dataRef.dataId, source)
519 photoCalib = dataRef.get(source)
520 exposure.setPhotoCalib(photoCalib)
522 photoCalib = exposure.getPhotoCalib()
524 if self.config.doApplyExternalSkyWcs:
525 source = f
"{self.config.externalSkyWcsName}_wcs"
526 self.log.debug(
"Applying external skyWcs to %s from %s", dataRef.dataId, source)
527 skyWcs = dataRef.get(source)
528 exposure.setWcs(skyWcs)
530 exposure.maskedImage = photoCalib.calibrateImage(exposure.maskedImage,
531 includeScaleUncertainty=self.config.includeCalibVar)
532 exposure.maskedImage /= photoCalib.getCalibrationMean()
538 def _prepareEmptyExposure(skyInfo):
539 """Produce an empty exposure for a given patch"""
540 exp = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
541 exp.getMaskedImage().set(numpy.nan, afwImage.Mask
542 .getPlaneBitMask(
"NO_DATA"), numpy.inf)
546 """Return list of requested warp types per the config.
549 if self.config.makeDirect:
550 warpTypeList.append(
"direct")
551 if self.config.makePsfMatched:
552 warpTypeList.append(
"psfMatched")
556 """Apply correction to the sky background level
558 Sky corrections can be generated with the 'skyCorrection.py'
559 executable in pipe_drivers. Because the sky model used by that
560 code extends over the entire focal plane, this can produce
561 better sky subtraction.
563 The calexp is updated in-place.
567 dataRef : `lsst.daf.persistence.ButlerDataRef`
568 Data reference for calexp.
569 calexp : `lsst.afw.image.Exposure` or `lsst.afw.image.MaskedImage`
572 bg = dataRef.get(
"skyCorr")
573 self.log.debug(
"Applying sky correction to %s", dataRef.dataId)
574 if isinstance(calexp, afwImage.Exposure):
575 calexp = calexp.getMaskedImage()
576 calexp -= bg.getImage()
580 dimensions=(
"tract",
"patch",
"skymap",
"instrument",
"visit"),
581 defaultTemplates={
"coaddName":
"deep",
582 "skyWcsName":
"jointcal",
583 "photoCalibName":
"fgcm",
585 calExpList = connectionTypes.Input(
586 doc=
"Input exposures to be resampled and optionally PSF-matched onto a SkyMap projection/patch",
587 name=
"{calexpType}calexp",
588 storageClass=
"ExposureF",
589 dimensions=(
"instrument",
"visit",
"detector"),
593 backgroundList = connectionTypes.Input(
594 doc=
"Input backgrounds to be added back into the calexp if bgSubtracted=False",
595 name=
"calexpBackground",
596 storageClass=
"Background",
597 dimensions=(
"instrument",
"visit",
"detector"),
600 skyCorrList = connectionTypes.Input(
601 doc=
"Input Sky Correction to be subtracted from the calexp if doApplySkyCorr=True",
603 storageClass=
"Background",
604 dimensions=(
"instrument",
"visit",
"detector"),
607 skyMap = connectionTypes.Input(
608 doc=
"Input definition of geometry/bbox and projection/wcs for warped exposures",
609 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
610 storageClass=
"SkyMap",
611 dimensions=(
"skymap",),
613 externalSkyWcsTractCatalog = connectionTypes.Input(
614 doc=(
"Per-tract, per-visit wcs calibrations. These catalogs use the detector "
615 "id for the catalog id, sorted on id for fast lookup."),
616 name=
"{skyWcsName}SkyWcsCatalog",
617 storageClass=
"ExposureCatalog",
618 dimensions=(
"instrument",
"visit",
"tract"),
620 externalSkyWcsGlobalCatalog = connectionTypes.Input(
621 doc=(
"Per-visit wcs calibrations computed globally (with no tract information). "
622 "These catalogs use the detector id for the catalog id, sorted on id for "
624 name=
"{skyWcsName}SkyWcsCatalog",
625 storageClass=
"ExposureCatalog",
626 dimensions=(
"instrument",
"visit"),
628 externalPhotoCalibTractCatalog = connectionTypes.Input(
629 doc=(
"Per-tract, per-visit photometric calibrations. These catalogs use the "
630 "detector id for the catalog id, sorted on id for fast lookup."),
631 name=
"{photoCalibName}PhotoCalibCatalog",
632 storageClass=
"ExposureCatalog",
633 dimensions=(
"instrument",
"visit",
"tract"),
635 externalPhotoCalibGlobalCatalog = connectionTypes.Input(
636 doc=(
"Per-visit photometric calibrations computed globally (with no tract "
637 "information). These catalogs use the detector id for the catalog id, "
638 "sorted on id for fast lookup."),
639 name=
"{photoCalibName}PhotoCalibCatalog",
640 storageClass=
"ExposureCatalog",
641 dimensions=(
"instrument",
"visit"),
643 direct = connectionTypes.Output(
644 doc=(
"Output direct warped exposure (previously called CoaddTempExp), produced by resampling ",
645 "calexps onto the skyMap patch geometry."),
646 name=
"{coaddName}Coadd_directWarp",
647 storageClass=
"ExposureF",
648 dimensions=(
"tract",
"patch",
"skymap",
"visit",
"instrument"),
650 psfMatched = connectionTypes.Output(
651 doc=(
"Output PSF-Matched warped exposure (previously called CoaddTempExp), produced by resampling ",
652 "calexps onto the skyMap patch geometry and PSF-matching to a model PSF."),
653 name=
"{coaddName}Coadd_psfMatchedWarp",
654 storageClass=
"ExposureF",
655 dimensions=(
"tract",
"patch",
"skymap",
"visit",
"instrument"),
658 wcsList = connectionTypes.Input(
659 doc=
"WCSs of calexps used by SelectImages subtask to determine if the calexp overlaps the patch",
660 name=
"{calexpType}calexp.wcs",
662 dimensions=(
"instrument",
"visit",
"detector"),
665 bboxList = connectionTypes.Input(
666 doc=
"BBoxes of calexps used by SelectImages subtask to determine if the calexp overlaps the patch",
667 name=
"{calexpType}calexp.bbox",
668 storageClass=
"Box2I",
669 dimensions=(
"instrument",
"visit",
"detector"),
672 srcList = connectionTypes.Input(
673 doc=
"src catalogs used by PsfWcsSelectImages subtask to further select on PSF stability",
675 storageClass=
"SourceCatalog",
676 dimensions=(
"instrument",
"visit",
"detector"),
679 psfList = connectionTypes.Input(
680 doc=
"PSF models used by BestSeeingWcsSelectImages subtask to futher select on seeing",
681 name=
"{calexpType}calexp.psf",
683 dimensions=(
"instrument",
"visit",
"detector"),
687 def __init__(self, *, config=None):
688 super().__init__(config=config)
689 if config.bgSubtracted:
690 self.inputs.remove(
"backgroundList")
691 if not config.doApplySkyCorr:
692 self.inputs.remove(
"skyCorrList")
693 if config.doApplyExternalSkyWcs:
694 if config.useGlobalExternalSkyWcs:
695 self.inputs.remove(
"externalSkyWcsTractCatalog")
697 self.inputs.remove(
"externalSkyWcsGlobalCatalog")
699 self.inputs.remove(
"externalSkyWcsTractCatalog")
700 self.inputs.remove(
"externalSkyWcsGlobalCatalog")
701 if config.doApplyExternalPhotoCalib:
702 if config.useGlobalExternalPhotoCalib:
703 self.inputs.remove(
"externalPhotoCalibTractCatalog")
705 self.inputs.remove(
"externalPhotoCalibGlobalCatalog")
707 self.inputs.remove(
"externalPhotoCalibTractCatalog")
708 self.inputs.remove(
"externalPhotoCalibGlobalCatalog")
709 if not config.makeDirect:
710 self.outputs.remove(
"direct")
711 if not config.makePsfMatched:
712 self.outputs.remove(
"psfMatched")
715 if config.select.target != lsst.pipe.tasks.selectImages.PsfWcsSelectImagesTask:
716 self.inputs.remove(
"srcList")
717 if config.select.target != lsst.pipe.tasks.selectImages.BestSeeingWcsSelectImagesTask:
718 self.inputs.remove(
"psfList")
722 pipelineConnections=MakeWarpConnections):
729 """Warp and optionally PSF-Match calexps onto an a common projection
731 ConfigClass = MakeWarpConfig
732 _DefaultName =
"makeWarp"
734 @utils.inheritDoc(pipeBase.PipelineTask)
735 def runQuantum(self, butlerQC, inputRefs, outputRefs):
739 Construct warps for requested warp type for single epoch
741 PipelineTask (Gen3) entry point to warp and optionally PSF-match
742 calexps. This method is analogous to `runDataRef`.
746 detectorOrder = [ref.datasetRef.dataId[
'detector']
for ref
in inputRefs.calExpList]
747 inputRefs = reorderRefs(inputRefs, detectorOrder, dataIdKey=
'detector')
750 inputs = butlerQC.get(inputRefs)
754 skyMap = inputs.pop(
"skyMap")
755 quantumDataId = butlerQC.quantum.dataId
756 skyInfo =
makeSkyInfo(skyMap, tractId=quantumDataId[
'tract'], patchId=quantumDataId[
'patch'])
759 dataIdList = [ref.datasetRef.dataId
for ref
in inputRefs.calExpList]
761 ccdIdList = [dataId.pack(
"visit_detector")
for dataId
in dataIdList]
766 coordList = [skyInfo.wcs.pixelToSky(pos)
for pos
in cornerPosList]
767 goodIndices = self.select.
run(**inputs, coordList=coordList, dataIds=dataIdList)
768 inputs = self.filterInputs(indices=goodIndices, inputs=inputs)
771 inputs[
'calExpList'] = [ref.get()
for ref
in inputs[
'calExpList']]
774 visits = [dataId[
'visit']
for dataId
in dataIdList]
777 if self.config.doApplyExternalSkyWcs:
778 if self.config.useGlobalExternalSkyWcs:
779 externalSkyWcsCatalog = inputs.pop(
"externalSkyWcsGlobalCatalog")
781 externalSkyWcsCatalog = inputs.pop(
"externalSkyWcsTractCatalog")
783 externalSkyWcsCatalog =
None
785 if self.config.doApplyExternalPhotoCalib:
786 if self.config.useGlobalExternalPhotoCalib:
787 externalPhotoCalibCatalog = inputs.pop(
"externalPhotoCalibGlobalCatalog")
789 externalPhotoCalibCatalog = inputs.pop(
"externalPhotoCalibTractCatalog")
791 externalPhotoCalibCatalog =
None
793 completeIndices = self.prepareCalibratedExposures(**inputs,
794 externalSkyWcsCatalog=externalSkyWcsCatalog,
795 externalPhotoCalibCatalog=externalPhotoCalibCatalog)
797 inputs = self.filterInputs(indices=completeIndices, inputs=inputs)
799 results = self.run(**inputs, visitId=visitId,
800 ccdIdList=[ccdIdList[i]
for i
in goodIndices],
801 dataIdList=[dataIdList[i]
for i
in goodIndices],
803 if self.config.makeDirect
and results.exposures[
"direct"]
is not None:
804 butlerQC.put(results.exposures[
"direct"], outputRefs.direct)
805 if self.config.makePsfMatched
and results.exposures[
"psfMatched"]
is not None:
806 butlerQC.put(results.exposures[
"psfMatched"], outputRefs.psfMatched)
808 def filterInputs(self, indices, inputs):
809 """Return task inputs with their lists filtered by indices
813 indices : `list` of integers
814 inputs : `dict` of `list` of input connections to be passed to run
816 for key
in inputs.keys():
818 if isinstance(inputs[key], list):
819 inputs[key] = [inputs[key][ind]
for ind
in indices]
822 def prepareCalibratedExposures(self, calExpList, backgroundList=None, skyCorrList=None,
823 externalSkyWcsCatalog=None, externalPhotoCalibCatalog=None,
825 """Calibrate and add backgrounds to input calExpList in place
829 calExpList : `list` of `lsst.afw.image.Exposure`
830 Sequence of calexps to be modified in place
831 backgroundList : `list` of `lsst.afw.math.backgroundList`, optional
832 Sequence of backgrounds to be added back in if bgSubtracted=False
833 skyCorrList : `list` of `lsst.afw.math.backgroundList`, optional
834 Sequence of background corrections to be subtracted if doApplySkyCorr=True
835 externalSkyWcsCatalog : `lsst.afw.table.ExposureCatalog`, optional
836 Exposure catalog with external skyWcs to be applied
837 if config.doApplyExternalSkyWcs=True. Catalog uses the detector id
838 for the catalog id, sorted on id for fast lookup.
839 externalPhotoCalibCatalog : `lsst.afw.table.ExposureCatalog`, optional
840 Exposure catalog with external photoCalib to be applied
841 if config.doApplyExternalPhotoCalib=True. Catalog uses the detector
842 id for the catalog id, sorted on id for fast lookup.
846 indices : `list` [`int`]
847 Indices of calExpList and friends that have valid photoCalib/skyWcs
849 backgroundList = len(calExpList)*[
None]
if backgroundList
is None else backgroundList
850 skyCorrList = len(calExpList)*[
None]
if skyCorrList
is None else skyCorrList
852 includeCalibVar = self.config.includeCalibVar
855 for index, (calexp, background, skyCorr)
in enumerate(zip(calExpList,
858 mi = calexp.maskedImage
859 if not self.config.bgSubtracted:
860 mi += background.getImage()
862 if externalSkyWcsCatalog
is not None or externalPhotoCalibCatalog
is not None:
863 detectorId = calexp.getInfo().getDetector().getId()
866 if externalPhotoCalibCatalog
is not None:
867 row = externalPhotoCalibCatalog.find(detectorId)
869 self.log.warning(
"Detector id %s not found in externalPhotoCalibCatalog "
870 "and will not be used in the warp.", detectorId)
872 photoCalib = row.getPhotoCalib()
873 if photoCalib
is None:
874 self.log.warning(
"Detector id %s has None for photoCalib in externalPhotoCalibCatalog "
875 "and will not be used in the warp.", detectorId)
877 calexp.setPhotoCalib(photoCalib)
879 photoCalib = calexp.getPhotoCalib()
880 if photoCalib
is None:
881 self.log.warning(
"Detector id %s has None for photoCalib in the calexp "
882 "and will not be used in the warp.", detectorId)
886 if externalSkyWcsCatalog
is not None:
887 row = externalSkyWcsCatalog.find(detectorId)
889 self.log.warning(
"Detector id %s not found in externalSkyWcsCatalog "
890 "and will not be used in the warp.", detectorId)
892 skyWcs = row.getWcs()
894 self.log.warning(
"Detector id %s has None for skyWcs in externalSkyWcsCatalog "
895 "and will not be used in the warp.", detectorId)
897 calexp.setWcs(skyWcs)
899 skyWcs = calexp.getWcs()
901 self.log.warning(
"Detector id %s has None for skyWcs in the calexp "
902 "and will not be used in the warp.", detectorId)
906 calexp.maskedImage = photoCalib.calibrateImage(calexp.maskedImage,
907 includeScaleUncertainty=includeCalibVar)
908 calexp.maskedImage /= photoCalib.getCalibrationMean()
913 if self.config.doApplySkyCorr:
914 mi -= skyCorr.getImage()
916 indices.append(index)
921 def reorderRefs(inputRefs, outputSortKeyOrder, dataIdKey):
922 """Reorder inputRefs per outputSortKeyOrder
924 Any inputRefs which are lists will be resorted per specified key e.g.,
925 'detector.' Only iterables will be reordered, and values can be of type
926 `lsst.pipe.base.connections.DeferredDatasetRef` or
927 `lsst.daf.butler.core.datasets.ref.DatasetRef`.
928 Returned lists of refs have the same length as the outputSortKeyOrder.
929 If an outputSortKey not in the inputRef, then it will be padded with None.
930 If an inputRef contains an inputSortKey that is not in the
931 outputSortKeyOrder it will be removed.
935 inputRefs : `lsst.pipe.base.connections.QuantizedConnection`
936 Input references to be reordered and padded.
937 outputSortKeyOrder : iterable
938 Iterable of values to be compared with inputRef's dataId[dataIdKey]
940 dataIdKey in the dataRefs to compare with the outputSortKeyOrder.
944 inputRefs: `lsst.pipe.base.connections.QuantizedConnection`
945 Quantized Connection with sorted DatasetRef values sorted if iterable.
947 for connectionName, refs
in inputRefs:
948 if isinstance(refs, Iterable):
949 if hasattr(refs[0],
"dataId"):
950 inputSortKeyOrder = [ref.dataId[dataIdKey]
for ref
in refs]
952 inputSortKeyOrder = [ref.datasetRef.dataId[dataIdKey]
for ref
in refs]
953 if inputSortKeyOrder != outputSortKeyOrder:
954 setattr(inputRefs, connectionName,
Base class for coaddition.
def getTempExpDatasetName(self, warpType="direct")
def selectExposures(self, patchRef, skyInfo=None, selectDataList=[])
Select exposures to coadd.
def getCoaddDatasetName(self, warpType="direct")
def getSkyInfo(self, patchRef)
Use getSkyinfo to return the skyMap, tract and patch information, wcs and the outer bbox of the patch...
def getBadPixelMask(self)
Convenience method to provide the bitmask from the mask plane names.
Warp and optionally PSF-Match calexps onto an a common projection.
def getCalibratedExposure(self, dataRef, bgSubtracted)
def run(self, calExpList, ccdIdList, skyInfo, visitId=0, dataIdList=None, **kwargs)
def __init__(self, reuse=False, **kwargs)
def _prepareEmptyExposure(skyInfo)
def runDataRef(self, patchRef, selectDataList=[])
Produce <coaddName>Coadd_<warpType>Warp images by warping and optionally PSF-matching.
def getWarpTypeList(self)
def applySkyCorr(self, dataRef, calexp)
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, altMaskList=None, mask=None, supplementaryData=None)
def reorderAndPadList(inputList, inputKeys, outputKeys, padWith=None)
def makeSkyInfo(skyMap, tractId, patchId)
def getGroupDataRef(butler, datasetType, groupTuple, keys)
def groupPatchExposures(patchDataRef, calexpDataRefList, coaddDatasetType="deepCoadd", tempExpDatasetType="deepCoadd_directWarp")