30 import lsst.pipe.base.connectionTypes
as connectionTypes
31 import lsst.utils
as utils
35 from .coaddBase
import CoaddBaseTask, makeSkyInfo, reorderAndPadList
36 from .warpAndPsfMatch
import WarpAndPsfMatchTask
37 from .coaddHelpers
import groupPatchExposures, getGroupDataRef
38 from collections.abc
import Iterable
40 __all__ = [
"MakeCoaddTempExpTask",
"MakeWarpTask",
"MakeWarpConfig"]
42 log = logging.getLogger(__name__.partition(
".")[2])
46 """Raised when data cannot be retrieved for an exposure.
47 When processing patches, sometimes one exposure is missing; this lets us
48 distinguish bewteen that case, and other errors.
54 """Config for MakeCoaddTempExpTask
56 warpAndPsfMatch = pexConfig.ConfigurableField(
57 target=WarpAndPsfMatchTask,
58 doc=
"Task to warp and PSF-match calexp",
60 doWrite = pexConfig.Field(
61 doc=
"persist <coaddName>Coadd_<warpType>Warp",
65 bgSubtracted = pexConfig.Field(
66 doc=
"Work with a background subtracted calexp?",
70 coaddPsf = pexConfig.ConfigField(
71 doc=
"Configuration for CoaddPsf",
74 makeDirect = pexConfig.Field(
75 doc=
"Make direct Warp/Coadds",
79 makePsfMatched = pexConfig.Field(
80 doc=
"Make Psf-Matched Warp/Coadd?",
85 doWriteEmptyWarps = pexConfig.Field(
88 doc=
"Write out warps even if they are empty"
91 hasFakes = pexConfig.Field(
92 doc=
"Should be set to True if fake sources have been inserted into the input data.",
96 doApplySkyCorr = pexConfig.Field(dtype=bool, default=
False, doc=
"Apply sky correction?")
99 CoaddBaseTask.ConfigClass.validate(self)
101 raise RuntimeError(
"At least one of config.makePsfMatched and config.makeDirect must be True")
104 log.warning(
"Config doPsfMatch deprecated. Setting makePsfMatched=True and makeDirect=False")
109 CoaddBaseTask.ConfigClass.setDefaults(self)
110 self.
warpAndPsfMatchwarpAndPsfMatch.psfMatch.kernel.active.kernelSize = self.matchingKernelSize
121 r"""!Warp and optionally PSF-Match calexps onto an a common projection.
123 @anchor MakeCoaddTempExpTask_
125 @section pipe_tasks_makeCoaddTempExp_Contents Contents
127 - @ref pipe_tasks_makeCoaddTempExp_Purpose
128 - @ref pipe_tasks_makeCoaddTempExp_Initialize
129 - @ref pipe_tasks_makeCoaddTempExp_IO
130 - @ref pipe_tasks_makeCoaddTempExp_Config
131 - @ref pipe_tasks_makeCoaddTempExp_Debug
132 - @ref pipe_tasks_makeCoaddTempExp_Example
134 @section pipe_tasks_makeCoaddTempExp_Purpose Description
136 Warp and optionally PSF-Match calexps onto a common projection, by
137 performing the following operations:
138 - Group calexps by visit/run
139 - For each visit, generate a Warp by calling method @ref makeTempExp.
140 makeTempExp loops over the visit's calexps calling @ref WarpAndPsfMatch
143 The result is a `directWarp` (and/or optionally a `psfMatchedWarp`).
145 @section pipe_tasks_makeCoaddTempExp_Initialize Task Initialization
147 @copydoc \_\_init\_\_
149 This task has one special keyword argument: passing reuse=True will cause
150 the task to skip the creation of warps that are already present in the
153 @section pipe_tasks_makeCoaddTempExp_IO Invoking the Task
155 This task is primarily designed to be run from the command line.
157 The main method is `runDataRef`, which takes a single butler data reference for the patch(es)
162 WarpType identifies the types of convolutions applied to Warps (previously CoaddTempExps).
163 Only two types are available: direct (for regular Warps/Coadds) and psfMatched
164 (for Warps/Coadds with homogenized PSFs). We expect to add a third type, likelihood,
165 for generating likelihood Coadds with Warps that have been correlated with their own PSF.
167 @section pipe_tasks_makeCoaddTempExp_Config Configuration parameters
169 See @ref MakeCoaddTempExpConfig and parameters inherited from
170 @link lsst.pipe.tasks.coaddBase.CoaddBaseConfig CoaddBaseConfig @endlink
172 @subsection pipe_tasks_MakeCoaddTempExp_psfMatching Guide to PSF-Matching Configs
174 To make `psfMatchedWarps`, select `config.makePsfMatched=True`. The subtask
175 @link lsst.ip.diffim.modelPsfMatch.ModelPsfMatchTask ModelPsfMatchTask @endlink
176 is responsible for the PSF-Matching, and its config is accessed via `config.warpAndPsfMatch.psfMatch`.
177 The optimal configuration depends on aspects of dataset: the pixel scale, average PSF FWHM and
178 dimensions of the PSF kernel. These configs include the requested model PSF, the matching kernel size,
179 padding of the science PSF thumbnail and spatial sampling frequency of the PSF.
181 *Config Guidelines*: The user must specify the size of the model PSF to which to match by setting
182 `config.modelPsf.defaultFwhm` in units of pixels. The appropriate values depends on science case.
183 In general, for a set of input images, this config should equal the FWHM of the visit
184 with the worst seeing. The smallest it should be set to is the median FWHM. The defaults
185 of the other config options offer a reasonable starting point.
186 The following list presents the most common problems that arise from a misconfigured
187 @link lsst.ip.diffim.modelPsfMatch.ModelPsfMatchTask ModelPsfMatchTask @endlink
188 and corresponding solutions. All assume the default Alard-Lupton kernel, with configs accessed via
189 ```config.warpAndPsfMatch.psfMatch.kernel['AL']```. Each item in the list is formatted as:
190 Problem: Explanation. *Solution*
192 *Troublshooting PSF-Matching Configuration:*
193 - Matched PSFs look boxy: The matching kernel is too small. _Increase the matching kernel size.
196 config.warpAndPsfMatch.psfMatch.kernel['AL'].kernelSize=27 # default 21
198 Note that increasing the kernel size also increases runtime.
199 - Matched PSFs look ugly (dipoles, quadropoles, donuts): unable to find good solution
200 for matching kernel. _Provide the matcher with more data by either increasing
201 the spatial sampling by decreasing the spatial cell size,_
203 config.warpAndPsfMatch.psfMatch.kernel['AL'].sizeCellX = 64 # default 128
204 config.warpAndPsfMatch.psfMatch.kernel['AL'].sizeCellY = 64 # default 128
206 _or increasing the padding around the Science PSF, for example:_
208 config.warpAndPsfMatch.psfMatch.autoPadPsfTo=1.6 # default 1.4
210 Increasing `autoPadPsfTo` increases the minimum ratio of input PSF dimensions to the
211 matching kernel dimensions, thus increasing the number of pixels available to fit
212 after convolving the PSF with the matching kernel.
213 Optionally, for debugging the effects of padding, the level of padding may be manually
214 controlled by setting turning off the automatic padding and setting the number
215 of pixels by which to pad the PSF:
217 config.warpAndPsfMatch.psfMatch.doAutoPadPsf = False # default True
218 config.warpAndPsfMatch.psfMatch.padPsfBy = 6 # pixels. default 0
220 - Deconvolution: Matching a large PSF to a smaller PSF produces
221 a telltale noise pattern which looks like ripples or a brain.
222 _Increase the size of the requested model PSF. For example:_
224 config.modelPsf.defaultFwhm = 11 # Gaussian sigma in units of pixels.
226 - High frequency (sometimes checkered) noise: The matching basis functions are too small.
227 _Increase the width of the Gaussian basis functions. For example:_
229 config.warpAndPsfMatch.psfMatch.kernel['AL'].alardSigGauss=[1.5, 3.0, 6.0]
230 # from default [0.7, 1.5, 3.0]
233 @section pipe_tasks_makeCoaddTempExp_Debug Debug variables
235 MakeCoaddTempExpTask has no debug output, but its subtasks do.
237 @section pipe_tasks_makeCoaddTempExp_Example A complete example of using MakeCoaddTempExpTask
239 This example uses the package ci_hsc to show how MakeCoaddTempExp fits
240 into the larger Data Release Processing.
245 # if not built already:
246 python $(which scons) # this will take a while
248 The following assumes that `processCcd.py` and `makeSkyMap.py` have previously been run
249 (e.g. by building `ci_hsc` above) to generate a repository of calexps and an
250 output respository with the desired SkyMap. The command,
252 makeCoaddTempExp.py $CI_HSC_DIR/DATA --rerun ci_hsc \
253 --id patch=5,4 tract=0 filter=HSC-I \
254 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 \
255 --selectId visit=903988 ccd=23 --selectId visit=903988 ccd=24 \
256 --config doApplyExternalPhotoCalib=False doApplyExternalSkyWcs=False \
257 makePsfMatched=True modelPsf.defaultFwhm=11
259 writes a direct and PSF-Matched Warp to
260 - `$CI_HSC_DIR/DATA/rerun/ci_hsc/deepCoadd/HSC-I/0/5,4/warp-HSC-I-0-5,4-903988.fits` and
261 - `$CI_HSC_DIR/DATA/rerun/ci_hsc/deepCoadd/HSC-I/0/5,4/psfMatchedWarp-HSC-I-0-5,4-903988.fits`
264 @note PSF-Matching in this particular dataset would benefit from adding
265 `--configfile ./matchingConfig.py` to
266 the command line arguments where `matchingConfig.py` is defined by:
269 config.warpAndPsfMatch.psfMatch.kernel['AL'].kernelSize=27
270 config.warpAndPsfMatch.psfMatch.kernel['AL'].alardSigGauss=[1.5, 3.0, 6.0]" > matchingConfig.py
273 Add the option `--help` to see more options.
275 ConfigClass = MakeCoaddTempExpConfig
276 _DefaultName =
"makeCoaddTempExp"
279 CoaddBaseTask.__init__(self, **kwargs)
281 self.makeSubtask(
"warpAndPsfMatch")
282 if self.config.hasFakes:
289 """!Produce <coaddName>Coadd_<warpType>Warp images by warping and optionally PSF-matching.
291 @param[in] patchRef: data reference for sky map patch. Must include keys "tract", "patch",
292 plus the camera-specific filter key (e.g. "filter" or "band")
293 @return: dataRefList: a list of data references for the new <coaddName>Coadd_directWarps
294 if direct or both warp types are requested and <coaddName>Coadd_psfMatchedWarps if only psfMatched
297 @warning: this task assumes that all exposures in a warp (coaddTempExp) have the same filter.
299 @warning: this task sets the PhotoCalib of the coaddTempExp to the PhotoCalib of the first calexp
300 with any good pixels in the patch. For a mosaic camera the resulting PhotoCalib should be ignored
301 (assembleCoadd should determine zeropoint scaling without referring to it).
306 if self.config.makePsfMatched
and not self.config.makeDirect:
311 calExpRefList = self.
selectExposuresselectExposures(patchRef, skyInfo, selectDataList=selectDataList)
313 if len(calExpRefList) == 0:
314 self.log.warning(
"No exposures to coadd for patch %s", patchRef.dataId)
316 self.log.info(
"Selected %d calexps for patch %s", len(calExpRefList), patchRef.dataId)
317 calExpRefList = [calExpRef
for calExpRef
in calExpRefList
if calExpRef.datasetExists(self.
calexpTypecalexpType)]
318 self.log.info(
"Processing %d existing calexps for patch %s", len(calExpRefList), patchRef.dataId)
322 self.log.info(
"Processing %d warp exposures for patch %s", len(groupData.groups), patchRef.dataId)
325 for i, (tempExpTuple, calexpRefList)
in enumerate(groupData.groups.items()):
327 tempExpTuple, groupData.keys)
328 if self.
reusereuse
and tempExpRef.datasetExists(datasetType=primaryWarpDataset, write=
True):
329 self.log.info(
"Skipping makeCoaddTempExp for %s; output already exists.", tempExpRef.dataId)
330 dataRefList.append(tempExpRef)
332 self.log.info(
"Processing Warp %d/%d: id=%s", i, len(groupData.groups), tempExpRef.dataId)
338 visitId = int(tempExpRef.dataId[
"visit"])
339 except (KeyError, ValueError):
346 for calExpInd, calExpRef
in enumerate(calexpRefList):
347 self.log.info(
"Reading calexp %s of %s for Warp id=%s", calExpInd+1, len(calexpRefList),
350 ccdId = calExpRef.get(
"ccdExposureId", immediate=
True)
357 calExpRef = calExpRef.butlerSubset.butler.dataRef(self.
calexpTypecalexpType,
358 dataId=calExpRef.dataId,
359 tract=skyInfo.tractInfo.getId())
360 calExp = self.
getCalibratedExposuregetCalibratedExposure(calExpRef, bgSubtracted=self.config.bgSubtracted)
361 except Exception
as e:
362 self.log.warning(
"Calexp %s not found; skipping it: %s", calExpRef.dataId, e)
365 if self.config.doApplySkyCorr:
368 calExpList.append(calExp)
369 ccdIdList.append(ccdId)
370 dataIdList.append(calExpRef.dataId)
372 exps = self.
runrun(calExpList, ccdIdList, skyInfo, visitId, dataIdList).exposures
374 if any(exps.values()):
375 dataRefList.append(tempExpRef)
377 self.log.warning(
"Warp %s could not be created", tempExpRef.dataId)
379 if self.config.doWrite:
380 for (warpType, exposure)
in exps.items():
381 if exposure
is not None:
388 def run(self, calExpList, ccdIdList, skyInfo, visitId=0, dataIdList=None, **kwargs):
389 """Create a Warp from inputs
391 We iterate over the multiple calexps in a single exposure to construct
392 the warp (previously called a coaddTempExp) of that exposure to the
393 supplied tract/patch.
395 Pixels that receive no pixels are set to NAN; this is not correct
396 (violates LSST algorithms group policy), but will be fixed up by
397 interpolating after the coaddition.
399 @param calexpRefList: List of data references for calexps that (may)
400 overlap the patch of interest
401 @param skyInfo: Struct from CoaddBaseTask.getSkyInfo() with geometric
402 information about the patch
403 @param visitId: integer identifier for visit, for the table that will
405 @return a pipeBase Struct containing:
406 - exposures: a dictionary containing the warps requested:
407 "direct": direct warp if config.makeDirect
408 "psfMatched": PSF-matched warp if config.makePsfMatched
412 totGoodPix = {warpType: 0
for warpType
in warpTypeList}
413 didSetMetadata = {warpType:
False for warpType
in warpTypeList}
414 coaddTempExps = {warpType: self.
_prepareEmptyExposure_prepareEmptyExposure(skyInfo)
for warpType
in warpTypeList}
415 inputRecorder = {warpType: self.inputRecorder.makeCoaddTempExpRecorder(visitId, len(calExpList))
416 for warpType
in warpTypeList}
418 modelPsf = self.config.modelPsf.apply()
if self.config.makePsfMatched
else None
419 if dataIdList
is None:
420 dataIdList = ccdIdList
422 for calExpInd, (calExp, ccdId, dataId)
in enumerate(zip(calExpList, ccdIdList, dataIdList)):
423 self.log.info(
"Processing calexp %d of %d for this Warp: id=%s",
424 calExpInd+1, len(calExpList), dataId)
427 warpedAndMatched = self.warpAndPsfMatch.
run(calExp, modelPsf=modelPsf,
428 wcs=skyInfo.wcs, maxBBox=skyInfo.bbox,
429 makeDirect=self.config.makeDirect,
430 makePsfMatched=self.config.makePsfMatched)
431 except Exception
as e:
432 self.log.warning(
"WarpAndPsfMatch failed for calexp %s; skipping it: %s", dataId, e)
435 numGoodPix = {warpType: 0
for warpType
in warpTypeList}
436 for warpType
in warpTypeList:
437 exposure = warpedAndMatched.getDict()[warpType]
440 coaddTempExp = coaddTempExps[warpType]
441 if didSetMetadata[warpType]:
442 mimg = exposure.getMaskedImage()
443 mimg *= (coaddTempExp.getPhotoCalib().getInstFluxAtZeroMagnitude()
444 / exposure.getPhotoCalib().getInstFluxAtZeroMagnitude())
446 numGoodPix[warpType] = coaddUtils.copyGoodPixels(
447 coaddTempExp.getMaskedImage(), exposure.getMaskedImage(), self.
getBadPixelMaskgetBadPixelMask())
448 totGoodPix[warpType] += numGoodPix[warpType]
449 self.log.debug(
"Calexp %s has %d good pixels in this patch (%.1f%%) for %s",
450 dataId, numGoodPix[warpType],
451 100.0*numGoodPix[warpType]/skyInfo.bbox.getArea(), warpType)
452 if numGoodPix[warpType] > 0
and not didSetMetadata[warpType]:
453 coaddTempExp.setPhotoCalib(exposure.getPhotoCalib())
454 coaddTempExp.setFilterLabel(exposure.getFilterLabel())
455 coaddTempExp.getInfo().setVisitInfo(exposure.getInfo().getVisitInfo())
457 coaddTempExp.setPsf(exposure.getPsf())
458 didSetMetadata[warpType] =
True
461 inputRecorder[warpType].addCalExp(calExp, ccdId, numGoodPix[warpType])
463 except Exception
as e:
464 self.log.warning(
"Error processing calexp %s; skipping it: %s", dataId, e)
467 for warpType
in warpTypeList:
468 self.log.info(
"%sWarp has %d good pixels (%.1f%%)",
469 warpType, totGoodPix[warpType], 100.0*totGoodPix[warpType]/skyInfo.bbox.getArea())
471 if totGoodPix[warpType] > 0
and didSetMetadata[warpType]:
472 inputRecorder[warpType].finish(coaddTempExps[warpType], totGoodPix[warpType])
473 if warpType ==
"direct":
474 coaddTempExps[warpType].setPsf(
475 CoaddPsf(inputRecorder[warpType].coaddInputs.ccds, skyInfo.wcs,
476 self.config.coaddPsf.makeControl()))
478 if not self.config.doWriteEmptyWarps:
480 coaddTempExps[warpType] =
None
485 result = pipeBase.Struct(exposures=coaddTempExps)
489 """Return one calibrated Exposure, possibly with an updated SkyWcs.
491 @param[in] dataRef a sensor-level data reference
492 @param[in] bgSubtracted return calexp with background subtracted? If False get the
493 calexp's background background model and add it to the calexp.
494 @return calibrated exposure
496 @raises MissingExposureError If data for the exposure is not available.
498 If config.doApplyExternalPhotoCalib is `True`, the photometric calibration
499 (`photoCalib`) is taken from `config.externalPhotoCalibName` via the
500 `name_photoCalib` dataset. Otherwise, the photometric calibration is
501 retrieved from the processed exposure. When
502 `config.doApplyExternalSkyWcs` is `True`, the astrometric calibration
503 is taken from `config.externalSkyWcsName` with the `name_wcs` dataset.
504 Otherwise, the astrometric calibration is taken from the processed
508 exposure = dataRef.get(self.
calexpTypecalexpType, immediate=
True)
509 except dafPersist.NoResults
as e:
513 background = dataRef.get(
"calexpBackground", immediate=
True)
514 mi = exposure.getMaskedImage()
515 mi += background.getImage()
518 if self.config.doApplyExternalPhotoCalib:
519 source = f
"{self.config.externalPhotoCalibName}_photoCalib"
520 self.log.debug(
"Applying external photoCalib to %s from %s", dataRef.dataId, source)
521 photoCalib = dataRef.get(source)
522 exposure.setPhotoCalib(photoCalib)
524 photoCalib = exposure.getPhotoCalib()
526 if self.config.doApplyExternalSkyWcs:
527 source = f
"{self.config.externalSkyWcsName}_wcs"
528 self.log.debug(
"Applying external skyWcs to %s from %s", dataRef.dataId, source)
529 skyWcs = dataRef.get(source)
530 exposure.setWcs(skyWcs)
532 exposure.maskedImage = photoCalib.calibrateImage(exposure.maskedImage,
533 includeScaleUncertainty=self.config.includeCalibVar)
534 exposure.maskedImage /= photoCalib.getCalibrationMean()
540 def _prepareEmptyExposure(skyInfo):
541 """Produce an empty exposure for a given patch"""
542 exp = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
543 exp.getMaskedImage().set(numpy.nan, afwImage.Mask
544 .getPlaneBitMask(
"NO_DATA"), numpy.inf)
548 """Return list of requested warp types per the config.
551 if self.config.makeDirect:
552 warpTypeList.append(
"direct")
553 if self.config.makePsfMatched:
554 warpTypeList.append(
"psfMatched")
558 """Apply correction to the sky background level
560 Sky corrections can be generated with the 'skyCorrection.py'
561 executable in pipe_drivers. Because the sky model used by that
562 code extends over the entire focal plane, this can produce
563 better sky subtraction.
565 The calexp is updated in-place.
569 dataRef : `lsst.daf.persistence.ButlerDataRef`
570 Data reference for calexp.
571 calexp : `lsst.afw.image.Exposure` or `lsst.afw.image.MaskedImage`
574 bg = dataRef.get(
"skyCorr")
575 self.log.debug(
"Applying sky correction to %s", dataRef.dataId)
576 if isinstance(calexp, afwImage.Exposure):
577 calexp = calexp.getMaskedImage()
578 calexp -= bg.getImage()
582 dimensions=(
"tract",
"patch",
"skymap",
"instrument",
"visit"),
583 defaultTemplates={
"coaddName":
"deep",
584 "skyWcsName":
"jointcal",
585 "photoCalibName":
"fgcm",
587 calExpList = connectionTypes.Input(
588 doc=
"Input exposures to be resampled and optionally PSF-matched onto a SkyMap projection/patch",
589 name=
"{calexpType}calexp",
590 storageClass=
"ExposureF",
591 dimensions=(
"instrument",
"visit",
"detector"),
595 backgroundList = connectionTypes.Input(
596 doc=
"Input backgrounds to be added back into the calexp if bgSubtracted=False",
597 name=
"calexpBackground",
598 storageClass=
"Background",
599 dimensions=(
"instrument",
"visit",
"detector"),
602 skyCorrList = connectionTypes.Input(
603 doc=
"Input Sky Correction to be subtracted from the calexp if doApplySkyCorr=True",
605 storageClass=
"Background",
606 dimensions=(
"instrument",
"visit",
"detector"),
609 skyMap = connectionTypes.Input(
610 doc=
"Input definition of geometry/bbox and projection/wcs for warped exposures",
611 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
612 storageClass=
"SkyMap",
613 dimensions=(
"skymap",),
615 externalSkyWcsTractCatalog = connectionTypes.Input(
616 doc=(
"Per-tract, per-visit wcs calibrations. These catalogs use the detector "
617 "id for the catalog id, sorted on id for fast lookup."),
618 name=
"{skyWcsName}SkyWcsCatalog",
619 storageClass=
"ExposureCatalog",
620 dimensions=(
"instrument",
"visit",
"tract"),
622 externalSkyWcsGlobalCatalog = connectionTypes.Input(
623 doc=(
"Per-visit wcs calibrations computed globally (with no tract information). "
624 "These catalogs use the detector id for the catalog id, sorted on id for "
626 name=
"{skyWcsName}SkyWcsCatalog",
627 storageClass=
"ExposureCatalog",
628 dimensions=(
"instrument",
"visit"),
630 externalPhotoCalibTractCatalog = connectionTypes.Input(
631 doc=(
"Per-tract, per-visit photometric calibrations. These catalogs use the "
632 "detector id for the catalog id, sorted on id for fast lookup."),
633 name=
"{photoCalibName}PhotoCalibCatalog",
634 storageClass=
"ExposureCatalog",
635 dimensions=(
"instrument",
"visit",
"tract"),
637 externalPhotoCalibGlobalCatalog = connectionTypes.Input(
638 doc=(
"Per-visit photometric calibrations computed globally (with no tract "
639 "information). These catalogs use the detector id for the catalog id, "
640 "sorted on id for fast lookup."),
641 name=
"{photoCalibName}PhotoCalibCatalog",
642 storageClass=
"ExposureCatalog",
643 dimensions=(
"instrument",
"visit"),
645 direct = connectionTypes.Output(
646 doc=(
"Output direct warped exposure (previously called CoaddTempExp), produced by resampling ",
647 "calexps onto the skyMap patch geometry."),
648 name=
"{coaddName}Coadd_directWarp",
649 storageClass=
"ExposureF",
650 dimensions=(
"tract",
"patch",
"skymap",
"visit",
"instrument"),
652 psfMatched = connectionTypes.Output(
653 doc=(
"Output PSF-Matched warped exposure (previously called CoaddTempExp), produced by resampling ",
654 "calexps onto the skyMap patch geometry and PSF-matching to a model PSF."),
655 name=
"{coaddName}Coadd_psfMatchedWarp",
656 storageClass=
"ExposureF",
657 dimensions=(
"tract",
"patch",
"skymap",
"visit",
"instrument"),
660 wcsList = connectionTypes.Input(
661 doc=
"WCSs of calexps used by SelectImages subtask to determine if the calexp overlaps the patch",
662 name=
"{calexpType}calexp.wcs",
664 dimensions=(
"instrument",
"visit",
"detector"),
667 bboxList = connectionTypes.Input(
668 doc=
"BBoxes of calexps used by SelectImages subtask to determine if the calexp overlaps the patch",
669 name=
"{calexpType}calexp.bbox",
670 storageClass=
"Box2I",
671 dimensions=(
"instrument",
"visit",
"detector"),
674 visitSummary = connectionTypes.Input(
675 doc=
"Consolidated exposure metadata from ConsolidateVisitSummaryTask",
676 name=
"{calexpType}visitSummary",
677 storageClass=
"ExposureCatalog",
678 dimensions=(
"instrument",
"visit",),
681 def __init__(self, *, config=None):
682 super().__init__(config=config)
683 if config.bgSubtracted:
684 self.inputs.remove(
"backgroundList")
685 if not config.doApplySkyCorr:
686 self.inputs.remove(
"skyCorrList")
687 if config.doApplyExternalSkyWcs:
688 if config.useGlobalExternalSkyWcs:
689 self.inputs.remove(
"externalSkyWcsTractCatalog")
691 self.inputs.remove(
"externalSkyWcsGlobalCatalog")
693 self.inputs.remove(
"externalSkyWcsTractCatalog")
694 self.inputs.remove(
"externalSkyWcsGlobalCatalog")
695 if config.doApplyExternalPhotoCalib:
696 if config.useGlobalExternalPhotoCalib:
697 self.inputs.remove(
"externalPhotoCalibTractCatalog")
699 self.inputs.remove(
"externalPhotoCalibGlobalCatalog")
701 self.inputs.remove(
"externalPhotoCalibTractCatalog")
702 self.inputs.remove(
"externalPhotoCalibGlobalCatalog")
703 if not config.makeDirect:
704 self.outputs.remove(
"direct")
705 if not config.makePsfMatched:
706 self.outputs.remove(
"psfMatched")
708 if config.select.target != lsst.pipe.tasks.selectImages.PsfWcsSelectImagesTask:
709 self.inputs.remove(
"visitSummary")
713 pipelineConnections=MakeWarpConnections):
720 """Warp and optionally PSF-Match calexps onto an a common projection
722 ConfigClass = MakeWarpConfig
723 _DefaultName =
"makeWarp"
725 @utils.inheritDoc(pipeBase.PipelineTask)
726 def runQuantum(self, butlerQC, inputRefs, outputRefs):
730 Construct warps for requested warp type for single epoch
732 PipelineTask (Gen3) entry point to warp and optionally PSF-match
733 calexps. This method is analogous to `runDataRef`.
737 detectorOrder = [ref.datasetRef.dataId[
'detector']
for ref
in inputRefs.calExpList]
738 inputRefs = reorderRefs(inputRefs, detectorOrder, dataIdKey=
'detector')
741 inputs = butlerQC.get(inputRefs)
745 skyMap = inputs.pop(
"skyMap")
746 quantumDataId = butlerQC.quantum.dataId
747 skyInfo =
makeSkyInfo(skyMap, tractId=quantumDataId[
'tract'], patchId=quantumDataId[
'patch'])
750 dataIdList = [ref.datasetRef.dataId
for ref
in inputRefs.calExpList]
752 ccdIdList = [dataId.pack(
"visit_detector")
for dataId
in dataIdList]
757 coordList = [skyInfo.wcs.pixelToSky(pos)
for pos
in cornerPosList]
758 goodIndices = self.select.run(**inputs, coordList=coordList, dataIds=dataIdList)
759 inputs = self.filterInputs(indices=goodIndices, inputs=inputs)
762 inputs[
'calExpList'] = [ref.get()
for ref
in inputs[
'calExpList']]
765 visits = [dataId[
'visit']
for dataId
in dataIdList]
768 if self.config.doApplyExternalSkyWcs:
769 if self.config.useGlobalExternalSkyWcs:
770 externalSkyWcsCatalog = inputs.pop(
"externalSkyWcsGlobalCatalog")
772 externalSkyWcsCatalog = inputs.pop(
"externalSkyWcsTractCatalog")
774 externalSkyWcsCatalog =
None
776 if self.config.doApplyExternalPhotoCalib:
777 if self.config.useGlobalExternalPhotoCalib:
778 externalPhotoCalibCatalog = inputs.pop(
"externalPhotoCalibGlobalCatalog")
780 externalPhotoCalibCatalog = inputs.pop(
"externalPhotoCalibTractCatalog")
782 externalPhotoCalibCatalog =
None
784 completeIndices = self.prepareCalibratedExposures(**inputs,
785 externalSkyWcsCatalog=externalSkyWcsCatalog,
786 externalPhotoCalibCatalog=externalPhotoCalibCatalog)
788 inputs = self.filterInputs(indices=completeIndices, inputs=inputs)
790 results = self.run(**inputs, visitId=visitId,
791 ccdIdList=[ccdIdList[i]
for i
in goodIndices],
792 dataIdList=[dataIdList[i]
for i
in goodIndices],
794 if self.config.makeDirect
and results.exposures[
"direct"]
is not None:
795 butlerQC.put(results.exposures[
"direct"], outputRefs.direct)
796 if self.config.makePsfMatched
and results.exposures[
"psfMatched"]
is not None:
797 butlerQC.put(results.exposures[
"psfMatched"], outputRefs.psfMatched)
799 def filterInputs(self, indices, inputs):
800 """Return task inputs with their lists filtered by indices
804 indices : `list` of integers
805 inputs : `dict` of `list` of input connections to be passed to run
807 for key
in inputs.keys():
809 if isinstance(inputs[key], list):
810 inputs[key] = [inputs[key][ind]
for ind
in indices]
813 def prepareCalibratedExposures(self, calExpList, backgroundList=None, skyCorrList=None,
814 externalSkyWcsCatalog=None, externalPhotoCalibCatalog=None,
816 """Calibrate and add backgrounds to input calExpList in place
820 calExpList : `list` of `lsst.afw.image.Exposure`
821 Sequence of calexps to be modified in place
822 backgroundList : `list` of `lsst.afw.math.backgroundList`, optional
823 Sequence of backgrounds to be added back in if bgSubtracted=False
824 skyCorrList : `list` of `lsst.afw.math.backgroundList`, optional
825 Sequence of background corrections to be subtracted if doApplySkyCorr=True
826 externalSkyWcsCatalog : `lsst.afw.table.ExposureCatalog`, optional
827 Exposure catalog with external skyWcs to be applied
828 if config.doApplyExternalSkyWcs=True. Catalog uses the detector id
829 for the catalog id, sorted on id for fast lookup.
830 externalPhotoCalibCatalog : `lsst.afw.table.ExposureCatalog`, optional
831 Exposure catalog with external photoCalib to be applied
832 if config.doApplyExternalPhotoCalib=True. Catalog uses the detector
833 id for the catalog id, sorted on id for fast lookup.
837 indices : `list` [`int`]
838 Indices of calExpList and friends that have valid photoCalib/skyWcs
840 backgroundList = len(calExpList)*[
None]
if backgroundList
is None else backgroundList
841 skyCorrList = len(calExpList)*[
None]
if skyCorrList
is None else skyCorrList
843 includeCalibVar = self.config.includeCalibVar
846 for index, (calexp, background, skyCorr)
in enumerate(zip(calExpList,
849 mi = calexp.maskedImage
850 if not self.config.bgSubtracted:
851 mi += background.getImage()
853 if externalSkyWcsCatalog
is not None or externalPhotoCalibCatalog
is not None:
854 detectorId = calexp.getInfo().getDetector().getId()
857 if externalPhotoCalibCatalog
is not None:
858 row = externalPhotoCalibCatalog.find(detectorId)
860 self.log.warning(
"Detector id %s not found in externalPhotoCalibCatalog "
861 "and will not be used in the warp.", detectorId)
863 photoCalib = row.getPhotoCalib()
864 if photoCalib
is None:
865 self.log.warning(
"Detector id %s has None for photoCalib in externalPhotoCalibCatalog "
866 "and will not be used in the warp.", detectorId)
868 calexp.setPhotoCalib(photoCalib)
870 photoCalib = calexp.getPhotoCalib()
871 if photoCalib
is None:
872 self.log.warning(
"Detector id %s has None for photoCalib in the calexp "
873 "and will not be used in the warp.", detectorId)
877 if externalSkyWcsCatalog
is not None:
878 row = externalSkyWcsCatalog.find(detectorId)
880 self.log.warning(
"Detector id %s not found in externalSkyWcsCatalog "
881 "and will not be used in the warp.", detectorId)
883 skyWcs = row.getWcs()
885 self.log.warning(
"Detector id %s has None for skyWcs in externalSkyWcsCatalog "
886 "and will not be used in the warp.", detectorId)
888 calexp.setWcs(skyWcs)
890 skyWcs = calexp.getWcs()
892 self.log.warning(
"Detector id %s has None for skyWcs in the calexp "
893 "and will not be used in the warp.", detectorId)
897 calexp.maskedImage = photoCalib.calibrateImage(calexp.maskedImage,
898 includeScaleUncertainty=includeCalibVar)
899 calexp.maskedImage /= photoCalib.getCalibrationMean()
904 if self.config.doApplySkyCorr:
905 mi -= skyCorr.getImage()
907 indices.append(index)
912 def reorderRefs(inputRefs, outputSortKeyOrder, dataIdKey):
913 """Reorder inputRefs per outputSortKeyOrder
915 Any inputRefs which are lists will be resorted per specified key e.g.,
916 'detector.' Only iterables will be reordered, and values can be of type
917 `lsst.pipe.base.connections.DeferredDatasetRef` or
918 `lsst.daf.butler.core.datasets.ref.DatasetRef`.
919 Returned lists of refs have the same length as the outputSortKeyOrder.
920 If an outputSortKey not in the inputRef, then it will be padded with None.
921 If an inputRef contains an inputSortKey that is not in the
922 outputSortKeyOrder it will be removed.
926 inputRefs : `lsst.pipe.base.connections.QuantizedConnection`
927 Input references to be reordered and padded.
928 outputSortKeyOrder : iterable
929 Iterable of values to be compared with inputRef's dataId[dataIdKey]
931 dataIdKey in the dataRefs to compare with the outputSortKeyOrder.
935 inputRefs: `lsst.pipe.base.connections.QuantizedConnection`
936 Quantized Connection with sorted DatasetRef values sorted if iterable.
938 for connectionName, refs
in inputRefs:
939 if isinstance(refs, Iterable):
940 if hasattr(refs[0],
"dataId"):
941 inputSortKeyOrder = [ref.dataId[dataIdKey]
for ref
in refs]
943 inputSortKeyOrder = [ref.datasetRef.dataId[dataIdKey]
for ref
in refs]
944 if inputSortKeyOrder != outputSortKeyOrder:
945 setattr(inputRefs, connectionName,
Base class for coaddition.
def getTempExpDatasetName(self, warpType="direct")
def selectExposures(self, patchRef, skyInfo=None, selectDataList=[])
Select exposures to coadd.
def getCoaddDatasetName(self, warpType="direct")
def getSkyInfo(self, patchRef)
Use getSkyinfo to return the skyMap, tract and patch information, wcs and the outer bbox of the patch...
def getBadPixelMask(self)
Convenience method to provide the bitmask from the mask plane names.
Warp and optionally PSF-Match calexps onto an a common projection.
def getCalibratedExposure(self, dataRef, bgSubtracted)
def run(self, calExpList, ccdIdList, skyInfo, visitId=0, dataIdList=None, **kwargs)
def __init__(self, reuse=False, **kwargs)
def _prepareEmptyExposure(skyInfo)
def runDataRef(self, patchRef, selectDataList=[])
Produce <coaddName>Coadd_<warpType>Warp images by warping and optionally PSF-matching.
def getWarpTypeList(self)
def applySkyCorr(self, dataRef, calexp)
def reorderAndPadList(inputList, inputKeys, outputKeys, padWith=None)
def makeSkyInfo(skyMap, tractId, patchId)
def getGroupDataRef(butler, datasetType, groupTuple, keys)
def groupPatchExposures(patchDataRef, calexpDataRefList, coaddDatasetType="deepCoadd", tempExpDatasetType="deepCoadd_directWarp")