30 import lsst.pipe.base.connectionTypes
as connectionTypes
31 import lsst.utils
as utils
35 from lsst.utils.timer
import timeMethod
36 from .coaddBase
import CoaddBaseTask, makeSkyInfo, reorderAndPadList
37 from .warpAndPsfMatch
import WarpAndPsfMatchTask
38 from .coaddHelpers
import groupPatchExposures, getGroupDataRef
39 from collections.abc
import Iterable
41 __all__ = [
"MakeCoaddTempExpTask",
"MakeWarpTask",
"MakeWarpConfig"]
43 log = logging.getLogger(__name__.partition(
".")[2])
47 """Raised when data cannot be retrieved for an exposure.
48 When processing patches, sometimes one exposure is missing; this lets us
49 distinguish bewteen that case, and other errors.
55 """Config for MakeCoaddTempExpTask
57 warpAndPsfMatch = pexConfig.ConfigurableField(
58 target=WarpAndPsfMatchTask,
59 doc=
"Task to warp and PSF-match calexp",
61 doWrite = pexConfig.Field(
62 doc=
"persist <coaddName>Coadd_<warpType>Warp",
66 bgSubtracted = pexConfig.Field(
67 doc=
"Work with a background subtracted calexp?",
71 coaddPsf = pexConfig.ConfigField(
72 doc=
"Configuration for CoaddPsf",
75 makeDirect = pexConfig.Field(
76 doc=
"Make direct Warp/Coadds",
80 makePsfMatched = pexConfig.Field(
81 doc=
"Make Psf-Matched Warp/Coadd?",
86 doWriteEmptyWarps = pexConfig.Field(
89 doc=
"Write out warps even if they are empty"
92 hasFakes = pexConfig.Field(
93 doc=
"Should be set to True if fake sources have been inserted into the input data.",
97 doApplySkyCorr = pexConfig.Field(dtype=bool, default=
False, doc=
"Apply sky correction?")
100 CoaddBaseTask.ConfigClass.validate(self)
102 raise RuntimeError(
"At least one of config.makePsfMatched and config.makeDirect must be True")
105 log.warning(
"Config doPsfMatch deprecated. Setting makePsfMatched=True and makeDirect=False")
110 CoaddBaseTask.ConfigClass.setDefaults(self)
111 self.
warpAndPsfMatchwarpAndPsfMatch.psfMatch.kernel.active.kernelSize = self.matchingKernelSize
122 r"""!Warp and optionally PSF-Match calexps onto an a common projection.
124 @anchor MakeCoaddTempExpTask_
126 @section pipe_tasks_makeCoaddTempExp_Contents Contents
128 - @ref pipe_tasks_makeCoaddTempExp_Purpose
129 - @ref pipe_tasks_makeCoaddTempExp_Initialize
130 - @ref pipe_tasks_makeCoaddTempExp_IO
131 - @ref pipe_tasks_makeCoaddTempExp_Config
132 - @ref pipe_tasks_makeCoaddTempExp_Debug
133 - @ref pipe_tasks_makeCoaddTempExp_Example
135 @section pipe_tasks_makeCoaddTempExp_Purpose Description
137 Warp and optionally PSF-Match calexps onto a common projection, by
138 performing the following operations:
139 - Group calexps by visit/run
140 - For each visit, generate a Warp by calling method @ref makeTempExp.
141 makeTempExp loops over the visit's calexps calling @ref WarpAndPsfMatch
144 The result is a `directWarp` (and/or optionally a `psfMatchedWarp`).
146 @section pipe_tasks_makeCoaddTempExp_Initialize Task Initialization
148 @copydoc \_\_init\_\_
150 This task has one special keyword argument: passing reuse=True will cause
151 the task to skip the creation of warps that are already present in the
154 @section pipe_tasks_makeCoaddTempExp_IO Invoking the Task
156 This task is primarily designed to be run from the command line.
158 The main method is `runDataRef`, which takes a single butler data reference for the patch(es)
163 WarpType identifies the types of convolutions applied to Warps (previously CoaddTempExps).
164 Only two types are available: direct (for regular Warps/Coadds) and psfMatched
165 (for Warps/Coadds with homogenized PSFs). We expect to add a third type, likelihood,
166 for generating likelihood Coadds with Warps that have been correlated with their own PSF.
168 @section pipe_tasks_makeCoaddTempExp_Config Configuration parameters
170 See @ref MakeCoaddTempExpConfig and parameters inherited from
171 @link lsst.pipe.tasks.coaddBase.CoaddBaseConfig CoaddBaseConfig @endlink
173 @subsection pipe_tasks_MakeCoaddTempExp_psfMatching Guide to PSF-Matching Configs
175 To make `psfMatchedWarps`, select `config.makePsfMatched=True`. The subtask
176 @link lsst.ip.diffim.modelPsfMatch.ModelPsfMatchTask ModelPsfMatchTask @endlink
177 is responsible for the PSF-Matching, and its config is accessed via `config.warpAndPsfMatch.psfMatch`.
178 The optimal configuration depends on aspects of dataset: the pixel scale, average PSF FWHM and
179 dimensions of the PSF kernel. These configs include the requested model PSF, the matching kernel size,
180 padding of the science PSF thumbnail and spatial sampling frequency of the PSF.
182 *Config Guidelines*: The user must specify the size of the model PSF to which to match by setting
183 `config.modelPsf.defaultFwhm` in units of pixels. The appropriate values depends on science case.
184 In general, for a set of input images, this config should equal the FWHM of the visit
185 with the worst seeing. The smallest it should be set to is the median FWHM. The defaults
186 of the other config options offer a reasonable starting point.
187 The following list presents the most common problems that arise from a misconfigured
188 @link lsst.ip.diffim.modelPsfMatch.ModelPsfMatchTask ModelPsfMatchTask @endlink
189 and corresponding solutions. All assume the default Alard-Lupton kernel, with configs accessed via
190 ```config.warpAndPsfMatch.psfMatch.kernel['AL']```. Each item in the list is formatted as:
191 Problem: Explanation. *Solution*
193 *Troublshooting PSF-Matching Configuration:*
194 - Matched PSFs look boxy: The matching kernel is too small. _Increase the matching kernel size.
197 config.warpAndPsfMatch.psfMatch.kernel['AL'].kernelSize=27 # default 21
199 Note that increasing the kernel size also increases runtime.
200 - Matched PSFs look ugly (dipoles, quadropoles, donuts): unable to find good solution
201 for matching kernel. _Provide the matcher with more data by either increasing
202 the spatial sampling by decreasing the spatial cell size,_
204 config.warpAndPsfMatch.psfMatch.kernel['AL'].sizeCellX = 64 # default 128
205 config.warpAndPsfMatch.psfMatch.kernel['AL'].sizeCellY = 64 # default 128
207 _or increasing the padding around the Science PSF, for example:_
209 config.warpAndPsfMatch.psfMatch.autoPadPsfTo=1.6 # default 1.4
211 Increasing `autoPadPsfTo` increases the minimum ratio of input PSF dimensions to the
212 matching kernel dimensions, thus increasing the number of pixels available to fit
213 after convolving the PSF with the matching kernel.
214 Optionally, for debugging the effects of padding, the level of padding may be manually
215 controlled by setting turning off the automatic padding and setting the number
216 of pixels by which to pad the PSF:
218 config.warpAndPsfMatch.psfMatch.doAutoPadPsf = False # default True
219 config.warpAndPsfMatch.psfMatch.padPsfBy = 6 # pixels. default 0
221 - Deconvolution: Matching a large PSF to a smaller PSF produces
222 a telltale noise pattern which looks like ripples or a brain.
223 _Increase the size of the requested model PSF. For example:_
225 config.modelPsf.defaultFwhm = 11 # Gaussian sigma in units of pixels.
227 - High frequency (sometimes checkered) noise: The matching basis functions are too small.
228 _Increase the width of the Gaussian basis functions. For example:_
230 config.warpAndPsfMatch.psfMatch.kernel['AL'].alardSigGauss=[1.5, 3.0, 6.0]
231 # from default [0.7, 1.5, 3.0]
234 @section pipe_tasks_makeCoaddTempExp_Debug Debug variables
236 MakeCoaddTempExpTask has no debug output, but its subtasks do.
238 @section pipe_tasks_makeCoaddTempExp_Example A complete example of using MakeCoaddTempExpTask
240 This example uses the package ci_hsc to show how MakeCoaddTempExp fits
241 into the larger Data Release Processing.
246 # if not built already:
247 python $(which scons) # this will take a while
249 The following assumes that `processCcd.py` and `makeSkyMap.py` have previously been run
250 (e.g. by building `ci_hsc` above) to generate a repository of calexps and an
251 output respository with the desired SkyMap. The command,
253 makeCoaddTempExp.py $CI_HSC_DIR/DATA --rerun ci_hsc \
254 --id patch=5,4 tract=0 filter=HSC-I \
255 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 \
256 --selectId visit=903988 ccd=23 --selectId visit=903988 ccd=24 \
257 --config doApplyExternalPhotoCalib=False doApplyExternalSkyWcs=False \
258 makePsfMatched=True modelPsf.defaultFwhm=11
260 writes a direct and PSF-Matched Warp to
261 - `$CI_HSC_DIR/DATA/rerun/ci_hsc/deepCoadd/HSC-I/0/5,4/warp-HSC-I-0-5,4-903988.fits` and
262 - `$CI_HSC_DIR/DATA/rerun/ci_hsc/deepCoadd/HSC-I/0/5,4/psfMatchedWarp-HSC-I-0-5,4-903988.fits`
265 @note PSF-Matching in this particular dataset would benefit from adding
266 `--configfile ./matchingConfig.py` to
267 the command line arguments where `matchingConfig.py` is defined by:
270 config.warpAndPsfMatch.psfMatch.kernel['AL'].kernelSize=27
271 config.warpAndPsfMatch.psfMatch.kernel['AL'].alardSigGauss=[1.5, 3.0, 6.0]" > matchingConfig.py
274 Add the option `--help` to see more options.
276 ConfigClass = MakeCoaddTempExpConfig
277 _DefaultName =
"makeCoaddTempExp"
280 CoaddBaseTask.__init__(self, **kwargs)
282 self.makeSubtask(
"warpAndPsfMatch")
283 if self.config.hasFakes:
290 """!Produce <coaddName>Coadd_<warpType>Warp images by warping and optionally PSF-matching.
292 @param[in] patchRef: data reference for sky map patch. Must include keys "tract", "patch",
293 plus the camera-specific filter key (e.g. "filter" or "band")
294 @return: dataRefList: a list of data references for the new <coaddName>Coadd_directWarps
295 if direct or both warp types are requested and <coaddName>Coadd_psfMatchedWarps if only psfMatched
298 @warning: this task assumes that all exposures in a warp (coaddTempExp) have the same filter.
300 @warning: this task sets the PhotoCalib of the coaddTempExp to the PhotoCalib of the first calexp
301 with any good pixels in the patch. For a mosaic camera the resulting PhotoCalib should be ignored
302 (assembleCoadd should determine zeropoint scaling without referring to it).
307 if self.config.makePsfMatched
and not self.config.makeDirect:
312 calExpRefList = self.
selectExposuresselectExposures(patchRef, skyInfo, selectDataList=selectDataList)
314 if len(calExpRefList) == 0:
315 self.log.warning(
"No exposures to coadd for patch %s", patchRef.dataId)
317 self.log.info(
"Selected %d calexps for patch %s", len(calExpRefList), patchRef.dataId)
318 calExpRefList = [calExpRef
for calExpRef
in calExpRefList
if calExpRef.datasetExists(self.
calexpTypecalexpType)]
319 self.log.info(
"Processing %d existing calexps for patch %s", len(calExpRefList), patchRef.dataId)
323 self.log.info(
"Processing %d warp exposures for patch %s", len(groupData.groups), patchRef.dataId)
326 for i, (tempExpTuple, calexpRefList)
in enumerate(groupData.groups.items()):
328 tempExpTuple, groupData.keys)
329 if self.
reusereuse
and tempExpRef.datasetExists(datasetType=primaryWarpDataset, write=
True):
330 self.log.info(
"Skipping makeCoaddTempExp for %s; output already exists.", tempExpRef.dataId)
331 dataRefList.append(tempExpRef)
333 self.log.info(
"Processing Warp %d/%d: id=%s", i, len(groupData.groups), tempExpRef.dataId)
339 visitId = int(tempExpRef.dataId[
"visit"])
340 except (KeyError, ValueError):
347 for calExpInd, calExpRef
in enumerate(calexpRefList):
348 self.log.info(
"Reading calexp %s of %s for Warp id=%s", calExpInd+1, len(calexpRefList),
351 ccdId = calExpRef.get(
"ccdExposureId", immediate=
True)
358 calExpRef = calExpRef.butlerSubset.butler.dataRef(self.
calexpTypecalexpType,
359 dataId=calExpRef.dataId,
360 tract=skyInfo.tractInfo.getId())
361 calExp = self.
getCalibratedExposuregetCalibratedExposure(calExpRef, bgSubtracted=self.config.bgSubtracted)
362 except Exception
as e:
363 self.log.warning(
"Calexp %s not found; skipping it: %s", calExpRef.dataId, e)
366 if self.config.doApplySkyCorr:
369 calExpList.append(calExp)
370 ccdIdList.append(ccdId)
371 dataIdList.append(calExpRef.dataId)
373 exps = self.
runrun(calExpList, ccdIdList, skyInfo, visitId, dataIdList).exposures
375 if any(exps.values()):
376 dataRefList.append(tempExpRef)
378 self.log.warning(
"Warp %s could not be created", tempExpRef.dataId)
380 if self.config.doWrite:
381 for (warpType, exposure)
in exps.items():
382 if exposure
is not None:
389 def run(self, calExpList, ccdIdList, skyInfo, visitId=0, dataIdList=None, **kwargs):
390 """Create a Warp from inputs
392 We iterate over the multiple calexps in a single exposure to construct
393 the warp (previously called a coaddTempExp) of that exposure to the
394 supplied tract/patch.
396 Pixels that receive no pixels are set to NAN; this is not correct
397 (violates LSST algorithms group policy), but will be fixed up by
398 interpolating after the coaddition.
400 @param calexpRefList: List of data references for calexps that (may)
401 overlap the patch of interest
402 @param skyInfo: Struct from CoaddBaseTask.getSkyInfo() with geometric
403 information about the patch
404 @param visitId: integer identifier for visit, for the table that will
406 @return a pipeBase Struct containing:
407 - exposures: a dictionary containing the warps requested:
408 "direct": direct warp if config.makeDirect
409 "psfMatched": PSF-matched warp if config.makePsfMatched
413 totGoodPix = {warpType: 0
for warpType
in warpTypeList}
414 didSetMetadata = {warpType:
False for warpType
in warpTypeList}
415 coaddTempExps = {warpType: self.
_prepareEmptyExposure_prepareEmptyExposure(skyInfo)
for warpType
in warpTypeList}
416 inputRecorder = {warpType: self.inputRecorder.makeCoaddTempExpRecorder(visitId, len(calExpList))
417 for warpType
in warpTypeList}
419 modelPsf = self.config.modelPsf.apply()
if self.config.makePsfMatched
else None
420 if dataIdList
is None:
421 dataIdList = ccdIdList
423 for calExpInd, (calExp, ccdId, dataId)
in enumerate(zip(calExpList, ccdIdList, dataIdList)):
424 self.log.info(
"Processing calexp %d of %d for this Warp: id=%s",
425 calExpInd+1, len(calExpList), dataId)
428 warpedAndMatched = self.warpAndPsfMatch.
run(calExp, modelPsf=modelPsf,
429 wcs=skyInfo.wcs, maxBBox=skyInfo.bbox,
430 makeDirect=self.config.makeDirect,
431 makePsfMatched=self.config.makePsfMatched)
432 except Exception
as e:
433 self.log.warning(
"WarpAndPsfMatch failed for calexp %s; skipping it: %s", dataId, e)
436 numGoodPix = {warpType: 0
for warpType
in warpTypeList}
437 for warpType
in warpTypeList:
438 exposure = warpedAndMatched.getDict()[warpType]
441 coaddTempExp = coaddTempExps[warpType]
442 if didSetMetadata[warpType]:
443 mimg = exposure.getMaskedImage()
444 mimg *= (coaddTempExp.getPhotoCalib().getInstFluxAtZeroMagnitude()
445 / exposure.getPhotoCalib().getInstFluxAtZeroMagnitude())
447 numGoodPix[warpType] = coaddUtils.copyGoodPixels(
448 coaddTempExp.getMaskedImage(), exposure.getMaskedImage(), self.
getBadPixelMaskgetBadPixelMask())
449 totGoodPix[warpType] += numGoodPix[warpType]
450 self.log.debug(
"Calexp %s has %d good pixels in this patch (%.1f%%) for %s",
451 dataId, numGoodPix[warpType],
452 100.0*numGoodPix[warpType]/skyInfo.bbox.getArea(), warpType)
453 if numGoodPix[warpType] > 0
and not didSetMetadata[warpType]:
454 coaddTempExp.setPhotoCalib(exposure.getPhotoCalib())
455 coaddTempExp.setFilterLabel(exposure.getFilterLabel())
456 coaddTempExp.getInfo().setVisitInfo(exposure.getInfo().getVisitInfo())
458 coaddTempExp.setPsf(exposure.getPsf())
459 didSetMetadata[warpType] =
True
462 inputRecorder[warpType].addCalExp(calExp, ccdId, numGoodPix[warpType])
464 except Exception
as e:
465 self.log.warning(
"Error processing calexp %s; skipping it: %s", dataId, e)
468 for warpType
in warpTypeList:
469 self.log.info(
"%sWarp has %d good pixels (%.1f%%)",
470 warpType, totGoodPix[warpType], 100.0*totGoodPix[warpType]/skyInfo.bbox.getArea())
472 if totGoodPix[warpType] > 0
and didSetMetadata[warpType]:
473 inputRecorder[warpType].finish(coaddTempExps[warpType], totGoodPix[warpType])
474 if warpType ==
"direct":
475 coaddTempExps[warpType].setPsf(
476 CoaddPsf(inputRecorder[warpType].coaddInputs.ccds, skyInfo.wcs,
477 self.config.coaddPsf.makeControl()))
479 if not self.config.doWriteEmptyWarps:
481 coaddTempExps[warpType] =
None
486 result = pipeBase.Struct(exposures=coaddTempExps)
490 """Return one calibrated Exposure, possibly with an updated SkyWcs.
492 @param[in] dataRef a sensor-level data reference
493 @param[in] bgSubtracted return calexp with background subtracted? If False get the
494 calexp's background background model and add it to the calexp.
495 @return calibrated exposure
497 @raises MissingExposureError If data for the exposure is not available.
499 If config.doApplyExternalPhotoCalib is `True`, the photometric calibration
500 (`photoCalib`) is taken from `config.externalPhotoCalibName` via the
501 `name_photoCalib` dataset. Otherwise, the photometric calibration is
502 retrieved from the processed exposure. When
503 `config.doApplyExternalSkyWcs` is `True`, the astrometric calibration
504 is taken from `config.externalSkyWcsName` with the `name_wcs` dataset.
505 Otherwise, the astrometric calibration is taken from the processed
509 exposure = dataRef.get(self.
calexpTypecalexpType, immediate=
True)
510 except dafPersist.NoResults
as e:
514 background = dataRef.get(
"calexpBackground", immediate=
True)
515 mi = exposure.getMaskedImage()
516 mi += background.getImage()
519 if self.config.doApplyExternalPhotoCalib:
520 source = f
"{self.config.externalPhotoCalibName}_photoCalib"
521 self.log.debug(
"Applying external photoCalib to %s from %s", dataRef.dataId, source)
522 photoCalib = dataRef.get(source)
523 exposure.setPhotoCalib(photoCalib)
525 photoCalib = exposure.getPhotoCalib()
527 if self.config.doApplyExternalSkyWcs:
528 source = f
"{self.config.externalSkyWcsName}_wcs"
529 self.log.debug(
"Applying external skyWcs to %s from %s", dataRef.dataId, source)
530 skyWcs = dataRef.get(source)
531 exposure.setWcs(skyWcs)
533 exposure.maskedImage = photoCalib.calibrateImage(exposure.maskedImage,
534 includeScaleUncertainty=self.config.includeCalibVar)
535 exposure.maskedImage /= photoCalib.getCalibrationMean()
541 def _prepareEmptyExposure(skyInfo):
542 """Produce an empty exposure for a given patch"""
543 exp = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
544 exp.getMaskedImage().set(numpy.nan, afwImage.Mask
545 .getPlaneBitMask(
"NO_DATA"), numpy.inf)
549 """Return list of requested warp types per the config.
552 if self.config.makeDirect:
553 warpTypeList.append(
"direct")
554 if self.config.makePsfMatched:
555 warpTypeList.append(
"psfMatched")
559 """Apply correction to the sky background level
561 Sky corrections can be generated with the 'skyCorrection.py'
562 executable in pipe_drivers. Because the sky model used by that
563 code extends over the entire focal plane, this can produce
564 better sky subtraction.
566 The calexp is updated in-place.
570 dataRef : `lsst.daf.persistence.ButlerDataRef`
571 Data reference for calexp.
572 calexp : `lsst.afw.image.Exposure` or `lsst.afw.image.MaskedImage`
575 bg = dataRef.get(
"skyCorr")
576 self.log.debug(
"Applying sky correction to %s", dataRef.dataId)
577 if isinstance(calexp, afwImage.Exposure):
578 calexp = calexp.getMaskedImage()
579 calexp -= bg.getImage()
583 dimensions=(
"tract",
"patch",
"skymap",
"instrument",
"visit"),
584 defaultTemplates={
"coaddName":
"deep",
585 "skyWcsName":
"jointcal",
586 "photoCalibName":
"fgcm",
588 calExpList = connectionTypes.Input(
589 doc=
"Input exposures to be resampled and optionally PSF-matched onto a SkyMap projection/patch",
590 name=
"{calexpType}calexp",
591 storageClass=
"ExposureF",
592 dimensions=(
"instrument",
"visit",
"detector"),
596 backgroundList = connectionTypes.Input(
597 doc=
"Input backgrounds to be added back into the calexp if bgSubtracted=False",
598 name=
"calexpBackground",
599 storageClass=
"Background",
600 dimensions=(
"instrument",
"visit",
"detector"),
603 skyCorrList = connectionTypes.Input(
604 doc=
"Input Sky Correction to be subtracted from the calexp if doApplySkyCorr=True",
606 storageClass=
"Background",
607 dimensions=(
"instrument",
"visit",
"detector"),
610 skyMap = connectionTypes.Input(
611 doc=
"Input definition of geometry/bbox and projection/wcs for warped exposures",
612 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
613 storageClass=
"SkyMap",
614 dimensions=(
"skymap",),
616 externalSkyWcsTractCatalog = connectionTypes.Input(
617 doc=(
"Per-tract, per-visit wcs calibrations. These catalogs use the detector "
618 "id for the catalog id, sorted on id for fast lookup."),
619 name=
"{skyWcsName}SkyWcsCatalog",
620 storageClass=
"ExposureCatalog",
621 dimensions=(
"instrument",
"visit",
"tract"),
623 externalSkyWcsGlobalCatalog = connectionTypes.Input(
624 doc=(
"Per-visit wcs calibrations computed globally (with no tract information). "
625 "These catalogs use the detector id for the catalog id, sorted on id for "
627 name=
"{skyWcsName}SkyWcsCatalog",
628 storageClass=
"ExposureCatalog",
629 dimensions=(
"instrument",
"visit"),
631 externalPhotoCalibTractCatalog = connectionTypes.Input(
632 doc=(
"Per-tract, per-visit photometric calibrations. These catalogs use the "
633 "detector id for the catalog id, sorted on id for fast lookup."),
634 name=
"{photoCalibName}PhotoCalibCatalog",
635 storageClass=
"ExposureCatalog",
636 dimensions=(
"instrument",
"visit",
"tract"),
638 externalPhotoCalibGlobalCatalog = connectionTypes.Input(
639 doc=(
"Per-visit photometric calibrations computed globally (with no tract "
640 "information). These catalogs use the detector id for the catalog id, "
641 "sorted on id for fast lookup."),
642 name=
"{photoCalibName}PhotoCalibCatalog",
643 storageClass=
"ExposureCatalog",
644 dimensions=(
"instrument",
"visit"),
646 direct = connectionTypes.Output(
647 doc=(
"Output direct warped exposure (previously called CoaddTempExp), produced by resampling ",
648 "calexps onto the skyMap patch geometry."),
649 name=
"{coaddName}Coadd_directWarp",
650 storageClass=
"ExposureF",
651 dimensions=(
"tract",
"patch",
"skymap",
"visit",
"instrument"),
653 psfMatched = connectionTypes.Output(
654 doc=(
"Output PSF-Matched warped exposure (previously called CoaddTempExp), produced by resampling ",
655 "calexps onto the skyMap patch geometry and PSF-matching to a model PSF."),
656 name=
"{coaddName}Coadd_psfMatchedWarp",
657 storageClass=
"ExposureF",
658 dimensions=(
"tract",
"patch",
"skymap",
"visit",
"instrument"),
661 wcsList = connectionTypes.Input(
662 doc=
"WCSs of calexps used by SelectImages subtask to determine if the calexp overlaps the patch",
663 name=
"{calexpType}calexp.wcs",
665 dimensions=(
"instrument",
"visit",
"detector"),
668 bboxList = connectionTypes.Input(
669 doc=
"BBoxes of calexps used by SelectImages subtask to determine if the calexp overlaps the patch",
670 name=
"{calexpType}calexp.bbox",
671 storageClass=
"Box2I",
672 dimensions=(
"instrument",
"visit",
"detector"),
675 srcList = connectionTypes.Input(
676 doc=
"src catalogs used by PsfWcsSelectImages subtask to further select on PSF stability",
678 storageClass=
"SourceCatalog",
679 dimensions=(
"instrument",
"visit",
"detector"),
682 psfList = connectionTypes.Input(
683 doc=
"PSF models used by BestSeeingWcsSelectImages subtask to futher select on seeing",
684 name=
"{calexpType}calexp.psf",
686 dimensions=(
"instrument",
"visit",
"detector"),
690 def __init__(self, *, config=None):
691 super().__init__(config=config)
692 if config.bgSubtracted:
693 self.inputs.remove(
"backgroundList")
694 if not config.doApplySkyCorr:
695 self.inputs.remove(
"skyCorrList")
696 if config.doApplyExternalSkyWcs:
697 if config.useGlobalExternalSkyWcs:
698 self.inputs.remove(
"externalSkyWcsTractCatalog")
700 self.inputs.remove(
"externalSkyWcsGlobalCatalog")
702 self.inputs.remove(
"externalSkyWcsTractCatalog")
703 self.inputs.remove(
"externalSkyWcsGlobalCatalog")
704 if config.doApplyExternalPhotoCalib:
705 if config.useGlobalExternalPhotoCalib:
706 self.inputs.remove(
"externalPhotoCalibTractCatalog")
708 self.inputs.remove(
"externalPhotoCalibGlobalCatalog")
710 self.inputs.remove(
"externalPhotoCalibTractCatalog")
711 self.inputs.remove(
"externalPhotoCalibGlobalCatalog")
712 if not config.makeDirect:
713 self.outputs.remove(
"direct")
714 if not config.makePsfMatched:
715 self.outputs.remove(
"psfMatched")
718 if config.select.target != lsst.pipe.tasks.selectImages.PsfWcsSelectImagesTask:
719 self.inputs.remove(
"srcList")
720 if config.select.target != lsst.pipe.tasks.selectImages.BestSeeingWcsSelectImagesTask:
721 self.inputs.remove(
"psfList")
725 pipelineConnections=MakeWarpConnections):
732 """Warp and optionally PSF-Match calexps onto an a common projection
734 ConfigClass = MakeWarpConfig
735 _DefaultName =
"makeWarp"
737 @utils.inheritDoc(pipeBase.PipelineTask)
738 def runQuantum(self, butlerQC, inputRefs, outputRefs):
742 Construct warps for requested warp type for single epoch
744 PipelineTask (Gen3) entry point to warp and optionally PSF-match
745 calexps. This method is analogous to `runDataRef`.
749 detectorOrder = [ref.datasetRef.dataId[
'detector']
for ref
in inputRefs.calExpList]
750 inputRefs = reorderRefs(inputRefs, detectorOrder, dataIdKey=
'detector')
753 inputs = butlerQC.get(inputRefs)
757 skyMap = inputs.pop(
"skyMap")
758 quantumDataId = butlerQC.quantum.dataId
759 skyInfo =
makeSkyInfo(skyMap, tractId=quantumDataId[
'tract'], patchId=quantumDataId[
'patch'])
762 dataIdList = [ref.datasetRef.dataId
for ref
in inputRefs.calExpList]
764 ccdIdList = [dataId.pack(
"visit_detector")
for dataId
in dataIdList]
769 coordList = [skyInfo.wcs.pixelToSky(pos)
for pos
in cornerPosList]
770 goodIndices = self.select.run(**inputs, coordList=coordList, dataIds=dataIdList)
771 inputs = self.filterInputs(indices=goodIndices, inputs=inputs)
774 inputs[
'calExpList'] = [ref.get()
for ref
in inputs[
'calExpList']]
777 visits = [dataId[
'visit']
for dataId
in dataIdList]
780 if self.config.doApplyExternalSkyWcs:
781 if self.config.useGlobalExternalSkyWcs:
782 externalSkyWcsCatalog = inputs.pop(
"externalSkyWcsGlobalCatalog")
784 externalSkyWcsCatalog = inputs.pop(
"externalSkyWcsTractCatalog")
786 externalSkyWcsCatalog =
None
788 if self.config.doApplyExternalPhotoCalib:
789 if self.config.useGlobalExternalPhotoCalib:
790 externalPhotoCalibCatalog = inputs.pop(
"externalPhotoCalibGlobalCatalog")
792 externalPhotoCalibCatalog = inputs.pop(
"externalPhotoCalibTractCatalog")
794 externalPhotoCalibCatalog =
None
796 completeIndices = self.prepareCalibratedExposures(**inputs,
797 externalSkyWcsCatalog=externalSkyWcsCatalog,
798 externalPhotoCalibCatalog=externalPhotoCalibCatalog)
800 inputs = self.filterInputs(indices=completeIndices, inputs=inputs)
802 results = self.run(**inputs, visitId=visitId,
803 ccdIdList=[ccdIdList[i]
for i
in goodIndices],
804 dataIdList=[dataIdList[i]
for i
in goodIndices],
806 if self.config.makeDirect
and results.exposures[
"direct"]
is not None:
807 butlerQC.put(results.exposures[
"direct"], outputRefs.direct)
808 if self.config.makePsfMatched
and results.exposures[
"psfMatched"]
is not None:
809 butlerQC.put(results.exposures[
"psfMatched"], outputRefs.psfMatched)
811 def filterInputs(self, indices, inputs):
812 """Return task inputs with their lists filtered by indices
816 indices : `list` of integers
817 inputs : `dict` of `list` of input connections to be passed to run
819 for key
in inputs.keys():
821 if isinstance(inputs[key], list):
822 inputs[key] = [inputs[key][ind]
for ind
in indices]
825 def prepareCalibratedExposures(self, calExpList, backgroundList=None, skyCorrList=None,
826 externalSkyWcsCatalog=None, externalPhotoCalibCatalog=None,
828 """Calibrate and add backgrounds to input calExpList in place
832 calExpList : `list` of `lsst.afw.image.Exposure`
833 Sequence of calexps to be modified in place
834 backgroundList : `list` of `lsst.afw.math.backgroundList`, optional
835 Sequence of backgrounds to be added back in if bgSubtracted=False
836 skyCorrList : `list` of `lsst.afw.math.backgroundList`, optional
837 Sequence of background corrections to be subtracted if doApplySkyCorr=True
838 externalSkyWcsCatalog : `lsst.afw.table.ExposureCatalog`, optional
839 Exposure catalog with external skyWcs to be applied
840 if config.doApplyExternalSkyWcs=True. Catalog uses the detector id
841 for the catalog id, sorted on id for fast lookup.
842 externalPhotoCalibCatalog : `lsst.afw.table.ExposureCatalog`, optional
843 Exposure catalog with external photoCalib to be applied
844 if config.doApplyExternalPhotoCalib=True. Catalog uses the detector
845 id for the catalog id, sorted on id for fast lookup.
849 indices : `list` [`int`]
850 Indices of calExpList and friends that have valid photoCalib/skyWcs
852 backgroundList = len(calExpList)*[
None]
if backgroundList
is None else backgroundList
853 skyCorrList = len(calExpList)*[
None]
if skyCorrList
is None else skyCorrList
855 includeCalibVar = self.config.includeCalibVar
858 for index, (calexp, background, skyCorr)
in enumerate(zip(calExpList,
861 mi = calexp.maskedImage
862 if not self.config.bgSubtracted:
863 mi += background.getImage()
865 if externalSkyWcsCatalog
is not None or externalPhotoCalibCatalog
is not None:
866 detectorId = calexp.getInfo().getDetector().getId()
869 if externalPhotoCalibCatalog
is not None:
870 row = externalPhotoCalibCatalog.find(detectorId)
872 self.log.warning(
"Detector id %s not found in externalPhotoCalibCatalog "
873 "and will not be used in the warp.", detectorId)
875 photoCalib = row.getPhotoCalib()
876 if photoCalib
is None:
877 self.log.warning(
"Detector id %s has None for photoCalib in externalPhotoCalibCatalog "
878 "and will not be used in the warp.", detectorId)
880 calexp.setPhotoCalib(photoCalib)
882 photoCalib = calexp.getPhotoCalib()
883 if photoCalib
is None:
884 self.log.warning(
"Detector id %s has None for photoCalib in the calexp "
885 "and will not be used in the warp.", detectorId)
889 if externalSkyWcsCatalog
is not None:
890 row = externalSkyWcsCatalog.find(detectorId)
892 self.log.warning(
"Detector id %s not found in externalSkyWcsCatalog "
893 "and will not be used in the warp.", detectorId)
895 skyWcs = row.getWcs()
897 self.log.warning(
"Detector id %s has None for skyWcs in externalSkyWcsCatalog "
898 "and will not be used in the warp.", detectorId)
900 calexp.setWcs(skyWcs)
902 skyWcs = calexp.getWcs()
904 self.log.warning(
"Detector id %s has None for skyWcs in the calexp "
905 "and will not be used in the warp.", detectorId)
909 calexp.maskedImage = photoCalib.calibrateImage(calexp.maskedImage,
910 includeScaleUncertainty=includeCalibVar)
911 calexp.maskedImage /= photoCalib.getCalibrationMean()
916 if self.config.doApplySkyCorr:
917 mi -= skyCorr.getImage()
919 indices.append(index)
924 def reorderRefs(inputRefs, outputSortKeyOrder, dataIdKey):
925 """Reorder inputRefs per outputSortKeyOrder
927 Any inputRefs which are lists will be resorted per specified key e.g.,
928 'detector.' Only iterables will be reordered, and values can be of type
929 `lsst.pipe.base.connections.DeferredDatasetRef` or
930 `lsst.daf.butler.core.datasets.ref.DatasetRef`.
931 Returned lists of refs have the same length as the outputSortKeyOrder.
932 If an outputSortKey not in the inputRef, then it will be padded with None.
933 If an inputRef contains an inputSortKey that is not in the
934 outputSortKeyOrder it will be removed.
938 inputRefs : `lsst.pipe.base.connections.QuantizedConnection`
939 Input references to be reordered and padded.
940 outputSortKeyOrder : iterable
941 Iterable of values to be compared with inputRef's dataId[dataIdKey]
943 dataIdKey in the dataRefs to compare with the outputSortKeyOrder.
947 inputRefs: `lsst.pipe.base.connections.QuantizedConnection`
948 Quantized Connection with sorted DatasetRef values sorted if iterable.
950 for connectionName, refs
in inputRefs:
951 if isinstance(refs, Iterable):
952 if hasattr(refs[0],
"dataId"):
953 inputSortKeyOrder = [ref.dataId[dataIdKey]
for ref
in refs]
955 inputSortKeyOrder = [ref.datasetRef.dataId[dataIdKey]
for ref
in refs]
956 if inputSortKeyOrder != outputSortKeyOrder:
957 setattr(inputRefs, connectionName,
Base class for coaddition.
def getTempExpDatasetName(self, warpType="direct")
def selectExposures(self, patchRef, skyInfo=None, selectDataList=[])
Select exposures to coadd.
def getCoaddDatasetName(self, warpType="direct")
def getSkyInfo(self, patchRef)
Use getSkyinfo to return the skyMap, tract and patch information, wcs and the outer bbox of the patch...
def getBadPixelMask(self)
Convenience method to provide the bitmask from the mask plane names.
Warp and optionally PSF-Match calexps onto an a common projection.
def getCalibratedExposure(self, dataRef, bgSubtracted)
def run(self, calExpList, ccdIdList, skyInfo, visitId=0, dataIdList=None, **kwargs)
def __init__(self, reuse=False, **kwargs)
def _prepareEmptyExposure(skyInfo)
def runDataRef(self, patchRef, selectDataList=[])
Produce <coaddName>Coadd_<warpType>Warp images by warping and optionally PSF-matching.
def getWarpTypeList(self)
def applySkyCorr(self, dataRef, calexp)
def reorderAndPadList(inputList, inputKeys, outputKeys, padWith=None)
def makeSkyInfo(skyMap, tractId, patchId)
def getGroupDataRef(butler, datasetType, groupTuple, keys)
def groupPatchExposures(patchDataRef, calexpDataRefList, coaddDatasetType="deepCoadd", tempExpDatasetType="deepCoadd_directWarp")