30import lsst.pipe.base.connectionTypes
as connectionTypes
31import lsst.utils
as utils
35from lsst.utils.timer
import timeMethod
36from .coaddBase
import CoaddBaseTask, makeSkyInfo, reorderAndPadList
37from .selectImages
import PsfWcsSelectImagesTask
38from .warpAndPsfMatch
import WarpAndPsfMatchTask
39from .coaddHelpers
import groupPatchExposures, getGroupDataRef
40from collections.abc
import Iterable
42__all__ = [
"MakeCoaddTempExpTask",
"MakeWarpTask",
"MakeWarpConfig"]
44log = logging.getLogger(__name__)
48 """Raised when data cannot be retrieved for an exposure.
49 When processing patches, sometimes one exposure is missing; this lets us
50 distinguish bewteen that case,
and other errors.
56 """Config for MakeCoaddTempExpTask
58 warpAndPsfMatch = pexConfig.ConfigurableField(
59 target=WarpAndPsfMatchTask,
60 doc="Task to warp and PSF-match calexp",
62 doWrite = pexConfig.Field(
63 doc=
"persist <coaddName>Coadd_<warpType>Warp",
67 bgSubtracted = pexConfig.Field(
68 doc=
"Work with a background subtracted calexp?",
72 coaddPsf = pexConfig.ConfigField(
73 doc=
"Configuration for CoaddPsf",
76 makeDirect = pexConfig.Field(
77 doc=
"Make direct Warp/Coadds",
81 makePsfMatched = pexConfig.Field(
82 doc=
"Make Psf-Matched Warp/Coadd?",
87 doWriteEmptyWarps = pexConfig.Field(
90 doc=
"Write out warps even if they are empty"
93 hasFakes = pexConfig.Field(
94 doc=
"Should be set to True if fake sources have been inserted into the input data.",
98 doApplySkyCorr = pexConfig.Field(dtype=bool, default=
False, doc=
"Apply sky correction?")
100 doApplyFinalizedPsf = pexConfig.Field(
101 doc=
"Whether to apply finalized psf models and aperture correction map.",
107 CoaddBaseTask.ConfigClass.validate(self)
109 raise RuntimeError(
"At least one of config.makePsfMatched and config.makeDirect must be True")
112 log.warning(
"Config doPsfMatch deprecated. Setting makePsfMatched=True and makeDirect=False")
117 CoaddBaseTask.ConfigClass.setDefaults(self)
118 self.
warpAndPsfMatchwarpAndPsfMatch.psfMatch.kernel.active.kernelSize = self.matchingKernelSize
119 self.select.retarget(PsfWcsSelectImagesTask)
130 r"""!Warp and optionally PSF-Match calexps onto an a common projection.
132 @anchor MakeCoaddTempExpTask_
134 @section pipe_tasks_makeCoaddTempExp_Contents Contents
136 -
@ref pipe_tasks_makeCoaddTempExp_Purpose
137 -
@ref pipe_tasks_makeCoaddTempExp_Initialize
138 -
@ref pipe_tasks_makeCoaddTempExp_IO
139 -
@ref pipe_tasks_makeCoaddTempExp_Config
140 -
@ref pipe_tasks_makeCoaddTempExp_Debug
141 -
@ref pipe_tasks_makeCoaddTempExp_Example
143 @section pipe_tasks_makeCoaddTempExp_Purpose Description
145 Warp
and optionally PSF-Match calexps onto a common projection, by
146 performing the following operations:
147 - Group calexps by visit/run
148 - For each visit, generate a Warp by calling method
@ref makeTempExp.
149 makeTempExp loops over the visit
's calexps calling @ref WarpAndPsfMatch
152 The result is a `directWarp` (
and/
or optionally a `psfMatchedWarp`).
154 @section pipe_tasks_makeCoaddTempExp_Initialize Task Initialization
156 @copydoc \_\_init\_\_
158 This task has one special keyword argument: passing reuse=
True will cause
159 the task to skip the creation of warps that are already present
in the
162 @section pipe_tasks_makeCoaddTempExp_IO Invoking the Task
164 This task
is primarily designed to be run
from the command line.
166 The main method
is `runDataRef`, which takes a single butler data reference
for the
patch(es)
171 WarpType identifies the types of convolutions applied to Warps (previously CoaddTempExps).
172 Only two types are available: direct (
for regular Warps/Coadds)
and psfMatched
173 (
for Warps/Coadds
with homogenized PSFs). We expect to add a third type, likelihood,
174 for generating likelihood Coadds
with Warps that have been correlated
with their own PSF.
176 @section pipe_tasks_makeCoaddTempExp_Config Configuration parameters
178 See
@ref MakeCoaddTempExpConfig
and parameters inherited
from
181 @subsection pipe_tasks_MakeCoaddTempExp_psfMatching Guide to PSF-Matching Configs
183 To make `psfMatchedWarps`, select `config.makePsfMatched=
True`. The subtask
185 is responsible
for the PSF-Matching,
and its config
is accessed via `config.warpAndPsfMatch.psfMatch`.
186 The optimal configuration depends on aspects of dataset: the pixel scale, average PSF FWHM
and
187 dimensions of the PSF kernel. These configs include the requested model PSF, the matching kernel size,
188 padding of the science PSF thumbnail
and spatial sampling frequency of the PSF.
190 *Config Guidelines*: The user must specify the size of the model PSF to which to match by setting
191 `config.modelPsf.defaultFwhm`
in units of pixels. The appropriate values depends on science case.
192 In general,
for a set of input images, this config should equal the FWHM of the visit
193 with the worst seeing. The smallest it should be set to
is the median FWHM. The defaults
194 of the other config options offer a reasonable starting point.
195 The following list presents the most common problems that arise
from a misconfigured
197 and corresponding solutions. All assume the default Alard-Lupton kernel,
with configs accessed via
198 ```config.warpAndPsfMatch.psfMatch.kernel[
'AL']```. Each item
in the list
is formatted
as:
199 Problem: Explanation. *Solution*
201 *Troublshooting PSF-Matching Configuration:*
202 - Matched PSFs look boxy: The matching kernel
is too small. _Increase the matching kernel size.
205 config.warpAndPsfMatch.psfMatch.kernel[
'AL'].kernelSize=27
207 Note that increasing the kernel size also increases runtime.
208 - Matched PSFs look ugly (dipoles, quadropoles, donuts): unable to find good solution
209 for matching kernel. _Provide the matcher
with more data by either increasing
210 the spatial sampling by decreasing the spatial cell size,_
212 config.warpAndPsfMatch.psfMatch.kernel[
'AL'].sizeCellX = 64
213 config.warpAndPsfMatch.psfMatch.kernel[
'AL'].sizeCellY = 64
215 _or increasing the padding around the Science PSF,
for example:_
217 config.warpAndPsfMatch.psfMatch.autoPadPsfTo=1.6
219 Increasing `autoPadPsfTo` increases the minimum ratio of input PSF dimensions to the
220 matching kernel dimensions, thus increasing the number of pixels available to fit
221 after convolving the PSF
with the matching kernel.
222 Optionally,
for debugging the effects of padding, the level of padding may be manually
223 controlled by setting turning off the automatic padding
and setting the number
224 of pixels by which to pad the PSF:
226 config.warpAndPsfMatch.psfMatch.doAutoPadPsf =
False
227 config.warpAndPsfMatch.psfMatch.padPsfBy = 6
229 - Deconvolution: Matching a large PSF to a smaller PSF produces
230 a telltale noise pattern which looks like ripples
or a brain.
231 _Increase the size of the requested model PSF. For example:_
233 config.modelPsf.defaultFwhm = 11
235 - High frequency (sometimes checkered) noise: The matching basis functions are too small.
236 _Increase the width of the Gaussian basis functions. For example:_
238 config.warpAndPsfMatch.psfMatch.kernel[
'AL'].alardSigGauss=[1.5, 3.0, 6.0]
242 @section pipe_tasks_makeCoaddTempExp_Debug Debug variables
244 MakeCoaddTempExpTask has no debug output, but its subtasks do.
246 @section pipe_tasks_makeCoaddTempExp_Example A complete example of using MakeCoaddTempExpTask
248 This example uses the package ci_hsc to show how MakeCoaddTempExp fits
249 into the larger Data Release Processing.
255 python $(which scons)
257 The following assumes that `processCcd.py`
and `makeSkyMap.py` have previously been run
258 (e.g. by building `ci_hsc` above) to generate a repository of calexps
and an
259 output respository
with the desired SkyMap. The command,
261 makeCoaddTempExp.py $CI_HSC_DIR/DATA --rerun ci_hsc \
262 --id patch=5,4 tract=0 filter=HSC-I \
263 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 \
264 --selectId visit=903988 ccd=23 --selectId visit=903988 ccd=24 \
265 --config doApplyExternalPhotoCalib=
False doApplyExternalSkyWcs=
False \
266 makePsfMatched=
True modelPsf.defaultFwhm=11
268 writes a direct
and PSF-Matched Warp to
269 - `$CI_HSC_DIR/DATA/rerun/ci_hsc/deepCoadd/HSC-I/0/5,4/warp-HSC-I-0-5,4-903988.fits`
and
270 - `$CI_HSC_DIR/DATA/rerun/ci_hsc/deepCoadd/HSC-I/0/5,4/psfMatchedWarp-HSC-I-0-5,4-903988.fits`
273 @note PSF-Matching
in this particular dataset would benefit
from adding
274 `--configfile ./matchingConfig.py` to
275 the command line arguments where `matchingConfig.py`
is defined by:
278 config.warpAndPsfMatch.psfMatch.kernel['AL'].kernelSize=27
279 config.warpAndPsfMatch.psfMatch.kernel[
'AL'].alardSigGauss=[1.5, 3.0, 6.0]
" > matchingConfig.py
282 Add the option `--help` to see more options.
284 ConfigClass = MakeCoaddTempExpConfig
285 _DefaultName = "makeCoaddTempExp"
288 CoaddBaseTask.__init__(self, **kwargs)
290 self.makeSubtask(
"warpAndPsfMatch")
291 if self.config.hasFakes:
298 """!Produce <coaddName>Coadd_<warpType>Warp images by warping and optionally PSF-matching.
300 @param[
in] patchRef: data reference
for sky map patch. Must include keys
"tract",
"patch",
301 plus the camera-specific filter key (e.g.
"filter" or "band")
302 @return: dataRefList: a list of data references
for the new <coaddName>Coadd_directWarps
303 if direct
or both warp types are requested
and <coaddName>Coadd_psfMatchedWarps
if only psfMatched
306 @warning: this task assumes that all exposures
in a warp (coaddTempExp) have the same filter.
308 @warning: this task sets the PhotoCalib of the coaddTempExp to the PhotoCalib of the first calexp
309 with any good pixels
in the patch. For a mosaic camera the resulting PhotoCalib should be ignored
310 (assembleCoadd should determine zeropoint scaling without referring to it).
315 if self.config.makePsfMatched
and not self.config.makeDirect:
320 calExpRefList = self.
selectExposuresselectExposures(patchRef, skyInfo, selectDataList=selectDataList)
322 if len(calExpRefList) == 0:
323 self.log.warning(
"No exposures to coadd for patch %s", patchRef.dataId)
325 self.log.info(
"Selected %d calexps for patch %s", len(calExpRefList), patchRef.dataId)
326 calExpRefList = [calExpRef
for calExpRef
in calExpRefList
if calExpRef.datasetExists(self.
calexpTypecalexpType)]
327 self.log.info(
"Processing %d existing calexps for patch %s", len(calExpRefList), patchRef.dataId)
331 self.log.info(
"Processing %d warp exposures for patch %s", len(groupData.groups), patchRef.dataId)
334 for i, (tempExpTuple, calexpRefList)
in enumerate(groupData.groups.items()):
336 tempExpTuple, groupData.keys)
337 if self.
reusereuse
and tempExpRef.datasetExists(datasetType=primaryWarpDataset, write=
True):
338 self.log.info(
"Skipping makeCoaddTempExp for %s; output already exists.", tempExpRef.dataId)
339 dataRefList.append(tempExpRef)
341 self.log.info(
"Processing Warp %d/%d: id=%s", i, len(groupData.groups), tempExpRef.dataId)
347 visitId = int(tempExpRef.dataId[
"visit"])
348 except (KeyError, ValueError):
355 for calExpInd, calExpRef
in enumerate(calexpRefList):
356 self.log.info(
"Reading calexp %s of %s for Warp id=%s", calExpInd+1, len(calexpRefList),
359 ccdId = calExpRef.get(
"ccdExposureId", immediate=
True)
366 calExpRef = calExpRef.butlerSubset.butler.dataRef(self.
calexpTypecalexpType,
367 dataId=calExpRef.dataId,
368 tract=skyInfo.tractInfo.getId())
369 calExp = self.
getCalibratedExposuregetCalibratedExposure(calExpRef, bgSubtracted=self.config.bgSubtracted)
370 except Exception
as e:
371 self.log.warning(
"Calexp %s not found; skipping it: %s", calExpRef.dataId, e)
374 if self.config.doApplySkyCorr:
377 calExpList.append(calExp)
378 ccdIdList.append(ccdId)
379 dataIdList.append(calExpRef.dataId)
381 exps = self.
runrun(calExpList, ccdIdList, skyInfo, visitId, dataIdList).exposures
383 if any(exps.values()):
384 dataRefList.append(tempExpRef)
386 self.log.warning(
"Warp %s could not be created", tempExpRef.dataId)
388 if self.config.doWrite:
389 for (warpType, exposure)
in exps.items():
390 if exposure
is not None:
397 def run(self, calExpList, ccdIdList, skyInfo, visitId=0, dataIdList=None, **kwargs):
398 """Create a Warp from inputs
400 We iterate over the multiple calexps in a single exposure to construct
401 the warp (previously called a coaddTempExp) of that exposure to the
402 supplied tract/patch.
404 Pixels that receive no pixels are set to NAN; this
is not correct
405 (violates LSST algorithms group policy), but will be fixed up by
406 interpolating after the coaddition.
408 @param calexpRefList: List of data references
for calexps that (may)
409 overlap the patch of interest
410 @param skyInfo: Struct
from CoaddBaseTask.getSkyInfo()
with geometric
411 information about the patch
412 @param visitId: integer identifier
for visit,
for the table that will
414 @return a pipeBase Struct containing:
415 - exposures: a dictionary containing the warps requested:
416 "direct": direct warp
if config.makeDirect
417 "psfMatched": PSF-matched warp
if config.makePsfMatched
421 totGoodPix = {warpType: 0 for warpType
in warpTypeList}
422 didSetMetadata = {warpType:
False for warpType
in warpTypeList}
423 coaddTempExps = {warpType: self.
_prepareEmptyExposure_prepareEmptyExposure(skyInfo)
for warpType
in warpTypeList}
424 inputRecorder = {warpType: self.inputRecorder.makeCoaddTempExpRecorder(visitId, len(calExpList))
425 for warpType
in warpTypeList}
427 modelPsf = self.config.modelPsf.apply()
if self.config.makePsfMatched
else None
428 if dataIdList
is None:
429 dataIdList = ccdIdList
431 for calExpInd, (calExp, ccdId, dataId)
in enumerate(zip(calExpList, ccdIdList, dataIdList)):
432 self.log.info(
"Processing calexp %d of %d for this Warp: id=%s",
433 calExpInd+1, len(calExpList), dataId)
436 warpedAndMatched = self.warpAndPsfMatch.
run(calExp, modelPsf=modelPsf,
437 wcs=skyInfo.wcs, maxBBox=skyInfo.bbox,
438 makeDirect=self.config.makeDirect,
439 makePsfMatched=self.config.makePsfMatched)
440 except Exception
as e:
441 self.log.warning(
"WarpAndPsfMatch failed for calexp %s; skipping it: %s", dataId, e)
444 numGoodPix = {warpType: 0
for warpType
in warpTypeList}
445 for warpType
in warpTypeList:
446 exposure = warpedAndMatched.getDict()[warpType]
449 coaddTempExp = coaddTempExps[warpType]
450 if didSetMetadata[warpType]:
451 mimg = exposure.getMaskedImage()
452 mimg *= (coaddTempExp.getPhotoCalib().getInstFluxAtZeroMagnitude()
453 / exposure.getPhotoCalib().getInstFluxAtZeroMagnitude())
455 numGoodPix[warpType] = coaddUtils.copyGoodPixels(
456 coaddTempExp.getMaskedImage(), exposure.getMaskedImage(), self.
getBadPixelMaskgetBadPixelMask())
457 totGoodPix[warpType] += numGoodPix[warpType]
458 self.log.debug(
"Calexp %s has %d good pixels in this patch (%.1f%%) for %s",
459 dataId, numGoodPix[warpType],
460 100.0*numGoodPix[warpType]/skyInfo.bbox.getArea(), warpType)
461 if numGoodPix[warpType] > 0
and not didSetMetadata[warpType]:
462 coaddTempExp.info.id = exposure.info.id
463 coaddTempExp.setPhotoCalib(exposure.getPhotoCalib())
464 coaddTempExp.setFilterLabel(exposure.getFilterLabel())
465 coaddTempExp.getInfo().setVisitInfo(exposure.getInfo().getVisitInfo())
467 coaddTempExp.setPsf(exposure.getPsf())
468 didSetMetadata[warpType] =
True
471 inputRecorder[warpType].addCalExp(calExp, ccdId, numGoodPix[warpType])
473 except Exception
as e:
474 self.log.warning(
"Error processing calexp %s; skipping it: %s", dataId, e)
477 for warpType
in warpTypeList:
478 self.log.info(
"%sWarp has %d good pixels (%.1f%%)",
479 warpType, totGoodPix[warpType], 100.0*totGoodPix[warpType]/skyInfo.bbox.getArea())
481 if totGoodPix[warpType] > 0
and didSetMetadata[warpType]:
482 inputRecorder[warpType].finish(coaddTempExps[warpType], totGoodPix[warpType])
483 if warpType ==
"direct":
484 coaddTempExps[warpType].setPsf(
485 CoaddPsf(inputRecorder[warpType].coaddInputs.ccds, skyInfo.wcs,
486 self.config.coaddPsf.makeControl()))
488 if not self.config.doWriteEmptyWarps:
490 coaddTempExps[warpType] =
None
495 result = pipeBase.Struct(exposures=coaddTempExps)
499 """Return one calibrated Exposure, possibly with an updated SkyWcs.
501 @param[
in] dataRef a sensor-level data reference
502 @param[
in] bgSubtracted
return calexp
with background subtracted? If
False get the
503 calexp
's background background model and add it to the calexp.
504 @return calibrated exposure
506 @raises MissingExposureError If data
for the exposure
is not available.
508 If config.doApplyExternalPhotoCalib
is `
True`, the photometric calibration
509 (`photoCalib`)
is taken
from `config.externalPhotoCalibName` via the
510 `name_photoCalib` dataset. Otherwise, the photometric calibration
is
511 retrieved
from the processed exposure. When
512 `config.doApplyExternalSkyWcs`
is `
True`, the astrometric calibration
513 is taken
from `config.externalSkyWcsName`
with the `name_wcs` dataset.
514 Otherwise, the astrometric calibration
is taken
from the processed
518 exposure = dataRef.get(self.
calexpTypecalexpType, immediate=
True)
519 except dafPersist.NoResults
as e:
523 background = dataRef.get(
"calexpBackground", immediate=
True)
524 mi = exposure.getMaskedImage()
525 mi += background.getImage()
528 if self.config.doApplyExternalPhotoCalib:
529 source = f
"{self.config.externalPhotoCalibName}_photoCalib"
530 self.log.debug(
"Applying external photoCalib to %s from %s", dataRef.dataId, source)
531 photoCalib = dataRef.get(source)
532 exposure.setPhotoCalib(photoCalib)
534 photoCalib = exposure.getPhotoCalib()
536 if self.config.doApplyExternalSkyWcs:
537 source = f
"{self.config.externalSkyWcsName}_wcs"
538 self.log.debug(
"Applying external skyWcs to %s from %s", dataRef.dataId, source)
539 skyWcs = dataRef.get(source)
540 exposure.setWcs(skyWcs)
542 exposure.maskedImage = photoCalib.calibrateImage(exposure.maskedImage,
543 includeScaleUncertainty=self.config.includeCalibVar)
544 exposure.maskedImage /= photoCalib.getCalibrationMean()
550 def _prepareEmptyExposure(skyInfo):
551 """Produce an empty exposure for a given patch"""
552 exp = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
553 exp.getMaskedImage().set(numpy.nan, afwImage.Mask
554 .getPlaneBitMask(
"NO_DATA"), numpy.inf)
558 """Return list of requested warp types per the config.
561 if self.config.makeDirect:
562 warpTypeList.append(
"direct")
563 if self.config.makePsfMatched:
564 warpTypeList.append(
"psfMatched")
568 """Apply correction to the sky background level
570 Sky corrections can be generated with the
'skyCorrection.py'
571 executable
in pipe_drivers. Because the sky model used by that
572 code extends over the entire focal plane, this can produce
573 better sky subtraction.
575 The calexp
is updated
in-place.
579 dataRef : `lsst.daf.persistence.ButlerDataRef`
580 Data reference
for calexp.
584 bg = dataRef.get("skyCorr")
585 self.log.debug(
"Applying sky correction to %s", dataRef.dataId)
586 if isinstance(calexp, afwImage.Exposure):
587 calexp = calexp.getMaskedImage()
588 calexp -= bg.getImage()
592 dimensions=(
"tract",
"patch",
"skymap",
"instrument",
"visit"),
593 defaultTemplates={
"coaddName":
"deep",
594 "skyWcsName":
"jointcal",
595 "photoCalibName":
"fgcm",
597 calExpList = connectionTypes.Input(
598 doc=
"Input exposures to be resampled and optionally PSF-matched onto a SkyMap projection/patch",
599 name=
"{calexpType}calexp",
600 storageClass=
"ExposureF",
601 dimensions=(
"instrument",
"visit",
"detector"),
605 backgroundList = connectionTypes.Input(
606 doc=
"Input backgrounds to be added back into the calexp if bgSubtracted=False",
607 name=
"calexpBackground",
608 storageClass=
"Background",
609 dimensions=(
"instrument",
"visit",
"detector"),
612 skyCorrList = connectionTypes.Input(
613 doc=
"Input Sky Correction to be subtracted from the calexp if doApplySkyCorr=True",
615 storageClass=
"Background",
616 dimensions=(
"instrument",
"visit",
"detector"),
619 skyMap = connectionTypes.Input(
620 doc=
"Input definition of geometry/bbox and projection/wcs for warped exposures",
621 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
622 storageClass=
"SkyMap",
623 dimensions=(
"skymap",),
625 externalSkyWcsTractCatalog = connectionTypes.Input(
626 doc=(
"Per-tract, per-visit wcs calibrations. These catalogs use the detector "
627 "id for the catalog id, sorted on id for fast lookup."),
628 name=
"{skyWcsName}SkyWcsCatalog",
629 storageClass=
"ExposureCatalog",
630 dimensions=(
"instrument",
"visit",
"tract"),
632 externalSkyWcsGlobalCatalog = connectionTypes.Input(
633 doc=(
"Per-visit wcs calibrations computed globally (with no tract information). "
634 "These catalogs use the detector id for the catalog id, sorted on id for "
636 name=
"{skyWcsName}SkyWcsCatalog",
637 storageClass=
"ExposureCatalog",
638 dimensions=(
"instrument",
"visit"),
640 externalPhotoCalibTractCatalog = connectionTypes.Input(
641 doc=(
"Per-tract, per-visit photometric calibrations. These catalogs use the "
642 "detector id for the catalog id, sorted on id for fast lookup."),
643 name=
"{photoCalibName}PhotoCalibCatalog",
644 storageClass=
"ExposureCatalog",
645 dimensions=(
"instrument",
"visit",
"tract"),
647 externalPhotoCalibGlobalCatalog = connectionTypes.Input(
648 doc=(
"Per-visit photometric calibrations computed globally (with no tract "
649 "information). These catalogs use the detector id for the catalog id, "
650 "sorted on id for fast lookup."),
651 name=
"{photoCalibName}PhotoCalibCatalog",
652 storageClass=
"ExposureCatalog",
653 dimensions=(
"instrument",
"visit"),
655 finalizedPsfApCorrCatalog = connectionTypes.Input(
656 doc=(
"Per-visit finalized psf models and aperture correction maps. "
657 "These catalogs use the detector id for the catalog id, "
658 "sorted on id for fast lookup."),
659 name=
"finalized_psf_ap_corr_catalog",
660 storageClass=
"ExposureCatalog",
661 dimensions=(
"instrument",
"visit"),
663 direct = connectionTypes.Output(
664 doc=(
"Output direct warped exposure (previously called CoaddTempExp), produced by resampling ",
665 "calexps onto the skyMap patch geometry."),
666 name=
"{coaddName}Coadd_directWarp",
667 storageClass=
"ExposureF",
668 dimensions=(
"tract",
"patch",
"skymap",
"visit",
"instrument"),
670 psfMatched = connectionTypes.Output(
671 doc=(
"Output PSF-Matched warped exposure (previously called CoaddTempExp), produced by resampling ",
672 "calexps onto the skyMap patch geometry and PSF-matching to a model PSF."),
673 name=
"{coaddName}Coadd_psfMatchedWarp",
674 storageClass=
"ExposureF",
675 dimensions=(
"tract",
"patch",
"skymap",
"visit",
"instrument"),
678 wcsList = connectionTypes.Input(
679 doc=
"WCSs of calexps used by SelectImages subtask to determine if the calexp overlaps the patch",
680 name=
"{calexpType}calexp.wcs",
682 dimensions=(
"instrument",
"visit",
"detector"),
685 bboxList = connectionTypes.Input(
686 doc=
"BBoxes of calexps used by SelectImages subtask to determine if the calexp overlaps the patch",
687 name=
"{calexpType}calexp.bbox",
688 storageClass=
"Box2I",
689 dimensions=(
"instrument",
"visit",
"detector"),
692 visitSummary = connectionTypes.Input(
693 doc=
"Consolidated exposure metadata from ConsolidateVisitSummaryTask",
694 name=
"{calexpType}visitSummary",
695 storageClass=
"ExposureCatalog",
696 dimensions=(
"instrument",
"visit",),
698 srcList = connectionTypes.Input(
699 doc=
"Source catalogs used by PsfWcsSelectImages subtask to further select on PSF stability",
701 storageClass=
"SourceCatalog",
702 dimensions=(
"instrument",
"visit",
"detector"),
706 def __init__(self, *, config=None):
707 super().__init__(config=config)
708 if config.bgSubtracted:
709 self.inputs.remove(
"backgroundList")
710 if not config.doApplySkyCorr:
711 self.inputs.remove(
"skyCorrList")
712 if config.doApplyExternalSkyWcs:
713 if config.useGlobalExternalSkyWcs:
714 self.inputs.remove(
"externalSkyWcsTractCatalog")
716 self.inputs.remove(
"externalSkyWcsGlobalCatalog")
718 self.inputs.remove(
"externalSkyWcsTractCatalog")
719 self.inputs.remove(
"externalSkyWcsGlobalCatalog")
720 if config.doApplyExternalPhotoCalib:
721 if config.useGlobalExternalPhotoCalib:
722 self.inputs.remove(
"externalPhotoCalibTractCatalog")
724 self.inputs.remove(
"externalPhotoCalibGlobalCatalog")
726 self.inputs.remove(
"externalPhotoCalibTractCatalog")
727 self.inputs.remove(
"externalPhotoCalibGlobalCatalog")
728 if not config.doApplyFinalizedPsf:
729 self.inputs.remove(
"finalizedPsfApCorrCatalog")
730 if not config.makeDirect:
731 self.outputs.remove(
"direct")
732 if not config.makePsfMatched:
733 self.outputs.remove(
"psfMatched")
735 if config.select.target != lsst.pipe.tasks.selectImages.PsfWcsSelectImagesTask:
736 self.inputs.remove(
"visitSummary")
737 self.inputs.remove(
"srcList")
738 elif not config.select.doLegacyStarSelectionComputation:
740 self.inputs.remove(
"srcList")
744 pipelineConnections=MakeWarpConnections):
751 """Warp and optionally PSF-Match calexps onto an a common projection
753 ConfigClass = MakeWarpConfig
754 _DefaultName = "makeWarp"
756 @utils.inheritDoc(pipeBase.PipelineTask)
757 def runQuantum(self, butlerQC, inputRefs, outputRefs):
761 Construct warps for requested warp type
for single epoch
763 PipelineTask (Gen3) entry point to warp
and optionally PSF-match
764 calexps. This method
is analogous to `runDataRef`.
769 detectorOrder = [ref.datasetRef.dataId[
'detector']
for ref
in inputRefs.calExpList]
771 inputRefs = reorderRefs(inputRefs, detectorOrder, dataIdKey=
'detector')
774 inputs = butlerQC.get(inputRefs)
778 skyMap = inputs.pop(
"skyMap")
779 quantumDataId = butlerQC.quantum.dataId
780 skyInfo =
makeSkyInfo(skyMap, tractId=quantumDataId[
'tract'], patchId=quantumDataId[
'patch'])
783 dataIdList = [ref.datasetRef.dataId
for ref
in inputRefs.calExpList]
785 ccdIdList = [dataId.pack(
"visit_detector")
for dataId
in dataIdList]
790 coordList = [skyInfo.wcs.pixelToSky(pos)
for pos
in cornerPosList]
791 goodIndices = self.select.run(**inputs, coordList=coordList, dataIds=dataIdList)
792 inputs = self.filterInputs(indices=goodIndices, inputs=inputs)
795 inputs[
'calExpList'] = [ref.get()
for ref
in inputs[
'calExpList']]
798 visits = [dataId[
'visit']
for dataId
in dataIdList]
801 if self.config.doApplyExternalSkyWcs:
802 if self.config.useGlobalExternalSkyWcs:
803 externalSkyWcsCatalog = inputs.pop(
"externalSkyWcsGlobalCatalog")
805 externalSkyWcsCatalog = inputs.pop(
"externalSkyWcsTractCatalog")
807 externalSkyWcsCatalog =
None
809 if self.config.doApplyExternalPhotoCalib:
810 if self.config.useGlobalExternalPhotoCalib:
811 externalPhotoCalibCatalog = inputs.pop(
"externalPhotoCalibGlobalCatalog")
813 externalPhotoCalibCatalog = inputs.pop(
"externalPhotoCalibTractCatalog")
815 externalPhotoCalibCatalog =
None
817 if self.config.doApplyFinalizedPsf:
818 finalizedPsfApCorrCatalog = inputs.pop(
"finalizedPsfApCorrCatalog")
820 finalizedPsfApCorrCatalog =
None
822 completeIndices = self.prepareCalibratedExposures(**inputs,
823 externalSkyWcsCatalog=externalSkyWcsCatalog,
824 externalPhotoCalibCatalog=externalPhotoCalibCatalog,
825 finalizedPsfApCorrCatalog=finalizedPsfApCorrCatalog)
827 inputs = self.filterInputs(indices=completeIndices, inputs=inputs)
829 results = self.run(**inputs, visitId=visitId,
830 ccdIdList=[ccdIdList[i]
for i
in goodIndices],
831 dataIdList=[dataIdList[i]
for i
in goodIndices],
833 if self.config.makeDirect
and results.exposures[
"direct"]
is not None:
834 butlerQC.put(results.exposures[
"direct"], outputRefs.direct)
835 if self.config.makePsfMatched
and results.exposures[
"psfMatched"]
is not None:
836 butlerQC.put(results.exposures[
"psfMatched"], outputRefs.psfMatched)
838 def filterInputs(self, indices, inputs):
839 """Return task inputs with their lists filtered by indices
843 indices : `list` of integers
844 inputs : `dict` of `list` of input connections to be passed to run
846 for key
in inputs.keys():
848 if isinstance(inputs[key], list):
849 inputs[key] = [inputs[key][ind]
for ind
in indices]
852 def prepareCalibratedExposures(self, calExpList, backgroundList=None, skyCorrList=None,
853 externalSkyWcsCatalog=None, externalPhotoCalibCatalog=None,
854 finalizedPsfApCorrCatalog=None,
856 """Calibrate and add backgrounds to input calExpList in place
861 Sequence of calexps to be modified in place
862 backgroundList : `list` of `lsst.afw.math.backgroundList`, optional
863 Sequence of backgrounds to be added back
in if bgSubtracted=
False
864 skyCorrList : `list` of `lsst.afw.math.backgroundList`, optional
865 Sequence of background corrections to be subtracted
if doApplySkyCorr=
True
867 Exposure catalog
with external skyWcs to be applied
868 if config.doApplyExternalSkyWcs=
True. Catalog uses the detector id
869 for the catalog id, sorted on id
for fast lookup.
871 Exposure catalog
with external photoCalib to be applied
872 if config.doApplyExternalPhotoCalib=
True. Catalog uses the detector
873 id
for the catalog id, sorted on id
for fast lookup.
875 Exposure catalog
with finalized psf models
and aperture correction
876 maps to be applied
if config.doApplyFinalizedPsf=
True. Catalog uses
877 the detector id
for the catalog id, sorted on id
for fast lookup.
881 indices : `list` [`int`]
882 Indices of calExpList
and friends that have valid photoCalib/skyWcs
884 backgroundList = len(calExpList)*[None]
if backgroundList
is None else backgroundList
885 skyCorrList = len(calExpList)*[
None]
if skyCorrList
is None else skyCorrList
887 includeCalibVar = self.config.includeCalibVar
890 for index, (calexp, background, skyCorr)
in enumerate(zip(calExpList,
893 if not self.config.bgSubtracted:
894 calexp.maskedImage += background.getImage()
896 detectorId = calexp.getInfo().getDetector().getId()
899 if externalPhotoCalibCatalog
is not None:
900 row = externalPhotoCalibCatalog.find(detectorId)
902 self.log.warning(
"Detector id %s not found in externalPhotoCalibCatalog "
903 "and will not be used in the warp.", detectorId)
905 photoCalib = row.getPhotoCalib()
906 if photoCalib
is None:
907 self.log.warning(
"Detector id %s has None for photoCalib in externalPhotoCalibCatalog "
908 "and will not be used in the warp.", detectorId)
910 calexp.setPhotoCalib(photoCalib)
912 photoCalib = calexp.getPhotoCalib()
913 if photoCalib
is None:
914 self.log.warning(
"Detector id %s has None for photoCalib in the calexp "
915 "and will not be used in the warp.", detectorId)
919 if externalSkyWcsCatalog
is not None:
920 row = externalSkyWcsCatalog.find(detectorId)
922 self.log.warning(
"Detector id %s not found in externalSkyWcsCatalog "
923 "and will not be used in the warp.", detectorId)
925 skyWcs = row.getWcs()
927 self.log.warning(
"Detector id %s has None for skyWcs in externalSkyWcsCatalog "
928 "and will not be used in the warp.", detectorId)
930 calexp.setWcs(skyWcs)
932 skyWcs = calexp.getWcs()
934 self.log.warning(
"Detector id %s has None for skyWcs in the calexp "
935 "and will not be used in the warp.", detectorId)
939 if finalizedPsfApCorrCatalog
is not None:
940 row = finalizedPsfApCorrCatalog.find(detectorId)
942 self.log.warning(
"Detector id %s not found in finalizedPsfApCorrCatalog "
943 "and will not be used in the warp.", detectorId)
947 self.log.warning(
"Detector id %s has None for psf in finalizedPsfApCorrCatalog "
948 "and will not be used in the warp.", detectorId)
951 apCorrMap = row.getApCorrMap()
952 if apCorrMap
is None:
953 self.log.warning(
"Detector id %s has None for ApCorrMap in finalizedPsfApCorrCatalog "
954 "and will not be used in the warp.", detectorId)
956 calexp.setApCorrMap(apCorrMap)
959 calexp.maskedImage = photoCalib.calibrateImage(calexp.maskedImage,
960 includeScaleUncertainty=includeCalibVar)
961 calexp.maskedImage /= photoCalib.getCalibrationMean()
966 if self.config.doApplySkyCorr:
967 calexp.maskedImage -= skyCorr.getImage()
969 indices.append(index)
974def reorderRefs(inputRefs, outputSortKeyOrder, dataIdKey):
975 """Reorder inputRefs per outputSortKeyOrder
977 Any inputRefs which are lists will be resorted per specified key e.g.,
978 'detector.' Only iterables will be reordered,
and values can be of type
979 `lsst.pipe.base.connections.DeferredDatasetRef`
or
980 `lsst.daf.butler.core.datasets.ref.DatasetRef`.
981 Returned lists of refs have the same length
as the outputSortKeyOrder.
982 If an outputSortKey
not in the inputRef, then it will be padded
with None.
983 If an inputRef contains an inputSortKey that
is not in the
984 outputSortKeyOrder it will be removed.
988 inputRefs : `lsst.pipe.base.connections.QuantizedConnection`
989 Input references to be reordered
and padded.
990 outputSortKeyOrder : iterable
991 Iterable of values to be compared
with inputRef
's dataId[dataIdKey]
993 dataIdKey in the dataRefs to compare
with the outputSortKeyOrder.
997 inputRefs: `lsst.pipe.base.connections.QuantizedConnection`
998 Quantized Connection
with sorted DatasetRef values sorted
if iterable.
1000 for connectionName, refs
in inputRefs:
1001 if isinstance(refs, Iterable):
1002 if hasattr(refs[0],
"dataId"):
1003 inputSortKeyOrder = [ref.dataId[dataIdKey]
for ref
in refs]
1005 inputSortKeyOrder = [ref.datasetRef.dataId[dataIdKey]
for ref
in refs]
1006 if inputSortKeyOrder != outputSortKeyOrder:
1007 setattr(inputRefs, connectionName,
Configuration parameters for CoaddBaseTask.
Base class for coaddition.
def getTempExpDatasetName(self, warpType="direct")
def selectExposures(self, patchRef, skyInfo=None, selectDataList=[])
Select exposures to coadd.
def getCoaddDatasetName(self, warpType="direct")
def getSkyInfo(self, patchRef)
Use getSkyinfo to return the skyMap, tract and patch information, wcs and the outer bbox of the patch...
def getBadPixelMask(self)
Convenience method to provide the bitmask from the mask plane names.
Warp and optionally PSF-Match calexps onto an a common projection.
def getCalibratedExposure(self, dataRef, bgSubtracted)
def run(self, calExpList, ccdIdList, skyInfo, visitId=0, dataIdList=None, **kwargs)
def __init__(self, reuse=False, **kwargs)
def _prepareEmptyExposure(skyInfo)
def runDataRef(self, patchRef, selectDataList=[])
Produce <coaddName>Coadd_<warpType>Warp images by warping and optionally PSF-matching.
def getWarpTypeList(self)
def applySkyCorr(self, dataRef, calexp)
def reorderAndPadList(inputList, inputKeys, outputKeys, padWith=None)
def makeSkyInfo(skyMap, tractId, patchId)
def getGroupDataRef(butler, datasetType, groupTuple, keys)
def groupPatchExposures(patchDataRef, calexpDataRefList, coaddDatasetType="deepCoadd", tempExpDatasetType="deepCoadd_directWarp")