30import lsst.pipe.base.connectionTypes
as connectionTypes
31import lsst.utils
as utils
35from lsst.utils.timer
import timeMethod
36from .coaddBase
import CoaddBaseTask, makeSkyInfo, reorderAndPadList
37from .selectImages
import PsfWcsSelectImagesTask
38from .warpAndPsfMatch
import WarpAndPsfMatchTask
39from .coaddHelpers
import groupPatchExposures, getGroupDataRef
40from collections.abc
import Iterable
42__all__ = [
"MakeCoaddTempExpTask",
"MakeWarpTask",
"MakeWarpConfig"]
44log = logging.getLogger(__name__)
48 """Raised when data cannot be retrieved for an exposure.
49 When processing patches, sometimes one exposure is missing; this lets us
50 distinguish bewteen that case,
and other errors.
56 """Config for MakeCoaddTempExpTask
58 warpAndPsfMatch = pexConfig.ConfigurableField(
59 target=WarpAndPsfMatchTask,
60 doc="Task to warp and PSF-match calexp",
62 doWrite = pexConfig.Field(
63 doc=
"persist <coaddName>Coadd_<warpType>Warp",
67 bgSubtracted = pexConfig.Field(
68 doc=
"Work with a background subtracted calexp?",
72 coaddPsf = pexConfig.ConfigField(
73 doc=
"Configuration for CoaddPsf",
76 makeDirect = pexConfig.Field(
77 doc=
"Make direct Warp/Coadds",
81 makePsfMatched = pexConfig.Field(
82 doc=
"Make Psf-Matched Warp/Coadd?",
87 doWriteEmptyWarps = pexConfig.Field(
90 doc=
"Write out warps even if they are empty"
93 hasFakes = pexConfig.Field(
94 doc=
"Should be set to True if fake sources have been inserted into the input data.",
98 doApplySkyCorr = pexConfig.Field(dtype=bool, default=
False, doc=
"Apply sky correction?")
101 CoaddBaseTask.ConfigClass.validate(self)
103 raise RuntimeError(
"At least one of config.makePsfMatched and config.makeDirect must be True")
106 log.warning(
"Config doPsfMatch deprecated. Setting makePsfMatched=True and makeDirect=False")
111 CoaddBaseTask.ConfigClass.setDefaults(self)
112 self.
warpAndPsfMatchwarpAndPsfMatch.psfMatch.kernel.active.kernelSize = self.matchingKernelSize
113 self.select.retarget(PsfWcsSelectImagesTask)
124 r"""!Warp and optionally PSF-Match calexps onto an a common projection.
126 @anchor MakeCoaddTempExpTask_
128 @section pipe_tasks_makeCoaddTempExp_Contents Contents
130 -
@ref pipe_tasks_makeCoaddTempExp_Purpose
131 -
@ref pipe_tasks_makeCoaddTempExp_Initialize
132 -
@ref pipe_tasks_makeCoaddTempExp_IO
133 -
@ref pipe_tasks_makeCoaddTempExp_Config
134 -
@ref pipe_tasks_makeCoaddTempExp_Debug
135 -
@ref pipe_tasks_makeCoaddTempExp_Example
137 @section pipe_tasks_makeCoaddTempExp_Purpose Description
139 Warp
and optionally PSF-Match calexps onto a common projection, by
140 performing the following operations:
141 - Group calexps by visit/run
142 - For each visit, generate a Warp by calling method
@ref makeTempExp.
143 makeTempExp loops over the visit
's calexps calling @ref WarpAndPsfMatch
146 The result is a `directWarp` (
and/
or optionally a `psfMatchedWarp`).
148 @section pipe_tasks_makeCoaddTempExp_Initialize Task Initialization
150 @copydoc \_\_init\_\_
152 This task has one special keyword argument: passing reuse=
True will cause
153 the task to skip the creation of warps that are already present
in the
156 @section pipe_tasks_makeCoaddTempExp_IO Invoking the Task
158 This task
is primarily designed to be run
from the command line.
160 The main method
is `runDataRef`, which takes a single butler data reference
for the
patch(es)
165 WarpType identifies the types of convolutions applied to Warps (previously CoaddTempExps).
166 Only two types are available: direct (
for regular Warps/Coadds)
and psfMatched
167 (
for Warps/Coadds
with homogenized PSFs). We expect to add a third type, likelihood,
168 for generating likelihood Coadds
with Warps that have been correlated
with their own PSF.
170 @section pipe_tasks_makeCoaddTempExp_Config Configuration parameters
172 See
@ref MakeCoaddTempExpConfig
and parameters inherited
from
175 @subsection pipe_tasks_MakeCoaddTempExp_psfMatching Guide to PSF-Matching Configs
177 To make `psfMatchedWarps`, select `config.makePsfMatched=
True`. The subtask
179 is responsible
for the PSF-Matching,
and its config
is accessed via `config.warpAndPsfMatch.psfMatch`.
180 The optimal configuration depends on aspects of dataset: the pixel scale, average PSF FWHM
and
181 dimensions of the PSF kernel. These configs include the requested model PSF, the matching kernel size,
182 padding of the science PSF thumbnail
and spatial sampling frequency of the PSF.
184 *Config Guidelines*: The user must specify the size of the model PSF to which to match by setting
185 `config.modelPsf.defaultFwhm`
in units of pixels. The appropriate values depends on science case.
186 In general,
for a set of input images, this config should equal the FWHM of the visit
187 with the worst seeing. The smallest it should be set to
is the median FWHM. The defaults
188 of the other config options offer a reasonable starting point.
189 The following list presents the most common problems that arise
from a misconfigured
191 and corresponding solutions. All assume the default Alard-Lupton kernel,
with configs accessed via
192 ```config.warpAndPsfMatch.psfMatch.kernel[
'AL']```. Each item
in the list
is formatted
as:
193 Problem: Explanation. *Solution*
195 *Troublshooting PSF-Matching Configuration:*
196 - Matched PSFs look boxy: The matching kernel
is too small. _Increase the matching kernel size.
199 config.warpAndPsfMatch.psfMatch.kernel[
'AL'].kernelSize=27
201 Note that increasing the kernel size also increases runtime.
202 - Matched PSFs look ugly (dipoles, quadropoles, donuts): unable to find good solution
203 for matching kernel. _Provide the matcher
with more data by either increasing
204 the spatial sampling by decreasing the spatial cell size,_
206 config.warpAndPsfMatch.psfMatch.kernel[
'AL'].sizeCellX = 64
207 config.warpAndPsfMatch.psfMatch.kernel[
'AL'].sizeCellY = 64
209 _or increasing the padding around the Science PSF,
for example:_
211 config.warpAndPsfMatch.psfMatch.autoPadPsfTo=1.6
213 Increasing `autoPadPsfTo` increases the minimum ratio of input PSF dimensions to the
214 matching kernel dimensions, thus increasing the number of pixels available to fit
215 after convolving the PSF
with the matching kernel.
216 Optionally,
for debugging the effects of padding, the level of padding may be manually
217 controlled by setting turning off the automatic padding
and setting the number
218 of pixels by which to pad the PSF:
220 config.warpAndPsfMatch.psfMatch.doAutoPadPsf =
False
221 config.warpAndPsfMatch.psfMatch.padPsfBy = 6
223 - Deconvolution: Matching a large PSF to a smaller PSF produces
224 a telltale noise pattern which looks like ripples
or a brain.
225 _Increase the size of the requested model PSF. For example:_
227 config.modelPsf.defaultFwhm = 11
229 - High frequency (sometimes checkered) noise: The matching basis functions are too small.
230 _Increase the width of the Gaussian basis functions. For example:_
232 config.warpAndPsfMatch.psfMatch.kernel[
'AL'].alardSigGauss=[1.5, 3.0, 6.0]
236 @section pipe_tasks_makeCoaddTempExp_Debug Debug variables
238 MakeCoaddTempExpTask has no debug output, but its subtasks do.
240 @section pipe_tasks_makeCoaddTempExp_Example A complete example of using MakeCoaddTempExpTask
242 This example uses the package ci_hsc to show how MakeCoaddTempExp fits
243 into the larger Data Release Processing.
249 python $(which scons)
251 The following assumes that `processCcd.py`
and `makeSkyMap.py` have previously been run
252 (e.g. by building `ci_hsc` above) to generate a repository of calexps
and an
253 output respository
with the desired SkyMap. The command,
255 makeCoaddTempExp.py $CI_HSC_DIR/DATA --rerun ci_hsc \
256 --id patch=5,4 tract=0 filter=HSC-I \
257 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 \
258 --selectId visit=903988 ccd=23 --selectId visit=903988 ccd=24 \
259 --config doApplyExternalPhotoCalib=
False doApplyExternalSkyWcs=
False \
260 makePsfMatched=
True modelPsf.defaultFwhm=11
262 writes a direct
and PSF-Matched Warp to
263 - `$CI_HSC_DIR/DATA/rerun/ci_hsc/deepCoadd/HSC-I/0/5,4/warp-HSC-I-0-5,4-903988.fits`
and
264 - `$CI_HSC_DIR/DATA/rerun/ci_hsc/deepCoadd/HSC-I/0/5,4/psfMatchedWarp-HSC-I-0-5,4-903988.fits`
267 @note PSF-Matching
in this particular dataset would benefit
from adding
268 `--configfile ./matchingConfig.py` to
269 the command line arguments where `matchingConfig.py`
is defined by:
272 config.warpAndPsfMatch.psfMatch.kernel['AL'].kernelSize=27
273 config.warpAndPsfMatch.psfMatch.kernel[
'AL'].alardSigGauss=[1.5, 3.0, 6.0]
" > matchingConfig.py
276 Add the option `--help` to see more options.
278 ConfigClass = MakeCoaddTempExpConfig
279 _DefaultName = "makeCoaddTempExp"
282 CoaddBaseTask.__init__(self, **kwargs)
284 self.makeSubtask(
"warpAndPsfMatch")
285 if self.config.hasFakes:
292 """!Produce <coaddName>Coadd_<warpType>Warp images by warping and optionally PSF-matching.
294 @param[
in] patchRef: data reference
for sky map patch. Must include keys
"tract",
"patch",
295 plus the camera-specific filter key (e.g.
"filter" or "band")
296 @return: dataRefList: a list of data references
for the new <coaddName>Coadd_directWarps
297 if direct
or both warp types are requested
and <coaddName>Coadd_psfMatchedWarps
if only psfMatched
300 @warning: this task assumes that all exposures
in a warp (coaddTempExp) have the same filter.
302 @warning: this task sets the PhotoCalib of the coaddTempExp to the PhotoCalib of the first calexp
303 with any good pixels
in the patch. For a mosaic camera the resulting PhotoCalib should be ignored
304 (assembleCoadd should determine zeropoint scaling without referring to it).
309 if self.config.makePsfMatched
and not self.config.makeDirect:
314 calExpRefList = self.
selectExposuresselectExposures(patchRef, skyInfo, selectDataList=selectDataList)
316 if len(calExpRefList) == 0:
317 self.log.warning(
"No exposures to coadd for patch %s", patchRef.dataId)
319 self.log.info(
"Selected %d calexps for patch %s", len(calExpRefList), patchRef.dataId)
320 calExpRefList = [calExpRef
for calExpRef
in calExpRefList
if calExpRef.datasetExists(self.
calexpTypecalexpType)]
321 self.log.info(
"Processing %d existing calexps for patch %s", len(calExpRefList), patchRef.dataId)
325 self.log.info(
"Processing %d warp exposures for patch %s", len(groupData.groups), patchRef.dataId)
328 for i, (tempExpTuple, calexpRefList)
in enumerate(groupData.groups.items()):
330 tempExpTuple, groupData.keys)
331 if self.
reusereuse
and tempExpRef.datasetExists(datasetType=primaryWarpDataset, write=
True):
332 self.log.info(
"Skipping makeCoaddTempExp for %s; output already exists.", tempExpRef.dataId)
333 dataRefList.append(tempExpRef)
335 self.log.info(
"Processing Warp %d/%d: id=%s", i, len(groupData.groups), tempExpRef.dataId)
341 visitId = int(tempExpRef.dataId[
"visit"])
342 except (KeyError, ValueError):
349 for calExpInd, calExpRef
in enumerate(calexpRefList):
350 self.log.info(
"Reading calexp %s of %s for Warp id=%s", calExpInd+1, len(calexpRefList),
353 ccdId = calExpRef.get(
"ccdExposureId", immediate=
True)
360 calExpRef = calExpRef.butlerSubset.butler.dataRef(self.
calexpTypecalexpType,
361 dataId=calExpRef.dataId,
362 tract=skyInfo.tractInfo.getId())
363 calExp = self.
getCalibratedExposuregetCalibratedExposure(calExpRef, bgSubtracted=self.config.bgSubtracted)
364 except Exception
as e:
365 self.log.warning(
"Calexp %s not found; skipping it: %s", calExpRef.dataId, e)
368 if self.config.doApplySkyCorr:
371 calExpList.append(calExp)
372 ccdIdList.append(ccdId)
373 dataIdList.append(calExpRef.dataId)
375 exps = self.
runrun(calExpList, ccdIdList, skyInfo, visitId, dataIdList).exposures
377 if any(exps.values()):
378 dataRefList.append(tempExpRef)
380 self.log.warning(
"Warp %s could not be created", tempExpRef.dataId)
382 if self.config.doWrite:
383 for (warpType, exposure)
in exps.items():
384 if exposure
is not None:
391 def run(self, calExpList, ccdIdList, skyInfo, visitId=0, dataIdList=None, **kwargs):
392 """Create a Warp from inputs
394 We iterate over the multiple calexps in a single exposure to construct
395 the warp (previously called a coaddTempExp) of that exposure to the
396 supplied tract/patch.
398 Pixels that receive no pixels are set to NAN; this
is not correct
399 (violates LSST algorithms group policy), but will be fixed up by
400 interpolating after the coaddition.
402 @param calexpRefList: List of data references
for calexps that (may)
403 overlap the patch of interest
404 @param skyInfo: Struct
from CoaddBaseTask.getSkyInfo()
with geometric
405 information about the patch
406 @param visitId: integer identifier
for visit,
for the table that will
408 @return a pipeBase Struct containing:
409 - exposures: a dictionary containing the warps requested:
410 "direct": direct warp
if config.makeDirect
411 "psfMatched": PSF-matched warp
if config.makePsfMatched
415 totGoodPix = {warpType: 0 for warpType
in warpTypeList}
416 didSetMetadata = {warpType:
False for warpType
in warpTypeList}
417 coaddTempExps = {warpType: self.
_prepareEmptyExposure_prepareEmptyExposure(skyInfo)
for warpType
in warpTypeList}
418 inputRecorder = {warpType: self.inputRecorder.makeCoaddTempExpRecorder(visitId, len(calExpList))
419 for warpType
in warpTypeList}
421 modelPsf = self.config.modelPsf.apply()
if self.config.makePsfMatched
else None
422 if dataIdList
is None:
423 dataIdList = ccdIdList
425 for calExpInd, (calExp, ccdId, dataId)
in enumerate(zip(calExpList, ccdIdList, dataIdList)):
426 self.log.info(
"Processing calexp %d of %d for this Warp: id=%s",
427 calExpInd+1, len(calExpList), dataId)
430 warpedAndMatched = self.warpAndPsfMatch.
run(calExp, modelPsf=modelPsf,
431 wcs=skyInfo.wcs, maxBBox=skyInfo.bbox,
432 makeDirect=self.config.makeDirect,
433 makePsfMatched=self.config.makePsfMatched)
434 except Exception
as e:
435 self.log.warning(
"WarpAndPsfMatch failed for calexp %s; skipping it: %s", dataId, e)
438 numGoodPix = {warpType: 0
for warpType
in warpTypeList}
439 for warpType
in warpTypeList:
440 exposure = warpedAndMatched.getDict()[warpType]
443 coaddTempExp = coaddTempExps[warpType]
444 if didSetMetadata[warpType]:
445 mimg = exposure.getMaskedImage()
446 mimg *= (coaddTempExp.getPhotoCalib().getInstFluxAtZeroMagnitude()
447 / exposure.getPhotoCalib().getInstFluxAtZeroMagnitude())
449 numGoodPix[warpType] = coaddUtils.copyGoodPixels(
450 coaddTempExp.getMaskedImage(), exposure.getMaskedImage(), self.
getBadPixelMaskgetBadPixelMask())
451 totGoodPix[warpType] += numGoodPix[warpType]
452 self.log.debug(
"Calexp %s has %d good pixels in this patch (%.1f%%) for %s",
453 dataId, numGoodPix[warpType],
454 100.0*numGoodPix[warpType]/skyInfo.bbox.getArea(), warpType)
455 if numGoodPix[warpType] > 0
and not didSetMetadata[warpType]:
456 coaddTempExp.info.id = exposure.info.id
457 coaddTempExp.setPhotoCalib(exposure.getPhotoCalib())
458 coaddTempExp.setFilterLabel(exposure.getFilterLabel())
459 coaddTempExp.getInfo().setVisitInfo(exposure.getInfo().getVisitInfo())
461 coaddTempExp.setPsf(exposure.getPsf())
462 didSetMetadata[warpType] =
True
465 inputRecorder[warpType].addCalExp(calExp, ccdId, numGoodPix[warpType])
467 except Exception
as e:
468 self.log.warning(
"Error processing calexp %s; skipping it: %s", dataId, e)
471 for warpType
in warpTypeList:
472 self.log.info(
"%sWarp has %d good pixels (%.1f%%)",
473 warpType, totGoodPix[warpType], 100.0*totGoodPix[warpType]/skyInfo.bbox.getArea())
475 if totGoodPix[warpType] > 0
and didSetMetadata[warpType]:
476 inputRecorder[warpType].finish(coaddTempExps[warpType], totGoodPix[warpType])
477 if warpType ==
"direct":
478 coaddTempExps[warpType].setPsf(
479 CoaddPsf(inputRecorder[warpType].coaddInputs.ccds, skyInfo.wcs,
480 self.config.coaddPsf.makeControl()))
482 if not self.config.doWriteEmptyWarps:
484 coaddTempExps[warpType] =
None
489 result = pipeBase.Struct(exposures=coaddTempExps)
493 """Return one calibrated Exposure, possibly with an updated SkyWcs.
495 @param[
in] dataRef a sensor-level data reference
496 @param[
in] bgSubtracted
return calexp
with background subtracted? If
False get the
497 calexp
's background background model and add it to the calexp.
498 @return calibrated exposure
500 @raises MissingExposureError If data
for the exposure
is not available.
502 If config.doApplyExternalPhotoCalib
is `
True`, the photometric calibration
503 (`photoCalib`)
is taken
from `config.externalPhotoCalibName` via the
504 `name_photoCalib` dataset. Otherwise, the photometric calibration
is
505 retrieved
from the processed exposure. When
506 `config.doApplyExternalSkyWcs`
is `
True`, the astrometric calibration
507 is taken
from `config.externalSkyWcsName`
with the `name_wcs` dataset.
508 Otherwise, the astrometric calibration
is taken
from the processed
512 exposure = dataRef.get(self.
calexpTypecalexpType, immediate=
True)
513 except dafPersist.NoResults
as e:
517 background = dataRef.get(
"calexpBackground", immediate=
True)
518 mi = exposure.getMaskedImage()
519 mi += background.getImage()
522 if self.config.doApplyExternalPhotoCalib:
523 source = f
"{self.config.externalPhotoCalibName}_photoCalib"
524 self.log.debug(
"Applying external photoCalib to %s from %s", dataRef.dataId, source)
525 photoCalib = dataRef.get(source)
526 exposure.setPhotoCalib(photoCalib)
528 photoCalib = exposure.getPhotoCalib()
530 if self.config.doApplyExternalSkyWcs:
531 source = f
"{self.config.externalSkyWcsName}_wcs"
532 self.log.debug(
"Applying external skyWcs to %s from %s", dataRef.dataId, source)
533 skyWcs = dataRef.get(source)
534 exposure.setWcs(skyWcs)
536 exposure.maskedImage = photoCalib.calibrateImage(exposure.maskedImage,
537 includeScaleUncertainty=self.config.includeCalibVar)
538 exposure.maskedImage /= photoCalib.getCalibrationMean()
544 def _prepareEmptyExposure(skyInfo):
545 """Produce an empty exposure for a given patch"""
546 exp = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
547 exp.getMaskedImage().set(numpy.nan, afwImage.Mask
548 .getPlaneBitMask(
"NO_DATA"), numpy.inf)
552 """Return list of requested warp types per the config.
555 if self.config.makeDirect:
556 warpTypeList.append(
"direct")
557 if self.config.makePsfMatched:
558 warpTypeList.append(
"psfMatched")
562 """Apply correction to the sky background level
564 Sky corrections can be generated with the
'skyCorrection.py'
565 executable
in pipe_drivers. Because the sky model used by that
566 code extends over the entire focal plane, this can produce
567 better sky subtraction.
569 The calexp
is updated
in-place.
573 dataRef : `lsst.daf.persistence.ButlerDataRef`
574 Data reference
for calexp.
578 bg = dataRef.get("skyCorr")
579 self.log.debug(
"Applying sky correction to %s", dataRef.dataId)
580 if isinstance(calexp, afwImage.Exposure):
581 calexp = calexp.getMaskedImage()
582 calexp -= bg.getImage()
586 dimensions=(
"tract",
"patch",
"skymap",
"instrument",
"visit"),
587 defaultTemplates={
"coaddName":
"deep",
588 "skyWcsName":
"jointcal",
589 "photoCalibName":
"fgcm",
591 calExpList = connectionTypes.Input(
592 doc=
"Input exposures to be resampled and optionally PSF-matched onto a SkyMap projection/patch",
593 name=
"{calexpType}calexp",
594 storageClass=
"ExposureF",
595 dimensions=(
"instrument",
"visit",
"detector"),
599 backgroundList = connectionTypes.Input(
600 doc=
"Input backgrounds to be added back into the calexp if bgSubtracted=False",
601 name=
"calexpBackground",
602 storageClass=
"Background",
603 dimensions=(
"instrument",
"visit",
"detector"),
606 skyCorrList = connectionTypes.Input(
607 doc=
"Input Sky Correction to be subtracted from the calexp if doApplySkyCorr=True",
609 storageClass=
"Background",
610 dimensions=(
"instrument",
"visit",
"detector"),
613 skyMap = connectionTypes.Input(
614 doc=
"Input definition of geometry/bbox and projection/wcs for warped exposures",
615 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
616 storageClass=
"SkyMap",
617 dimensions=(
"skymap",),
619 externalSkyWcsTractCatalog = connectionTypes.Input(
620 doc=(
"Per-tract, per-visit wcs calibrations. These catalogs use the detector "
621 "id for the catalog id, sorted on id for fast lookup."),
622 name=
"{skyWcsName}SkyWcsCatalog",
623 storageClass=
"ExposureCatalog",
624 dimensions=(
"instrument",
"visit",
"tract"),
626 externalSkyWcsGlobalCatalog = connectionTypes.Input(
627 doc=(
"Per-visit wcs calibrations computed globally (with no tract information). "
628 "These catalogs use the detector id for the catalog id, sorted on id for "
630 name=
"{skyWcsName}SkyWcsCatalog",
631 storageClass=
"ExposureCatalog",
632 dimensions=(
"instrument",
"visit"),
634 externalPhotoCalibTractCatalog = connectionTypes.Input(
635 doc=(
"Per-tract, per-visit photometric calibrations. These catalogs use the "
636 "detector id for the catalog id, sorted on id for fast lookup."),
637 name=
"{photoCalibName}PhotoCalibCatalog",
638 storageClass=
"ExposureCatalog",
639 dimensions=(
"instrument",
"visit",
"tract"),
641 externalPhotoCalibGlobalCatalog = connectionTypes.Input(
642 doc=(
"Per-visit photometric calibrations computed globally (with no tract "
643 "information). These catalogs use the detector id for the catalog id, "
644 "sorted on id for fast lookup."),
645 name=
"{photoCalibName}PhotoCalibCatalog",
646 storageClass=
"ExposureCatalog",
647 dimensions=(
"instrument",
"visit"),
649 direct = connectionTypes.Output(
650 doc=(
"Output direct warped exposure (previously called CoaddTempExp), produced by resampling ",
651 "calexps onto the skyMap patch geometry."),
652 name=
"{coaddName}Coadd_directWarp",
653 storageClass=
"ExposureF",
654 dimensions=(
"tract",
"patch",
"skymap",
"visit",
"instrument"),
656 psfMatched = connectionTypes.Output(
657 doc=(
"Output PSF-Matched warped exposure (previously called CoaddTempExp), produced by resampling ",
658 "calexps onto the skyMap patch geometry and PSF-matching to a model PSF."),
659 name=
"{coaddName}Coadd_psfMatchedWarp",
660 storageClass=
"ExposureF",
661 dimensions=(
"tract",
"patch",
"skymap",
"visit",
"instrument"),
664 wcsList = connectionTypes.Input(
665 doc=
"WCSs of calexps used by SelectImages subtask to determine if the calexp overlaps the patch",
666 name=
"{calexpType}calexp.wcs",
668 dimensions=(
"instrument",
"visit",
"detector"),
671 bboxList = connectionTypes.Input(
672 doc=
"BBoxes of calexps used by SelectImages subtask to determine if the calexp overlaps the patch",
673 name=
"{calexpType}calexp.bbox",
674 storageClass=
"Box2I",
675 dimensions=(
"instrument",
"visit",
"detector"),
678 visitSummary = connectionTypes.Input(
679 doc=
"Consolidated exposure metadata from ConsolidateVisitSummaryTask",
680 name=
"{calexpType}visitSummary",
681 storageClass=
"ExposureCatalog",
682 dimensions=(
"instrument",
"visit",),
684 srcList = connectionTypes.Input(
685 doc=
"Source catalogs used by PsfWcsSelectImages subtask to further select on PSF stability",
687 storageClass=
"SourceCatalog",
688 dimensions=(
"instrument",
"visit",
"detector"),
692 def __init__(self, *, config=None):
693 super().__init__(config=config)
694 if config.bgSubtracted:
695 self.inputs.remove(
"backgroundList")
696 if not config.doApplySkyCorr:
697 self.inputs.remove(
"skyCorrList")
698 if config.doApplyExternalSkyWcs:
699 if config.useGlobalExternalSkyWcs:
700 self.inputs.remove(
"externalSkyWcsTractCatalog")
702 self.inputs.remove(
"externalSkyWcsGlobalCatalog")
704 self.inputs.remove(
"externalSkyWcsTractCatalog")
705 self.inputs.remove(
"externalSkyWcsGlobalCatalog")
706 if config.doApplyExternalPhotoCalib:
707 if config.useGlobalExternalPhotoCalib:
708 self.inputs.remove(
"externalPhotoCalibTractCatalog")
710 self.inputs.remove(
"externalPhotoCalibGlobalCatalog")
712 self.inputs.remove(
"externalPhotoCalibTractCatalog")
713 self.inputs.remove(
"externalPhotoCalibGlobalCatalog")
714 if not config.makeDirect:
715 self.outputs.remove(
"direct")
716 if not config.makePsfMatched:
717 self.outputs.remove(
"psfMatched")
719 if config.select.target != lsst.pipe.tasks.selectImages.PsfWcsSelectImagesTask:
720 self.inputs.remove(
"visitSummary")
721 self.inputs.remove(
"srcList")
722 elif not config.select.doLegacyStarSelectionComputation:
724 self.inputs.remove(
"srcList")
728 pipelineConnections=MakeWarpConnections):
735 """Warp and optionally PSF-Match calexps onto an a common projection
737 ConfigClass = MakeWarpConfig
738 _DefaultName = "makeWarp"
740 @utils.inheritDoc(pipeBase.PipelineTask)
741 def runQuantum(self, butlerQC, inputRefs, outputRefs):
745 Construct warps for requested warp type
for single epoch
747 PipelineTask (Gen3) entry point to warp
and optionally PSF-match
748 calexps. This method
is analogous to `runDataRef`.
753 detectorOrder = [ref.datasetRef.dataId[
'detector']
for ref
in inputRefs.calExpList]
755 inputRefs = reorderRefs(inputRefs, detectorOrder, dataIdKey=
'detector')
758 inputs = butlerQC.get(inputRefs)
762 skyMap = inputs.pop(
"skyMap")
763 quantumDataId = butlerQC.quantum.dataId
764 skyInfo =
makeSkyInfo(skyMap, tractId=quantumDataId[
'tract'], patchId=quantumDataId[
'patch'])
767 dataIdList = [ref.datasetRef.dataId
for ref
in inputRefs.calExpList]
769 ccdIdList = [dataId.pack(
"visit_detector")
for dataId
in dataIdList]
774 coordList = [skyInfo.wcs.pixelToSky(pos)
for pos
in cornerPosList]
775 goodIndices = self.select.run(**inputs, coordList=coordList, dataIds=dataIdList)
776 inputs = self.filterInputs(indices=goodIndices, inputs=inputs)
779 inputs[
'calExpList'] = [ref.get()
for ref
in inputs[
'calExpList']]
782 visits = [dataId[
'visit']
for dataId
in dataIdList]
785 if self.config.doApplyExternalSkyWcs:
786 if self.config.useGlobalExternalSkyWcs:
787 externalSkyWcsCatalog = inputs.pop(
"externalSkyWcsGlobalCatalog")
789 externalSkyWcsCatalog = inputs.pop(
"externalSkyWcsTractCatalog")
791 externalSkyWcsCatalog =
None
793 if self.config.doApplyExternalPhotoCalib:
794 if self.config.useGlobalExternalPhotoCalib:
795 externalPhotoCalibCatalog = inputs.pop(
"externalPhotoCalibGlobalCatalog")
797 externalPhotoCalibCatalog = inputs.pop(
"externalPhotoCalibTractCatalog")
799 externalPhotoCalibCatalog =
None
801 completeIndices = self.prepareCalibratedExposures(**inputs,
802 externalSkyWcsCatalog=externalSkyWcsCatalog,
803 externalPhotoCalibCatalog=externalPhotoCalibCatalog)
805 inputs = self.filterInputs(indices=completeIndices, inputs=inputs)
807 results = self.run(**inputs, visitId=visitId,
808 ccdIdList=[ccdIdList[i]
for i
in goodIndices],
809 dataIdList=[dataIdList[i]
for i
in goodIndices],
811 if self.config.makeDirect
and results.exposures[
"direct"]
is not None:
812 butlerQC.put(results.exposures[
"direct"], outputRefs.direct)
813 if self.config.makePsfMatched
and results.exposures[
"psfMatched"]
is not None:
814 butlerQC.put(results.exposures[
"psfMatched"], outputRefs.psfMatched)
816 def filterInputs(self, indices, inputs):
817 """Return task inputs with their lists filtered by indices
821 indices : `list` of integers
822 inputs : `dict` of `list` of input connections to be passed to run
824 for key
in inputs.keys():
826 if isinstance(inputs[key], list):
827 inputs[key] = [inputs[key][ind]
for ind
in indices]
830 def prepareCalibratedExposures(self, calExpList, backgroundList=None, skyCorrList=None,
831 externalSkyWcsCatalog=None, externalPhotoCalibCatalog=None,
833 """Calibrate and add backgrounds to input calExpList in place
838 Sequence of calexps to be modified in place
839 backgroundList : `list` of `lsst.afw.math.backgroundList`, optional
840 Sequence of backgrounds to be added back
in if bgSubtracted=
False
841 skyCorrList : `list` of `lsst.afw.math.backgroundList`, optional
842 Sequence of background corrections to be subtracted
if doApplySkyCorr=
True
844 Exposure catalog
with external skyWcs to be applied
845 if config.doApplyExternalSkyWcs=
True. Catalog uses the detector id
846 for the catalog id, sorted on id
for fast lookup.
848 Exposure catalog
with external photoCalib to be applied
849 if config.doApplyExternalPhotoCalib=
True. Catalog uses the detector
850 id
for the catalog id, sorted on id
for fast lookup.
854 indices : `list` [`int`]
855 Indices of calExpList
and friends that have valid photoCalib/skyWcs
857 backgroundList = len(calExpList)*[None]
if backgroundList
is None else backgroundList
858 skyCorrList = len(calExpList)*[
None]
if skyCorrList
is None else skyCorrList
860 includeCalibVar = self.config.includeCalibVar
863 for index, (calexp, background, skyCorr)
in enumerate(zip(calExpList,
866 if not self.config.bgSubtracted:
867 calexp.maskedImage += background.getImage()
869 if externalSkyWcsCatalog
is not None or externalPhotoCalibCatalog
is not None:
870 detectorId = calexp.getInfo().getDetector().getId()
873 if externalPhotoCalibCatalog
is not None:
874 row = externalPhotoCalibCatalog.find(detectorId)
876 self.log.warning(
"Detector id %s not found in externalPhotoCalibCatalog "
877 "and will not be used in the warp.", detectorId)
879 photoCalib = row.getPhotoCalib()
880 if photoCalib
is None:
881 self.log.warning(
"Detector id %s has None for photoCalib in externalPhotoCalibCatalog "
882 "and will not be used in the warp.", detectorId)
884 calexp.setPhotoCalib(photoCalib)
886 photoCalib = calexp.getPhotoCalib()
887 if photoCalib
is None:
888 self.log.warning(
"Detector id %s has None for photoCalib in the calexp "
889 "and will not be used in the warp.", detectorId)
893 if externalSkyWcsCatalog
is not None:
894 row = externalSkyWcsCatalog.find(detectorId)
896 self.log.warning(
"Detector id %s not found in externalSkyWcsCatalog "
897 "and will not be used in the warp.", detectorId)
899 skyWcs = row.getWcs()
901 self.log.warning(
"Detector id %s has None for skyWcs in externalSkyWcsCatalog "
902 "and will not be used in the warp.", detectorId)
904 calexp.setWcs(skyWcs)
906 skyWcs = calexp.getWcs()
908 self.log.warning(
"Detector id %s has None for skyWcs in the calexp "
909 "and will not be used in the warp.", detectorId)
913 calexp.maskedImage = photoCalib.calibrateImage(calexp.maskedImage,
914 includeScaleUncertainty=includeCalibVar)
915 calexp.maskedImage /= photoCalib.getCalibrationMean()
920 if self.config.doApplySkyCorr:
921 calexp.maskedImage -= skyCorr.getImage()
923 indices.append(index)
928def reorderRefs(inputRefs, outputSortKeyOrder, dataIdKey):
929 """Reorder inputRefs per outputSortKeyOrder
931 Any inputRefs which are lists will be resorted per specified key e.g.,
932 'detector.' Only iterables will be reordered,
and values can be of type
933 `lsst.pipe.base.connections.DeferredDatasetRef`
or
934 `lsst.daf.butler.core.datasets.ref.DatasetRef`.
935 Returned lists of refs have the same length
as the outputSortKeyOrder.
936 If an outputSortKey
not in the inputRef, then it will be padded
with None.
937 If an inputRef contains an inputSortKey that
is not in the
938 outputSortKeyOrder it will be removed.
942 inputRefs : `lsst.pipe.base.connections.QuantizedConnection`
943 Input references to be reordered
and padded.
944 outputSortKeyOrder : iterable
945 Iterable of values to be compared
with inputRef
's dataId[dataIdKey]
947 dataIdKey in the dataRefs to compare
with the outputSortKeyOrder.
951 inputRefs: `lsst.pipe.base.connections.QuantizedConnection`
952 Quantized Connection
with sorted DatasetRef values sorted
if iterable.
954 for connectionName, refs
in inputRefs:
955 if isinstance(refs, Iterable):
956 if hasattr(refs[0],
"dataId"):
957 inputSortKeyOrder = [ref.dataId[dataIdKey]
for ref
in refs]
959 inputSortKeyOrder = [ref.datasetRef.dataId[dataIdKey]
for ref
in refs]
960 if inputSortKeyOrder != outputSortKeyOrder:
961 setattr(inputRefs, connectionName,
Configuration parameters for CoaddBaseTask.
Base class for coaddition.
def getTempExpDatasetName(self, warpType="direct")
def selectExposures(self, patchRef, skyInfo=None, selectDataList=[])
Select exposures to coadd.
def getCoaddDatasetName(self, warpType="direct")
def getSkyInfo(self, patchRef)
Use getSkyinfo to return the skyMap, tract and patch information, wcs and the outer bbox of the patch...
def getBadPixelMask(self)
Convenience method to provide the bitmask from the mask plane names.
Warp and optionally PSF-Match calexps onto an a common projection.
def getCalibratedExposure(self, dataRef, bgSubtracted)
def run(self, calExpList, ccdIdList, skyInfo, visitId=0, dataIdList=None, **kwargs)
def __init__(self, reuse=False, **kwargs)
def _prepareEmptyExposure(skyInfo)
def runDataRef(self, patchRef, selectDataList=[])
Produce <coaddName>Coadd_<warpType>Warp images by warping and optionally PSF-matching.
def getWarpTypeList(self)
def applySkyCorr(self, dataRef, calexp)
def reorderAndPadList(inputList, inputKeys, outputKeys, padWith=None)
def makeSkyInfo(skyMap, tractId, patchId)
def getGroupDataRef(butler, datasetType, groupTuple, keys)
def groupPatchExposures(patchDataRef, calexpDataRefList, coaddDatasetType="deepCoadd", tempExpDatasetType="deepCoadd_directWarp")