30import lsst.pipe.base.connectionTypes
as connectionTypes
31import lsst.utils
as utils
35from lsst.utils.timer
import timeMethod
36from .coaddBase
import CoaddBaseTask, makeSkyInfo, reorderAndPadList
37from .warpAndPsfMatch
import WarpAndPsfMatchTask
38from .coaddHelpers
import groupPatchExposures, getGroupDataRef
39from collections.abc
import Iterable
41__all__ = [
"MakeCoaddTempExpTask",
"MakeWarpTask",
"MakeWarpConfig"]
43log = logging.getLogger(__name__)
47 """Raised when data cannot be retrieved for an exposure.
48 When processing patches, sometimes one exposure is missing; this lets us
49 distinguish bewteen that case,
and other errors.
55 """Config for MakeCoaddTempExpTask
57 warpAndPsfMatch = pexConfig.ConfigurableField(
58 target=WarpAndPsfMatchTask,
59 doc="Task to warp and PSF-match calexp",
61 doWrite = pexConfig.Field(
62 doc=
"persist <coaddName>Coadd_<warpType>Warp",
66 bgSubtracted = pexConfig.Field(
67 doc=
"Work with a background subtracted calexp?",
71 coaddPsf = pexConfig.ConfigField(
72 doc=
"Configuration for CoaddPsf",
75 makeDirect = pexConfig.Field(
76 doc=
"Make direct Warp/Coadds",
80 makePsfMatched = pexConfig.Field(
81 doc=
"Make Psf-Matched Warp/Coadd?",
86 doWriteEmptyWarps = pexConfig.Field(
89 doc=
"Write out warps even if they are empty"
92 hasFakes = pexConfig.Field(
93 doc=
"Should be set to True if fake sources have been inserted into the input data.",
97 doApplySkyCorr = pexConfig.Field(dtype=bool, default=
False, doc=
"Apply sky correction?")
100 CoaddBaseTask.ConfigClass.validate(self)
102 raise RuntimeError(
"At least one of config.makePsfMatched and config.makeDirect must be True")
105 log.warning(
"Config doPsfMatch deprecated. Setting makePsfMatched=True and makeDirect=False")
110 CoaddBaseTask.ConfigClass.setDefaults(self)
111 self.
warpAndPsfMatchwarpAndPsfMatch.psfMatch.kernel.active.kernelSize = self.matchingKernelSize
122 r"""!Warp and optionally PSF-Match calexps onto an a common projection.
124 @anchor MakeCoaddTempExpTask_
126 @section pipe_tasks_makeCoaddTempExp_Contents Contents
128 -
@ref pipe_tasks_makeCoaddTempExp_Purpose
129 -
@ref pipe_tasks_makeCoaddTempExp_Initialize
130 -
@ref pipe_tasks_makeCoaddTempExp_IO
131 -
@ref pipe_tasks_makeCoaddTempExp_Config
132 -
@ref pipe_tasks_makeCoaddTempExp_Debug
133 -
@ref pipe_tasks_makeCoaddTempExp_Example
135 @section pipe_tasks_makeCoaddTempExp_Purpose Description
137 Warp
and optionally PSF-Match calexps onto a common projection, by
138 performing the following operations:
139 - Group calexps by visit/run
140 - For each visit, generate a Warp by calling method
@ref makeTempExp.
141 makeTempExp loops over the visit
's calexps calling @ref WarpAndPsfMatch
144 The result is a `directWarp` (
and/
or optionally a `psfMatchedWarp`).
146 @section pipe_tasks_makeCoaddTempExp_Initialize Task Initialization
148 @copydoc \_\_init\_\_
150 This task has one special keyword argument: passing reuse=
True will cause
151 the task to skip the creation of warps that are already present
in the
154 @section pipe_tasks_makeCoaddTempExp_IO Invoking the Task
156 This task
is primarily designed to be run
from the command line.
158 The main method
is `runDataRef`, which takes a single butler data reference
for the
patch(es)
163 WarpType identifies the types of convolutions applied to Warps (previously CoaddTempExps).
164 Only two types are available: direct (
for regular Warps/Coadds)
and psfMatched
165 (
for Warps/Coadds
with homogenized PSFs). We expect to add a third type, likelihood,
166 for generating likelihood Coadds
with Warps that have been correlated
with their own PSF.
168 @section pipe_tasks_makeCoaddTempExp_Config Configuration parameters
170 See
@ref MakeCoaddTempExpConfig
and parameters inherited
from
173 @subsection pipe_tasks_MakeCoaddTempExp_psfMatching Guide to PSF-Matching Configs
175 To make `psfMatchedWarps`, select `config.makePsfMatched=
True`. The subtask
177 is responsible
for the PSF-Matching,
and its config
is accessed via `config.warpAndPsfMatch.psfMatch`.
178 The optimal configuration depends on aspects of dataset: the pixel scale, average PSF FWHM
and
179 dimensions of the PSF kernel. These configs include the requested model PSF, the matching kernel size,
180 padding of the science PSF thumbnail
and spatial sampling frequency of the PSF.
182 *Config Guidelines*: The user must specify the size of the model PSF to which to match by setting
183 `config.modelPsf.defaultFwhm`
in units of pixels. The appropriate values depends on science case.
184 In general,
for a set of input images, this config should equal the FWHM of the visit
185 with the worst seeing. The smallest it should be set to
is the median FWHM. The defaults
186 of the other config options offer a reasonable starting point.
187 The following list presents the most common problems that arise
from a misconfigured
189 and corresponding solutions. All assume the default Alard-Lupton kernel,
with configs accessed via
190 ```config.warpAndPsfMatch.psfMatch.kernel[
'AL']```. Each item
in the list
is formatted
as:
191 Problem: Explanation. *Solution*
193 *Troublshooting PSF-Matching Configuration:*
194 - Matched PSFs look boxy: The matching kernel
is too small. _Increase the matching kernel size.
197 config.warpAndPsfMatch.psfMatch.kernel[
'AL'].kernelSize=27
199 Note that increasing the kernel size also increases runtime.
200 - Matched PSFs look ugly (dipoles, quadropoles, donuts): unable to find good solution
201 for matching kernel. _Provide the matcher
with more data by either increasing
202 the spatial sampling by decreasing the spatial cell size,_
204 config.warpAndPsfMatch.psfMatch.kernel[
'AL'].sizeCellX = 64
205 config.warpAndPsfMatch.psfMatch.kernel[
'AL'].sizeCellY = 64
207 _or increasing the padding around the Science PSF,
for example:_
209 config.warpAndPsfMatch.psfMatch.autoPadPsfTo=1.6
211 Increasing `autoPadPsfTo` increases the minimum ratio of input PSF dimensions to the
212 matching kernel dimensions, thus increasing the number of pixels available to fit
213 after convolving the PSF
with the matching kernel.
214 Optionally,
for debugging the effects of padding, the level of padding may be manually
215 controlled by setting turning off the automatic padding
and setting the number
216 of pixels by which to pad the PSF:
218 config.warpAndPsfMatch.psfMatch.doAutoPadPsf =
False
219 config.warpAndPsfMatch.psfMatch.padPsfBy = 6
221 - Deconvolution: Matching a large PSF to a smaller PSF produces
222 a telltale noise pattern which looks like ripples
or a brain.
223 _Increase the size of the requested model PSF. For example:_
225 config.modelPsf.defaultFwhm = 11
227 - High frequency (sometimes checkered) noise: The matching basis functions are too small.
228 _Increase the width of the Gaussian basis functions. For example:_
230 config.warpAndPsfMatch.psfMatch.kernel[
'AL'].alardSigGauss=[1.5, 3.0, 6.0]
234 @section pipe_tasks_makeCoaddTempExp_Debug Debug variables
236 MakeCoaddTempExpTask has no debug output, but its subtasks do.
238 @section pipe_tasks_makeCoaddTempExp_Example A complete example of using MakeCoaddTempExpTask
240 This example uses the package ci_hsc to show how MakeCoaddTempExp fits
241 into the larger Data Release Processing.
247 python $(which scons)
249 The following assumes that `processCcd.py`
and `makeSkyMap.py` have previously been run
250 (e.g. by building `ci_hsc` above) to generate a repository of calexps
and an
251 output respository
with the desired SkyMap. The command,
253 makeCoaddTempExp.py $CI_HSC_DIR/DATA --rerun ci_hsc \
254 --id patch=5,4 tract=0 filter=HSC-I \
255 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 \
256 --selectId visit=903988 ccd=23 --selectId visit=903988 ccd=24 \
257 --config doApplyExternalPhotoCalib=
False doApplyExternalSkyWcs=
False \
258 makePsfMatched=
True modelPsf.defaultFwhm=11
260 writes a direct
and PSF-Matched Warp to
261 - `$CI_HSC_DIR/DATA/rerun/ci_hsc/deepCoadd/HSC-I/0/5,4/warp-HSC-I-0-5,4-903988.fits`
and
262 - `$CI_HSC_DIR/DATA/rerun/ci_hsc/deepCoadd/HSC-I/0/5,4/psfMatchedWarp-HSC-I-0-5,4-903988.fits`
265 @note PSF-Matching
in this particular dataset would benefit
from adding
266 `--configfile ./matchingConfig.py` to
267 the command line arguments where `matchingConfig.py`
is defined by:
270 config.warpAndPsfMatch.psfMatch.kernel['AL'].kernelSize=27
271 config.warpAndPsfMatch.psfMatch.kernel[
'AL'].alardSigGauss=[1.5, 3.0, 6.0]
" > matchingConfig.py
274 Add the option `--help` to see more options.
276 ConfigClass = MakeCoaddTempExpConfig
277 _DefaultName = "makeCoaddTempExp"
280 CoaddBaseTask.__init__(self, **kwargs)
282 self.makeSubtask(
"warpAndPsfMatch")
283 if self.config.hasFakes:
290 """!Produce <coaddName>Coadd_<warpType>Warp images by warping and optionally PSF-matching.
292 @param[
in] patchRef: data reference
for sky map patch. Must include keys
"tract",
"patch",
293 plus the camera-specific filter key (e.g.
"filter" or "band")
294 @return: dataRefList: a list of data references
for the new <coaddName>Coadd_directWarps
295 if direct
or both warp types are requested
and <coaddName>Coadd_psfMatchedWarps
if only psfMatched
298 @warning: this task assumes that all exposures
in a warp (coaddTempExp) have the same filter.
300 @warning: this task sets the PhotoCalib of the coaddTempExp to the PhotoCalib of the first calexp
301 with any good pixels
in the patch. For a mosaic camera the resulting PhotoCalib should be ignored
302 (assembleCoadd should determine zeropoint scaling without referring to it).
307 if self.config.makePsfMatched
and not self.config.makeDirect:
312 calExpRefList = self.
selectExposuresselectExposures(patchRef, skyInfo, selectDataList=selectDataList)
314 if len(calExpRefList) == 0:
315 self.log.warning(
"No exposures to coadd for patch %s", patchRef.dataId)
317 self.log.info(
"Selected %d calexps for patch %s", len(calExpRefList), patchRef.dataId)
318 calExpRefList = [calExpRef
for calExpRef
in calExpRefList
if calExpRef.datasetExists(self.
calexpTypecalexpType)]
319 self.log.info(
"Processing %d existing calexps for patch %s", len(calExpRefList), patchRef.dataId)
323 self.log.info(
"Processing %d warp exposures for patch %s", len(groupData.groups), patchRef.dataId)
326 for i, (tempExpTuple, calexpRefList)
in enumerate(groupData.groups.items()):
328 tempExpTuple, groupData.keys)
329 if self.
reusereuse
and tempExpRef.datasetExists(datasetType=primaryWarpDataset, write=
True):
330 self.log.info(
"Skipping makeCoaddTempExp for %s; output already exists.", tempExpRef.dataId)
331 dataRefList.append(tempExpRef)
333 self.log.info(
"Processing Warp %d/%d: id=%s", i, len(groupData.groups), tempExpRef.dataId)
339 visitId = int(tempExpRef.dataId[
"visit"])
340 except (KeyError, ValueError):
347 for calExpInd, calExpRef
in enumerate(calexpRefList):
348 self.log.info(
"Reading calexp %s of %s for Warp id=%s", calExpInd+1, len(calexpRefList),
351 ccdId = calExpRef.get(
"ccdExposureId", immediate=
True)
358 calExpRef = calExpRef.butlerSubset.butler.dataRef(self.
calexpTypecalexpType,
359 dataId=calExpRef.dataId,
360 tract=skyInfo.tractInfo.getId())
361 calExp = self.
getCalibratedExposuregetCalibratedExposure(calExpRef, bgSubtracted=self.config.bgSubtracted)
362 except Exception
as e:
363 self.log.warning(
"Calexp %s not found; skipping it: %s", calExpRef.dataId, e)
366 if self.config.doApplySkyCorr:
369 calExpList.append(calExp)
370 ccdIdList.append(ccdId)
371 dataIdList.append(calExpRef.dataId)
373 exps = self.
runrun(calExpList, ccdIdList, skyInfo, visitId, dataIdList).exposures
375 if any(exps.values()):
376 dataRefList.append(tempExpRef)
378 self.log.warning(
"Warp %s could not be created", tempExpRef.dataId)
380 if self.config.doWrite:
381 for (warpType, exposure)
in exps.items():
382 if exposure
is not None:
389 def run(self, calExpList, ccdIdList, skyInfo, visitId=0, dataIdList=None, **kwargs):
390 """Create a Warp from inputs
392 We iterate over the multiple calexps in a single exposure to construct
393 the warp (previously called a coaddTempExp) of that exposure to the
394 supplied tract/patch.
396 Pixels that receive no pixels are set to NAN; this
is not correct
397 (violates LSST algorithms group policy), but will be fixed up by
398 interpolating after the coaddition.
400 @param calexpRefList: List of data references
for calexps that (may)
401 overlap the patch of interest
402 @param skyInfo: Struct
from CoaddBaseTask.getSkyInfo()
with geometric
403 information about the patch
404 @param visitId: integer identifier
for visit,
for the table that will
406 @return a pipeBase Struct containing:
407 - exposures: a dictionary containing the warps requested:
408 "direct": direct warp
if config.makeDirect
409 "psfMatched": PSF-matched warp
if config.makePsfMatched
413 totGoodPix = {warpType: 0 for warpType
in warpTypeList}
414 didSetMetadata = {warpType:
False for warpType
in warpTypeList}
415 coaddTempExps = {warpType: self.
_prepareEmptyExposure_prepareEmptyExposure(skyInfo)
for warpType
in warpTypeList}
416 inputRecorder = {warpType: self.inputRecorder.makeCoaddTempExpRecorder(visitId, len(calExpList))
417 for warpType
in warpTypeList}
419 modelPsf = self.config.modelPsf.apply()
if self.config.makePsfMatched
else None
420 if dataIdList
is None:
421 dataIdList = ccdIdList
423 for calExpInd, (calExp, ccdId, dataId)
in enumerate(zip(calExpList, ccdIdList, dataIdList)):
424 self.log.info(
"Processing calexp %d of %d for this Warp: id=%s",
425 calExpInd+1, len(calExpList), dataId)
428 warpedAndMatched = self.warpAndPsfMatch.
run(calExp, modelPsf=modelPsf,
429 wcs=skyInfo.wcs, maxBBox=skyInfo.bbox,
430 makeDirect=self.config.makeDirect,
431 makePsfMatched=self.config.makePsfMatched)
432 except Exception
as e:
433 self.log.warning(
"WarpAndPsfMatch failed for calexp %s; skipping it: %s", dataId, e)
436 numGoodPix = {warpType: 0
for warpType
in warpTypeList}
437 for warpType
in warpTypeList:
438 exposure = warpedAndMatched.getDict()[warpType]
441 coaddTempExp = coaddTempExps[warpType]
442 if didSetMetadata[warpType]:
443 mimg = exposure.getMaskedImage()
444 mimg *= (coaddTempExp.getPhotoCalib().getInstFluxAtZeroMagnitude()
445 / exposure.getPhotoCalib().getInstFluxAtZeroMagnitude())
447 numGoodPix[warpType] = coaddUtils.copyGoodPixels(
448 coaddTempExp.getMaskedImage(), exposure.getMaskedImage(), self.
getBadPixelMaskgetBadPixelMask())
449 totGoodPix[warpType] += numGoodPix[warpType]
450 self.log.debug(
"Calexp %s has %d good pixels in this patch (%.1f%%) for %s",
451 dataId, numGoodPix[warpType],
452 100.0*numGoodPix[warpType]/skyInfo.bbox.getArea(), warpType)
453 if numGoodPix[warpType] > 0
and not didSetMetadata[warpType]:
454 coaddTempExp.info.id = exposure.info.id
455 coaddTempExp.setPhotoCalib(exposure.getPhotoCalib())
456 coaddTempExp.setFilterLabel(exposure.getFilterLabel())
457 coaddTempExp.getInfo().setVisitInfo(exposure.getInfo().getVisitInfo())
459 coaddTempExp.setPsf(exposure.getPsf())
460 didSetMetadata[warpType] =
True
463 inputRecorder[warpType].addCalExp(calExp, ccdId, numGoodPix[warpType])
465 except Exception
as e:
466 self.log.warning(
"Error processing calexp %s; skipping it: %s", dataId, e)
469 for warpType
in warpTypeList:
470 self.log.info(
"%sWarp has %d good pixels (%.1f%%)",
471 warpType, totGoodPix[warpType], 100.0*totGoodPix[warpType]/skyInfo.bbox.getArea())
473 if totGoodPix[warpType] > 0
and didSetMetadata[warpType]:
474 inputRecorder[warpType].finish(coaddTempExps[warpType], totGoodPix[warpType])
475 if warpType ==
"direct":
476 coaddTempExps[warpType].setPsf(
477 CoaddPsf(inputRecorder[warpType].coaddInputs.ccds, skyInfo.wcs,
478 self.config.coaddPsf.makeControl()))
480 if not self.config.doWriteEmptyWarps:
482 coaddTempExps[warpType] =
None
487 result = pipeBase.Struct(exposures=coaddTempExps)
491 """Return one calibrated Exposure, possibly with an updated SkyWcs.
493 @param[
in] dataRef a sensor-level data reference
494 @param[
in] bgSubtracted
return calexp
with background subtracted? If
False get the
495 calexp
's background background model and add it to the calexp.
496 @return calibrated exposure
498 @raises MissingExposureError If data
for the exposure
is not available.
500 If config.doApplyExternalPhotoCalib
is `
True`, the photometric calibration
501 (`photoCalib`)
is taken
from `config.externalPhotoCalibName` via the
502 `name_photoCalib` dataset. Otherwise, the photometric calibration
is
503 retrieved
from the processed exposure. When
504 `config.doApplyExternalSkyWcs`
is `
True`, the astrometric calibration
505 is taken
from `config.externalSkyWcsName`
with the `name_wcs` dataset.
506 Otherwise, the astrometric calibration
is taken
from the processed
510 exposure = dataRef.get(self.
calexpTypecalexpType, immediate=
True)
511 except dafPersist.NoResults
as e:
515 background = dataRef.get(
"calexpBackground", immediate=
True)
516 mi = exposure.getMaskedImage()
517 mi += background.getImage()
520 if self.config.doApplyExternalPhotoCalib:
521 source = f
"{self.config.externalPhotoCalibName}_photoCalib"
522 self.log.debug(
"Applying external photoCalib to %s from %s", dataRef.dataId, source)
523 photoCalib = dataRef.get(source)
524 exposure.setPhotoCalib(photoCalib)
526 photoCalib = exposure.getPhotoCalib()
528 if self.config.doApplyExternalSkyWcs:
529 source = f
"{self.config.externalSkyWcsName}_wcs"
530 self.log.debug(
"Applying external skyWcs to %s from %s", dataRef.dataId, source)
531 skyWcs = dataRef.get(source)
532 exposure.setWcs(skyWcs)
534 exposure.maskedImage = photoCalib.calibrateImage(exposure.maskedImage,
535 includeScaleUncertainty=self.config.includeCalibVar)
536 exposure.maskedImage /= photoCalib.getCalibrationMean()
542 def _prepareEmptyExposure(skyInfo):
543 """Produce an empty exposure for a given patch"""
544 exp = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
545 exp.getMaskedImage().set(numpy.nan, afwImage.Mask
546 .getPlaneBitMask(
"NO_DATA"), numpy.inf)
550 """Return list of requested warp types per the config.
553 if self.config.makeDirect:
554 warpTypeList.append(
"direct")
555 if self.config.makePsfMatched:
556 warpTypeList.append(
"psfMatched")
560 """Apply correction to the sky background level
562 Sky corrections can be generated with the
'skyCorrection.py'
563 executable
in pipe_drivers. Because the sky model used by that
564 code extends over the entire focal plane, this can produce
565 better sky subtraction.
567 The calexp
is updated
in-place.
571 dataRef : `lsst.daf.persistence.ButlerDataRef`
572 Data reference
for calexp.
576 bg = dataRef.get("skyCorr")
577 self.log.debug(
"Applying sky correction to %s", dataRef.dataId)
578 if isinstance(calexp, afwImage.Exposure):
579 calexp = calexp.getMaskedImage()
580 calexp -= bg.getImage()
584 dimensions=(
"tract",
"patch",
"skymap",
"instrument",
"visit"),
585 defaultTemplates={
"coaddName":
"deep",
586 "skyWcsName":
"jointcal",
587 "photoCalibName":
"fgcm",
589 calExpList = connectionTypes.Input(
590 doc=
"Input exposures to be resampled and optionally PSF-matched onto a SkyMap projection/patch",
591 name=
"{calexpType}calexp",
592 storageClass=
"ExposureF",
593 dimensions=(
"instrument",
"visit",
"detector"),
597 backgroundList = connectionTypes.Input(
598 doc=
"Input backgrounds to be added back into the calexp if bgSubtracted=False",
599 name=
"calexpBackground",
600 storageClass=
"Background",
601 dimensions=(
"instrument",
"visit",
"detector"),
604 skyCorrList = connectionTypes.Input(
605 doc=
"Input Sky Correction to be subtracted from the calexp if doApplySkyCorr=True",
607 storageClass=
"Background",
608 dimensions=(
"instrument",
"visit",
"detector"),
611 skyMap = connectionTypes.Input(
612 doc=
"Input definition of geometry/bbox and projection/wcs for warped exposures",
613 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
614 storageClass=
"SkyMap",
615 dimensions=(
"skymap",),
617 externalSkyWcsTractCatalog = connectionTypes.Input(
618 doc=(
"Per-tract, per-visit wcs calibrations. These catalogs use the detector "
619 "id for the catalog id, sorted on id for fast lookup."),
620 name=
"{skyWcsName}SkyWcsCatalog",
621 storageClass=
"ExposureCatalog",
622 dimensions=(
"instrument",
"visit",
"tract"),
624 externalSkyWcsGlobalCatalog = connectionTypes.Input(
625 doc=(
"Per-visit wcs calibrations computed globally (with no tract information). "
626 "These catalogs use the detector id for the catalog id, sorted on id for "
628 name=
"{skyWcsName}SkyWcsCatalog",
629 storageClass=
"ExposureCatalog",
630 dimensions=(
"instrument",
"visit"),
632 externalPhotoCalibTractCatalog = connectionTypes.Input(
633 doc=(
"Per-tract, per-visit photometric calibrations. These catalogs use the "
634 "detector id for the catalog id, sorted on id for fast lookup."),
635 name=
"{photoCalibName}PhotoCalibCatalog",
636 storageClass=
"ExposureCatalog",
637 dimensions=(
"instrument",
"visit",
"tract"),
639 externalPhotoCalibGlobalCatalog = connectionTypes.Input(
640 doc=(
"Per-visit photometric calibrations computed globally (with no tract "
641 "information). These catalogs use the detector id for the catalog id, "
642 "sorted on id for fast lookup."),
643 name=
"{photoCalibName}PhotoCalibCatalog",
644 storageClass=
"ExposureCatalog",
645 dimensions=(
"instrument",
"visit"),
647 direct = connectionTypes.Output(
648 doc=(
"Output direct warped exposure (previously called CoaddTempExp), produced by resampling ",
649 "calexps onto the skyMap patch geometry."),
650 name=
"{coaddName}Coadd_directWarp",
651 storageClass=
"ExposureF",
652 dimensions=(
"tract",
"patch",
"skymap",
"visit",
"instrument"),
654 psfMatched = connectionTypes.Output(
655 doc=(
"Output PSF-Matched warped exposure (previously called CoaddTempExp), produced by resampling ",
656 "calexps onto the skyMap patch geometry and PSF-matching to a model PSF."),
657 name=
"{coaddName}Coadd_psfMatchedWarp",
658 storageClass=
"ExposureF",
659 dimensions=(
"tract",
"patch",
"skymap",
"visit",
"instrument"),
662 wcsList = connectionTypes.Input(
663 doc=
"WCSs of calexps used by SelectImages subtask to determine if the calexp overlaps the patch",
664 name=
"{calexpType}calexp.wcs",
666 dimensions=(
"instrument",
"visit",
"detector"),
669 bboxList = connectionTypes.Input(
670 doc=
"BBoxes of calexps used by SelectImages subtask to determine if the calexp overlaps the patch",
671 name=
"{calexpType}calexp.bbox",
672 storageClass=
"Box2I",
673 dimensions=(
"instrument",
"visit",
"detector"),
676 visitSummary = connectionTypes.Input(
677 doc=
"Consolidated exposure metadata from ConsolidateVisitSummaryTask",
678 name=
"{calexpType}visitSummary",
679 storageClass=
"ExposureCatalog",
680 dimensions=(
"instrument",
"visit",),
682 srcList = connectionTypes.Input(
683 doc=
"Source catalogs used by PsfWcsSelectImages subtask to further select on PSF stability",
685 storageClass=
"SourceCatalog",
686 dimensions=(
"instrument",
"visit",
"detector"),
690 def __init__(self, *, config=None):
691 super().__init__(config=config)
692 if config.bgSubtracted:
693 self.inputs.remove(
"backgroundList")
694 if not config.doApplySkyCorr:
695 self.inputs.remove(
"skyCorrList")
696 if config.doApplyExternalSkyWcs:
697 if config.useGlobalExternalSkyWcs:
698 self.inputs.remove(
"externalSkyWcsTractCatalog")
700 self.inputs.remove(
"externalSkyWcsGlobalCatalog")
702 self.inputs.remove(
"externalSkyWcsTractCatalog")
703 self.inputs.remove(
"externalSkyWcsGlobalCatalog")
704 if config.doApplyExternalPhotoCalib:
705 if config.useGlobalExternalPhotoCalib:
706 self.inputs.remove(
"externalPhotoCalibTractCatalog")
708 self.inputs.remove(
"externalPhotoCalibGlobalCatalog")
710 self.inputs.remove(
"externalPhotoCalibTractCatalog")
711 self.inputs.remove(
"externalPhotoCalibGlobalCatalog")
712 if not config.makeDirect:
713 self.outputs.remove(
"direct")
714 if not config.makePsfMatched:
715 self.outputs.remove(
"psfMatched")
717 if config.select.target != lsst.pipe.tasks.selectImages.PsfWcsSelectImagesTask:
718 self.inputs.remove(
"visitSummary")
719 self.inputs.remove(
"srcList")
720 elif not config.select.doLegacyStarSelectionComputation:
722 self.inputs.remove(
"srcList")
726 pipelineConnections=MakeWarpConnections):
733 """Warp and optionally PSF-Match calexps onto an a common projection
735 ConfigClass = MakeWarpConfig
736 _DefaultName = "makeWarp"
738 @utils.inheritDoc(pipeBase.PipelineTask)
739 def runQuantum(self, butlerQC, inputRefs, outputRefs):
743 Construct warps for requested warp type
for single epoch
745 PipelineTask (Gen3) entry point to warp
and optionally PSF-match
746 calexps. This method
is analogous to `runDataRef`.
750 detectorOrder = [ref.datasetRef.dataId[
'detector']
for ref
in inputRefs.calExpList]
751 inputRefs = reorderRefs(inputRefs, detectorOrder, dataIdKey=
'detector')
754 inputs = butlerQC.get(inputRefs)
758 skyMap = inputs.pop(
"skyMap")
759 quantumDataId = butlerQC.quantum.dataId
760 skyInfo =
makeSkyInfo(skyMap, tractId=quantumDataId[
'tract'], patchId=quantumDataId[
'patch'])
763 dataIdList = [ref.datasetRef.dataId
for ref
in inputRefs.calExpList]
765 ccdIdList = [dataId.pack(
"visit_detector")
for dataId
in dataIdList]
770 coordList = [skyInfo.wcs.pixelToSky(pos)
for pos
in cornerPosList]
771 goodIndices = self.select.run(**inputs, coordList=coordList, dataIds=dataIdList)
772 inputs = self.filterInputs(indices=goodIndices, inputs=inputs)
775 inputs[
'calExpList'] = [ref.get()
for ref
in inputs[
'calExpList']]
778 visits = [dataId[
'visit']
for dataId
in dataIdList]
781 if self.config.doApplyExternalSkyWcs:
782 if self.config.useGlobalExternalSkyWcs:
783 externalSkyWcsCatalog = inputs.pop(
"externalSkyWcsGlobalCatalog")
785 externalSkyWcsCatalog = inputs.pop(
"externalSkyWcsTractCatalog")
787 externalSkyWcsCatalog =
None
789 if self.config.doApplyExternalPhotoCalib:
790 if self.config.useGlobalExternalPhotoCalib:
791 externalPhotoCalibCatalog = inputs.pop(
"externalPhotoCalibGlobalCatalog")
793 externalPhotoCalibCatalog = inputs.pop(
"externalPhotoCalibTractCatalog")
795 externalPhotoCalibCatalog =
None
797 completeIndices = self.prepareCalibratedExposures(**inputs,
798 externalSkyWcsCatalog=externalSkyWcsCatalog,
799 externalPhotoCalibCatalog=externalPhotoCalibCatalog)
801 inputs = self.filterInputs(indices=completeIndices, inputs=inputs)
803 results = self.run(**inputs, visitId=visitId,
804 ccdIdList=[ccdIdList[i]
for i
in goodIndices],
805 dataIdList=[dataIdList[i]
for i
in goodIndices],
807 if self.config.makeDirect
and results.exposures[
"direct"]
is not None:
808 butlerQC.put(results.exposures[
"direct"], outputRefs.direct)
809 if self.config.makePsfMatched
and results.exposures[
"psfMatched"]
is not None:
810 butlerQC.put(results.exposures[
"psfMatched"], outputRefs.psfMatched)
812 def filterInputs(self, indices, inputs):
813 """Return task inputs with their lists filtered by indices
817 indices : `list` of integers
818 inputs : `dict` of `list` of input connections to be passed to run
820 for key
in inputs.keys():
822 if isinstance(inputs[key], list):
823 inputs[key] = [inputs[key][ind]
for ind
in indices]
826 def prepareCalibratedExposures(self, calExpList, backgroundList=None, skyCorrList=None,
827 externalSkyWcsCatalog=None, externalPhotoCalibCatalog=None,
829 """Calibrate and add backgrounds to input calExpList in place
834 Sequence of calexps to be modified in place
835 backgroundList : `list` of `lsst.afw.math.backgroundList`, optional
836 Sequence of backgrounds to be added back
in if bgSubtracted=
False
837 skyCorrList : `list` of `lsst.afw.math.backgroundList`, optional
838 Sequence of background corrections to be subtracted
if doApplySkyCorr=
True
840 Exposure catalog
with external skyWcs to be applied
841 if config.doApplyExternalSkyWcs=
True. Catalog uses the detector id
842 for the catalog id, sorted on id
for fast lookup.
844 Exposure catalog
with external photoCalib to be applied
845 if config.doApplyExternalPhotoCalib=
True. Catalog uses the detector
846 id
for the catalog id, sorted on id
for fast lookup.
850 indices : `list` [`int`]
851 Indices of calExpList
and friends that have valid photoCalib/skyWcs
853 backgroundList = len(calExpList)*[None]
if backgroundList
is None else backgroundList
854 skyCorrList = len(calExpList)*[
None]
if skyCorrList
is None else skyCorrList
856 includeCalibVar = self.config.includeCalibVar
859 for index, (calexp, background, skyCorr)
in enumerate(zip(calExpList,
862 if not self.config.bgSubtracted:
863 calexp.maskedImage += background.getImage()
865 if externalSkyWcsCatalog
is not None or externalPhotoCalibCatalog
is not None:
866 detectorId = calexp.getInfo().getDetector().getId()
869 if externalPhotoCalibCatalog
is not None:
870 row = externalPhotoCalibCatalog.find(detectorId)
872 self.log.warning(
"Detector id %s not found in externalPhotoCalibCatalog "
873 "and will not be used in the warp.", detectorId)
875 photoCalib = row.getPhotoCalib()
876 if photoCalib
is None:
877 self.log.warning(
"Detector id %s has None for photoCalib in externalPhotoCalibCatalog "
878 "and will not be used in the warp.", detectorId)
880 calexp.setPhotoCalib(photoCalib)
882 photoCalib = calexp.getPhotoCalib()
883 if photoCalib
is None:
884 self.log.warning(
"Detector id %s has None for photoCalib in the calexp "
885 "and will not be used in the warp.", detectorId)
889 if externalSkyWcsCatalog
is not None:
890 row = externalSkyWcsCatalog.find(detectorId)
892 self.log.warning(
"Detector id %s not found in externalSkyWcsCatalog "
893 "and will not be used in the warp.", detectorId)
895 skyWcs = row.getWcs()
897 self.log.warning(
"Detector id %s has None for skyWcs in externalSkyWcsCatalog "
898 "and will not be used in the warp.", detectorId)
900 calexp.setWcs(skyWcs)
902 skyWcs = calexp.getWcs()
904 self.log.warning(
"Detector id %s has None for skyWcs in the calexp "
905 "and will not be used in the warp.", detectorId)
909 calexp.maskedImage = photoCalib.calibrateImage(calexp.maskedImage,
910 includeScaleUncertainty=includeCalibVar)
911 calexp.maskedImage /= photoCalib.getCalibrationMean()
916 if self.config.doApplySkyCorr:
917 calexp.maskedImage -= skyCorr.getImage()
919 indices.append(index)
924def reorderRefs(inputRefs, outputSortKeyOrder, dataIdKey):
925 """Reorder inputRefs per outputSortKeyOrder
927 Any inputRefs which are lists will be resorted per specified key e.g.,
928 'detector.' Only iterables will be reordered,
and values can be of type
929 `lsst.pipe.base.connections.DeferredDatasetRef`
or
930 `lsst.daf.butler.core.datasets.ref.DatasetRef`.
931 Returned lists of refs have the same length
as the outputSortKeyOrder.
932 If an outputSortKey
not in the inputRef, then it will be padded
with None.
933 If an inputRef contains an inputSortKey that
is not in the
934 outputSortKeyOrder it will be removed.
938 inputRefs : `lsst.pipe.base.connections.QuantizedConnection`
939 Input references to be reordered
and padded.
940 outputSortKeyOrder : iterable
941 Iterable of values to be compared
with inputRef
's dataId[dataIdKey]
943 dataIdKey in the dataRefs to compare
with the outputSortKeyOrder.
947 inputRefs: `lsst.pipe.base.connections.QuantizedConnection`
948 Quantized Connection
with sorted DatasetRef values sorted
if iterable.
950 for connectionName, refs
in inputRefs:
951 if isinstance(refs, Iterable):
952 if hasattr(refs[0],
"dataId"):
953 inputSortKeyOrder = [ref.dataId[dataIdKey]
for ref
in refs]
955 inputSortKeyOrder = [ref.datasetRef.dataId[dataIdKey]
for ref
in refs]
956 if inputSortKeyOrder != outputSortKeyOrder:
957 setattr(inputRefs, connectionName,
Configuration parameters for CoaddBaseTask.
Base class for coaddition.
def getTempExpDatasetName(self, warpType="direct")
def selectExposures(self, patchRef, skyInfo=None, selectDataList=[])
Select exposures to coadd.
def getCoaddDatasetName(self, warpType="direct")
def getSkyInfo(self, patchRef)
Use getSkyinfo to return the skyMap, tract and patch information, wcs and the outer bbox of the patch...
def getBadPixelMask(self)
Convenience method to provide the bitmask from the mask plane names.
Warp and optionally PSF-Match calexps onto an a common projection.
def getCalibratedExposure(self, dataRef, bgSubtracted)
def run(self, calExpList, ccdIdList, skyInfo, visitId=0, dataIdList=None, **kwargs)
def __init__(self, reuse=False, **kwargs)
def _prepareEmptyExposure(skyInfo)
def runDataRef(self, patchRef, selectDataList=[])
Produce <coaddName>Coadd_<warpType>Warp images by warping and optionally PSF-matching.
def getWarpTypeList(self)
def applySkyCorr(self, dataRef, calexp)
def reorderAndPadList(inputList, inputKeys, outputKeys, padWith=None)
def makeSkyInfo(skyMap, tractId, patchId)
def getGroupDataRef(butler, datasetType, groupTuple, keys)
def groupPatchExposures(patchDataRef, calexpDataRefList, coaddDatasetType="deepCoadd", tempExpDatasetType="deepCoadd_directWarp")