30import lsst.pipe.base.connectionTypes
as connectionTypes
31import lsst.utils
as utils
35from .coaddBase
import CoaddBaseTask, makeSkyInfo, reorderAndPadList
36from .selectImages
import PsfWcsSelectImagesTask
37from .warpAndPsfMatch
import WarpAndPsfMatchTask
38from .coaddHelpers
import groupPatchExposures, getGroupDataRef
39from collections.abc
import Iterable
41__all__ = [
"MakeCoaddTempExpTask",
"MakeWarpTask",
"MakeWarpConfig"]
43log = logging.getLogger(__name__.partition(
".")[2])
47 """Raised when data cannot be retrieved for an exposure.
48 When processing patches, sometimes one exposure is missing; this lets us
49 distinguish bewteen that case,
and other errors.
55 """Config for MakeCoaddTempExpTask
57 warpAndPsfMatch = pexConfig.ConfigurableField(
58 target=WarpAndPsfMatchTask,
59 doc="Task to warp and PSF-match calexp",
61 doWrite = pexConfig.Field(
62 doc=
"persist <coaddName>Coadd_<warpType>Warp",
66 bgSubtracted = pexConfig.Field(
67 doc=
"Work with a background subtracted calexp?",
71 coaddPsf = pexConfig.ConfigField(
72 doc=
"Configuration for CoaddPsf",
75 makeDirect = pexConfig.Field(
76 doc=
"Make direct Warp/Coadds",
80 makePsfMatched = pexConfig.Field(
81 doc=
"Make Psf-Matched Warp/Coadd?",
86 doWriteEmptyWarps = pexConfig.Field(
89 doc=
"Write out warps even if they are empty"
92 hasFakes = pexConfig.Field(
93 doc=
"Should be set to True if fake sources have been inserted into the input data.",
97 doApplySkyCorr = pexConfig.Field(dtype=bool, default=
False, doc=
"Apply sky correction?")
100 CoaddBaseTask.ConfigClass.validate(self)
102 raise RuntimeError(
"At least one of config.makePsfMatched and config.makeDirect must be True")
105 log.warning(
"Config doPsfMatch deprecated. Setting makePsfMatched=True and makeDirect=False")
110 CoaddBaseTask.ConfigClass.setDefaults(self)
111 self.
warpAndPsfMatch.psfMatch.kernel.active.kernelSize = self.matchingKernelSize
112 self.select.retarget(PsfWcsSelectImagesTask)
123 r"""!Warp and optionally PSF-Match calexps onto an a common projection.
125 @anchor MakeCoaddTempExpTask_
127 @section pipe_tasks_makeCoaddTempExp_Contents Contents
129 -
@ref pipe_tasks_makeCoaddTempExp_Purpose
130 -
@ref pipe_tasks_makeCoaddTempExp_Initialize
131 -
@ref pipe_tasks_makeCoaddTempExp_IO
132 -
@ref pipe_tasks_makeCoaddTempExp_Config
133 -
@ref pipe_tasks_makeCoaddTempExp_Debug
134 -
@ref pipe_tasks_makeCoaddTempExp_Example
136 @section pipe_tasks_makeCoaddTempExp_Purpose Description
138 Warp
and optionally PSF-Match calexps onto a common projection, by
139 performing the following operations:
140 - Group calexps by visit/run
141 - For each visit, generate a Warp by calling method
@ref makeTempExp.
142 makeTempExp loops over the visit
's calexps calling @ref WarpAndPsfMatch
145 The result is a `directWarp` (
and/
or optionally a `psfMatchedWarp`).
147 @section pipe_tasks_makeCoaddTempExp_Initialize Task Initialization
149 @copydoc \_\_init\_\_
151 This task has one special keyword argument: passing reuse=
True will cause
152 the task to skip the creation of warps that are already present
in the
155 @section pipe_tasks_makeCoaddTempExp_IO Invoking the Task
157 This task
is primarily designed to be run
from the command line.
159 The main method
is `runDataRef`, which takes a single butler data reference
for the
patch(es)
164 WarpType identifies the types of convolutions applied to Warps (previously CoaddTempExps).
165 Only two types are available: direct (
for regular Warps/Coadds)
and psfMatched
166 (
for Warps/Coadds
with homogenized PSFs). We expect to add a third type, likelihood,
167 for generating likelihood Coadds
with Warps that have been correlated
with their own PSF.
169 @section pipe_tasks_makeCoaddTempExp_Config Configuration parameters
171 See
@ref MakeCoaddTempExpConfig
and parameters inherited
from
174 @subsection pipe_tasks_MakeCoaddTempExp_psfMatching Guide to PSF-Matching Configs
176 To make `psfMatchedWarps`, select `config.makePsfMatched=
True`. The subtask
178 is responsible
for the PSF-Matching,
and its config
is accessed via `config.warpAndPsfMatch.psfMatch`.
179 The optimal configuration depends on aspects of dataset: the pixel scale, average PSF FWHM
and
180 dimensions of the PSF kernel. These configs include the requested model PSF, the matching kernel size,
181 padding of the science PSF thumbnail
and spatial sampling frequency of the PSF.
183 *Config Guidelines*: The user must specify the size of the model PSF to which to match by setting
184 `config.modelPsf.defaultFwhm`
in units of pixels. The appropriate values depends on science case.
185 In general,
for a set of input images, this config should equal the FWHM of the visit
186 with the worst seeing. The smallest it should be set to
is the median FWHM. The defaults
187 of the other config options offer a reasonable starting point.
188 The following list presents the most common problems that arise
from a misconfigured
190 and corresponding solutions. All assume the default Alard-Lupton kernel,
with configs accessed via
191 ```config.warpAndPsfMatch.psfMatch.kernel[
'AL']```. Each item
in the list
is formatted
as:
192 Problem: Explanation. *Solution*
194 *Troublshooting PSF-Matching Configuration:*
195 - Matched PSFs look boxy: The matching kernel
is too small. _Increase the matching kernel size.
198 config.warpAndPsfMatch.psfMatch.kernel[
'AL'].kernelSize=27
200 Note that increasing the kernel size also increases runtime.
201 - Matched PSFs look ugly (dipoles, quadropoles, donuts): unable to find good solution
202 for matching kernel. _Provide the matcher
with more data by either increasing
203 the spatial sampling by decreasing the spatial cell size,_
205 config.warpAndPsfMatch.psfMatch.kernel[
'AL'].sizeCellX = 64
206 config.warpAndPsfMatch.psfMatch.kernel[
'AL'].sizeCellY = 64
208 _or increasing the padding around the Science PSF,
for example:_
210 config.warpAndPsfMatch.psfMatch.autoPadPsfTo=1.6
212 Increasing `autoPadPsfTo` increases the minimum ratio of input PSF dimensions to the
213 matching kernel dimensions, thus increasing the number of pixels available to fit
214 after convolving the PSF
with the matching kernel.
215 Optionally,
for debugging the effects of padding, the level of padding may be manually
216 controlled by setting turning off the automatic padding
and setting the number
217 of pixels by which to pad the PSF:
219 config.warpAndPsfMatch.psfMatch.doAutoPadPsf =
False
220 config.warpAndPsfMatch.psfMatch.padPsfBy = 6
222 - Deconvolution: Matching a large PSF to a smaller PSF produces
223 a telltale noise pattern which looks like ripples
or a brain.
224 _Increase the size of the requested model PSF. For example:_
226 config.modelPsf.defaultFwhm = 11
228 - High frequency (sometimes checkered) noise: The matching basis functions are too small.
229 _Increase the width of the Gaussian basis functions. For example:_
231 config.warpAndPsfMatch.psfMatch.kernel[
'AL'].alardSigGauss=[1.5, 3.0, 6.0]
235 @section pipe_tasks_makeCoaddTempExp_Debug Debug variables
237 MakeCoaddTempExpTask has no debug output, but its subtasks do.
239 @section pipe_tasks_makeCoaddTempExp_Example A complete example of using MakeCoaddTempExpTask
241 This example uses the package ci_hsc to show how MakeCoaddTempExp fits
242 into the larger Data Release Processing.
248 python $(which scons)
250 The following assumes that `processCcd.py`
and `makeSkyMap.py` have previously been run
251 (e.g. by building `ci_hsc` above) to generate a repository of calexps
and an
252 output respository
with the desired SkyMap. The command,
254 makeCoaddTempExp.py $CI_HSC_DIR/DATA --rerun ci_hsc \
255 --id patch=5,4 tract=0 filter=HSC-I \
256 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 \
257 --selectId visit=903988 ccd=23 --selectId visit=903988 ccd=24 \
258 --config doApplyExternalPhotoCalib=
False doApplyExternalSkyWcs=
False \
259 makePsfMatched=
True modelPsf.defaultFwhm=11
261 writes a direct
and PSF-Matched Warp to
262 - `$CI_HSC_DIR/DATA/rerun/ci_hsc/deepCoadd/HSC-I/0/5,4/warp-HSC-I-0-5,4-903988.fits`
and
263 - `$CI_HSC_DIR/DATA/rerun/ci_hsc/deepCoadd/HSC-I/0/5,4/psfMatchedWarp-HSC-I-0-5,4-903988.fits`
266 @note PSF-Matching
in this particular dataset would benefit
from adding
267 `--configfile ./matchingConfig.py` to
268 the command line arguments where `matchingConfig.py`
is defined by:
271 config.warpAndPsfMatch.psfMatch.kernel['AL'].kernelSize=27
272 config.warpAndPsfMatch.psfMatch.kernel[
'AL'].alardSigGauss=[1.5, 3.0, 6.0]
" > matchingConfig.py
275 Add the option `--help` to see more options.
277 ConfigClass = MakeCoaddTempExpConfig
278 _DefaultName = "makeCoaddTempExp"
281 CoaddBaseTask.__init__(self, **kwargs)
283 self.makeSubtask(
"warpAndPsfMatch")
284 if self.config.hasFakes:
291 """!Produce <coaddName>Coadd_<warpType>Warp images by warping and optionally PSF-matching.
293 @param[
in] patchRef: data reference
for sky map patch. Must include keys
"tract",
"patch",
294 plus the camera-specific filter key (e.g.
"filter" or "band")
295 @return: dataRefList: a list of data references
for the new <coaddName>Coadd_directWarps
296 if direct
or both warp types are requested
and <coaddName>Coadd_psfMatchedWarps
if only psfMatched
299 @warning: this task assumes that all exposures
in a warp (coaddTempExp) have the same filter.
301 @warning: this task sets the PhotoCalib of the coaddTempExp to the PhotoCalib of the first calexp
302 with any good pixels
in the patch. For a mosaic camera the resulting PhotoCalib should be ignored
303 (assembleCoadd should determine zeropoint scaling without referring to it).
308 if self.config.makePsfMatched
and not self.config.makeDirect:
313 calExpRefList = self.
selectExposures(patchRef, skyInfo, selectDataList=selectDataList)
315 if len(calExpRefList) == 0:
316 self.log.warning(
"No exposures to coadd for patch %s", patchRef.dataId)
318 self.log.info(
"Selected %d calexps for patch %s", len(calExpRefList), patchRef.dataId)
319 calExpRefList = [calExpRef
for calExpRef
in calExpRefList
if calExpRef.datasetExists(self.
calexpType)]
320 self.log.info(
"Processing %d existing calexps for patch %s", len(calExpRefList), patchRef.dataId)
324 self.log.info(
"Processing %d warp exposures for patch %s", len(groupData.groups), patchRef.dataId)
327 for i, (tempExpTuple, calexpRefList)
in enumerate(groupData.groups.items()):
329 tempExpTuple, groupData.keys)
330 if self.
reuse and tempExpRef.datasetExists(datasetType=primaryWarpDataset, write=
True):
331 self.log.info(
"Skipping makeCoaddTempExp for %s; output already exists.", tempExpRef.dataId)
332 dataRefList.append(tempExpRef)
334 self.log.info(
"Processing Warp %d/%d: id=%s", i, len(groupData.groups), tempExpRef.dataId)
340 visitId = int(tempExpRef.dataId[
"visit"])
341 except (KeyError, ValueError):
348 for calExpInd, calExpRef
in enumerate(calexpRefList):
349 self.log.info(
"Reading calexp %s of %s for Warp id=%s", calExpInd+1, len(calexpRefList),
352 ccdId = calExpRef.get(
"ccdExposureId", immediate=
True)
359 calExpRef = calExpRef.butlerSubset.butler.dataRef(self.
calexpType,
360 dataId=calExpRef.dataId,
361 tract=skyInfo.tractInfo.getId())
363 except Exception
as e:
364 self.log.warning(
"Calexp %s not found; skipping it: %s", calExpRef.dataId, e)
367 if self.config.doApplySkyCorr:
370 calExpList.append(calExp)
371 ccdIdList.append(ccdId)
372 dataIdList.append(calExpRef.dataId)
374 exps = self.
run(calExpList, ccdIdList, skyInfo, visitId, dataIdList).exposures
376 if any(exps.values()):
377 dataRefList.append(tempExpRef)
379 self.log.warning(
"Warp %s could not be created", tempExpRef.dataId)
381 if self.config.doWrite:
382 for (warpType, exposure)
in exps.items():
383 if exposure
is not None:
390 def run(self, calExpList, ccdIdList, skyInfo, visitId=0, dataIdList=None, **kwargs):
391 """Create a Warp from inputs
393 We iterate over the multiple calexps in a single exposure to construct
394 the warp (previously called a coaddTempExp) of that exposure to the
395 supplied tract/patch.
397 Pixels that receive no pixels are set to NAN; this
is not correct
398 (violates LSST algorithms group policy), but will be fixed up by
399 interpolating after the coaddition.
401 @param calexpRefList: List of data references
for calexps that (may)
402 overlap the patch of interest
403 @param skyInfo: Struct
from CoaddBaseTask.getSkyInfo()
with geometric
404 information about the patch
405 @param visitId: integer identifier
for visit,
for the table that will
407 @return a pipeBase Struct containing:
408 - exposures: a dictionary containing the warps requested:
409 "direct": direct warp
if config.makeDirect
410 "psfMatched": PSF-matched warp
if config.makePsfMatched
414 totGoodPix = {warpType: 0 for warpType
in warpTypeList}
415 didSetMetadata = {warpType:
False for warpType
in warpTypeList}
417 inputRecorder = {warpType: self.inputRecorder.makeCoaddTempExpRecorder(visitId, len(calExpList))
418 for warpType
in warpTypeList}
420 modelPsf = self.config.modelPsf.apply()
if self.config.makePsfMatched
else None
421 if dataIdList
is None:
422 dataIdList = ccdIdList
424 for calExpInd, (calExp, ccdId, dataId)
in enumerate(zip(calExpList, ccdIdList, dataIdList)):
425 self.log.info(
"Processing calexp %d of %d for this Warp: id=%s",
426 calExpInd+1, len(calExpList), dataId)
429 warpedAndMatched = self.warpAndPsfMatch.
run(calExp, modelPsf=modelPsf,
430 wcs=skyInfo.wcs, maxBBox=skyInfo.bbox,
431 makeDirect=self.config.makeDirect,
432 makePsfMatched=self.config.makePsfMatched)
433 except Exception
as e:
434 self.log.warning(
"WarpAndPsfMatch failed for calexp %s; skipping it: %s", dataId, e)
437 numGoodPix = {warpType: 0
for warpType
in warpTypeList}
438 for warpType
in warpTypeList:
439 exposure = warpedAndMatched.getDict()[warpType]
442 coaddTempExp = coaddTempExps[warpType]
443 if didSetMetadata[warpType]:
444 mimg = exposure.getMaskedImage()
445 mimg *= (coaddTempExp.getPhotoCalib().getInstFluxAtZeroMagnitude()
446 / exposure.getPhotoCalib().getInstFluxAtZeroMagnitude())
448 numGoodPix[warpType] = coaddUtils.copyGoodPixels(
449 coaddTempExp.getMaskedImage(), exposure.getMaskedImage(), self.
getBadPixelMask())
450 totGoodPix[warpType] += numGoodPix[warpType]
451 self.log.debug(
"Calexp %s has %d good pixels in this patch (%.1f%%) for %s",
452 dataId, numGoodPix[warpType],
453 100.0*numGoodPix[warpType]/skyInfo.bbox.getArea(), warpType)
454 if numGoodPix[warpType] > 0
and not didSetMetadata[warpType]:
455 coaddTempExp.setPhotoCalib(exposure.getPhotoCalib())
456 coaddTempExp.setFilterLabel(exposure.getFilterLabel())
457 coaddTempExp.getInfo().setVisitInfo(exposure.getInfo().getVisitInfo())
459 coaddTempExp.setPsf(exposure.getPsf())
460 didSetMetadata[warpType] =
True
463 inputRecorder[warpType].addCalExp(calExp, ccdId, numGoodPix[warpType])
465 except Exception
as e:
466 self.log.warning(
"Error processing calexp %s; skipping it: %s", dataId, e)
469 for warpType
in warpTypeList:
470 self.log.info(
"%sWarp has %d good pixels (%.1f%%)",
471 warpType, totGoodPix[warpType], 100.0*totGoodPix[warpType]/skyInfo.bbox.getArea())
473 if totGoodPix[warpType] > 0
and didSetMetadata[warpType]:
474 inputRecorder[warpType].finish(coaddTempExps[warpType], totGoodPix[warpType])
475 if warpType ==
"direct":
476 coaddTempExps[warpType].setPsf(
477 CoaddPsf(inputRecorder[warpType].coaddInputs.ccds, skyInfo.wcs,
478 self.config.coaddPsf.makeControl()))
480 if not self.config.doWriteEmptyWarps:
482 coaddTempExps[warpType] =
None
487 result = pipeBase.Struct(exposures=coaddTempExps)
491 """Return one calibrated Exposure, possibly with an updated SkyWcs.
493 @param[
in] dataRef a sensor-level data reference
494 @param[
in] bgSubtracted
return calexp
with background subtracted? If
False get the
495 calexp
's background background model and add it to the calexp.
496 @return calibrated exposure
498 @raises MissingExposureError If data
for the exposure
is not available.
500 If config.doApplyExternalPhotoCalib
is `
True`, the photometric calibration
501 (`photoCalib`)
is taken
from `config.externalPhotoCalibName` via the
502 `name_photoCalib` dataset. Otherwise, the photometric calibration
is
503 retrieved
from the processed exposure. When
504 `config.doApplyExternalSkyWcs`
is `
True`, the astrometric calibration
505 is taken
from `config.externalSkyWcsName`
with the `name_wcs` dataset.
506 Otherwise, the astrometric calibration
is taken
from the processed
510 exposure = dataRef.get(self.
calexpType, immediate=
True)
511 except dafPersist.NoResults
as e:
515 background = dataRef.get(
"calexpBackground", immediate=
True)
516 mi = exposure.getMaskedImage()
517 mi += background.getImage()
520 if self.config.doApplyExternalPhotoCalib:
521 source = f
"{self.config.externalPhotoCalibName}_photoCalib"
522 self.log.debug(
"Applying external photoCalib to %s from %s", dataRef.dataId, source)
523 photoCalib = dataRef.get(source)
524 exposure.setPhotoCalib(photoCalib)
526 photoCalib = exposure.getPhotoCalib()
528 if self.config.doApplyExternalSkyWcs:
529 source = f
"{self.config.externalSkyWcsName}_wcs"
530 self.log.debug(
"Applying external skyWcs to %s from %s", dataRef.dataId, source)
531 skyWcs = dataRef.get(source)
532 exposure.setWcs(skyWcs)
534 exposure.maskedImage = photoCalib.calibrateImage(exposure.maskedImage,
535 includeScaleUncertainty=self.config.includeCalibVar)
536 exposure.maskedImage /= photoCalib.getCalibrationMean()
542 def _prepareEmptyExposure(skyInfo):
543 """Produce an empty exposure for a given patch"""
544 exp = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
545 exp.getMaskedImage().set(numpy.nan, afwImage.Mask
546 .getPlaneBitMask(
"NO_DATA"), numpy.inf)
550 """Return list of requested warp types per the config.
553 if self.config.makeDirect:
554 warpTypeList.append(
"direct")
555 if self.config.makePsfMatched:
556 warpTypeList.append(
"psfMatched")
560 """Apply correction to the sky background level
562 Sky corrections can be generated with the
'skyCorrection.py'
563 executable
in pipe_drivers. Because the sky model used by that
564 code extends over the entire focal plane, this can produce
565 better sky subtraction.
567 The calexp
is updated
in-place.
571 dataRef : `lsst.daf.persistence.ButlerDataRef`
572 Data reference
for calexp.
576 bg = dataRef.get("skyCorr")
577 self.log.debug(
"Applying sky correction to %s", dataRef.dataId)
578 if isinstance(calexp, afwImage.Exposure):
579 calexp = calexp.getMaskedImage()
580 calexp -= bg.getImage()
584 dimensions=(
"tract",
"patch",
"skymap",
"instrument",
"visit"),
585 defaultTemplates={
"coaddName":
"deep",
586 "skyWcsName":
"jointcal",
587 "photoCalibName":
"fgcm",
589 calExpList = connectionTypes.Input(
590 doc=
"Input exposures to be resampled and optionally PSF-matched onto a SkyMap projection/patch",
591 name=
"{calexpType}calexp",
592 storageClass=
"ExposureF",
593 dimensions=(
"instrument",
"visit",
"detector"),
597 backgroundList = connectionTypes.Input(
598 doc=
"Input backgrounds to be added back into the calexp if bgSubtracted=False",
599 name=
"calexpBackground",
600 storageClass=
"Background",
601 dimensions=(
"instrument",
"visit",
"detector"),
604 skyCorrList = connectionTypes.Input(
605 doc=
"Input Sky Correction to be subtracted from the calexp if doApplySkyCorr=True",
607 storageClass=
"Background",
608 dimensions=(
"instrument",
"visit",
"detector"),
611 skyMap = connectionTypes.Input(
612 doc=
"Input definition of geometry/bbox and projection/wcs for warped exposures",
613 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
614 storageClass=
"SkyMap",
615 dimensions=(
"skymap",),
617 externalSkyWcsTractCatalog = connectionTypes.Input(
618 doc=(
"Per-tract, per-visit wcs calibrations. These catalogs use the detector "
619 "id for the catalog id, sorted on id for fast lookup."),
620 name=
"{skyWcsName}SkyWcsCatalog",
621 storageClass=
"ExposureCatalog",
622 dimensions=(
"instrument",
"visit",
"tract"),
624 externalSkyWcsGlobalCatalog = connectionTypes.Input(
625 doc=(
"Per-visit wcs calibrations computed globally (with no tract information). "
626 "These catalogs use the detector id for the catalog id, sorted on id for "
628 name=
"{skyWcsName}SkyWcsCatalog",
629 storageClass=
"ExposureCatalog",
630 dimensions=(
"instrument",
"visit"),
632 externalPhotoCalibTractCatalog = connectionTypes.Input(
633 doc=(
"Per-tract, per-visit photometric calibrations. These catalogs use the "
634 "detector id for the catalog id, sorted on id for fast lookup."),
635 name=
"{photoCalibName}PhotoCalibCatalog",
636 storageClass=
"ExposureCatalog",
637 dimensions=(
"instrument",
"visit",
"tract"),
639 externalPhotoCalibGlobalCatalog = connectionTypes.Input(
640 doc=(
"Per-visit photometric calibrations computed globally (with no tract "
641 "information). These catalogs use the detector id for the catalog id, "
642 "sorted on id for fast lookup."),
643 name=
"{photoCalibName}PhotoCalibCatalog",
644 storageClass=
"ExposureCatalog",
645 dimensions=(
"instrument",
"visit"),
647 direct = connectionTypes.Output(
648 doc=(
"Output direct warped exposure (previously called CoaddTempExp), produced by resampling ",
649 "calexps onto the skyMap patch geometry."),
650 name=
"{coaddName}Coadd_directWarp",
651 storageClass=
"ExposureF",
652 dimensions=(
"tract",
"patch",
"skymap",
"visit",
"instrument"),
654 psfMatched = connectionTypes.Output(
655 doc=(
"Output PSF-Matched warped exposure (previously called CoaddTempExp), produced by resampling ",
656 "calexps onto the skyMap patch geometry and PSF-matching to a model PSF."),
657 name=
"{coaddName}Coadd_psfMatchedWarp",
658 storageClass=
"ExposureF",
659 dimensions=(
"tract",
"patch",
"skymap",
"visit",
"instrument"),
662 wcsList = connectionTypes.Input(
663 doc=
"WCSs of calexps used by SelectImages subtask to determine if the calexp overlaps the patch",
664 name=
"{calexpType}calexp.wcs",
666 dimensions=(
"instrument",
"visit",
"detector"),
669 bboxList = connectionTypes.Input(
670 doc=
"BBoxes of calexps used by SelectImages subtask to determine if the calexp overlaps the patch",
671 name=
"{calexpType}calexp.bbox",
672 storageClass=
"Box2I",
673 dimensions=(
"instrument",
"visit",
"detector"),
676 visitSummary = connectionTypes.Input(
677 doc=
"Consolidated exposure metadata from ConsolidateVisitSummaryTask",
678 name=
"{calexpType}visitSummary",
679 storageClass=
"ExposureCatalog",
680 dimensions=(
"instrument",
"visit",),
682 srcList = connectionTypes.Input(
683 doc=
"Source catalogs used by PsfWcsSelectImages subtask to further select on PSF stability",
685 storageClass=
"SourceCatalog",
686 dimensions=(
"instrument",
"visit",
"detector"),
690 def __init__(self, *, config=None):
691 super().__init__(config=config)
692 if config.bgSubtracted:
693 self.inputs.remove(
"backgroundList")
694 if not config.doApplySkyCorr:
695 self.inputs.remove(
"skyCorrList")
696 if config.doApplyExternalSkyWcs:
697 if config.useGlobalExternalSkyWcs:
698 self.inputs.remove(
"externalSkyWcsTractCatalog")
700 self.inputs.remove(
"externalSkyWcsGlobalCatalog")
702 self.inputs.remove(
"externalSkyWcsTractCatalog")
703 self.inputs.remove(
"externalSkyWcsGlobalCatalog")
704 if config.doApplyExternalPhotoCalib:
705 if config.useGlobalExternalPhotoCalib:
706 self.inputs.remove(
"externalPhotoCalibTractCatalog")
708 self.inputs.remove(
"externalPhotoCalibGlobalCatalog")
710 self.inputs.remove(
"externalPhotoCalibTractCatalog")
711 self.inputs.remove(
"externalPhotoCalibGlobalCatalog")
712 if not config.makeDirect:
713 self.outputs.remove(
"direct")
714 if not config.makePsfMatched:
715 self.outputs.remove(
"psfMatched")
717 if config.select.target != lsst.pipe.tasks.selectImages.PsfWcsSelectImagesTask:
718 self.inputs.remove(
"visitSummary")
719 self.inputs.remove(
"srcList")
720 elif not config.select.doLegacyStarSelectionComputation:
722 self.inputs.remove(
"srcList")
726 pipelineConnections=MakeWarpConnections):
733 """Warp and optionally PSF-Match calexps onto an a common projection
735 ConfigClass = MakeWarpConfig
736 _DefaultName = "makeWarp"
738 @utils.inheritDoc(pipeBase.PipelineTask)
739 def runQuantum(self, butlerQC, inputRefs, outputRefs):
743 Construct warps for requested warp type
for single epoch
745 PipelineTask (Gen3) entry point to warp
and optionally PSF-match
746 calexps. This method
is analogous to `runDataRef`.
750 detectorOrder = [ref.datasetRef.dataId[
'detector']
for ref
in inputRefs.calExpList]
751 inputRefs = reorderRefs(inputRefs, detectorOrder, dataIdKey=
'detector')
754 inputs = butlerQC.get(inputRefs)
758 skyMap = inputs.pop(
"skyMap")
759 quantumDataId = butlerQC.quantum.dataId
760 skyInfo =
makeSkyInfo(skyMap, tractId=quantumDataId[
'tract'], patchId=quantumDataId[
'patch'])
763 dataIdList = [ref.datasetRef.dataId
for ref
in inputRefs.calExpList]
765 ccdIdList = [dataId.pack(
"visit_detector")
for dataId
in dataIdList]
770 coordList = [skyInfo.wcs.pixelToSky(pos)
for pos
in cornerPosList]
771 goodIndices = self.select.run(**inputs, coordList=coordList, dataIds=dataIdList)
772 inputs = self.filterInputs(indices=goodIndices, inputs=inputs)
775 inputs[
'calExpList'] = [ref.get()
for ref
in inputs[
'calExpList']]
778 visits = [dataId[
'visit']
for dataId
in dataIdList]
781 if self.config.doApplyExternalSkyWcs:
782 if self.config.useGlobalExternalSkyWcs:
783 externalSkyWcsCatalog = inputs.pop(
"externalSkyWcsGlobalCatalog")
785 externalSkyWcsCatalog = inputs.pop(
"externalSkyWcsTractCatalog")
787 externalSkyWcsCatalog =
None
789 if self.config.doApplyExternalPhotoCalib:
790 if self.config.useGlobalExternalPhotoCalib:
791 externalPhotoCalibCatalog = inputs.pop(
"externalPhotoCalibGlobalCatalog")
793 externalPhotoCalibCatalog = inputs.pop(
"externalPhotoCalibTractCatalog")
795 externalPhotoCalibCatalog =
None
797 completeIndices = self.prepareCalibratedExposures(**inputs,
798 externalSkyWcsCatalog=externalSkyWcsCatalog,
799 externalPhotoCalibCatalog=externalPhotoCalibCatalog)
801 inputs = self.filterInputs(indices=completeIndices, inputs=inputs)
803 results = self.run(**inputs, visitId=visitId,
804 ccdIdList=[ccdIdList[i]
for i
in goodIndices],
805 dataIdList=[dataIdList[i]
for i
in goodIndices],
807 if self.config.makeDirect
and results.exposures[
"direct"]
is not None:
808 butlerQC.put(results.exposures[
"direct"], outputRefs.direct)
809 if self.config.makePsfMatched
and results.exposures[
"psfMatched"]
is not None:
810 butlerQC.put(results.exposures[
"psfMatched"], outputRefs.psfMatched)
812 def filterInputs(self, indices, inputs):
813 """Return task inputs with their lists filtered by indices
817 indices : `list` of integers
818 inputs : `dict` of `list` of input connections to be passed to run
820 for key
in inputs.keys():
822 if isinstance(inputs[key], list):
823 inputs[key] = [inputs[key][ind]
for ind
in indices]
826 def prepareCalibratedExposures(self, calExpList, backgroundList=None, skyCorrList=None,
827 externalSkyWcsCatalog=None, externalPhotoCalibCatalog=None,
829 """Calibrate and add backgrounds to input calExpList in place
834 Sequence of calexps to be modified in place
835 backgroundList : `list` of `lsst.afw.math.backgroundList`, optional
836 Sequence of backgrounds to be added back
in if bgSubtracted=
False
837 skyCorrList : `list` of `lsst.afw.math.backgroundList`, optional
838 Sequence of background corrections to be subtracted
if doApplySkyCorr=
True
840 Exposure catalog
with external skyWcs to be applied
841 if config.doApplyExternalSkyWcs=
True. Catalog uses the detector id
842 for the catalog id, sorted on id
for fast lookup.
844 Exposure catalog
with external photoCalib to be applied
845 if config.doApplyExternalPhotoCalib=
True. Catalog uses the detector
846 id
for the catalog id, sorted on id
for fast lookup.
850 indices : `list` [`int`]
851 Indices of calExpList
and friends that have valid photoCalib/skyWcs
853 backgroundList = len(calExpList)*[None]
if backgroundList
is None else backgroundList
854 skyCorrList = len(calExpList)*[
None]
if skyCorrList
is None else skyCorrList
856 includeCalibVar = self.config.includeCalibVar
859 for index, (calexp, background, skyCorr)
in enumerate(zip(calExpList,
862 if not self.config.bgSubtracted:
863 calexp.maskedImage += background.getImage()
865 if externalSkyWcsCatalog
is not None or externalPhotoCalibCatalog
is not None:
866 detectorId = calexp.getInfo().getDetector().getId()
869 if externalPhotoCalibCatalog
is not None:
870 row = externalPhotoCalibCatalog.find(detectorId)
872 self.log.warning(
"Detector id %s not found in externalPhotoCalibCatalog "
873 "and will not be used in the warp.", detectorId)
875 photoCalib = row.getPhotoCalib()
876 if photoCalib
is None:
877 self.log.warning(
"Detector id %s has None for photoCalib in externalPhotoCalibCatalog "
878 "and will not be used in the warp.", detectorId)
880 calexp.setPhotoCalib(photoCalib)
882 photoCalib = calexp.getPhotoCalib()
883 if photoCalib
is None:
884 self.log.warning(
"Detector id %s has None for photoCalib in the calexp "
885 "and will not be used in the warp.", detectorId)
889 if externalSkyWcsCatalog
is not None:
890 row = externalSkyWcsCatalog.find(detectorId)
892 self.log.warning(
"Detector id %s not found in externalSkyWcsCatalog "
893 "and will not be used in the warp.", detectorId)
895 skyWcs = row.getWcs()
897 self.log.warning(
"Detector id %s has None for skyWcs in externalSkyWcsCatalog "
898 "and will not be used in the warp.", detectorId)
900 calexp.setWcs(skyWcs)
902 skyWcs = calexp.getWcs()
904 self.log.warning(
"Detector id %s has None for skyWcs in the calexp "
905 "and will not be used in the warp.", detectorId)
909 calexp.maskedImage = photoCalib.calibrateImage(calexp.maskedImage,
910 includeScaleUncertainty=includeCalibVar)
911 calexp.maskedImage /= photoCalib.getCalibrationMean()
916 if self.config.doApplySkyCorr:
917 calexp.maskedImage -= skyCorr.getImage()
919 indices.append(index)
924def reorderRefs(inputRefs, outputSortKeyOrder, dataIdKey):
925 """Reorder inputRefs per outputSortKeyOrder
927 Any inputRefs which are lists will be resorted per specified key e.g.,
928 'detector.' Only iterables will be reordered,
and values can be of type
929 `lsst.pipe.base.connections.DeferredDatasetRef`
or
930 `lsst.daf.butler.core.datasets.ref.DatasetRef`.
931 Returned lists of refs have the same length
as the outputSortKeyOrder.
932 If an outputSortKey
not in the inputRef, then it will be padded
with None.
933 If an inputRef contains an inputSortKey that
is not in the
934 outputSortKeyOrder it will be removed.
938 inputRefs : `lsst.pipe.base.connections.QuantizedConnection`
939 Input references to be reordered
and padded.
940 outputSortKeyOrder : iterable
941 Iterable of values to be compared
with inputRef
's dataId[dataIdKey]
943 dataIdKey in the dataRefs to compare
with the outputSortKeyOrder.
947 inputRefs: `lsst.pipe.base.connections.QuantizedConnection`
948 Quantized Connection
with sorted DatasetRef values sorted
if iterable.
950 for connectionName, refs
in inputRefs:
951 if isinstance(refs, Iterable):
952 if hasattr(refs[0],
"dataId"):
953 inputSortKeyOrder = [ref.dataId[dataIdKey]
for ref
in refs]
955 inputSortKeyOrder = [ref.datasetRef.dataId[dataIdKey]
for ref
in refs]
956 if inputSortKeyOrder != outputSortKeyOrder:
957 setattr(inputRefs, connectionName,
Configuration parameters for CoaddBaseTask.
Base class for coaddition.
def getTempExpDatasetName(self, warpType="direct")
def selectExposures(self, patchRef, skyInfo=None, selectDataList=[])
Select exposures to coadd.
def getCoaddDatasetName(self, warpType="direct")
def getSkyInfo(self, patchRef)
Use getSkyinfo to return the skyMap, tract and patch information, wcs and the outer bbox of the patch...
def getBadPixelMask(self)
Convenience method to provide the bitmask from the mask plane names.
Warp and optionally PSF-Match calexps onto an a common projection.
def getCalibratedExposure(self, dataRef, bgSubtracted)
def run(self, calExpList, ccdIdList, skyInfo, visitId=0, dataIdList=None, **kwargs)
def __init__(self, reuse=False, **kwargs)
def _prepareEmptyExposure(skyInfo)
def runDataRef(self, patchRef, selectDataList=[])
Produce <coaddName>Coadd_<warpType>Warp images by warping and optionally PSF-matching.
def getWarpTypeList(self)
def applySkyCorr(self, dataRef, calexp)
def reorderAndPadList(inputList, inputKeys, outputKeys, padWith=None)
def makeSkyInfo(skyMap, tractId, patchId)
def getGroupDataRef(butler, datasetType, groupTuple, keys)
def groupPatchExposures(patchDataRef, calexpDataRefList, coaddDatasetType="deepCoadd", tempExpDatasetType="deepCoadd_directWarp")