29import lsst.pipe.base.connectionTypes
as connectionTypes
30import lsst.utils
as utils
34from lsst.utils.timer
import timeMethod
35from .coaddBase
import CoaddBaseTask, makeSkyInfo, reorderAndPadList
36from .warpAndPsfMatch
import WarpAndPsfMatchTask
37from collections.abc
import Iterable
39__all__ = [
"MakeWarpTask",
"MakeWarpConfig"]
41log = logging.getLogger(__name__)
45 dimensions=(
"tract",
"patch",
"skymap",
"instrument",
"visit"),
46 defaultTemplates={
"coaddName":
"deep",
47 "skyWcsName":
"jointcal",
48 "photoCalibName":
"fgcm",
50 calExpList = connectionTypes.Input(
51 doc=
"Input exposures to be resampled and optionally PSF-matched onto a SkyMap projection/patch",
52 name=
"{calexpType}calexp",
53 storageClass=
"ExposureF",
54 dimensions=(
"instrument",
"visit",
"detector"),
58 backgroundList = connectionTypes.Input(
59 doc=
"Input backgrounds to be added back into the calexp if bgSubtracted=False",
60 name=
"calexpBackground",
61 storageClass=
"Background",
62 dimensions=(
"instrument",
"visit",
"detector"),
65 skyCorrList = connectionTypes.Input(
66 doc=
"Input Sky Correction to be subtracted from the calexp if doApplySkyCorr=True",
68 storageClass=
"Background",
69 dimensions=(
"instrument",
"visit",
"detector"),
72 skyMap = connectionTypes.Input(
73 doc=
"Input definition of geometry/bbox and projection/wcs for warped exposures",
74 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
75 storageClass=
"SkyMap",
76 dimensions=(
"skymap",),
78 externalSkyWcsTractCatalog = connectionTypes.Input(
79 doc=(
"Per-tract, per-visit wcs calibrations. These catalogs use the detector "
80 "id for the catalog id, sorted on id for fast lookup."),
81 name=
"{skyWcsName}SkyWcsCatalog",
82 storageClass=
"ExposureCatalog",
83 dimensions=(
"instrument",
"visit",
"tract"),
85 externalSkyWcsGlobalCatalog = connectionTypes.Input(
86 doc=(
"Per-visit wcs calibrations computed globally (with no tract information). "
87 "These catalogs use the detector id for the catalog id, sorted on id for "
89 name=
"{skyWcsName}SkyWcsCatalog",
90 storageClass=
"ExposureCatalog",
91 dimensions=(
"instrument",
"visit"),
93 externalPhotoCalibTractCatalog = connectionTypes.Input(
94 doc=(
"Per-tract, per-visit photometric calibrations. These catalogs use the "
95 "detector id for the catalog id, sorted on id for fast lookup."),
96 name=
"{photoCalibName}PhotoCalibCatalog",
97 storageClass=
"ExposureCatalog",
98 dimensions=(
"instrument",
"visit",
"tract"),
100 externalPhotoCalibGlobalCatalog = connectionTypes.Input(
101 doc=(
"Per-visit photometric calibrations computed globally (with no tract "
102 "information). These catalogs use the detector id for the catalog id, "
103 "sorted on id for fast lookup."),
104 name=
"{photoCalibName}PhotoCalibCatalog",
105 storageClass=
"ExposureCatalog",
106 dimensions=(
"instrument",
"visit"),
108 finalizedPsfApCorrCatalog = connectionTypes.Input(
109 doc=(
"Per-visit finalized psf models and aperture correction maps. "
110 "These catalogs use the detector id for the catalog id, "
111 "sorted on id for fast lookup."),
112 name=
"finalized_psf_ap_corr_catalog",
113 storageClass=
"ExposureCatalog",
114 dimensions=(
"instrument",
"visit"),
116 direct = connectionTypes.Output(
117 doc=(
"Output direct warped exposure (previously called CoaddTempExp), produced by resampling ",
118 "calexps onto the skyMap patch geometry."),
119 name=
"{coaddName}Coadd_directWarp",
120 storageClass=
"ExposureF",
121 dimensions=(
"tract",
"patch",
"skymap",
"visit",
"instrument"),
123 psfMatched = connectionTypes.Output(
124 doc=(
"Output PSF-Matched warped exposure (previously called CoaddTempExp), produced by resampling ",
125 "calexps onto the skyMap patch geometry and PSF-matching to a model PSF."),
126 name=
"{coaddName}Coadd_psfMatchedWarp",
127 storageClass=
"ExposureF",
128 dimensions=(
"tract",
"patch",
"skymap",
"visit",
"instrument"),
131 wcsList = connectionTypes.Input(
132 doc=
"WCSs of calexps used by SelectImages subtask to determine if the calexp overlaps the patch",
133 name=
"{calexpType}calexp.wcs",
135 dimensions=(
"instrument",
"visit",
"detector"),
138 bboxList = connectionTypes.Input(
139 doc=
"BBoxes of calexps used by SelectImages subtask to determine if the calexp overlaps the patch",
140 name=
"{calexpType}calexp.bbox",
141 storageClass=
"Box2I",
142 dimensions=(
"instrument",
"visit",
"detector"),
145 visitSummary = connectionTypes.Input(
146 doc=
"Consolidated exposure metadata from ConsolidateVisitSummaryTask",
147 name=
"{calexpType}visitSummary",
148 storageClass=
"ExposureCatalog",
149 dimensions=(
"instrument",
"visit",),
152 def __init__(self, *, config=None):
153 super().__init__(config=config)
154 if config.bgSubtracted:
155 self.inputs.remove(
"backgroundList")
156 if not config.doApplySkyCorr:
157 self.inputs.remove(
"skyCorrList")
158 if config.doApplyExternalSkyWcs:
159 if config.useGlobalExternalSkyWcs:
160 self.inputs.remove(
"externalSkyWcsTractCatalog")
162 self.inputs.remove(
"externalSkyWcsGlobalCatalog")
164 self.inputs.remove(
"externalSkyWcsTractCatalog")
165 self.inputs.remove(
"externalSkyWcsGlobalCatalog")
166 if config.doApplyExternalPhotoCalib:
167 if config.useGlobalExternalPhotoCalib:
168 self.inputs.remove(
"externalPhotoCalibTractCatalog")
170 self.inputs.remove(
"externalPhotoCalibGlobalCatalog")
172 self.inputs.remove(
"externalPhotoCalibTractCatalog")
173 self.inputs.remove(
"externalPhotoCalibGlobalCatalog")
174 if not config.doApplyFinalizedPsf:
175 self.inputs.remove(
"finalizedPsfApCorrCatalog")
176 if not config.makeDirect:
177 self.outputs.remove(
"direct")
178 if not config.makePsfMatched:
179 self.outputs.remove(
"psfMatched")
181 if config.select.target != lsst.pipe.tasks.selectImages.PsfWcsSelectImagesTask:
182 self.inputs.remove(
"visitSummary")
185class MakeWarpConfig(pipeBase.PipelineTaskConfig, CoaddBaseTask.ConfigClass,
186 pipelineConnections=MakeWarpConnections):
187 """Config for MakeWarpTask."""
188 warpAndPsfMatch = pexConfig.ConfigurableField(
189 target=WarpAndPsfMatchTask,
190 doc=
"Task to warp and PSF-match calexp",
192 doWrite = pexConfig.Field(
193 doc=
"persist <coaddName>Coadd_<warpType>Warp",
197 bgSubtracted = pexConfig.Field(
198 doc=
"Work with a background subtracted calexp?",
202 coaddPsf = pexConfig.ConfigField(
203 doc=
"Configuration for CoaddPsf",
204 dtype=CoaddPsfConfig,
206 makeDirect = pexConfig.Field(
207 doc=
"Make direct Warp/Coadds",
211 makePsfMatched = pexConfig.Field(
212 doc=
"Make Psf-Matched Warp/Coadd?",
216 doWriteEmptyWarps = pexConfig.Field(
219 doc=
"Write out warps even if they are empty"
221 hasFakes = pexConfig.Field(
222 doc=
"Should be set to True if fake sources have been inserted into the input data.",
226 doApplySkyCorr = pexConfig.Field(
229 doc=
"Apply sky correction?",
231 doApplyFinalizedPsf = pexConfig.Field(
232 doc=
"Whether to apply finalized psf models and aperture correction map.",
238 CoaddBaseTask.ConfigClass.validate(self)
240 if not self.makePsfMatched
and not self.makeDirect:
241 raise RuntimeError(
"At least one of config.makePsfMatched and config.makeDirect must be True")
244 log.warning(
"Config doPsfMatch deprecated. Setting makePsfMatched=True and makeDirect=False")
245 self.makePsfMatched =
True
246 self.makeDirect =
False
248 def setDefaults(self):
249 CoaddBaseTask.ConfigClass.setDefaults(self)
250 self.warpAndPsfMatch.psfMatch.kernel.active.kernelSize = self.matchingKernelSize
254 """Warp and optionally PSF-Match calexps onto an a common projection
256 ConfigClass = MakeWarpConfig
257 _DefaultName = "makeWarp"
259 def __init__(self, **kwargs):
260 CoaddBaseTask.__init__(self, **kwargs)
261 self.makeSubtask(
"warpAndPsfMatch")
262 if self.config.hasFakes:
263 self.calexpType =
"fakes_calexp"
265 self.calexpType =
"calexp"
267 @utils.inheritDoc(pipeBase.PipelineTask)
268 def runQuantum(self, butlerQC, inputRefs, outputRefs):
272 Construct warps for requested warp type
for single epoch
277 detectorOrder = [ref.datasetRef.dataId[
'detector']
for ref
in inputRefs.calExpList]
279 inputRefs = reorderRefs(inputRefs, detectorOrder, dataIdKey=
'detector')
282 inputs = butlerQC.get(inputRefs)
286 skyMap = inputs.pop(
"skyMap")
287 quantumDataId = butlerQC.quantum.dataId
288 skyInfo = makeSkyInfo(skyMap, tractId=quantumDataId[
'tract'], patchId=quantumDataId[
'patch'])
291 dataIdList = [ref.datasetRef.dataId
for ref
in inputRefs.calExpList]
293 ccdIdList = [dataId.pack(
"visit_detector")
for dataId
in dataIdList]
298 coordList = [skyInfo.wcs.pixelToSky(pos)
for pos
in cornerPosList]
299 goodIndices = self.select.run(**inputs, coordList=coordList, dataIds=dataIdList)
300 inputs = self.filterInputs(indices=goodIndices, inputs=inputs)
303 inputs[
'calExpList'] = [ref.get()
for ref
in inputs[
'calExpList']]
306 visits = [dataId[
'visit']
for dataId
in dataIdList]
309 if self.config.doApplyExternalSkyWcs:
310 if self.config.useGlobalExternalSkyWcs:
311 externalSkyWcsCatalog = inputs.pop(
"externalSkyWcsGlobalCatalog")
313 externalSkyWcsCatalog = inputs.pop(
"externalSkyWcsTractCatalog")
315 externalSkyWcsCatalog =
None
317 if self.config.doApplyExternalPhotoCalib:
318 if self.config.useGlobalExternalPhotoCalib:
319 externalPhotoCalibCatalog = inputs.pop(
"externalPhotoCalibGlobalCatalog")
321 externalPhotoCalibCatalog = inputs.pop(
"externalPhotoCalibTractCatalog")
323 externalPhotoCalibCatalog =
None
325 if self.config.doApplyFinalizedPsf:
326 finalizedPsfApCorrCatalog = inputs.pop(
"finalizedPsfApCorrCatalog")
328 finalizedPsfApCorrCatalog =
None
330 completeIndices = self.prepareCalibratedExposures(**inputs,
331 externalSkyWcsCatalog=externalSkyWcsCatalog,
332 externalPhotoCalibCatalog=externalPhotoCalibCatalog,
333 finalizedPsfApCorrCatalog=finalizedPsfApCorrCatalog)
335 inputs = self.filterInputs(indices=completeIndices, inputs=inputs)
337 results = self.run(**inputs, visitId=visitId,
338 ccdIdList=[ccdIdList[i]
for i
in goodIndices],
339 dataIdList=[dataIdList[i]
for i
in goodIndices],
341 if self.config.makeDirect
and results.exposures[
"direct"]
is not None:
342 butlerQC.put(results.exposures[
"direct"], outputRefs.direct)
343 if self.config.makePsfMatched
and results.exposures[
"psfMatched"]
is not None:
344 butlerQC.put(results.exposures[
"psfMatched"], outputRefs.psfMatched)
347 def run(self, calExpList, ccdIdList, skyInfo, visitId=0, dataIdList=None, **kwargs):
348 """Create a Warp from inputs
350 We iterate over the multiple calexps in a single exposure to construct
351 the warp (previously called a coaddTempExp) of that exposure to the
352 supplied tract/patch.
354 Pixels that receive no pixels are set to NAN; this
is not correct
355 (violates LSST algorithms group policy), but will be fixed up by
356 interpolating after the coaddition.
358 @param calexpRefList: List of data references
for calexps that (may)
359 overlap the patch of interest
360 @param skyInfo: Struct
from CoaddBaseTask.getSkyInfo()
with geometric
361 information about the patch
362 @param visitId: integer identifier
for visit,
for the table that will
364 @return a pipeBase Struct containing:
365 - exposures: a dictionary containing the warps requested:
366 "direct": direct warp
if config.makeDirect
367 "psfMatched": PSF-matched warp
if config.makePsfMatched
369 warpTypeList = self.getWarpTypeList()
371 totGoodPix = {warpType: 0 for warpType
in warpTypeList}
372 didSetMetadata = {warpType:
False for warpType
in warpTypeList}
373 warps = {warpType: self._prepareEmptyExposure(skyInfo)
for warpType
in warpTypeList}
374 inputRecorder = {warpType: self.inputRecorder.makeCoaddTempExpRecorder(visitId, len(calExpList))
375 for warpType
in warpTypeList}
377 modelPsf = self.config.modelPsf.apply()
if self.config.makePsfMatched
else None
378 if dataIdList
is None:
379 dataIdList = ccdIdList
381 for calExpInd, (calExp, ccdId, dataId)
in enumerate(zip(calExpList, ccdIdList, dataIdList)):
382 self.log.info(
"Processing calexp %d of %d for this Warp: id=%s",
383 calExpInd+1, len(calExpList), dataId)
386 warpedAndMatched = self.warpAndPsfMatch.run(calExp, modelPsf=modelPsf,
387 wcs=skyInfo.wcs, maxBBox=skyInfo.bbox,
388 makeDirect=self.config.makeDirect,
389 makePsfMatched=self.config.makePsfMatched)
390 except Exception
as e:
391 self.log.warning(
"WarpAndPsfMatch failed for calexp %s; skipping it: %s", dataId, e)
394 numGoodPix = {warpType: 0
for warpType
in warpTypeList}
395 for warpType
in warpTypeList:
396 exposure = warpedAndMatched.getDict()[warpType]
399 warp = warps[warpType]
400 if didSetMetadata[warpType]:
401 mimg = exposure.getMaskedImage()
402 mimg *= (warp.getPhotoCalib().getInstFluxAtZeroMagnitude()
403 / exposure.getPhotoCalib().getInstFluxAtZeroMagnitude())
405 numGoodPix[warpType] = coaddUtils.copyGoodPixels(
406 warp.getMaskedImage(), exposure.getMaskedImage(), self.getBadPixelMask())
407 totGoodPix[warpType] += numGoodPix[warpType]
408 self.log.debug(
"Calexp %s has %d good pixels in this patch (%.1f%%) for %s",
409 dataId, numGoodPix[warpType],
410 100.0*numGoodPix[warpType]/skyInfo.bbox.getArea(), warpType)
411 if numGoodPix[warpType] > 0
and not didSetMetadata[warpType]:
412 warp.info.id = exposure.info.id
413 warp.setPhotoCalib(exposure.getPhotoCalib())
414 warp.setFilter(exposure.getFilter())
415 warp.getInfo().setVisitInfo(exposure.getInfo().getVisitInfo())
417 warp.setPsf(exposure.getPsf())
418 didSetMetadata[warpType] =
True
421 inputRecorder[warpType].addCalExp(calExp, ccdId, numGoodPix[warpType])
423 except Exception
as e:
424 self.log.warning(
"Error processing calexp %s; skipping it: %s", dataId, e)
427 for warpType
in warpTypeList:
428 self.log.info(
"%sWarp has %d good pixels (%.1f%%)",
429 warpType, totGoodPix[warpType], 100.0*totGoodPix[warpType]/skyInfo.bbox.getArea())
431 if totGoodPix[warpType] > 0
and didSetMetadata[warpType]:
432 inputRecorder[warpType].finish(warps[warpType], totGoodPix[warpType])
433 if warpType ==
"direct":
434 warps[warpType].setPsf(
435 CoaddPsf(inputRecorder[warpType].coaddInputs.ccds, skyInfo.wcs,
436 self.config.coaddPsf.makeControl()))
438 if not self.config.doWriteEmptyWarps:
440 warps[warpType] =
None
444 result = pipeBase.Struct(exposures=warps)
447 def filterInputs(self, indices, inputs):
448 """Return task inputs with their lists filtered by indices
452 indices : `list` of integers
453 inputs : `dict` of `list` of input connections to be passed to run
455 for key
in inputs.keys():
457 if isinstance(inputs[key], list):
458 inputs[key] = [inputs[key][ind]
for ind
in indices]
461 def prepareCalibratedExposures(self, calExpList, backgroundList=None, skyCorrList=None,
462 externalSkyWcsCatalog=None, externalPhotoCalibCatalog=None,
463 finalizedPsfApCorrCatalog=None,
465 """Calibrate and add backgrounds to input calExpList in place
470 Sequence of calexps to be modified in place
471 backgroundList : `list` of `lsst.afw.math.backgroundList`, optional
472 Sequence of backgrounds to be added back
in if bgSubtracted=
False
473 skyCorrList : `list` of `lsst.afw.math.backgroundList`, optional
474 Sequence of background corrections to be subtracted
if doApplySkyCorr=
True
476 Exposure catalog
with external skyWcs to be applied
477 if config.doApplyExternalSkyWcs=
True. Catalog uses the detector id
478 for the catalog id, sorted on id
for fast lookup.
480 Exposure catalog
with external photoCalib to be applied
481 if config.doApplyExternalPhotoCalib=
True. Catalog uses the detector
482 id
for the catalog id, sorted on id
for fast lookup.
484 Exposure catalog
with finalized psf models
and aperture correction
485 maps to be applied
if config.doApplyFinalizedPsf=
True. Catalog uses
486 the detector id
for the catalog id, sorted on id
for fast lookup.
490 indices : `list` [`int`]
491 Indices of calExpList
and friends that have valid photoCalib/skyWcs
493 backgroundList = len(calExpList)*[None]
if backgroundList
is None else backgroundList
494 skyCorrList = len(calExpList)*[
None]
if skyCorrList
is None else skyCorrList
496 includeCalibVar = self.config.includeCalibVar
499 for index, (calexp, background, skyCorr)
in enumerate(zip(calExpList,
502 if not self.config.bgSubtracted:
503 calexp.maskedImage += background.getImage()
505 detectorId = calexp.getInfo().getDetector().getId()
508 if externalPhotoCalibCatalog
is not None:
509 row = externalPhotoCalibCatalog.find(detectorId)
511 self.log.warning(
"Detector id %s not found in externalPhotoCalibCatalog "
512 "and will not be used in the warp.", detectorId)
514 photoCalib = row.getPhotoCalib()
515 if photoCalib
is None:
516 self.log.warning(
"Detector id %s has None for photoCalib in externalPhotoCalibCatalog "
517 "and will not be used in the warp.", detectorId)
519 calexp.setPhotoCalib(photoCalib)
521 photoCalib = calexp.getPhotoCalib()
522 if photoCalib
is None:
523 self.log.warning(
"Detector id %s has None for photoCalib in the calexp "
524 "and will not be used in the warp.", detectorId)
528 if externalSkyWcsCatalog
is not None:
529 row = externalSkyWcsCatalog.find(detectorId)
531 self.log.warning(
"Detector id %s not found in externalSkyWcsCatalog "
532 "and will not be used in the warp.", detectorId)
534 skyWcs = row.getWcs()
536 self.log.warning(
"Detector id %s has None for skyWcs in externalSkyWcsCatalog "
537 "and will not be used in the warp.", detectorId)
539 calexp.setWcs(skyWcs)
541 skyWcs = calexp.getWcs()
543 self.log.warning(
"Detector id %s has None for skyWcs in the calexp "
544 "and will not be used in the warp.", detectorId)
548 if finalizedPsfApCorrCatalog
is not None:
549 row = finalizedPsfApCorrCatalog.find(detectorId)
551 self.log.warning(
"Detector id %s not found in finalizedPsfApCorrCatalog "
552 "and will not be used in the warp.", detectorId)
556 self.log.warning(
"Detector id %s has None for psf in finalizedPsfApCorrCatalog "
557 "and will not be used in the warp.", detectorId)
560 apCorrMap = row.getApCorrMap()
561 if apCorrMap
is None:
562 self.log.warning(
"Detector id %s has None for ApCorrMap in finalizedPsfApCorrCatalog "
563 "and will not be used in the warp.", detectorId)
565 calexp.info.setApCorrMap(apCorrMap)
568 calexp.maskedImage = photoCalib.calibrateImage(calexp.maskedImage,
569 includeScaleUncertainty=includeCalibVar)
570 calexp.maskedImage /= photoCalib.getCalibrationMean()
575 if self.config.doApplySkyCorr:
576 calexp.maskedImage -= skyCorr.getImage()
578 indices.append(index)
583 def _prepareEmptyExposure(skyInfo):
584 """Produce an empty exposure for a given patch"""
585 exp = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
586 exp.getMaskedImage().set(numpy.nan, afwImage.Mask
587 .getPlaneBitMask(
"NO_DATA"), numpy.inf)
590 def getWarpTypeList(self):
591 """Return list of requested warp types per the config.
594 if self.config.makeDirect:
595 warpTypeList.append(
"direct")
596 if self.config.makePsfMatched:
597 warpTypeList.append(
"psfMatched")
601def reorderRefs(inputRefs, outputSortKeyOrder, dataIdKey):
602 """Reorder inputRefs per outputSortKeyOrder
604 Any inputRefs which are lists will be resorted per specified key e.g.,
605 'detector.' Only iterables will be reordered,
and values can be of type
606 `lsst.pipe.base.connections.DeferredDatasetRef`
or
607 `lsst.daf.butler.core.datasets.ref.DatasetRef`.
608 Returned lists of refs have the same length
as the outputSortKeyOrder.
609 If an outputSortKey
not in the inputRef, then it will be padded
with None.
610 If an inputRef contains an inputSortKey that
is not in the
611 outputSortKeyOrder it will be removed.
615 inputRefs : `lsst.pipe.base.connections.QuantizedConnection`
616 Input references to be reordered
and padded.
617 outputSortKeyOrder : iterable
618 Iterable of values to be compared
with inputRef
's dataId[dataIdKey]
620 dataIdKey in the dataRefs to compare
with the outputSortKeyOrder.
624 inputRefs: `lsst.pipe.base.connections.QuantizedConnection`
625 Quantized Connection
with sorted DatasetRef values sorted
if iterable.
627 for connectionName, refs
in inputRefs:
628 if isinstance(refs, Iterable):
629 if hasattr(refs[0],
"dataId"):
630 inputSortKeyOrder = [ref.dataId[dataIdKey]
for ref
in refs]
632 inputSortKeyOrder = [ref.datasetRef.dataId[dataIdKey]
for ref
in refs]
633 if inputSortKeyOrder != outputSortKeyOrder:
634 setattr(inputRefs, connectionName,
635 reorderAndPadList(refs, inputSortKeyOrder, outputSortKeyOrder))
Base class for coaddition.