lsst.pipe.tasks g74b9dd4314+e6fb14f238
Loading...
Searching...
No Matches
makeWarp.py
Go to the documentation of this file.
1# This file is part of pipe_tasks.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21
22__all__ = ["MakeWarpTask", "MakeWarpConfig"]
23
24from deprecated.sphinx import deprecated
25import logging
26import numpy
27
28import lsst.pex.config as pexConfig
29import lsst.afw.image as afwImage
30import lsst.coadd.utils as coaddUtils
31import lsst.pipe.base as pipeBase
32import lsst.pipe.base.connectionTypes as connectionTypes
33import lsst.utils as utils
34import lsst.geom
35from lsst.daf.butler import DeferredDatasetHandle
36from lsst.meas.base import DetectorVisitIdGeneratorConfig
37from lsst.meas.algorithms import CoaddPsf, CoaddPsfConfig
38from lsst.skymap import BaseSkyMap
39from lsst.utils.timer import timeMethod
40from .coaddBase import CoaddBaseTask, makeSkyInfo, reorderAndPadList
41from .warpAndPsfMatch import WarpAndPsfMatchTask
42from collections.abc import Iterable
43
44log = logging.getLogger(__name__)
45
46
47class MakeWarpConnections(pipeBase.PipelineTaskConnections,
48 dimensions=("tract", "patch", "skymap", "instrument", "visit"),
49 defaultTemplates={"coaddName": "deep",
50 "skyWcsName": "gbdesAstrometricFit",
51 "photoCalibName": "fgcm",
52 "calexpType": ""}):
53 calExpList = connectionTypes.Input(
54 doc="Input exposures to be resampled and optionally PSF-matched onto a SkyMap projection/patch",
55 name="{calexpType}calexp",
56 storageClass="ExposureF",
57 dimensions=("instrument", "visit", "detector"),
58 multiple=True,
59 deferLoad=True,
60 )
61 backgroundList = connectionTypes.Input(
62 doc="Input backgrounds to be added back into the calexp if bgSubtracted=False",
63 name="calexpBackground",
64 storageClass="Background",
65 dimensions=("instrument", "visit", "detector"),
66 multiple=True,
67 )
68 skyCorrList = connectionTypes.Input(
69 doc="Input Sky Correction to be subtracted from the calexp if doApplySkyCorr=True",
70 name="skyCorr",
71 storageClass="Background",
72 dimensions=("instrument", "visit", "detector"),
73 multiple=True,
74 )
75 skyMap = connectionTypes.Input(
76 doc="Input definition of geometry/bbox and projection/wcs for warped exposures",
77 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
78 storageClass="SkyMap",
79 dimensions=("skymap",),
80 )
81 externalSkyWcsTractCatalog = connectionTypes.Input(
82 doc=("Per-tract, per-visit wcs calibrations. These catalogs use the detector "
83 "id for the catalog id, sorted on id for fast lookup."),
84 name="{skyWcsName}SkyWcsCatalog",
85 storageClass="ExposureCatalog",
86 dimensions=("instrument", "visit", "tract"),
87 )
88 externalSkyWcsGlobalCatalog = connectionTypes.Input(
89 doc=("Per-visit wcs calibrations computed globally (with no tract information). "
90 "These catalogs use the detector id for the catalog id, sorted on id for "
91 "fast lookup."),
92 name="finalVisitSummary",
93 storageClass="ExposureCatalog",
94 dimensions=("instrument", "visit"),
95 )
96 externalPhotoCalibTractCatalog = connectionTypes.Input(
97 doc=("Per-tract, per-visit photometric calibrations. These catalogs use the "
98 "detector id for the catalog id, sorted on id for fast lookup."),
99 name="{photoCalibName}PhotoCalibCatalog",
100 storageClass="ExposureCatalog",
101 dimensions=("instrument", "visit", "tract"),
102 )
103 externalPhotoCalibGlobalCatalog = connectionTypes.Input(
104 doc=("Per-visit photometric calibrations computed globally (with no tract "
105 "information). These catalogs use the detector id for the catalog id, "
106 "sorted on id for fast lookup."),
107 name="finalVisitSummary",
108 storageClass="ExposureCatalog",
109 dimensions=("instrument", "visit"),
110 )
111 finalizedPsfApCorrCatalog = connectionTypes.Input(
112 doc=("Per-visit finalized psf models and aperture correction maps. "
113 "These catalogs use the detector id for the catalog id, "
114 "sorted on id for fast lookup."),
115 name="finalVisitSummary",
116 storageClass="ExposureCatalog",
117 dimensions=("instrument", "visit"),
118 )
119 direct = connectionTypes.Output(
120 doc=("Output direct warped exposure (previously called CoaddTempExp), produced by resampling ",
121 "calexps onto the skyMap patch geometry."),
122 name="{coaddName}Coadd_directWarp",
123 storageClass="ExposureF",
124 dimensions=("tract", "patch", "skymap", "visit", "instrument"),
125 )
126 psfMatched = connectionTypes.Output(
127 doc=("Output PSF-Matched warped exposure (previously called CoaddTempExp), produced by resampling ",
128 "calexps onto the skyMap patch geometry and PSF-matching to a model PSF."),
129 name="{coaddName}Coadd_psfMatchedWarp",
130 storageClass="ExposureF",
131 dimensions=("tract", "patch", "skymap", "visit", "instrument"),
132 )
133 # TODO DM-28769, have selectImages subtask indicate which connections they
134 # need:
135 wcsList = connectionTypes.Input(
136 doc="WCSs of calexps used by SelectImages subtask to determine if the calexp overlaps the patch",
137 name="{calexpType}calexp.wcs",
138 storageClass="Wcs",
139 dimensions=("instrument", "visit", "detector"),
140 multiple=True,
141 )
142 bboxList = connectionTypes.Input(
143 doc="BBoxes of calexps used by SelectImages subtask to determine if the calexp overlaps the patch",
144 name="{calexpType}calexp.bbox",
145 storageClass="Box2I",
146 dimensions=("instrument", "visit", "detector"),
147 multiple=True,
148 )
149 visitSummary = connectionTypes.Input(
150 doc="Consolidated exposure metadata",
151 name="finalVisitSummary",
152 storageClass="ExposureCatalog",
153 dimensions=("instrument", "visit",),
154 )
155
156 def __init__(self, *, config=None):
157 super().__init__(config=config)
158 if config.bgSubtracted:
159 self.inputs.remove("backgroundList")
160 if not config.doApplySkyCorr:
161 self.inputs.remove("skyCorrList")
162 if config.doApplyExternalSkyWcs:
163 if config.useGlobalExternalSkyWcs:
164 self.inputs.remove("externalSkyWcsTractCatalog")
165 else:
166 self.inputs.remove("externalSkyWcsGlobalCatalog")
167 else:
168 self.inputs.remove("externalSkyWcsTractCatalog")
169 self.inputs.remove("externalSkyWcsGlobalCatalog")
170 if config.doApplyExternalPhotoCalib:
171 if config.useGlobalExternalPhotoCalib:
172 self.inputs.remove("externalPhotoCalibTractCatalog")
173 else:
174 self.inputs.remove("externalPhotoCalibGlobalCatalog")
175 else:
176 self.inputs.remove("externalPhotoCalibTractCatalog")
177 self.inputs.remove("externalPhotoCalibGlobalCatalog")
178 if not config.doApplyFinalizedPsf:
179 self.inputs.remove("finalizedPsfApCorrCatalog")
180 if not config.makeDirect:
181 self.outputs.remove("direct")
182 if not config.makePsfMatched:
183 self.outputs.remove("psfMatched")
184 # TODO DM-28769: add connection per selectImages connections
185 if config.select.target != lsst.pipe.tasks.selectImages.PsfWcsSelectImagesTask:
186 self.inputs.remove("visitSummary")
187
188
189class MakeWarpConfig(pipeBase.PipelineTaskConfig, CoaddBaseTask.ConfigClass,
190 pipelineConnections=MakeWarpConnections):
191 """Config for MakeWarpTask."""
192
193 warpAndPsfMatch = pexConfig.ConfigurableField(
194 target=WarpAndPsfMatchTask,
195 doc="Task to warp and PSF-match calexp",
196 )
197 doWrite = pexConfig.Field(
198 doc="persist <coaddName>Coadd_<warpType>Warp",
199 dtype=bool,
200 default=True,
201 )
202 bgSubtracted = pexConfig.Field(
203 doc="Work with a background subtracted calexp?",
204 dtype=bool,
205 default=True,
206 )
207 coaddPsf = pexConfig.ConfigField(
208 doc="Configuration for CoaddPsf",
209 dtype=CoaddPsfConfig,
210 )
211 makeDirect = pexConfig.Field(
212 doc="Make direct Warp/Coadds",
213 dtype=bool,
214 default=True,
215 )
216 makePsfMatched = pexConfig.Field(
217 doc="Make Psf-Matched Warp/Coadd?",
218 dtype=bool,
219 default=False,
220 )
221 doWriteEmptyWarps = pexConfig.Field(
222 dtype=bool,
223 default=False,
224 doc="Write out warps even if they are empty"
225 )
226 hasFakes = pexConfig.Field(
227 doc="Should be set to True if fake sources have been inserted into the input data.",
228 dtype=bool,
229 default=False,
230 )
231 doApplySkyCorr = pexConfig.Field(
232 dtype=bool,
233 default=False,
234 doc="Apply sky correction?",
235 )
236 doApplyFinalizedPsf = pexConfig.Field(
237 doc="Whether to apply finalized psf models and aperture correction map.",
238 dtype=bool,
239 default=True,
240 )
241 idGenerator = DetectorVisitIdGeneratorConfig.make_field()
242
243 def validate(self):
244 CoaddBaseTask.ConfigClass.validate(self)
245
246 if not self.makePsfMatched and not self.makeDirect:
247 raise RuntimeError("At least one of config.makePsfMatched and config.makeDirect must be True")
248 if self.doPsfMatch:
249 # Backwards compatibility.
250 log.warning("Config doPsfMatch deprecated. Setting makePsfMatched=True and makeDirect=False")
251 self.makePsfMatched = True
252 self.makeDirect = False
253
254 def setDefaults(self):
255 CoaddBaseTask.ConfigClass.setDefaults(self)
256 self.warpAndPsfMatch.psfMatch.kernel.active.kernelSize = self.matchingKernelSize
257
258
259class MakeWarpTask(CoaddBaseTask):
260 """Warp and optionally PSF-Match calexps onto an a common projection.
261
262 Warp and optionally PSF-Match calexps onto a common projection, by
263 performing the following operations:
264 - Group calexps by visit/run
265 - For each visit, generate a Warp by calling method @ref run.
266 `run` loops over the visit's calexps calling
268
269 Notes
270 -----
271 WarpType identifies the types of convolutions applied to Warps
272 (previously CoaddTempExps). Only two types are available: direct
273 (for regular Warps/Coadds) and psfMatched(for Warps/Coadds with
274 homogenized PSFs). We expect to add a third type, likelihood, for
275 generating likelihood Coadds with Warps that have been correlated with
276 their own PSF.
277
278 To make `psfMatchedWarps`, select `config.makePsfMatched=True`. The subtask
280 is responsible for the PSF-Matching, and its config is accessed via
281 `config.warpAndPsfMatch.psfMatch`.
282
283 The optimal configuration depends on aspects of dataset: the pixel scale,
284 average PSF FWHM and dimensions of the PSF kernel. These configs include
285 the requested model PSF, the matching kernel size, padding of the science
286 PSF thumbnail and spatial sampling frequency of the PSF.
287
288 *Config Guidelines*: The user must specify the size of the model PSF to
289 which to match by setting `config.modelPsf.defaultFwhm` in units of pixels.
290 The appropriate values depends on science case. In general, for a set of
291 input images, this config should equal the FWHM of the visit with the worst
292 seeing. The smallest it should be set to is the median FWHM. The defaults
293 of the other config options offer a reasonable starting point.
294
295 The following list presents the most common problems that arise from a
296 misconfigured
297 @link ip::diffim::modelPsfMatch::ModelPsfMatchTask ModelPsfMatchTask
298 @endlink
299 and corresponding solutions. All assume the default Alard-Lupton kernel,
300 with configs accessed via
301 ```config.warpAndPsfMatch.psfMatch.kernel['AL']```. Each item in the list
302 is formatted as:
303 Problem: Explanation. *Solution*
304
305 *Troublshooting PSF-Matching Configuration:*
306 - Matched PSFs look boxy: The matching kernel is too small.
307 _Increase the matching kernel size.
308 For example:_
309 config.warpAndPsfMatch.psfMatch.kernel['AL'].kernelSize=27
310 # default 21
311 Note that increasing the kernel size also increases runtime.
312 - Matched PSFs look ugly (dipoles, quadropoles, donuts): unable to find
313 good solution for matching kernel.
314 _Provide the matcher with more data by either increasing
315 the spatial sampling by decreasing the spatial cell size,_
316 config.warpAndPsfMatch.psfMatch.kernel['AL'].sizeCellX = 64
317 # default 128
318 config.warpAndPsfMatch.psfMatch.kernel['AL'].sizeCellY = 64
319 # default 128
320 _or increasing the padding around the Science PSF, for example:_
321 config.warpAndPsfMatch.psfMatch.autoPadPsfTo=1.6 # default 1.4
322 Increasing `autoPadPsfTo` increases the minimum ratio of input PSF
323 dimensions to the matching kernel dimensions, thus increasing the
324 number of pixels available to fit after convolving the PSF with the
325 matching kernel. Optionally, for debugging the effects of padding, the
326 level of padding may be manually controlled by setting turning off the
327 automatic padding and setting the number of pixels by which to pad the
328 PSF:
329 config.warpAndPsfMatch.psfMatch.doAutoPadPsf = False
330 # default True
331 config.warpAndPsfMatch.psfMatch.padPsfBy = 6
332 # pixels. default 0
333 - Deconvolution: Matching a large PSF to a smaller PSF produces
334 a telltale noise pattern which looks like ripples or a brain.
335 _Increase the size of the requested model PSF. For example:_
336 config.modelPsf.defaultFwhm = 11 # Gaussian sigma in units of
337 pixels.
338 - High frequency (sometimes checkered) noise: The matching basis functions
339 are too small.
340 _Increase the width of the Gaussian basis functions. For example:_
341 config.warpAndPsfMatch.psfMatch.kernel['AL'].alardSigGauss=
342 [1.5, 3.0, 6.0] # from default [0.7, 1.5, 3.0]
343 """
344 ConfigClass = MakeWarpConfig
345 _DefaultName = "makeWarp"
346
347 def __init__(self, **kwargs):
348 CoaddBaseTask.__init__(self, **kwargs)
349 self.makeSubtask("warpAndPsfMatch")
350 if self.config.hasFakes:
351 self.calexpType = "fakes_calexp"
352 else:
353 self.calexpType = "calexp"
354
355 @utils.inheritDoc(pipeBase.PipelineTask)
356 def runQuantum(self, butlerQC, inputRefs, outputRefs):
357 # Obtain the list of input detectors from calExpList. Sort them by
358 # detector order (to ensure reproducibility). Then ensure all input
359 # lists are in the same sorted detector order.
360 detectorOrder = [ref.datasetRef.dataId['detector'] for ref in inputRefs.calExpList]
361 detectorOrder.sort()
362 inputRefs = reorderRefs(inputRefs, detectorOrder, dataIdKey='detector')
363
364 # Read in all inputs.
365 inputs = butlerQC.get(inputRefs)
366
367 # Construct skyInfo expected by `run`. We remove the SkyMap itself
368 # from the dictionary so we can pass it as kwargs later.
369 skyMap = inputs.pop("skyMap")
370 quantumDataId = butlerQC.quantum.dataId
371 skyInfo = makeSkyInfo(skyMap, tractId=quantumDataId['tract'], patchId=quantumDataId['patch'])
372
373 # Construct list of input DataIds expected by `run`.
374 dataIdList = [ref.datasetRef.dataId for ref in inputRefs.calExpList]
375 # Construct list of packed integer IDs expected by `run`.
376 ccdIdList = [
377 self.config.idGenerator.apply(dataId).catalog_id
378 for dataId in dataIdList
379 ]
380
381 if self.config.doApplyExternalSkyWcs:
382 if self.config.useGlobalExternalSkyWcs:
383 externalSkyWcsCatalog = inputs.pop("externalSkyWcsGlobalCatalog")
384 else:
385 externalSkyWcsCatalog = inputs.pop("externalSkyWcsTractCatalog")
386 else:
387 externalSkyWcsCatalog = None
388
389 if self.config.doApplyExternalPhotoCalib:
390 if self.config.useGlobalExternalPhotoCalib:
391 externalPhotoCalibCatalog = inputs.pop("externalPhotoCalibGlobalCatalog")
392 else:
393 externalPhotoCalibCatalog = inputs.pop("externalPhotoCalibTractCatalog")
394 else:
395 externalPhotoCalibCatalog = None
396
397 if self.config.doApplyFinalizedPsf:
398 finalizedPsfApCorrCatalog = inputs.pop("finalizedPsfApCorrCatalog")
399 else:
400 finalizedPsfApCorrCatalog = None
401
402 # Do an initial selection on inputs with complete wcs/photoCalib info.
403 # Qualifying calexps will be read in the following call.
404 completeIndices = self._prepareCalibratedExposures(
405 **inputs,
406 externalSkyWcsCatalog=externalSkyWcsCatalog,
407 externalPhotoCalibCatalog=externalPhotoCalibCatalog,
408 finalizedPsfApCorrCatalog=finalizedPsfApCorrCatalog)
409 inputs = self.filterInputs(indices=completeIndices, inputs=inputs)
410
411 # Do another selection based on the configured selection task
412 # (using updated WCSs to determine patch overlap if an external
413 # calibration was applied).
414 cornerPosList = lsst.geom.Box2D(skyInfo.bbox).getCorners()
415 coordList = [skyInfo.wcs.pixelToSky(pos) for pos in cornerPosList]
416 goodIndices = self.select.run(**inputs, coordList=coordList, dataIds=dataIdList)
417 inputs = self.filterInputs(indices=goodIndices, inputs=inputs)
418
419 # Extract integer visitId requested by `run`.
420 visitId = dataIdList[0]["visit"]
421
422 results = self.run(**inputs, visitId=visitId,
423 ccdIdList=[ccdIdList[i] for i in goodIndices],
424 dataIdList=[dataIdList[i] for i in goodIndices],
425 skyInfo=skyInfo)
426 if self.config.makeDirect and results.exposures["direct"] is not None:
427 butlerQC.put(results.exposures["direct"], outputRefs.direct)
428 if self.config.makePsfMatched and results.exposures["psfMatched"] is not None:
429 butlerQC.put(results.exposures["psfMatched"], outputRefs.psfMatched)
430
431 @timeMethod
432 def run(self, calExpList, ccdIdList, skyInfo, visitId=0, dataIdList=None, **kwargs):
433 """Create a Warp from inputs.
434
435 We iterate over the multiple calexps in a single exposure to construct
436 the warp (previously called a coaddTempExp) of that exposure to the
437 supplied tract/patch.
438
439 Pixels that receive no pixels are set to NAN; this is not correct
440 (violates LSST algorithms group policy), but will be fixed up by
441 interpolating after the coaddition.
442
443 calexpRefList : `list`
444 List of data references for calexps that (may)
445 overlap the patch of interest.
446 skyInfo : `lsst.pipe.base.Struct`
447 Struct from `~lsst.pipe.base.coaddBase.makeSkyInfo()` with
448 geometric information about the patch.
449 visitId : `int`
450 Integer identifier for visit, for the table that will
451 produce the CoaddPsf.
452
453 Returns
454 -------
455 result : `lsst.pipe.base.Struct`
456 Results as a struct with attributes:
457
458 ``exposures``
459 A dictionary containing the warps requested:
460 "direct": direct warp if ``config.makeDirect``
461 "psfMatched": PSF-matched warp if ``config.makePsfMatched``
462 (`dict`).
463 """
464 warpTypeList = self.getWarpTypeList()
465
466 totGoodPix = {warpType: 0 for warpType in warpTypeList}
467 didSetMetadata = {warpType: False for warpType in warpTypeList}
468 warps = {warpType: self._prepareEmptyExposure(skyInfo) for warpType in warpTypeList}
469 inputRecorder = {warpType: self.inputRecorder.makeCoaddTempExpRecorder(visitId, len(calExpList))
470 for warpType in warpTypeList}
471
472 modelPsf = self.config.modelPsf.apply() if self.config.makePsfMatched else None
473 if dataIdList is None:
474 dataIdList = ccdIdList
475
476 for calExpInd, (calExp, ccdId, dataId) in enumerate(zip(calExpList, ccdIdList, dataIdList)):
477 self.log.info("Processing calexp %d of %d for this Warp: id=%s",
478 calExpInd+1, len(calExpList), dataId)
479 # TODO: The following conditional is only required for backwards
480 # compatibility with the deprecated prepareCalibratedExposures()
481 # method. Can remove with its removal after the deprecation
482 # period.
483 if isinstance(calExp, DeferredDatasetHandle):
484 calExp = calExp.get()
485 try:
486 warpedAndMatched = self.warpAndPsfMatch.run(calExp, modelPsf=modelPsf,
487 wcs=skyInfo.wcs, maxBBox=skyInfo.bbox,
488 makeDirect=self.config.makeDirect,
489 makePsfMatched=self.config.makePsfMatched)
490 except Exception as e:
491 self.log.warning("WarpAndPsfMatch failed for calexp %s; skipping it: %s", dataId, e)
492 continue
493 try:
494 numGoodPix = {warpType: 0 for warpType in warpTypeList}
495 for warpType in warpTypeList:
496 exposure = warpedAndMatched.getDict()[warpType]
497 if exposure is None:
498 continue
499 warp = warps[warpType]
500 if didSetMetadata[warpType]:
501 mimg = exposure.getMaskedImage()
502 mimg *= (warp.getPhotoCalib().getInstFluxAtZeroMagnitude()
503 / exposure.getPhotoCalib().getInstFluxAtZeroMagnitude())
504 del mimg
505 numGoodPix[warpType] = coaddUtils.copyGoodPixels(
506 warp.getMaskedImage(), exposure.getMaskedImage(), self.getBadPixelMask())
507 totGoodPix[warpType] += numGoodPix[warpType]
508 self.log.debug("Calexp %s has %d good pixels in this patch (%.1f%%) for %s",
509 dataId, numGoodPix[warpType],
510 100.0*numGoodPix[warpType]/skyInfo.bbox.getArea(), warpType)
511 if numGoodPix[warpType] > 0 and not didSetMetadata[warpType]:
512 warp.info.id = exposure.info.id
513 warp.setPhotoCalib(exposure.getPhotoCalib())
514 warp.setFilter(exposure.getFilter())
515 warp.getInfo().setVisitInfo(exposure.getInfo().getVisitInfo())
516 # PSF replaced with CoaddPsf after loop if and only if
517 # creating direct warp.
518 warp.setPsf(exposure.getPsf())
519 didSetMetadata[warpType] = True
520
521 # Need inputRecorder for CoaddApCorrMap for both direct and
522 # PSF-matched.
523 inputRecorder[warpType].addCalExp(calExp, ccdId, numGoodPix[warpType])
524
525 except Exception as e:
526 self.log.warning("Error processing calexp %s; skipping it: %s", dataId, e)
527 continue
528
529 for warpType in warpTypeList:
530 self.log.info("%sWarp has %d good pixels (%.1f%%)",
531 warpType, totGoodPix[warpType], 100.0*totGoodPix[warpType]/skyInfo.bbox.getArea())
532
533 if totGoodPix[warpType] > 0 and didSetMetadata[warpType]:
534 inputRecorder[warpType].finish(warps[warpType], totGoodPix[warpType])
535 if warpType == "direct":
536 warps[warpType].setPsf(
537 CoaddPsf(inputRecorder[warpType].coaddInputs.ccds, skyInfo.wcs,
538 self.config.coaddPsf.makeControl()))
539 else:
540 if not self.config.doWriteEmptyWarps:
541 # No good pixels. Exposure still empty.
542 warps[warpType] = None
543 # NoWorkFound is unnecessary as the downstream tasks will
544 # adjust the quantum accordingly.
545
546 result = pipeBase.Struct(exposures=warps)
547 return result
548
549 def filterInputs(self, indices, inputs):
550 """Filter task inputs by their indices.
551
552 Parameters
553 ----------
554 indices : `list` [`int`]
555 inputs : `dict` [`list`]
556 A dictionary of input connections to be passed to run.
557
558 Returns
559 -------
560 inputs : `dict` [`list`]
561 Task inputs with their lists filtered by indices.
562 """
563 for key in inputs.keys():
564 # Only down-select on list inputs
565 if isinstance(inputs[key], list):
566 inputs[key] = [inputs[key][ind] for ind in indices]
567 return inputs
568
569 @deprecated(reason="This method is deprecated in favor of its leading underscore version, "
570 "_prepareCalibratedfExposures(). Will be removed after v25.",
571 version="v25.0", category=FutureWarning)
572 def prepareCalibratedExposures(self, calExpList, backgroundList=None, skyCorrList=None,
573 externalSkyWcsCatalog=None, externalPhotoCalibCatalog=None,
574 finalizedPsfApCorrCatalog=None,
575 **kwargs):
576 """Deprecated function.
577
578 Please use _prepareCalibratedExposure(), which this delegates to and
579 noting its slightly updated API, instead.
580 """
581 # Read in all calexps.
582 calExpList = [ref.get() for ref in calExpList]
583 # Populate wcsList as required by new underscored version of function.
584 wcsList = [calexp.getWcs() for calexp in calExpList]
585
586 indices = self._prepareCalibratedExposures(calExpList=calExpList, wcsList=wcsList,
587 backgroundList=backgroundList, skyCorrList=skyCorrList,
588 externalSkyWcsCatalog=externalSkyWcsCatalog,
589 externalPhotoCalibCatalog=externalPhotoCalibCatalog,
590 finalizedPsfApCorrCatalog=finalizedPsfApCorrCatalog)
591 return indices
592
593 def _prepareCalibratedExposures(self, calExpList=[], wcsList=None, backgroundList=None, skyCorrList=None,
594 externalSkyWcsCatalog=None, externalPhotoCalibCatalog=None,
595 finalizedPsfApCorrCatalog=None, **kwargs):
596 """Calibrate and add backgrounds to input calExpList in place.
597
598 Parameters
599 ----------
600 calExpList : `list` [`lsst.afw.image.Exposure` or
601 `lsst.daf.butler.DeferredDatasetHandle`]
602 Sequence of calexps to be modified in place.
603 wcsList : `list` [`lsst.afw.geom.SkyWcs`]
604 The WCSs of the calexps in ``calExpList``. When
605 ``externalSkyCatalog`` is `None`, these are used to determine if
606 the calexp should be included in the warp, namely checking that it
607 is not `None`. If ``externalSkyCatalog`` is not `None`, this list
608 will be dynamically updated with the external sky WCS.
609 backgroundList : `list` [`lsst.afw.math.backgroundList`], optional
610 Sequence of backgrounds to be added back in if bgSubtracted=False.
611 skyCorrList : `list` [`lsst.afw.math.backgroundList`], optional
612 Sequence of background corrections to be subtracted if
613 doApplySkyCorr=True.
614 externalSkyWcsCatalog : `lsst.afw.table.ExposureCatalog`, optional
615 Exposure catalog with external skyWcs to be applied
616 if config.doApplyExternalSkyWcs=True. Catalog uses the detector id
617 for the catalog id, sorted on id for fast lookup.
618 externalPhotoCalibCatalog : `lsst.afw.table.ExposureCatalog`, optional
619 Exposure catalog with external photoCalib to be applied
620 if config.doApplyExternalPhotoCalib=True. Catalog uses the
621 detector id for the catalog id, sorted on id for fast lookup.
622 finalizedPsfApCorrCatalog : `lsst.afw.table.ExposureCatalog`, optional
623 Exposure catalog with finalized psf models and aperture correction
624 maps to be applied if config.doApplyFinalizedPsf=True. Catalog
625 uses the detector id for the catalog id, sorted on id for fast
626 lookup.
627 **kwargs
628 Additional keyword arguments.
629
630 Returns
631 -------
632 indices : `list` [`int`]
633 Indices of ``calExpList`` and friends that have valid
634 photoCalib/skyWcs.
635 """
636 wcsList = len(calExpList)*[None] if wcsList is None else wcsList
637 backgroundList = len(calExpList)*[None] if backgroundList is None else backgroundList
638 skyCorrList = len(calExpList)*[None] if skyCorrList is None else skyCorrList
639
640 includeCalibVar = self.config.includeCalibVar
641
642 indices = []
643 for index, (calexp, wcs, background, skyCorr) in enumerate(zip(calExpList,
644 wcsList,
645 backgroundList,
646 skyCorrList)):
647 if externalSkyWcsCatalog is None and wcs is None:
648 self.log.warning("Detector id %d for visit %d has None for skyWcs and will not be "
649 "used in the warp", calexp.dataId["detector"], calexp.dataId["visit"])
650 continue
651
652 if isinstance(calexp, DeferredDatasetHandle):
653 calexp = calexp.get()
654
655 if not self.config.bgSubtracted:
656 calexp.maskedImage += background.getImage()
657
658 detectorId = calexp.info.getDetector().getId()
659
660 # Find the external photoCalib.
661 if externalPhotoCalibCatalog is not None:
662 row = externalPhotoCalibCatalog.find(detectorId)
663 if row is None:
664 self.log.warning("Detector id %s not found in externalPhotoCalibCatalog "
665 "and will not be used in the warp.", detectorId)
666 continue
667 photoCalib = row.getPhotoCalib()
668 if photoCalib is None:
669 self.log.warning("Detector id %s has None for photoCalib in externalPhotoCalibCatalog "
670 "and will not be used in the warp.", detectorId)
671 continue
672 calexp.setPhotoCalib(photoCalib)
673 else:
674 photoCalib = calexp.getPhotoCalib()
675 if photoCalib is None:
676 self.log.warning("Detector id %s has None for photoCalib in the calexp "
677 "and will not be used in the warp.", detectorId)
678 continue
679
680 # Find and apply external skyWcs.
681 if externalSkyWcsCatalog is not None:
682 row = externalSkyWcsCatalog.find(detectorId)
683 if row is None:
684 self.log.warning("Detector id %s not found in externalSkyWcsCatalog "
685 "and will not be used in the warp.", detectorId)
686 continue
687 skyWcs = row.getWcs()
688 wcsList[index] = skyWcs
689 if skyWcs is None:
690 self.log.warning("Detector id %s has None for skyWcs in externalSkyWcsCatalog "
691 "and will not be used in the warp.", detectorId)
692 continue
693 calexp.setWcs(skyWcs)
694 else:
695 skyWcs = calexp.getWcs()
696 wcsList[index] = skyWcs
697 if skyWcs is None:
698 self.log.warning("Detector id %s has None for skyWcs in the calexp "
699 "and will not be used in the warp.", detectorId)
700 continue
701
702 # Find and apply finalized psf and aperture correction.
703 if finalizedPsfApCorrCatalog is not None:
704 row = finalizedPsfApCorrCatalog.find(detectorId)
705 if row is None:
706 self.log.warning("Detector id %s not found in finalizedPsfApCorrCatalog "
707 "and will not be used in the warp.", detectorId)
708 continue
709 psf = row.getPsf()
710 if psf is None:
711 self.log.warning("Detector id %s has None for psf in finalizedPsfApCorrCatalog "
712 "and will not be used in the warp.", detectorId)
713 continue
714 calexp.setPsf(psf)
715 apCorrMap = row.getApCorrMap()
716 if apCorrMap is None:
717 self.log.warning("Detector id %s has None for ApCorrMap in finalizedPsfApCorrCatalog "
718 "and will not be used in the warp.", detectorId)
719 continue
720 calexp.info.setApCorrMap(apCorrMap)
721 else:
722 # Ensure that calexp has valid aperture correction map.
723 if calexp.info.getApCorrMap() is None:
724 self.log.warning("Detector id %s has None for ApCorrMap in the calexp "
725 "and will not be used in the warp.", detectorId)
726 continue
727
728 # Calibrate the image.
729 calexp.maskedImage = photoCalib.calibrateImage(calexp.maskedImage,
730 includeScaleUncertainty=includeCalibVar)
731 calexp.maskedImage /= photoCalib.getCalibrationMean()
732 # TODO: The images will have a calibration of 1.0 everywhere once
733 # RFC-545 is implemented.
734 # exposure.setCalib(afwImage.Calib(1.0))
735
736 # Apply skycorr
737 if self.config.doApplySkyCorr:
738 calexp.maskedImage -= skyCorr.getImage()
739
740 indices.append(index)
741 calExpList[index] = calexp
742
743 return indices
744
745 @staticmethod
746 def _prepareEmptyExposure(skyInfo):
747 """Produce an empty exposure for a given patch.
748
749 Parameters
750 ----------
751 skyInfo : `lsst.pipe.base.Struct`
752 Struct from `~lsst.pipe.base.coaddBase.makeSkyInfo()` with
753 geometric information about the patch.
754
755 Returns
756 -------
757 exp : `lsst.afw.image.exposure.ExposureF`
758 An empty exposure for a given patch.
759 """
760 exp = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
761 exp.getMaskedImage().set(numpy.nan, afwImage.Mask
762 .getPlaneBitMask("NO_DATA"), numpy.inf)
763 return exp
764
765 def getWarpTypeList(self):
766 """Return list of requested warp types per the config.
767 """
768 warpTypeList = []
769 if self.config.makeDirect:
770 warpTypeList.append("direct")
771 if self.config.makePsfMatched:
772 warpTypeList.append("psfMatched")
773 return warpTypeList
774
775
776def reorderRefs(inputRefs, outputSortKeyOrder, dataIdKey):
777 """Reorder inputRefs per outputSortKeyOrder.
778
779 Any inputRefs which are lists will be resorted per specified key e.g.,
780 'detector.' Only iterables will be reordered, and values can be of type
781 `lsst.pipe.base.connections.DeferredDatasetRef` or
782 `lsst.daf.butler.core.datasets.ref.DatasetRef`.
783
784 Returned lists of refs have the same length as the outputSortKeyOrder.
785 If an outputSortKey not in the inputRef, then it will be padded with None.
786 If an inputRef contains an inputSortKey that is not in the
787 outputSortKeyOrder it will be removed.
788
789 Parameters
790 ----------
791 inputRefs : `lsst.pipe.base.connections.QuantizedConnection`
792 Input references to be reordered and padded.
793 outputSortKeyOrder : `iterable`
794 Iterable of values to be compared with inputRef's dataId[dataIdKey].
795 dataIdKey : `str`
796 The data ID key in the dataRefs to compare with the outputSortKeyOrder.
797
798 Returns
799 -------
800 inputRefs : `lsst.pipe.base.connections.QuantizedConnection`
801 Quantized Connection with sorted DatasetRef values sorted if iterable.
802 """
803 for connectionName, refs in inputRefs:
804 if isinstance(refs, Iterable):
805 if hasattr(refs[0], "dataId"):
806 inputSortKeyOrder = [ref.dataId[dataIdKey] for ref in refs]
807 else:
808 inputSortKeyOrder = [ref.datasetRef.dataId[dataIdKey] for ref in refs]
809 if inputSortKeyOrder != outputSortKeyOrder:
810 setattr(inputRefs, connectionName,
811 reorderAndPadList(refs, inputSortKeyOrder, outputSortKeyOrder))
812 return inputRefs