lsst.pipe.tasks g369a80f31c+ba1a864b1f
Loading...
Searching...
No Matches
makeWarp.py
Go to the documentation of this file.
1# This file is part of pipe_tasks.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21
22__all__ = ["MakeWarpTask", "MakeWarpConfig"]
23
24import logging
25import numpy
26
27import lsst.pex.config as pexConfig
28import lsst.afw.image as afwImage
29import lsst.coadd.utils as coaddUtils
30import lsst.pipe.base as pipeBase
31import lsst.pipe.base.connectionTypes as connectionTypes
32import lsst.utils as utils
33import lsst.geom
34from lsst.meas.algorithms import CoaddPsf, CoaddPsfConfig
35from lsst.skymap import BaseSkyMap
36from lsst.utils.timer import timeMethod
37from .coaddBase import CoaddBaseTask, makeSkyInfo, reorderAndPadList
38from .warpAndPsfMatch import WarpAndPsfMatchTask
39from collections.abc import Iterable
40
41log = logging.getLogger(__name__)
42
43
44class MakeWarpConnections(pipeBase.PipelineTaskConnections,
45 dimensions=("tract", "patch", "skymap", "instrument", "visit"),
46 defaultTemplates={"coaddName": "deep",
47 "skyWcsName": "jointcal",
48 "photoCalibName": "fgcm",
49 "calexpType": ""}):
50 calExpList = connectionTypes.Input(
51 doc="Input exposures to be resampled and optionally PSF-matched onto a SkyMap projection/patch",
52 name="{calexpType}calexp",
53 storageClass="ExposureF",
54 dimensions=("instrument", "visit", "detector"),
55 multiple=True,
56 deferLoad=True,
57 )
58 backgroundList = connectionTypes.Input(
59 doc="Input backgrounds to be added back into the calexp if bgSubtracted=False",
60 name="calexpBackground",
61 storageClass="Background",
62 dimensions=("instrument", "visit", "detector"),
63 multiple=True,
64 )
65 skyCorrList = connectionTypes.Input(
66 doc="Input Sky Correction to be subtracted from the calexp if doApplySkyCorr=True",
67 name="skyCorr",
68 storageClass="Background",
69 dimensions=("instrument", "visit", "detector"),
70 multiple=True,
71 )
72 skyMap = connectionTypes.Input(
73 doc="Input definition of geometry/bbox and projection/wcs for warped exposures",
74 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
75 storageClass="SkyMap",
76 dimensions=("skymap",),
77 )
78 externalSkyWcsTractCatalog = connectionTypes.Input(
79 doc=("Per-tract, per-visit wcs calibrations. These catalogs use the detector "
80 "id for the catalog id, sorted on id for fast lookup."),
81 name="{skyWcsName}SkyWcsCatalog",
82 storageClass="ExposureCatalog",
83 dimensions=("instrument", "visit", "tract"),
84 )
85 externalSkyWcsGlobalCatalog = connectionTypes.Input(
86 doc=("Per-visit wcs calibrations computed globally (with no tract information). "
87 "These catalogs use the detector id for the catalog id, sorted on id for "
88 "fast lookup."),
89 name="{skyWcsName}SkyWcsCatalog",
90 storageClass="ExposureCatalog",
91 dimensions=("instrument", "visit"),
92 )
93 externalPhotoCalibTractCatalog = connectionTypes.Input(
94 doc=("Per-tract, per-visit photometric calibrations. These catalogs use the "
95 "detector id for the catalog id, sorted on id for fast lookup."),
96 name="{photoCalibName}PhotoCalibCatalog",
97 storageClass="ExposureCatalog",
98 dimensions=("instrument", "visit", "tract"),
99 )
100 externalPhotoCalibGlobalCatalog = connectionTypes.Input(
101 doc=("Per-visit photometric calibrations computed globally (with no tract "
102 "information). These catalogs use the detector id for the catalog id, "
103 "sorted on id for fast lookup."),
104 name="{photoCalibName}PhotoCalibCatalog",
105 storageClass="ExposureCatalog",
106 dimensions=("instrument", "visit"),
107 )
108 finalizedPsfApCorrCatalog = connectionTypes.Input(
109 doc=("Per-visit finalized psf models and aperture correction maps. "
110 "These catalogs use the detector id for the catalog id, "
111 "sorted on id for fast lookup."),
112 name="finalized_psf_ap_corr_catalog",
113 storageClass="ExposureCatalog",
114 dimensions=("instrument", "visit"),
115 )
116 direct = connectionTypes.Output(
117 doc=("Output direct warped exposure (previously called CoaddTempExp), produced by resampling ",
118 "calexps onto the skyMap patch geometry."),
119 name="{coaddName}Coadd_directWarp",
120 storageClass="ExposureF",
121 dimensions=("tract", "patch", "skymap", "visit", "instrument"),
122 )
123 psfMatched = connectionTypes.Output(
124 doc=("Output PSF-Matched warped exposure (previously called CoaddTempExp), produced by resampling ",
125 "calexps onto the skyMap patch geometry and PSF-matching to a model PSF."),
126 name="{coaddName}Coadd_psfMatchedWarp",
127 storageClass="ExposureF",
128 dimensions=("tract", "patch", "skymap", "visit", "instrument"),
129 )
130 # TODO DM-28769, have selectImages subtask indicate which connections they need:
131 wcsList = connectionTypes.Input(
132 doc="WCSs of calexps used by SelectImages subtask to determine if the calexp overlaps the patch",
133 name="{calexpType}calexp.wcs",
134 storageClass="Wcs",
135 dimensions=("instrument", "visit", "detector"),
136 multiple=True,
137 )
138 bboxList = connectionTypes.Input(
139 doc="BBoxes of calexps used by SelectImages subtask to determine if the calexp overlaps the patch",
140 name="{calexpType}calexp.bbox",
141 storageClass="Box2I",
142 dimensions=("instrument", "visit", "detector"),
143 multiple=True,
144 )
145 visitSummary = connectionTypes.Input(
146 doc="Consolidated exposure metadata from ConsolidateVisitSummaryTask",
147 name="{calexpType}visitSummary",
148 storageClass="ExposureCatalog",
149 dimensions=("instrument", "visit",),
150 )
151
152 def __init__(self, *, config=None):
153 super().__init__(config=config)
154 if config.bgSubtracted:
155 self.inputs.remove("backgroundList")
156 if not config.doApplySkyCorr:
157 self.inputs.remove("skyCorrList")
158 if config.doApplyExternalSkyWcs:
159 if config.useGlobalExternalSkyWcs:
160 self.inputs.remove("externalSkyWcsTractCatalog")
161 else:
162 self.inputs.remove("externalSkyWcsGlobalCatalog")
163 else:
164 self.inputs.remove("externalSkyWcsTractCatalog")
165 self.inputs.remove("externalSkyWcsGlobalCatalog")
166 if config.doApplyExternalPhotoCalib:
167 if config.useGlobalExternalPhotoCalib:
168 self.inputs.remove("externalPhotoCalibTractCatalog")
169 else:
170 self.inputs.remove("externalPhotoCalibGlobalCatalog")
171 else:
172 self.inputs.remove("externalPhotoCalibTractCatalog")
173 self.inputs.remove("externalPhotoCalibGlobalCatalog")
174 if not config.doApplyFinalizedPsf:
175 self.inputs.remove("finalizedPsfApCorrCatalog")
176 if not config.makeDirect:
177 self.outputs.remove("direct")
178 if not config.makePsfMatched:
179 self.outputs.remove("psfMatched")
180 # TODO DM-28769: add connection per selectImages connections
181 if config.select.target != lsst.pipe.tasks.selectImages.PsfWcsSelectImagesTask:
182 self.inputs.remove("visitSummary")
183
184
185class MakeWarpConfig(pipeBase.PipelineTaskConfig, CoaddBaseTask.ConfigClass,
186 pipelineConnections=MakeWarpConnections):
187 """Config for MakeWarpTask."""
188
189 warpAndPsfMatch = pexConfig.ConfigurableField(
190 target=WarpAndPsfMatchTask,
191 doc="Task to warp and PSF-match calexp",
192 )
193 doWrite = pexConfig.Field(
194 doc="persist <coaddName>Coadd_<warpType>Warp",
195 dtype=bool,
196 default=True,
197 )
198 bgSubtracted = pexConfig.Field(
199 doc="Work with a background subtracted calexp?",
200 dtype=bool,
201 default=True,
202 )
203 coaddPsf = pexConfig.ConfigField(
204 doc="Configuration for CoaddPsf",
205 dtype=CoaddPsfConfig,
206 )
207 makeDirect = pexConfig.Field(
208 doc="Make direct Warp/Coadds",
209 dtype=bool,
210 default=True,
211 )
212 makePsfMatched = pexConfig.Field(
213 doc="Make Psf-Matched Warp/Coadd?",
214 dtype=bool,
215 default=False,
216 )
217 doWriteEmptyWarps = pexConfig.Field(
218 dtype=bool,
219 default=False,
220 doc="Write out warps even if they are empty"
221 )
222 hasFakes = pexConfig.Field(
223 doc="Should be set to True if fake sources have been inserted into the input data.",
224 dtype=bool,
225 default=False,
226 )
227 doApplySkyCorr = pexConfig.Field(
228 dtype=bool,
229 default=False,
230 doc="Apply sky correction?",
231 )
232 doApplyFinalizedPsf = pexConfig.Field(
233 doc="Whether to apply finalized psf models and aperture correction map.",
234 dtype=bool,
235 default=True,
236 )
237
238 def validate(self):
239 CoaddBaseTask.ConfigClass.validate(self)
240
241 if not self.makePsfMatched and not self.makeDirect:
242 raise RuntimeError("At least one of config.makePsfMatched and config.makeDirect must be True")
243 if self.doPsfMatch:
244 # Backwards compatibility.
245 log.warning("Config doPsfMatch deprecated. Setting makePsfMatched=True and makeDirect=False")
246 self.makePsfMatched = True
247 self.makeDirect = False
248
249 def setDefaults(self):
250 CoaddBaseTask.ConfigClass.setDefaults(self)
251 self.warpAndPsfMatch.psfMatch.kernel.active.kernelSize = self.matchingKernelSize
252
253
254class MakeWarpTask(CoaddBaseTask):
255 """Warp and optionally PSF-Match calexps onto an a common projection.
256
257 Warp and optionally PSF-Match calexps onto a common projection, by
258 performing the following operations:
259 - Group calexps by visit/run
260 - For each visit, generate a Warp by calling method @ref run.
261 `run` loops over the visit's calexps calling
263
264 Notes
265 -----
266 WarpType identifies the types of convolutions applied to Warps
267 (previously CoaddTempExps). Only two types are available: direct
268 (for regular Warps/Coadds) and psfMatched(for Warps/Coadds with
269 homogenized PSFs). We expect to add a third type, likelihood, for
270 generating likelihood Coadds with Warps that have been correlated with
271 their own PSF.
272
273 To make `psfMatchedWarps`, select `config.makePsfMatched=True`. The subtask
275 is responsible for the PSF-Matching, and its config is accessed via `config.warpAndPsfMatch.psfMatch`.
276 The optimal configuration depends on aspects of dataset: the pixel scale, average PSF FWHM and
277 dimensions of the PSF kernel. These configs include the requested model PSF, the matching kernel size,
278 padding of the science PSF thumbnail and spatial sampling frequency of the PSF.
279 *Config Guidelines*: The user must specify the size of the model PSF to which to match by setting
280 `config.modelPsf.defaultFwhm` in units of pixels. The appropriate values depends on science case.
281 In general, for a set of input images, this config should equal the FWHM of the visit
282 with the worst seeing. The smallest it should be set to is the median FWHM. The defaults
283 of the other config options offer a reasonable starting point.
284 The following list presents the most common problems that arise from a misconfigured
285 @link ip::diffim::modelPsfMatch::ModelPsfMatchTask ModelPsfMatchTask @endlink
286 and corresponding solutions. All assume the default Alard-Lupton kernel, with configs accessed via
287 ```config.warpAndPsfMatch.psfMatch.kernel['AL']```. Each item in the list is formatted as:
288 Problem: Explanation. *Solution*
289 *Troublshooting PSF-Matching Configuration:*
290 - Matched PSFs look boxy: The matching kernel is too small. _Increase the matching kernel size.
291 For example:_
292 config.warpAndPsfMatch.psfMatch.kernel['AL'].kernelSize=27 # default 21
293 Note that increasing the kernel size also increases runtime.
294 - Matched PSFs look ugly (dipoles, quadropoles, donuts): unable to find good solution
295 for matching kernel. _Provide the matcher with more data by either increasing
296 the spatial sampling by decreasing the spatial cell size,_
297 config.warpAndPsfMatch.psfMatch.kernel['AL'].sizeCellX = 64 # default 128
298 config.warpAndPsfMatch.psfMatch.kernel['AL'].sizeCellY = 64 # default 128
299 _or increasing the padding around the Science PSF, for example:_
300 config.warpAndPsfMatch.psfMatch.autoPadPsfTo=1.6 # default 1.4
301 Increasing `autoPadPsfTo` increases the minimum ratio of input PSF dimensions to the
302 matching kernel dimensions, thus increasing the number of pixels available to fit
303 after convolving the PSF with the matching kernel.
304 Optionally, for debugging the effects of padding, the level of padding may be manually
305 controlled by setting turning off the automatic padding and setting the number
306 of pixels by which to pad the PSF:
307 config.warpAndPsfMatch.psfMatch.doAutoPadPsf = False # default True
308 config.warpAndPsfMatch.psfMatch.padPsfBy = 6 # pixels. default 0
309 - Deconvolution: Matching a large PSF to a smaller PSF produces
310 a telltale noise pattern which looks like ripples or a brain.
311 _Increase the size of the requested model PSF. For example:_
312 config.modelPsf.defaultFwhm = 11 # Gaussian sigma in units of pixels.
313 - High frequency (sometimes checkered) noise: The matching basis functions are too small.
314 _Increase the width of the Gaussian basis functions. For example:_
315 config.warpAndPsfMatch.psfMatch.kernel['AL'].alardSigGauss=[1.5, 3.0, 6.0]
316 # from default [0.7, 1.5, 3.0]
317 """
318 ConfigClass = MakeWarpConfig
319 _DefaultName = "makeWarp"
320
321 def __init__(self, **kwargs):
322 CoaddBaseTask.__init__(self, **kwargs)
323 self.makeSubtask("warpAndPsfMatch")
324 if self.config.hasFakes:
325 self.calexpType = "fakes_calexp"
326 else:
327 self.calexpType = "calexp"
328
329 @utils.inheritDoc(pipeBase.PipelineTask)
330 def runQuantum(self, butlerQC, inputRefs, outputRefs):
331 # Obtain the list of input detectors from calExpList. Sort them by
332 # detector order (to ensure reproducibility). Then ensure all input
333 # lists are in the same sorted detector order.
334 detectorOrder = [ref.datasetRef.dataId['detector'] for ref in inputRefs.calExpList]
335 detectorOrder.sort()
336 inputRefs = reorderRefs(inputRefs, detectorOrder, dataIdKey='detector')
337
338 # Read in all inputs.
339 inputs = butlerQC.get(inputRefs)
340
341 # Construct skyInfo expected by `run`. We remove the SkyMap itself
342 # from the dictionary so we can pass it as kwargs later.
343 skyMap = inputs.pop("skyMap")
344 quantumDataId = butlerQC.quantum.dataId
345 skyInfo = makeSkyInfo(skyMap, tractId=quantumDataId['tract'], patchId=quantumDataId['patch'])
346
347 # Construct list of input DataIds expected by `run`
348 dataIdList = [ref.datasetRef.dataId for ref in inputRefs.calExpList]
349 # Construct list of packed integer IDs expected by `run`
350 ccdIdList = [dataId.pack("visit_detector") for dataId in dataIdList]
351
352 # Run the selector and filter out calexps that were not selected
353 # primarily because they do not overlap the patch
354 cornerPosList = lsst.geom.Box2D(skyInfo.bbox).getCorners()
355 coordList = [skyInfo.wcs.pixelToSky(pos) for pos in cornerPosList]
356 goodIndices = self.select.run(**inputs, coordList=coordList, dataIds=dataIdList)
357 inputs = self.filterInputs(indices=goodIndices, inputs=inputs)
358
359 # Read from disk only the selected calexps
360 inputs['calExpList'] = [ref.get() for ref in inputs['calExpList']]
361
362 # Extract integer visitId requested by `run`
363 visits = [dataId['visit'] for dataId in dataIdList]
364 visitId = visits[0]
365
366 if self.config.doApplyExternalSkyWcs:
367 if self.config.useGlobalExternalSkyWcs:
368 externalSkyWcsCatalog = inputs.pop("externalSkyWcsGlobalCatalog")
369 else:
370 externalSkyWcsCatalog = inputs.pop("externalSkyWcsTractCatalog")
371 else:
372 externalSkyWcsCatalog = None
373
374 if self.config.doApplyExternalPhotoCalib:
375 if self.config.useGlobalExternalPhotoCalib:
376 externalPhotoCalibCatalog = inputs.pop("externalPhotoCalibGlobalCatalog")
377 else:
378 externalPhotoCalibCatalog = inputs.pop("externalPhotoCalibTractCatalog")
379 else:
380 externalPhotoCalibCatalog = None
381
382 if self.config.doApplyFinalizedPsf:
383 finalizedPsfApCorrCatalog = inputs.pop("finalizedPsfApCorrCatalog")
384 else:
385 finalizedPsfApCorrCatalog = None
386
387 completeIndices = self.prepareCalibratedExposures(**inputs,
388 externalSkyWcsCatalog=externalSkyWcsCatalog,
389 externalPhotoCalibCatalog=externalPhotoCalibCatalog,
390 finalizedPsfApCorrCatalog=finalizedPsfApCorrCatalog)
391 # Redo the input selection with inputs with complete wcs/photocalib info.
392 inputs = self.filterInputs(indices=completeIndices, inputs=inputs)
393
394 results = self.run(**inputs, visitId=visitId,
395 ccdIdList=[ccdIdList[i] for i in goodIndices],
396 dataIdList=[dataIdList[i] for i in goodIndices],
397 skyInfo=skyInfo)
398 if self.config.makeDirect and results.exposures["direct"] is not None:
399 butlerQC.put(results.exposures["direct"], outputRefs.direct)
400 if self.config.makePsfMatched and results.exposures["psfMatched"] is not None:
401 butlerQC.put(results.exposures["psfMatched"], outputRefs.psfMatched)
402
403 @timeMethod
404 def run(self, calExpList, ccdIdList, skyInfo, visitId=0, dataIdList=None, **kwargs):
405 """Create a Warp from inputs.
406
407 We iterate over the multiple calexps in a single exposure to construct
408 the warp (previously called a coaddTempExp) of that exposure to the
409 supplied tract/patch.
410
411 Pixels that receive no pixels are set to NAN; this is not correct
412 (violates LSST algorithms group policy), but will be fixed up by
413 interpolating after the coaddition.
414
415 calexpRefList : `list`
416 List of data references for calexps that (may)
417 overlap the patch of interest.
418 skyInfo : `lsst.pipe.base.Struct`
419 Struct from CoaddBaseTask.getSkyInfo() with geometric
420 information about the patch.
421 visitId : `int`
422 Integer identifier for visit, for the table that will
423 produce the CoaddPsf.
424
425 Returns
426 -------
427 result : `lsst.pipe.base.Struct`
428 Results as a struct with attributes:
429
430 ``exposures``
431 A dictionary containing the warps requested:
432 "direct": direct warp if ``config.makeDirect``
433 "psfMatched": PSF-matched warp if ``config.makePsfMatched`` (`dict`).
434 """
435 warpTypeList = self.getWarpTypeList()
436
437 totGoodPix = {warpType: 0 for warpType in warpTypeList}
438 didSetMetadata = {warpType: False for warpType in warpTypeList}
439 warps = {warpType: self._prepareEmptyExposure(skyInfo) for warpType in warpTypeList}
440 inputRecorder = {warpType: self.inputRecorder.makeCoaddTempExpRecorder(visitId, len(calExpList))
441 for warpType in warpTypeList}
442
443 modelPsf = self.config.modelPsf.apply() if self.config.makePsfMatched else None
444 if dataIdList is None:
445 dataIdList = ccdIdList
446
447 for calExpInd, (calExp, ccdId, dataId) in enumerate(zip(calExpList, ccdIdList, dataIdList)):
448 self.log.info("Processing calexp %d of %d for this Warp: id=%s",
449 calExpInd+1, len(calExpList), dataId)
450
451 try:
452 warpedAndMatched = self.warpAndPsfMatch.run(calExp, modelPsf=modelPsf,
453 wcs=skyInfo.wcs, maxBBox=skyInfo.bbox,
454 makeDirect=self.config.makeDirect,
455 makePsfMatched=self.config.makePsfMatched)
456 except Exception as e:
457 self.log.warning("WarpAndPsfMatch failed for calexp %s; skipping it: %s", dataId, e)
458 continue
459 try:
460 numGoodPix = {warpType: 0 for warpType in warpTypeList}
461 for warpType in warpTypeList:
462 exposure = warpedAndMatched.getDict()[warpType]
463 if exposure is None:
464 continue
465 warp = warps[warpType]
466 if didSetMetadata[warpType]:
467 mimg = exposure.getMaskedImage()
468 mimg *= (warp.getPhotoCalib().getInstFluxAtZeroMagnitude()
469 / exposure.getPhotoCalib().getInstFluxAtZeroMagnitude())
470 del mimg
471 numGoodPix[warpType] = coaddUtils.copyGoodPixels(
472 warp.getMaskedImage(), exposure.getMaskedImage(), self.getBadPixelMask())
473 totGoodPix[warpType] += numGoodPix[warpType]
474 self.log.debug("Calexp %s has %d good pixels in this patch (%.1f%%) for %s",
475 dataId, numGoodPix[warpType],
476 100.0*numGoodPix[warpType]/skyInfo.bbox.getArea(), warpType)
477 if numGoodPix[warpType] > 0 and not didSetMetadata[warpType]:
478 warp.info.id = exposure.info.id
479 warp.setPhotoCalib(exposure.getPhotoCalib())
480 warp.setFilter(exposure.getFilter())
481 warp.getInfo().setVisitInfo(exposure.getInfo().getVisitInfo())
482 # PSF replaced with CoaddPsf after loop if and only if creating direct warp
483 warp.setPsf(exposure.getPsf())
484 didSetMetadata[warpType] = True
485
486 # Need inputRecorder for CoaddApCorrMap for both direct and PSF-matched
487 inputRecorder[warpType].addCalExp(calExp, ccdId, numGoodPix[warpType])
488
489 except Exception as e:
490 self.log.warning("Error processing calexp %s; skipping it: %s", dataId, e)
491 continue
492
493 for warpType in warpTypeList:
494 self.log.info("%sWarp has %d good pixels (%.1f%%)",
495 warpType, totGoodPix[warpType], 100.0*totGoodPix[warpType]/skyInfo.bbox.getArea())
496
497 if totGoodPix[warpType] > 0 and didSetMetadata[warpType]:
498 inputRecorder[warpType].finish(warps[warpType], totGoodPix[warpType])
499 if warpType == "direct":
500 warps[warpType].setPsf(
501 CoaddPsf(inputRecorder[warpType].coaddInputs.ccds, skyInfo.wcs,
502 self.config.coaddPsf.makeControl()))
503 else:
504 if not self.config.doWriteEmptyWarps:
505 # No good pixels. Exposure still empty
506 warps[warpType] = None
507 # NoWorkFound is unnecessary as the downstream tasks will
508 # adjust the quantum accordingly.
509
510 result = pipeBase.Struct(exposures=warps)
511 return result
512
513 def filterInputs(self, indices, inputs):
514 """Filter task inputs by their indices.
515
516 Parameters
517 ----------
518 indices : `list` of `int`
519 inputs : `dict` of `list`
520 A dictionary of input connections to be passed to run.
521
522 Returns
523 -------
524 inputs : `dict` of `list`
525 Task inputs with their lists filtered by indices.
526 """
527 for key in inputs.keys():
528 # Only down-select on list inputs
529 if isinstance(inputs[key], list):
530 inputs[key] = [inputs[key][ind] for ind in indices]
531 return inputs
532
533 def prepareCalibratedExposures(self, calExpList, backgroundList=None, skyCorrList=None,
534 externalSkyWcsCatalog=None, externalPhotoCalibCatalog=None,
535 finalizedPsfApCorrCatalog=None,
536 **kwargs):
537 """Calibrate and add backgrounds to input calExpList in place.
538
539 Parameters
540 ----------
541 calExpList : `list` of `lsst.afw.image.Exposure`
542 Sequence of calexps to be modified in place.
543 backgroundList : `list` of `lsst.afw.math.backgroundList`, optional
544 Sequence of backgrounds to be added back in if bgSubtracted=False.
545 skyCorrList : `list` of `lsst.afw.math.backgroundList`, optional
546 Sequence of background corrections to be subtracted if doApplySkyCorr=True.
547 externalSkyWcsCatalog : `lsst.afw.table.ExposureCatalog`, optional
548 Exposure catalog with external skyWcs to be applied
549 if config.doApplyExternalSkyWcs=True. Catalog uses the detector id
550 for the catalog id, sorted on id for fast lookup.
551 externalPhotoCalibCatalog : `lsst.afw.table.ExposureCatalog`, optional
552 Exposure catalog with external photoCalib to be applied
553 if config.doApplyExternalPhotoCalib=True. Catalog uses the detector
554 id for the catalog id, sorted on id for fast lookup.
555 finalizedPsfApCorrCatalog : `lsst.afw.table.ExposureCatalog`, optional
556 Exposure catalog with finalized psf models and aperture correction
557 maps to be applied if config.doApplyFinalizedPsf=True. Catalog uses
558 the detector id for the catalog id, sorted on id for fast lookup.
559 **kwargs
560 Additional keyword arguments.
561
562 Returns
563 -------
564 indices : `list` [`int`]
565 Indices of calExpList and friends that have valid photoCalib/skyWcs.
566 """
567 backgroundList = len(calExpList)*[None] if backgroundList is None else backgroundList
568 skyCorrList = len(calExpList)*[None] if skyCorrList is None else skyCorrList
569
570 includeCalibVar = self.config.includeCalibVar
571
572 indices = []
573 for index, (calexp, background, skyCorr) in enumerate(zip(calExpList,
574 backgroundList,
575 skyCorrList)):
576 if not self.config.bgSubtracted:
577 calexp.maskedImage += background.getImage()
578
579 detectorId = calexp.getInfo().getDetector().getId()
580
581 # Find the external photoCalib
582 if externalPhotoCalibCatalog is not None:
583 row = externalPhotoCalibCatalog.find(detectorId)
584 if row is None:
585 self.log.warning("Detector id %s not found in externalPhotoCalibCatalog "
586 "and will not be used in the warp.", detectorId)
587 continue
588 photoCalib = row.getPhotoCalib()
589 if photoCalib is None:
590 self.log.warning("Detector id %s has None for photoCalib in externalPhotoCalibCatalog "
591 "and will not be used in the warp.", detectorId)
592 continue
593 calexp.setPhotoCalib(photoCalib)
594 else:
595 photoCalib = calexp.getPhotoCalib()
596 if photoCalib is None:
597 self.log.warning("Detector id %s has None for photoCalib in the calexp "
598 "and will not be used in the warp.", detectorId)
599 continue
600
601 # Find and apply external skyWcs
602 if externalSkyWcsCatalog is not None:
603 row = externalSkyWcsCatalog.find(detectorId)
604 if row is None:
605 self.log.warning("Detector id %s not found in externalSkyWcsCatalog "
606 "and will not be used in the warp.", detectorId)
607 continue
608 skyWcs = row.getWcs()
609 if skyWcs is None:
610 self.log.warning("Detector id %s has None for skyWcs in externalSkyWcsCatalog "
611 "and will not be used in the warp.", detectorId)
612 continue
613 calexp.setWcs(skyWcs)
614 else:
615 skyWcs = calexp.getWcs()
616 if skyWcs is None:
617 self.log.warning("Detector id %s has None for skyWcs in the calexp "
618 "and will not be used in the warp.", detectorId)
619 continue
620
621 # Find and apply finalized psf and aperture correction
622 if finalizedPsfApCorrCatalog is not None:
623 row = finalizedPsfApCorrCatalog.find(detectorId)
624 if row is None:
625 self.log.warning("Detector id %s not found in finalizedPsfApCorrCatalog "
626 "and will not be used in the warp.", detectorId)
627 continue
628 psf = row.getPsf()
629 if psf is None:
630 self.log.warning("Detector id %s has None for psf in finalizedPsfApCorrCatalog "
631 "and will not be used in the warp.", detectorId)
632 continue
633 calexp.setPsf(psf)
634 apCorrMap = row.getApCorrMap()
635 if apCorrMap is None:
636 self.log.warning("Detector id %s has None for ApCorrMap in finalizedPsfApCorrCatalog "
637 "and will not be used in the warp.", detectorId)
638 continue
639 calexp.info.setApCorrMap(apCorrMap)
640
641 # Calibrate the image
642 calexp.maskedImage = photoCalib.calibrateImage(calexp.maskedImage,
643 includeScaleUncertainty=includeCalibVar)
644 calexp.maskedImage /= photoCalib.getCalibrationMean()
645 # TODO: The images will have a calibration of 1.0 everywhere once RFC-545 is implemented.
646 # exposure.setCalib(afwImage.Calib(1.0))
647
648 # Apply skycorr
649 if self.config.doApplySkyCorr:
650 calexp.maskedImage -= skyCorr.getImage()
651
652 indices.append(index)
653
654 return indices
655
656 @staticmethod
657 def _prepareEmptyExposure(skyInfo):
658 """Produce an empty exposure for a given patch.
659
660 Parameters
661 ----------
662 skyInfo : `lsst.pipe.base.Struct`
663 Struct from CoaddBaseTask.getSkyInfo() with geometric
664 information about the patch.
665
666 Returns
667 -------
668 exp : `lsst.afw.image.exposure.ExposureF`
669 An empty exposure for a given patch.
670 """
671 exp = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
672 exp.getMaskedImage().set(numpy.nan, afwImage.Mask
673 .getPlaneBitMask("NO_DATA"), numpy.inf)
674 return exp
675
676 def getWarpTypeList(self):
677 """Return list of requested warp types per the config.
678 """
679 warpTypeList = []
680 if self.config.makeDirect:
681 warpTypeList.append("direct")
682 if self.config.makePsfMatched:
683 warpTypeList.append("psfMatched")
684 return warpTypeList
685
686
687def reorderRefs(inputRefs, outputSortKeyOrder, dataIdKey):
688 """Reorder inputRefs per outputSortKeyOrder.
689
690 Any inputRefs which are lists will be resorted per specified key e.g.,
691 'detector.' Only iterables will be reordered, and values can be of type
692 `lsst.pipe.base.connections.DeferredDatasetRef` or
693 `lsst.daf.butler.core.datasets.ref.DatasetRef`.
694
695 Returned lists of refs have the same length as the outputSortKeyOrder.
696 If an outputSortKey not in the inputRef, then it will be padded with None.
697 If an inputRef contains an inputSortKey that is not in the
698 outputSortKeyOrder it will be removed.
699
700 Parameters
701 ----------
702 inputRefs : `lsst.pipe.base.connections.QuantizedConnection`
703 Input references to be reordered and padded.
704 outputSortKeyOrder : `iterable`
705 Iterable of values to be compared with inputRef's dataId[dataIdKey].
706 dataIdKey : `str`
707 The data ID key in the dataRefs to compare with the outputSortKeyOrder.
708
709 Returns
710 -------
711 inputRefs: `lsst.pipe.base.connections.QuantizedConnection`
712 Quantized Connection with sorted DatasetRef values sorted if iterable.
713 """
714 for connectionName, refs in inputRefs:
715 if isinstance(refs, Iterable):
716 if hasattr(refs[0], "dataId"):
717 inputSortKeyOrder = [ref.dataId[dataIdKey] for ref in refs]
718 else:
719 inputSortKeyOrder = [ref.datasetRef.dataId[dataIdKey] for ref in refs]
720 if inputSortKeyOrder != outputSortKeyOrder:
721 setattr(inputRefs, connectionName,
722 reorderAndPadList(refs, inputSortKeyOrder, outputSortKeyOrder))
723 return inputRefs