Coverage for python/lsst/pipe/tasks/skyCorrection.py: 20%
159 statements
« prev ^ index » next coverage.py v7.5.0, created at 2024-05-03 03:41 -0700
« prev ^ index » next coverage.py v7.5.0, created at 2024-05-03 03:41 -0700
1# This file is part of pipe_tasks.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22__all__ = ["SkyCorrectionTask", "SkyCorrectionConfig"]
24import warnings
26import lsst.afw.image as afwImage
27import lsst.afw.math as afwMath
28import lsst.pipe.base.connectionTypes as cT
29import numpy as np
30from lsst.pex.config import Config, ConfigField, ConfigurableField, Field
31from lsst.pipe.base import PipelineTask, PipelineTaskConfig, PipelineTaskConnections, Struct
32from lsst.pipe.tasks.background import (
33 FocalPlaneBackground,
34 FocalPlaneBackgroundConfig,
35 MaskObjectsTask,
36 SkyMeasurementTask,
37)
38from lsst.pipe.tasks.visualizeVisit import VisualizeMosaicExpConfig, VisualizeMosaicExpTask
41def _skyFrameLookup(datasetType, registry, quantumDataId, collections):
42 """Lookup function to identify sky frames.
44 Parameters
45 ----------
46 datasetType : `lsst.daf.butler.DatasetType`
47 Dataset to lookup.
48 registry : `lsst.daf.butler.Registry`
49 Butler registry to query.
50 quantumDataId : `lsst.daf.butler.DataCoordinate`
51 Data id to transform to find sky frames.
52 The ``detector`` entry will be stripped.
53 collections : `lsst.daf.butler.CollectionSearch`
54 Collections to search through.
56 Returns
57 -------
58 results : `list` [`lsst.daf.butler.DatasetRef`]
59 List of datasets that will be used as sky calibration frames.
60 """
61 newDataId = quantumDataId.subset(registry.dimensions.conform(["instrument", "visit"]))
62 skyFrames = []
63 for dataId in registry.queryDataIds(["visit", "detector"], dataId=newDataId).expanded():
64 skyFrame = registry.findDataset(
65 datasetType, dataId, collections=collections, timespan=dataId.timespan
66 )
67 skyFrames.append(skyFrame)
68 return skyFrames
71def _reorderAndPadList(inputList, inputKeys, outputKeys, padWith=None):
72 """Match the order of one list to another, padding if necessary.
74 Parameters
75 ----------
76 inputList : `list`
77 List to be reordered and padded. Elements can be any type.
78 inputKeys : iterable
79 Iterable of values to be compared with outputKeys.
80 Length must match `inputList`.
81 outputKeys : iterable
82 Iterable of values to be compared with inputKeys.
83 padWith :
84 Any value to be inserted where one of inputKeys is not in outputKeys.
86 Returns
87 -------
88 outputList : `list`
89 Copy of inputList reordered per outputKeys and padded with `padWith`
90 so that the length matches length of outputKeys.
91 """
92 outputList = []
93 for outputKey in outputKeys:
94 if outputKey in inputKeys:
95 outputList.append(inputList[inputKeys.index(outputKey)])
96 else:
97 outputList.append(padWith)
98 return outputList
101class SkyCorrectionConnections(PipelineTaskConnections, dimensions=("instrument", "visit")):
102 rawLinker = cT.Input(
103 doc="Raw data to provide exp-visit linkage to connect calExp inputs to camera/sky calibs.",
104 name="raw",
105 multiple=True,
106 deferLoad=True,
107 storageClass="Exposure",
108 dimensions=["instrument", "exposure", "detector"],
109 )
110 calExps = cT.Input(
111 doc="Background-subtracted calibrated exposures.",
112 name="calexp",
113 multiple=True,
114 storageClass="ExposureF",
115 dimensions=["instrument", "visit", "detector"],
116 )
117 calBkgs = cT.Input(
118 doc="Subtracted backgrounds for input calibrated exposures.",
119 multiple=True,
120 name="calexpBackground",
121 storageClass="Background",
122 dimensions=["instrument", "visit", "detector"],
123 )
124 skyFrames = cT.PrerequisiteInput(
125 doc="Calibration sky frames.",
126 name="sky",
127 multiple=True,
128 storageClass="ExposureF",
129 dimensions=["instrument", "physical_filter", "detector"],
130 isCalibration=True,
131 lookupFunction=_skyFrameLookup,
132 )
133 camera = cT.PrerequisiteInput(
134 doc="Input camera.",
135 name="camera",
136 storageClass="Camera",
137 dimensions=["instrument"],
138 isCalibration=True,
139 )
140 skyCorr = cT.Output(
141 doc="Sky correction data, to be subtracted from the calibrated exposures.",
142 name="skyCorr",
143 multiple=True,
144 storageClass="Background",
145 dimensions=["instrument", "visit", "detector"],
146 )
147 calExpMosaic = cT.Output(
148 doc="Full focal plane mosaicked image of the sky corrected calibrated exposures.",
149 name="calexp_skyCorr_visit_mosaic",
150 storageClass="ImageF",
151 dimensions=["instrument", "visit"],
152 )
153 calBkgMosaic = cT.Output(
154 doc="Full focal plane mosaicked image of the sky corrected calibrated exposure backgrounds.",
155 name="calexpBackground_skyCorr_visit_mosaic",
156 storageClass="ImageF",
157 dimensions=["instrument", "visit"],
158 )
161class SkyCorrectionConfig(PipelineTaskConfig, pipelineConnections=SkyCorrectionConnections):
162 maskObjects = ConfigurableField(
163 target=MaskObjectsTask,
164 doc="Mask Objects",
165 )
166 doMaskObjects = Field(
167 dtype=bool,
168 default=True,
169 doc="Iteratively mask objects to find good sky?",
170 )
171 bgModel1 = ConfigField(
172 dtype=FocalPlaneBackgroundConfig,
173 doc="Initial background model, prior to sky frame subtraction",
174 )
175 sky = ConfigurableField(
176 target=SkyMeasurementTask,
177 doc="Sky measurement",
178 )
179 doSky = Field(
180 dtype=bool,
181 default=True,
182 doc="Do sky frame subtraction?",
183 )
184 bgModel2 = ConfigField(
185 dtype=FocalPlaneBackgroundConfig,
186 doc="Final (cleanup) background model, after sky frame subtraction",
187 )
188 doBgModel2 = Field(
189 dtype=bool,
190 default=True,
191 doc="Do final (cleanup) background model subtraction, after sky frame subtraction?",
192 )
193 binning = Field(
194 dtype=int,
195 default=8,
196 doc="Binning factor for constructing full focal plane '*_camera' output datasets",
197 )
199 def setDefaults(self):
200 Config.setDefaults(self)
201 self.bgModel2.doSmooth = True
202 self.bgModel2.minFrac = 0.5
203 self.bgModel2.xSize = 256
204 self.bgModel2.ySize = 256
205 self.bgModel2.smoothScale = 1.0
208class SkyCorrectionTask(PipelineTask):
209 """Perform a full focal plane sky correction."""
211 ConfigClass = SkyCorrectionConfig
212 _DefaultName = "skyCorr"
214 def __init__(self, *args, **kwargs):
215 super().__init__(**kwargs)
216 self.makeSubtask("sky")
217 self.makeSubtask("maskObjects")
219 def runQuantum(self, butlerQC, inputRefs, outputRefs):
220 # Sort the calExps, calBkgs and skyFrames inputRefs and the
221 # skyCorr outputRef by detector ID to ensure reproducibility.
222 detectorOrder = [ref.dataId["detector"] for ref in inputRefs.calExps]
223 detectorOrder.sort()
224 inputRefs.calExps = _reorderAndPadList(
225 inputRefs.calExps, [ref.dataId["detector"] for ref in inputRefs.calExps], detectorOrder
226 )
227 inputRefs.calBkgs = _reorderAndPadList(
228 inputRefs.calBkgs, [ref.dataId["detector"] for ref in inputRefs.calBkgs], detectorOrder
229 )
230 inputRefs.skyFrames = _reorderAndPadList(
231 inputRefs.skyFrames, [ref.dataId["detector"] for ref in inputRefs.skyFrames], detectorOrder
232 )
233 outputRefs.skyCorr = _reorderAndPadList(
234 outputRefs.skyCorr, [ref.dataId["detector"] for ref in outputRefs.skyCorr], detectorOrder
235 )
236 inputs = butlerQC.get(inputRefs)
237 inputs.pop("rawLinker", None)
238 outputs = self.run(**inputs)
239 butlerQC.put(outputs, outputRefs)
241 def run(self, calExps, calBkgs, skyFrames, camera):
242 """Perform sky correction on a visit.
244 The original visit-level background is first restored to the calibrated
245 exposure and the existing background model is inverted in-place. If
246 doMaskObjects is True, the mask map associated with this exposure will
247 be iteratively updated (over nIter loops) by re-estimating the
248 background each iteration and redetecting footprints.
250 An initial full focal plane sky subtraction (bgModel1) will take place
251 prior to scaling and subtracting the sky frame.
253 If doSky is True, the sky frame will be scaled to the flux in the input
254 visit.
256 If doBgModel2 is True, a final full focal plane sky subtraction will
257 take place after the sky frame has been subtracted.
259 The first N elements of the returned skyCorr will consist of inverted
260 elements of the calexpBackground model (i.e., subtractive). All
261 subsequent elements appended to skyCorr thereafter will be additive
262 such that, when skyCorr is subtracted from a calexp, the net result
263 will be to undo the initial per-detector background solution and then
264 apply the skyCorr model thereafter. Adding skyCorr to a
265 calexpBackground will effectively negate the calexpBackground,
266 returning only the additive background components of the skyCorr
267 background model.
269 Parameters
270 ----------
271 calExps : `list` [`lsst.afw.image.exposure.ExposureF`]
272 Detector calibrated exposure images for the visit.
273 calBkgs : `list` [`lsst.afw.math.BackgroundList`]
274 Detector background lists matching the calibrated exposures.
275 skyFrames : `list` [`lsst.afw.image.exposure.ExposureF`]
276 Sky frame calibration data for the input detectors.
277 camera : `lsst.afw.cameraGeom.Camera`
278 Camera matching the input data to process.
280 Returns
281 -------
282 results : `Struct` containing:
283 skyCorr : `list` [`lsst.afw.math.BackgroundList`]
284 Detector-level sky correction background lists.
285 calExpMosaic : `lsst.afw.image.exposure.ExposureF`
286 Visit-level mosaic of the sky corrected data, binned.
287 Analogous to `calexp - skyCorr`.
288 calBkgMosaic : `lsst.afw.image.exposure.ExposureF`
289 Visit-level mosaic of the sky correction background, binned.
290 Analogous to `calexpBackground + skyCorr`.
291 """
292 # Restore original backgrounds in-place; optionally refine mask maps
293 numOrigBkgElements = [len(calBkg) for calBkg in calBkgs]
294 _ = self._restoreBackgroundRefineMask(calExps, calBkgs)
296 # Bin exposures, generate full-fp bg, map to CCDs and subtract in-place
297 _ = self._subtractVisitBackground(calExps, calBkgs, camera, self.config.bgModel1)
299 # Subtract a scaled sky frame from all input exposures
300 if self.config.doSky:
301 self._subtractSkyFrame(calExps, skyFrames, calBkgs)
303 # Bin exposures, generate full-fp bg, map to CCDs and subtract in-place
304 if self.config.doBgModel2:
305 _ = self._subtractVisitBackground(calExps, calBkgs, camera, self.config.bgModel2)
307 # Make camera-level images of bg subtracted calexps and subtracted bgs
308 calExpIds = [exp.getDetector().getId() for exp in calExps]
309 skyCorrExtras = []
310 for calBkg, num in zip(calBkgs, numOrigBkgElements):
311 skyCorrExtra = calBkg.clone()
312 skyCorrExtra._backgrounds = skyCorrExtra._backgrounds[num:]
313 skyCorrExtras.append(skyCorrExtra)
314 calExpMosaic = self._binAndMosaic(calExps, camera, self.config.binning, ids=calExpIds, refExps=None)
315 calBkgMosaic = self._binAndMosaic(
316 skyCorrExtras, camera, self.config.binning, ids=calExpIds, refExps=calExps
317 )
319 return Struct(skyCorr=calBkgs, calExpMosaic=calExpMosaic, calBkgMosaic=calBkgMosaic)
321 def _restoreBackgroundRefineMask(self, calExps, calBkgs):
322 """Restore original background to each calexp and invert the related
323 background model; optionally refine the mask plane.
325 The original visit-level background is restored to each calibrated
326 exposure and the existing background model is inverted in-place. If
327 doMaskObjects is True, the mask map associated with the exposure will
328 be iteratively updated (over nIter loops) by re-estimating the
329 background each iteration and redetecting footprints.
331 The background model modified in-place in this method will comprise the
332 first N elements of the skyCorr dataset type, i.e., these N elements
333 are the inverse of the calexpBackground model. All subsequent elements
334 appended to skyCorr will be additive such that, when skyCorr is
335 subtracted from a calexp, the net result will be to undo the initial
336 per-detector background solution and then apply the skyCorr model
337 thereafter. Adding skyCorr to a calexpBackground will effectively
338 negate the calexpBackground, returning only the additive background
339 components of the skyCorr background model.
341 Parameters
342 ----------
343 calExps : `lsst.afw.image.exposure.ExposureF`
344 Detector level calexp images to process.
345 calBkgs : `lsst.afw.math._backgroundList.BackgroundList`
346 Detector level background lists associated with the calexps.
348 Returns
349 -------
350 calExps : `lsst.afw.image.exposure.ExposureF`
351 The calexps with the initially subtracted background restored.
352 skyCorrBases : `lsst.afw.math._backgroundList.BackgroundList`
353 The inverted initial background models; the genesis for skyCorrs.
354 """
355 skyCorrBases = []
356 for calExp, calBkg in zip(calExps, calBkgs):
357 image = calExp.getMaskedImage()
359 # Invert all elements of the existing bg model; restore in calexp
360 for calBkgElement in calBkg:
361 statsImage = calBkgElement[0].getStatsImage()
362 statsImage *= -1
363 skyCorrBase = calBkg.getImage()
364 image -= skyCorrBase
366 # Iteratively subtract bg, re-detect sources, and add bg back on
367 if self.config.doMaskObjects:
368 self.maskObjects.findObjects(calExp)
370 stats = np.nanpercentile(skyCorrBase.array, [50, 75, 25])
371 self.log.info(
372 "Detector %d: Initial background restored; BG median = %.1f counts, BG IQR = %.1f counts",
373 calExp.getDetector().getId(),
374 -stats[0],
375 np.subtract(*stats[1:]),
376 )
377 skyCorrBases.append(skyCorrBase)
378 return calExps, skyCorrBases
380 def _subtractVisitBackground(self, calExps, calBkgs, camera, config):
381 """Perform a full focal-plane background subtraction for a visit.
383 Generate a full focal plane background model, binning all masked
384 detectors into bins of [bgModelN.xSize, bgModelN.ySize]. After,
385 subtract the resultant background model (translated back into CCD
386 coordinates) from the original detector exposure.
388 Return a list of background subtracted images and a list of full focal
389 plane background parameters.
391 Parameters
392 ----------
393 calExps : `list` [`lsst.afw.image.exposure.ExposureF`]
394 Calibrated exposures to be background subtracted.
395 calBkgs : `list` [`lsst.afw.math._backgroundList.BackgroundList`]
396 Background lists associated with the input calibrated exposures.
397 camera : `lsst.afw.cameraGeom.Camera`
398 Camera description.
399 config : `lsst.pipe.tasks.background.FocalPlaneBackgroundConfig`
400 Configuration to use for background subtraction.
402 Returns
403 -------
404 calExps : `list` [`lsst.afw.image.maskedImage.MaskedImageF`]
405 Background subtracted exposures for creating a focal plane image.
406 calBkgs : `list` [`lsst.afw.math._backgroundList.BackgroundList`]
407 Updated background lists with a visit-level model appended.
408 """
409 # Set up empty full focal plane background model object
410 bgModelBase = FocalPlaneBackground.fromCamera(config, camera)
412 # Loop over each detector, bin into [xSize, ySize] bins, and update
413 # summed flux (_values) and number of contributing pixels (_numbers)
414 # in focal plane coordinates. Append outputs to bgModels.
415 bgModels = []
416 for calExp in calExps:
417 bgModel = bgModelBase.clone()
418 bgModel.addCcd(calExp)
419 bgModels.append(bgModel)
421 # Merge detector models to make a single full focal plane bg model
422 for bgModel, calExp in zip(bgModels, calExps):
423 msg = (
424 "Detector %d: Merging %d unmasked pixels (%.1f%s of detector area) into focal plane "
425 "background model"
426 )
427 self.log.debug(
428 msg,
429 calExp.getDetector().getId(),
430 bgModel._numbers.getArray().sum(),
431 100 * bgModel._numbers.getArray().sum() / calExp.getBBox().getArea(),
432 "%",
433 )
434 bgModelBase.merge(bgModel)
436 # Map full focal plane bg solution to detector; subtract from exposure
437 calBkgElements = []
438 for calExp in calExps:
439 _, calBkgElement = self._subtractDetectorBackground(calExp, bgModelBase)
440 calBkgElements.append(calBkgElement)
442 msg = (
443 "Focal plane background model constructed using %.2f x %.2f mm (%d x %d pixel) superpixels; "
444 "FP BG median = %.1f counts, FP BG IQR = %.1f counts"
445 )
446 with warnings.catch_warnings():
447 warnings.filterwarnings("ignore", r"invalid value encountered")
448 stats = np.nanpercentile(bgModelBase.getStatsImage().array, [50, 75, 25])
449 self.log.info(
450 msg,
451 config.xSize,
452 config.ySize,
453 int(config.xSize / config.pixelSize),
454 int(config.ySize / config.pixelSize),
455 stats[0],
456 np.subtract(*stats[1:]),
457 )
459 for calBkg, calBkgElement in zip(calBkgs, calBkgElements):
460 calBkg.append(calBkgElement[0])
461 return calExps, calBkgs
463 def _subtractDetectorBackground(self, calExp, bgModel):
464 """Generate CCD background model and subtract from image.
466 Translate the full focal plane background into CCD coordinates and
467 subtract from the original science exposure image.
469 Parameters
470 ----------
471 calExp : `lsst.afw.image.exposure.ExposureF`
472 Exposure to subtract the background model from.
473 bgModel : `lsst.pipe.tasks.background.FocalPlaneBackground`
474 Full focal plane camera-level background model.
476 Returns
477 -------
478 calExp : `lsst.afw.image.exposure.ExposureF`
479 Background subtracted input exposure.
480 calBkgElement : `lsst.afw.math._backgroundList.BackgroundList`
481 Detector level realization of the full focal plane bg model.
482 """
483 image = calExp.getMaskedImage()
484 with warnings.catch_warnings():
485 warnings.filterwarnings("ignore", r"invalid value encountered")
486 calBkgElement = bgModel.toCcdBackground(calExp.getDetector(), image.getBBox())
487 image -= calBkgElement.getImage()
488 return calExp, calBkgElement
490 def _subtractSkyFrame(self, calExps, skyFrames, calBkgs):
491 """Determine the full focal plane sky frame scale factor relative to
492 an input list of calibrated exposures and subtract.
494 This method measures the sky frame scale on all inputs, resulting in
495 values equal to the background method solveScales(). The sky frame is
496 then subtracted as in subtractSkyFrame() using the appropriate scale.
498 Input calExps and calBkgs are updated in-place, returning sky frame
499 subtracted calExps and sky frame updated calBkgs, respectively.
501 Parameters
502 ----------
503 calExps : `list` [`lsst.afw.image.exposure.ExposureF`]
504 Calibrated exposures to be background subtracted.
505 skyFrames : `list` [`lsst.afw.image.exposure.ExposureF`]
506 Sky frame calibration data for the input detectors.
507 calBkgs : `list` [`lsst.afw.math._backgroundList.BackgroundList`]
508 Background lists associated with the input calibrated exposures.
509 """
510 skyFrameBgModels = []
511 scales = []
512 for calExp, skyFrame in zip(calExps, skyFrames):
513 skyFrameBgModel = self.sky.exposureToBackground(skyFrame)
514 skyFrameBgModels.append(skyFrameBgModel)
515 # return a tuple of gridded image and sky frame clipped means
516 samples = self.sky.measureScale(calExp.getMaskedImage(), skyFrameBgModel)
517 scales.append(samples)
518 scale = self.sky.solveScales(scales)
519 for calExp, skyFrameBgModel, calBkg in zip(calExps, skyFrameBgModels, calBkgs):
520 # subtract the scaled sky frame model from each calExp in-place,
521 # also updating the calBkg list in-place
522 self.sky.subtractSkyFrame(calExp.getMaskedImage(), skyFrameBgModel, scale, calBkg)
523 self.log.info("Sky frame subtracted with a scale factor of %.5f", scale)
525 def _binAndMosaic(self, exposures, camera, binning, ids=None, refExps=None):
526 """Bin input exposures and mosaic across the entire focal plane.
528 Input exposures are binned and then mosaicked at the position of
529 the detector in the focal plane of the camera.
531 Parameters
532 ----------
533 exposures : `list`
534 Detector level list of either calexp `ExposureF` types or
535 calexpBackground `BackgroundList` types.
536 camera : `lsst.afw.cameraGeom.Camera`
537 Camera matching the input data to process.
538 binning : `int`
539 Binning size to be applied to input images.
540 ids : `list` [`int`], optional
541 List of detector ids to iterate over.
542 refExps : `list` [`lsst.afw.image.exposure.ExposureF`], optional
543 If supplied, mask planes from these reference images will be used.
544 Returns
545 -------
546 mosaicImage : `lsst.afw.image.exposure.ExposureF`
547 Mosaicked full focal plane image.
548 """
549 refExps = np.resize(refExps, len(exposures)) # type: ignore
550 binnedImages = []
551 for exp, refExp in zip(exposures, refExps):
552 try:
553 nativeImage = exp.getMaskedImage()
554 except AttributeError:
555 nativeImage = afwImage.makeMaskedImage(exp.getImage())
556 if refExp:
557 nativeImage.setMask(refExp.getMask())
558 binnedImage = afwMath.binImage(nativeImage, binning)
559 binnedImages.append(binnedImage)
560 mosConfig = VisualizeMosaicExpConfig()
561 mosConfig.binning = binning
562 mosTask = VisualizeMosaicExpTask(config=mosConfig)
563 imageStruct = mosTask.run(binnedImages, camera, inputIds=ids)
564 mosaicImage = imageStruct.outputData
565 return mosaicImage