Coverage for python/lsst/pipe/tasks/skyCorrection.py: 21%
168 statements
« prev ^ index » next coverage.py v7.4.1, created at 2024-02-20 12:27 +0000
« prev ^ index » next coverage.py v7.4.1, created at 2024-02-20 12:27 +0000
1# This file is part of pipe_tasks.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22__all__ = ["SkyCorrectionTask", "SkyCorrectionConfig"]
24import warnings
26import lsst.afw.image as afwImage
27import lsst.afw.math as afwMath
28import lsst.pipe.base.connectionTypes as cT
29import numpy as np
30from lsst.pex.config import Config, ConfigField, ConfigurableField, Field, FieldValidationError
31from lsst.pipe.base import PipelineTask, PipelineTaskConfig, PipelineTaskConnections, Struct
32from lsst.pipe.tasks.background import (
33 FocalPlaneBackground,
34 FocalPlaneBackgroundConfig,
35 MaskObjectsTask,
36 SkyMeasurementTask,
37)
38from lsst.pipe.tasks.visualizeVisit import VisualizeMosaicExpConfig, VisualizeMosaicExpTask
41def _skyFrameLookup(datasetType, registry, quantumDataId, collections):
42 """Lookup function to identify sky frames.
44 Parameters
45 ----------
46 datasetType : `lsst.daf.butler.DatasetType`
47 Dataset to lookup.
48 registry : `lsst.daf.butler.Registry`
49 Butler registry to query.
50 quantumDataId : `lsst.daf.butler.DataCoordinate`
51 Data id to transform to find sky frames.
52 The ``detector`` entry will be stripped.
53 collections : `lsst.daf.butler.CollectionSearch`
54 Collections to search through.
56 Returns
57 -------
58 results : `list` [`lsst.daf.butler.DatasetRef`]
59 List of datasets that will be used as sky calibration frames.
60 """
61 newDataId = quantumDataId.subset(registry.dimensions.conform(["instrument", "visit"]))
62 skyFrames = []
63 for dataId in registry.queryDataIds(["visit", "detector"], dataId=newDataId).expanded():
64 skyFrame = registry.findDataset(
65 datasetType, dataId, collections=collections, timespan=dataId.timespan
66 )
67 skyFrames.append(skyFrame)
68 return skyFrames
71def _reorderAndPadList(inputList, inputKeys, outputKeys, padWith=None):
72 """Match the order of one list to another, padding if necessary.
74 Parameters
75 ----------
76 inputList : `list`
77 List to be reordered and padded. Elements can be any type.
78 inputKeys : iterable
79 Iterable of values to be compared with outputKeys.
80 Length must match `inputList`.
81 outputKeys : iterable
82 Iterable of values to be compared with inputKeys.
83 padWith :
84 Any value to be inserted where one of inputKeys is not in outputKeys.
86 Returns
87 -------
88 outputList : `list`
89 Copy of inputList reordered per outputKeys and padded with `padWith`
90 so that the length matches length of outputKeys.
91 """
92 outputList = []
93 for outputKey in outputKeys:
94 if outputKey in inputKeys:
95 outputList.append(inputList[inputKeys.index(outputKey)])
96 else:
97 outputList.append(padWith)
98 return outputList
101class SkyCorrectionConnections(PipelineTaskConnections, dimensions=("instrument", "visit")):
102 rawLinker = cT.Input(
103 doc="Raw data to provide exp-visit linkage to connect calExp inputs to camera/sky calibs.",
104 name="raw",
105 multiple=True,
106 deferLoad=True,
107 storageClass="Exposure",
108 dimensions=["instrument", "exposure", "detector"],
109 )
110 calExps = cT.Input(
111 doc="Background-subtracted calibrated exposures.",
112 name="calexp",
113 multiple=True,
114 storageClass="ExposureF",
115 dimensions=["instrument", "visit", "detector"],
116 )
117 calBkgs = cT.Input(
118 doc="Subtracted backgrounds for input calibrated exposures.",
119 multiple=True,
120 name="calexpBackground",
121 storageClass="Background",
122 dimensions=["instrument", "visit", "detector"],
123 )
124 skyFrames = cT.PrerequisiteInput(
125 doc="Calibration sky frames.",
126 name="sky",
127 multiple=True,
128 storageClass="ExposureF",
129 dimensions=["instrument", "physical_filter", "detector"],
130 isCalibration=True,
131 lookupFunction=_skyFrameLookup,
132 )
133 camera = cT.PrerequisiteInput(
134 doc="Input camera.",
135 name="camera",
136 storageClass="Camera",
137 dimensions=["instrument"],
138 isCalibration=True,
139 )
140 skyCorr = cT.Output(
141 doc="Sky correction data, to be subtracted from the calibrated exposures.",
142 name="skyCorr",
143 multiple=True,
144 storageClass="Background",
145 dimensions=["instrument", "visit", "detector"],
146 )
147 calExpMosaic = cT.Output(
148 doc="Full focal plane mosaicked image of the sky corrected calibrated exposures.",
149 name="calexp_skyCorr_visit_mosaic",
150 storageClass="ImageF",
151 dimensions=["instrument", "visit"],
152 )
153 calBkgMosaic = cT.Output(
154 doc="Full focal plane mosaicked image of the sky corrected calibrated exposure backgrounds.",
155 name="calexpBackground_skyCorr_visit_mosaic",
156 storageClass="ImageF",
157 dimensions=["instrument", "visit"],
158 )
161class SkyCorrectionConfig(PipelineTaskConfig, pipelineConnections=SkyCorrectionConnections):
162 maskObjects = ConfigurableField(
163 target=MaskObjectsTask,
164 doc="Mask Objects",
165 )
166 doMaskObjects = Field(
167 dtype=bool,
168 default=True,
169 doc="Iteratively mask objects to find good sky?",
170 )
171 bgModel = ConfigField(
172 dtype=Config,
173 doc="Initial background model, prior to sky frame subtraction",
174 deprecated="This field is deprecated and will be removed after v26. Please use bgModel1 instead.",
175 )
176 doBgModel = Field(
177 dtype=bool,
178 default=None,
179 doc="Do initial background model subtraction (prior to sky frame subtraction)?",
180 optional=True,
181 deprecated="This field is deprecated and will be removed after v26. See RFC-898 for further details.",
182 )
183 bgModel1 = ConfigField(
184 dtype=FocalPlaneBackgroundConfig,
185 doc="Initial background model, prior to sky frame subtraction",
186 )
187 doBgModel1 = Field(
188 dtype=bool,
189 default=True,
190 doc="Do initial background model subtraction (prior to sky frame subtraction)?",
191 deprecated="This field is deprecated and will be removed after v26. See RFC-898 for further details.",
192 )
193 sky = ConfigurableField(
194 target=SkyMeasurementTask,
195 doc="Sky measurement",
196 )
197 doSky = Field(
198 dtype=bool,
199 default=True,
200 doc="Do sky frame subtraction?",
201 )
202 bgModel2 = ConfigField(
203 dtype=FocalPlaneBackgroundConfig,
204 doc="Final (cleanup) background model, after sky frame subtraction",
205 )
206 doBgModel2 = Field(
207 dtype=bool,
208 default=True,
209 doc="Do final (cleanup) background model subtraction, after sky frame subtraction?",
210 )
211 binning = Field(
212 dtype=int,
213 default=8,
214 doc="Binning factor for constructing full focal plane '*_camera' output datasets",
215 )
217 def setDefaults(self):
218 Config.setDefaults(self)
219 self.bgModel2.doSmooth = True
220 self.bgModel2.minFrac = 0.5
221 self.bgModel2.xSize = 256
222 self.bgModel2.ySize = 256
223 self.bgModel2.smoothScale = 1.0
225 def validate(self):
226 # TODO: Entire validate method may be removed after v26 (a la DM-37242)
227 super().validate()
228 if self.doBgModel is not None and self.doBgModel != self.doBgModel1:
229 msg = "The doBgModel field will be removed after v26."
230 raise FieldValidationError(self.__class__.doBgModel, self, msg)
233class SkyCorrectionTask(PipelineTask):
234 """Perform a full focal plane sky correction."""
236 ConfigClass = SkyCorrectionConfig
237 _DefaultName = "skyCorr"
239 def __init__(self, *args, **kwargs):
240 super().__init__(**kwargs)
241 self.makeSubtask("sky")
242 self.makeSubtask("maskObjects")
244 def runQuantum(self, butlerQC, inputRefs, outputRefs):
245 # Sort the calExps, calBkgs and skyFrames inputRefs and the
246 # skyCorr outputRef by detector ID to ensure reproducibility.
247 detectorOrder = [ref.dataId["detector"] for ref in inputRefs.calExps]
248 detectorOrder.sort()
249 inputRefs.calExps = _reorderAndPadList(
250 inputRefs.calExps, [ref.dataId["detector"] for ref in inputRefs.calExps], detectorOrder
251 )
252 inputRefs.calBkgs = _reorderAndPadList(
253 inputRefs.calBkgs, [ref.dataId["detector"] for ref in inputRefs.calBkgs], detectorOrder
254 )
255 inputRefs.skyFrames = _reorderAndPadList(
256 inputRefs.skyFrames, [ref.dataId["detector"] for ref in inputRefs.skyFrames], detectorOrder
257 )
258 outputRefs.skyCorr = _reorderAndPadList(
259 outputRefs.skyCorr, [ref.dataId["detector"] for ref in outputRefs.skyCorr], detectorOrder
260 )
261 inputs = butlerQC.get(inputRefs)
262 inputs.pop("rawLinker", None)
263 outputs = self.run(**inputs)
264 butlerQC.put(outputs, outputRefs)
266 def run(self, calExps, calBkgs, skyFrames, camera):
267 """Perform sky correction on a visit.
269 The original visit-level background is first restored to the calibrated
270 exposure and the existing background model is inverted in-place. If
271 doMaskObjects is True, the mask map associated with this exposure will
272 be iteratively updated (over nIter loops) by re-estimating the
273 background each iteration and redetecting footprints.
275 If doBgModel1 is True, an initial full focal plane sky subtraction will
276 take place prior to scaling and subtracting the sky frame.
278 If doSky is True, the sky frame will be scaled to the flux in the input
279 visit.
281 If doBgModel2 is True, a final full focal plane sky subtraction will
282 take place after the sky frame has been subtracted.
284 The first N elements of the returned skyCorr will consist of inverted
285 elements of the calexpBackground model (i.e., subtractive). All
286 subsequent elements appended to skyCorr thereafter will be additive
287 such that, when skyCorr is subtracted from a calexp, the net result
288 will be to undo the initial per-detector background solution and then
289 apply the skyCorr model thereafter. Adding skyCorr to a
290 calexpBackground will effectively negate the calexpBackground,
291 returning only the additive background components of the skyCorr
292 background model.
294 Parameters
295 ----------
296 calExps : `list` [`lsst.afw.image.exposure.ExposureF`]
297 Detector calibrated exposure images for the visit.
298 calBkgs : `list` [`lsst.afw.math.BackgroundList`]
299 Detector background lists matching the calibrated exposures.
300 skyFrames : `list` [`lsst.afw.image.exposure.ExposureF`]
301 Sky frame calibration data for the input detectors.
302 camera : `lsst.afw.cameraGeom.Camera`
303 Camera matching the input data to process.
305 Returns
306 -------
307 results : `Struct` containing:
308 skyCorr : `list` [`lsst.afw.math.BackgroundList`]
309 Detector-level sky correction background lists.
310 calExpMosaic : `lsst.afw.image.exposure.ExposureF`
311 Visit-level mosaic of the sky corrected data, binned.
312 Analogous to `calexp - skyCorr`.
313 calBkgMosaic : `lsst.afw.image.exposure.ExposureF`
314 Visit-level mosaic of the sky correction background, binned.
315 Analogous to `calexpBackground + skyCorr`.
316 """
317 # Restore original backgrounds in-place; optionally refine mask maps
318 numOrigBkgElements = [len(calBkg) for calBkg in calBkgs]
319 _ = self._restoreBackgroundRefineMask(calExps, calBkgs)
321 # Bin exposures, generate full-fp bg, map to CCDs and subtract in-place
322 if self.config.doBgModel1:
323 _ = self._subtractVisitBackground(calExps, calBkgs, camera, self.config.bgModel1)
325 # Subtract a scaled sky frame from all input exposures
326 if self.config.doSky:
327 self._subtractSkyFrame(calExps, skyFrames, calBkgs)
329 # Bin exposures, generate full-fp bg, map to CCDs and subtract in-place
330 if self.config.doBgModel2:
331 _ = self._subtractVisitBackground(calExps, calBkgs, camera, self.config.bgModel2)
333 # Make camera-level images of bg subtracted calexps and subtracted bgs
334 calExpIds = [exp.getDetector().getId() for exp in calExps]
335 skyCorrExtras = []
336 for calBkg, num in zip(calBkgs, numOrigBkgElements):
337 skyCorrExtra = calBkg.clone()
338 skyCorrExtra._backgrounds = skyCorrExtra._backgrounds[num:]
339 skyCorrExtras.append(skyCorrExtra)
340 calExpMosaic = self._binAndMosaic(calExps, camera, self.config.binning, ids=calExpIds, refExps=None)
341 calBkgMosaic = self._binAndMosaic(
342 skyCorrExtras, camera, self.config.binning, ids=calExpIds, refExps=calExps
343 )
345 return Struct(skyCorr=calBkgs, calExpMosaic=calExpMosaic, calBkgMosaic=calBkgMosaic)
347 def _restoreBackgroundRefineMask(self, calExps, calBkgs):
348 """Restore original background to each calexp and invert the related
349 background model; optionally refine the mask plane.
351 The original visit-level background is restored to each calibrated
352 exposure and the existing background model is inverted in-place. If
353 doMaskObjects is True, the mask map associated with the exposure will
354 be iteratively updated (over nIter loops) by re-estimating the
355 background each iteration and redetecting footprints.
357 The background model modified in-place in this method will comprise the
358 first N elements of the skyCorr dataset type, i.e., these N elements
359 are the inverse of the calexpBackground model. All subsequent elements
360 appended to skyCorr will be additive such that, when skyCorr is
361 subtracted from a calexp, the net result will be to undo the initial
362 per-detector background solution and then apply the skyCorr model
363 thereafter. Adding skyCorr to a calexpBackground will effectively
364 negate the calexpBackground, returning only the additive background
365 components of the skyCorr background model.
367 Parameters
368 ----------
369 calExps : `lsst.afw.image.exposure.ExposureF`
370 Detector level calexp images to process.
371 calBkgs : `lsst.afw.math._backgroundList.BackgroundList`
372 Detector level background lists associated with the calexps.
374 Returns
375 -------
376 calExps : `lsst.afw.image.exposure.ExposureF`
377 The calexps with the initially subtracted background restored.
378 skyCorrBases : `lsst.afw.math._backgroundList.BackgroundList`
379 The inverted initial background models; the genesis for skyCorrs.
380 """
381 skyCorrBases = []
382 for calExp, calBkg in zip(calExps, calBkgs):
383 image = calExp.getMaskedImage()
385 # Invert all elements of the existing bg model; restore in calexp
386 for calBkgElement in calBkg:
387 statsImage = calBkgElement[0].getStatsImage()
388 statsImage *= -1
389 skyCorrBase = calBkg.getImage()
390 image -= skyCorrBase
392 # Iteratively subtract bg, re-detect sources, and add bg back on
393 if self.config.doMaskObjects:
394 self.maskObjects.findObjects(calExp)
396 stats = np.nanpercentile(skyCorrBase.array, [50, 75, 25])
397 self.log.info(
398 "Detector %d: Initial background restored; BG median = %.1f counts, BG IQR = %.1f counts",
399 calExp.getDetector().getId(),
400 -stats[0],
401 np.subtract(*stats[1:]),
402 )
403 skyCorrBases.append(skyCorrBase)
404 return calExps, skyCorrBases
406 def _subtractVisitBackground(self, calExps, calBkgs, camera, config):
407 """Perform a full focal-plane background subtraction for a visit.
409 Generate a full focal plane background model, binning all masked
410 detectors into bins of [bgModelN.xSize, bgModelN.ySize]. After,
411 subtract the resultant background model (translated back into CCD
412 coordinates) from the original detector exposure.
414 Return a list of background subtracted images and a list of full focal
415 plane background parameters.
417 Parameters
418 ----------
419 calExps : `list` [`lsst.afw.image.exposure.ExposureF`]
420 Calibrated exposures to be background subtracted.
421 calBkgs : `list` [`lsst.afw.math._backgroundList.BackgroundList`]
422 Background lists associated with the input calibrated exposures.
423 camera : `lsst.afw.cameraGeom.Camera`
424 Camera description.
425 config : `lsst.pipe.tasks.background.FocalPlaneBackgroundConfig`
426 Configuration to use for background subtraction.
428 Returns
429 -------
430 calExps : `list` [`lsst.afw.image.maskedImage.MaskedImageF`]
431 Background subtracted exposures for creating a focal plane image.
432 calBkgs : `list` [`lsst.afw.math._backgroundList.BackgroundList`]
433 Updated background lists with a visit-level model appended.
434 """
435 # Set up empty full focal plane background model object
436 bgModelBase = FocalPlaneBackground.fromCamera(config, camera)
438 # Loop over each detector, bin into [xSize, ySize] bins, and update
439 # summed flux (_values) and number of contributing pixels (_numbers)
440 # in focal plane coordinates. Append outputs to bgModels.
441 bgModels = []
442 for calExp in calExps:
443 bgModel = bgModelBase.clone()
444 bgModel.addCcd(calExp)
445 bgModels.append(bgModel)
447 # Merge detector models to make a single full focal plane bg model
448 for bgModel, calExp in zip(bgModels, calExps):
449 msg = (
450 "Detector %d: Merging %d unmasked pixels (%.1f%s of detector area) into focal plane "
451 "background model"
452 )
453 self.log.debug(
454 msg,
455 calExp.getDetector().getId(),
456 bgModel._numbers.getArray().sum(),
457 100 * bgModel._numbers.getArray().sum() / calExp.getBBox().getArea(),
458 "%",
459 )
460 bgModelBase.merge(bgModel)
462 # Map full focal plane bg solution to detector; subtract from exposure
463 calBkgElements = []
464 for calExp in calExps:
465 _, calBkgElement = self._subtractDetectorBackground(calExp, bgModelBase)
466 calBkgElements.append(calBkgElement)
468 msg = (
469 "Focal plane background model constructed using %.2f x %.2f mm (%d x %d pixel) superpixels; "
470 "FP BG median = %.1f counts, FP BG IQR = %.1f counts"
471 )
472 with warnings.catch_warnings():
473 warnings.filterwarnings("ignore", r"invalid value encountered")
474 stats = np.nanpercentile(bgModelBase.getStatsImage().array, [50, 75, 25])
475 self.log.info(
476 msg,
477 config.xSize,
478 config.ySize,
479 int(config.xSize / config.pixelSize),
480 int(config.ySize / config.pixelSize),
481 stats[0],
482 np.subtract(*stats[1:]),
483 )
485 for calBkg, calBkgElement in zip(calBkgs, calBkgElements):
486 calBkg.append(calBkgElement[0])
487 return calExps, calBkgs
489 def _subtractDetectorBackground(self, calExp, bgModel):
490 """Generate CCD background model and subtract from image.
492 Translate the full focal plane background into CCD coordinates and
493 subtract from the original science exposure image.
495 Parameters
496 ----------
497 calExp : `lsst.afw.image.exposure.ExposureF`
498 Exposure to subtract the background model from.
499 bgModel : `lsst.pipe.tasks.background.FocalPlaneBackground`
500 Full focal plane camera-level background model.
502 Returns
503 -------
504 calExp : `lsst.afw.image.exposure.ExposureF`
505 Background subtracted input exposure.
506 calBkgElement : `lsst.afw.math._backgroundList.BackgroundList`
507 Detector level realization of the full focal plane bg model.
508 """
509 image = calExp.getMaskedImage()
510 with warnings.catch_warnings():
511 warnings.filterwarnings("ignore", r"invalid value encountered")
512 calBkgElement = bgModel.toCcdBackground(calExp.getDetector(), image.getBBox())
513 image -= calBkgElement.getImage()
514 return calExp, calBkgElement
516 def _subtractSkyFrame(self, calExps, skyFrames, calBkgs):
517 """Determine the full focal plane sky frame scale factor relative to
518 an input list of calibrated exposures and subtract.
520 This method measures the sky frame scale on all inputs, resulting in
521 values equal to the background method solveScales(). The sky frame is
522 then subtracted as in subtractSkyFrame() using the appropriate scale.
524 Input calExps and calBkgs are updated in-place, returning sky frame
525 subtracted calExps and sky frame updated calBkgs, respectively.
527 Parameters
528 ----------
529 calExps : `list` [`lsst.afw.image.exposure.ExposureF`]
530 Calibrated exposures to be background subtracted.
531 skyFrames : `list` [`lsst.afw.image.exposure.ExposureF`]
532 Sky frame calibration data for the input detectors.
533 calBkgs : `list` [`lsst.afw.math._backgroundList.BackgroundList`]
534 Background lists associated with the input calibrated exposures.
535 """
536 skyFrameBgModels = []
537 scales = []
538 for calExp, skyFrame in zip(calExps, skyFrames):
539 skyFrameBgModel = self.sky.exposureToBackground(skyFrame)
540 skyFrameBgModels.append(skyFrameBgModel)
541 # return a tuple of gridded image and sky frame clipped means
542 samples = self.sky.measureScale(calExp.getMaskedImage(), skyFrameBgModel)
543 scales.append(samples)
544 scale = self.sky.solveScales(scales)
545 for calExp, skyFrameBgModel, calBkg in zip(calExps, skyFrameBgModels, calBkgs):
546 # subtract the scaled sky frame model from each calExp in-place,
547 # also updating the calBkg list in-place
548 self.sky.subtractSkyFrame(calExp.getMaskedImage(), skyFrameBgModel, scale, calBkg)
549 self.log.info("Sky frame subtracted with a scale factor of %.5f", scale)
551 def _binAndMosaic(self, exposures, camera, binning, ids=None, refExps=None):
552 """Bin input exposures and mosaic across the entire focal plane.
554 Input exposures are binned and then mosaicked at the position of
555 the detector in the focal plane of the camera.
557 Parameters
558 ----------
559 exposures : `list`
560 Detector level list of either calexp `ExposureF` types or
561 calexpBackground `BackgroundList` types.
562 camera : `lsst.afw.cameraGeom.Camera`
563 Camera matching the input data to process.
564 binning : `int`
565 Binning size to be applied to input images.
566 ids : `list` [`int`], optional
567 List of detector ids to iterate over.
568 refExps : `list` [`lsst.afw.image.exposure.ExposureF`], optional
569 If supplied, mask planes from these reference images will be used.
570 Returns
571 -------
572 mosaicImage : `lsst.afw.image.exposure.ExposureF`
573 Mosaicked full focal plane image.
574 """
575 refExps = np.resize(refExps, len(exposures)) # type: ignore
576 binnedImages = []
577 for exp, refExp in zip(exposures, refExps):
578 try:
579 nativeImage = exp.getMaskedImage()
580 except AttributeError:
581 nativeImage = afwImage.makeMaskedImage(exp.getImage())
582 if refExp:
583 nativeImage.setMask(refExp.getMask())
584 binnedImage = afwMath.binImage(nativeImage, binning)
585 binnedImages.append(binnedImage)
586 mosConfig = VisualizeMosaicExpConfig()
587 mosConfig.binning = binning
588 mosTask = VisualizeMosaicExpTask(config=mosConfig)
589 imageStruct = mosTask.run(binnedImages, camera, inputIds=ids)
590 mosaicImage = imageStruct.outputData
591 return mosaicImage