Coverage for python/lsst/pipe/tasks/skyCorrection.py: 21%
169 statements
« prev ^ index » next coverage.py v7.3.1, created at 2023-09-23 10:45 +0000
« prev ^ index » next coverage.py v7.3.1, created at 2023-09-23 10:45 +0000
1# This file is part of pipe_tasks.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22__all__ = ["SkyCorrectionTask", "SkyCorrectionConfig"]
24import warnings
26import lsst.afw.image as afwImage
27import lsst.afw.math as afwMath
28import lsst.pipe.base.connectionTypes as cT
29import numpy as np
30from lsst.daf.butler import DimensionGraph
31from lsst.pex.config import Config, ConfigField, ConfigurableField, Field, FieldValidationError
32from lsst.pipe.base import PipelineTask, PipelineTaskConfig, PipelineTaskConnections, Struct
33from lsst.pipe.tasks.background import (
34 FocalPlaneBackground,
35 FocalPlaneBackgroundConfig,
36 MaskObjectsTask,
37 SkyMeasurementTask,
38)
39from lsst.pipe.tasks.visualizeVisit import VisualizeMosaicExpConfig, VisualizeMosaicExpTask
42def _skyFrameLookup(datasetType, registry, quantumDataId, collections):
43 """Lookup function to identify sky frames.
45 Parameters
46 ----------
47 datasetType : `lsst.daf.butler.DatasetType`
48 Dataset to lookup.
49 registry : `lsst.daf.butler.Registry`
50 Butler registry to query.
51 quantumDataId : `lsst.daf.butler.DataCoordinate`
52 Data id to transform to find sky frames.
53 The ``detector`` entry will be stripped.
54 collections : `lsst.daf.butler.CollectionSearch`
55 Collections to search through.
57 Returns
58 -------
59 results : `list` [`lsst.daf.butler.DatasetRef`]
60 List of datasets that will be used as sky calibration frames.
61 """
62 newDataId = quantumDataId.subset(DimensionGraph(registry.dimensions, names=["instrument", "visit"]))
63 skyFrames = []
64 for dataId in registry.queryDataIds(["visit", "detector"], dataId=newDataId).expanded():
65 skyFrame = registry.findDataset(
66 datasetType, dataId, collections=collections, timespan=dataId.timespan
67 )
68 skyFrames.append(skyFrame)
69 return skyFrames
72def _reorderAndPadList(inputList, inputKeys, outputKeys, padWith=None):
73 """Match the order of one list to another, padding if necessary.
75 Parameters
76 ----------
77 inputList : `list`
78 List to be reordered and padded. Elements can be any type.
79 inputKeys : iterable
80 Iterable of values to be compared with outputKeys.
81 Length must match `inputList`.
82 outputKeys : iterable
83 Iterable of values to be compared with inputKeys.
84 padWith :
85 Any value to be inserted where one of inputKeys is not in outputKeys.
87 Returns
88 -------
89 outputList : `list`
90 Copy of inputList reordered per outputKeys and padded with `padWith`
91 so that the length matches length of outputKeys.
92 """
93 outputList = []
94 for outputKey in outputKeys:
95 if outputKey in inputKeys:
96 outputList.append(inputList[inputKeys.index(outputKey)])
97 else:
98 outputList.append(padWith)
99 return outputList
102class SkyCorrectionConnections(PipelineTaskConnections, dimensions=("instrument", "visit")):
103 rawLinker = cT.Input(
104 doc="Raw data to provide exp-visit linkage to connect calExp inputs to camera/sky calibs.",
105 name="raw",
106 multiple=True,
107 deferLoad=True,
108 storageClass="Exposure",
109 dimensions=["instrument", "exposure", "detector"],
110 )
111 calExps = cT.Input(
112 doc="Background-subtracted calibrated exposures.",
113 name="calexp",
114 multiple=True,
115 storageClass="ExposureF",
116 dimensions=["instrument", "visit", "detector"],
117 )
118 calBkgs = cT.Input(
119 doc="Subtracted backgrounds for input calibrated exposures.",
120 multiple=True,
121 name="calexpBackground",
122 storageClass="Background",
123 dimensions=["instrument", "visit", "detector"],
124 )
125 skyFrames = cT.PrerequisiteInput(
126 doc="Calibration sky frames.",
127 name="sky",
128 multiple=True,
129 storageClass="ExposureF",
130 dimensions=["instrument", "physical_filter", "detector"],
131 isCalibration=True,
132 lookupFunction=_skyFrameLookup,
133 )
134 camera = cT.PrerequisiteInput(
135 doc="Input camera.",
136 name="camera",
137 storageClass="Camera",
138 dimensions=["instrument"],
139 isCalibration=True,
140 )
141 skyCorr = cT.Output(
142 doc="Sky correction data, to be subtracted from the calibrated exposures.",
143 name="skyCorr",
144 multiple=True,
145 storageClass="Background",
146 dimensions=["instrument", "visit", "detector"],
147 )
148 calExpMosaic = cT.Output(
149 doc="Full focal plane mosaicked image of the sky corrected calibrated exposures.",
150 name="calexp_skyCorr_visit_mosaic",
151 storageClass="ImageF",
152 dimensions=["instrument", "visit"],
153 )
154 calBkgMosaic = cT.Output(
155 doc="Full focal plane mosaicked image of the sky corrected calibrated exposure backgrounds.",
156 name="calexpBackground_skyCorr_visit_mosaic",
157 storageClass="ImageF",
158 dimensions=["instrument", "visit"],
159 )
162class SkyCorrectionConfig(PipelineTaskConfig, pipelineConnections=SkyCorrectionConnections):
163 maskObjects = ConfigurableField(
164 target=MaskObjectsTask,
165 doc="Mask Objects",
166 )
167 doMaskObjects = Field(
168 dtype=bool,
169 default=True,
170 doc="Iteratively mask objects to find good sky?",
171 )
172 bgModel = ConfigField(
173 dtype=Config,
174 doc="Initial background model, prior to sky frame subtraction",
175 deprecated="This field is deprecated and will be removed after v26. Please use bgModel1 instead.",
176 )
177 doBgModel = Field(
178 dtype=bool,
179 default=None,
180 doc="Do initial background model subtraction (prior to sky frame subtraction)?",
181 optional=True,
182 deprecated="This field is deprecated and will be removed after v26. See RFC-898 for further details.",
183 )
184 bgModel1 = ConfigField(
185 dtype=FocalPlaneBackgroundConfig,
186 doc="Initial background model, prior to sky frame subtraction",
187 )
188 doBgModel1 = Field(
189 dtype=bool,
190 default=True,
191 doc="Do initial background model subtraction (prior to sky frame subtraction)?",
192 deprecated="This field is deprecated and will be removed after v26. See RFC-898 for further details.",
193 )
194 sky = ConfigurableField(
195 target=SkyMeasurementTask,
196 doc="Sky measurement",
197 )
198 doSky = Field(
199 dtype=bool,
200 default=True,
201 doc="Do sky frame subtraction?",
202 )
203 bgModel2 = ConfigField(
204 dtype=FocalPlaneBackgroundConfig,
205 doc="Final (cleanup) background model, after sky frame subtraction",
206 )
207 doBgModel2 = Field(
208 dtype=bool,
209 default=True,
210 doc="Do final (cleanup) background model subtraction, after sky frame subtraction?",
211 )
212 binning = Field(
213 dtype=int,
214 default=8,
215 doc="Binning factor for constructing full focal plane '*_camera' output datasets",
216 )
218 def setDefaults(self):
219 Config.setDefaults(self)
220 self.bgModel2.doSmooth = True
221 self.bgModel2.minFrac = 0.5
222 self.bgModel2.xSize = 256
223 self.bgModel2.ySize = 256
224 self.bgModel2.smoothScale = 1.0
226 def validate(self):
227 # TODO: Entire validate method may be removed after v26 (a la DM-37242)
228 super().validate()
229 if self.doBgModel is not None and self.doBgModel != self.doBgModel1:
230 msg = "The doBgModel field will be removed after v26."
231 raise FieldValidationError(self.__class__.doBgModel, self, msg)
234class SkyCorrectionTask(PipelineTask):
235 """Perform a full focal plane sky correction."""
237 ConfigClass = SkyCorrectionConfig
238 _DefaultName = "skyCorr"
240 def __init__(self, *args, **kwargs):
241 super().__init__(**kwargs)
242 self.makeSubtask("sky")
243 self.makeSubtask("maskObjects")
245 def runQuantum(self, butlerQC, inputRefs, outputRefs):
246 # Sort the calExps, calBkgs and skyFrames inputRefs and the
247 # skyCorr outputRef by detector ID to ensure reproducibility.
248 detectorOrder = [ref.dataId["detector"] for ref in inputRefs.calExps]
249 detectorOrder.sort()
250 inputRefs.calExps = _reorderAndPadList(
251 inputRefs.calExps, [ref.dataId["detector"] for ref in inputRefs.calExps], detectorOrder
252 )
253 inputRefs.calBkgs = _reorderAndPadList(
254 inputRefs.calBkgs, [ref.dataId["detector"] for ref in inputRefs.calBkgs], detectorOrder
255 )
256 inputRefs.skyFrames = _reorderAndPadList(
257 inputRefs.skyFrames, [ref.dataId["detector"] for ref in inputRefs.skyFrames], detectorOrder
258 )
259 outputRefs.skyCorr = _reorderAndPadList(
260 outputRefs.skyCorr, [ref.dataId["detector"] for ref in outputRefs.skyCorr], detectorOrder
261 )
262 inputs = butlerQC.get(inputRefs)
263 inputs.pop("rawLinker", None)
264 outputs = self.run(**inputs)
265 butlerQC.put(outputs, outputRefs)
267 def run(self, calExps, calBkgs, skyFrames, camera):
268 """Perform sky correction on a visit.
270 The original visit-level background is first restored to the calibrated
271 exposure and the existing background model is inverted in-place. If
272 doMaskObjects is True, the mask map associated with this exposure will
273 be iteratively updated (over nIter loops) by re-estimating the
274 background each iteration and redetecting footprints.
276 If doBgModel1 is True, an initial full focal plane sky subtraction will
277 take place prior to scaling and subtracting the sky frame.
279 If doSky is True, the sky frame will be scaled to the flux in the input
280 visit.
282 If doBgModel2 is True, a final full focal plane sky subtraction will
283 take place after the sky frame has been subtracted.
285 The first N elements of the returned skyCorr will consist of inverted
286 elements of the calexpBackground model (i.e., subtractive). All
287 subsequent elements appended to skyCorr thereafter will be additive
288 such that, when skyCorr is subtracted from a calexp, the net result
289 will be to undo the initial per-detector background solution and then
290 apply the skyCorr model thereafter. Adding skyCorr to a
291 calexpBackground will effectively negate the calexpBackground,
292 returning only the additive background components of the skyCorr
293 background model.
295 Parameters
296 ----------
297 calExps : `list` [`lsst.afw.image.exposure.ExposureF`]
298 Detector calibrated exposure images for the visit.
299 calBkgs : `list` [`lsst.afw.math.BackgroundList`]
300 Detector background lists matching the calibrated exposures.
301 skyFrames : `list` [`lsst.afw.image.exposure.ExposureF`]
302 Sky frame calibration data for the input detectors.
303 camera : `lsst.afw.cameraGeom.Camera`
304 Camera matching the input data to process.
306 Returns
307 -------
308 results : `Struct` containing:
309 skyCorr : `list` [`lsst.afw.math.BackgroundList`]
310 Detector-level sky correction background lists.
311 calExpMosaic : `lsst.afw.image.exposure.ExposureF`
312 Visit-level mosaic of the sky corrected data, binned.
313 Analogous to `calexp - skyCorr`.
314 calBkgMosaic : `lsst.afw.image.exposure.ExposureF`
315 Visit-level mosaic of the sky correction background, binned.
316 Analogous to `calexpBackground + skyCorr`.
317 """
318 # Restore original backgrounds in-place; optionally refine mask maps
319 numOrigBkgElements = [len(calBkg) for calBkg in calBkgs]
320 _ = self._restoreBackgroundRefineMask(calExps, calBkgs)
322 # Bin exposures, generate full-fp bg, map to CCDs and subtract in-place
323 if self.config.doBgModel1:
324 _ = self._subtractVisitBackground(calExps, calBkgs, camera, self.config.bgModel1)
326 # Subtract a scaled sky frame from all input exposures
327 if self.config.doSky:
328 self._subtractSkyFrame(calExps, skyFrames, calBkgs)
330 # Bin exposures, generate full-fp bg, map to CCDs and subtract in-place
331 if self.config.doBgModel2:
332 _ = self._subtractVisitBackground(calExps, calBkgs, camera, self.config.bgModel2)
334 # Make camera-level images of bg subtracted calexps and subtracted bgs
335 calExpIds = [exp.getDetector().getId() for exp in calExps]
336 skyCorrExtras = []
337 for calBkg, num in zip(calBkgs, numOrigBkgElements):
338 skyCorrExtra = calBkg.clone()
339 skyCorrExtra._backgrounds = skyCorrExtra._backgrounds[num:]
340 skyCorrExtras.append(skyCorrExtra)
341 calExpMosaic = self._binAndMosaic(calExps, camera, self.config.binning, ids=calExpIds, refExps=None)
342 calBkgMosaic = self._binAndMosaic(
343 skyCorrExtras, camera, self.config.binning, ids=calExpIds, refExps=calExps
344 )
346 return Struct(skyCorr=calBkgs, calExpMosaic=calExpMosaic, calBkgMosaic=calBkgMosaic)
348 def _restoreBackgroundRefineMask(self, calExps, calBkgs):
349 """Restore original background to each calexp and invert the related
350 background model; optionally refine the mask plane.
352 The original visit-level background is restored to each calibrated
353 exposure and the existing background model is inverted in-place. If
354 doMaskObjects is True, the mask map associated with the exposure will
355 be iteratively updated (over nIter loops) by re-estimating the
356 background each iteration and redetecting footprints.
358 The background model modified in-place in this method will comprise the
359 first N elements of the skyCorr dataset type, i.e., these N elements
360 are the inverse of the calexpBackground model. All subsequent elements
361 appended to skyCorr will be additive such that, when skyCorr is
362 subtracted from a calexp, the net result will be to undo the initial
363 per-detector background solution and then apply the skyCorr model
364 thereafter. Adding skyCorr to a calexpBackground will effectively
365 negate the calexpBackground, returning only the additive background
366 components of the skyCorr background model.
368 Parameters
369 ----------
370 calExps : `lsst.afw.image.exposure.ExposureF`
371 Detector level calexp images to process.
372 calBkgs : `lsst.afw.math._backgroundList.BackgroundList`
373 Detector level background lists associated with the calexps.
375 Returns
376 -------
377 calExps : `lsst.afw.image.exposure.ExposureF`
378 The calexps with the initially subtracted background restored.
379 skyCorrBases : `lsst.afw.math._backgroundList.BackgroundList`
380 The inverted initial background models; the genesis for skyCorrs.
381 """
382 skyCorrBases = []
383 for calExp, calBkg in zip(calExps, calBkgs):
384 image = calExp.getMaskedImage()
386 # Invert all elements of the existing bg model; restore in calexp
387 for calBkgElement in calBkg:
388 statsImage = calBkgElement[0].getStatsImage()
389 statsImage *= -1
390 skyCorrBase = calBkg.getImage()
391 image -= skyCorrBase
393 # Iteratively subtract bg, re-detect sources, and add bg back on
394 if self.config.doMaskObjects:
395 self.maskObjects.findObjects(calExp)
397 stats = np.nanpercentile(skyCorrBase.array, [50, 75, 25])
398 self.log.info(
399 "Detector %d: Initial background restored; BG median = %.1f counts, BG IQR = %.1f counts",
400 calExp.getDetector().getId(),
401 -stats[0],
402 np.subtract(*stats[1:]),
403 )
404 skyCorrBases.append(skyCorrBase)
405 return calExps, skyCorrBases
407 def _subtractVisitBackground(self, calExps, calBkgs, camera, config):
408 """Perform a full focal-plane background subtraction for a visit.
410 Generate a full focal plane background model, binning all masked
411 detectors into bins of [bgModelN.xSize, bgModelN.ySize]. After,
412 subtract the resultant background model (translated back into CCD
413 coordinates) from the original detector exposure.
415 Return a list of background subtracted images and a list of full focal
416 plane background parameters.
418 Parameters
419 ----------
420 calExps : `list` [`lsst.afw.image.exposure.ExposureF`]
421 Calibrated exposures to be background subtracted.
422 calBkgs : `list` [`lsst.afw.math._backgroundList.BackgroundList`]
423 Background lists associated with the input calibrated exposures.
424 camera : `lsst.afw.cameraGeom.Camera`
425 Camera description.
426 config : `lsst.pipe.tasks.background.FocalPlaneBackgroundConfig`
427 Configuration to use for background subtraction.
429 Returns
430 -------
431 calExps : `list` [`lsst.afw.image.maskedImage.MaskedImageF`]
432 Background subtracted exposures for creating a focal plane image.
433 calBkgs : `list` [`lsst.afw.math._backgroundList.BackgroundList`]
434 Updated background lists with a visit-level model appended.
435 """
436 # Set up empty full focal plane background model object
437 bgModelBase = FocalPlaneBackground.fromCamera(config, camera)
439 # Loop over each detector, bin into [xSize, ySize] bins, and update
440 # summed flux (_values) and number of contributing pixels (_numbers)
441 # in focal plane coordinates. Append outputs to bgModels.
442 bgModels = []
443 for calExp in calExps:
444 bgModel = bgModelBase.clone()
445 bgModel.addCcd(calExp)
446 bgModels.append(bgModel)
448 # Merge detector models to make a single full focal plane bg model
449 for bgModel, calExp in zip(bgModels, calExps):
450 msg = (
451 "Detector %d: Merging %d unmasked pixels (%.1f%s of detector area) into focal plane "
452 "background model"
453 )
454 self.log.debug(
455 msg,
456 calExp.getDetector().getId(),
457 bgModel._numbers.getArray().sum(),
458 100 * bgModel._numbers.getArray().sum() / calExp.getBBox().getArea(),
459 "%",
460 )
461 bgModelBase.merge(bgModel)
463 # Map full focal plane bg solution to detector; subtract from exposure
464 calBkgElements = []
465 for calExp in calExps:
466 _, calBkgElement = self._subtractDetectorBackground(calExp, bgModelBase)
467 calBkgElements.append(calBkgElement)
469 msg = (
470 "Focal plane background model constructed using %.2f x %.2f mm (%d x %d pixel) superpixels; "
471 "FP BG median = %.1f counts, FP BG IQR = %.1f counts"
472 )
473 with warnings.catch_warnings():
474 warnings.filterwarnings("ignore", r"invalid value encountered")
475 stats = np.nanpercentile(bgModelBase.getStatsImage().array, [50, 75, 25])
476 self.log.info(
477 msg,
478 config.xSize,
479 config.ySize,
480 int(config.xSize / config.pixelSize),
481 int(config.ySize / config.pixelSize),
482 stats[0],
483 np.subtract(*stats[1:]),
484 )
486 for calBkg, calBkgElement in zip(calBkgs, calBkgElements):
487 calBkg.append(calBkgElement[0])
488 return calExps, calBkgs
490 def _subtractDetectorBackground(self, calExp, bgModel):
491 """Generate CCD background model and subtract from image.
493 Translate the full focal plane background into CCD coordinates and
494 subtract from the original science exposure image.
496 Parameters
497 ----------
498 calExp : `lsst.afw.image.exposure.ExposureF`
499 Exposure to subtract the background model from.
500 bgModel : `lsst.pipe.tasks.background.FocalPlaneBackground`
501 Full focal plane camera-level background model.
503 Returns
504 -------
505 calExp : `lsst.afw.image.exposure.ExposureF`
506 Background subtracted input exposure.
507 calBkgElement : `lsst.afw.math._backgroundList.BackgroundList`
508 Detector level realization of the full focal plane bg model.
509 """
510 image = calExp.getMaskedImage()
511 with warnings.catch_warnings():
512 warnings.filterwarnings("ignore", r"invalid value encountered")
513 calBkgElement = bgModel.toCcdBackground(calExp.getDetector(), image.getBBox())
514 image -= calBkgElement.getImage()
515 return calExp, calBkgElement
517 def _subtractSkyFrame(self, calExps, skyFrames, calBkgs):
518 """Determine the full focal plane sky frame scale factor relative to
519 an input list of calibrated exposures and subtract.
521 This method measures the sky frame scale on all inputs, resulting in
522 values equal to the background method solveScales(). The sky frame is
523 then subtracted as in subtractSkyFrame() using the appropriate scale.
525 Input calExps and calBkgs are updated in-place, returning sky frame
526 subtracted calExps and sky frame updated calBkgs, respectively.
528 Parameters
529 ----------
530 calExps : `list` [`lsst.afw.image.exposure.ExposureF`]
531 Calibrated exposures to be background subtracted.
532 skyFrames : `list` [`lsst.afw.image.exposure.ExposureF`]
533 Sky frame calibration data for the input detectors.
534 calBkgs : `list` [`lsst.afw.math._backgroundList.BackgroundList`]
535 Background lists associated with the input calibrated exposures.
536 """
537 skyFrameBgModels = []
538 scales = []
539 for calExp, skyFrame in zip(calExps, skyFrames):
540 skyFrameBgModel = self.sky.exposureToBackground(skyFrame)
541 skyFrameBgModels.append(skyFrameBgModel)
542 # return a tuple of gridded image and sky frame clipped means
543 samples = self.sky.measureScale(calExp.getMaskedImage(), skyFrameBgModel)
544 scales.append(samples)
545 scale = self.sky.solveScales(scales)
546 for calExp, skyFrameBgModel, calBkg in zip(calExps, skyFrameBgModels, calBkgs):
547 # subtract the scaled sky frame model from each calExp in-place,
548 # also updating the calBkg list in-place
549 self.sky.subtractSkyFrame(calExp.getMaskedImage(), skyFrameBgModel, scale, calBkg)
550 self.log.info("Sky frame subtracted with a scale factor of %.5f", scale)
552 def _binAndMosaic(self, exposures, camera, binning, ids=None, refExps=None):
553 """Bin input exposures and mosaic across the entire focal plane.
555 Input exposures are binned and then mosaicked at the position of
556 the detector in the focal plane of the camera.
558 Parameters
559 ----------
560 exposures : `list`
561 Detector level list of either calexp `ExposureF` types or
562 calexpBackground `BackgroundList` types.
563 camera : `lsst.afw.cameraGeom.Camera`
564 Camera matching the input data to process.
565 binning : `int`
566 Binning size to be applied to input images.
567 ids : `list` [`int`], optional
568 List of detector ids to iterate over.
569 refExps : `list` [`lsst.afw.image.exposure.ExposureF`], optional
570 If supplied, mask planes from these reference images will be used.
571 Returns
572 -------
573 mosaicImage : `lsst.afw.image.exposure.ExposureF`
574 Mosaicked full focal plane image.
575 """
576 refExps = np.resize(refExps, len(exposures)) # type: ignore
577 binnedImages = []
578 for exp, refExp in zip(exposures, refExps):
579 try:
580 nativeImage = exp.getMaskedImage()
581 except AttributeError:
582 nativeImage = afwImage.makeMaskedImage(exp.getImage())
583 if refExp:
584 nativeImage.setMask(refExp.getMask())
585 binnedImage = afwMath.binImage(nativeImage, binning)
586 binnedImages.append(binnedImage)
587 mosConfig = VisualizeMosaicExpConfig()
588 mosConfig.binning = binning
589 mosTask = VisualizeMosaicExpTask(config=mosConfig)
590 imageStruct = mosTask.run(binnedImages, camera, inputIds=ids)
591 mosaicImage = imageStruct.outputData
592 return mosaicImage