lsst.pipe.tasks gc750389ca5+10447145b3
Loading...
Searching...
No Matches
skyCorrection.py
Go to the documentation of this file.
1# This file is part of pipe_tasks.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21
22__all__ = ["SkyCorrectionTask", "SkyCorrectionConfig"]
23
24import lsst.afw.image as afwImage
25import lsst.afw.math as afwMath
26import lsst.pipe.base.connectionTypes as cT
27import numpy as np
28from lsst.daf.butler import DimensionGraph
29from lsst.pex.config import Config, ConfigField, ConfigurableField, Field, FieldValidationError
30from lsst.pipe.base import PipelineTask, PipelineTaskConfig, PipelineTaskConnections, Struct
32 FocalPlaneBackground,
33 FocalPlaneBackgroundConfig,
34 MaskObjectsTask,
35 SkyMeasurementTask,
36)
37from lsst.pipe.tasks.visualizeVisit import VisualizeMosaicExpConfig, VisualizeMosaicExpTask
38
39
40def _skyFrameLookup(datasetType, registry, quantumDataId, collections):
41 """Lookup function to identify sky frames.
42
43 Parameters
44 ----------
45 datasetType : `lsst.daf.butler.DatasetType`
46 Dataset to lookup.
47 registry : `lsst.daf.butler.Registry`
48 Butler registry to query.
49 quantumDataId : `lsst.daf.butler.DataCoordinate`
50 Data id to transform to find sky frames.
51 The ``detector`` entry will be stripped.
52 collections : `lsst.daf.butler.CollectionSearch`
53 Collections to search through.
54
55 Returns
56 -------
57 results : `list` [`lsst.daf.butler.DatasetRef`]
58 List of datasets that will be used as sky calibration frames.
59 """
60 newDataId = quantumDataId.subset(DimensionGraph(registry.dimensions, names=["instrument", "visit"]))
61 skyFrames = []
62 for dataId in registry.queryDataIds(["visit", "detector"], dataId=newDataId).expanded():
63 skyFrame = registry.findDataset(
64 datasetType, dataId, collections=collections, timespan=dataId.timespan
65 )
66 skyFrames.append(skyFrame)
67 return skyFrames
68
69
70def _reorderAndPadList(inputList, inputKeys, outputKeys, padWith=None):
71 """Match the order of one list to another, padding if necessary.
72
73 Parameters
74 ----------
75 inputList : `list`
76 List to be reordered and padded. Elements can be any type.
77 inputKeys : iterable
78 Iterable of values to be compared with outputKeys.
79 Length must match `inputList`.
80 outputKeys : iterable
81 Iterable of values to be compared with inputKeys.
82 padWith :
83 Any value to be inserted where one of inputKeys is not in outputKeys.
84
85 Returns
86 -------
87 outputList : `list`
88 Copy of inputList reordered per outputKeys and padded with `padWith`
89 so that the length matches length of outputKeys.
90 """
91 outputList = []
92 for outputKey in outputKeys:
93 if outputKey in inputKeys:
94 outputList.append(inputList[inputKeys.index(outputKey)])
95 else:
96 outputList.append(padWith)
97 return outputList
98
99
100class SkyCorrectionConnections(PipelineTaskConnections, dimensions=("instrument", "visit")):
101 rawLinker = cT.Input(
102 doc="Raw data to provide exp-visit linkage to connect calExp inputs to camera/sky calibs.",
103 name="raw",
104 multiple=True,
105 deferLoad=True,
106 storageClass="Exposure",
107 dimensions=["instrument", "exposure", "detector"],
108 )
109 calExps = cT.Input(
110 doc="Background-subtracted calibrated exposures.",
111 name="calexp",
112 multiple=True,
113 storageClass="ExposureF",
114 dimensions=["instrument", "visit", "detector"],
115 )
116 calBkgs = cT.Input(
117 doc="Subtracted backgrounds for input calibrated exposures.",
118 multiple=True,
119 name="calexpBackground",
120 storageClass="Background",
121 dimensions=["instrument", "visit", "detector"],
122 )
123 skyFrames = cT.PrerequisiteInput(
124 doc="Calibration sky frames.",
125 name="sky",
126 multiple=True,
127 storageClass="ExposureF",
128 dimensions=["instrument", "physical_filter", "detector"],
129 isCalibration=True,
130 lookupFunction=_skyFrameLookup,
131 )
132 camera = cT.PrerequisiteInput(
133 doc="Input camera.",
134 name="camera",
135 storageClass="Camera",
136 dimensions=["instrument"],
137 isCalibration=True,
138 )
139 skyCorr = cT.Output(
140 doc="Sky correction data, to be subtracted from the calibrated exposures.",
141 name="skyCorr",
142 multiple=True,
143 storageClass="Background",
144 dimensions=["instrument", "visit", "detector"],
145 )
146 calExpMosaic = cT.Output(
147 doc="Full focal plane mosaicked image of the sky corrected calibrated exposures.",
148 name="calexp_skyCorr_visit_mosaic",
149 storageClass="ImageF",
150 dimensions=["instrument", "visit"],
151 )
152 calBkgMosaic = cT.Output(
153 doc="Full focal plane mosaicked image of the sky corrected calibrated exposure backgrounds.",
154 name="calexpBackground_skyCorr_visit_mosaic",
155 storageClass="ImageF",
156 dimensions=["instrument", "visit"],
157 )
158
159
160class SkyCorrectionConfig(PipelineTaskConfig, pipelineConnections=SkyCorrectionConnections):
161 maskObjects = ConfigurableField(
162 target=MaskObjectsTask,
163 doc="Mask Objects",
164 )
165 doMaskObjects = Field(
166 dtype=bool,
167 default=True,
168 doc="Iteratively mask objects to find good sky?",
169 )
170 bgModel = ConfigField(
171 dtype=Config,
172 doc="Initial background model, prior to sky frame subtraction",
173 deprecated="This field is deprecated and will be removed after v26. Please use bgModel1 instead.",
174 )
175 doBgModel = Field(
176 dtype=bool,
177 default=None,
178 doc="Do initial background model subtraction (prior to sky frame subtraction)?",
179 optional=True,
180 deprecated="This field is deprecated and will be removed after v26. See RFC-898 for further details.",
181 )
182 bgModel1 = ConfigField(
183 dtype=FocalPlaneBackgroundConfig,
184 doc="Initial background model, prior to sky frame subtraction",
185 )
186 doBgModel1 = Field(
187 dtype=bool,
188 default=True,
189 doc="Do initial background model subtraction (prior to sky frame subtraction)?",
190 deprecated="This field is deprecated and will be removed after v26. See RFC-898 for further details.",
191 )
192 sky = ConfigurableField(
193 target=SkyMeasurementTask,
194 doc="Sky measurement",
195 )
196 doSky = Field(
197 dtype=bool,
198 default=True,
199 doc="Do sky frame subtraction?",
200 )
201 bgModel2 = ConfigField(
202 dtype=FocalPlaneBackgroundConfig,
203 doc="Final (cleanup) background model, after sky frame subtraction",
204 )
205 doBgModel2 = Field(
206 dtype=bool,
207 default=True,
208 doc="Do final (cleanup) background model subtraction, after sky frame subtraction?",
209 )
210 binning = Field(
211 dtype=int,
212 default=8,
213 doc="Binning factor for constructing full focal plane '*_camera' output datasets",
214 )
215
216 def setDefaults(self):
217 Config.setDefaults(self)
218 self.bgModel2.doSmooth = True
219 self.bgModel2.minFrac = 0.5
220 self.bgModel2.xSize = 256
221 self.bgModel2.ySize = 256
222 self.bgModel2.smoothScale = 1.0
223
224 def validate(self):
225 # TODO: Entire validate method may be removed after v26 (a la DM-37242)
226 super().validate()
227 if self.doBgModel is not None and self.doBgModel != self.doBgModel1:
228 msg = "The doBgModel field will be removed after v26."
229 raise FieldValidationError(self.__class__.doBgModel, self, msg)
230
231
232class SkyCorrectionTask(PipelineTask):
233 """Perform a full focal plane sky correction."""
234
235 ConfigClass = SkyCorrectionConfig
236 _DefaultName = "skyCorr"
237
238 def __init__(self, *args, **kwargs):
239 super().__init__(**kwargs)
240 self.makeSubtask("sky")
241 self.makeSubtask("maskObjects")
242
243 def runQuantum(self, butlerQC, inputRefs, outputRefs):
244 # Sort the calExps, calBkgs and skyFrames inputRefs and the
245 # skyCorr outputRef by detector ID to ensure reproducibility.
246 detectorOrder = [ref.dataId["detector"] for ref in inputRefs.calExps]
247 detectorOrder.sort()
248 inputRefs.calExps = _reorderAndPadList(
249 inputRefs.calExps, [ref.dataId["detector"] for ref in inputRefs.calExps], detectorOrder
250 )
251 inputRefs.calBkgs = _reorderAndPadList(
252 inputRefs.calBkgs, [ref.dataId["detector"] for ref in inputRefs.calBkgs], detectorOrder
253 )
254 inputRefs.skyFrames = _reorderAndPadList(
255 inputRefs.skyFrames, [ref.dataId["detector"] for ref in inputRefs.skyFrames], detectorOrder
256 )
257 outputRefs.skyCorr = _reorderAndPadList(
258 outputRefs.skyCorr, [ref.dataId["detector"] for ref in outputRefs.skyCorr], detectorOrder
259 )
260 inputs = butlerQC.get(inputRefs)
261 inputs.pop("rawLinker", None)
262 outputs = self.run(**inputs)
263 butlerQC.put(outputs, outputRefs)
264
265 def run(self, calExps, calBkgs, skyFrames, camera):
266 """Perform sky correction on a visit.
267
268 The original visit-level background is first restored to the calibrated
269 exposure and the existing background model is inverted in-place. If
270 doMaskObjects is True, the mask map associated with this exposure will
271 be iteratively updated (over nIter loops) by re-estimating the
272 background each iteration and redetecting footprints.
273
274 If doBgModel1 is True, an initial full focal plane sky subtraction will
275 take place prior to scaling and subtracting the sky frame.
276
277 If doSky is True, the sky frame will be scaled to the flux in the input
278 visit.
279
280 If doBgModel2 is True, a final full focal plane sky subtraction will
281 take place after the sky frame has been subtracted.
282
283 The first N elements of the returned skyCorr will consist of inverted
284 elements of the calexpBackground model (i.e., subtractive). All
285 subsequent elements appended to skyCorr thereafter will be additive
286 such that, when skyCorr is subtracted from a calexp, the net result
287 will be to undo the initial per-detector background solution and then
288 apply the skyCorr model thereafter. Adding skyCorr to a
289 calexpBackground will effectively negate the calexpBackground,
290 returning only the additive background components of the skyCorr
291 background model.
292
293 Parameters
294 ----------
295 calExps : `list` [`lsst.afw.image.exposure.ExposureF`]
296 Detector calibrated exposure images for the visit.
297 calBkgs : `list` [`lsst.afw.math.BackgroundList`]
298 Detector background lists matching the calibrated exposures.
299 skyFrames : `list` [`lsst.afw.image.exposure.ExposureF`]
300 Sky frame calibration data for the input detectors.
302 Camera matching the input data to process.
303
304 Returns
305 -------
306 results : `Struct` containing:
307 skyCorr : `list` [`lsst.afw.math.BackgroundList`]
308 Detector-level sky correction background lists.
309 calExpMosaic : `lsst.afw.image.exposure.ExposureF`
310 Visit-level mosaic of the sky corrected data, binned.
311 Analogous to `calexp - skyCorr`.
312 calBkgMosaic : `lsst.afw.image.exposure.ExposureF`
313 Visit-level mosaic of the sky correction background, binned.
314 Analogous to `calexpBackground + skyCorr`.
315 """
316 # Restore original backgrounds in-place; optionally refine mask maps
317 numOrigBkgElements = [len(calBkg) for calBkg in calBkgs]
318 _ = self._restoreBackgroundRefineMask(calExps, calBkgs)
319
320 # Bin exposures, generate full-fp bg, map to CCDs and subtract in-place
321 if self.config.doBgModel1:
322 _ = self._subtractVisitBackground(calExps, calBkgs, camera, self.config.bgModel1)
323
324 # Subtract a scaled sky frame from all input exposures
325 if self.config.doSky:
326 self._subtractSkyFrame(calExps, skyFrames, calBkgs)
327
328 # Bin exposures, generate full-fp bg, map to CCDs and subtract in-place
329 if self.config.doBgModel2:
330 _ = self._subtractVisitBackground(calExps, calBkgs, camera, self.config.bgModel2)
331
332 # Make camera-level images of bg subtracted calexps and subtracted bgs
333 calExpIds = [exp.getDetector().getId() for exp in calExps]
334 skyCorrExtras = []
335 for calBkg, num in zip(calBkgs, numOrigBkgElements):
336 skyCorrExtra = calBkg.clone()
337 skyCorrExtra._backgrounds = skyCorrExtra._backgrounds[num:]
338 skyCorrExtras.append(skyCorrExtra)
339 calExpMosaic = self._binAndMosaic(calExps, camera, self.config.binning, ids=calExpIds, refExps=None)
340 calBkgMosaic = self._binAndMosaic(
341 skyCorrExtras, camera, self.config.binning, ids=calExpIds, refExps=calExps
342 )
343
344 return Struct(skyCorr=calBkgs, calExpMosaic=calExpMosaic, calBkgMosaic=calBkgMosaic)
345
346 def _restoreBackgroundRefineMask(self, calExps, calBkgs):
347 """Restore original background to each calexp and invert the related
348 background model; optionally refine the mask plane.
349
350 The original visit-level background is restored to each calibrated
351 exposure and the existing background model is inverted in-place. If
352 doMaskObjects is True, the mask map associated with the exposure will
353 be iteratively updated (over nIter loops) by re-estimating the
354 background each iteration and redetecting footprints.
355
356 The background model modified in-place in this method will comprise the
357 first N elements of the skyCorr dataset type, i.e., these N elements
358 are the inverse of the calexpBackground model. All subsequent elements
359 appended to skyCorr will be additive such that, when skyCorr is
360 subtracted from a calexp, the net result will be to undo the initial
361 per-detector background solution and then apply the skyCorr model
362 thereafter. Adding skyCorr to a calexpBackground will effectively
363 negate the calexpBackground, returning only the additive background
364 components of the skyCorr background model.
365
366 Parameters
367 ----------
368 calExps : `lsst.afw.image.exposure.ExposureF`
369 Detector level calexp images to process.
371 Detector level background lists associated with the calexps.
372
373 Returns
374 -------
375 calExps : `lsst.afw.image.exposure.ExposureF`
376 The calexps with the initially subtracted background restored.
378 The inverted initial background models; the genesis for skyCorrs.
379 """
380 skyCorrBases = []
381 for calExp, calBkg in zip(calExps, calBkgs):
382 image = calExp.getMaskedImage()
383
384 # Invert all elements of the existing bg model; restore in calexp
385 for calBkgElement in calBkg:
386 statsImage = calBkgElement[0].getStatsImage()
387 statsImage *= -1
388 skyCorrBase = calBkg.getImage()
389 image -= skyCorrBase
390
391 # Iteratively subtract bg, re-detect sources, and add bg back on
392 if self.config.doMaskObjects:
393 self.maskObjects.findObjects(calExp)
394
395 stats = np.nanpercentile(skyCorrBase.array, [50, 75, 25])
396 self.log.info(
397 "Detector %d: Initial background restored; BG median = %.1f counts, BG IQR = %.1f counts",
398 calExp.getDetector().getId(),
399 -stats[0],
400 np.subtract(*stats[1:]),
401 )
402 skyCorrBases.append(skyCorrBase)
403 return calExps, skyCorrBases
404
405 def _subtractVisitBackground(self, calExps, calBkgs, camera, config):
406 """Perform a full focal-plane background subtraction for a visit.
407
408 Generate a full focal plane background model, binning all masked
409 detectors into bins of [bgModelN.xSize, bgModelN.ySize]. After,
410 subtract the resultant background model (translated back into CCD
411 coordinates) from the original detector exposure.
412
413 Return a list of background subtracted images and a list of full focal
414 plane background parameters.
415
416 Parameters
417 ----------
418 calExps : `list` [`lsst.afw.image.exposure.ExposureF`]
419 Calibrated exposures to be background subtracted.
421 Background lists associated with the input calibrated exposures.
423 Camera description.
425 Configuration to use for background subtraction.
426
427 Returns
428 -------
429 calExps : `list` [`lsst.afw.image.maskedImage.MaskedImageF`]
430 Background subtracted exposures for creating a focal plane image.
432 Updated background lists with a visit-level model appended.
433 """
434 # Set up empty full focal plane background model object
435 bgModelBase = FocalPlaneBackground.fromCamera(config, camera)
436
437 # Loop over each detector, bin into [xSize, ySize] bins, and update
438 # summed flux (_values) and number of contributing pixels (_numbers)
439 # in focal plane coordinates. Append outputs to bgModels.
440 bgModels = []
441 for calExp in calExps:
442 bgModel = bgModelBase.clone()
443 bgModel.addCcd(calExp)
444 bgModels.append(bgModel)
445
446 # Merge detector models to make a single full focal plane bg model
447 for bgModel, calExp in zip(bgModels, calExps):
448 msg = (
449 "Detector %d: Merging %d unmasked pixels (%.1f%s of detector area) into focal plane "
450 "background model"
451 )
452 self.log.debug(
453 msg,
454 calExp.getDetector().getId(),
455 bgModel._numbers.getArray().sum(),
456 100 * bgModel._numbers.getArray().sum() / calExp.getBBox().getArea(),
457 "%",
458 )
459 bgModelBase.merge(bgModel)
460
461 # Map full focal plane bg solution to detector; subtract from exposure
462 calBkgElements = []
463 for calExp in calExps:
464 _, calBkgElement = self._subtractDetectorBackground(calExp, bgModelBase)
465 calBkgElements.append(calBkgElement)
466
467 msg = (
468 "Focal plane background model constructed using %.2f x %.2f mm (%d x %d pixel) superpixels; "
469 "FP BG median = %.1f counts, FP BG IQR = %.1f counts"
470 )
471 with np.warnings.catch_warnings():
472 np.warnings.filterwarnings("ignore", r"invalid value encountered")
473 stats = np.nanpercentile(bgModelBase.getStatsImage().array, [50, 75, 25])
474 self.log.info(
475 msg,
476 config.xSize,
477 config.ySize,
478 int(config.xSize / config.pixelSize),
479 int(config.ySize / config.pixelSize),
480 stats[0],
481 np.subtract(*stats[1:]),
482 )
483
484 for calBkg, calBkgElement in zip(calBkgs, calBkgElements):
485 calBkg.append(calBkgElement[0])
486 return calExps, calBkgs
487
488 def _subtractDetectorBackground(self, calExp, bgModel):
489 """Generate CCD background model and subtract from image.
490
491 Translate the full focal plane background into CCD coordinates and
492 subtract from the original science exposure image.
493
494 Parameters
495 ----------
496 calExp : `lsst.afw.image.exposure.ExposureF`
497 Exposure to subtract the background model from.
499 Full focal plane camera-level background model.
500
501 Returns
502 -------
503 calExp : `lsst.afw.image.exposure.ExposureF`
504 Background subtracted input exposure.
506 Detector level realization of the full focal plane bg model.
507 """
508 image = calExp.getMaskedImage()
509 with np.warnings.catch_warnings():
510 np.warnings.filterwarnings("ignore", r"invalid value encountered")
511 calBkgElement = bgModel.toCcdBackground(calExp.getDetector(), image.getBBox())
512 image -= calBkgElement.getImage()
513 return calExp, calBkgElement
514
515 def _subtractSkyFrame(self, calExps, skyFrames, calBkgs):
516 """Determine the full focal plane sky frame scale factor relative to
517 an input list of calibrated exposures and subtract.
518
519 This method measures the sky frame scale on all inputs, resulting in
520 values equal to the background method solveScales(). The sky frame is
521 then subtracted as in subtractSkyFrame() using the appropriate scale.
522
523 Input calExps and calBkgs are updated in-place, returning sky frame
524 subtracted calExps and sky frame updated calBkgs, respectively.
525
526 Parameters
527 ----------
528 calExps : `list` [`lsst.afw.image.exposure.ExposureF`]
529 Calibrated exposures to be background subtracted.
530 skyFrames : `list` [`lsst.afw.image.exposure.ExposureF`]
531 Sky frame calibration data for the input detectors.
533 Background lists associated with the input calibrated exposures.
534 """
535 skyFrameBgModels = []
536 scales = []
537 for calExp, skyFrame in zip(calExps, skyFrames):
538 skyFrameBgModel = self.sky.exposureToBackground(skyFrame)
539 skyFrameBgModels.append(skyFrameBgModel)
540 # return a tuple of gridded image and sky frame clipped means
541 samples = self.sky.measureScale(calExp.getMaskedImage(), skyFrameBgModel)
542 scales.append(samples)
543 scale = self.sky.solveScales(scales)
544 for calExp, skyFrameBgModel, calBkg in zip(calExps, skyFrameBgModels, calBkgs):
545 # subtract the scaled sky frame model from each calExp in-place,
546 # also updating the calBkg list in-place
547 self.sky.subtractSkyFrame(calExp.getMaskedImage(), skyFrameBgModel, scale, calBkg)
548 self.log.info("Sky frame subtracted with a scale factor of %.5f", scale)
549
550 def _binAndMosaic(self, exposures, camera, binning, ids=None, refExps=None):
551 """Bin input exposures and mosaic across the entire focal plane.
552
553 Input exposures are binned and then mosaicked at the position of
554 the detector in the focal plane of the camera.
555
556 Parameters
557 ----------
558 exposures : `list`
559 Detector level list of either calexp `ExposureF` types or
560 calexpBackground `BackgroundList` types.
562 Camera matching the input data to process.
563 binning : `int`
564 Binning size to be applied to input images.
565 ids : `list` [`int`], optional
566 List of detector ids to iterate over.
567 refExps : `list` [`lsst.afw.image.exposure.ExposureF`], optional
568 If supplied, mask planes from these reference images will be used.
569 Returns
570 -------
571 mosaicImage : `lsst.afw.image.exposure.ExposureF`
572 Mosaicked full focal plane image.
573 """
574 refExps = np.resize(refExps, len(exposures)) # type: ignore
575 binnedImages = []
576 for exp, refExp in zip(exposures, refExps):
577 try:
578 nativeImage = exp.getMaskedImage()
579 except AttributeError:
580 nativeImage = afwImage.makeMaskedImage(exp.getImage())
581 if refExp:
582 nativeImage.setMask(refExp.getMask())
583 binnedImage = afwMath.binImage(nativeImage, binning)
584 binnedImages.append(binnedImage)
585 mosConfig = VisualizeMosaicExpConfig()
586 mosConfig.binning = binning
587 mosTask = VisualizeMosaicExpTask(config=mosConfig)
588 imageStruct = mosTask.run(binnedImages, camera, inputIds=ids)
589 mosaicImage = imageStruct.outputData
590 return mosaicImage
def _restoreBackgroundRefineMask(self, calExps, calBkgs)
def _subtractVisitBackground(self, calExps, calBkgs, camera, config)
def _binAndMosaic(self, exposures, camera, binning, ids=None, refExps=None)
def _subtractDetectorBackground(self, calExp, bgModel)
def runQuantum(self, butlerQC, inputRefs, outputRefs)
def _subtractSkyFrame(self, calExps, skyFrames, calBkgs)
def run(self, calExps, calBkgs, skyFrames, camera)