22__all__ = [
"SkyCorrectionTask",
"SkyCorrectionConfig"]
28import lsst.pipe.base.connectionTypes
as cT
30from lsst.pex.config import Config, ConfigField, ConfigurableField, Field
31from lsst.pipe.base
import PipelineTask, PipelineTaskConfig, PipelineTaskConnections, Struct
34 FocalPlaneBackgroundConfig,
42 """Lookup function to identify sky frames.
46 datasetType : `lsst.daf.butler.DatasetType`
48 registry : `lsst.daf.butler.Registry`
49 Butler registry to query.
50 quantumDataId : `lsst.daf.butler.DataCoordinate`
51 Data id to transform to find sky frames.
52 The ``detector`` entry will be stripped.
53 collections : `lsst.daf.butler.CollectionSearch`
54 Collections to search through.
58 results : `list` [`lsst.daf.butler.DatasetRef`]
59 List of datasets that will be used as sky calibration frames.
61 newDataId = quantumDataId.subset(registry.dimensions.conform([
"instrument",
"visit"]))
63 for dataId
in registry.queryDataIds([
"visit",
"detector"], dataId=newDataId).expanded():
64 skyFrame = registry.findDataset(
65 datasetType, dataId, collections=collections, timespan=dataId.timespan
67 skyFrames.append(skyFrame)
72 """Match the order of one list to another, padding if necessary.
77 List to be reordered and padded. Elements can be any type.
79 Iterable of values to be compared with outputKeys.
80 Length must match `inputList`.
82 Iterable of values to be compared with inputKeys.
84 Any value to be inserted where one of inputKeys is not in outputKeys.
89 Copy of inputList reordered per outputKeys and padded with `padWith`
90 so that the length matches length of outputKeys.
93 for outputKey
in outputKeys:
94 if outputKey
in inputKeys:
95 outputList.append(inputList[inputKeys.index(outputKey)])
97 outputList.append(padWith)
102 rawLinker = cT.Input(
103 doc=
"Raw data to provide exp-visit linkage to connect calExp inputs to camera/sky calibs.",
107 storageClass=
"Exposure",
108 dimensions=[
"instrument",
"exposure",
"detector"],
111 doc=
"Background-subtracted calibrated exposures.",
114 storageClass=
"ExposureF",
115 dimensions=[
"instrument",
"visit",
"detector"],
118 doc=
"Subtracted backgrounds for input calibrated exposures.",
120 name=
"calexpBackground",
121 storageClass=
"Background",
122 dimensions=[
"instrument",
"visit",
"detector"],
124 skyFrames = cT.PrerequisiteInput(
125 doc=
"Calibration sky frames.",
128 storageClass=
"ExposureF",
129 dimensions=[
"instrument",
"physical_filter",
"detector"],
131 lookupFunction=_skyFrameLookup,
133 camera = cT.PrerequisiteInput(
136 storageClass=
"Camera",
137 dimensions=[
"instrument"],
141 doc=
"Sky correction data, to be subtracted from the calibrated exposures.",
144 storageClass=
"Background",
145 dimensions=[
"instrument",
"visit",
"detector"],
147 calExpMosaic = cT.Output(
148 doc=
"Full focal plane mosaicked image of the sky corrected calibrated exposures.",
149 name=
"calexp_skyCorr_visit_mosaic",
150 storageClass=
"ImageF",
151 dimensions=[
"instrument",
"visit"],
153 calBkgMosaic = cT.Output(
154 doc=
"Full focal plane mosaicked image of the sky corrected calibrated exposure backgrounds.",
155 name=
"calexpBackground_skyCorr_visit_mosaic",
156 storageClass=
"ImageF",
157 dimensions=[
"instrument",
"visit"],
160 def __init__(self, *, config:
"SkyCorrectionConfig | None" =
None):
162 assert config
is not None
168 maskObjects = ConfigurableField(
169 target=MaskObjectsTask,
172 doMaskObjects = Field(
175 doc=
"Iteratively mask objects to find good sky?",
177 bgModel1 = ConfigField(
178 dtype=FocalPlaneBackgroundConfig,
179 doc=
"Initial background model, prior to sky frame subtraction",
181 sky = ConfigurableField(
182 target=SkyMeasurementTask,
183 doc=
"Sky measurement",
188 doc=
"Do sky frame subtraction?",
190 bgModel2 = ConfigField(
191 dtype=FocalPlaneBackgroundConfig,
192 doc=
"Final (cleanup) background model, after sky frame subtraction",
197 doc=
"Do final (cleanup) background model subtraction, after sky frame subtraction?",
202 doc=
"Binning factor for constructing full focal plane '*_camera' output datasets",
206 Config.setDefaults(self)
215 """Perform a full focal plane sky correction."""
217 ConfigClass = SkyCorrectionConfig
218 _DefaultName =
"skyCorr"
222 self.makeSubtask(
"sky")
223 self.makeSubtask(
"maskObjects")
228 detectorOrder = [ref.dataId[
"detector"]
for ref
in inputRefs.calExps]
231 inputRefs.calExps, [ref.dataId[
"detector"]
for ref
in inputRefs.calExps], detectorOrder
234 inputRefs.calBkgs, [ref.dataId[
"detector"]
for ref
in inputRefs.calBkgs], detectorOrder
237 if self.config.doSky:
239 inputRefs.skyFrames, [ref.dataId[
"detector"]
for ref
in inputRefs.skyFrames], detectorOrder
242 inputRefs.skyFrames = []
244 outputRefs.skyCorr, [ref.dataId[
"detector"]
for ref
in outputRefs.skyCorr], detectorOrder
246 inputs = butlerQC.get(inputRefs)
247 inputs.pop(
"rawLinker",
None)
248 outputs = self.
run(**inputs)
249 butlerQC.put(outputs, outputRefs)
251 def run(self, calExps, calBkgs, skyFrames, camera):
252 """Perform sky correction on a visit.
254 The original visit-level background is first restored to the calibrated
255 exposure and the existing background model is inverted in-place. If
256 doMaskObjects is True, the mask map associated with this exposure will
257 be iteratively updated (over nIter loops) by re-estimating the
258 background each iteration and redetecting footprints.
260 An initial full focal plane sky subtraction (bgModel1) will take place
261 prior to scaling and subtracting the sky frame.
263 If doSky is True, the sky frame will be scaled to the flux in the input
266 If doBgModel2 is True, a final full focal plane sky subtraction will
267 take place after the sky frame has been subtracted.
269 The first N elements of the returned skyCorr will consist of inverted
270 elements of the calexpBackground model (i.e., subtractive). All
271 subsequent elements appended to skyCorr thereafter will be additive
272 such that, when skyCorr is subtracted from a calexp, the net result
273 will be to undo the initial per-detector background solution and then
274 apply the skyCorr model thereafter. Adding skyCorr to a
275 calexpBackground will effectively negate the calexpBackground,
276 returning only the additive background components of the skyCorr
281 calExps : `list` [`lsst.afw.image.exposure.ExposureF`]
282 Detector calibrated exposure images for the visit.
283 calBkgs : `list` [`lsst.afw.math.BackgroundList`]
284 Detector background lists matching the calibrated exposures.
285 skyFrames : `list` [`lsst.afw.image.exposure.ExposureF`]
286 Sky frame calibration data for the input detectors.
287 camera : `lsst.afw.cameraGeom.Camera`
288 Camera matching the input data to process.
292 results : `Struct` containing:
293 skyCorr : `list` [`lsst.afw.math.BackgroundList`]
294 Detector-level sky correction background lists.
295 calExpMosaic : `lsst.afw.image.exposure.ExposureF`
296 Visit-level mosaic of the sky corrected data, binned.
297 Analogous to `calexp - skyCorr`.
298 calBkgMosaic : `lsst.afw.image.exposure.ExposureF`
299 Visit-level mosaic of the sky correction background, binned.
300 Analogous to `calexpBackground + skyCorr`.
303 numOrigBkgElements = [len(calBkg)
for calBkg
in calBkgs]
310 if self.config.doSky:
314 if self.config.doBgModel2:
318 calExpIds = [exp.getDetector().getId()
for exp
in calExps]
320 for calBkg, num
in zip(calBkgs, numOrigBkgElements):
321 skyCorrExtra = calBkg.clone()
322 skyCorrExtra._backgrounds = skyCorrExtra._backgrounds[num:]
323 skyCorrExtras.append(skyCorrExtra)
324 calExpMosaic = self.
_binAndMosaic(calExps, camera, self.config.binning, ids=calExpIds, refExps=
None)
326 skyCorrExtras, camera, self.config.binning, ids=calExpIds, refExps=calExps
329 return Struct(skyCorr=calBkgs, calExpMosaic=calExpMosaic, calBkgMosaic=calBkgMosaic)
332 """Restore original background to each calexp and invert the related
333 background model; optionally refine the mask plane.
335 The original visit-level background is restored to each calibrated
336 exposure and the existing background model is inverted in-place. If
337 doMaskObjects is True, the mask map associated with the exposure will
338 be iteratively updated (over nIter loops) by re-estimating the
339 background each iteration and redetecting footprints.
341 The background model modified in-place in this method will comprise the
342 first N elements of the skyCorr dataset type, i.e., these N elements
343 are the inverse of the calexpBackground model. All subsequent elements
344 appended to skyCorr will be additive such that, when skyCorr is
345 subtracted from a calexp, the net result will be to undo the initial
346 per-detector background solution and then apply the skyCorr model
347 thereafter. Adding skyCorr to a calexpBackground will effectively
348 negate the calexpBackground, returning only the additive background
349 components of the skyCorr background model.
353 calExps : `lsst.afw.image.exposure.ExposureF`
354 Detector level calexp images to process.
355 calBkgs : `lsst.afw.math._backgroundList.BackgroundList`
356 Detector level background lists associated with the calexps.
360 calExps : `lsst.afw.image.exposure.ExposureF`
361 The calexps with the initially subtracted background restored.
362 skyCorrBases : `lsst.afw.math._backgroundList.BackgroundList`
363 The inverted initial background models; the genesis for skyCorrs.
366 for calExp, calBkg
in zip(calExps, calBkgs):
367 image = calExp.getMaskedImage()
370 for calBkgElement
in calBkg:
371 statsImage = calBkgElement[0].getStatsImage()
373 skyCorrBase = calBkg.getImage()
377 if self.config.doMaskObjects:
378 self.maskObjects.findObjects(calExp)
380 stats = np.nanpercentile(skyCorrBase.array, [50, 75, 25])
382 "Detector %d: Initial background restored; BG median = %.1f counts, BG IQR = %.1f counts",
383 calExp.getDetector().getId(),
385 np.subtract(*stats[1:]),
387 skyCorrBases.append(skyCorrBase)
388 return calExps, skyCorrBases
391 """Perform a full focal-plane background subtraction for a visit.
393 Generate a full focal plane background model, binning all masked
394 detectors into bins of [bgModelN.xSize, bgModelN.ySize]. After,
395 subtract the resultant background model (translated back into CCD
396 coordinates) from the original detector exposure.
398 Return a list of background subtracted images and a list of full focal
399 plane background parameters.
403 calExps : `list` [`lsst.afw.image.exposure.ExposureF`]
404 Calibrated exposures to be background subtracted.
405 calBkgs : `list` [`lsst.afw.math._backgroundList.BackgroundList`]
406 Background lists associated with the input calibrated exposures.
407 camera : `lsst.afw.cameraGeom.Camera`
409 config : `lsst.pipe.tasks.background.FocalPlaneBackgroundConfig`
410 Configuration to use for background subtraction.
414 calExps : `list` [`lsst.afw.image.maskedImage.MaskedImageF`]
415 Background subtracted exposures for creating a focal plane image.
416 calBkgs : `list` [`lsst.afw.math._backgroundList.BackgroundList`]
417 Updated background lists with a visit-level model appended.
420 bgModelBase = FocalPlaneBackground.fromCamera(config, camera)
426 for calExp
in calExps:
427 bgModel = bgModelBase.clone()
428 bgModel.addCcd(calExp)
429 bgModels.append(bgModel)
432 for bgModel, calExp
in zip(bgModels, calExps):
434 "Detector %d: Merging %d unmasked pixels (%.1f%s of detector area) into focal plane "
439 calExp.getDetector().getId(),
440 bgModel._numbers.getArray().sum(),
441 100 * bgModel._numbers.getArray().sum() / calExp.getBBox().getArea(),
444 bgModelBase.merge(bgModel)
448 for calExp
in calExps:
450 calBkgElements.append(calBkgElement)
453 "Focal plane background model constructed using %.2f x %.2f mm (%d x %d pixel) superpixels; "
454 "FP BG median = %.1f counts, FP BG IQR = %.1f counts"
456 with warnings.catch_warnings():
457 warnings.filterwarnings(
"ignore",
r"invalid value encountered")
458 stats = np.nanpercentile(bgModelBase.getStatsImage().array, [50, 75, 25])
463 int(config.xSize / config.pixelSize),
464 int(config.ySize / config.pixelSize),
466 np.subtract(*stats[1:]),
469 for calBkg, calBkgElement
in zip(calBkgs, calBkgElements):
470 calBkg.append(calBkgElement[0])
471 return calExps, calBkgs
474 """Generate CCD background model and subtract from image.
476 Translate the full focal plane background into CCD coordinates and
477 subtract from the original science exposure image.
481 calExp : `lsst.afw.image.exposure.ExposureF`
482 Exposure to subtract the background model from.
483 bgModel : `lsst.pipe.tasks.background.FocalPlaneBackground`
484 Full focal plane camera-level background model.
488 calExp : `lsst.afw.image.exposure.ExposureF`
489 Background subtracted input exposure.
490 calBkgElement : `lsst.afw.math._backgroundList.BackgroundList`
491 Detector level realization of the full focal plane bg model.
493 image = calExp.getMaskedImage()
494 with warnings.catch_warnings():
495 warnings.filterwarnings(
"ignore",
r"invalid value encountered")
496 calBkgElement = bgModel.toCcdBackground(calExp.getDetector(), image.getBBox())
497 image -= calBkgElement.getImage()
498 return calExp, calBkgElement
501 """Determine the full focal plane sky frame scale factor relative to
502 an input list of calibrated exposures and subtract.
504 This method measures the sky frame scale on all inputs, resulting in
505 values equal to the background method solveScales(). The sky frame is
506 then subtracted as in subtractSkyFrame() using the appropriate scale.
508 Input calExps and calBkgs are updated in-place, returning sky frame
509 subtracted calExps and sky frame updated calBkgs, respectively.
513 calExps : `list` [`lsst.afw.image.exposure.ExposureF`]
514 Calibrated exposures to be background subtracted.
515 skyFrames : `list` [`lsst.afw.image.exposure.ExposureF`]
516 Sky frame calibration data for the input detectors.
517 calBkgs : `list` [`lsst.afw.math._backgroundList.BackgroundList`]
518 Background lists associated with the input calibrated exposures.
520 skyFrameBgModels = []
522 for calExp, skyFrame
in zip(calExps, skyFrames):
523 skyFrameBgModel = self.sky.exposureToBackground(skyFrame)
524 skyFrameBgModels.append(skyFrameBgModel)
526 samples = self.sky.measureScale(calExp.getMaskedImage(), skyFrameBgModel)
527 scales.append(samples)
528 scale = self.sky.solveScales(scales)
529 for calExp, skyFrameBgModel, calBkg
in zip(calExps, skyFrameBgModels, calBkgs):
532 self.sky.subtractSkyFrame(calExp.getMaskedImage(), skyFrameBgModel, scale, calBkg)
533 self.log.info(
"Sky frame subtracted with a scale factor of %.5f", scale)
535 def _binAndMosaic(self, exposures, camera, binning, ids=None, refExps=None):
536 """Bin input exposures and mosaic across the entire focal plane.
538 Input exposures are binned and then mosaicked at the position of
539 the detector in the focal plane of the camera.
544 Detector level list of either calexp `ExposureF` types or
545 calexpBackground `BackgroundList` types.
546 camera : `lsst.afw.cameraGeom.Camera`
547 Camera matching the input data to process.
549 Binning size to be applied to input images.
550 ids : `list` [`int`], optional
551 List of detector ids to iterate over.
552 refExps : `list` [`lsst.afw.image.exposure.ExposureF`], optional
553 If supplied, mask planes from these reference images will be used.
556 mosaicImage : `lsst.afw.image.exposure.ExposureF`
557 Mosaicked full focal plane image.
559 refExps = np.resize(refExps, len(exposures))
561 for exp, refExp
in zip(exposures, refExps):
563 nativeImage = exp.getMaskedImage()
564 except AttributeError:
565 nativeImage = afwImage.makeMaskedImage(exp.getImage())
567 nativeImage.setMask(refExp.getMask())
568 binnedImage = afwMath.binImage(nativeImage, binning)
569 binnedImages.append(binnedImage)
571 mosConfig.binning = binning
573 imageStruct = mosTask.run(binnedImages, camera, inputIds=ids)
574 mosaicImage = imageStruct.outputData
__init__(self, *"SkyCorrectionConfig | None" config=None)
_subtractVisitBackground(self, calExps, calBkgs, camera, config)
run(self, calExps, calBkgs, skyFrames, camera)
_subtractSkyFrame(self, calExps, skyFrames, calBkgs)
_restoreBackgroundRefineMask(self, calExps, calBkgs)
runQuantum(self, butlerQC, inputRefs, outputRefs)
_subtractDetectorBackground(self, calExp, bgModel)
_binAndMosaic(self, exposures, camera, binning, ids=None, refExps=None)
__init__(self, *args, **kwargs)
_reorderAndPadList(inputList, inputKeys, outputKeys, padWith=None)
_skyFrameLookup(datasetType, registry, quantumDataId, collections)