22__all__ = [
"SkyCorrectionTask",
"SkyCorrectionConfig"]
28import lsst.pipe.base.connectionTypes
as cT
30from lsst.pex.config import Config, ConfigField, ConfigurableField, Field, FieldValidationError
31from lsst.pipe.base
import PipelineTask, PipelineTaskConfig, PipelineTaskConnections, Struct
34 FocalPlaneBackgroundConfig,
42 """Lookup function to identify sky frames.
46 datasetType : `lsst.daf.butler.DatasetType`
48 registry : `lsst.daf.butler.Registry`
49 Butler registry to query.
50 quantumDataId : `lsst.daf.butler.DataCoordinate`
51 Data id to transform to find sky frames.
52 The ``detector`` entry will be stripped.
53 collections : `lsst.daf.butler.CollectionSearch`
54 Collections to search through.
58 results : `list` [`lsst.daf.butler.DatasetRef`]
59 List of datasets that will be used as sky calibration frames.
61 newDataId = quantumDataId.subset(registry.dimensions.conform([
"instrument",
"visit"]))
63 for dataId
in registry.queryDataIds([
"visit",
"detector"], dataId=newDataId).expanded():
64 skyFrame = registry.findDataset(
65 datasetType, dataId, collections=collections, timespan=dataId.timespan
67 skyFrames.append(skyFrame)
72 """Match the order of one list to another, padding if necessary.
77 List to be reordered and padded. Elements can be any type.
79 Iterable of values to be compared with outputKeys.
80 Length must match `inputList`.
82 Iterable of values to be compared with inputKeys.
84 Any value to be inserted where one of inputKeys is not in outputKeys.
89 Copy of inputList reordered per outputKeys and padded with `padWith`
90 so that the length matches length of outputKeys.
93 for outputKey
in outputKeys:
94 if outputKey
in inputKeys:
95 outputList.append(inputList[inputKeys.index(outputKey)])
97 outputList.append(padWith)
102 rawLinker = cT.Input(
103 doc=
"Raw data to provide exp-visit linkage to connect calExp inputs to camera/sky calibs.",
107 storageClass=
"Exposure",
108 dimensions=[
"instrument",
"exposure",
"detector"],
111 doc=
"Background-subtracted calibrated exposures.",
114 storageClass=
"ExposureF",
115 dimensions=[
"instrument",
"visit",
"detector"],
118 doc=
"Subtracted backgrounds for input calibrated exposures.",
120 name=
"calexpBackground",
121 storageClass=
"Background",
122 dimensions=[
"instrument",
"visit",
"detector"],
124 skyFrames = cT.PrerequisiteInput(
125 doc=
"Calibration sky frames.",
128 storageClass=
"ExposureF",
129 dimensions=[
"instrument",
"physical_filter",
"detector"],
131 lookupFunction=_skyFrameLookup,
133 camera = cT.PrerequisiteInput(
136 storageClass=
"Camera",
137 dimensions=[
"instrument"],
141 doc=
"Sky correction data, to be subtracted from the calibrated exposures.",
144 storageClass=
"Background",
145 dimensions=[
"instrument",
"visit",
"detector"],
147 calExpMosaic = cT.Output(
148 doc=
"Full focal plane mosaicked image of the sky corrected calibrated exposures.",
149 name=
"calexp_skyCorr_visit_mosaic",
150 storageClass=
"ImageF",
151 dimensions=[
"instrument",
"visit"],
153 calBkgMosaic = cT.Output(
154 doc=
"Full focal plane mosaicked image of the sky corrected calibrated exposure backgrounds.",
155 name=
"calexpBackground_skyCorr_visit_mosaic",
156 storageClass=
"ImageF",
157 dimensions=[
"instrument",
"visit"],
162 maskObjects = ConfigurableField(
163 target=MaskObjectsTask,
166 doMaskObjects = Field(
169 doc=
"Iteratively mask objects to find good sky?",
171 bgModel = ConfigField(
173 doc=
"Initial background model, prior to sky frame subtraction",
174 deprecated=
"This field is deprecated and will be removed after v26. Please use bgModel1 instead.",
179 doc=
"Do initial background model subtraction (prior to sky frame subtraction)?",
181 deprecated=
"This field is deprecated and will be removed after v26. See RFC-898 for further details.",
183 bgModel1 = ConfigField(
184 dtype=FocalPlaneBackgroundConfig,
185 doc=
"Initial background model, prior to sky frame subtraction",
190 doc=
"Do initial background model subtraction (prior to sky frame subtraction)?",
191 deprecated=
"This field is deprecated and will be removed after v26. See RFC-898 for further details.",
193 sky = ConfigurableField(
194 target=SkyMeasurementTask,
195 doc=
"Sky measurement",
200 doc=
"Do sky frame subtraction?",
202 bgModel2 = ConfigField(
203 dtype=FocalPlaneBackgroundConfig,
204 doc=
"Final (cleanup) background model, after sky frame subtraction",
209 doc=
"Do final (cleanup) background model subtraction, after sky frame subtraction?",
214 doc=
"Binning factor for constructing full focal plane '*_camera' output datasets",
218 Config.setDefaults(self)
229 msg =
"The doBgModel field will be removed after v26."
230 raise FieldValidationError(self.__class__.doBgModel, self, msg)
234 """Perform a full focal plane sky correction."""
236 ConfigClass = SkyCorrectionConfig
237 _DefaultName =
"skyCorr"
241 self.makeSubtask(
"sky")
242 self.makeSubtask(
"maskObjects")
247 detectorOrder = [ref.dataId[
"detector"]
for ref
in inputRefs.calExps]
250 inputRefs.calExps, [ref.dataId[
"detector"]
for ref
in inputRefs.calExps], detectorOrder
253 inputRefs.calBkgs, [ref.dataId[
"detector"]
for ref
in inputRefs.calBkgs], detectorOrder
256 inputRefs.skyFrames, [ref.dataId[
"detector"]
for ref
in inputRefs.skyFrames], detectorOrder
259 outputRefs.skyCorr, [ref.dataId[
"detector"]
for ref
in outputRefs.skyCorr], detectorOrder
261 inputs = butlerQC.get(inputRefs)
262 inputs.pop(
"rawLinker",
None)
263 outputs = self.
run(**inputs)
264 butlerQC.put(outputs, outputRefs)
266 def run(self, calExps, calBkgs, skyFrames, camera):
267 """Perform sky correction on a visit.
269 The original visit-level background is first restored to the calibrated
270 exposure and the existing background model is inverted in-place. If
271 doMaskObjects is True, the mask map associated with this exposure will
272 be iteratively updated (over nIter loops) by re-estimating the
273 background each iteration and redetecting footprints.
275 If doBgModel1 is True, an initial full focal plane sky subtraction will
276 take place prior to scaling and subtracting the sky frame.
278 If doSky is True, the sky frame will be scaled to the flux in the input
281 If doBgModel2 is True, a final full focal plane sky subtraction will
282 take place after the sky frame has been subtracted.
284 The first N elements of the returned skyCorr will consist of inverted
285 elements of the calexpBackground model (i.e., subtractive). All
286 subsequent elements appended to skyCorr thereafter will be additive
287 such that, when skyCorr is subtracted from a calexp, the net result
288 will be to undo the initial per-detector background solution and then
289 apply the skyCorr model thereafter. Adding skyCorr to a
290 calexpBackground will effectively negate the calexpBackground,
291 returning only the additive background components of the skyCorr
296 calExps : `list` [`lsst.afw.image.exposure.ExposureF`]
297 Detector calibrated exposure images for the visit.
298 calBkgs : `list` [`lsst.afw.math.BackgroundList`]
299 Detector background lists matching the calibrated exposures.
300 skyFrames : `list` [`lsst.afw.image.exposure.ExposureF`]
301 Sky frame calibration data for the input detectors.
302 camera : `lsst.afw.cameraGeom.Camera`
303 Camera matching the input data to process.
307 results : `Struct` containing:
308 skyCorr : `list` [`lsst.afw.math.BackgroundList`]
309 Detector-level sky correction background lists.
310 calExpMosaic : `lsst.afw.image.exposure.ExposureF`
311 Visit-level mosaic of the sky corrected data, binned.
312 Analogous to `calexp - skyCorr`.
313 calBkgMosaic : `lsst.afw.image.exposure.ExposureF`
314 Visit-level mosaic of the sky correction background, binned.
315 Analogous to `calexpBackground + skyCorr`.
318 numOrigBkgElements = [len(calBkg)
for calBkg
in calBkgs]
322 if self.config.doBgModel1:
326 if self.config.doSky:
330 if self.config.doBgModel2:
334 calExpIds = [exp.getDetector().getId()
for exp
in calExps]
336 for calBkg, num
in zip(calBkgs, numOrigBkgElements):
337 skyCorrExtra = calBkg.clone()
338 skyCorrExtra._backgrounds = skyCorrExtra._backgrounds[num:]
339 skyCorrExtras.append(skyCorrExtra)
340 calExpMosaic = self.
_binAndMosaic(calExps, camera, self.config.binning, ids=calExpIds, refExps=
None)
342 skyCorrExtras, camera, self.config.binning, ids=calExpIds, refExps=calExps
345 return Struct(skyCorr=calBkgs, calExpMosaic=calExpMosaic, calBkgMosaic=calBkgMosaic)
348 """Restore original background to each calexp and invert the related
349 background model; optionally refine the mask plane.
351 The original visit-level background is restored to each calibrated
352 exposure and the existing background model is inverted in-place. If
353 doMaskObjects is True, the mask map associated with the exposure will
354 be iteratively updated (over nIter loops) by re-estimating the
355 background each iteration and redetecting footprints.
357 The background model modified in-place in this method will comprise the
358 first N elements of the skyCorr dataset type, i.e., these N elements
359 are the inverse of the calexpBackground model. All subsequent elements
360 appended to skyCorr will be additive such that, when skyCorr is
361 subtracted from a calexp, the net result will be to undo the initial
362 per-detector background solution and then apply the skyCorr model
363 thereafter. Adding skyCorr to a calexpBackground will effectively
364 negate the calexpBackground, returning only the additive background
365 components of the skyCorr background model.
369 calExps : `lsst.afw.image.exposure.ExposureF`
370 Detector level calexp images to process.
371 calBkgs : `lsst.afw.math._backgroundList.BackgroundList`
372 Detector level background lists associated with the calexps.
376 calExps : `lsst.afw.image.exposure.ExposureF`
377 The calexps with the initially subtracted background restored.
378 skyCorrBases : `lsst.afw.math._backgroundList.BackgroundList`
379 The inverted initial background models; the genesis for skyCorrs.
382 for calExp, calBkg
in zip(calExps, calBkgs):
383 image = calExp.getMaskedImage()
386 for calBkgElement
in calBkg:
387 statsImage = calBkgElement[0].getStatsImage()
389 skyCorrBase = calBkg.getImage()
393 if self.config.doMaskObjects:
394 self.maskObjects.findObjects(calExp)
396 stats = np.nanpercentile(skyCorrBase.array, [50, 75, 25])
398 "Detector %d: Initial background restored; BG median = %.1f counts, BG IQR = %.1f counts",
399 calExp.getDetector().getId(),
401 np.subtract(*stats[1:]),
403 skyCorrBases.append(skyCorrBase)
404 return calExps, skyCorrBases
407 """Perform a full focal-plane background subtraction for a visit.
409 Generate a full focal plane background model, binning all masked
410 detectors into bins of [bgModelN.xSize, bgModelN.ySize]. After,
411 subtract the resultant background model (translated back into CCD
412 coordinates) from the original detector exposure.
414 Return a list of background subtracted images and a list of full focal
415 plane background parameters.
419 calExps : `list` [`lsst.afw.image.exposure.ExposureF`]
420 Calibrated exposures to be background subtracted.
421 calBkgs : `list` [`lsst.afw.math._backgroundList.BackgroundList`]
422 Background lists associated with the input calibrated exposures.
423 camera : `lsst.afw.cameraGeom.Camera`
425 config : `lsst.pipe.tasks.background.FocalPlaneBackgroundConfig`
426 Configuration to use for background subtraction.
430 calExps : `list` [`lsst.afw.image.maskedImage.MaskedImageF`]
431 Background subtracted exposures for creating a focal plane image.
432 calBkgs : `list` [`lsst.afw.math._backgroundList.BackgroundList`]
433 Updated background lists with a visit-level model appended.
436 bgModelBase = FocalPlaneBackground.fromCamera(config, camera)
442 for calExp
in calExps:
443 bgModel = bgModelBase.clone()
444 bgModel.addCcd(calExp)
445 bgModels.append(bgModel)
448 for bgModel, calExp
in zip(bgModels, calExps):
450 "Detector %d: Merging %d unmasked pixels (%.1f%s of detector area) into focal plane "
455 calExp.getDetector().getId(),
456 bgModel._numbers.getArray().sum(),
457 100 * bgModel._numbers.getArray().sum() / calExp.getBBox().getArea(),
460 bgModelBase.merge(bgModel)
464 for calExp
in calExps:
466 calBkgElements.append(calBkgElement)
469 "Focal plane background model constructed using %.2f x %.2f mm (%d x %d pixel) superpixels; "
470 "FP BG median = %.1f counts, FP BG IQR = %.1f counts"
472 with warnings.catch_warnings():
473 warnings.filterwarnings(
"ignore",
r"invalid value encountered")
474 stats = np.nanpercentile(bgModelBase.getStatsImage().array, [50, 75, 25])
479 int(config.xSize / config.pixelSize),
480 int(config.ySize / config.pixelSize),
482 np.subtract(*stats[1:]),
485 for calBkg, calBkgElement
in zip(calBkgs, calBkgElements):
486 calBkg.append(calBkgElement[0])
487 return calExps, calBkgs
490 """Generate CCD background model and subtract from image.
492 Translate the full focal plane background into CCD coordinates and
493 subtract from the original science exposure image.
497 calExp : `lsst.afw.image.exposure.ExposureF`
498 Exposure to subtract the background model from.
499 bgModel : `lsst.pipe.tasks.background.FocalPlaneBackground`
500 Full focal plane camera-level background model.
504 calExp : `lsst.afw.image.exposure.ExposureF`
505 Background subtracted input exposure.
506 calBkgElement : `lsst.afw.math._backgroundList.BackgroundList`
507 Detector level realization of the full focal plane bg model.
509 image = calExp.getMaskedImage()
510 with warnings.catch_warnings():
511 warnings.filterwarnings(
"ignore",
r"invalid value encountered")
512 calBkgElement = bgModel.toCcdBackground(calExp.getDetector(), image.getBBox())
513 image -= calBkgElement.getImage()
514 return calExp, calBkgElement
517 """Determine the full focal plane sky frame scale factor relative to
518 an input list of calibrated exposures and subtract.
520 This method measures the sky frame scale on all inputs, resulting in
521 values equal to the background method solveScales(). The sky frame is
522 then subtracted as in subtractSkyFrame() using the appropriate scale.
524 Input calExps and calBkgs are updated in-place, returning sky frame
525 subtracted calExps and sky frame updated calBkgs, respectively.
529 calExps : `list` [`lsst.afw.image.exposure.ExposureF`]
530 Calibrated exposures to be background subtracted.
531 skyFrames : `list` [`lsst.afw.image.exposure.ExposureF`]
532 Sky frame calibration data for the input detectors.
533 calBkgs : `list` [`lsst.afw.math._backgroundList.BackgroundList`]
534 Background lists associated with the input calibrated exposures.
536 skyFrameBgModels = []
538 for calExp, skyFrame
in zip(calExps, skyFrames):
539 skyFrameBgModel = self.sky.exposureToBackground(skyFrame)
540 skyFrameBgModels.append(skyFrameBgModel)
542 samples = self.sky.measureScale(calExp.getMaskedImage(), skyFrameBgModel)
543 scales.append(samples)
544 scale = self.sky.solveScales(scales)
545 for calExp, skyFrameBgModel, calBkg
in zip(calExps, skyFrameBgModels, calBkgs):
548 self.sky.subtractSkyFrame(calExp.getMaskedImage(), skyFrameBgModel, scale, calBkg)
549 self.log.info(
"Sky frame subtracted with a scale factor of %.5f", scale)
551 def _binAndMosaic(self, exposures, camera, binning, ids=None, refExps=None):
552 """Bin input exposures and mosaic across the entire focal plane.
554 Input exposures are binned and then mosaicked at the position of
555 the detector in the focal plane of the camera.
560 Detector level list of either calexp `ExposureF` types or
561 calexpBackground `BackgroundList` types.
562 camera : `lsst.afw.cameraGeom.Camera`
563 Camera matching the input data to process.
565 Binning size to be applied to input images.
566 ids : `list` [`int`], optional
567 List of detector ids to iterate over.
568 refExps : `list` [`lsst.afw.image.exposure.ExposureF`], optional
569 If supplied, mask planes from these reference images will be used.
572 mosaicImage : `lsst.afw.image.exposure.ExposureF`
573 Mosaicked full focal plane image.
575 refExps = np.resize(refExps, len(exposures))
577 for exp, refExp
in zip(exposures, refExps):
579 nativeImage = exp.getMaskedImage()
580 except AttributeError:
581 nativeImage = afwImage.makeMaskedImage(exp.getImage())
583 nativeImage.setMask(refExp.getMask())
584 binnedImage = afwMath.binImage(nativeImage, binning)
585 binnedImages.append(binnedImage)
587 mosConfig.binning = binning
589 imageStruct = mosTask.run(binnedImages, camera, inputIds=ids)
590 mosaicImage = imageStruct.outputData
_subtractVisitBackground(self, calExps, calBkgs, camera, config)
run(self, calExps, calBkgs, skyFrames, camera)
_subtractSkyFrame(self, calExps, skyFrames, calBkgs)
_restoreBackgroundRefineMask(self, calExps, calBkgs)
runQuantum(self, butlerQC, inputRefs, outputRefs)
_subtractDetectorBackground(self, calExp, bgModel)
_binAndMosaic(self, exposures, camera, binning, ids=None, refExps=None)
__init__(self, *args, **kwargs)
_reorderAndPadList(inputList, inputKeys, outputKeys, padWith=None)
_skyFrameLookup(datasetType, registry, quantumDataId, collections)