22__all__ = [
"SkyCorrectionTask",
"SkyCorrectionConfig"]
26import lsst.pipe.base.connectionTypes
as cT
28from lsst.daf.butler
import DimensionGraph
29from lsst.pex.config import Config, ConfigField, ConfigurableField, Field, FieldValidationError
30from lsst.pipe.base import PipelineTask, PipelineTaskConfig, PipelineTaskConnections, Struct
33 FocalPlaneBackgroundConfig,
40def _skyFrameLookup(datasetType, registry, quantumDataId, collections):
41 """Lookup function to identify sky frames.
45 datasetType : `lsst.daf.butler.DatasetType`
47 registry : `lsst.daf.butler.Registry`
48 Butler registry to query.
49 quantumDataId : `lsst.daf.butler.DataCoordinate`
50 Data id to transform to find sky frames.
51 The ``detector`` entry will be stripped.
52 collections : `lsst.daf.butler.CollectionSearch`
53 Collections to search through.
57 results : `list` [`lsst.daf.butler.DatasetRef`]
58 List of datasets that will be used as sky calibration frames.
60 newDataId = quantumDataId.subset(DimensionGraph(registry.dimensions, names=["instrument",
"visit"]))
62 for dataId
in registry.queryDataIds([
"visit",
"detector"], dataId=newDataId).expanded():
63 skyFrame = registry.findDataset(
64 datasetType, dataId, collections=collections, timespan=dataId.timespan
66 skyFrames.append(skyFrame)
70def _reorderAndPadList(inputList, inputKeys, outputKeys, padWith=None):
71 """Match the order of one list to another, padding if necessary.
76 List to be reordered and padded. Elements can be any type.
78 Iterable of values to be compared
with outputKeys.
79 Length must match `inputList`.
81 Iterable of values to be compared
with inputKeys.
83 Any value to be inserted where one of inputKeys
is not in outputKeys.
88 Copy of inputList reordered per outputKeys
and padded
with `padWith`
89 so that the length matches length of outputKeys.
92 for outputKey
in outputKeys:
93 if outputKey
in inputKeys:
94 outputList.append(inputList[inputKeys.index(outputKey)])
96 outputList.append(padWith)
101 rawLinker = cT.Input(
102 doc=
"Raw data to provide exp-visit linkage to connect calExp inputs to camera/sky calibs.",
106 storageClass=
"Exposure",
107 dimensions=[
"instrument",
"exposure",
"detector"],
110 doc=
"Background-subtracted calibrated exposures.",
113 storageClass=
"ExposureF",
114 dimensions=[
"instrument",
"visit",
"detector"],
117 doc=
"Subtracted backgrounds for input calibrated exposures.",
119 name=
"calexpBackground",
120 storageClass=
"Background",
121 dimensions=[
"instrument",
"visit",
"detector"],
123 skyFrames = cT.PrerequisiteInput(
124 doc=
"Calibration sky frames.",
127 storageClass=
"ExposureF",
128 dimensions=[
"instrument",
"physical_filter",
"detector"],
130 lookupFunction=_skyFrameLookup,
132 camera = cT.PrerequisiteInput(
135 storageClass=
"Camera",
136 dimensions=[
"instrument"],
140 doc=
"Sky correction data, to be subtracted from the calibrated exposures.",
143 storageClass=
"Background",
144 dimensions=[
"instrument",
"visit",
"detector"],
146 calExpMosaic = cT.Output(
147 doc=
"Full focal plane mosaicked image of the sky corrected calibrated exposures.",
148 name=
"calexp_skyCorr_visit_mosaic",
149 storageClass=
"ImageF",
150 dimensions=[
"instrument",
"visit"],
152 calBkgMosaic = cT.Output(
153 doc=
"Full focal plane mosaicked image of the sky corrected calibrated exposure backgrounds.",
154 name=
"calexpBackground_skyCorr_visit_mosaic",
155 storageClass=
"ImageF",
156 dimensions=[
"instrument",
"visit"],
161 maskObjects = ConfigurableField(
162 target=MaskObjectsTask,
165 doMaskObjects = Field(
168 doc=
"Iteratively mask objects to find good sky?",
170 bgModel = ConfigField(
172 doc=
"Initial background model, prior to sky frame subtraction",
173 deprecated=
"This field is deprecated and will be removed after v26. Please use bgModel1 instead.",
178 doc=
"Do initial background model subtraction (prior to sky frame subtraction)?",
180 deprecated=
"This field is deprecated and will be removed after v26. See RFC-898 for further details.",
182 bgModel1 = ConfigField(
183 dtype=FocalPlaneBackgroundConfig,
184 doc=
"Initial background model, prior to sky frame subtraction",
189 doc=
"Do initial background model subtraction (prior to sky frame subtraction)?",
190 deprecated=
"This field is deprecated and will be removed after v26. See RFC-898 for further details.",
192 sky = ConfigurableField(
193 target=SkyMeasurementTask,
194 doc=
"Sky measurement",
199 doc=
"Do sky frame subtraction?",
201 bgModel2 = ConfigField(
202 dtype=FocalPlaneBackgroundConfig,
203 doc=
"Final (cleanup) background model, after sky frame subtraction",
208 doc=
"Do final (cleanup) background model subtraction, after sky frame subtraction?",
213 doc=
"Binning factor for constructing full focal plane '*_camera' output datasets",
217 Config.setDefaults(self)
228 msg =
"The doBgModel field will be removed after v26."
229 raise FieldValidationError(self.__class__.doBgModel, self, msg)
233 """Perform a full focal plane sky correction."""
235 ConfigClass = SkyCorrectionConfig
236 _DefaultName =
"skyCorr"
240 self.makeSubtask(
"sky")
241 self.makeSubtask(
"maskObjects")
246 detectorOrder = [ref.dataId[
"detector"]
for ref
in inputRefs.calExps]
248 inputRefs.calExps = _reorderAndPadList(
249 inputRefs.calExps, [ref.dataId[
"detector"]
for ref
in inputRefs.calExps], detectorOrder
251 inputRefs.calBkgs = _reorderAndPadList(
252 inputRefs.calBkgs, [ref.dataId[
"detector"]
for ref
in inputRefs.calBkgs], detectorOrder
254 inputRefs.skyFrames = _reorderAndPadList(
255 inputRefs.skyFrames, [ref.dataId[
"detector"]
for ref
in inputRefs.skyFrames], detectorOrder
257 outputRefs.skyCorr = _reorderAndPadList(
258 outputRefs.skyCorr, [ref.dataId[
"detector"]
for ref
in outputRefs.skyCorr], detectorOrder
260 inputs = butlerQC.get(inputRefs)
261 inputs.pop(
"rawLinker",
None)
262 outputs = self.
run(**inputs)
263 butlerQC.put(outputs, outputRefs)
265 def run(self, calExps, calBkgs, skyFrames, camera):
266 """Perform sky correction on a visit.
268 The original visit-level background is first restored to the calibrated
269 exposure
and the existing background model
is inverted
in-place. If
270 doMaskObjects
is True, the mask map associated
with this exposure will
271 be iteratively updated (over nIter loops) by re-estimating the
272 background each iteration
and redetecting footprints.
274 If doBgModel1
is True, an initial full focal plane sky subtraction will
275 take place prior to scaling
and subtracting the sky frame.
277 If doSky
is True, the sky frame will be scaled to the flux
in the input
280 If doBgModel2
is True, a final full focal plane sky subtraction will
281 take place after the sky frame has been subtracted.
283 The first N elements of the returned skyCorr will consist of inverted
284 elements of the calexpBackground model (i.e., subtractive). All
285 subsequent elements appended to skyCorr thereafter will be additive
286 such that, when skyCorr
is subtracted
from a calexp, the net result
287 will be to undo the initial per-detector background solution
and then
288 apply the skyCorr model thereafter. Adding skyCorr to a
289 calexpBackground will effectively negate the calexpBackground,
290 returning only the additive background components of the skyCorr
295 calExps : `list` [`lsst.afw.image.exposure.ExposureF`]
296 Detector calibrated exposure images
for the visit.
297 calBkgs : `list` [`lsst.afw.math.BackgroundList`]
298 Detector background lists matching the calibrated exposures.
299 skyFrames : `list` [`lsst.afw.image.exposure.ExposureF`]
300 Sky frame calibration data
for the input detectors.
302 Camera matching the input data to process.
306 results : `Struct` containing:
307 skyCorr : `list` [`lsst.afw.math.BackgroundList`]
308 Detector-level sky correction background lists.
309 calExpMosaic : `lsst.afw.image.exposure.ExposureF`
310 Visit-level mosaic of the sky corrected data, binned.
311 Analogous to `calexp - skyCorr`.
312 calBkgMosaic : `lsst.afw.image.exposure.ExposureF`
313 Visit-level mosaic of the sky correction background, binned.
314 Analogous to `calexpBackground + skyCorr`.
317 numOrigBkgElements = [len(calBkg)
for calBkg
in calBkgs]
321 if self.config.doBgModel1:
325 if self.config.doSky:
329 if self.config.doBgModel2:
333 calExpIds = [exp.getDetector().getId()
for exp
in calExps]
335 for calBkg, num
in zip(calBkgs, numOrigBkgElements):
336 skyCorrExtra = calBkg.clone()
337 skyCorrExtra._backgrounds = skyCorrExtra._backgrounds[num:]
338 skyCorrExtras.append(skyCorrExtra)
339 calExpMosaic = self.
_binAndMosaic(calExps, camera, self.config.binning, ids=calExpIds, refExps=
None)
341 skyCorrExtras, camera, self.config.binning, ids=calExpIds, refExps=calExps
344 return Struct(skyCorr=calBkgs, calExpMosaic=calExpMosaic, calBkgMosaic=calBkgMosaic)
346 def _restoreBackgroundRefineMask(self, calExps, calBkgs):
347 """Restore original background to each calexp and invert the related
348 background model; optionally refine the mask plane.
350 The original visit-level background is restored to each calibrated
351 exposure
and the existing background model
is inverted
in-place. If
352 doMaskObjects
is True, the mask map associated
with the exposure will
353 be iteratively updated (over nIter loops) by re-estimating the
354 background each iteration
and redetecting footprints.
356 The background model modified
in-place
in this method will comprise the
357 first N elements of the skyCorr dataset type, i.e., these N elements
358 are the inverse of the calexpBackground model. All subsequent elements
359 appended to skyCorr will be additive such that, when skyCorr
is
360 subtracted
from a calexp, the net result will be to undo the initial
361 per-detector background solution
and then apply the skyCorr model
362 thereafter. Adding skyCorr to a calexpBackground will effectively
363 negate the calexpBackground, returning only the additive background
364 components of the skyCorr background model.
368 calExps : `lsst.afw.image.exposure.ExposureF`
369 Detector level calexp images to process.
371 Detector level background lists associated
with the calexps.
375 calExps : `lsst.afw.image.exposure.ExposureF`
376 The calexps
with the initially subtracted background restored.
378 The inverted initial background models; the genesis
for skyCorrs.
381 for calExp, calBkg
in zip(calExps, calBkgs):
382 image = calExp.getMaskedImage()
385 for calBkgElement
in calBkg:
386 statsImage = calBkgElement[0].getStatsImage()
388 skyCorrBase = calBkg.getImage()
392 if self.config.doMaskObjects:
393 self.maskObjects.findObjects(calExp)
395 stats = np.nanpercentile(skyCorrBase.array, [50, 75, 25])
397 "Detector %d: Initial background restored; BG median = %.1f counts, BG IQR = %.1f counts",
398 calExp.getDetector().getId(),
400 np.subtract(*stats[1:]),
402 skyCorrBases.append(skyCorrBase)
403 return calExps, skyCorrBases
405 def _subtractVisitBackground(self, calExps, calBkgs, camera, config):
406 """Perform a full focal-plane background subtraction for a visit.
408 Generate a full focal plane background model, binning all masked
409 detectors into bins of [bgModelN.xSize, bgModelN.ySize]. After,
410 subtract the resultant background model (translated back into CCD
411 coordinates) from the original detector exposure.
413 Return a list of background subtracted images
and a list of full focal
414 plane background parameters.
418 calExps : `list` [`lsst.afw.image.exposure.ExposureF`]
419 Calibrated exposures to be background subtracted.
421 Background lists associated
with the input calibrated exposures.
425 Configuration to use
for background subtraction.
429 calExps : `list` [`lsst.afw.image.maskedImage.MaskedImageF`]
430 Background subtracted exposures
for creating a focal plane image.
432 Updated background lists
with a visit-level model appended.
435 bgModelBase = FocalPlaneBackground.fromCamera(config, camera)
441 for calExp
in calExps:
442 bgModel = bgModelBase.clone()
443 bgModel.addCcd(calExp)
444 bgModels.append(bgModel)
447 for bgModel, calExp
in zip(bgModels, calExps):
449 "Detector %d: Merging %d unmasked pixels (%.1f%s of detector area) into focal plane "
454 calExp.getDetector().getId(),
455 bgModel._numbers.getArray().sum(),
456 100 * bgModel._numbers.getArray().sum() / calExp.getBBox().getArea(),
459 bgModelBase.merge(bgModel)
463 for calExp
in calExps:
465 calBkgElements.append(calBkgElement)
468 "Focal plane background model constructed using %.2f x %.2f mm (%d x %d pixel) superpixels; "
469 "FP BG median = %.1f counts, FP BG IQR = %.1f counts"
471 with np.warnings.catch_warnings():
472 np.warnings.filterwarnings(
"ignore",
r"invalid value encountered")
473 stats = np.nanpercentile(bgModelBase.getStatsImage().array, [50, 75, 25])
478 int(config.xSize / config.pixelSize),
479 int(config.ySize / config.pixelSize),
481 np.subtract(*stats[1:]),
484 for calBkg, calBkgElement
in zip(calBkgs, calBkgElements):
485 calBkg.append(calBkgElement[0])
486 return calExps, calBkgs
488 def _subtractDetectorBackground(self, calExp, bgModel):
489 """Generate CCD background model and subtract from image.
491 Translate the full focal plane background into CCD coordinates and
492 subtract
from the original science exposure image.
496 calExp : `lsst.afw.image.exposure.ExposureF`
497 Exposure to subtract the background model
from.
499 Full focal plane camera-level background model.
503 calExp : `lsst.afw.image.exposure.ExposureF`
504 Background subtracted input exposure.
506 Detector level realization of the full focal plane bg model.
508 image = calExp.getMaskedImage()
509 with np.warnings.catch_warnings():
510 np.warnings.filterwarnings(
"ignore",
r"invalid value encountered")
511 calBkgElement = bgModel.toCcdBackground(calExp.getDetector(), image.getBBox())
512 image -= calBkgElement.getImage()
513 return calExp, calBkgElement
515 def _subtractSkyFrame(self, calExps, skyFrames, calBkgs):
516 """Determine the full focal plane sky frame scale factor relative to
517 an input list of calibrated exposures and subtract.
519 This method measures the sky frame scale on all inputs, resulting
in
520 values equal to the background method solveScales(). The sky frame
is
521 then subtracted
as in subtractSkyFrame() using the appropriate scale.
523 Input calExps
and calBkgs are updated
in-place, returning sky frame
524 subtracted calExps
and sky frame updated calBkgs, respectively.
528 calExps : `list` [`lsst.afw.image.exposure.ExposureF`]
529 Calibrated exposures to be background subtracted.
530 skyFrames : `list` [`lsst.afw.image.exposure.ExposureF`]
531 Sky frame calibration data
for the input detectors.
533 Background lists associated
with the input calibrated exposures.
535 skyFrameBgModels = []
537 for calExp, skyFrame
in zip(calExps, skyFrames):
538 skyFrameBgModel = self.sky.exposureToBackground(skyFrame)
539 skyFrameBgModels.append(skyFrameBgModel)
541 samples = self.sky.measureScale(calExp.getMaskedImage(), skyFrameBgModel)
542 scales.append(samples)
543 scale = self.sky.solveScales(scales)
544 for calExp, skyFrameBgModel, calBkg
in zip(calExps, skyFrameBgModels, calBkgs):
547 self.sky.subtractSkyFrame(calExp.getMaskedImage(), skyFrameBgModel, scale, calBkg)
548 self.log.info(
"Sky frame subtracted with a scale factor of %.5f", scale)
550 def _binAndMosaic(self, exposures, camera, binning, ids=None, refExps=None):
551 """Bin input exposures and mosaic across the entire focal plane.
553 Input exposures are binned and then mosaicked at the position of
554 the detector
in the focal plane of the camera.
559 Detector level list of either calexp `ExposureF` types
or
560 calexpBackground `BackgroundList` types.
562 Camera matching the input data to process.
564 Binning size to be applied to input images.
565 ids : `list` [`int`], optional
566 List of detector ids to iterate over.
567 refExps : `list` [`lsst.afw.image.exposure.ExposureF`], optional
568 If supplied, mask planes
from these reference images will be used.
571 mosaicImage : `lsst.afw.image.exposure.ExposureF`
572 Mosaicked full focal plane image.
574 refExps = np.resize(refExps, len(exposures))
576 for exp, refExp
in zip(exposures, refExps):
578 nativeImage = exp.getMaskedImage()
579 except AttributeError:
580 nativeImage = afwImage.makeMaskedImage(exp.getImage())
582 nativeImage.setMask(refExp.getMask())
583 binnedImage = afwMath.binImage(nativeImage, binning)
584 binnedImages.append(binnedImage)
586 mosConfig.binning = binning
588 imageStruct = mosTask.run(binnedImages, camera, inputIds=ids)
589 mosaicImage = imageStruct.outputData
def _restoreBackgroundRefineMask(self, calExps, calBkgs)
def _subtractVisitBackground(self, calExps, calBkgs, camera, config)
def _binAndMosaic(self, exposures, camera, binning, ids=None, refExps=None)
def _subtractDetectorBackground(self, calExp, bgModel)
def __init__(self, *args, **kwargs)
def runQuantum(self, butlerQC, inputRefs, outputRefs)
def _subtractSkyFrame(self, calExps, skyFrames, calBkgs)
def run(self, calExps, calBkgs, skyFrames, camera)