22__all__ = [
"SkyCorrectionTask",
"SkyCorrectionConfig"]
28import lsst.pipe.base.connectionTypes
as cT
30from lsst.daf.butler
import DimensionGraph
31from lsst.pex.config import Config, ConfigField, ConfigurableField, Field, FieldValidationError
32from lsst.pipe.base import PipelineTask, PipelineTaskConfig, PipelineTaskConnections, Struct
35 FocalPlaneBackgroundConfig,
42def _skyFrameLookup(datasetType, registry, quantumDataId, collections):
43 """Lookup function to identify sky frames.
47 datasetType : `lsst.daf.butler.DatasetType`
49 registry : `lsst.daf.butler.Registry`
50 Butler registry to query.
51 quantumDataId : `lsst.daf.butler.DataCoordinate`
52 Data id to transform to find sky frames.
53 The ``detector`` entry will be stripped.
54 collections : `lsst.daf.butler.CollectionSearch`
55 Collections to search through.
59 results : `list` [`lsst.daf.butler.DatasetRef`]
60 List of datasets that will be used as sky calibration frames.
62 newDataId = quantumDataId.subset(DimensionGraph(registry.dimensions, names=["instrument",
"visit"]))
64 for dataId
in registry.queryDataIds([
"visit",
"detector"], dataId=newDataId).expanded():
65 skyFrame = registry.findDataset(
66 datasetType, dataId, collections=collections, timespan=dataId.timespan
68 skyFrames.append(skyFrame)
72def _reorderAndPadList(inputList, inputKeys, outputKeys, padWith=None):
73 """Match the order of one list to another, padding if necessary.
78 List to be reordered and padded. Elements can be any type.
80 Iterable of values to be compared
with outputKeys.
81 Length must match `inputList`.
83 Iterable of values to be compared
with inputKeys.
85 Any value to be inserted where one of inputKeys
is not in outputKeys.
90 Copy of inputList reordered per outputKeys
and padded
with `padWith`
91 so that the length matches length of outputKeys.
94 for outputKey
in outputKeys:
95 if outputKey
in inputKeys:
96 outputList.append(inputList[inputKeys.index(outputKey)])
98 outputList.append(padWith)
103 rawLinker = cT.Input(
104 doc=
"Raw data to provide exp-visit linkage to connect calExp inputs to camera/sky calibs.",
108 storageClass=
"Exposure",
109 dimensions=[
"instrument",
"exposure",
"detector"],
112 doc=
"Background-subtracted calibrated exposures.",
115 storageClass=
"ExposureF",
116 dimensions=[
"instrument",
"visit",
"detector"],
119 doc=
"Subtracted backgrounds for input calibrated exposures.",
121 name=
"calexpBackground",
122 storageClass=
"Background",
123 dimensions=[
"instrument",
"visit",
"detector"],
125 skyFrames = cT.PrerequisiteInput(
126 doc=
"Calibration sky frames.",
129 storageClass=
"ExposureF",
130 dimensions=[
"instrument",
"physical_filter",
"detector"],
132 lookupFunction=_skyFrameLookup,
134 camera = cT.PrerequisiteInput(
137 storageClass=
"Camera",
138 dimensions=[
"instrument"],
142 doc=
"Sky correction data, to be subtracted from the calibrated exposures.",
145 storageClass=
"Background",
146 dimensions=[
"instrument",
"visit",
"detector"],
148 calExpMosaic = cT.Output(
149 doc=
"Full focal plane mosaicked image of the sky corrected calibrated exposures.",
150 name=
"calexp_skyCorr_visit_mosaic",
151 storageClass=
"ImageF",
152 dimensions=[
"instrument",
"visit"],
154 calBkgMosaic = cT.Output(
155 doc=
"Full focal plane mosaicked image of the sky corrected calibrated exposure backgrounds.",
156 name=
"calexpBackground_skyCorr_visit_mosaic",
157 storageClass=
"ImageF",
158 dimensions=[
"instrument",
"visit"],
163 maskObjects = ConfigurableField(
164 target=MaskObjectsTask,
167 doMaskObjects = Field(
170 doc=
"Iteratively mask objects to find good sky?",
172 bgModel = ConfigField(
174 doc=
"Initial background model, prior to sky frame subtraction",
175 deprecated=
"This field is deprecated and will be removed after v26. Please use bgModel1 instead.",
180 doc=
"Do initial background model subtraction (prior to sky frame subtraction)?",
182 deprecated=
"This field is deprecated and will be removed after v26. See RFC-898 for further details.",
184 bgModel1 = ConfigField(
185 dtype=FocalPlaneBackgroundConfig,
186 doc=
"Initial background model, prior to sky frame subtraction",
191 doc=
"Do initial background model subtraction (prior to sky frame subtraction)?",
192 deprecated=
"This field is deprecated and will be removed after v26. See RFC-898 for further details.",
194 sky = ConfigurableField(
195 target=SkyMeasurementTask,
196 doc=
"Sky measurement",
201 doc=
"Do sky frame subtraction?",
203 bgModel2 = ConfigField(
204 dtype=FocalPlaneBackgroundConfig,
205 doc=
"Final (cleanup) background model, after sky frame subtraction",
210 doc=
"Do final (cleanup) background model subtraction, after sky frame subtraction?",
215 doc=
"Binning factor for constructing full focal plane '*_camera' output datasets",
219 Config.setDefaults(self)
230 msg =
"The doBgModel field will be removed after v26."
231 raise FieldValidationError(self.__class__.doBgModel, self, msg)
235 """Perform a full focal plane sky correction."""
237 ConfigClass = SkyCorrectionConfig
238 _DefaultName =
"skyCorr"
242 self.makeSubtask(
"sky")
243 self.makeSubtask(
"maskObjects")
248 detectorOrder = [ref.dataId[
"detector"]
for ref
in inputRefs.calExps]
250 inputRefs.calExps = _reorderAndPadList(
251 inputRefs.calExps, [ref.dataId[
"detector"]
for ref
in inputRefs.calExps], detectorOrder
253 inputRefs.calBkgs = _reorderAndPadList(
254 inputRefs.calBkgs, [ref.dataId[
"detector"]
for ref
in inputRefs.calBkgs], detectorOrder
256 inputRefs.skyFrames = _reorderAndPadList(
257 inputRefs.skyFrames, [ref.dataId[
"detector"]
for ref
in inputRefs.skyFrames], detectorOrder
259 outputRefs.skyCorr = _reorderAndPadList(
260 outputRefs.skyCorr, [ref.dataId[
"detector"]
for ref
in outputRefs.skyCorr], detectorOrder
262 inputs = butlerQC.get(inputRefs)
263 inputs.pop(
"rawLinker",
None)
264 outputs = self.
run(**inputs)
265 butlerQC.put(outputs, outputRefs)
267 def run(self, calExps, calBkgs, skyFrames, camera):
268 """Perform sky correction on a visit.
270 The original visit-level background is first restored to the calibrated
271 exposure
and the existing background model
is inverted
in-place. If
272 doMaskObjects
is True, the mask map associated
with this exposure will
273 be iteratively updated (over nIter loops) by re-estimating the
274 background each iteration
and redetecting footprints.
276 If doBgModel1
is True, an initial full focal plane sky subtraction will
277 take place prior to scaling
and subtracting the sky frame.
279 If doSky
is True, the sky frame will be scaled to the flux
in the input
282 If doBgModel2
is True, a final full focal plane sky subtraction will
283 take place after the sky frame has been subtracted.
285 The first N elements of the returned skyCorr will consist of inverted
286 elements of the calexpBackground model (i.e., subtractive). All
287 subsequent elements appended to skyCorr thereafter will be additive
288 such that, when skyCorr
is subtracted
from a calexp, the net result
289 will be to undo the initial per-detector background solution
and then
290 apply the skyCorr model thereafter. Adding skyCorr to a
291 calexpBackground will effectively negate the calexpBackground,
292 returning only the additive background components of the skyCorr
297 calExps : `list` [`lsst.afw.image.exposure.ExposureF`]
298 Detector calibrated exposure images
for the visit.
299 calBkgs : `list` [`lsst.afw.math.BackgroundList`]
300 Detector background lists matching the calibrated exposures.
301 skyFrames : `list` [`lsst.afw.image.exposure.ExposureF`]
302 Sky frame calibration data
for the input detectors.
304 Camera matching the input data to process.
308 results : `Struct` containing:
309 skyCorr : `list` [`lsst.afw.math.BackgroundList`]
310 Detector-level sky correction background lists.
311 calExpMosaic : `lsst.afw.image.exposure.ExposureF`
312 Visit-level mosaic of the sky corrected data, binned.
313 Analogous to `calexp - skyCorr`.
314 calBkgMosaic : `lsst.afw.image.exposure.ExposureF`
315 Visit-level mosaic of the sky correction background, binned.
316 Analogous to `calexpBackground + skyCorr`.
319 numOrigBkgElements = [len(calBkg)
for calBkg
in calBkgs]
323 if self.config.doBgModel1:
327 if self.config.doSky:
331 if self.config.doBgModel2:
335 calExpIds = [exp.getDetector().getId()
for exp
in calExps]
337 for calBkg, num
in zip(calBkgs, numOrigBkgElements):
338 skyCorrExtra = calBkg.clone()
339 skyCorrExtra._backgrounds = skyCorrExtra._backgrounds[num:]
340 skyCorrExtras.append(skyCorrExtra)
341 calExpMosaic = self.
_binAndMosaic(calExps, camera, self.config.binning, ids=calExpIds, refExps=
None)
343 skyCorrExtras, camera, self.config.binning, ids=calExpIds, refExps=calExps
346 return Struct(skyCorr=calBkgs, calExpMosaic=calExpMosaic, calBkgMosaic=calBkgMosaic)
348 def _restoreBackgroundRefineMask(self, calExps, calBkgs):
349 """Restore original background to each calexp and invert the related
350 background model; optionally refine the mask plane.
352 The original visit-level background is restored to each calibrated
353 exposure
and the existing background model
is inverted
in-place. If
354 doMaskObjects
is True, the mask map associated
with the exposure will
355 be iteratively updated (over nIter loops) by re-estimating the
356 background each iteration
and redetecting footprints.
358 The background model modified
in-place
in this method will comprise the
359 first N elements of the skyCorr dataset type, i.e., these N elements
360 are the inverse of the calexpBackground model. All subsequent elements
361 appended to skyCorr will be additive such that, when skyCorr
is
362 subtracted
from a calexp, the net result will be to undo the initial
363 per-detector background solution
and then apply the skyCorr model
364 thereafter. Adding skyCorr to a calexpBackground will effectively
365 negate the calexpBackground, returning only the additive background
366 components of the skyCorr background model.
370 calExps : `lsst.afw.image.exposure.ExposureF`
371 Detector level calexp images to process.
373 Detector level background lists associated
with the calexps.
377 calExps : `lsst.afw.image.exposure.ExposureF`
378 The calexps
with the initially subtracted background restored.
380 The inverted initial background models; the genesis
for skyCorrs.
383 for calExp, calBkg
in zip(calExps, calBkgs):
384 image = calExp.getMaskedImage()
387 for calBkgElement
in calBkg:
388 statsImage = calBkgElement[0].getStatsImage()
390 skyCorrBase = calBkg.getImage()
394 if self.config.doMaskObjects:
395 self.maskObjects.findObjects(calExp)
397 stats = np.nanpercentile(skyCorrBase.array, [50, 75, 25])
399 "Detector %d: Initial background restored; BG median = %.1f counts, BG IQR = %.1f counts",
400 calExp.getDetector().getId(),
402 np.subtract(*stats[1:]),
404 skyCorrBases.append(skyCorrBase)
405 return calExps, skyCorrBases
407 def _subtractVisitBackground(self, calExps, calBkgs, camera, config):
408 """Perform a full focal-plane background subtraction for a visit.
410 Generate a full focal plane background model, binning all masked
411 detectors into bins of [bgModelN.xSize, bgModelN.ySize]. After,
412 subtract the resultant background model (translated back into CCD
413 coordinates) from the original detector exposure.
415 Return a list of background subtracted images
and a list of full focal
416 plane background parameters.
420 calExps : `list` [`lsst.afw.image.exposure.ExposureF`]
421 Calibrated exposures to be background subtracted.
423 Background lists associated
with the input calibrated exposures.
427 Configuration to use
for background subtraction.
431 calExps : `list` [`lsst.afw.image.maskedImage.MaskedImageF`]
432 Background subtracted exposures
for creating a focal plane image.
434 Updated background lists
with a visit-level model appended.
437 bgModelBase = FocalPlaneBackground.fromCamera(config, camera)
443 for calExp
in calExps:
444 bgModel = bgModelBase.clone()
445 bgModel.addCcd(calExp)
446 bgModels.append(bgModel)
449 for bgModel, calExp
in zip(bgModels, calExps):
451 "Detector %d: Merging %d unmasked pixels (%.1f%s of detector area) into focal plane "
456 calExp.getDetector().getId(),
457 bgModel._numbers.getArray().sum(),
458 100 * bgModel._numbers.getArray().sum() / calExp.getBBox().getArea(),
461 bgModelBase.merge(bgModel)
465 for calExp
in calExps:
467 calBkgElements.append(calBkgElement)
470 "Focal plane background model constructed using %.2f x %.2f mm (%d x %d pixel) superpixels; "
471 "FP BG median = %.1f counts, FP BG IQR = %.1f counts"
473 with warnings.catch_warnings():
474 warnings.filterwarnings(
"ignore",
r"invalid value encountered")
475 stats = np.nanpercentile(bgModelBase.getStatsImage().array, [50, 75, 25])
480 int(config.xSize / config.pixelSize),
481 int(config.ySize / config.pixelSize),
483 np.subtract(*stats[1:]),
486 for calBkg, calBkgElement
in zip(calBkgs, calBkgElements):
487 calBkg.append(calBkgElement[0])
488 return calExps, calBkgs
490 def _subtractDetectorBackground(self, calExp, bgModel):
491 """Generate CCD background model and subtract from image.
493 Translate the full focal plane background into CCD coordinates and
494 subtract
from the original science exposure image.
498 calExp : `lsst.afw.image.exposure.ExposureF`
499 Exposure to subtract the background model
from.
501 Full focal plane camera-level background model.
505 calExp : `lsst.afw.image.exposure.ExposureF`
506 Background subtracted input exposure.
508 Detector level realization of the full focal plane bg model.
510 image = calExp.getMaskedImage()
511 with warnings.catch_warnings():
512 warnings.filterwarnings(
"ignore",
r"invalid value encountered")
513 calBkgElement = bgModel.toCcdBackground(calExp.getDetector(), image.getBBox())
514 image -= calBkgElement.getImage()
515 return calExp, calBkgElement
517 def _subtractSkyFrame(self, calExps, skyFrames, calBkgs):
518 """Determine the full focal plane sky frame scale factor relative to
519 an input list of calibrated exposures and subtract.
521 This method measures the sky frame scale on all inputs, resulting
in
522 values equal to the background method solveScales(). The sky frame
is
523 then subtracted
as in subtractSkyFrame() using the appropriate scale.
525 Input calExps
and calBkgs are updated
in-place, returning sky frame
526 subtracted calExps
and sky frame updated calBkgs, respectively.
530 calExps : `list` [`lsst.afw.image.exposure.ExposureF`]
531 Calibrated exposures to be background subtracted.
532 skyFrames : `list` [`lsst.afw.image.exposure.ExposureF`]
533 Sky frame calibration data
for the input detectors.
535 Background lists associated
with the input calibrated exposures.
537 skyFrameBgModels = []
539 for calExp, skyFrame
in zip(calExps, skyFrames):
540 skyFrameBgModel = self.sky.exposureToBackground(skyFrame)
541 skyFrameBgModels.append(skyFrameBgModel)
543 samples = self.sky.measureScale(calExp.getMaskedImage(), skyFrameBgModel)
544 scales.append(samples)
545 scale = self.sky.solveScales(scales)
546 for calExp, skyFrameBgModel, calBkg
in zip(calExps, skyFrameBgModels, calBkgs):
549 self.sky.subtractSkyFrame(calExp.getMaskedImage(), skyFrameBgModel, scale, calBkg)
550 self.log.info(
"Sky frame subtracted with a scale factor of %.5f", scale)
552 def _binAndMosaic(self, exposures, camera, binning, ids=None, refExps=None):
553 """Bin input exposures and mosaic across the entire focal plane.
555 Input exposures are binned and then mosaicked at the position of
556 the detector
in the focal plane of the camera.
561 Detector level list of either calexp `ExposureF` types
or
562 calexpBackground `BackgroundList` types.
564 Camera matching the input data to process.
566 Binning size to be applied to input images.
567 ids : `list` [`int`], optional
568 List of detector ids to iterate over.
569 refExps : `list` [`lsst.afw.image.exposure.ExposureF`], optional
570 If supplied, mask planes
from these reference images will be used.
573 mosaicImage : `lsst.afw.image.exposure.ExposureF`
574 Mosaicked full focal plane image.
576 refExps = np.resize(refExps, len(exposures))
578 for exp, refExp
in zip(exposures, refExps):
580 nativeImage = exp.getMaskedImage()
581 except AttributeError:
582 nativeImage = afwImage.makeMaskedImage(exp.getImage())
584 nativeImage.setMask(refExp.getMask())
585 binnedImage = afwMath.binImage(nativeImage, binning)
586 binnedImages.append(binnedImage)
588 mosConfig.binning = binning
590 imageStruct = mosTask.run(binnedImages, camera, inputIds=ids)
591 mosaicImage = imageStruct.outputData
def _restoreBackgroundRefineMask(self, calExps, calBkgs)
def _subtractVisitBackground(self, calExps, calBkgs, camera, config)
def _binAndMosaic(self, exposures, camera, binning, ids=None, refExps=None)
def _subtractDetectorBackground(self, calExp, bgModel)
def __init__(self, *args, **kwargs)
def runQuantum(self, butlerQC, inputRefs, outputRefs)
def _subtractSkyFrame(self, calExps, skyFrames, calBkgs)
def run(self, calExps, calBkgs, skyFrames, camera)