22__all__ = [
"SkyCorrectionTask",
"SkyCorrectionConfig"]
28import lsst.pipe.base.connectionTypes
as cT
30from lsst.pex.config import Config, ConfigField, ConfigurableField, Field
31from lsst.pipe.base
import PipelineTask, PipelineTaskConfig, PipelineTaskConnections, Struct
34 FocalPlaneBackgroundConfig,
42 """Lookup function to identify sky frames.
46 datasetType : `lsst.daf.butler.DatasetType`
48 registry : `lsst.daf.butler.Registry`
49 Butler registry to query.
50 quantumDataId : `lsst.daf.butler.DataCoordinate`
51 Data id to transform to find sky frames.
52 The ``detector`` entry will be stripped.
53 collections : `lsst.daf.butler.CollectionSearch`
54 Collections to search through.
58 results : `list` [`lsst.daf.butler.DatasetRef`]
59 List of datasets that will be used as sky calibration frames.
61 newDataId = quantumDataId.subset(registry.dimensions.conform([
"instrument",
"visit"]))
63 for dataId
in registry.queryDataIds([
"visit",
"detector"], dataId=newDataId).expanded():
64 skyFrame = registry.findDataset(
65 datasetType, dataId, collections=collections, timespan=dataId.timespan
67 skyFrames.append(skyFrame)
72 """Match the order of one list to another, padding if necessary.
77 List to be reordered and padded. Elements can be any type.
79 Iterable of values to be compared with outputKeys.
80 Length must match `inputList`.
82 Iterable of values to be compared with inputKeys.
84 Any value to be inserted where one of inputKeys is not in outputKeys.
89 Copy of inputList reordered per outputKeys and padded with `padWith`
90 so that the length matches length of outputKeys.
93 for outputKey
in outputKeys:
94 if outputKey
in inputKeys:
95 outputList.append(inputList[inputKeys.index(outputKey)])
97 outputList.append(padWith)
102 rawLinker = cT.Input(
103 doc=
"Raw data to provide exp-visit linkage to connect calExp inputs to camera/sky calibs.",
107 storageClass=
"Exposure",
108 dimensions=[
"instrument",
"exposure",
"detector"],
111 doc=
"Background-subtracted calibrated exposures.",
114 storageClass=
"ExposureF",
115 dimensions=[
"instrument",
"visit",
"detector"],
118 doc=
"Subtracted backgrounds for input calibrated exposures.",
120 name=
"calexpBackground",
121 storageClass=
"Background",
122 dimensions=[
"instrument",
"visit",
"detector"],
124 skyFrames = cT.PrerequisiteInput(
125 doc=
"Calibration sky frames.",
128 storageClass=
"ExposureF",
129 dimensions=[
"instrument",
"physical_filter",
"detector"],
131 lookupFunction=_skyFrameLookup,
133 camera = cT.PrerequisiteInput(
136 storageClass=
"Camera",
137 dimensions=[
"instrument"],
141 doc=
"Sky correction data, to be subtracted from the calibrated exposures.",
144 storageClass=
"Background",
145 dimensions=[
"instrument",
"visit",
"detector"],
147 calExpMosaic = cT.Output(
148 doc=
"Full focal plane mosaicked image of the sky corrected calibrated exposures.",
149 name=
"calexp_skyCorr_visit_mosaic",
150 storageClass=
"ImageF",
151 dimensions=[
"instrument",
"visit"],
153 calBkgMosaic = cT.Output(
154 doc=
"Full focal plane mosaicked image of the sky corrected calibrated exposure backgrounds.",
155 name=
"calexpBackground_skyCorr_visit_mosaic",
156 storageClass=
"ImageF",
157 dimensions=[
"instrument",
"visit"],
162 maskObjects = ConfigurableField(
163 target=MaskObjectsTask,
166 doMaskObjects = Field(
169 doc=
"Iteratively mask objects to find good sky?",
171 bgModel1 = ConfigField(
172 dtype=FocalPlaneBackgroundConfig,
173 doc=
"Initial background model, prior to sky frame subtraction",
175 sky = ConfigurableField(
176 target=SkyMeasurementTask,
177 doc=
"Sky measurement",
182 doc=
"Do sky frame subtraction?",
184 bgModel2 = ConfigField(
185 dtype=FocalPlaneBackgroundConfig,
186 doc=
"Final (cleanup) background model, after sky frame subtraction",
191 doc=
"Do final (cleanup) background model subtraction, after sky frame subtraction?",
196 doc=
"Binning factor for constructing full focal plane '*_camera' output datasets",
200 Config.setDefaults(self)
209 """Perform a full focal plane sky correction."""
211 ConfigClass = SkyCorrectionConfig
212 _DefaultName =
"skyCorr"
216 self.makeSubtask(
"sky")
217 self.makeSubtask(
"maskObjects")
222 detectorOrder = [ref.dataId[
"detector"]
for ref
in inputRefs.calExps]
225 inputRefs.calExps, [ref.dataId[
"detector"]
for ref
in inputRefs.calExps], detectorOrder
228 inputRefs.calBkgs, [ref.dataId[
"detector"]
for ref
in inputRefs.calBkgs], detectorOrder
231 inputRefs.skyFrames, [ref.dataId[
"detector"]
for ref
in inputRefs.skyFrames], detectorOrder
234 outputRefs.skyCorr, [ref.dataId[
"detector"]
for ref
in outputRefs.skyCorr], detectorOrder
236 inputs = butlerQC.get(inputRefs)
237 inputs.pop(
"rawLinker",
None)
238 outputs = self.
run(**inputs)
239 butlerQC.put(outputs, outputRefs)
241 def run(self, calExps, calBkgs, skyFrames, camera):
242 """Perform sky correction on a visit.
244 The original visit-level background is first restored to the calibrated
245 exposure and the existing background model is inverted in-place. If
246 doMaskObjects is True, the mask map associated with this exposure will
247 be iteratively updated (over nIter loops) by re-estimating the
248 background each iteration and redetecting footprints.
250 An initial full focal plane sky subtraction (bgModel1) will take place
251 prior to scaling and subtracting the sky frame.
253 If doSky is True, the sky frame will be scaled to the flux in the input
256 If doBgModel2 is True, a final full focal plane sky subtraction will
257 take place after the sky frame has been subtracted.
259 The first N elements of the returned skyCorr will consist of inverted
260 elements of the calexpBackground model (i.e., subtractive). All
261 subsequent elements appended to skyCorr thereafter will be additive
262 such that, when skyCorr is subtracted from a calexp, the net result
263 will be to undo the initial per-detector background solution and then
264 apply the skyCorr model thereafter. Adding skyCorr to a
265 calexpBackground will effectively negate the calexpBackground,
266 returning only the additive background components of the skyCorr
271 calExps : `list` [`lsst.afw.image.exposure.ExposureF`]
272 Detector calibrated exposure images for the visit.
273 calBkgs : `list` [`lsst.afw.math.BackgroundList`]
274 Detector background lists matching the calibrated exposures.
275 skyFrames : `list` [`lsst.afw.image.exposure.ExposureF`]
276 Sky frame calibration data for the input detectors.
277 camera : `lsst.afw.cameraGeom.Camera`
278 Camera matching the input data to process.
282 results : `Struct` containing:
283 skyCorr : `list` [`lsst.afw.math.BackgroundList`]
284 Detector-level sky correction background lists.
285 calExpMosaic : `lsst.afw.image.exposure.ExposureF`
286 Visit-level mosaic of the sky corrected data, binned.
287 Analogous to `calexp - skyCorr`.
288 calBkgMosaic : `lsst.afw.image.exposure.ExposureF`
289 Visit-level mosaic of the sky correction background, binned.
290 Analogous to `calexpBackground + skyCorr`.
293 numOrigBkgElements = [len(calBkg)
for calBkg
in calBkgs]
300 if self.config.doSky:
304 if self.config.doBgModel2:
308 calExpIds = [exp.getDetector().getId()
for exp
in calExps]
310 for calBkg, num
in zip(calBkgs, numOrigBkgElements):
311 skyCorrExtra = calBkg.clone()
312 skyCorrExtra._backgrounds = skyCorrExtra._backgrounds[num:]
313 skyCorrExtras.append(skyCorrExtra)
314 calExpMosaic = self.
_binAndMosaic(calExps, camera, self.config.binning, ids=calExpIds, refExps=
None)
316 skyCorrExtras, camera, self.config.binning, ids=calExpIds, refExps=calExps
319 return Struct(skyCorr=calBkgs, calExpMosaic=calExpMosaic, calBkgMosaic=calBkgMosaic)
322 """Restore original background to each calexp and invert the related
323 background model; optionally refine the mask plane.
325 The original visit-level background is restored to each calibrated
326 exposure and the existing background model is inverted in-place. If
327 doMaskObjects is True, the mask map associated with the exposure will
328 be iteratively updated (over nIter loops) by re-estimating the
329 background each iteration and redetecting footprints.
331 The background model modified in-place in this method will comprise the
332 first N elements of the skyCorr dataset type, i.e., these N elements
333 are the inverse of the calexpBackground model. All subsequent elements
334 appended to skyCorr will be additive such that, when skyCorr is
335 subtracted from a calexp, the net result will be to undo the initial
336 per-detector background solution and then apply the skyCorr model
337 thereafter. Adding skyCorr to a calexpBackground will effectively
338 negate the calexpBackground, returning only the additive background
339 components of the skyCorr background model.
343 calExps : `lsst.afw.image.exposure.ExposureF`
344 Detector level calexp images to process.
345 calBkgs : `lsst.afw.math._backgroundList.BackgroundList`
346 Detector level background lists associated with the calexps.
350 calExps : `lsst.afw.image.exposure.ExposureF`
351 The calexps with the initially subtracted background restored.
352 skyCorrBases : `lsst.afw.math._backgroundList.BackgroundList`
353 The inverted initial background models; the genesis for skyCorrs.
356 for calExp, calBkg
in zip(calExps, calBkgs):
357 image = calExp.getMaskedImage()
360 for calBkgElement
in calBkg:
361 statsImage = calBkgElement[0].getStatsImage()
363 skyCorrBase = calBkg.getImage()
367 if self.config.doMaskObjects:
368 self.maskObjects.findObjects(calExp)
370 stats = np.nanpercentile(skyCorrBase.array, [50, 75, 25])
372 "Detector %d: Initial background restored; BG median = %.1f counts, BG IQR = %.1f counts",
373 calExp.getDetector().getId(),
375 np.subtract(*stats[1:]),
377 skyCorrBases.append(skyCorrBase)
378 return calExps, skyCorrBases
381 """Perform a full focal-plane background subtraction for a visit.
383 Generate a full focal plane background model, binning all masked
384 detectors into bins of [bgModelN.xSize, bgModelN.ySize]. After,
385 subtract the resultant background model (translated back into CCD
386 coordinates) from the original detector exposure.
388 Return a list of background subtracted images and a list of full focal
389 plane background parameters.
393 calExps : `list` [`lsst.afw.image.exposure.ExposureF`]
394 Calibrated exposures to be background subtracted.
395 calBkgs : `list` [`lsst.afw.math._backgroundList.BackgroundList`]
396 Background lists associated with the input calibrated exposures.
397 camera : `lsst.afw.cameraGeom.Camera`
399 config : `lsst.pipe.tasks.background.FocalPlaneBackgroundConfig`
400 Configuration to use for background subtraction.
404 calExps : `list` [`lsst.afw.image.maskedImage.MaskedImageF`]
405 Background subtracted exposures for creating a focal plane image.
406 calBkgs : `list` [`lsst.afw.math._backgroundList.BackgroundList`]
407 Updated background lists with a visit-level model appended.
410 bgModelBase = FocalPlaneBackground.fromCamera(config, camera)
416 for calExp
in calExps:
417 bgModel = bgModelBase.clone()
418 bgModel.addCcd(calExp)
419 bgModels.append(bgModel)
422 for bgModel, calExp
in zip(bgModels, calExps):
424 "Detector %d: Merging %d unmasked pixels (%.1f%s of detector area) into focal plane "
429 calExp.getDetector().getId(),
430 bgModel._numbers.getArray().sum(),
431 100 * bgModel._numbers.getArray().sum() / calExp.getBBox().getArea(),
434 bgModelBase.merge(bgModel)
438 for calExp
in calExps:
440 calBkgElements.append(calBkgElement)
443 "Focal plane background model constructed using %.2f x %.2f mm (%d x %d pixel) superpixels; "
444 "FP BG median = %.1f counts, FP BG IQR = %.1f counts"
446 with warnings.catch_warnings():
447 warnings.filterwarnings(
"ignore",
r"invalid value encountered")
448 stats = np.nanpercentile(bgModelBase.getStatsImage().array, [50, 75, 25])
453 int(config.xSize / config.pixelSize),
454 int(config.ySize / config.pixelSize),
456 np.subtract(*stats[1:]),
459 for calBkg, calBkgElement
in zip(calBkgs, calBkgElements):
460 calBkg.append(calBkgElement[0])
461 return calExps, calBkgs
464 """Generate CCD background model and subtract from image.
466 Translate the full focal plane background into CCD coordinates and
467 subtract from the original science exposure image.
471 calExp : `lsst.afw.image.exposure.ExposureF`
472 Exposure to subtract the background model from.
473 bgModel : `lsst.pipe.tasks.background.FocalPlaneBackground`
474 Full focal plane camera-level background model.
478 calExp : `lsst.afw.image.exposure.ExposureF`
479 Background subtracted input exposure.
480 calBkgElement : `lsst.afw.math._backgroundList.BackgroundList`
481 Detector level realization of the full focal plane bg model.
483 image = calExp.getMaskedImage()
484 with warnings.catch_warnings():
485 warnings.filterwarnings(
"ignore",
r"invalid value encountered")
486 calBkgElement = bgModel.toCcdBackground(calExp.getDetector(), image.getBBox())
487 image -= calBkgElement.getImage()
488 return calExp, calBkgElement
491 """Determine the full focal plane sky frame scale factor relative to
492 an input list of calibrated exposures and subtract.
494 This method measures the sky frame scale on all inputs, resulting in
495 values equal to the background method solveScales(). The sky frame is
496 then subtracted as in subtractSkyFrame() using the appropriate scale.
498 Input calExps and calBkgs are updated in-place, returning sky frame
499 subtracted calExps and sky frame updated calBkgs, respectively.
503 calExps : `list` [`lsst.afw.image.exposure.ExposureF`]
504 Calibrated exposures to be background subtracted.
505 skyFrames : `list` [`lsst.afw.image.exposure.ExposureF`]
506 Sky frame calibration data for the input detectors.
507 calBkgs : `list` [`lsst.afw.math._backgroundList.BackgroundList`]
508 Background lists associated with the input calibrated exposures.
510 skyFrameBgModels = []
512 for calExp, skyFrame
in zip(calExps, skyFrames):
513 skyFrameBgModel = self.sky.exposureToBackground(skyFrame)
514 skyFrameBgModels.append(skyFrameBgModel)
516 samples = self.sky.measureScale(calExp.getMaskedImage(), skyFrameBgModel)
517 scales.append(samples)
518 scale = self.sky.solveScales(scales)
519 for calExp, skyFrameBgModel, calBkg
in zip(calExps, skyFrameBgModels, calBkgs):
522 self.sky.subtractSkyFrame(calExp.getMaskedImage(), skyFrameBgModel, scale, calBkg)
523 self.log.info(
"Sky frame subtracted with a scale factor of %.5f", scale)
525 def _binAndMosaic(self, exposures, camera, binning, ids=None, refExps=None):
526 """Bin input exposures and mosaic across the entire focal plane.
528 Input exposures are binned and then mosaicked at the position of
529 the detector in the focal plane of the camera.
534 Detector level list of either calexp `ExposureF` types or
535 calexpBackground `BackgroundList` types.
536 camera : `lsst.afw.cameraGeom.Camera`
537 Camera matching the input data to process.
539 Binning size to be applied to input images.
540 ids : `list` [`int`], optional
541 List of detector ids to iterate over.
542 refExps : `list` [`lsst.afw.image.exposure.ExposureF`], optional
543 If supplied, mask planes from these reference images will be used.
546 mosaicImage : `lsst.afw.image.exposure.ExposureF`
547 Mosaicked full focal plane image.
549 refExps = np.resize(refExps, len(exposures))
551 for exp, refExp
in zip(exposures, refExps):
553 nativeImage = exp.getMaskedImage()
554 except AttributeError:
555 nativeImage = afwImage.makeMaskedImage(exp.getImage())
557 nativeImage.setMask(refExp.getMask())
558 binnedImage = afwMath.binImage(nativeImage, binning)
559 binnedImages.append(binnedImage)
561 mosConfig.binning = binning
563 imageStruct = mosTask.run(binnedImages, camera, inputIds=ids)
564 mosaicImage = imageStruct.outputData
_subtractVisitBackground(self, calExps, calBkgs, camera, config)
run(self, calExps, calBkgs, skyFrames, camera)
_subtractSkyFrame(self, calExps, skyFrames, calBkgs)
_restoreBackgroundRefineMask(self, calExps, calBkgs)
runQuantum(self, butlerQC, inputRefs, outputRefs)
_subtractDetectorBackground(self, calExp, bgModel)
_binAndMosaic(self, exposures, camera, binning, ids=None, refExps=None)
__init__(self, *args, **kwargs)
_reorderAndPadList(inputList, inputKeys, outputKeys, padWith=None)
_skyFrameLookup(datasetType, registry, quantumDataId, collections)