26 from lsst.pex.config import Config, Field, ConfigurableField, ConfigField
30 FocalPlaneBackgroundConfig, MaskObjectsTask)
34 __all__ = [
"SkyCorrectionConfig",
"SkyCorrectionTask"]
40 """Make and write an image of an entire focal plane
44 camera : `lsst.afw.cameraGeom.Camera`
46 exposures : `list` of `tuple` of `int` and `lsst.afw.image.Exposure`
47 List of detector ID and CCD exposures (binned by `binning`).
48 filename : `str`, optional
51 Binning size that has been applied to images.
53 image = visualizeVisit.makeCameraImage(camera, dict(exp
for exp
in exposures
if exp
is not None), binning)
54 if filename
is not None:
55 image.writeFits(filename)
61 doc=
"Raw data to provide exp-visit linkage to connect calExp inputs to camera/sky calibs.",
65 storageClass=
"ExposureU",
66 dimensions=[
"instrument",
"exposure",
"detector"],
68 calExpArray = cT.Input(
69 doc=
"Input exposures to process",
72 storageClass=
"ExposureF",
73 dimensions=[
"instrument",
"visit",
"detector"],
75 calBkgArray = cT.Input(
76 doc=
"Input background files to use",
78 name=
"calexpBackground",
79 storageClass=
"Background",
80 dimensions=[
"instrument",
"visit",
"detector"],
82 camera = cT.PrerequisiteInput(
83 doc=
"Input camera to use.",
85 storageClass=
"Camera",
86 dimensions=[
"instrument",
"calibration_label"],
88 skyCalibs = cT.PrerequisiteInput(
89 doc=
"Input sky calibrations to use.",
92 storageClass=
"ExposureF",
93 dimensions=[
"instrument",
"physical_filter",
"detector",
"calibration_label"],
95 calExpCamera = cT.Output(
96 doc=
"Output camera image.",
98 storageClass=
"ImageF",
99 dimensions=[
"instrument",
"visit"],
102 doc=
"Output sky corrected images.",
105 storageClass=
"Background",
106 dimensions=[
"instrument",
"visit",
"detector"],
111 """Configuration for SkyCorrectionTask"""
112 bgModel = ConfigField(dtype=FocalPlaneBackgroundConfig, doc=
"Background model")
113 bgModel2 = ConfigField(dtype=FocalPlaneBackgroundConfig, doc=
"2nd Background model")
114 sky = ConfigurableField(target=SkyMeasurementTask, doc=
"Sky measurement")
115 maskObjects = ConfigurableField(target=MaskObjectsTask, doc=
"Mask Objects")
116 doMaskObjects = Field(dtype=bool, default=
True, doc=
"Mask objects to find good sky?")
117 doBgModel = Field(dtype=bool, default=
True, doc=
"Do background model subtraction?")
118 doBgModel2 = Field(dtype=bool, default=
True, doc=
"Do cleanup background model subtraction?")
119 doSky = Field(dtype=bool, default=
True, doc=
"Do sky frame subtraction?")
120 binning = Field(dtype=int, default=8, doc=
"Binning factor for constructing focal-plane images")
121 calexpType = Field(dtype=str, default=
"calexp",
122 doc=
"Should be set to fakes_calexp if you want to process calexps with fakes in.")
125 Config.setDefaults(self)
126 self.
bgModel2bgModel2.doSmooth =
True
130 self.
bgModel2bgModel2.smoothScale = 1.0
134 """Correct sky over entire focal plane"""
135 ConfigClass = SkyCorrectionConfig
136 _DefaultName =
"skyCorr"
139 inputs = butlerQC.get(inputRefs)
140 inputs.pop(
"rawLinker",
None)
141 outputs = self.
runrun(**inputs)
142 butlerQC.put(outputs, outputRefs)
147 self.makeSubtask(
"sky")
148 self.makeSubtask(
"maskObjects")
151 def _makeArgumentParser(cls, *args, **kwargs):
152 kwargs.pop(
"doBatch",
False)
153 datasetType = ConfigDatasetType(name=
"calexpType")
154 parser = ArgumentParser(name=
"skyCorr", *args, **kwargs)
155 parser.add_id_argument(
"--id", datasetType=datasetType, level=
"visit",
156 help=
"data ID, e.g. --id visit=12345")
161 """Return walltime request for batch job
163 Subclasses should override if the walltime should be calculated
164 differently (e.g., addition of some serial time).
169 Requested time per iteration.
170 parsedCmd : `argparse.Namespace`
171 Results of argument parsing.
175 numTargets = len(cls.RunnerClass.getTargetList(parsedCmd))
176 return time*numTargets
179 """Perform sky correction on an exposure
181 We restore the original sky, and remove it again using multiple
182 algorithms. We optionally apply:
184 1. A large-scale background model.
185 This step removes very-large-scale sky such as moonlight.
187 3. A medium-scale background model.
188 This step removes residual sky (This is smooth on the focal plane).
190 Only the master node executes this method. The data is held on
191 the slave nodes, which do all the hard work.
195 expRef : `lsst.daf.persistence.ButlerDataRef`
196 Data reference for exposure.
200 ~lsst.pipe.drivers.SkyCorrectionTask.run
203 extension =
"-%(visit)d.fits" % expRef.dataId
208 pool.storeSet(butler=expRef.getButler())
209 camera = expRef.get(
"camera")
211 dataIdList = [ccdRef.dataId
for ccdRef
in expRef.subItems(
"ccd")
if
212 ccdRef.datasetExists(self.config.calexpType)]
214 exposures = pool.map(self.
loadImageloadImage, dataIdList)
217 exposures = pool.mapToPrevious(self.
collectOriginalcollectOriginal, dataIdList)
219 exposures = pool.mapToPrevious(self.
collectMaskcollectMask, dataIdList)
222 if self.config.doBgModel:
223 exposures = self.
focalPlaneBackgroundfocalPlaneBackground(camera, pool, dataIdList, self.config.bgModel)
225 if self.config.doSky:
226 measScales = pool.mapToPrevious(self.
measureSkyFramemeasureSkyFrame, dataIdList)
227 scale = self.sky.solveScales(measScales)
228 self.log.info(
"Sky frame scale: %s" % (scale,))
230 exposures = pool.mapToPrevious(self.
subtractSkyFramesubtractSkyFrame, dataIdList, scale)
233 calibs = pool.mapToPrevious(self.
collectSkycollectSky, dataIdList)
236 if self.config.doBgModel2:
237 exposures = self.
focalPlaneBackgroundfocalPlaneBackground(camera, pool, dataIdList, self.config.bgModel2)
241 expRef.put(image,
"calexp_camera")
243 pool.mapToPrevious(self.
writewrite, dataIdList)
246 """Perform full focal-plane background subtraction
248 This method runs on the master node.
252 camera : `lsst.afw.cameraGeom.Camera`
254 pool : `lsst.ctrl.pool.Pool`
256 dataIdList : iterable of `dict`
257 List of data identifiers for the CCDs.
258 config : `lsst.pipe.drivers.background.FocalPlaneBackgroundConfig`
259 Configuration to use for background subtraction.
263 exposures : `list` of `lsst.afw.image.Image`
264 List of binned images, for creating focal plane image.
266 bgModel = FocalPlaneBackground.fromCamera(config, camera)
267 data = [pipeBase.Struct(dataId=dataId, bgModel=bgModel.clone())
for dataId
in dataIdList]
268 bgModelList = pool.mapToPrevious(self.
accumulateModelaccumulateModel, data)
269 for ii, bg
in enumerate(bgModelList):
270 self.log.info(
"Background %d: %d pixels", ii, bg._numbers.array.sum())
272 return pool.mapToPrevious(self.
subtractModelsubtractModel, dataIdList, bgModel)
275 """Perform full focal-plane background subtraction
277 This method runs on the master node.
281 camera : `lsst.afw.cameraGeom.Camera`
283 cacheExposures : `list` of `lsst.afw.image.Exposures`
284 List of loaded and processed input calExp.
285 idList : `list` of `int`
286 List of detector ids to iterate over.
287 config : `lsst.pipe.drivers.background.FocalPlaneBackgroundConfig`
288 Configuration to use for background subtraction.
292 exposures : `list` of `lsst.afw.image.Image`
293 List of binned images, for creating focal plane image.
294 newCacheBgList : `list` of `lsst.afwMath.backgroundList`
295 Background lists generated.
296 cacheBgModel : `FocalPlaneBackground`
297 Full focal plane background model.
299 bgModel = FocalPlaneBackground.fromCamera(config, camera)
300 data = [pipeBase.Struct(id=id, bgModel=bgModel.clone())
for id
in idList]
303 for nodeData, cacheExp
in zip(data, cacheExposures):
304 nodeData.bgModel.addCcd(cacheExp)
305 bgModelList.append(nodeData.bgModel)
307 for ii, bg
in enumerate(bgModelList):
308 self.log.info(
"Background %d: %d pixels", ii, bg._numbers.getArray().sum())
314 for cacheExp
in cacheExposures:
315 nodeExp, nodeBgModel, nodeBgList = self.
subtractModelRunsubtractModelRun(cacheExp, bgModel)
316 exposures.append(afwMath.binImage(nodeExp.getMaskedImage(), self.config.binning))
317 cacheBgModel.append(nodeBgModel)
318 newCacheBgList.append(nodeBgList)
320 return exposures, newCacheBgList, cacheBgModel
322 def run(self, calExpArray, calBkgArray, skyCalibs, camera):
323 """Duplicate runDataRef method without ctrl_pool for Gen3.
327 calExpArray : `list` of `lsst.afw.image.Exposure`
328 Array of detector input calExp images for the exposure to
330 calBkgArray : `list` of `lsst.afw.math.BackgroundList`
331 Array of detector input background lists matching the
333 skyCalibs : `list` of `lsst.afw.image.Exposure`
334 Array of SKY calibrations for the input detectors to be
336 camera : `lsst.afw.cameraGeom.Camera`
337 Camera matching the input data to process.
341 results : `pipeBase.Struct` containing
342 calExpCamera : `lsst.afw.image.Exposure`
343 Full camera image of the sky-corrected data.
344 skyCorr : `list` of `lsst.afw.math.BackgroundList`
345 Detector-level sky-corrected background lists.
349 ~lsst.pipe.drivers.SkyCorrectionTask.runDataRef()
366 idList = [exp.getDetector().getId()
for exp
in calExpArray]
373 for calExp, calBgModel
in zip(calExpArray, calBkgArray):
374 nodeExp, nodeBgList = self.
loadImageRunloadImageRun(calExp, calBgModel)
375 cacheExposures.append(nodeExp)
376 cacheBgList.append(nodeBgList)
377 exposures.append(afwMath.binImage(nodeExp.getMaskedImage(), self.config.binning))
379 if self.config.doBgModel:
382 camera, cacheExposures, idList, self.config.bgModel
384 for cacheBg, newBg
in zip(cacheBgList, newCacheBgList):
385 cacheBg.append(newBg)
387 if self.config.doSky:
393 for cacheExp, skyCalib
in zip(cacheExposures, skyCalibs):
394 skyExp = self.sky.exposureToBackground(skyCalib)
395 cacheSky.append(skyExp)
396 scale = self.sky.measureScale(cacheExp.getMaskedImage(), skyExp)
397 measScales.append(scale)
399 scale = self.sky.solveScales(measScales)
400 self.log.info(
"Sky frame scale: %s" % (scale, ))
406 for cacheExp, nodeSky, nodeBgList
in zip(cacheExposures, cacheSky, cacheBgList):
407 self.sky.
subtractSkyFrame(cacheExp.getMaskedImage(), nodeSky, scale, nodeBgList)
408 exposures.append(afwMath.binImage(cacheExp.getMaskedImage(), self.config.binning))
410 if self.config.doBgModel2:
414 camera, cacheExposures, idList, self.config.bgModel2
416 for cacheBg, newBg
in zip(cacheBgList, newBgList):
417 cacheBg.append(newBg)
423 return pipeBase.Struct(
429 """Load original image and restore the sky
431 This method runs on the slave nodes.
435 cache : `lsst.pipe.base.Struct`
442 exposure : `lsst.afw.image.Exposure`
445 cache.dataId = dataId
446 cache.exposure = cache.butler.get(self.config.calexpType, dataId, immediate=
True).clone()
447 bgOld = cache.butler.get(
"calexpBackground", dataId, immediate=
True)
448 image = cache.exposure.getMaskedImage()
452 statsImage = bgData[0].getStatsImage()
455 image -= bgOld.getImage()
456 cache.bgList = afwMath.BackgroundList()
458 cache.bgList.append(bgData)
460 if self.config.doMaskObjects:
461 self.maskObjects.findObjects(cache.exposure)
463 return self.
collectcollect(cache)
466 """Serial implementation of self.loadImage() for Gen3.
468 Load and restore background to calExp and calExpBkg.
472 calExp : `lsst.afw.image.Exposure`
473 Detector level calExp image to process.
474 calExpBkg : `lsst.afw.math.BackgroundList`
475 Detector level background list associated with the calExp.
479 calExp : `lsst.afw.image.Exposure`
480 Background restored calExp.
481 bgList : `lsst.afw.math.BackgroundList`
482 New background list containing the restoration background.
484 image = calExp.getMaskedImage()
486 for bgOld
in calExpBkg:
487 statsImage = bgOld[0].getStatsImage()
490 image -= calExpBkg.getImage()
491 bgList = afwMath.BackgroundList()
492 for bgData
in calExpBkg:
493 bgList.append(bgData)
495 if self.config.doMaskObjects:
496 self.maskObjects.findObjects(calExp)
498 return (calExp, bgList)
501 """Measure scale for sky frame
503 This method runs on the slave nodes.
507 cache : `lsst.pipe.base.Struct`
517 assert cache.dataId == dataId
518 cache.sky = self.sky.getSkyData(cache.butler, dataId)
519 scale = self.sky.measureScale(cache.exposure.getMaskedImage(), cache.sky)
523 """Subtract sky frame
525 This method runs on the slave nodes.
529 cache : `lsst.pipe.base.Struct`
538 exposure : `lsst.afw.image.Exposure`
541 assert cache.dataId == dataId
542 self.sky.
subtractSkyFrame(cache.exposure.getMaskedImage(), cache.sky, scale, cache.bgList)
543 return self.
collectcollect(cache)
546 """Fit background model for CCD
548 This method runs on the slave nodes.
552 cache : `lsst.pipe.base.Struct`
554 data : `lsst.pipe.base.Struct`
555 Data identifier, with `dataId` (data identifier) and `bgModel`
556 (background model) elements.
560 bgModel : `lsst.pipe.drivers.background.FocalPlaneBackground`
563 assert cache.dataId == data.dataId
564 data.bgModel.addCcd(cache.exposure)
568 """Subtract background model
570 This method runs on the slave nodes.
574 cache : `lsst.pipe.base.Struct`
578 bgModel : `lsst.pipe.drivers.background.FocalPlaneBackround`
583 exposure : `lsst.afw.image.Exposure`
586 assert cache.dataId == dataId
587 exposure = cache.exposure
588 image = exposure.getMaskedImage()
589 detector = exposure.getDetector()
590 bbox = image.getBBox()
592 cache.bgModel = bgModel.toCcdBackground(detector, bbox)
593 image -= cache.bgModel.getImage()
595 self.log.error(f
"There was an error processing {dataId}, no calib file produced")
597 cache.bgList.append(cache.bgModel[0])
598 return self.
collectcollect(cache)
601 """Serial implementation of self.subtractModel() for Gen3.
603 Load and restore background to calExp and calExpBkg.
607 exposure : `lsst.afw.image.Exposure`
608 Exposure to subtract the background model from.
609 bgModel : `lsst.pipe.drivers.background.FocalPlaneBackground`
610 Full camera level background model.
614 exposure : `lsst.afw.image.Exposure`
615 Background subtracted input exposure.
616 bgModelCcd : `lsst.afw.math.BackgroundList`
617 Detector level realization of the full background model.
618 bgModelMaskedImage : `lsst.afw.image.MaskedImage`
619 Background model from the bgModelCcd realization.
621 image = exposure.getMaskedImage()
622 detector = exposure.getDetector()
623 bbox = image.getBBox()
624 bgModelCcd = bgModel.toCcdBackground(detector, bbox)
625 image -= bgModelCcd.getImage()
627 return (exposure, bgModelCcd, bgModelCcd[0])
630 """Generate an image of the background model for visualisation
632 Useful for debugging.
636 cache : `lsst.pipe.base.Struct`
640 bgModel : `lsst.pipe.drivers.background.FocalPlaneBackround`
647 image : `lsst.afw.image.MaskedImage`
648 Binned background model image.
650 assert cache.dataId == dataId
651 exposure = cache.exposure
652 detector = exposure.getDetector()
653 bbox = exposure.getMaskedImage().getBBox()
654 image = bgModel.toCcdBackground(detector, bbox).getImage()
658 """Return the binned image required for visualization
660 This method just helps to cut down on boilerplate.
664 image : `lsst.afw.image.MaskedImage`
665 Image to go into visualisation.
671 image : `lsst.afw.image.MaskedImage`
674 return (exposure.getDetector().getId(), afwMath.binImage(image, self.config.binning))
677 """Collect exposure for potential visualisation
679 This method runs on the slave nodes.
683 cache : `lsst.pipe.base.Struct`
690 image : `lsst.afw.image.MaskedImage`
693 return self.
collectBinnedImagecollectBinnedImage(cache.exposure, cache.exposure.maskedImage)
696 """Collect original image for visualisation
698 This method runs on the slave nodes.
702 cache : `lsst.pipe.base.Struct`
711 image : `lsst.afw.image.MaskedImage`
714 exposure = cache.butler.get(
"calexp", dataId, immediate=
True)
718 """Collect original image for visualisation
720 This method runs on the slave nodes.
724 cache : `lsst.pipe.base.Struct`
733 image : `lsst.afw.image.MaskedImage`
736 return self.
collectBinnedImagecollectBinnedImage(cache.exposure, cache.sky.getImage())
739 """Collect mask for visualisation
741 This method runs on the slave nodes.
745 cache : `lsst.pipe.base.Struct`
754 image : `lsst.afw.image.Image`
758 image = afwImage.ImageF(cache.exposure.maskedImage.getBBox())
759 image.array[:] = cache.exposure.maskedImage.mask.array
763 """Write resultant background list
765 This method runs on the slave nodes.
769 cache : `lsst.pipe.base.Struct`
774 cache.butler.put(cache.bgList,
"skyCorr", dataId)
776 def _getMetadataName(self):
777 """There's no metadata to write out"""
def logOperation(self, operation, catch=False, trace=True)
def run(self, calExpArray, calBkgArray, skyCalibs, camera)
def subtractModel(self, cache, dataId, bgModel)
def focalPlaneBackground(self, camera, pool, dataIdList, config)
def subtractModelRun(self, exposure, bgModel)
def __init__(self, *args, **kwargs)
def accumulateModel(self, cache, data)
def subtractSkyFrame(self, cache, dataId, scale)
def measureSkyFrame(self, cache, dataId)
def collectBinnedImage(self, exposure, image)
def focalPlaneBackgroundRun(self, camera, cacheExposures, idList, config)
def loadImage(self, cache, dataId)
def collectMask(self, cache, dataId)
def runQuantum(self, butlerQC, inputRefs, outputRefs)
def batchWallTime(cls, time, parsedCmd, numCores)
def collectOriginal(self, cache, dataId)
def runDataRef(self, expRef)
def realiseModel(self, cache, dataId, bgModel)
def write(self, cache, dataId)
def collectSky(self, cache, dataId)
def loadImageRun(self, calExp, calExpBkg)
def makeCameraImage(camera, exposures, filename=None, binning=8)