21 from __future__
import absolute_import, division, print_function
27 from lsst.pex.config import Config, Field, ConfigurableField, ConfigField
31 FocalPlaneBackgroundConfig, MaskObjectsTask)
34 __all__ = [
"SkyCorrectionConfig",
"SkyCorrectionTask"]
40 """Make and write an image of an entire focal plane 44 camera : `lsst.afw.cameraGeom.Camera` 46 exposures : `list` of `tuple` of `int` and `lsst.afw.image.Exposure` 47 List of detector ID and CCD exposures (binned by `binning`). 48 filename : `str`, optional 51 Binning size that has been applied to images. 53 image = visualizeVisit.makeCameraImage(camera, dict(exp
for exp
in exposures
if exp
is not None), binning)
54 if filename
is not None:
55 image.writeFits(filename)
60 """Configuration for SkyCorrectionTask""" 62 rawLinker = pipeBase.InputDatasetField(
63 doc=
"Raw data to provide exp-visit linkage to connect calExp inputs to camera/sky calibs.",
66 storageClass=
"ExposureU",
67 dimensions=[
"instrument",
"exposure",
"detector"],
70 calExpArray = pipeBase.InputDatasetField(
71 doc=
"Input exposures to process",
74 storageClass=
"ExposureF",
75 dimensions=[
"instrument",
"visit",
"detector"],
77 calBkgArray = pipeBase.InputDatasetField(
78 doc=
"Input background files to use",
79 name=
"calexpBackground",
81 storageClass=
"Background",
82 dimensions=[
"instrument",
"visit",
"detector"],
85 camera = pipeBase.InputDatasetField(
86 doc=
"Input camera to use.",
89 storageClass=
"TablePersistableCamera",
90 dimensions=[
"instrument",
"calibration_label"],
92 skyCalibs = pipeBase.InputDatasetField(
93 doc=
"Input sky calibrations to use.",
96 storageClass=
"ExposureF",
97 dimensions=[
"instrument",
"physical_filter",
"detector",
"calibration_label"],
100 calExpCamera = pipeBase.OutputDatasetField(
101 doc=
"Output camera image.",
102 name=
'calexp_camera',
104 storageClass=
"ImageF",
105 dimensions=[
"instrument",
"visit"],
107 skyCorr = pipeBase.OutputDatasetField(
108 doc=
"Output sky corrected images.",
111 storageClass=
"Background",
112 dimensions=[
"instrument",
"visit",
"detector"],
115 bgModel = ConfigField(dtype=FocalPlaneBackgroundConfig, doc=
"Background model")
116 bgModel2 = ConfigField(dtype=FocalPlaneBackgroundConfig, doc=
"2nd Background model")
117 sky = ConfigurableField(target=SkyMeasurementTask, doc=
"Sky measurement")
118 maskObjects = ConfigurableField(target=MaskObjectsTask, doc=
"Mask Objects")
119 doMaskObjects = Field(dtype=bool, default=
True, doc=
"Mask objects to find good sky?")
120 doBgModel = Field(dtype=bool, default=
True, doc=
"Do background model subtraction?")
121 doBgModel2 = Field(dtype=bool, default=
True, doc=
"Do cleanup background model subtraction?")
122 doSky = Field(dtype=bool, default=
True, doc=
"Do sky frame subtraction?")
123 binning = Field(dtype=int, default=8, doc=
"Binning factor for constructing focal-plane images")
126 Config.setDefaults(self)
133 self.quantum.dimensions = (
"instrument",
"visit")
137 """Correct sky over entire focal plane""" 138 ConfigClass = SkyCorrectionConfig
139 _DefaultName =
"skyCorr" 142 inputData.pop(
"rawLinker",
None)
143 return super().
adaptArgsAndRun(inputData, inputDataIds, outputDataIds, butler)
148 names.add(
'skyCalibs')
154 return frozenset([
"calibration_label"])
159 self.makeSubtask(
"sky")
160 self.makeSubtask(
"maskObjects")
163 def _makeArgumentParser(cls, *args, **kwargs):
164 kwargs.pop(
"doBatch",
False)
165 parser = pipeBase.ArgumentParser(name=
"skyCorr", *args, **kwargs)
166 parser.add_id_argument(
"--id", datasetType=
"calexp", level=
"visit",
167 help=
"data ID, e.g. --id visit=12345")
172 """Return walltime request for batch job 174 Subclasses should override if the walltime should be calculated 175 differently (e.g., addition of some serial time). 180 Requested time per iteration. 181 parsedCmd : `argparse.Namespace` 182 Results of argument parsing. 186 numTargets = len(cls.RunnerClass.getTargetList(parsedCmd))
187 return time*numTargets
190 """Perform sky correction on an exposure 192 We restore the original sky, and remove it again using multiple 193 algorithms. We optionally apply: 195 1. A large-scale background model. 196 This step removes very-large-scale sky such as moonlight. 198 3. A medium-scale background model. 199 This step removes residual sky (This is smooth on the focal plane). 201 Only the master node executes this method. The data is held on 202 the slave nodes, which do all the hard work. 206 expRef : `lsst.daf.persistence.ButlerDataRef` 207 Data reference for exposure. 211 ~lsst.pipe.drivers.SkyCorrectionTask.run 214 extension =
"-%(visit)d.fits" % expRef.dataId
216 with self.
logOperation(
"processing %s" % (expRef.dataId,)):
219 pool.storeSet(butler=expRef.getButler())
220 camera = expRef.get(
"camera")
222 dataIdList = [ccdRef.dataId
for ccdRef
in expRef.subItems(
"ccd")
if 223 ccdRef.datasetExists(
"calexp")]
225 exposures = pool.map(self.
loadImage, dataIdList)
230 exposures = pool.mapToPrevious(self.
collectMask, dataIdList)
233 if self.config.doBgModel:
236 if self.config.doSky:
238 scale = self.sky.solveScales(measScales)
239 self.log.info(
"Sky frame scale: %s" % (scale,))
244 calibs = pool.mapToPrevious(self.
collectSky, dataIdList)
247 if self.config.doBgModel2:
252 expRef.put(image,
"calexp_camera")
254 pool.mapToPrevious(self.
write, dataIdList)
257 """Perform full focal-plane background subtraction 259 This method runs on the master node. 263 camera : `lsst.afw.cameraGeom.Camera` 265 pool : `lsst.ctrl.pool.Pool` 267 dataIdList : iterable of `dict` 268 List of data identifiers for the CCDs. 269 config : `lsst.pipe.drivers.background.FocalPlaneBackgroundConfig` 270 Configuration to use for background subtraction. 274 exposures : `list` of `lsst.afw.image.Image` 275 List of binned images, for creating focal plane image. 277 bgModel = FocalPlaneBackground.fromCamera(config, camera)
278 data = [pipeBase.Struct(dataId=dataId, bgModel=bgModel.clone())
for dataId
in dataIdList]
280 for ii, bg
in enumerate(bgModelList):
281 self.log.info(
"Background %d: %d pixels", ii, bg._numbers.array.sum())
283 return pool.mapToPrevious(self.
subtractModel, dataIdList, bgModel)
286 """Perform full focal-plane background subtraction 288 This method runs on the master node. 292 camera : `lsst.afw.cameraGeom.Camera` 294 cacheExposures : `list` of `lsst.afw.image.Exposures` 295 List of loaded and processed input calExp. 296 idList : `list` of `int` 297 List of detector ids to iterate over. 298 config : `lsst.pipe.drivers.background.FocalPlaneBackgroundConfig` 299 Configuration to use for background subtraction. 303 exposures : `list` of `lsst.afw.image.Image` 304 List of binned images, for creating focal plane image. 305 newCacheBgList : `list` of `lsst.afwMath.backgroundList` 306 Background lists generated. 307 cacheBgModel : `FocalPlaneBackground` 308 Full focal plane background model. 310 bgModel = FocalPlaneBackground.fromCamera(config, camera)
311 data = [pipeBase.Struct(id=id, bgModel=bgModel.clone())
for id
in idList]
314 for nodeData, cacheExp
in zip(data, cacheExposures):
315 nodeData.bgModel.addCcd(cacheExp)
316 bgModelList.append(nodeData.bgModel)
318 for ii, bg
in enumerate(bgModelList):
319 self.log.info(
"Background %d: %d pixels", ii, bg._numbers.getArray().sum())
325 for cacheExp
in cacheExposures:
327 exposures.append(afwMath.binImage(nodeExp.getMaskedImage(), self.config.binning))
328 cacheBgModel.append(nodeBgModel)
329 newCacheBgList.append(nodeBgList)
331 return exposures, newCacheBgList, cacheBgModel
333 def run(self, calExpArray, calBkgArray, skyCalibs, camera):
334 """Duplicate runDataRef method without ctrl_pool for Gen3. 338 calExpArray : `list` of `lsst.afw.image.Exposure` 339 Array of detector input calExp images for the exposure to 341 calBkgArray : `list` of `lsst.afw.math.BackgroundList` 342 Array of detector input background lists matching the 344 skyCalibs : `list` of `lsst.afw.image.Exposure` 345 Array of SKY calibrations for the input detectors to be 347 camera : `lsst.afw.cameraGeom.Camera` 348 Camera matching the input data to process. 352 results : `pipeBase.Struct` containing 353 calExpCamera : `lsst.afw.image.Exposure` 354 Full camera image of the sky-corrected data. 355 skyCorr : `list` of `lsst.afw.math.BackgroundList` 356 Detector-level sky-corrected background lists. 360 ~lsst.pipe.drivers.SkyCorrectionTask.runDataRef() 377 idList = [exp.getDetector().getId()
for exp
in calExpArray]
384 for calExp, calBgModel
in zip(calExpArray, calBkgArray):
385 nodeExp, nodeBgList = self.
loadImageRun(calExp, calBgModel)
386 cacheExposures.append(nodeExp)
387 cacheBgList.append(nodeBgList)
388 exposures.append(afwMath.binImage(nodeExp.getMaskedImage(), self.config.binning))
390 if self.config.doBgModel:
393 camera, cacheExposures, idList, self.config.bgModel
395 for cacheBg, newBg
in zip(cacheBgList, newCacheBgList):
396 cacheBg.append(newBg)
398 if self.config.doSky:
404 for cacheExp, skyCalib
in zip(cacheExposures, skyCalibs):
405 skyExp = self.sky.exposureToBackground(skyCalib)
406 cacheSky.append(skyExp)
407 scale = self.sky.measureScale(cacheExp.getMaskedImage(), skyExp)
408 measScales.append(scale)
410 scale = self.sky.solveScales(measScales)
411 self.log.info(
"Sky frame scale: %s" % (scale, ))
417 for cacheExp, nodeSky, nodeBgList
in zip(cacheExposures, cacheSky, cacheBgList):
418 self.sky.
subtractSkyFrame(cacheExp.getMaskedImage(), nodeSky, scale, nodeBgList)
419 exposures.append(afwMath.binImage(cacheExp.getMaskedImage(), self.config.binning))
421 if self.config.doBgModel2:
425 camera, cacheExposures, idList, self.config.bgModel2
427 for cacheBg, newBg
in zip(cacheBgList, newBgList):
428 cacheBg.append(newBg)
434 return pipeBase.Struct(
440 """Load original image and restore the sky 442 This method runs on the slave nodes. 446 cache : `lsst.pipe.base.Struct` 453 exposure : `lsst.afw.image.Exposure` 456 cache.dataId = dataId
457 cache.exposure = cache.butler.get(
"calexp", dataId, immediate=
True).clone()
458 bgOld = cache.butler.get(
"calexpBackground", dataId, immediate=
True)
459 image = cache.exposure.getMaskedImage()
463 statsImage = bgData[0].getStatsImage()
466 image -= bgOld.getImage()
467 cache.bgList = afwMath.BackgroundList()
469 cache.bgList.append(bgData)
471 if self.config.doMaskObjects:
472 self.maskObjects.findObjects(cache.exposure)
477 """Serial implementation of self.loadImage() for Gen3. 479 Load and restore background to calExp and calExpBkg. 483 calExp : `lsst.afw.image.Exposure` 484 Detector level calExp image to process. 485 calExpBkg : `lsst.afw.math.BackgroundList` 486 Detector level background list associated with the calExp. 490 calExp : `lsst.afw.image.Exposure` 491 Background restored calExp. 492 bgList : `lsst.afw.math.BackgroundList` 493 New background list containing the restoration background. 495 image = calExp.getMaskedImage()
497 for bgOld
in calExpBkg:
498 statsImage = bgOld[0].getStatsImage()
501 image -= calExpBkg.getImage()
502 bgList = afwMath.BackgroundList()
503 for bgData
in calExpBkg:
504 bgList.append(bgData)
506 if self.config.doMaskObjects:
507 self.maskObjects.findObjects(calExp)
509 return (calExp, bgList)
512 """Measure scale for sky frame 514 This method runs on the slave nodes. 518 cache : `lsst.pipe.base.Struct` 528 assert cache.dataId == dataId
529 cache.sky = self.sky.getSkyData(cache.butler, dataId)
530 scale = self.sky.measureScale(cache.exposure.getMaskedImage(), cache.sky)
534 """Subtract sky frame 536 This method runs on the slave nodes. 540 cache : `lsst.pipe.base.Struct` 549 exposure : `lsst.afw.image.Exposure` 552 assert cache.dataId == dataId
553 self.sky.
subtractSkyFrame(cache.exposure.getMaskedImage(), cache.sky, scale, cache.bgList)
557 """Fit background model for CCD 559 This method runs on the slave nodes. 563 cache : `lsst.pipe.base.Struct` 565 data : `lsst.pipe.base.Struct` 566 Data identifier, with `dataId` (data identifier) and `bgModel` 567 (background model) elements. 571 bgModel : `lsst.pipe.drivers.background.FocalPlaneBackground` 574 assert cache.dataId == data.dataId
575 data.bgModel.addCcd(cache.exposure)
579 """Subtract background model 581 This method runs on the slave nodes. 585 cache : `lsst.pipe.base.Struct` 589 bgModel : `lsst.pipe.drivers.background.FocalPlaneBackround` 594 exposure : `lsst.afw.image.Exposure` 597 assert cache.dataId == dataId
598 exposure = cache.exposure
599 image = exposure.getMaskedImage()
600 detector = exposure.getDetector()
601 bbox = image.getBBox()
602 cache.bgModel = bgModel.toCcdBackground(detector, bbox)
603 image -= cache.bgModel.getImage()
604 cache.bgList.append(cache.bgModel[0])
608 """Serial implementation of self.subtractModel() for Gen3. 610 Load and restore background to calExp and calExpBkg. 614 exposure : `lsst.afw.image.Exposure` 615 Exposure to subtract the background model from. 616 bgModel : `lsst.pipe.drivers.background.FocalPlaneBackground` 617 Full camera level background model. 621 exposure : `lsst.afw.image.Exposure` 622 Background subtracted input exposure. 623 bgModelCcd : `lsst.afw.math.BackgroundList` 624 Detector level realization of the full background model. 625 bgModelMaskedImage : `lsst.afw.image.MaskedImage` 626 Background model from the bgModelCcd realization. 628 image = exposure.getMaskedImage()
629 detector = exposure.getDetector()
630 bbox = image.getBBox()
631 bgModelCcd = bgModel.toCcdBackground(detector, bbox)
632 image -= bgModelCcd.getImage()
634 return (exposure, bgModelCcd, bgModelCcd[0])
637 """Generate an image of the background model for visualisation 639 Useful for debugging. 643 cache : `lsst.pipe.base.Struct` 647 bgModel : `lsst.pipe.drivers.background.FocalPlaneBackround` 654 image : `lsst.afw.image.MaskedImage` 655 Binned background model image. 657 assert cache.dataId == dataId
658 exposure = cache.exposure
659 detector = exposure.getDetector()
660 bbox = exposure.getMaskedImage().getBBox()
661 image = bgModel.toCcdBackground(detector, bbox).getImage()
665 """Return the binned image required for visualization 667 This method just helps to cut down on boilerplate. 671 image : `lsst.afw.image.MaskedImage` 672 Image to go into visualisation. 678 image : `lsst.afw.image.MaskedImage` 681 return (exposure.getDetector().getId(), afwMath.binImage(image, self.config.binning))
684 """Collect exposure for potential visualisation 686 This method runs on the slave nodes. 690 cache : `lsst.pipe.base.Struct` 697 image : `lsst.afw.image.MaskedImage` 703 """Collect original image for visualisation 705 This method runs on the slave nodes. 709 cache : `lsst.pipe.base.Struct` 718 image : `lsst.afw.image.MaskedImage` 721 exposure = cache.butler.get(
"calexp", dataId, immediate=
True)
725 """Collect original image for visualisation 727 This method runs on the slave nodes. 731 cache : `lsst.pipe.base.Struct` 740 image : `lsst.afw.image.MaskedImage` 746 """Collect mask for visualisation 748 This method runs on the slave nodes. 752 cache : `lsst.pipe.base.Struct` 761 image : `lsst.afw.image.Image` 765 image = afwImage.ImageF(cache.exposure.maskedImage.getBBox())
766 image.array[:] = cache.exposure.maskedImage.mask.array
770 """Write resultant background list 772 This method runs on the slave nodes. 776 cache : `lsst.pipe.base.Struct` 781 cache.butler.put(cache.bgList,
"skyCorr", dataId)
783 def _getMetadataName(self):
784 """There's no metadata to write out"""
def subtractModel(self, cache, dataId, bgModel)
def getPrerequisiteDatasetTypes(cls, config)
def batchWallTime(cls, time, parsedCmd, numCores)
def runDataRef(self, expRef)
def makeCameraImage(camera, exposures, filename=None, binning=8)
def focalPlaneBackground(self, camera, pool, dataIdList, config)
def run(self, calExpArray, calBkgArray, skyCalibs, camera)
def accumulateModel(self, cache, data)
def subtractModelRun(self, exposure, bgModel)
def loadImage(self, cache, dataId)
def collectBinnedImage(self, exposure, image)
def realiseModel(self, cache, dataId, bgModel)
def collectOriginal(self, cache, dataId)
def focalPlaneBackgroundRun(self, camera, cacheExposures, idList, config)
def subtractSkyFrame(self, cache, dataId, scale)
def write(self, cache, dataId)
def logOperation(self, operation, catch=False, trace=True)
def getPerDatasetTypeDimensions(cls, config)
def loadImageRun(self, calExp, calExpBkg)
def __init__(self, args, kwargs)
def collectSky(self, cache, dataId)
def collectMask(self, cache, dataId)
def measureSkyFrame(self, cache, dataId)
def adaptArgsAndRun(self, inputData, inputDataIds, outputDataIds, butler)