21 from __future__
import absolute_import, division, print_function
28 from lsst.pex.config import Config, Field, ConfigurableField, ConfigField
32 FocalPlaneBackgroundConfig, MaskObjectsTask)
35 __all__ = [
"SkyCorrectionConfig",
"SkyCorrectionTask"]
41 """Make and write an image of an entire focal plane 45 camera : `lsst.afw.cameraGeom.Camera` 47 exposures : `list` of `tuple` of `int` and `lsst.afw.image.Exposure` 48 List of detector ID and CCD exposures (binned by `binning`). 49 filename : `str`, optional 52 Binning size that has been applied to images. 54 image = visualizeVisit.makeCameraImage(camera, dict(exp
for exp
in exposures
if exp
is not None), binning)
55 if filename
is not None:
56 image.writeFits(filename)
61 """Configuration for SkyCorrectionTask""" 63 rawLinker = pipeBase.InputDatasetField(
64 doc=
"Raw data to provide exp-visit linkage to connect calExp inputs to camera/sky calibs.",
67 storageClass=
"ExposureU",
68 dimensions=[
"instrument",
"exposure",
"detector"],
71 calExpArray = pipeBase.InputDatasetField(
72 doc=
"Input exposures to process",
75 storageClass=
"ExposureF",
76 dimensions=[
"instrument",
"visit",
"detector"],
78 calBkgArray = pipeBase.InputDatasetField(
79 doc=
"Input background files to use",
80 name=
"calexpBackground",
82 storageClass=
"Background",
83 dimensions=[
"instrument",
"visit",
"detector"],
86 camera = pipeBase.InputDatasetField(
87 doc=
"Input camera to use.",
90 storageClass=
"TablePersistableCamera",
91 dimensions=[
"instrument",
"calibration_label"],
93 skyCalibs = pipeBase.InputDatasetField(
94 doc=
"Input sky calibrations to use.",
97 storageClass=
"ExposureF",
98 dimensions=[
"instrument",
"physical_filter",
"detector",
"calibration_label"],
101 calExpCamera = pipeBase.OutputDatasetField(
102 doc=
"Output camera image.",
103 name=
'calexp_camera',
105 storageClass=
"ImageF",
106 dimensions=[
"instrument",
"visit"],
108 skyCorr = pipeBase.OutputDatasetField(
109 doc=
"Output sky corrected images.",
112 storageClass=
"Background",
113 dimensions=[
"instrument",
"visit",
"detector"],
116 bgModel = ConfigField(dtype=FocalPlaneBackgroundConfig, doc=
"Background model")
117 bgModel2 = ConfigField(dtype=FocalPlaneBackgroundConfig, doc=
"2nd Background model")
118 sky = ConfigurableField(target=SkyMeasurementTask, doc=
"Sky measurement")
119 maskObjects = ConfigurableField(target=MaskObjectsTask, doc=
"Mask Objects")
120 doMaskObjects = Field(dtype=bool, default=
True, doc=
"Mask objects to find good sky?")
121 doBgModel = Field(dtype=bool, default=
True, doc=
"Do background model subtraction?")
122 doBgModel2 = Field(dtype=bool, default=
True, doc=
"Do cleanup background model subtraction?")
123 doSky = Field(dtype=bool, default=
True, doc=
"Do sky frame subtraction?")
124 binning = Field(dtype=int, default=8, doc=
"Binning factor for constructing focal-plane images")
125 calexpType = Field(dtype=str, default=
"calexp",
126 doc=
"Should be set to fakes_calexp if you want to process calexps with fakes in.")
129 Config.setDefaults(self)
136 self.quantum.dimensions = (
"instrument",
"visit")
140 """Correct sky over entire focal plane""" 141 ConfigClass = SkyCorrectionConfig
142 _DefaultName =
"skyCorr" 145 inputData.pop(
"rawLinker",
None)
146 return super().
adaptArgsAndRun(inputData, inputDataIds, outputDataIds, butler)
151 names.add(
'skyCalibs')
157 return frozenset([
"calibration_label"])
162 self.makeSubtask(
"sky")
163 self.makeSubtask(
"maskObjects")
166 def _makeArgumentParser(cls, *args, **kwargs):
167 kwargs.pop(
"doBatch",
False)
168 datasetType = ConfigDatasetType(name=
"calexpType")
169 parser = ArgumentParser(name=
"skyCorr", *args, **kwargs)
170 parser.add_id_argument(
"--id", datasetType=datasetType, level=
"visit",
171 help=
"data ID, e.g. --id visit=12345")
176 """Return walltime request for batch job 178 Subclasses should override if the walltime should be calculated 179 differently (e.g., addition of some serial time). 184 Requested time per iteration. 185 parsedCmd : `argparse.Namespace` 186 Results of argument parsing. 190 numTargets = len(cls.RunnerClass.getTargetList(parsedCmd))
191 return time*numTargets
194 """Perform sky correction on an exposure 196 We restore the original sky, and remove it again using multiple 197 algorithms. We optionally apply: 199 1. A large-scale background model. 200 This step removes very-large-scale sky such as moonlight. 202 3. A medium-scale background model. 203 This step removes residual sky (This is smooth on the focal plane). 205 Only the master node executes this method. The data is held on 206 the slave nodes, which do all the hard work. 210 expRef : `lsst.daf.persistence.ButlerDataRef` 211 Data reference for exposure. 215 ~lsst.pipe.drivers.SkyCorrectionTask.run 218 extension =
"-%(visit)d.fits" % expRef.dataId
220 with self.
logOperation(
"processing %s" % (expRef.dataId,)):
223 pool.storeSet(butler=expRef.getButler())
224 camera = expRef.get(
"camera")
226 dataIdList = [ccdRef.dataId
for ccdRef
in expRef.subItems(
"ccd")
if 227 ccdRef.datasetExists(self.config.calexpType)]
229 exposures = pool.map(self.
loadImage, dataIdList)
234 exposures = pool.mapToPrevious(self.
collectMask, dataIdList)
237 if self.config.doBgModel:
240 if self.config.doSky:
242 scale = self.sky.solveScales(measScales)
243 self.log.info(
"Sky frame scale: %s" % (scale,))
248 calibs = pool.mapToPrevious(self.
collectSky, dataIdList)
251 if self.config.doBgModel2:
256 expRef.put(image,
"calexp_camera")
258 pool.mapToPrevious(self.
write, dataIdList)
261 """Perform full focal-plane background subtraction 263 This method runs on the master node. 267 camera : `lsst.afw.cameraGeom.Camera` 269 pool : `lsst.ctrl.pool.Pool` 271 dataIdList : iterable of `dict` 272 List of data identifiers for the CCDs. 273 config : `lsst.pipe.drivers.background.FocalPlaneBackgroundConfig` 274 Configuration to use for background subtraction. 278 exposures : `list` of `lsst.afw.image.Image` 279 List of binned images, for creating focal plane image. 281 bgModel = FocalPlaneBackground.fromCamera(config, camera)
282 data = [pipeBase.Struct(dataId=dataId, bgModel=bgModel.clone())
for dataId
in dataIdList]
284 for ii, bg
in enumerate(bgModelList):
285 self.log.info(
"Background %d: %d pixels", ii, bg._numbers.array.sum())
287 return pool.mapToPrevious(self.
subtractModel, dataIdList, bgModel)
290 """Perform full focal-plane background subtraction 292 This method runs on the master node. 296 camera : `lsst.afw.cameraGeom.Camera` 298 cacheExposures : `list` of `lsst.afw.image.Exposures` 299 List of loaded and processed input calExp. 300 idList : `list` of `int` 301 List of detector ids to iterate over. 302 config : `lsst.pipe.drivers.background.FocalPlaneBackgroundConfig` 303 Configuration to use for background subtraction. 307 exposures : `list` of `lsst.afw.image.Image` 308 List of binned images, for creating focal plane image. 309 newCacheBgList : `list` of `lsst.afwMath.backgroundList` 310 Background lists generated. 311 cacheBgModel : `FocalPlaneBackground` 312 Full focal plane background model. 314 bgModel = FocalPlaneBackground.fromCamera(config, camera)
315 data = [pipeBase.Struct(id=id, bgModel=bgModel.clone())
for id
in idList]
318 for nodeData, cacheExp
in zip(data, cacheExposures):
319 nodeData.bgModel.addCcd(cacheExp)
320 bgModelList.append(nodeData.bgModel)
322 for ii, bg
in enumerate(bgModelList):
323 self.log.info(
"Background %d: %d pixels", ii, bg._numbers.getArray().sum())
329 for cacheExp
in cacheExposures:
331 exposures.append(afwMath.binImage(nodeExp.getMaskedImage(), self.config.binning))
332 cacheBgModel.append(nodeBgModel)
333 newCacheBgList.append(nodeBgList)
335 return exposures, newCacheBgList, cacheBgModel
337 def run(self, calExpArray, calBkgArray, skyCalibs, camera):
338 """Duplicate runDataRef method without ctrl_pool for Gen3. 342 calExpArray : `list` of `lsst.afw.image.Exposure` 343 Array of detector input calExp images for the exposure to 345 calBkgArray : `list` of `lsst.afw.math.BackgroundList` 346 Array of detector input background lists matching the 348 skyCalibs : `list` of `lsst.afw.image.Exposure` 349 Array of SKY calibrations for the input detectors to be 351 camera : `lsst.afw.cameraGeom.Camera` 352 Camera matching the input data to process. 356 results : `pipeBase.Struct` containing 357 calExpCamera : `lsst.afw.image.Exposure` 358 Full camera image of the sky-corrected data. 359 skyCorr : `list` of `lsst.afw.math.BackgroundList` 360 Detector-level sky-corrected background lists. 364 ~lsst.pipe.drivers.SkyCorrectionTask.runDataRef() 381 idList = [exp.getDetector().getId()
for exp
in calExpArray]
388 for calExp, calBgModel
in zip(calExpArray, calBkgArray):
389 nodeExp, nodeBgList = self.
loadImageRun(calExp, calBgModel)
390 cacheExposures.append(nodeExp)
391 cacheBgList.append(nodeBgList)
392 exposures.append(afwMath.binImage(nodeExp.getMaskedImage(), self.config.binning))
394 if self.config.doBgModel:
397 camera, cacheExposures, idList, self.config.bgModel
399 for cacheBg, newBg
in zip(cacheBgList, newCacheBgList):
400 cacheBg.append(newBg)
402 if self.config.doSky:
408 for cacheExp, skyCalib
in zip(cacheExposures, skyCalibs):
409 skyExp = self.sky.exposureToBackground(skyCalib)
410 cacheSky.append(skyExp)
411 scale = self.sky.measureScale(cacheExp.getMaskedImage(), skyExp)
412 measScales.append(scale)
414 scale = self.sky.solveScales(measScales)
415 self.log.info(
"Sky frame scale: %s" % (scale, ))
421 for cacheExp, nodeSky, nodeBgList
in zip(cacheExposures, cacheSky, cacheBgList):
422 self.sky.
subtractSkyFrame(cacheExp.getMaskedImage(), nodeSky, scale, nodeBgList)
423 exposures.append(afwMath.binImage(cacheExp.getMaskedImage(), self.config.binning))
425 if self.config.doBgModel2:
429 camera, cacheExposures, idList, self.config.bgModel2
431 for cacheBg, newBg
in zip(cacheBgList, newBgList):
432 cacheBg.append(newBg)
438 return pipeBase.Struct(
444 """Load original image and restore the sky 446 This method runs on the slave nodes. 450 cache : `lsst.pipe.base.Struct` 457 exposure : `lsst.afw.image.Exposure` 460 cache.dataId = dataId
461 cache.exposure = cache.butler.get(self.config.calexpType, dataId, immediate=
True).clone()
462 bgOld = cache.butler.get(
"calexpBackground", dataId, immediate=
True)
463 image = cache.exposure.getMaskedImage()
467 statsImage = bgData[0].getStatsImage()
470 image -= bgOld.getImage()
471 cache.bgList = afwMath.BackgroundList()
473 cache.bgList.append(bgData)
475 if self.config.doMaskObjects:
476 self.maskObjects.findObjects(cache.exposure)
481 """Serial implementation of self.loadImage() for Gen3. 483 Load and restore background to calExp and calExpBkg. 487 calExp : `lsst.afw.image.Exposure` 488 Detector level calExp image to process. 489 calExpBkg : `lsst.afw.math.BackgroundList` 490 Detector level background list associated with the calExp. 494 calExp : `lsst.afw.image.Exposure` 495 Background restored calExp. 496 bgList : `lsst.afw.math.BackgroundList` 497 New background list containing the restoration background. 499 image = calExp.getMaskedImage()
501 for bgOld
in calExpBkg:
502 statsImage = bgOld[0].getStatsImage()
505 image -= calExpBkg.getImage()
506 bgList = afwMath.BackgroundList()
507 for bgData
in calExpBkg:
508 bgList.append(bgData)
510 if self.config.doMaskObjects:
511 self.maskObjects.findObjects(calExp)
513 return (calExp, bgList)
516 """Measure scale for sky frame 518 This method runs on the slave nodes. 522 cache : `lsst.pipe.base.Struct` 532 assert cache.dataId == dataId
533 cache.sky = self.sky.getSkyData(cache.butler, dataId)
534 scale = self.sky.measureScale(cache.exposure.getMaskedImage(), cache.sky)
538 """Subtract sky frame 540 This method runs on the slave nodes. 544 cache : `lsst.pipe.base.Struct` 553 exposure : `lsst.afw.image.Exposure` 556 assert cache.dataId == dataId
557 self.sky.
subtractSkyFrame(cache.exposure.getMaskedImage(), cache.sky, scale, cache.bgList)
561 """Fit background model for CCD 563 This method runs on the slave nodes. 567 cache : `lsst.pipe.base.Struct` 569 data : `lsst.pipe.base.Struct` 570 Data identifier, with `dataId` (data identifier) and `bgModel` 571 (background model) elements. 575 bgModel : `lsst.pipe.drivers.background.FocalPlaneBackground` 578 assert cache.dataId == data.dataId
579 data.bgModel.addCcd(cache.exposure)
583 """Subtract background model 585 This method runs on the slave nodes. 589 cache : `lsst.pipe.base.Struct` 593 bgModel : `lsst.pipe.drivers.background.FocalPlaneBackround` 598 exposure : `lsst.afw.image.Exposure` 601 assert cache.dataId == dataId
602 exposure = cache.exposure
603 image = exposure.getMaskedImage()
604 detector = exposure.getDetector()
605 bbox = image.getBBox()
606 cache.bgModel = bgModel.toCcdBackground(detector, bbox)
607 image -= cache.bgModel.getImage()
608 cache.bgList.append(cache.bgModel[0])
612 """Serial implementation of self.subtractModel() for Gen3. 614 Load and restore background to calExp and calExpBkg. 618 exposure : `lsst.afw.image.Exposure` 619 Exposure to subtract the background model from. 620 bgModel : `lsst.pipe.drivers.background.FocalPlaneBackground` 621 Full camera level background model. 625 exposure : `lsst.afw.image.Exposure` 626 Background subtracted input exposure. 627 bgModelCcd : `lsst.afw.math.BackgroundList` 628 Detector level realization of the full background model. 629 bgModelMaskedImage : `lsst.afw.image.MaskedImage` 630 Background model from the bgModelCcd realization. 632 image = exposure.getMaskedImage()
633 detector = exposure.getDetector()
634 bbox = image.getBBox()
635 bgModelCcd = bgModel.toCcdBackground(detector, bbox)
636 image -= bgModelCcd.getImage()
638 return (exposure, bgModelCcd, bgModelCcd[0])
641 """Generate an image of the background model for visualisation 643 Useful for debugging. 647 cache : `lsst.pipe.base.Struct` 651 bgModel : `lsst.pipe.drivers.background.FocalPlaneBackround` 658 image : `lsst.afw.image.MaskedImage` 659 Binned background model image. 661 assert cache.dataId == dataId
662 exposure = cache.exposure
663 detector = exposure.getDetector()
664 bbox = exposure.getMaskedImage().getBBox()
665 image = bgModel.toCcdBackground(detector, bbox).getImage()
669 """Return the binned image required for visualization 671 This method just helps to cut down on boilerplate. 675 image : `lsst.afw.image.MaskedImage` 676 Image to go into visualisation. 682 image : `lsst.afw.image.MaskedImage` 685 return (exposure.getDetector().getId(), afwMath.binImage(image, self.config.binning))
688 """Collect exposure for potential visualisation 690 This method runs on the slave nodes. 694 cache : `lsst.pipe.base.Struct` 701 image : `lsst.afw.image.MaskedImage` 707 """Collect original image for visualisation 709 This method runs on the slave nodes. 713 cache : `lsst.pipe.base.Struct` 722 image : `lsst.afw.image.MaskedImage` 725 exposure = cache.butler.get(
"calexp", dataId, immediate=
True)
729 """Collect original image for visualisation 731 This method runs on the slave nodes. 735 cache : `lsst.pipe.base.Struct` 744 image : `lsst.afw.image.MaskedImage` 750 """Collect mask for visualisation 752 This method runs on the slave nodes. 756 cache : `lsst.pipe.base.Struct` 765 image : `lsst.afw.image.Image` 769 image = afwImage.ImageF(cache.exposure.maskedImage.getBBox())
770 image.array[:] = cache.exposure.maskedImage.mask.array
774 """Write resultant background list 776 This method runs on the slave nodes. 780 cache : `lsst.pipe.base.Struct` 785 cache.butler.put(cache.bgList,
"skyCorr", dataId)
787 def _getMetadataName(self):
788 """There's no metadata to write out"""
def subtractModel(self, cache, dataId, bgModel)
def getPrerequisiteDatasetTypes(cls, config)
def batchWallTime(cls, time, parsedCmd, numCores)
def runDataRef(self, expRef)
def makeCameraImage(camera, exposures, filename=None, binning=8)
def focalPlaneBackground(self, camera, pool, dataIdList, config)
def run(self, calExpArray, calBkgArray, skyCalibs, camera)
def accumulateModel(self, cache, data)
def subtractModelRun(self, exposure, bgModel)
def loadImage(self, cache, dataId)
def collectBinnedImage(self, exposure, image)
def realiseModel(self, cache, dataId, bgModel)
def collectOriginal(self, cache, dataId)
def focalPlaneBackgroundRun(self, camera, cacheExposures, idList, config)
def subtractSkyFrame(self, cache, dataId, scale)
def write(self, cache, dataId)
def logOperation(self, operation, catch=False, trace=True)
def getPerDatasetTypeDimensions(cls, config)
def loadImageRun(self, calExp, calExpBkg)
def __init__(self, args, kwargs)
def collectSky(self, cache, dataId)
def collectMask(self, cache, dataId)
def measureSkyFrame(self, cache, dataId)
def adaptArgsAndRun(self, inputData, inputDataIds, outputDataIds, butler)