21 from __future__
import absolute_import, division, print_function
28 from lsst.pex.config
import Config, Field, ConfigurableField, ConfigField
32 FocalPlaneBackgroundConfig, MaskObjectsTask)
36 __all__ = [
"SkyCorrectionConfig",
"SkyCorrectionTask"]
42 """Make and write an image of an entire focal plane 46 camera : `lsst.afw.cameraGeom.Camera` 48 exposures : `list` of `tuple` of `int` and `lsst.afw.image.Exposure` 49 List of detector ID and CCD exposures (binned by `binning`). 50 filename : `str`, optional 53 Binning size that has been applied to images. 55 image = visualizeVisit.makeCameraImage(camera, dict(exp
for exp
in exposures
if exp
is not None), binning)
56 if filename
is not None:
57 image.writeFits(filename)
63 doc=
"Raw data to provide exp-visit linkage to connect calExp inputs to camera/sky calibs.",
67 storageClass=
"ExposureU",
68 dimensions=[
"instrument",
"exposure",
"detector"],
70 calExpArray = cT.Input(
71 doc=
"Input exposures to process",
74 storageClass=
"ExposureF",
75 dimensions=[
"instrument",
"visit",
"detector"],
77 calBkgArray = cT.Input(
78 doc=
"Input background files to use",
80 name=
"calexpBackground",
81 storageClass=
"Background",
82 dimensions=[
"instrument",
"visit",
"detector"],
84 camera = cT.PrerequisiteInput(
85 doc=
"Input camera to use.",
87 storageClass=
"Camera",
88 dimensions=[
"instrument",
"calibration_label"],
90 skyCalibs = cT.PrerequisiteInput(
91 doc=
"Input sky calibrations to use.",
94 storageClass=
"ExposureF",
95 dimensions=[
"instrument",
"physical_filter",
"detector",
"calibration_label"],
97 calExpCamera = cT.Output(
98 doc=
"Output camera image.",
100 storageClass=
"ImageF",
101 dimensions=[
"instrument",
"visit"],
104 doc=
"Output sky corrected images.",
107 storageClass=
"Background",
108 dimensions=[
"instrument",
"visit",
"detector"],
113 """Configuration for SkyCorrectionTask""" 114 bgModel = ConfigField(dtype=FocalPlaneBackgroundConfig, doc=
"Background model")
115 bgModel2 = ConfigField(dtype=FocalPlaneBackgroundConfig, doc=
"2nd Background model")
116 sky = ConfigurableField(target=SkyMeasurementTask, doc=
"Sky measurement")
117 maskObjects = ConfigurableField(target=MaskObjectsTask, doc=
"Mask Objects")
118 doMaskObjects = Field(dtype=bool, default=
True, doc=
"Mask objects to find good sky?")
119 doBgModel = Field(dtype=bool, default=
True, doc=
"Do background model subtraction?")
120 doBgModel2 = Field(dtype=bool, default=
True, doc=
"Do cleanup background model subtraction?")
121 doSky = Field(dtype=bool, default=
True, doc=
"Do sky frame subtraction?")
122 binning = Field(dtype=int, default=8, doc=
"Binning factor for constructing focal-plane images")
123 calexpType = Field(dtype=str, default=
"calexp",
124 doc=
"Should be set to fakes_calexp if you want to process calexps with fakes in.")
127 Config.setDefaults(self)
136 """Correct sky over entire focal plane""" 137 ConfigClass = SkyCorrectionConfig
138 _DefaultName =
"skyCorr" 141 inputs = butlerQC.get(inputRefs)
142 inputs.pop(
"rawLinker",
None)
143 outputs = self.
run(**inputs)
144 butlerQC.put(outputs, outputRefs)
149 self.makeSubtask(
"sky")
150 self.makeSubtask(
"maskObjects")
153 def _makeArgumentParser(cls, *args, **kwargs):
154 kwargs.pop(
"doBatch",
False)
155 datasetType = ConfigDatasetType(name=
"calexpType")
156 parser = ArgumentParser(name=
"skyCorr", *args, **kwargs)
157 parser.add_id_argument(
"--id", datasetType=datasetType, level=
"visit",
158 help=
"data ID, e.g. --id visit=12345")
163 """Return walltime request for batch job 165 Subclasses should override if the walltime should be calculated 166 differently (e.g., addition of some serial time). 171 Requested time per iteration. 172 parsedCmd : `argparse.Namespace` 173 Results of argument parsing. 177 numTargets = len(cls.RunnerClass.getTargetList(parsedCmd))
178 return time*numTargets
181 """Perform sky correction on an exposure 183 We restore the original sky, and remove it again using multiple 184 algorithms. We optionally apply: 186 1. A large-scale background model. 187 This step removes very-large-scale sky such as moonlight. 189 3. A medium-scale background model. 190 This step removes residual sky (This is smooth on the focal plane). 192 Only the master node executes this method. The data is held on 193 the slave nodes, which do all the hard work. 197 expRef : `lsst.daf.persistence.ButlerDataRef` 198 Data reference for exposure. 202 ~lsst.pipe.drivers.SkyCorrectionTask.run 205 extension =
"-%(visit)d.fits" % expRef.dataId
207 with self.
logOperation(
"processing %s" % (expRef.dataId,)):
210 pool.storeSet(butler=expRef.getButler())
211 camera = expRef.get(
"camera")
213 dataIdList = [ccdRef.dataId
for ccdRef
in expRef.subItems(
"ccd")
if 214 ccdRef.datasetExists(self.config.calexpType)]
216 exposures = pool.map(self.
loadImage, dataIdList)
221 exposures = pool.mapToPrevious(self.
collectMask, dataIdList)
224 if self.config.doBgModel:
227 if self.config.doSky:
229 scale = self.sky.solveScales(measScales)
230 self.log.info(
"Sky frame scale: %s" % (scale,))
235 calibs = pool.mapToPrevious(self.
collectSky, dataIdList)
238 if self.config.doBgModel2:
243 expRef.put(image,
"calexp_camera")
245 pool.mapToPrevious(self.
write, dataIdList)
248 """Perform full focal-plane background subtraction 250 This method runs on the master node. 254 camera : `lsst.afw.cameraGeom.Camera` 256 pool : `lsst.ctrl.pool.Pool` 258 dataIdList : iterable of `dict` 259 List of data identifiers for the CCDs. 260 config : `lsst.pipe.drivers.background.FocalPlaneBackgroundConfig` 261 Configuration to use for background subtraction. 265 exposures : `list` of `lsst.afw.image.Image` 266 List of binned images, for creating focal plane image. 268 bgModel = FocalPlaneBackground.fromCamera(config, camera)
269 data = [pipeBase.Struct(dataId=dataId, bgModel=bgModel.clone())
for dataId
in dataIdList]
271 for ii, bg
in enumerate(bgModelList):
272 self.log.info(
"Background %d: %d pixels", ii, bg._numbers.array.sum())
274 return pool.mapToPrevious(self.
subtractModel, dataIdList, bgModel)
277 """Perform full focal-plane background subtraction 279 This method runs on the master node. 283 camera : `lsst.afw.cameraGeom.Camera` 285 cacheExposures : `list` of `lsst.afw.image.Exposures` 286 List of loaded and processed input calExp. 287 idList : `list` of `int` 288 List of detector ids to iterate over. 289 config : `lsst.pipe.drivers.background.FocalPlaneBackgroundConfig` 290 Configuration to use for background subtraction. 294 exposures : `list` of `lsst.afw.image.Image` 295 List of binned images, for creating focal plane image. 296 newCacheBgList : `list` of `lsst.afwMath.backgroundList` 297 Background lists generated. 298 cacheBgModel : `FocalPlaneBackground` 299 Full focal plane background model. 301 bgModel = FocalPlaneBackground.fromCamera(config, camera)
302 data = [pipeBase.Struct(id=id, bgModel=bgModel.clone())
for id
in idList]
305 for nodeData, cacheExp
in zip(data, cacheExposures):
306 nodeData.bgModel.addCcd(cacheExp)
307 bgModelList.append(nodeData.bgModel)
309 for ii, bg
in enumerate(bgModelList):
310 self.log.info(
"Background %d: %d pixels", ii, bg._numbers.getArray().sum())
316 for cacheExp
in cacheExposures:
318 exposures.append(afwMath.binImage(nodeExp.getMaskedImage(), self.config.binning))
319 cacheBgModel.append(nodeBgModel)
320 newCacheBgList.append(nodeBgList)
322 return exposures, newCacheBgList, cacheBgModel
324 def run(self, calExpArray, calBkgArray, skyCalibs, camera):
325 """Duplicate runDataRef method without ctrl_pool for Gen3. 329 calExpArray : `list` of `lsst.afw.image.Exposure` 330 Array of detector input calExp images for the exposure to 332 calBkgArray : `list` of `lsst.afw.math.BackgroundList` 333 Array of detector input background lists matching the 335 skyCalibs : `list` of `lsst.afw.image.Exposure` 336 Array of SKY calibrations for the input detectors to be 338 camera : `lsst.afw.cameraGeom.Camera` 339 Camera matching the input data to process. 343 results : `pipeBase.Struct` containing 344 calExpCamera : `lsst.afw.image.Exposure` 345 Full camera image of the sky-corrected data. 346 skyCorr : `list` of `lsst.afw.math.BackgroundList` 347 Detector-level sky-corrected background lists. 351 ~lsst.pipe.drivers.SkyCorrectionTask.runDataRef() 368 idList = [exp.getDetector().getId()
for exp
in calExpArray]
375 for calExp, calBgModel
in zip(calExpArray, calBkgArray):
376 nodeExp, nodeBgList = self.
loadImageRun(calExp, calBgModel)
377 cacheExposures.append(nodeExp)
378 cacheBgList.append(nodeBgList)
379 exposures.append(afwMath.binImage(nodeExp.getMaskedImage(), self.config.binning))
381 if self.config.doBgModel:
384 camera, cacheExposures, idList, self.config.bgModel
386 for cacheBg, newBg
in zip(cacheBgList, newCacheBgList):
387 cacheBg.append(newBg)
389 if self.config.doSky:
395 for cacheExp, skyCalib
in zip(cacheExposures, skyCalibs):
396 skyExp = self.sky.exposureToBackground(skyCalib)
397 cacheSky.append(skyExp)
398 scale = self.sky.measureScale(cacheExp.getMaskedImage(), skyExp)
399 measScales.append(scale)
401 scale = self.sky.solveScales(measScales)
402 self.log.info(
"Sky frame scale: %s" % (scale, ))
408 for cacheExp, nodeSky, nodeBgList
in zip(cacheExposures, cacheSky, cacheBgList):
409 self.sky.
subtractSkyFrame(cacheExp.getMaskedImage(), nodeSky, scale, nodeBgList)
410 exposures.append(afwMath.binImage(cacheExp.getMaskedImage(), self.config.binning))
412 if self.config.doBgModel2:
416 camera, cacheExposures, idList, self.config.bgModel2
418 for cacheBg, newBg
in zip(cacheBgList, newBgList):
419 cacheBg.append(newBg)
425 return pipeBase.Struct(
431 """Load original image and restore the sky 433 This method runs on the slave nodes. 437 cache : `lsst.pipe.base.Struct` 444 exposure : `lsst.afw.image.Exposure` 447 cache.dataId = dataId
448 cache.exposure = cache.butler.get(self.config.calexpType, dataId, immediate=
True).clone()
449 bgOld = cache.butler.get(
"calexpBackground", dataId, immediate=
True)
450 image = cache.exposure.getMaskedImage()
454 statsImage = bgData[0].getStatsImage()
457 image -= bgOld.getImage()
458 cache.bgList = afwMath.BackgroundList()
460 cache.bgList.append(bgData)
462 if self.config.doMaskObjects:
463 self.maskObjects.findObjects(cache.exposure)
468 """Serial implementation of self.loadImage() for Gen3. 470 Load and restore background to calExp and calExpBkg. 474 calExp : `lsst.afw.image.Exposure` 475 Detector level calExp image to process. 476 calExpBkg : `lsst.afw.math.BackgroundList` 477 Detector level background list associated with the calExp. 481 calExp : `lsst.afw.image.Exposure` 482 Background restored calExp. 483 bgList : `lsst.afw.math.BackgroundList` 484 New background list containing the restoration background. 486 image = calExp.getMaskedImage()
488 for bgOld
in calExpBkg:
489 statsImage = bgOld[0].getStatsImage()
492 image -= calExpBkg.getImage()
493 bgList = afwMath.BackgroundList()
494 for bgData
in calExpBkg:
495 bgList.append(bgData)
497 if self.config.doMaskObjects:
498 self.maskObjects.findObjects(calExp)
500 return (calExp, bgList)
503 """Measure scale for sky frame 505 This method runs on the slave nodes. 509 cache : `lsst.pipe.base.Struct` 519 assert cache.dataId == dataId
520 cache.sky = self.sky.getSkyData(cache.butler, dataId)
521 scale = self.sky.measureScale(cache.exposure.getMaskedImage(), cache.sky)
525 """Subtract sky frame 527 This method runs on the slave nodes. 531 cache : `lsst.pipe.base.Struct` 540 exposure : `lsst.afw.image.Exposure` 543 assert cache.dataId == dataId
544 self.sky.
subtractSkyFrame(cache.exposure.getMaskedImage(), cache.sky, scale, cache.bgList)
548 """Fit background model for CCD 550 This method runs on the slave nodes. 554 cache : `lsst.pipe.base.Struct` 556 data : `lsst.pipe.base.Struct` 557 Data identifier, with `dataId` (data identifier) and `bgModel` 558 (background model) elements. 562 bgModel : `lsst.pipe.drivers.background.FocalPlaneBackground` 565 assert cache.dataId == data.dataId
566 data.bgModel.addCcd(cache.exposure)
570 """Subtract background model 572 This method runs on the slave nodes. 576 cache : `lsst.pipe.base.Struct` 580 bgModel : `lsst.pipe.drivers.background.FocalPlaneBackround` 585 exposure : `lsst.afw.image.Exposure` 588 assert cache.dataId == dataId
589 exposure = cache.exposure
590 image = exposure.getMaskedImage()
591 detector = exposure.getDetector()
592 bbox = image.getBBox()
593 cache.bgModel = bgModel.toCcdBackground(detector, bbox)
594 image -= cache.bgModel.getImage()
595 cache.bgList.append(cache.bgModel[0])
599 """Serial implementation of self.subtractModel() for Gen3. 601 Load and restore background to calExp and calExpBkg. 605 exposure : `lsst.afw.image.Exposure` 606 Exposure to subtract the background model from. 607 bgModel : `lsst.pipe.drivers.background.FocalPlaneBackground` 608 Full camera level background model. 612 exposure : `lsst.afw.image.Exposure` 613 Background subtracted input exposure. 614 bgModelCcd : `lsst.afw.math.BackgroundList` 615 Detector level realization of the full background model. 616 bgModelMaskedImage : `lsst.afw.image.MaskedImage` 617 Background model from the bgModelCcd realization. 619 image = exposure.getMaskedImage()
620 detector = exposure.getDetector()
621 bbox = image.getBBox()
622 bgModelCcd = bgModel.toCcdBackground(detector, bbox)
623 image -= bgModelCcd.getImage()
625 return (exposure, bgModelCcd, bgModelCcd[0])
628 """Generate an image of the background model for visualisation 630 Useful for debugging. 634 cache : `lsst.pipe.base.Struct` 638 bgModel : `lsst.pipe.drivers.background.FocalPlaneBackround` 645 image : `lsst.afw.image.MaskedImage` 646 Binned background model image. 648 assert cache.dataId == dataId
649 exposure = cache.exposure
650 detector = exposure.getDetector()
651 bbox = exposure.getMaskedImage().getBBox()
652 image = bgModel.toCcdBackground(detector, bbox).getImage()
656 """Return the binned image required for visualization 658 This method just helps to cut down on boilerplate. 662 image : `lsst.afw.image.MaskedImage` 663 Image to go into visualisation. 669 image : `lsst.afw.image.MaskedImage` 672 return (exposure.getDetector().getId(), afwMath.binImage(image, self.config.binning))
675 """Collect exposure for potential visualisation 677 This method runs on the slave nodes. 681 cache : `lsst.pipe.base.Struct` 688 image : `lsst.afw.image.MaskedImage` 694 """Collect original image for visualisation 696 This method runs on the slave nodes. 700 cache : `lsst.pipe.base.Struct` 709 image : `lsst.afw.image.MaskedImage` 712 exposure = cache.butler.get(
"calexp", dataId, immediate=
True)
716 """Collect original image for visualisation 718 This method runs on the slave nodes. 722 cache : `lsst.pipe.base.Struct` 731 image : `lsst.afw.image.MaskedImage` 737 """Collect mask for visualisation 739 This method runs on the slave nodes. 743 cache : `lsst.pipe.base.Struct` 752 image : `lsst.afw.image.Image` 756 image = afwImage.ImageF(cache.exposure.maskedImage.getBBox())
757 image.array[:] = cache.exposure.maskedImage.mask.array
761 """Write resultant background list 763 This method runs on the slave nodes. 767 cache : `lsst.pipe.base.Struct` 772 cache.butler.put(cache.bgList,
"skyCorr", dataId)
774 def _getMetadataName(self):
775 """There's no metadata to write out"""
def subtractModel(self, cache, dataId, bgModel)
def batchWallTime(cls, time, parsedCmd, numCores)
def runDataRef(self, expRef)
def makeCameraImage(camera, exposures, filename=None, binning=8)
def focalPlaneBackground(self, camera, pool, dataIdList, config)
def run(self, calExpArray, calBkgArray, skyCalibs, camera)
def accumulateModel(self, cache, data)
def subtractModelRun(self, exposure, bgModel)
def loadImage(self, cache, dataId)
def collectBinnedImage(self, exposure, image)
def realiseModel(self, cache, dataId, bgModel)
def collectOriginal(self, cache, dataId)
def focalPlaneBackgroundRun(self, camera, cacheExposures, idList, config)
def subtractSkyFrame(self, cache, dataId, scale)
def write(self, cache, dataId)
def runQuantum(self, butlerQC, inputRefs, outputRefs)
def logOperation(self, operation, catch=False, trace=True)
def loadImageRun(self, calExp, calExpBkg)
def __init__(self, args, kwargs)
def collectSky(self, cache, dataId)
def collectMask(self, cache, dataId)
def measureSkyFrame(self, cache, dataId)