26 from lsst.pex.config
import Config, Field, ConfigurableField, ConfigField
30 FocalPlaneBackgroundConfig, MaskObjectsTask)
34 __all__ = [
"SkyCorrectionConfig",
"SkyCorrectionTask"]
40 """Make and write an image of an entire focal plane
44 camera : `lsst.afw.cameraGeom.Camera`
46 exposures : `list` of `tuple` of `int` and `lsst.afw.image.Exposure`
47 List of detector ID and CCD exposures (binned by `binning`).
48 filename : `str`, optional
51 Binning size that has been applied to images.
53 image = visualizeVisit.makeCameraImage(camera, dict(exp
for exp
in exposures
if exp
is not None), binning)
54 if filename
is not None:
55 image.writeFits(filename)
61 doc=
"Raw data to provide exp-visit linkage to connect calExp inputs to camera/sky calibs.",
65 storageClass=
"ExposureU",
66 dimensions=[
"instrument",
"exposure",
"detector"],
68 calExpArray = cT.Input(
69 doc=
"Input exposures to process",
72 storageClass=
"ExposureF",
73 dimensions=[
"instrument",
"visit",
"detector"],
75 calBkgArray = cT.Input(
76 doc=
"Input background files to use",
78 name=
"calexpBackground",
79 storageClass=
"Background",
80 dimensions=[
"instrument",
"visit",
"detector"],
82 camera = cT.PrerequisiteInput(
83 doc=
"Input camera to use.",
85 storageClass=
"Camera",
86 dimensions=[
"instrument",
"calibration_label"],
88 skyCalibs = cT.PrerequisiteInput(
89 doc=
"Input sky calibrations to use.",
92 storageClass=
"ExposureF",
93 dimensions=[
"instrument",
"physical_filter",
"detector",
"calibration_label"],
95 calExpCamera = cT.Output(
96 doc=
"Output camera image.",
98 storageClass=
"ImageF",
99 dimensions=[
"instrument",
"visit"],
102 doc=
"Output sky corrected images.",
105 storageClass=
"Background",
106 dimensions=[
"instrument",
"visit",
"detector"],
111 """Configuration for SkyCorrectionTask"""
112 bgModel = ConfigField(dtype=FocalPlaneBackgroundConfig, doc=
"Background model")
113 bgModel2 = ConfigField(dtype=FocalPlaneBackgroundConfig, doc=
"2nd Background model")
114 sky = ConfigurableField(target=SkyMeasurementTask, doc=
"Sky measurement")
115 maskObjects = ConfigurableField(target=MaskObjectsTask, doc=
"Mask Objects")
116 doMaskObjects = Field(dtype=bool, default=
True, doc=
"Mask objects to find good sky?")
117 doBgModel = Field(dtype=bool, default=
True, doc=
"Do background model subtraction?")
118 doBgModel2 = Field(dtype=bool, default=
True, doc=
"Do cleanup background model subtraction?")
119 doSky = Field(dtype=bool, default=
True, doc=
"Do sky frame subtraction?")
120 binning = Field(dtype=int, default=8, doc=
"Binning factor for constructing focal-plane images")
121 calexpType = Field(dtype=str, default=
"calexp",
122 doc=
"Should be set to fakes_calexp if you want to process calexps with fakes in.")
125 Config.setDefaults(self)
134 """Correct sky over entire focal plane"""
135 ConfigClass = SkyCorrectionConfig
136 _DefaultName =
"skyCorr"
139 inputs = butlerQC.get(inputRefs)
140 inputs.pop(
"rawLinker",
None)
141 outputs = self.
run(**inputs)
142 butlerQC.put(outputs, outputRefs)
147 self.makeSubtask(
"sky")
148 self.makeSubtask(
"maskObjects")
151 def _makeArgumentParser(cls, *args, **kwargs):
152 kwargs.pop(
"doBatch",
False)
153 datasetType = ConfigDatasetType(name=
"calexpType")
154 parser = ArgumentParser(name=
"skyCorr", *args, **kwargs)
155 parser.add_id_argument(
"--id", datasetType=datasetType, level=
"visit",
156 help=
"data ID, e.g. --id visit=12345")
161 """Return walltime request for batch job
163 Subclasses should override if the walltime should be calculated
164 differently (e.g., addition of some serial time).
169 Requested time per iteration.
170 parsedCmd : `argparse.Namespace`
171 Results of argument parsing.
175 numTargets = len(cls.RunnerClass.getTargetList(parsedCmd))
176 return time*numTargets
179 """Perform sky correction on an exposure
181 We restore the original sky, and remove it again using multiple
182 algorithms. We optionally apply:
184 1. A large-scale background model.
185 This step removes very-large-scale sky such as moonlight.
187 3. A medium-scale background model.
188 This step removes residual sky (This is smooth on the focal plane).
190 Only the master node executes this method. The data is held on
191 the slave nodes, which do all the hard work.
195 expRef : `lsst.daf.persistence.ButlerDataRef`
196 Data reference for exposure.
200 ~lsst.pipe.drivers.SkyCorrectionTask.run
203 extension =
"-%(visit)d.fits" % expRef.dataId
205 with self.
logOperation(
"processing %s" % (expRef.dataId,)):
208 pool.storeSet(butler=expRef.getButler())
209 camera = expRef.get(
"camera")
211 dataIdList = [ccdRef.dataId
for ccdRef
in expRef.subItems(
"ccd")
if
212 ccdRef.datasetExists(self.config.calexpType)]
214 exposures = pool.map(self.
loadImage, dataIdList)
219 exposures = pool.mapToPrevious(self.
collectMask, dataIdList)
222 if self.config.doBgModel:
225 if self.config.doSky:
227 scale = self.sky.solveScales(measScales)
228 self.log.info(
"Sky frame scale: %s" % (scale,))
233 calibs = pool.mapToPrevious(self.
collectSky, dataIdList)
236 if self.config.doBgModel2:
241 expRef.put(image,
"calexp_camera")
243 pool.mapToPrevious(self.
write, dataIdList)
246 """Perform full focal-plane background subtraction
248 This method runs on the master node.
252 camera : `lsst.afw.cameraGeom.Camera`
254 pool : `lsst.ctrl.pool.Pool`
256 dataIdList : iterable of `dict`
257 List of data identifiers for the CCDs.
258 config : `lsst.pipe.drivers.background.FocalPlaneBackgroundConfig`
259 Configuration to use for background subtraction.
263 exposures : `list` of `lsst.afw.image.Image`
264 List of binned images, for creating focal plane image.
266 bgModel = FocalPlaneBackground.fromCamera(config, camera)
267 data = [pipeBase.Struct(dataId=dataId, bgModel=bgModel.clone())
for dataId
in dataIdList]
269 for ii, bg
in enumerate(bgModelList):
270 self.log.info(
"Background %d: %d pixels", ii, bg._numbers.array.sum())
272 return pool.mapToPrevious(self.
subtractModel, dataIdList, bgModel)
275 """Perform full focal-plane background subtraction
277 This method runs on the master node.
281 camera : `lsst.afw.cameraGeom.Camera`
283 cacheExposures : `list` of `lsst.afw.image.Exposures`
284 List of loaded and processed input calExp.
285 idList : `list` of `int`
286 List of detector ids to iterate over.
287 config : `lsst.pipe.drivers.background.FocalPlaneBackgroundConfig`
288 Configuration to use for background subtraction.
292 exposures : `list` of `lsst.afw.image.Image`
293 List of binned images, for creating focal plane image.
294 newCacheBgList : `list` of `lsst.afwMath.backgroundList`
295 Background lists generated.
296 cacheBgModel : `FocalPlaneBackground`
297 Full focal plane background model.
299 bgModel = FocalPlaneBackground.fromCamera(config, camera)
300 data = [pipeBase.Struct(id=id, bgModel=bgModel.clone())
for id
in idList]
303 for nodeData, cacheExp
in zip(data, cacheExposures):
304 nodeData.bgModel.addCcd(cacheExp)
305 bgModelList.append(nodeData.bgModel)
307 for ii, bg
in enumerate(bgModelList):
308 self.log.info(
"Background %d: %d pixels", ii, bg._numbers.getArray().sum())
314 for cacheExp
in cacheExposures:
316 exposures.append(afwMath.binImage(nodeExp.getMaskedImage(), self.config.binning))
317 cacheBgModel.append(nodeBgModel)
318 newCacheBgList.append(nodeBgList)
320 return exposures, newCacheBgList, cacheBgModel
322 def run(self, calExpArray, calBkgArray, skyCalibs, camera):
323 """Duplicate runDataRef method without ctrl_pool for Gen3.
327 calExpArray : `list` of `lsst.afw.image.Exposure`
328 Array of detector input calExp images for the exposure to
330 calBkgArray : `list` of `lsst.afw.math.BackgroundList`
331 Array of detector input background lists matching the
333 skyCalibs : `list` of `lsst.afw.image.Exposure`
334 Array of SKY calibrations for the input detectors to be
336 camera : `lsst.afw.cameraGeom.Camera`
337 Camera matching the input data to process.
341 results : `pipeBase.Struct` containing
342 calExpCamera : `lsst.afw.image.Exposure`
343 Full camera image of the sky-corrected data.
344 skyCorr : `list` of `lsst.afw.math.BackgroundList`
345 Detector-level sky-corrected background lists.
349 ~lsst.pipe.drivers.SkyCorrectionTask.runDataRef()
366 idList = [exp.getDetector().getId()
for exp
in calExpArray]
373 for calExp, calBgModel
in zip(calExpArray, calBkgArray):
374 nodeExp, nodeBgList = self.
loadImageRun(calExp, calBgModel)
375 cacheExposures.append(nodeExp)
376 cacheBgList.append(nodeBgList)
377 exposures.append(afwMath.binImage(nodeExp.getMaskedImage(), self.config.binning))
379 if self.config.doBgModel:
382 camera, cacheExposures, idList, self.config.bgModel
384 for cacheBg, newBg
in zip(cacheBgList, newCacheBgList):
385 cacheBg.append(newBg)
387 if self.config.doSky:
393 for cacheExp, skyCalib
in zip(cacheExposures, skyCalibs):
394 skyExp = self.sky.exposureToBackground(skyCalib)
395 cacheSky.append(skyExp)
396 scale = self.sky.measureScale(cacheExp.getMaskedImage(), skyExp)
397 measScales.append(scale)
399 scale = self.sky.solveScales(measScales)
400 self.log.info(
"Sky frame scale: %s" % (scale, ))
406 for cacheExp, nodeSky, nodeBgList
in zip(cacheExposures, cacheSky, cacheBgList):
407 self.sky.
subtractSkyFrame(cacheExp.getMaskedImage(), nodeSky, scale, nodeBgList)
408 exposures.append(afwMath.binImage(cacheExp.getMaskedImage(), self.config.binning))
410 if self.config.doBgModel2:
414 camera, cacheExposures, idList, self.config.bgModel2
416 for cacheBg, newBg
in zip(cacheBgList, newBgList):
417 cacheBg.append(newBg)
423 return pipeBase.Struct(
429 """Load original image and restore the sky
431 This method runs on the slave nodes.
435 cache : `lsst.pipe.base.Struct`
442 exposure : `lsst.afw.image.Exposure`
445 cache.dataId = dataId
446 cache.exposure = cache.butler.get(self.config.calexpType, dataId, immediate=
True).clone()
447 bgOld = cache.butler.get(
"calexpBackground", dataId, immediate=
True)
448 image = cache.exposure.getMaskedImage()
452 statsImage = bgData[0].getStatsImage()
455 image -= bgOld.getImage()
456 cache.bgList = afwMath.BackgroundList()
458 cache.bgList.append(bgData)
460 if self.config.doMaskObjects:
461 self.maskObjects.findObjects(cache.exposure)
466 """Serial implementation of self.loadImage() for Gen3.
468 Load and restore background to calExp and calExpBkg.
472 calExp : `lsst.afw.image.Exposure`
473 Detector level calExp image to process.
474 calExpBkg : `lsst.afw.math.BackgroundList`
475 Detector level background list associated with the calExp.
479 calExp : `lsst.afw.image.Exposure`
480 Background restored calExp.
481 bgList : `lsst.afw.math.BackgroundList`
482 New background list containing the restoration background.
484 image = calExp.getMaskedImage()
486 for bgOld
in calExpBkg:
487 statsImage = bgOld[0].getStatsImage()
490 image -= calExpBkg.getImage()
491 bgList = afwMath.BackgroundList()
492 for bgData
in calExpBkg:
493 bgList.append(bgData)
495 if self.config.doMaskObjects:
496 self.maskObjects.findObjects(calExp)
498 return (calExp, bgList)
501 """Measure scale for sky frame
503 This method runs on the slave nodes.
507 cache : `lsst.pipe.base.Struct`
517 assert cache.dataId == dataId
518 cache.sky = self.sky.getSkyData(cache.butler, dataId)
519 scale = self.sky.measureScale(cache.exposure.getMaskedImage(), cache.sky)
523 """Subtract sky frame
525 This method runs on the slave nodes.
529 cache : `lsst.pipe.base.Struct`
538 exposure : `lsst.afw.image.Exposure`
541 assert cache.dataId == dataId
542 self.sky.
subtractSkyFrame(cache.exposure.getMaskedImage(), cache.sky, scale, cache.bgList)
546 """Fit background model for CCD
548 This method runs on the slave nodes.
552 cache : `lsst.pipe.base.Struct`
554 data : `lsst.pipe.base.Struct`
555 Data identifier, with `dataId` (data identifier) and `bgModel`
556 (background model) elements.
560 bgModel : `lsst.pipe.drivers.background.FocalPlaneBackground`
563 assert cache.dataId == data.dataId
564 data.bgModel.addCcd(cache.exposure)
568 """Subtract background model
570 This method runs on the slave nodes.
574 cache : `lsst.pipe.base.Struct`
578 bgModel : `lsst.pipe.drivers.background.FocalPlaneBackround`
583 exposure : `lsst.afw.image.Exposure`
586 assert cache.dataId == dataId
587 exposure = cache.exposure
588 image = exposure.getMaskedImage()
589 detector = exposure.getDetector()
590 bbox = image.getBBox()
591 cache.bgModel = bgModel.toCcdBackground(detector, bbox)
592 image -= cache.bgModel.getImage()
593 cache.bgList.append(cache.bgModel[0])
597 """Serial implementation of self.subtractModel() for Gen3.
599 Load and restore background to calExp and calExpBkg.
603 exposure : `lsst.afw.image.Exposure`
604 Exposure to subtract the background model from.
605 bgModel : `lsst.pipe.drivers.background.FocalPlaneBackground`
606 Full camera level background model.
610 exposure : `lsst.afw.image.Exposure`
611 Background subtracted input exposure.
612 bgModelCcd : `lsst.afw.math.BackgroundList`
613 Detector level realization of the full background model.
614 bgModelMaskedImage : `lsst.afw.image.MaskedImage`
615 Background model from the bgModelCcd realization.
617 image = exposure.getMaskedImage()
618 detector = exposure.getDetector()
619 bbox = image.getBBox()
620 bgModelCcd = bgModel.toCcdBackground(detector, bbox)
621 image -= bgModelCcd.getImage()
623 return (exposure, bgModelCcd, bgModelCcd[0])
626 """Generate an image of the background model for visualisation
628 Useful for debugging.
632 cache : `lsst.pipe.base.Struct`
636 bgModel : `lsst.pipe.drivers.background.FocalPlaneBackround`
643 image : `lsst.afw.image.MaskedImage`
644 Binned background model image.
646 assert cache.dataId == dataId
647 exposure = cache.exposure
648 detector = exposure.getDetector()
649 bbox = exposure.getMaskedImage().getBBox()
650 image = bgModel.toCcdBackground(detector, bbox).getImage()
654 """Return the binned image required for visualization
656 This method just helps to cut down on boilerplate.
660 image : `lsst.afw.image.MaskedImage`
661 Image to go into visualisation.
667 image : `lsst.afw.image.MaskedImage`
670 return (exposure.getDetector().getId(), afwMath.binImage(image, self.config.binning))
673 """Collect exposure for potential visualisation
675 This method runs on the slave nodes.
679 cache : `lsst.pipe.base.Struct`
686 image : `lsst.afw.image.MaskedImage`
692 """Collect original image for visualisation
694 This method runs on the slave nodes.
698 cache : `lsst.pipe.base.Struct`
707 image : `lsst.afw.image.MaskedImage`
710 exposure = cache.butler.get(
"calexp", dataId, immediate=
True)
714 """Collect original image for visualisation
716 This method runs on the slave nodes.
720 cache : `lsst.pipe.base.Struct`
729 image : `lsst.afw.image.MaskedImage`
735 """Collect mask for visualisation
737 This method runs on the slave nodes.
741 cache : `lsst.pipe.base.Struct`
750 image : `lsst.afw.image.Image`
754 image = afwImage.ImageF(cache.exposure.maskedImage.getBBox())
755 image.array[:] = cache.exposure.maskedImage.mask.array
759 """Write resultant background list
761 This method runs on the slave nodes.
765 cache : `lsst.pipe.base.Struct`
770 cache.butler.put(cache.bgList,
"skyCorr", dataId)
772 def _getMetadataName(self):
773 """There's no metadata to write out"""