lsst.pipe.drivers g96f01af41f+7dc01da196
skyCorrection.py
Go to the documentation of this file.
1# This file is part of pipe_drivers.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21
22from deprecated.sphinx import deprecated
23
24import lsst.afw.math as afwMath
25import lsst.afw.image as afwImage
26import lsst.pipe.base as pipeBase
27
28from lsst.daf.butler import DimensionGraph
29from lsst.pex.config import Config, Field, ConfigurableField, ConfigField
30from lsst.pipe.drivers.background import (SkyMeasurementTask, FocalPlaneBackground,
31 FocalPlaneBackgroundConfig, MaskObjectsTask)
32import lsst.pipe.drivers.visualizeVisit as visualizeVisit
33import lsst.pipe.base.connectionTypes as cT
34
35try:
36 from lsst.pipe.base import ArgumentParser
37 from lsst.pipe.base import ConfigDatasetType
38
39except ImportError:
40 from argparse import ArgumentParser
41
43 def __init__(**kwargs):
44 raise NotImplementedError()
45
46try:
47 from lsst.ctrl.pool.pool import Pool
48except ImportError:
49 class Pool:
50 pass
51
52try:
53 from lsst.ctrl.pool.parallel import BatchPoolTask
54except ImportError:
55 from lsst.pipe.base import Task as BatchPoolTask
56
57__all__ = ["SkyCorrectionConfig", "SkyCorrectionTask"]
58
59DEBUG = False # Debugging outputs?
60
61
62def reorderAndPadList(inputList, inputKeys, outputKeys, padWith=None):
63 """Match the order of one list to another, padding if necessary
64
65 Parameters
66 ----------
67 inputList : list
68 List to be reordered and padded. Elements can be any type.
69 inputKeys : iterable
70 Iterable of values to be compared with outputKeys.
71 Length must match `inputList`
72 outputKeys : iterable
73 Iterable of values to be compared with inputKeys.
74 padWith :
75 Any value to be inserted where inputKey not in outputKeys
76
77 Returns
78 -------
79 list
80 Copy of inputList reordered per outputKeys and padded with `padWith`
81 so that the length matches length of outputKeys.
82 """
83 outputList = []
84 for d in outputKeys:
85 if d in inputKeys:
86 outputList.append(inputList[inputKeys.index(d)])
87 else:
88 outputList.append(padWith)
89 return outputList
90
91
92def makeCameraImage(camera, exposures, filename=None, binning=8):
93 """Make and write an image of an entire focal plane
94
95 Parameters
96 ----------
98 Camera description.
99 exposures : `list` of `tuple` of `int` and `lsst.afw.image.Exposure`
100 List of detector ID and CCD exposures (binned by `binning`).
101 filename : `str`, optional
102 Output filename.
103 binning : `int`
104 Binning size that has been applied to images.
105 """
106 image = visualizeVisit.makeCameraImage(camera, dict(exp for exp in exposures if exp is not None), binning)
107 if filename is not None:
108 image.writeFits(filename)
109 return image
110
111
112def _skyLookup(datasetType, registry, quantumDataId, collections):
113 """Lookup function to identify sky frames
114
115 Parameters
116 ----------
117 datasetType : `lsst.daf.butler.DatasetType`
118 Dataset to lookup.
119 registry : `lsst.daf.butler.Registry`
120 Butler registry to query.
121 quantumDataId : `lsst.daf.butler.DataCoordinate`
122 Data id to transform to find sky frames.
123 The ``detector`` entry will be stripped.
124 collections : `lsst.daf.butler.CollectionSearch`
125 Collections to search through.
126
127 Returns
128 -------
129 results : `list` [`lsst.daf.butler.DatasetRef`]
130 List of datasets that will be used as sky calibration frames
131 """
132 newDataId = quantumDataId.subset(DimensionGraph(registry.dimensions, names=["instrument", "visit"]))
133 skyFrames = []
134 for dataId in registry.queryDataIds(["visit", "detector"], dataId=newDataId).expanded():
135 skyFrame = registry.findDataset(datasetType, dataId, collections=collections,
136 timespan=dataId.timespan)
137 skyFrames.append(skyFrame)
138
139 return skyFrames
140
141
142class SkyCorrectionConnections(pipeBase.PipelineTaskConnections, dimensions=("instrument", "visit")):
143 rawLinker = cT.Input(
144 doc="Raw data to provide exp-visit linkage to connect calExp inputs to camera/sky calibs.",
145 name="raw",
146 multiple=True,
147 deferLoad=True,
148 storageClass="Exposure",
149 dimensions=["instrument", "exposure", "detector"],
150 )
151 calExpArray = cT.Input(
152 doc="Input exposures to process",
153 name="calexp",
154 multiple=True,
155 storageClass="ExposureF",
156 dimensions=["instrument", "visit", "detector"],
157 )
158 calBkgArray = cT.Input(
159 doc="Input background files to use",
160 multiple=True,
161 name="calexpBackground",
162 storageClass="Background",
163 dimensions=["instrument", "visit", "detector"],
164 )
165 camera = cT.PrerequisiteInput(
166 doc="Input camera to use.",
167 name="camera",
168 storageClass="Camera",
169 dimensions=["instrument"],
170 isCalibration=True,
171 )
172 skyCalibs = cT.PrerequisiteInput(
173 doc="Input sky calibrations to use.",
174 name="sky",
175 multiple=True,
176 storageClass="ExposureF",
177 dimensions=["instrument", "physical_filter", "detector"],
178 isCalibration=True,
179 lookupFunction=_skyLookup,
180 )
181 calExpCamera = cT.Output(
182 doc="Output camera image.",
183 name='calexp_camera',
184 storageClass="ImageF",
185 dimensions=["instrument", "visit"],
186 )
187 skyCorr = cT.Output(
188 doc="Output sky corrected images.",
189 name='skyCorr',
190 multiple=True,
191 storageClass="Background",
192 dimensions=["instrument", "visit", "detector"],
193 )
194
195
196class SkyCorrectionConfig(pipeBase.PipelineTaskConfig, pipelineConnections=SkyCorrectionConnections):
197 """Configuration for SkyCorrectionTask"""
198 bgModel = ConfigField(dtype=FocalPlaneBackgroundConfig, doc="Background model")
199 bgModel2 = ConfigField(dtype=FocalPlaneBackgroundConfig, doc="2nd Background model")
200 sky = ConfigurableField(target=SkyMeasurementTask, doc="Sky measurement")
201 maskObjects = ConfigurableField(target=MaskObjectsTask, doc="Mask Objects")
202 doMaskObjects = Field(dtype=bool, default=True, doc="Mask objects to find good sky?")
203 doBgModel = Field(dtype=bool, default=True, doc="Do background model subtraction?")
204 doBgModel2 = Field(dtype=bool, default=True, doc="Do cleanup background model subtraction?")
205 doSky = Field(dtype=bool, default=True, doc="Do sky frame subtraction?")
206 binning = Field(dtype=int, default=8, doc="Binning factor for constructing focal-plane images")
207 calexpType = Field(dtype=str, default="calexp",
208 doc="Should be set to fakes_calexp if you want to process calexps with fakes in.")
209
210 def setDefaults(self):
211 Config.setDefaults(self)
212 self.bgModel2.doSmooth = True
213 self.bgModel2.minFrac = 0.5
214 self.bgModel2.xSize = 256
215 self.bgModel2.ySize = 256
216 self.bgModel2.smoothScale = 1.0
217
218
219@deprecated(
220 reason="pipe_drivers is deprecated. It will be removed after v25. "
221 "Please use lsst.pipe.tasks.skyCorrection.SkyCorrectionTask instead.",
222 version="v25.0",
223 category=FutureWarning,
224)
225class SkyCorrectionTask(pipeBase.PipelineTask, BatchPoolTask):
226 """Correct sky over entire focal plane"""
227 ConfigClass = SkyCorrectionConfig
228 _DefaultName = "skyCorr"
229
230 def runQuantum(self, butlerQC, inputRefs, outputRefs):
231
232 # Reorder the skyCalibs, calBkgArray, and calExpArray inputRefs and the
233 # skyCorr outputRef sorted by detector id to ensure reproducibility.
234 detectorOrder = [ref.dataId['detector'] for ref in inputRefs.calExpArray]
235 detectorOrder.sort()
236 inputRefs.calExpArray = reorderAndPadList(inputRefs.calExpArray,
237 [ref.dataId['detector'] for ref in inputRefs.calExpArray],
238 detectorOrder)
239 inputRefs.skyCalibs = reorderAndPadList(inputRefs.skyCalibs,
240 [ref.dataId['detector'] for ref in inputRefs.skyCalibs],
241 detectorOrder)
242 inputRefs.calBkgArray = reorderAndPadList(inputRefs.calBkgArray,
243 [ref.dataId['detector'] for ref in inputRefs.calBkgArray],
244 detectorOrder)
245 outputRefs.skyCorr = reorderAndPadList(outputRefs.skyCorr,
246 [ref.dataId['detector'] for ref in outputRefs.skyCorr],
247 detectorOrder)
248 inputs = butlerQC.get(inputRefs)
249 inputs.pop("rawLinker", None)
250 outputs = self.run(**inputs)
251 butlerQC.put(outputs, outputRefs)
252
253 def __init__(self, *args, **kwargs):
254 super().__init__(**kwargs)
255
256 self.makeSubtask("sky")
257 self.makeSubtask("maskObjects")
258
259 @classmethod
260 def _makeArgumentParser(cls, *args, **kwargs):
261 kwargs.pop("doBatch", False)
262 datasetType = ConfigDatasetType(name="calexpType")
263 parser = ArgumentParser(name="skyCorr", *args, **kwargs)
264 parser.add_id_argument("--id", datasetType=datasetType, level="visit",
265 help="data ID, e.g. --id visit=12345")
266 return parser
267
268 @classmethod
269 def batchWallTime(cls, time, parsedCmd, numCores):
270 """Return walltime request for batch job
271
272 Subclasses should override if the walltime should be calculated
273 differently (e.g., addition of some serial time).
274
275 Parameters
276 ----------
277 time : `float`
278 Requested time per iteration.
279 parsedCmd : `argparse.Namespace`
280 Results of argument parsing.
281 numCores : `int`
282 Number of cores.
283 """
284 numTargets = len(cls.RunnerClass.getTargetList(parsedCmd))
285 return time*numTargets
286
287 def runDataRef(self, expRef):
288 """Perform sky correction on an exposure
289
290 We restore the original sky, and remove it again using multiple
291 algorithms. We optionally apply:
292
293 1. A large-scale background model.
294 This step removes very-large-scale sky such as moonlight.
295 2. A sky frame.
296 3. A medium-scale background model.
297 This step removes residual sky (This is smooth on the focal plane).
298
299 Only the master node executes this method. The data is held on
300 the slave nodes, which do all the hard work.
301
302 Parameters
303 ----------
304 expRef : `lsst.daf.persistence.ButlerDataRef`
305 Data reference for exposure.
306
307 See Also
308 --------
309 ~lsst.pipe.drivers.SkyCorrectionTask.run
310 """
311 if DEBUG:
312 extension = "-%(visit)d.fits" % expRef.dataId
313
314 with self.logOperation("processing %s" % (expRef.dataId,)):
315 pool = Pool()
316 pool.cacheClear()
317 pool.storeSet(butler=expRef.getButler())
318 camera = expRef.get("camera")
319
320 dataIdList = [ccdRef.dataId for ccdRef in expRef.subItems("ccd") if
321 ccdRef.datasetExists(self.config.calexpType)]
322 ccdList = [dataId["ccd"] for dataId in dataIdList]
323 # Order inputs sorted by detector id to ensure reproducibility
324 dataIdList = [dataId for _, dataId in sorted(zip(ccdList, dataIdList))]
325
326 exposures = pool.map(self.loadImage, dataIdList)
327 if DEBUG:
328 makeCameraImage(camera, exposures, "restored" + extension)
329 exposures = pool.mapToPrevious(self.collectOriginal, dataIdList)
330 makeCameraImage(camera, exposures, "original" + extension)
331 exposures = pool.mapToPrevious(self.collectMask, dataIdList)
332 makeCameraImage(camera, exposures, "mask" + extension)
333
334 if self.config.doBgModel:
335 exposures = self.focalPlaneBackground(camera, pool, dataIdList, self.config.bgModel)
336
337 if self.config.doSky:
338 measScales = pool.mapToPrevious(self.measureSkyFrame, dataIdList)
339 scale = self.sky.solveScales(measScales)
340 self.log.info("Sky frame scale: %s" % (scale,))
341
342 exposures = pool.mapToPrevious(self.subtractSkyFrame, dataIdList, scale)
343 if DEBUG:
344 makeCameraImage(camera, exposures, "skysub" + extension)
345 calibs = pool.mapToPrevious(self.collectSky, dataIdList)
346 makeCameraImage(camera, calibs, "sky" + extension)
347
348 if self.config.doBgModel2:
349 exposures = self.focalPlaneBackground(camera, pool, dataIdList, self.config.bgModel2)
350
351 # Persist camera-level image of calexp
352 image = makeCameraImage(camera, exposures)
353 expRef.put(image, "calexp_camera")
354
355 pool.mapToPrevious(self.write, dataIdList)
356
357 def focalPlaneBackground(self, camera, pool, dataIdList, config):
358 """Perform full focal-plane background subtraction
359
360 This method runs on the master node.
361
362 Parameters
363 ----------
365 Camera description.
366 pool : `lsst.ctrl.pool.Pool`
367 Process pool.
368 dataIdList : iterable of `dict`
369 List of data identifiers for the CCDs.
371 Configuration to use for background subtraction.
372
373 Returns
374 -------
375 exposures : `list` of `lsst.afw.image.Image`
376 List of binned images, for creating focal plane image.
377 """
378 bgModel = FocalPlaneBackground.fromCamera(config, camera)
379 data = [pipeBase.Struct(dataId=dataId, bgModel=bgModel.clone()) for dataId in dataIdList]
380 bgModelList = pool.mapToPrevious(self.accumulateModel, data)
381 for ii, bg in enumerate(bgModelList):
382 self.log.info("Background %d: %d pixels", ii, bg._numbers.array.sum())
383 bgModel.merge(bg)
384 return pool.mapToPrevious(self.subtractModel, dataIdList, bgModel)
385
386 def focalPlaneBackgroundRun(self, camera, cacheExposures, idList, config):
387 """Perform full focal-plane background subtraction
388
389 This method runs on the master node.
390
391 Parameters
392 ----------
394 Camera description.
395 cacheExposures : `list` of `lsst.afw.image.Exposures`
396 List of loaded and processed input calExp.
397 idList : `list` of `int`
398 List of detector ids to iterate over.
400 Configuration to use for background subtraction.
401
402 Returns
403 -------
404 exposures : `list` of `lsst.afw.image.Image`
405 List of binned images, for creating focal plane image.
406 newCacheBgList : `list` of `lsst.afwMath.backgroundList`
407 Background lists generated.
408 cacheBgModel : `FocalPlaneBackground`
409 Full focal plane background model.
410 """
411 bgModel = FocalPlaneBackground.fromCamera(config, camera)
412 data = [pipeBase.Struct(id=id, bgModel=bgModel.clone()) for id in idList]
413
414 bgModelList = []
415 for nodeData, cacheExp in zip(data, cacheExposures):
416 nodeData.bgModel.addCcd(cacheExp)
417 bgModelList.append(nodeData.bgModel)
418
419 for ii, bg in enumerate(bgModelList):
420 self.log.info("Background %d: %d pixels", ii, bg._numbers.getArray().sum())
421 bgModel.merge(bg)
422
423 exposures = []
424 newCacheBgList = []
425 cacheBgModel = []
426 for cacheExp in cacheExposures:
427 nodeExp, nodeBgModel, nodeBgList = self.subtractModelRun(cacheExp, bgModel)
428 exposures.append(afwMath.binImage(nodeExp.getMaskedImage(), self.config.binning))
429 cacheBgModel.append(nodeBgModel)
430 newCacheBgList.append(nodeBgList)
431
432 return exposures, newCacheBgList, cacheBgModel
433
434 def run(self, calExpArray, calBkgArray, skyCalibs, camera):
435 """Duplicate runDataRef method without ctrl_pool for Gen3.
436
437 Parameters
438 ----------
439 calExpArray : `list` of `lsst.afw.image.Exposure`
440 Array of detector input calExp images for the exposure to
441 process.
442 calBkgArray : `list` of `lsst.afw.math.BackgroundList`
443 Array of detector input background lists matching the
444 calExps to process.
445 skyCalibs : `list` of `lsst.afw.image.Exposure`
446 Array of SKY calibrations for the input detectors to be
447 processed.
449 Camera matching the input data to process.
450
451 Returns
452 -------
453 results : `pipeBase.Struct` containing
454 calExpCamera : `lsst.afw.image.Exposure`
455 Full camera image of the sky-corrected data.
456 skyCorr : `list` of `lsst.afw.math.BackgroundList`
457 Detector-level sky-corrected background lists.
458
459 See Also
460 --------
461 ~lsst.pipe.drivers.SkyCorrectionTask.runDataRef()
462 """
463 # To allow SkyCorrectionTask to run in the Gen3 butler
464 # environment, a new run() method was added that performs the
465 # same operations in a serial environment (pipetask processing
466 # does not support MPI processing as of 2019-05-03). Methods
467 # used in runDataRef() are used as appropriate in run(), but
468 # some have been rewritten in serial form. Please ensure that
469 # any updates to runDataRef() or the methods it calls with
470 # pool.mapToPrevious() are duplicated in run() and its
471 # methods.
472 #
473 # Variable names here should match those in runDataRef() as
474 # closely as possible. Variables matching data stored in the
475 # pool cache have a prefix indicating this. Variables that
476 # would be local to an MPI processing client have a prefix
477 # "node".
478 idList = [exp.getDetector().getId() for exp in calExpArray]
479
480 # Construct arrays that match the cache in self.runDataRef() after
481 # self.loadImage() is map/reduced.
482 cacheExposures = []
483 cacheBgList = []
484 exposures = []
485 for calExp, calBgModel in zip(calExpArray, calBkgArray):
486 nodeExp, nodeBgList = self.loadImageRun(calExp, calBgModel)
487 cacheExposures.append(nodeExp)
488 cacheBgList.append(nodeBgList)
489 exposures.append(afwMath.binImage(nodeExp.getMaskedImage(), self.config.binning))
490
491 if self.config.doBgModel:
492 # Generate focal plane background, updating backgrounds in the "cache".
493 exposures, newCacheBgList, cacheBgModel = self.focalPlaneBackgroundRun(
494 camera, cacheExposures, idList, self.config.bgModel
495 )
496 for cacheBg, newBg in zip(cacheBgList, newCacheBgList):
497 cacheBg.append(newBg)
498
499 if self.config.doSky:
500 # Measure the sky frame scale on all inputs. Results in
501 # values equal to self.measureSkyFrame() and
502 # self.sky.solveScales() in runDataRef().
503 cacheSky = []
504 measScales = []
505 for cacheExp, skyCalib in zip(cacheExposures, skyCalibs):
506 skyExp = self.sky.exposureToBackground(skyCalib)
507 cacheSky.append(skyExp)
508 scale = self.sky.measureScale(cacheExp.getMaskedImage(), skyExp)
509 measScales.append(scale)
510
511 scale = self.sky.solveScales(measScales)
512 self.log.info("Sky frame scale: %s" % (scale, ))
513
514 # Subtract sky frame, as in self.subtractSkyFrame(), with
515 # appropriate scale from the "cache".
516 exposures = []
517 newBgList = []
518 for cacheExp, nodeSky, nodeBgList in zip(cacheExposures, cacheSky, cacheBgList):
519 self.sky.subtractSkyFrame(cacheExp.getMaskedImage(), nodeSky, scale, nodeBgList)
520 exposures.append(afwMath.binImage(cacheExp.getMaskedImage(), self.config.binning))
521
522 if self.config.doBgModel2:
523 # As above, generate a focal plane background model and
524 # update the cache models.
525 exposures, newBgList, cacheBgModel = self.focalPlaneBackgroundRun(
526 camera, cacheExposures, idList, self.config.bgModel2
527 )
528 for cacheBg, newBg in zip(cacheBgList, newBgList):
529 cacheBg.append(newBg)
530
531 # Generate camera-level image of calexp and return it along
532 # with the list of sky corrected background models.
533 image = makeCameraImage(camera, zip(idList, exposures))
534
535 return pipeBase.Struct(
536 calExpCamera=image,
537 skyCorr=cacheBgList,
538 )
539
540 def loadImage(self, cache, dataId):
541 """Load original image and restore the sky
542
543 This method runs on the slave nodes.
544
545 Parameters
546 ----------
547 cache : `lsst.pipe.base.Struct`
548 Process pool cache.
549 dataId : `dict`
550 Data identifier.
551
552 Returns
553 -------
554 exposure : `lsst.afw.image.Exposure`
555 Resultant exposure.
556 """
557 cache.dataId = dataId
558 cache.exposure = cache.butler.get(self.config.calexpType, dataId, immediate=True).clone()
559 bgOld = cache.butler.get("calexpBackground", dataId, immediate=True)
560 image = cache.exposure.getMaskedImage()
561
562 # We're removing the old background, so change the sense of all its components
563 for bgData in bgOld:
564 statsImage = bgData[0].getStatsImage()
565 statsImage *= -1
566
567 image -= bgOld.getImage()
568 cache.bgList = afwMath.BackgroundList()
569 for bgData in bgOld:
570 cache.bgList.append(bgData)
571
572 if self.config.doMaskObjects:
573 self.maskObjects.findObjects(cache.exposure)
574
575 return self.collect(cache)
576
577 def loadImageRun(self, calExp, calExpBkg):
578 """Serial implementation of self.loadImage() for Gen3.
579
580 Load and restore background to calExp and calExpBkg.
581
582 Parameters
583 ----------
584 calExp : `lsst.afw.image.Exposure`
585 Detector level calExp image to process.
586 calExpBkg : `lsst.afw.math.BackgroundList`
587 Detector level background list associated with the calExp.
588
589 Returns
590 -------
591 calExp : `lsst.afw.image.Exposure`
592 Background restored calExp.
593 bgList : `lsst.afw.math.BackgroundList`
594 New background list containing the restoration background.
595 """
596 image = calExp.getMaskedImage()
597
598 for bgOld in calExpBkg:
599 statsImage = bgOld[0].getStatsImage()
600 statsImage *= -1
601
602 image -= calExpBkg.getImage()
603 bgList = afwMath.BackgroundList()
604 for bgData in calExpBkg:
605 bgList.append(bgData)
606
607 if self.config.doMaskObjects:
608 self.maskObjects.findObjects(calExp)
609
610 return (calExp, bgList)
611
612 def measureSkyFrame(self, cache, dataId):
613 """Measure scale for sky frame
614
615 This method runs on the slave nodes.
616
617 Parameters
618 ----------
619 cache : `lsst.pipe.base.Struct`
620 Process pool cache.
621 dataId : `dict`
622 Data identifier.
623
624 Returns
625 -------
626 scale : `float`
627 Scale for sky frame.
628 """
629 assert cache.dataId == dataId
630 cache.sky = self.sky.getSkyData(cache.butler, dataId)
631 scale = self.sky.measureScale(cache.exposure.getMaskedImage(), cache.sky)
632 return scale
633
634 def subtractSkyFrame(self, cache, dataId, scale):
635 """Subtract sky frame
636
637 This method runs on the slave nodes.
638
639 Parameters
640 ----------
641 cache : `lsst.pipe.base.Struct`
642 Process pool cache.
643 dataId : `dict`
644 Data identifier.
645 scale : `float`
646 Scale for sky frame.
647
648 Returns
649 -------
650 exposure : `lsst.afw.image.Exposure`
651 Resultant exposure.
652 """
653 assert cache.dataId == dataId
654 self.sky.subtractSkyFrame(cache.exposure.getMaskedImage(), cache.sky, scale, cache.bgList)
655 return self.collect(cache)
656
657 def accumulateModel(self, cache, data):
658 """Fit background model for CCD
659
660 This method runs on the slave nodes.
661
662 Parameters
663 ----------
664 cache : `lsst.pipe.base.Struct`
665 Process pool cache.
666 data : `lsst.pipe.base.Struct`
667 Data identifier, with `dataId` (data identifier) and `bgModel`
668 (background model) elements.
669
670 Returns
671 -------
673 Background model.
674 """
675 assert cache.dataId == data.dataId
676 data.bgModel.addCcd(cache.exposure)
677 return data.bgModel
678
679 def subtractModel(self, cache, dataId, bgModel):
680 """Subtract background model
681
682 This method runs on the slave nodes.
683
684 Parameters
685 ----------
686 cache : `lsst.pipe.base.Struct`
687 Process pool cache.
688 dataId : `dict`
689 Data identifier.
690 bgModel : `lsst.pipe.drivers.background.FocalPlaneBackround`
691 Background model.
692
693 Returns
694 -------
695 exposure : `lsst.afw.image.Exposure`
696 Resultant exposure.
697 """
698 assert cache.dataId == dataId
699 exposure = cache.exposure
700 image = exposure.getMaskedImage()
701 detector = exposure.getDetector()
702 bbox = image.getBBox()
703 try:
704 cache.bgModel = bgModel.toCcdBackground(detector, bbox)
705 image -= cache.bgModel.getImage()
706 except RuntimeError:
707 self.log.error(f"There was an error processing {dataId}, no calib file produced")
708 return
709 cache.bgList.append(cache.bgModel[0])
710 return self.collect(cache)
711
712 def subtractModelRun(self, exposure, bgModel):
713 """Serial implementation of self.subtractModel() for Gen3.
714
715 Load and restore background to calExp and calExpBkg.
716
717 Parameters
718 ----------
719 exposure : `lsst.afw.image.Exposure`
720 Exposure to subtract the background model from.
722 Full camera level background model.
723
724 Returns
725 -------
726 exposure : `lsst.afw.image.Exposure`
727 Background subtracted input exposure.
728 bgModelCcd : `lsst.afw.math.BackgroundList`
729 Detector level realization of the full background model.
730 bgModelMaskedImage : `lsst.afw.image.MaskedImage`
731 Background model from the bgModelCcd realization.
732 """
733 image = exposure.getMaskedImage()
734 detector = exposure.getDetector()
735 bbox = image.getBBox()
736 bgModelCcd = bgModel.toCcdBackground(detector, bbox)
737 image -= bgModelCcd.getImage()
738
739 return (exposure, bgModelCcd, bgModelCcd[0])
740
741 def realiseModel(self, cache, dataId, bgModel):
742 """Generate an image of the background model for visualisation
743
744 Useful for debugging.
745
746 Parameters
747 ----------
748 cache : `lsst.pipe.base.Struct`
749 Process pool cache.
750 dataId : `dict`
751 Data identifier.
752 bgModel : `lsst.pipe.drivers.background.FocalPlaneBackround`
753 Background model.
754
755 Returns
756 -------
757 detId : `int`
758 Detector identifier.
760 Binned background model image.
761 """
762 assert cache.dataId == dataId
763 exposure = cache.exposure
764 detector = exposure.getDetector()
765 bbox = exposure.getMaskedImage().getBBox()
766 image = bgModel.toCcdBackground(detector, bbox).getImage()
767 return self.collectBinnedImage(exposure, image)
768
769 def collectBinnedImage(self, exposure, image):
770 """Return the binned image required for visualization
771
772 This method just helps to cut down on boilerplate.
773
774 Parameters
775 ----------
777 Image to go into visualisation.
778
779 Returns
780 -------
781 detId : `int`
782 Detector identifier.
784 Binned image.
785 """
786 return (exposure.getDetector().getId(), afwMath.binImage(image, self.config.binning))
787
788 def collect(self, cache):
789 """Collect exposure for potential visualisation
790
791 This method runs on the slave nodes.
792
793 Parameters
794 ----------
795 cache : `lsst.pipe.base.Struct`
796 Process pool cache.
797
798 Returns
799 -------
800 detId : `int`
801 Detector identifier.
803 Binned image.
804 """
805 return self.collectBinnedImage(cache.exposure, cache.exposure.maskedImage)
806
807 def collectOriginal(self, cache, dataId):
808 """Collect original image for visualisation
809
810 This method runs on the slave nodes.
811
812 Parameters
813 ----------
814 cache : `lsst.pipe.base.Struct`
815 Process pool cache.
816 dataId : `dict`
817 Data identifier.
818
819 Returns
820 -------
821 detId : `int`
822 Detector identifier.
824 Binned image.
825 """
826 exposure = cache.butler.get("calexp", dataId, immediate=True)
827 return self.collectBinnedImage(exposure, exposure.maskedImage)
828
829 def collectSky(self, cache, dataId):
830 """Collect original image for visualisation
831
832 This method runs on the slave nodes.
833
834 Parameters
835 ----------
836 cache : `lsst.pipe.base.Struct`
837 Process pool cache.
838 dataId : `dict`
839 Data identifier.
840
841 Returns
842 -------
843 detId : `int`
844 Detector identifier.
846 Binned image.
847 """
848 return self.collectBinnedImage(cache.exposure, cache.sky.getImage())
849
850 def collectMask(self, cache, dataId):
851 """Collect mask for visualisation
852
853 This method runs on the slave nodes.
854
855 Parameters
856 ----------
857 cache : `lsst.pipe.base.Struct`
858 Process pool cache.
859 dataId : `dict`
860 Data identifier.
861
862 Returns
863 -------
864 detId : `int`
865 Detector identifier.
866 image : `lsst.afw.image.Image`
867 Binned image.
868 """
869 # Convert Mask to floating-point image, because that's what's required for focal plane construction
870 image = afwImage.ImageF(cache.exposure.maskedImage.getBBox())
871 image.array[:] = cache.exposure.maskedImage.mask.array
872 return self.collectBinnedImage(cache.exposure, image)
873
874 def write(self, cache, dataId):
875 """Write resultant background list
876
877 This method runs on the slave nodes.
878
879 Parameters
880 ----------
881 cache : `lsst.pipe.base.Struct`
882 Process pool cache.
883 dataId : `dict`
884 Data identifier.
885 """
886 cache.butler.put(cache.bgList, "skyCorr", dataId)
887
888 def _getMetadataName(self):
889 """There's no metadata to write out"""
890 return None
def logOperation(self, operation, catch=False, trace=True)
def run(self, calExpArray, calBkgArray, skyCalibs, camera)
def subtractModel(self, cache, dataId, bgModel)
def focalPlaneBackground(self, camera, pool, dataIdList, config)
def subtractSkyFrame(self, cache, dataId, scale)
def focalPlaneBackgroundRun(self, camera, cacheExposures, idList, config)
def runQuantum(self, butlerQC, inputRefs, outputRefs)
def batchWallTime(cls, time, parsedCmd, numCores)
def realiseModel(self, cache, dataId, bgModel)
def makeCameraImage(camera, exposures, filename=None, binning=8)
def reorderAndPadList(inputList, inputKeys, outputKeys, padWith=None)