lsst.pipe.drivers g1ca1189e32+c8a88ba758
skyCorrection.py
Go to the documentation of this file.
1# This file is part of pipe_drivers.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21
22from deprecated.sphinx import deprecated
23
24import lsst.afw.math as afwMath
25import lsst.afw.image as afwImage
26import lsst.pipe.base as pipeBase
27
28from lsst.pipe.base import ArgumentParser, ConfigDatasetType
29from lsst.daf.butler import DimensionGraph
30from lsst.pex.config import Config, Field, ConfigurableField, ConfigField
31from lsst.ctrl.pool.pool import Pool
32from lsst.ctrl.pool.parallel import BatchPoolTask
33from lsst.pipe.drivers.background import (SkyMeasurementTask, FocalPlaneBackground,
34 FocalPlaneBackgroundConfig, MaskObjectsTask)
35import lsst.pipe.drivers.visualizeVisit as visualizeVisit
36import lsst.pipe.base.connectionTypes as cT
37
38__all__ = ["SkyCorrectionConfig", "SkyCorrectionTask"]
39
40DEBUG = False # Debugging outputs?
41
42
43def reorderAndPadList(inputList, inputKeys, outputKeys, padWith=None):
44 """Match the order of one list to another, padding if necessary
45
46 Parameters
47 ----------
48 inputList : list
49 List to be reordered and padded. Elements can be any type.
50 inputKeys : iterable
51 Iterable of values to be compared with outputKeys.
52 Length must match `inputList`
53 outputKeys : iterable
54 Iterable of values to be compared with inputKeys.
55 padWith :
56 Any value to be inserted where inputKey not in outputKeys
57
58 Returns
59 -------
60 list
61 Copy of inputList reordered per outputKeys and padded with `padWith`
62 so that the length matches length of outputKeys.
63 """
64 outputList = []
65 for d in outputKeys:
66 if d in inputKeys:
67 outputList.append(inputList[inputKeys.index(d)])
68 else:
69 outputList.append(padWith)
70 return outputList
71
72
73def makeCameraImage(camera, exposures, filename=None, binning=8):
74 """Make and write an image of an entire focal plane
75
76 Parameters
77 ----------
79 Camera description.
80 exposures : `list` of `tuple` of `int` and `lsst.afw.image.Exposure`
81 List of detector ID and CCD exposures (binned by `binning`).
82 filename : `str`, optional
83 Output filename.
84 binning : `int`
85 Binning size that has been applied to images.
86 """
87 image = visualizeVisit.makeCameraImage(camera, dict(exp for exp in exposures if exp is not None), binning)
88 if filename is not None:
89 image.writeFits(filename)
90 return image
91
92
93def _skyLookup(datasetType, registry, quantumDataId, collections):
94 """Lookup function to identify sky frames
95
96 Parameters
97 ----------
98 datasetType : `lsst.daf.butler.DatasetType`
99 Dataset to lookup.
100 registry : `lsst.daf.butler.Registry`
101 Butler registry to query.
102 quantumDataId : `lsst.daf.butler.DataCoordinate`
103 Data id to transform to find sky frames.
104 The ``detector`` entry will be stripped.
105 collections : `lsst.daf.butler.CollectionSearch`
106 Collections to search through.
107
108 Returns
109 -------
110 results : `list` [`lsst.daf.butler.DatasetRef`]
111 List of datasets that will be used as sky calibration frames
112 """
113 newDataId = quantumDataId.subset(DimensionGraph(registry.dimensions, names=["instrument", "visit"]))
114 skyFrames = []
115 for dataId in registry.queryDataIds(["visit", "detector"], dataId=newDataId).expanded():
116 skyFrame = registry.findDataset(datasetType, dataId, collections=collections,
117 timespan=dataId.timespan)
118 skyFrames.append(skyFrame)
119
120 return skyFrames
121
122
123class SkyCorrectionConnections(pipeBase.PipelineTaskConnections, dimensions=("instrument", "visit")):
124 rawLinker = cT.Input(
125 doc="Raw data to provide exp-visit linkage to connect calExp inputs to camera/sky calibs.",
126 name="raw",
127 multiple=True,
128 deferLoad=True,
129 storageClass="Exposure",
130 dimensions=["instrument", "exposure", "detector"],
131 )
132 calExpArray = cT.Input(
133 doc="Input exposures to process",
134 name="calexp",
135 multiple=True,
136 storageClass="ExposureF",
137 dimensions=["instrument", "visit", "detector"],
138 )
139 calBkgArray = cT.Input(
140 doc="Input background files to use",
141 multiple=True,
142 name="calexpBackground",
143 storageClass="Background",
144 dimensions=["instrument", "visit", "detector"],
145 )
146 camera = cT.PrerequisiteInput(
147 doc="Input camera to use.",
148 name="camera",
149 storageClass="Camera",
150 dimensions=["instrument"],
151 isCalibration=True,
152 )
153 skyCalibs = cT.PrerequisiteInput(
154 doc="Input sky calibrations to use.",
155 name="sky",
156 multiple=True,
157 storageClass="ExposureF",
158 dimensions=["instrument", "physical_filter", "detector"],
159 isCalibration=True,
160 lookupFunction=_skyLookup,
161 )
162 calExpCamera = cT.Output(
163 doc="Output camera image.",
164 name='calexp_camera',
165 storageClass="ImageF",
166 dimensions=["instrument", "visit"],
167 )
168 skyCorr = cT.Output(
169 doc="Output sky corrected images.",
170 name='skyCorr',
171 multiple=True,
172 storageClass="Background",
173 dimensions=["instrument", "visit", "detector"],
174 )
175
176
177class SkyCorrectionConfig(pipeBase.PipelineTaskConfig, pipelineConnections=SkyCorrectionConnections):
178 """Configuration for SkyCorrectionTask"""
179 bgModel = ConfigField(dtype=FocalPlaneBackgroundConfig, doc="Background model")
180 bgModel2 = ConfigField(dtype=FocalPlaneBackgroundConfig, doc="2nd Background model")
181 sky = ConfigurableField(target=SkyMeasurementTask, doc="Sky measurement")
182 maskObjects = ConfigurableField(target=MaskObjectsTask, doc="Mask Objects")
183 doMaskObjects = Field(dtype=bool, default=True, doc="Mask objects to find good sky?")
184 doBgModel = Field(dtype=bool, default=True, doc="Do background model subtraction?")
185 doBgModel2 = Field(dtype=bool, default=True, doc="Do cleanup background model subtraction?")
186 doSky = Field(dtype=bool, default=True, doc="Do sky frame subtraction?")
187 binning = Field(dtype=int, default=8, doc="Binning factor for constructing focal-plane images")
188 calexpType = Field(dtype=str, default="calexp",
189 doc="Should be set to fakes_calexp if you want to process calexps with fakes in.")
190
191 def setDefaults(self):
192 Config.setDefaults(self)
193 self.bgModel2.doSmooth = True
194 self.bgModel2.minFrac = 0.5
195 self.bgModel2.xSize = 256
196 self.bgModel2.ySize = 256
197 self.bgModel2.smoothScale = 1.0
198
199
200@deprecated(
201 reason="pipe_drivers is deprecated. It will be removed after v25. "
202 "Please use lsst.pipe.tasks.skyCorrection.SkyCorrectionTask instead.",
203 version="v25.0",
204 category=FutureWarning,
205)
206class SkyCorrectionTask(pipeBase.PipelineTask, BatchPoolTask):
207 """Correct sky over entire focal plane"""
208 ConfigClass = SkyCorrectionConfig
209 _DefaultName = "skyCorr"
210
211 def runQuantum(self, butlerQC, inputRefs, outputRefs):
212
213 # Reorder the skyCalibs, calBkgArray, and calExpArray inputRefs and the
214 # skyCorr outputRef sorted by detector id to ensure reproducibility.
215 detectorOrder = [ref.dataId['detector'] for ref in inputRefs.calExpArray]
216 detectorOrder.sort()
217 inputRefs.calExpArray = reorderAndPadList(inputRefs.calExpArray,
218 [ref.dataId['detector'] for ref in inputRefs.calExpArray],
219 detectorOrder)
220 inputRefs.skyCalibs = reorderAndPadList(inputRefs.skyCalibs,
221 [ref.dataId['detector'] for ref in inputRefs.skyCalibs],
222 detectorOrder)
223 inputRefs.calBkgArray = reorderAndPadList(inputRefs.calBkgArray,
224 [ref.dataId['detector'] for ref in inputRefs.calBkgArray],
225 detectorOrder)
226 outputRefs.skyCorr = reorderAndPadList(outputRefs.skyCorr,
227 [ref.dataId['detector'] for ref in outputRefs.skyCorr],
228 detectorOrder)
229 inputs = butlerQC.get(inputRefs)
230 inputs.pop("rawLinker", None)
231 outputs = self.run(**inputs)
232 butlerQC.put(outputs, outputRefs)
233
234 def __init__(self, *args, **kwargs):
235 super().__init__(**kwargs)
236
237 self.makeSubtask("sky")
238 self.makeSubtask("maskObjects")
239
240 @classmethod
241 def _makeArgumentParser(cls, *args, **kwargs):
242 kwargs.pop("doBatch", False)
243 datasetType = ConfigDatasetType(name="calexpType")
244 parser = ArgumentParser(name="skyCorr", *args, **kwargs)
245 parser.add_id_argument("--id", datasetType=datasetType, level="visit",
246 help="data ID, e.g. --id visit=12345")
247 return parser
248
249 @classmethod
250 def batchWallTime(cls, time, parsedCmd, numCores):
251 """Return walltime request for batch job
252
253 Subclasses should override if the walltime should be calculated
254 differently (e.g., addition of some serial time).
255
256 Parameters
257 ----------
258 time : `float`
259 Requested time per iteration.
260 parsedCmd : `argparse.Namespace`
261 Results of argument parsing.
262 numCores : `int`
263 Number of cores.
264 """
265 numTargets = len(cls.RunnerClass.getTargetList(parsedCmd))
266 return time*numTargets
267
268 def runDataRef(self, expRef):
269 """Perform sky correction on an exposure
270
271 We restore the original sky, and remove it again using multiple
272 algorithms. We optionally apply:
273
274 1. A large-scale background model.
275 This step removes very-large-scale sky such as moonlight.
276 2. A sky frame.
277 3. A medium-scale background model.
278 This step removes residual sky (This is smooth on the focal plane).
279
280 Only the master node executes this method. The data is held on
281 the slave nodes, which do all the hard work.
282
283 Parameters
284 ----------
285 expRef : `lsst.daf.persistence.ButlerDataRef`
286 Data reference for exposure.
287
288 See Also
289 --------
290 ~lsst.pipe.drivers.SkyCorrectionTask.run
291 """
292 if DEBUG:
293 extension = "-%(visit)d.fits" % expRef.dataId
294
295 with self.logOperation("processing %s" % (expRef.dataId,)):
296 pool = Pool()
297 pool.cacheClear()
298 pool.storeSet(butler=expRef.getButler())
299 camera = expRef.get("camera")
300
301 dataIdList = [ccdRef.dataId for ccdRef in expRef.subItems("ccd") if
302 ccdRef.datasetExists(self.config.calexpType)]
303 ccdList = [dataId["ccd"] for dataId in dataIdList]
304 # Order inputs sorted by detector id to ensure reproducibility
305 dataIdList = [dataId for _, dataId in sorted(zip(ccdList, dataIdList))]
306
307 exposures = pool.map(self.loadImage, dataIdList)
308 if DEBUG:
309 makeCameraImage(camera, exposures, "restored" + extension)
310 exposures = pool.mapToPrevious(self.collectOriginal, dataIdList)
311 makeCameraImage(camera, exposures, "original" + extension)
312 exposures = pool.mapToPrevious(self.collectMask, dataIdList)
313 makeCameraImage(camera, exposures, "mask" + extension)
314
315 if self.config.doBgModel:
316 exposures = self.focalPlaneBackground(camera, pool, dataIdList, self.config.bgModel)
317
318 if self.config.doSky:
319 measScales = pool.mapToPrevious(self.measureSkyFrame, dataIdList)
320 scale = self.sky.solveScales(measScales)
321 self.log.info("Sky frame scale: %s" % (scale,))
322
323 exposures = pool.mapToPrevious(self.subtractSkyFrame, dataIdList, scale)
324 if DEBUG:
325 makeCameraImage(camera, exposures, "skysub" + extension)
326 calibs = pool.mapToPrevious(self.collectSky, dataIdList)
327 makeCameraImage(camera, calibs, "sky" + extension)
328
329 if self.config.doBgModel2:
330 exposures = self.focalPlaneBackground(camera, pool, dataIdList, self.config.bgModel2)
331
332 # Persist camera-level image of calexp
333 image = makeCameraImage(camera, exposures)
334 expRef.put(image, "calexp_camera")
335
336 pool.mapToPrevious(self.write, dataIdList)
337
338 def focalPlaneBackground(self, camera, pool, dataIdList, config):
339 """Perform full focal-plane background subtraction
340
341 This method runs on the master node.
342
343 Parameters
344 ----------
346 Camera description.
347 pool : `lsst.ctrl.pool.Pool`
348 Process pool.
349 dataIdList : iterable of `dict`
350 List of data identifiers for the CCDs.
352 Configuration to use for background subtraction.
353
354 Returns
355 -------
356 exposures : `list` of `lsst.afw.image.Image`
357 List of binned images, for creating focal plane image.
358 """
359 bgModel = FocalPlaneBackground.fromCamera(config, camera)
360 data = [pipeBase.Struct(dataId=dataId, bgModel=bgModel.clone()) for dataId in dataIdList]
361 bgModelList = pool.mapToPrevious(self.accumulateModel, data)
362 for ii, bg in enumerate(bgModelList):
363 self.log.info("Background %d: %d pixels", ii, bg._numbers.array.sum())
364 bgModel.merge(bg)
365 return pool.mapToPrevious(self.subtractModel, dataIdList, bgModel)
366
367 def focalPlaneBackgroundRun(self, camera, cacheExposures, idList, config):
368 """Perform full focal-plane background subtraction
369
370 This method runs on the master node.
371
372 Parameters
373 ----------
375 Camera description.
376 cacheExposures : `list` of `lsst.afw.image.Exposures`
377 List of loaded and processed input calExp.
378 idList : `list` of `int`
379 List of detector ids to iterate over.
381 Configuration to use for background subtraction.
382
383 Returns
384 -------
385 exposures : `list` of `lsst.afw.image.Image`
386 List of binned images, for creating focal plane image.
387 newCacheBgList : `list` of `lsst.afwMath.backgroundList`
388 Background lists generated.
389 cacheBgModel : `FocalPlaneBackground`
390 Full focal plane background model.
391 """
392 bgModel = FocalPlaneBackground.fromCamera(config, camera)
393 data = [pipeBase.Struct(id=id, bgModel=bgModel.clone()) for id in idList]
394
395 bgModelList = []
396 for nodeData, cacheExp in zip(data, cacheExposures):
397 nodeData.bgModel.addCcd(cacheExp)
398 bgModelList.append(nodeData.bgModel)
399
400 for ii, bg in enumerate(bgModelList):
401 self.log.info("Background %d: %d pixels", ii, bg._numbers.getArray().sum())
402 bgModel.merge(bg)
403
404 exposures = []
405 newCacheBgList = []
406 cacheBgModel = []
407 for cacheExp in cacheExposures:
408 nodeExp, nodeBgModel, nodeBgList = self.subtractModelRun(cacheExp, bgModel)
409 exposures.append(afwMath.binImage(nodeExp.getMaskedImage(), self.config.binning))
410 cacheBgModel.append(nodeBgModel)
411 newCacheBgList.append(nodeBgList)
412
413 return exposures, newCacheBgList, cacheBgModel
414
415 def run(self, calExpArray, calBkgArray, skyCalibs, camera):
416 """Duplicate runDataRef method without ctrl_pool for Gen3.
417
418 Parameters
419 ----------
420 calExpArray : `list` of `lsst.afw.image.Exposure`
421 Array of detector input calExp images for the exposure to
422 process.
423 calBkgArray : `list` of `lsst.afw.math.BackgroundList`
424 Array of detector input background lists matching the
425 calExps to process.
426 skyCalibs : `list` of `lsst.afw.image.Exposure`
427 Array of SKY calibrations for the input detectors to be
428 processed.
430 Camera matching the input data to process.
431
432 Returns
433 -------
434 results : `pipeBase.Struct` containing
435 calExpCamera : `lsst.afw.image.Exposure`
436 Full camera image of the sky-corrected data.
437 skyCorr : `list` of `lsst.afw.math.BackgroundList`
438 Detector-level sky-corrected background lists.
439
440 See Also
441 --------
442 ~lsst.pipe.drivers.SkyCorrectionTask.runDataRef()
443 """
444 # To allow SkyCorrectionTask to run in the Gen3 butler
445 # environment, a new run() method was added that performs the
446 # same operations in a serial environment (pipetask processing
447 # does not support MPI processing as of 2019-05-03). Methods
448 # used in runDataRef() are used as appropriate in run(), but
449 # some have been rewritten in serial form. Please ensure that
450 # any updates to runDataRef() or the methods it calls with
451 # pool.mapToPrevious() are duplicated in run() and its
452 # methods.
453 #
454 # Variable names here should match those in runDataRef() as
455 # closely as possible. Variables matching data stored in the
456 # pool cache have a prefix indicating this. Variables that
457 # would be local to an MPI processing client have a prefix
458 # "node".
459 idList = [exp.getDetector().getId() for exp in calExpArray]
460
461 # Construct arrays that match the cache in self.runDataRef() after
462 # self.loadImage() is map/reduced.
463 cacheExposures = []
464 cacheBgList = []
465 exposures = []
466 for calExp, calBgModel in zip(calExpArray, calBkgArray):
467 nodeExp, nodeBgList = self.loadImageRun(calExp, calBgModel)
468 cacheExposures.append(nodeExp)
469 cacheBgList.append(nodeBgList)
470 exposures.append(afwMath.binImage(nodeExp.getMaskedImage(), self.config.binning))
471
472 if self.config.doBgModel:
473 # Generate focal plane background, updating backgrounds in the "cache".
474 exposures, newCacheBgList, cacheBgModel = self.focalPlaneBackgroundRun(
475 camera, cacheExposures, idList, self.config.bgModel
476 )
477 for cacheBg, newBg in zip(cacheBgList, newCacheBgList):
478 cacheBg.append(newBg)
479
480 if self.config.doSky:
481 # Measure the sky frame scale on all inputs. Results in
482 # values equal to self.measureSkyFrame() and
483 # self.sky.solveScales() in runDataRef().
484 cacheSky = []
485 measScales = []
486 for cacheExp, skyCalib in zip(cacheExposures, skyCalibs):
487 skyExp = self.sky.exposureToBackground(skyCalib)
488 cacheSky.append(skyExp)
489 scale = self.sky.measureScale(cacheExp.getMaskedImage(), skyExp)
490 measScales.append(scale)
491
492 scale = self.sky.solveScales(measScales)
493 self.log.info("Sky frame scale: %s" % (scale, ))
494
495 # Subtract sky frame, as in self.subtractSkyFrame(), with
496 # appropriate scale from the "cache".
497 exposures = []
498 newBgList = []
499 for cacheExp, nodeSky, nodeBgList in zip(cacheExposures, cacheSky, cacheBgList):
500 self.sky.subtractSkyFrame(cacheExp.getMaskedImage(), nodeSky, scale, nodeBgList)
501 exposures.append(afwMath.binImage(cacheExp.getMaskedImage(), self.config.binning))
502
503 if self.config.doBgModel2:
504 # As above, generate a focal plane background model and
505 # update the cache models.
506 exposures, newBgList, cacheBgModel = self.focalPlaneBackgroundRun(
507 camera, cacheExposures, idList, self.config.bgModel2
508 )
509 for cacheBg, newBg in zip(cacheBgList, newBgList):
510 cacheBg.append(newBg)
511
512 # Generate camera-level image of calexp and return it along
513 # with the list of sky corrected background models.
514 image = makeCameraImage(camera, zip(idList, exposures))
515
516 return pipeBase.Struct(
517 calExpCamera=image,
518 skyCorr=cacheBgList,
519 )
520
521 def loadImage(self, cache, dataId):
522 """Load original image and restore the sky
523
524 This method runs on the slave nodes.
525
526 Parameters
527 ----------
528 cache : `lsst.pipe.base.Struct`
529 Process pool cache.
530 dataId : `dict`
531 Data identifier.
532
533 Returns
534 -------
535 exposure : `lsst.afw.image.Exposure`
536 Resultant exposure.
537 """
538 cache.dataId = dataId
539 cache.exposure = cache.butler.get(self.config.calexpType, dataId, immediate=True).clone()
540 bgOld = cache.butler.get("calexpBackground", dataId, immediate=True)
541 image = cache.exposure.getMaskedImage()
542
543 # We're removing the old background, so change the sense of all its components
544 for bgData in bgOld:
545 statsImage = bgData[0].getStatsImage()
546 statsImage *= -1
547
548 image -= bgOld.getImage()
549 cache.bgList = afwMath.BackgroundList()
550 for bgData in bgOld:
551 cache.bgList.append(bgData)
552
553 if self.config.doMaskObjects:
554 self.maskObjects.findObjects(cache.exposure)
555
556 return self.collect(cache)
557
558 def loadImageRun(self, calExp, calExpBkg):
559 """Serial implementation of self.loadImage() for Gen3.
560
561 Load and restore background to calExp and calExpBkg.
562
563 Parameters
564 ----------
565 calExp : `lsst.afw.image.Exposure`
566 Detector level calExp image to process.
567 calExpBkg : `lsst.afw.math.BackgroundList`
568 Detector level background list associated with the calExp.
569
570 Returns
571 -------
572 calExp : `lsst.afw.image.Exposure`
573 Background restored calExp.
574 bgList : `lsst.afw.math.BackgroundList`
575 New background list containing the restoration background.
576 """
577 image = calExp.getMaskedImage()
578
579 for bgOld in calExpBkg:
580 statsImage = bgOld[0].getStatsImage()
581 statsImage *= -1
582
583 image -= calExpBkg.getImage()
584 bgList = afwMath.BackgroundList()
585 for bgData in calExpBkg:
586 bgList.append(bgData)
587
588 if self.config.doMaskObjects:
589 self.maskObjects.findObjects(calExp)
590
591 return (calExp, bgList)
592
593 def measureSkyFrame(self, cache, dataId):
594 """Measure scale for sky frame
595
596 This method runs on the slave nodes.
597
598 Parameters
599 ----------
600 cache : `lsst.pipe.base.Struct`
601 Process pool cache.
602 dataId : `dict`
603 Data identifier.
604
605 Returns
606 -------
607 scale : `float`
608 Scale for sky frame.
609 """
610 assert cache.dataId == dataId
611 cache.sky = self.sky.getSkyData(cache.butler, dataId)
612 scale = self.sky.measureScale(cache.exposure.getMaskedImage(), cache.sky)
613 return scale
614
615 def subtractSkyFrame(self, cache, dataId, scale):
616 """Subtract sky frame
617
618 This method runs on the slave nodes.
619
620 Parameters
621 ----------
622 cache : `lsst.pipe.base.Struct`
623 Process pool cache.
624 dataId : `dict`
625 Data identifier.
626 scale : `float`
627 Scale for sky frame.
628
629 Returns
630 -------
631 exposure : `lsst.afw.image.Exposure`
632 Resultant exposure.
633 """
634 assert cache.dataId == dataId
635 self.sky.subtractSkyFrame(cache.exposure.getMaskedImage(), cache.sky, scale, cache.bgList)
636 return self.collect(cache)
637
638 def accumulateModel(self, cache, data):
639 """Fit background model for CCD
640
641 This method runs on the slave nodes.
642
643 Parameters
644 ----------
645 cache : `lsst.pipe.base.Struct`
646 Process pool cache.
647 data : `lsst.pipe.base.Struct`
648 Data identifier, with `dataId` (data identifier) and `bgModel`
649 (background model) elements.
650
651 Returns
652 -------
654 Background model.
655 """
656 assert cache.dataId == data.dataId
657 data.bgModel.addCcd(cache.exposure)
658 return data.bgModel
659
660 def subtractModel(self, cache, dataId, bgModel):
661 """Subtract background model
662
663 This method runs on the slave nodes.
664
665 Parameters
666 ----------
667 cache : `lsst.pipe.base.Struct`
668 Process pool cache.
669 dataId : `dict`
670 Data identifier.
671 bgModel : `lsst.pipe.drivers.background.FocalPlaneBackround`
672 Background model.
673
674 Returns
675 -------
676 exposure : `lsst.afw.image.Exposure`
677 Resultant exposure.
678 """
679 assert cache.dataId == dataId
680 exposure = cache.exposure
681 image = exposure.getMaskedImage()
682 detector = exposure.getDetector()
683 bbox = image.getBBox()
684 try:
685 cache.bgModel = bgModel.toCcdBackground(detector, bbox)
686 image -= cache.bgModel.getImage()
687 except RuntimeError:
688 self.log.error(f"There was an error processing {dataId}, no calib file produced")
689 return
690 cache.bgList.append(cache.bgModel[0])
691 return self.collect(cache)
692
693 def subtractModelRun(self, exposure, bgModel):
694 """Serial implementation of self.subtractModel() for Gen3.
695
696 Load and restore background to calExp and calExpBkg.
697
698 Parameters
699 ----------
700 exposure : `lsst.afw.image.Exposure`
701 Exposure to subtract the background model from.
703 Full camera level background model.
704
705 Returns
706 -------
707 exposure : `lsst.afw.image.Exposure`
708 Background subtracted input exposure.
709 bgModelCcd : `lsst.afw.math.BackgroundList`
710 Detector level realization of the full background model.
711 bgModelMaskedImage : `lsst.afw.image.MaskedImage`
712 Background model from the bgModelCcd realization.
713 """
714 image = exposure.getMaskedImage()
715 detector = exposure.getDetector()
716 bbox = image.getBBox()
717 bgModelCcd = bgModel.toCcdBackground(detector, bbox)
718 image -= bgModelCcd.getImage()
719
720 return (exposure, bgModelCcd, bgModelCcd[0])
721
722 def realiseModel(self, cache, dataId, bgModel):
723 """Generate an image of the background model for visualisation
724
725 Useful for debugging.
726
727 Parameters
728 ----------
729 cache : `lsst.pipe.base.Struct`
730 Process pool cache.
731 dataId : `dict`
732 Data identifier.
733 bgModel : `lsst.pipe.drivers.background.FocalPlaneBackround`
734 Background model.
735
736 Returns
737 -------
738 detId : `int`
739 Detector identifier.
741 Binned background model image.
742 """
743 assert cache.dataId == dataId
744 exposure = cache.exposure
745 detector = exposure.getDetector()
746 bbox = exposure.getMaskedImage().getBBox()
747 image = bgModel.toCcdBackground(detector, bbox).getImage()
748 return self.collectBinnedImage(exposure, image)
749
750 def collectBinnedImage(self, exposure, image):
751 """Return the binned image required for visualization
752
753 This method just helps to cut down on boilerplate.
754
755 Parameters
756 ----------
758 Image to go into visualisation.
759
760 Returns
761 -------
762 detId : `int`
763 Detector identifier.
765 Binned image.
766 """
767 return (exposure.getDetector().getId(), afwMath.binImage(image, self.config.binning))
768
769 def collect(self, cache):
770 """Collect exposure for potential visualisation
771
772 This method runs on the slave nodes.
773
774 Parameters
775 ----------
776 cache : `lsst.pipe.base.Struct`
777 Process pool cache.
778
779 Returns
780 -------
781 detId : `int`
782 Detector identifier.
784 Binned image.
785 """
786 return self.collectBinnedImage(cache.exposure, cache.exposure.maskedImage)
787
788 def collectOriginal(self, cache, dataId):
789 """Collect original image for visualisation
790
791 This method runs on the slave nodes.
792
793 Parameters
794 ----------
795 cache : `lsst.pipe.base.Struct`
796 Process pool cache.
797 dataId : `dict`
798 Data identifier.
799
800 Returns
801 -------
802 detId : `int`
803 Detector identifier.
805 Binned image.
806 """
807 exposure = cache.butler.get("calexp", dataId, immediate=True)
808 return self.collectBinnedImage(exposure, exposure.maskedImage)
809
810 def collectSky(self, cache, dataId):
811 """Collect original image for visualisation
812
813 This method runs on the slave nodes.
814
815 Parameters
816 ----------
817 cache : `lsst.pipe.base.Struct`
818 Process pool cache.
819 dataId : `dict`
820 Data identifier.
821
822 Returns
823 -------
824 detId : `int`
825 Detector identifier.
827 Binned image.
828 """
829 return self.collectBinnedImage(cache.exposure, cache.sky.getImage())
830
831 def collectMask(self, cache, dataId):
832 """Collect mask for visualisation
833
834 This method runs on the slave nodes.
835
836 Parameters
837 ----------
838 cache : `lsst.pipe.base.Struct`
839 Process pool cache.
840 dataId : `dict`
841 Data identifier.
842
843 Returns
844 -------
845 detId : `int`
846 Detector identifier.
847 image : `lsst.afw.image.Image`
848 Binned image.
849 """
850 # Convert Mask to floating-point image, because that's what's required for focal plane construction
851 image = afwImage.ImageF(cache.exposure.maskedImage.getBBox())
852 image.array[:] = cache.exposure.maskedImage.mask.array
853 return self.collectBinnedImage(cache.exposure, image)
854
855 def write(self, cache, dataId):
856 """Write resultant background list
857
858 This method runs on the slave nodes.
859
860 Parameters
861 ----------
862 cache : `lsst.pipe.base.Struct`
863 Process pool cache.
864 dataId : `dict`
865 Data identifier.
866 """
867 cache.butler.put(cache.bgList, "skyCorr", dataId)
868
869 def _getMetadataName(self):
870 """There's no metadata to write out"""
871 return None
def logOperation(self, operation, catch=False, trace=True)
def run(self, calExpArray, calBkgArray, skyCalibs, camera)
def subtractModel(self, cache, dataId, bgModel)
def focalPlaneBackground(self, camera, pool, dataIdList, config)
def subtractSkyFrame(self, cache, dataId, scale)
def focalPlaneBackgroundRun(self, camera, cacheExposures, idList, config)
def runQuantum(self, butlerQC, inputRefs, outputRefs)
def batchWallTime(cls, time, parsedCmd, numCores)
def realiseModel(self, cache, dataId, bgModel)
def makeCameraImage(camera, exposures, filename=None, binning=8)
def reorderAndPadList(inputList, inputKeys, outputKeys, padWith=None)