lsst.pipe.drivers gef6abdcebb+02680e40ae
skyCorrection.py
Go to the documentation of this file.
1# This file is part of pipe_drivers.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21import lsst.afw.math as afwMath
22import lsst.afw.image as afwImage
23import lsst.pipe.base as pipeBase
24
25from lsst.pipe.base import ArgumentParser, ConfigDatasetType
26from lsst.daf.butler import DimensionGraph
27from lsst.pex.config import Config, Field, ConfigurableField, ConfigField
28from lsst.ctrl.pool.pool import Pool
29from lsst.ctrl.pool.parallel import BatchPoolTask
30from lsst.pipe.drivers.background import (SkyMeasurementTask, FocalPlaneBackground,
31 FocalPlaneBackgroundConfig, MaskObjectsTask)
32import lsst.pipe.drivers.visualizeVisit as visualizeVisit
33import lsst.pipe.base.connectionTypes as cT
34
35__all__ = ["SkyCorrectionConfig", "SkyCorrectionTask"]
36
37DEBUG = False # Debugging outputs?
38
39
40def reorderAndPadList(inputList, inputKeys, outputKeys, padWith=None):
41 """Match the order of one list to another, padding if necessary
42
43 Parameters
44 ----------
45 inputList : list
46 List to be reordered and padded. Elements can be any type.
47 inputKeys : iterable
48 Iterable of values to be compared with outputKeys.
49 Length must match `inputList`
50 outputKeys : iterable
51 Iterable of values to be compared with inputKeys.
52 padWith :
53 Any value to be inserted where inputKey not in outputKeys
54
55 Returns
56 -------
57 list
58 Copy of inputList reordered per outputKeys and padded with `padWith`
59 so that the length matches length of outputKeys.
60 """
61 outputList = []
62 for d in outputKeys:
63 if d in inputKeys:
64 outputList.append(inputList[inputKeys.index(d)])
65 else:
66 outputList.append(padWith)
67 return outputList
68
69
70def makeCameraImage(camera, exposures, filename=None, binning=8):
71 """Make and write an image of an entire focal plane
72
73 Parameters
74 ----------
76 Camera description.
77 exposures : `list` of `tuple` of `int` and `lsst.afw.image.Exposure`
78 List of detector ID and CCD exposures (binned by `binning`).
79 filename : `str`, optional
80 Output filename.
81 binning : `int`
82 Binning size that has been applied to images.
83 """
84 image = visualizeVisit.makeCameraImage(camera, dict(exp for exp in exposures if exp is not None), binning)
85 if filename is not None:
86 image.writeFits(filename)
87 return image
88
89
90def _skyLookup(datasetType, registry, quantumDataId, collections):
91 """Lookup function to identify sky frames
92
93 Parameters
94 ----------
95 datasetType : `lsst.daf.butler.DatasetType`
96 Dataset to lookup.
97 registry : `lsst.daf.butler.Registry`
98 Butler registry to query.
99 quantumDataId : `lsst.daf.butler.DataCoordinate`
100 Data id to transform to find sky frames.
101 The ``detector`` entry will be stripped.
102 collections : `lsst.daf.butler.CollectionSearch`
103 Collections to search through.
104
105 Returns
106 -------
107 results : `list` [`lsst.daf.butler.DatasetRef`]
108 List of datasets that will be used as sky calibration frames
109 """
110 newDataId = quantumDataId.subset(DimensionGraph(registry.dimensions, names=["instrument", "visit"]))
111 skyFrames = []
112 for dataId in registry.queryDataIds(["visit", "detector"], dataId=newDataId).expanded():
113 skyFrame = registry.findDataset(datasetType, dataId, collections=collections,
114 timespan=dataId.timespan)
115 skyFrames.append(skyFrame)
116
117 return skyFrames
118
119
120class SkyCorrectionConnections(pipeBase.PipelineTaskConnections, dimensions=("instrument", "visit")):
121 rawLinker = cT.Input(
122 doc="Raw data to provide exp-visit linkage to connect calExp inputs to camera/sky calibs.",
123 name="raw",
124 multiple=True,
125 deferLoad=True,
126 storageClass="Exposure",
127 dimensions=["instrument", "exposure", "detector"],
128 )
129 calExpArray = cT.Input(
130 doc="Input exposures to process",
131 name="calexp",
132 multiple=True,
133 storageClass="ExposureF",
134 dimensions=["instrument", "visit", "detector"],
135 )
136 calBkgArray = cT.Input(
137 doc="Input background files to use",
138 multiple=True,
139 name="calexpBackground",
140 storageClass="Background",
141 dimensions=["instrument", "visit", "detector"],
142 )
143 camera = cT.PrerequisiteInput(
144 doc="Input camera to use.",
145 name="camera",
146 storageClass="Camera",
147 dimensions=["instrument"],
148 isCalibration=True,
149 )
150 skyCalibs = cT.PrerequisiteInput(
151 doc="Input sky calibrations to use.",
152 name="sky",
153 multiple=True,
154 storageClass="ExposureF",
155 dimensions=["instrument", "physical_filter", "detector"],
156 isCalibration=True,
157 lookupFunction=_skyLookup,
158 )
159 calExpCamera = cT.Output(
160 doc="Output camera image.",
161 name='calexp_camera',
162 storageClass="ImageF",
163 dimensions=["instrument", "visit"],
164 )
165 skyCorr = cT.Output(
166 doc="Output sky corrected images.",
167 name='skyCorr',
168 multiple=True,
169 storageClass="Background",
170 dimensions=["instrument", "visit", "detector"],
171 )
172
173
174class SkyCorrectionConfig(pipeBase.PipelineTaskConfig, pipelineConnections=SkyCorrectionConnections):
175 """Configuration for SkyCorrectionTask"""
176 bgModel = ConfigField(dtype=FocalPlaneBackgroundConfig, doc="Background model")
177 bgModel2 = ConfigField(dtype=FocalPlaneBackgroundConfig, doc="2nd Background model")
178 sky = ConfigurableField(target=SkyMeasurementTask, doc="Sky measurement")
179 maskObjects = ConfigurableField(target=MaskObjectsTask, doc="Mask Objects")
180 doMaskObjects = Field(dtype=bool, default=True, doc="Mask objects to find good sky?")
181 doBgModel = Field(dtype=bool, default=True, doc="Do background model subtraction?")
182 doBgModel2 = Field(dtype=bool, default=True, doc="Do cleanup background model subtraction?")
183 doSky = Field(dtype=bool, default=True, doc="Do sky frame subtraction?")
184 binning = Field(dtype=int, default=8, doc="Binning factor for constructing focal-plane images")
185 calexpType = Field(dtype=str, default="calexp",
186 doc="Should be set to fakes_calexp if you want to process calexps with fakes in.")
187
188 def setDefaults(self):
189 Config.setDefaults(self)
190 self.bgModel2bgModel2.doSmooth = True
191 self.bgModel2bgModel2.minFrac = 0.5
192 self.bgModel2bgModel2.xSize = 256
193 self.bgModel2bgModel2.ySize = 256
194 self.bgModel2bgModel2.smoothScale = 1.0
195
196
197class SkyCorrectionTask(pipeBase.PipelineTask, BatchPoolTask):
198 """Correct sky over entire focal plane"""
199 ConfigClass = SkyCorrectionConfig
200 _DefaultName = "skyCorr"
201
202 def runQuantum(self, butlerQC, inputRefs, outputRefs):
203
204 # Reorder the skyCalibs, calBkgArray, and calExpArray inputRefs and the
205 # skyCorr outputRef sorted by detector id to ensure reproducibility.
206 detectorOrder = [ref.dataId['detector'] for ref in inputRefs.calExpArray]
207 detectorOrder.sort()
208 inputRefs.calExpArray = reorderAndPadList(inputRefs.calExpArray,
209 [ref.dataId['detector'] for ref in inputRefs.calExpArray],
210 detectorOrder)
211 inputRefs.skyCalibs = reorderAndPadList(inputRefs.skyCalibs,
212 [ref.dataId['detector'] for ref in inputRefs.skyCalibs],
213 detectorOrder)
214 inputRefs.calBkgArray = reorderAndPadList(inputRefs.calBkgArray,
215 [ref.dataId['detector'] for ref in inputRefs.calBkgArray],
216 detectorOrder)
217 outputRefs.skyCorr = reorderAndPadList(outputRefs.skyCorr,
218 [ref.dataId['detector'] for ref in outputRefs.skyCorr],
219 detectorOrder)
220 inputs = butlerQC.get(inputRefs)
221 inputs.pop("rawLinker", None)
222 outputs = self.runrun(**inputs)
223 butlerQC.put(outputs, outputRefs)
224
225 def __init__(self, *args, **kwargs):
226 super().__init__(**kwargs)
227
228 self.makeSubtask("sky")
229 self.makeSubtask("maskObjects")
230
231 @classmethod
232 def _makeArgumentParser(cls, *args, **kwargs):
233 kwargs.pop("doBatch", False)
234 datasetType = ConfigDatasetType(name="calexpType")
235 parser = ArgumentParser(name="skyCorr", *args, **kwargs)
236 parser.add_id_argument("--id", datasetType=datasetType, level="visit",
237 help="data ID, e.g. --id visit=12345")
238 return parser
239
240 @classmethod
241 def batchWallTime(cls, time, parsedCmd, numCores):
242 """Return walltime request for batch job
243
244 Subclasses should override if the walltime should be calculated
245 differently (e.g., addition of some serial time).
246
247 Parameters
248 ----------
249 time : `float`
250 Requested time per iteration.
251 parsedCmd : `argparse.Namespace`
252 Results of argument parsing.
253 numCores : `int`
254 Number of cores.
255 """
256 numTargets = len(cls.RunnerClass.getTargetList(parsedCmd))
257 return time*numTargets
258
259 def runDataRef(self, expRef):
260 """Perform sky correction on an exposure
261
262 We restore the original sky, and remove it again using multiple
263 algorithms. We optionally apply:
264
265 1. A large-scale background model.
266 This step removes very-large-scale sky such as moonlight.
267 2. A sky frame.
268 3. A medium-scale background model.
269 This step removes residual sky (This is smooth on the focal plane).
270
271 Only the master node executes this method. The data is held on
272 the slave nodes, which do all the hard work.
273
274 Parameters
275 ----------
276 expRef : `lsst.daf.persistence.ButlerDataRef`
277 Data reference for exposure.
278
279 See Also
280 --------
281 ~lsst.pipe.drivers.SkyCorrectionTask.run
282 """
283 if DEBUG:
284 extension = "-%(visit)d.fits" % expRef.dataId
285
286 with self.logOperationlogOperation("processing %s" % (expRef.dataId,)):
287 pool = Pool()
288 pool.cacheClear()
289 pool.storeSet(butler=expRef.getButler())
290 camera = expRef.get("camera")
291
292 dataIdList = [ccdRef.dataId for ccdRef in expRef.subItems("ccd") if
293 ccdRef.datasetExists(self.config.calexpType)]
294 ccdList = [dataId["ccd"] for dataId in dataIdList]
295 # Order inputs sorted by detector id to ensure reproducibility
296 dataIdList = [dataId for _, dataId in sorted(zip(ccdList, dataIdList))]
297
298 exposures = pool.map(self.loadImageloadImage, dataIdList)
299 if DEBUG:
300 makeCameraImage(camera, exposures, "restored" + extension)
301 exposures = pool.mapToPrevious(self.collectOriginalcollectOriginal, dataIdList)
302 makeCameraImage(camera, exposures, "original" + extension)
303 exposures = pool.mapToPrevious(self.collectMaskcollectMask, dataIdList)
304 makeCameraImage(camera, exposures, "mask" + extension)
305
306 if self.config.doBgModel:
307 exposures = self.focalPlaneBackgroundfocalPlaneBackground(camera, pool, dataIdList, self.config.bgModel)
308
309 if self.config.doSky:
310 measScales = pool.mapToPrevious(self.measureSkyFramemeasureSkyFrame, dataIdList)
311 scale = self.sky.solveScales(measScales)
312 self.log.info("Sky frame scale: %s" % (scale,))
313
314 exposures = pool.mapToPrevious(self.subtractSkyFramesubtractSkyFrame, dataIdList, scale)
315 if DEBUG:
316 makeCameraImage(camera, exposures, "skysub" + extension)
317 calibs = pool.mapToPrevious(self.collectSkycollectSky, dataIdList)
318 makeCameraImage(camera, calibs, "sky" + extension)
319
320 if self.config.doBgModel2:
321 exposures = self.focalPlaneBackgroundfocalPlaneBackground(camera, pool, dataIdList, self.config.bgModel2)
322
323 # Persist camera-level image of calexp
324 image = makeCameraImage(camera, exposures)
325 expRef.put(image, "calexp_camera")
326
327 pool.mapToPrevious(self.writewrite, dataIdList)
328
329 def focalPlaneBackground(self, camera, pool, dataIdList, config):
330 """Perform full focal-plane background subtraction
331
332 This method runs on the master node.
333
334 Parameters
335 ----------
337 Camera description.
338 pool : `lsst.ctrl.pool.Pool`
339 Process pool.
340 dataIdList : iterable of `dict`
341 List of data identifiers for the CCDs.
343 Configuration to use for background subtraction.
344
345 Returns
346 -------
347 exposures : `list` of `lsst.afw.image.Image`
348 List of binned images, for creating focal plane image.
349 """
350 bgModel = FocalPlaneBackground.fromCamera(config, camera)
351 data = [pipeBase.Struct(dataId=dataId, bgModel=bgModel.clone()) for dataId in dataIdList]
352 bgModelList = pool.mapToPrevious(self.accumulateModelaccumulateModel, data)
353 for ii, bg in enumerate(bgModelList):
354 self.log.info("Background %d: %d pixels", ii, bg._numbers.array.sum())
355 bgModel.merge(bg)
356 return pool.mapToPrevious(self.subtractModelsubtractModel, dataIdList, bgModel)
357
358 def focalPlaneBackgroundRun(self, camera, cacheExposures, idList, config):
359 """Perform full focal-plane background subtraction
360
361 This method runs on the master node.
362
363 Parameters
364 ----------
366 Camera description.
367 cacheExposures : `list` of `lsst.afw.image.Exposures`
368 List of loaded and processed input calExp.
369 idList : `list` of `int`
370 List of detector ids to iterate over.
372 Configuration to use for background subtraction.
373
374 Returns
375 -------
376 exposures : `list` of `lsst.afw.image.Image`
377 List of binned images, for creating focal plane image.
378 newCacheBgList : `list` of `lsst.afwMath.backgroundList`
379 Background lists generated.
380 cacheBgModel : `FocalPlaneBackground`
381 Full focal plane background model.
382 """
383 bgModel = FocalPlaneBackground.fromCamera(config, camera)
384 data = [pipeBase.Struct(id=id, bgModel=bgModel.clone()) for id in idList]
385
386 bgModelList = []
387 for nodeData, cacheExp in zip(data, cacheExposures):
388 nodeData.bgModel.addCcd(cacheExp)
389 bgModelList.append(nodeData.bgModel)
390
391 for ii, bg in enumerate(bgModelList):
392 self.log.info("Background %d: %d pixels", ii, bg._numbers.getArray().sum())
393 bgModel.merge(bg)
394
395 exposures = []
396 newCacheBgList = []
397 cacheBgModel = []
398 for cacheExp in cacheExposures:
399 nodeExp, nodeBgModel, nodeBgList = self.subtractModelRunsubtractModelRun(cacheExp, bgModel)
400 exposures.append(afwMath.binImage(nodeExp.getMaskedImage(), self.config.binning))
401 cacheBgModel.append(nodeBgModel)
402 newCacheBgList.append(nodeBgList)
403
404 return exposures, newCacheBgList, cacheBgModel
405
406 def run(self, calExpArray, calBkgArray, skyCalibs, camera):
407 """Duplicate runDataRef method without ctrl_pool for Gen3.
408
409 Parameters
410 ----------
411 calExpArray : `list` of `lsst.afw.image.Exposure`
412 Array of detector input calExp images for the exposure to
413 process.
414 calBkgArray : `list` of `lsst.afw.math.BackgroundList`
415 Array of detector input background lists matching the
416 calExps to process.
417 skyCalibs : `list` of `lsst.afw.image.Exposure`
418 Array of SKY calibrations for the input detectors to be
419 processed.
421 Camera matching the input data to process.
422
423 Returns
424 -------
425 results : `pipeBase.Struct` containing
426 calExpCamera : `lsst.afw.image.Exposure`
427 Full camera image of the sky-corrected data.
428 skyCorr : `list` of `lsst.afw.math.BackgroundList`
429 Detector-level sky-corrected background lists.
430
431 See Also
432 --------
433 ~lsst.pipe.drivers.SkyCorrectionTask.runDataRef()
434 """
435 # To allow SkyCorrectionTask to run in the Gen3 butler
436 # environment, a new run() method was added that performs the
437 # same operations in a serial environment (pipetask processing
438 # does not support MPI processing as of 2019-05-03). Methods
439 # used in runDataRef() are used as appropriate in run(), but
440 # some have been rewritten in serial form. Please ensure that
441 # any updates to runDataRef() or the methods it calls with
442 # pool.mapToPrevious() are duplicated in run() and its
443 # methods.
444 #
445 # Variable names here should match those in runDataRef() as
446 # closely as possible. Variables matching data stored in the
447 # pool cache have a prefix indicating this. Variables that
448 # would be local to an MPI processing client have a prefix
449 # "node".
450 idList = [exp.getDetector().getId() for exp in calExpArray]
451
452 # Construct arrays that match the cache in self.runDataRef() after
453 # self.loadImage() is map/reduced.
454 cacheExposures = []
455 cacheBgList = []
456 exposures = []
457 for calExp, calBgModel in zip(calExpArray, calBkgArray):
458 nodeExp, nodeBgList = self.loadImageRunloadImageRun(calExp, calBgModel)
459 cacheExposures.append(nodeExp)
460 cacheBgList.append(nodeBgList)
461 exposures.append(afwMath.binImage(nodeExp.getMaskedImage(), self.config.binning))
462
463 if self.config.doBgModel:
464 # Generate focal plane background, updating backgrounds in the "cache".
465 exposures, newCacheBgList, cacheBgModel = self.focalPlaneBackgroundRunfocalPlaneBackgroundRun(
466 camera, cacheExposures, idList, self.config.bgModel
467 )
468 for cacheBg, newBg in zip(cacheBgList, newCacheBgList):
469 cacheBg.append(newBg)
470
471 if self.config.doSky:
472 # Measure the sky frame scale on all inputs. Results in
473 # values equal to self.measureSkyFrame() and
474 # self.sky.solveScales() in runDataRef().
475 cacheSky = []
476 measScales = []
477 for cacheExp, skyCalib in zip(cacheExposures, skyCalibs):
478 skyExp = self.sky.exposureToBackground(skyCalib)
479 cacheSky.append(skyExp)
480 scale = self.sky.measureScale(cacheExp.getMaskedImage(), skyExp)
481 measScales.append(scale)
482
483 scale = self.sky.solveScales(measScales)
484 self.log.info("Sky frame scale: %s" % (scale, ))
485
486 # Subtract sky frame, as in self.subtractSkyFrame(), with
487 # appropriate scale from the "cache".
488 exposures = []
489 newBgList = []
490 for cacheExp, nodeSky, nodeBgList in zip(cacheExposures, cacheSky, cacheBgList):
491 self.sky.subtractSkyFrame(cacheExp.getMaskedImage(), nodeSky, scale, nodeBgList)
492 exposures.append(afwMath.binImage(cacheExp.getMaskedImage(), self.config.binning))
493
494 if self.config.doBgModel2:
495 # As above, generate a focal plane background model and
496 # update the cache models.
497 exposures, newBgList, cacheBgModel = self.focalPlaneBackgroundRunfocalPlaneBackgroundRun(
498 camera, cacheExposures, idList, self.config.bgModel2
499 )
500 for cacheBg, newBg in zip(cacheBgList, newBgList):
501 cacheBg.append(newBg)
502
503 # Generate camera-level image of calexp and return it along
504 # with the list of sky corrected background models.
505 image = makeCameraImage(camera, zip(idList, exposures))
506
507 return pipeBase.Struct(
508 calExpCamera=image,
509 skyCorr=cacheBgList,
510 )
511
512 def loadImage(self, cache, dataId):
513 """Load original image and restore the sky
514
515 This method runs on the slave nodes.
516
517 Parameters
518 ----------
519 cache : `lsst.pipe.base.Struct`
520 Process pool cache.
521 dataId : `dict`
522 Data identifier.
523
524 Returns
525 -------
526 exposure : `lsst.afw.image.Exposure`
527 Resultant exposure.
528 """
529 cache.dataId = dataId
530 cache.exposure = cache.butler.get(self.config.calexpType, dataId, immediate=True).clone()
531 bgOld = cache.butler.get("calexpBackground", dataId, immediate=True)
532 image = cache.exposure.getMaskedImage()
533
534 # We're removing the old background, so change the sense of all its components
535 for bgData in bgOld:
536 statsImage = bgData[0].getStatsImage()
537 statsImage *= -1
538
539 image -= bgOld.getImage()
540 cache.bgList = afwMath.BackgroundList()
541 for bgData in bgOld:
542 cache.bgList.append(bgData)
543
544 if self.config.doMaskObjects:
545 self.maskObjects.findObjects(cache.exposure)
546
547 return self.collectcollect(cache)
548
549 def loadImageRun(self, calExp, calExpBkg):
550 """Serial implementation of self.loadImage() for Gen3.
551
552 Load and restore background to calExp and calExpBkg.
553
554 Parameters
555 ----------
556 calExp : `lsst.afw.image.Exposure`
557 Detector level calExp image to process.
558 calExpBkg : `lsst.afw.math.BackgroundList`
559 Detector level background list associated with the calExp.
560
561 Returns
562 -------
563 calExp : `lsst.afw.image.Exposure`
564 Background restored calExp.
565 bgList : `lsst.afw.math.BackgroundList`
566 New background list containing the restoration background.
567 """
568 image = calExp.getMaskedImage()
569
570 for bgOld in calExpBkg:
571 statsImage = bgOld[0].getStatsImage()
572 statsImage *= -1
573
574 image -= calExpBkg.getImage()
575 bgList = afwMath.BackgroundList()
576 for bgData in calExpBkg:
577 bgList.append(bgData)
578
579 if self.config.doMaskObjects:
580 self.maskObjects.findObjects(calExp)
581
582 return (calExp, bgList)
583
584 def measureSkyFrame(self, cache, dataId):
585 """Measure scale for sky frame
586
587 This method runs on the slave nodes.
588
589 Parameters
590 ----------
591 cache : `lsst.pipe.base.Struct`
592 Process pool cache.
593 dataId : `dict`
594 Data identifier.
595
596 Returns
597 -------
598 scale : `float`
599 Scale for sky frame.
600 """
601 assert cache.dataId == dataId
602 cache.sky = self.sky.getSkyData(cache.butler, dataId)
603 scale = self.sky.measureScale(cache.exposure.getMaskedImage(), cache.sky)
604 return scale
605
606 def subtractSkyFrame(self, cache, dataId, scale):
607 """Subtract sky frame
608
609 This method runs on the slave nodes.
610
611 Parameters
612 ----------
613 cache : `lsst.pipe.base.Struct`
614 Process pool cache.
615 dataId : `dict`
616 Data identifier.
617 scale : `float`
618 Scale for sky frame.
619
620 Returns
621 -------
622 exposure : `lsst.afw.image.Exposure`
623 Resultant exposure.
624 """
625 assert cache.dataId == dataId
626 self.sky.subtractSkyFrame(cache.exposure.getMaskedImage(), cache.sky, scale, cache.bgList)
627 return self.collectcollect(cache)
628
629 def accumulateModel(self, cache, data):
630 """Fit background model for CCD
631
632 This method runs on the slave nodes.
633
634 Parameters
635 ----------
636 cache : `lsst.pipe.base.Struct`
637 Process pool cache.
638 data : `lsst.pipe.base.Struct`
639 Data identifier, with `dataId` (data identifier) and `bgModel`
640 (background model) elements.
641
642 Returns
643 -------
645 Background model.
646 """
647 assert cache.dataId == data.dataId
648 data.bgModel.addCcd(cache.exposure)
649 return data.bgModel
650
651 def subtractModel(self, cache, dataId, bgModel):
652 """Subtract background model
653
654 This method runs on the slave nodes.
655
656 Parameters
657 ----------
658 cache : `lsst.pipe.base.Struct`
659 Process pool cache.
660 dataId : `dict`
661 Data identifier.
662 bgModel : `lsst.pipe.drivers.background.FocalPlaneBackround`
663 Background model.
664
665 Returns
666 -------
667 exposure : `lsst.afw.image.Exposure`
668 Resultant exposure.
669 """
670 assert cache.dataId == dataId
671 exposure = cache.exposure
672 image = exposure.getMaskedImage()
673 detector = exposure.getDetector()
674 bbox = image.getBBox()
675 try:
676 cache.bgModel = bgModel.toCcdBackground(detector, bbox)
677 image -= cache.bgModel.getImage()
678 except RuntimeError:
679 self.log.error(f"There was an error processing {dataId}, no calib file produced")
680 return
681 cache.bgList.append(cache.bgModel[0])
682 return self.collectcollect(cache)
683
684 def subtractModelRun(self, exposure, bgModel):
685 """Serial implementation of self.subtractModel() for Gen3.
686
687 Load and restore background to calExp and calExpBkg.
688
689 Parameters
690 ----------
691 exposure : `lsst.afw.image.Exposure`
692 Exposure to subtract the background model from.
694 Full camera level background model.
695
696 Returns
697 -------
698 exposure : `lsst.afw.image.Exposure`
699 Background subtracted input exposure.
700 bgModelCcd : `lsst.afw.math.BackgroundList`
701 Detector level realization of the full background model.
702 bgModelMaskedImage : `lsst.afw.image.MaskedImage`
703 Background model from the bgModelCcd realization.
704 """
705 image = exposure.getMaskedImage()
706 detector = exposure.getDetector()
707 bbox = image.getBBox()
708 bgModelCcd = bgModel.toCcdBackground(detector, bbox)
709 image -= bgModelCcd.getImage()
710
711 return (exposure, bgModelCcd, bgModelCcd[0])
712
713 def realiseModel(self, cache, dataId, bgModel):
714 """Generate an image of the background model for visualisation
715
716 Useful for debugging.
717
718 Parameters
719 ----------
720 cache : `lsst.pipe.base.Struct`
721 Process pool cache.
722 dataId : `dict`
723 Data identifier.
724 bgModel : `lsst.pipe.drivers.background.FocalPlaneBackround`
725 Background model.
726
727 Returns
728 -------
729 detId : `int`
730 Detector identifier.
732 Binned background model image.
733 """
734 assert cache.dataId == dataId
735 exposure = cache.exposure
736 detector = exposure.getDetector()
737 bbox = exposure.getMaskedImage().getBBox()
738 image = bgModel.toCcdBackground(detector, bbox).getImage()
739 return self.collectBinnedImagecollectBinnedImage(exposure, image)
740
741 def collectBinnedImage(self, exposure, image):
742 """Return the binned image required for visualization
743
744 This method just helps to cut down on boilerplate.
745
746 Parameters
747 ----------
749 Image to go into visualisation.
750
751 Returns
752 -------
753 detId : `int`
754 Detector identifier.
756 Binned image.
757 """
758 return (exposure.getDetector().getId(), afwMath.binImage(image, self.config.binning))
759
760 def collect(self, cache):
761 """Collect exposure for potential visualisation
762
763 This method runs on the slave nodes.
764
765 Parameters
766 ----------
767 cache : `lsst.pipe.base.Struct`
768 Process pool cache.
769
770 Returns
771 -------
772 detId : `int`
773 Detector identifier.
775 Binned image.
776 """
777 return self.collectBinnedImagecollectBinnedImage(cache.exposure, cache.exposure.maskedImage)
778
779 def collectOriginal(self, cache, dataId):
780 """Collect original image for visualisation
781
782 This method runs on the slave nodes.
783
784 Parameters
785 ----------
786 cache : `lsst.pipe.base.Struct`
787 Process pool cache.
788 dataId : `dict`
789 Data identifier.
790
791 Returns
792 -------
793 detId : `int`
794 Detector identifier.
796 Binned image.
797 """
798 exposure = cache.butler.get("calexp", dataId, immediate=True)
799 return self.collectBinnedImagecollectBinnedImage(exposure, exposure.maskedImage)
800
801 def collectSky(self, cache, dataId):
802 """Collect original image for visualisation
803
804 This method runs on the slave nodes.
805
806 Parameters
807 ----------
808 cache : `lsst.pipe.base.Struct`
809 Process pool cache.
810 dataId : `dict`
811 Data identifier.
812
813 Returns
814 -------
815 detId : `int`
816 Detector identifier.
818 Binned image.
819 """
820 return self.collectBinnedImagecollectBinnedImage(cache.exposure, cache.sky.getImage())
821
822 def collectMask(self, cache, dataId):
823 """Collect mask for visualisation
824
825 This method runs on the slave nodes.
826
827 Parameters
828 ----------
829 cache : `lsst.pipe.base.Struct`
830 Process pool cache.
831 dataId : `dict`
832 Data identifier.
833
834 Returns
835 -------
836 detId : `int`
837 Detector identifier.
838 image : `lsst.afw.image.Image`
839 Binned image.
840 """
841 # Convert Mask to floating-point image, because that's what's required for focal plane construction
842 image = afwImage.ImageF(cache.exposure.maskedImage.getBBox())
843 image.array[:] = cache.exposure.maskedImage.mask.array
844 return self.collectBinnedImagecollectBinnedImage(cache.exposure, image)
845
846 def write(self, cache, dataId):
847 """Write resultant background list
848
849 This method runs on the slave nodes.
850
851 Parameters
852 ----------
853 cache : `lsst.pipe.base.Struct`
854 Process pool cache.
855 dataId : `dict`
856 Data identifier.
857 """
858 cache.butler.put(cache.bgList, "skyCorr", dataId)
859
860 def _getMetadataName(self):
861 """There's no metadata to write out"""
862 return None
def logOperation(self, operation, catch=False, trace=True)
def run(self, calExpArray, calBkgArray, skyCalibs, camera)
def subtractModel(self, cache, dataId, bgModel)
def focalPlaneBackground(self, camera, pool, dataIdList, config)
def subtractSkyFrame(self, cache, dataId, scale)
def focalPlaneBackgroundRun(self, camera, cacheExposures, idList, config)
def runQuantum(self, butlerQC, inputRefs, outputRefs)
def batchWallTime(cls, time, parsedCmd, numCores)
def realiseModel(self, cache, dataId, bgModel)
def makeCameraImage(camera, exposures, filename=None, binning=8)
def reorderAndPadList(inputList, inputKeys, outputKeys, padWith=None)