lsst.pipe.drivers gd00fb1e517+b2d1123003
skyCorrection.py
Go to the documentation of this file.
1# This file is part of pipe_drivers.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21import lsst.afw.math as afwMath
22import lsst.afw.image as afwImage
23import lsst.pipe.base as pipeBase
24
25from lsst.pipe.base import ArgumentParser, ConfigDatasetType
26from lsst.daf.butler import DimensionGraph
27from lsst.pex.config import Config, Field, ConfigurableField, ConfigField
28from lsst.ctrl.pool.pool import Pool
29from lsst.ctrl.pool.parallel import BatchPoolTask
30from lsst.pipe.drivers.background import (SkyMeasurementTask, FocalPlaneBackground,
31 FocalPlaneBackgroundConfig, MaskObjectsTask)
32import lsst.pipe.drivers.visualizeVisit as visualizeVisit
33import lsst.pipe.base.connectionTypes as cT
34
35__all__ = ["SkyCorrectionConfig", "SkyCorrectionTask"]
36
37DEBUG = False # Debugging outputs?
38
39
40def reorderAndPadList(inputList, inputKeys, outputKeys, padWith=None):
41 """Match the order of one list to another, padding if necessary
42
43 Parameters
44 ----------
45 inputList : list
46 List to be reordered and padded. Elements can be any type.
47 inputKeys : iterable
48 Iterable of values to be compared with outputKeys.
49 Length must match `inputList`
50 outputKeys : iterable
51 Iterable of values to be compared with inputKeys.
52 padWith :
53 Any value to be inserted where inputKey not in outputKeys
54
55 Returns
56 -------
57 list
58 Copy of inputList reordered per outputKeys and padded with `padWith`
59 so that the length matches length of outputKeys.
60 """
61 outputList = []
62 for d in outputKeys:
63 if d in inputKeys:
64 outputList.append(inputList[inputKeys.index(d)])
65 else:
66 outputList.append(padWith)
67 return outputList
68
69
70def makeCameraImage(camera, exposures, filename=None, binning=8):
71 """Make and write an image of an entire focal plane
72
73 Parameters
74 ----------
76 Camera description.
77 exposures : `list` of `tuple` of `int` and `lsst.afw.image.Exposure`
78 List of detector ID and CCD exposures (binned by `binning`).
79 filename : `str`, optional
80 Output filename.
81 binning : `int`
82 Binning size that has been applied to images.
83 """
84 image = visualizeVisit.makeCameraImage(camera, dict(exp for exp in exposures if exp is not None), binning)
85 if filename is not None:
86 image.writeFits(filename)
87 return image
88
89
90def _skyLookup(datasetType, registry, quantumDataId, collections):
91 """Lookup function to identify sky frames
92
93 Parameters
94 ----------
95 datasetType : `lsst.daf.butler.DatasetType`
96 Dataset to lookup.
97 registry : `lsst.daf.butler.Registry`
98 Butler registry to query.
99 quantumDataId : `lsst.daf.butler.DataCoordinate`
100 Data id to transform to find sky frames.
101 The ``detector`` entry will be stripped.
102 collections : `lsst.daf.butler.CollectionSearch`
103 Collections to search through.
104
105 Returns
106 -------
107 results : `list` [`lsst.daf.butler.DatasetRef`]
108 List of datasets that will be used as sky calibration frames
109 """
110 newDataId = quantumDataId.subset(DimensionGraph(registry.dimensions, names=["instrument", "visit"]))
111 skyFrames = []
112 for dataId in registry.queryDataIds(["visit", "detector"], dataId=newDataId).expanded():
113 skyFrame = registry.findDataset(datasetType, dataId, collections=collections,
114 timespan=dataId.timespan)
115 skyFrames.append(skyFrame)
116
117 return skyFrames
118
119
120class SkyCorrectionConnections(pipeBase.PipelineTaskConnections, dimensions=("instrument", "visit")):
121 rawLinker = cT.Input(
122 doc="Raw data to provide exp-visit linkage to connect calExp inputs to camera/sky calibs.",
123 name="raw",
124 multiple=True,
125 deferLoad=True,
126 storageClass="Exposure",
127 dimensions=["instrument", "exposure", "detector"],
128 )
129 calExpArray = cT.Input(
130 doc="Input exposures to process",
131 name="calexp",
132 multiple=True,
133 storageClass="ExposureF",
134 dimensions=["instrument", "visit", "detector"],
135 )
136 calBkgArray = cT.Input(
137 doc="Input background files to use",
138 multiple=True,
139 name="calexpBackground",
140 storageClass="Background",
141 dimensions=["instrument", "visit", "detector"],
142 )
143 camera = cT.PrerequisiteInput(
144 doc="Input camera to use.",
145 name="camera",
146 storageClass="Camera",
147 dimensions=["instrument"],
148 isCalibration=True,
149 )
150 skyCalibs = cT.PrerequisiteInput(
151 doc="Input sky calibrations to use.",
152 name="sky",
153 multiple=True,
154 storageClass="ExposureF",
155 dimensions=["instrument", "physical_filter", "detector"],
156 isCalibration=True,
157 lookupFunction=_skyLookup,
158 )
159 calExpCamera = cT.Output(
160 doc="Output camera image.",
161 name='calexp_camera',
162 storageClass="ImageF",
163 dimensions=["instrument", "visit"],
164 )
165 skyCorr = cT.Output(
166 doc="Output sky corrected images.",
167 name='skyCorr',
168 multiple=True,
169 storageClass="Background",
170 dimensions=["instrument", "visit", "detector"],
171 )
172
173
174class SkyCorrectionConfig(pipeBase.PipelineTaskConfig, pipelineConnections=SkyCorrectionConnections):
175 """Configuration for SkyCorrectionTask"""
176 bgModel = ConfigField(dtype=FocalPlaneBackgroundConfig, doc="Background model")
177 bgModel2 = ConfigField(dtype=FocalPlaneBackgroundConfig, doc="2nd Background model")
178 sky = ConfigurableField(target=SkyMeasurementTask, doc="Sky measurement")
179 maskObjects = ConfigurableField(target=MaskObjectsTask, doc="Mask Objects")
180 doMaskObjects = Field(dtype=bool, default=True, doc="Mask objects to find good sky?")
181 doBgModel = Field(dtype=bool, default=True, doc="Do background model subtraction?")
182 doBgModel2 = Field(dtype=bool, default=True, doc="Do cleanup background model subtraction?")
183 doSky = Field(dtype=bool, default=True, doc="Do sky frame subtraction?")
184 binning = Field(dtype=int, default=8, doc="Binning factor for constructing focal-plane images")
185 calexpType = Field(dtype=str, default="calexp",
186 doc="Should be set to fakes_calexp if you want to process calexps with fakes in.")
187
188 def setDefaults(self):
189 Config.setDefaults(self)
190 self.bgModel2bgModel2.doSmooth = True
191 self.bgModel2bgModel2.minFrac = 0.5
192 self.bgModel2bgModel2.xSize = 256
193 self.bgModel2bgModel2.ySize = 256
194 self.bgModel2bgModel2.smoothScale = 1.0
195
196
197class SkyCorrectionTask(pipeBase.PipelineTask, BatchPoolTask):
198 """Correct sky over entire focal plane"""
199 ConfigClass = SkyCorrectionConfig
200 _DefaultName = "skyCorr"
201
202 def runQuantum(self, butlerQC, inputRefs, outputRefs):
203
204 # reorder skyCalibs and calBkgArray per calExpArray
205 detectorOrder = [ref.dataId['detector'] for ref in inputRefs.calExpArray]
206 inputRefs.skyCalibs = reorderAndPadList(inputRefs.skyCalibs,
207 [ref.dataId['detector'] for ref in inputRefs.skyCalibs],
208 detectorOrder)
209 inputRefs.calBkgArray = reorderAndPadList(inputRefs.calBkgArray,
210 [ref.dataId['detector'] for ref in inputRefs.calBkgArray],
211 detectorOrder)
212 outputRefs.skyCorr = reorderAndPadList(outputRefs.skyCorr,
213 [ref.dataId['detector'] for ref in outputRefs.skyCorr],
214 detectorOrder)
215 inputs = butlerQC.get(inputRefs)
216 inputs.pop("rawLinker", None)
217 outputs = self.runrun(**inputs)
218 butlerQC.put(outputs, outputRefs)
219
220 def __init__(self, *args, **kwargs):
221 super().__init__(**kwargs)
222
223 self.makeSubtask("sky")
224 self.makeSubtask("maskObjects")
225
226 @classmethod
227 def _makeArgumentParser(cls, *args, **kwargs):
228 kwargs.pop("doBatch", False)
229 datasetType = ConfigDatasetType(name="calexpType")
230 parser = ArgumentParser(name="skyCorr", *args, **kwargs)
231 parser.add_id_argument("--id", datasetType=datasetType, level="visit",
232 help="data ID, e.g. --id visit=12345")
233 return parser
234
235 @classmethod
236 def batchWallTime(cls, time, parsedCmd, numCores):
237 """Return walltime request for batch job
238
239 Subclasses should override if the walltime should be calculated
240 differently (e.g., addition of some serial time).
241
242 Parameters
243 ----------
244 time : `float`
245 Requested time per iteration.
246 parsedCmd : `argparse.Namespace`
247 Results of argument parsing.
248 numCores : `int`
249 Number of cores.
250 """
251 numTargets = len(cls.RunnerClass.getTargetList(parsedCmd))
252 return time*numTargets
253
254 def runDataRef(self, expRef):
255 """Perform sky correction on an exposure
256
257 We restore the original sky, and remove it again using multiple
258 algorithms. We optionally apply:
259
260 1. A large-scale background model.
261 This step removes very-large-scale sky such as moonlight.
262 2. A sky frame.
263 3. A medium-scale background model.
264 This step removes residual sky (This is smooth on the focal plane).
265
266 Only the master node executes this method. The data is held on
267 the slave nodes, which do all the hard work.
268
269 Parameters
270 ----------
271 expRef : `lsst.daf.persistence.ButlerDataRef`
272 Data reference for exposure.
273
274 See Also
275 --------
276 ~lsst.pipe.drivers.SkyCorrectionTask.run
277 """
278 if DEBUG:
279 extension = "-%(visit)d.fits" % expRef.dataId
280
281 with self.logOperationlogOperation("processing %s" % (expRef.dataId,)):
282 pool = Pool()
283 pool.cacheClear()
284 pool.storeSet(butler=expRef.getButler())
285 camera = expRef.get("camera")
286
287 dataIdList = [ccdRef.dataId for ccdRef in expRef.subItems("ccd") if
288 ccdRef.datasetExists(self.config.calexpType)]
289
290 exposures = pool.map(self.loadImageloadImage, dataIdList)
291 if DEBUG:
292 makeCameraImage(camera, exposures, "restored" + extension)
293 exposures = pool.mapToPrevious(self.collectOriginalcollectOriginal, dataIdList)
294 makeCameraImage(camera, exposures, "original" + extension)
295 exposures = pool.mapToPrevious(self.collectMaskcollectMask, dataIdList)
296 makeCameraImage(camera, exposures, "mask" + extension)
297
298 if self.config.doBgModel:
299 exposures = self.focalPlaneBackgroundfocalPlaneBackground(camera, pool, dataIdList, self.config.bgModel)
300
301 if self.config.doSky:
302 measScales = pool.mapToPrevious(self.measureSkyFramemeasureSkyFrame, dataIdList)
303 scale = self.sky.solveScales(measScales)
304 self.log.info("Sky frame scale: %s" % (scale,))
305
306 exposures = pool.mapToPrevious(self.subtractSkyFramesubtractSkyFrame, dataIdList, scale)
307 if DEBUG:
308 makeCameraImage(camera, exposures, "skysub" + extension)
309 calibs = pool.mapToPrevious(self.collectSkycollectSky, dataIdList)
310 makeCameraImage(camera, calibs, "sky" + extension)
311
312 if self.config.doBgModel2:
313 exposures = self.focalPlaneBackgroundfocalPlaneBackground(camera, pool, dataIdList, self.config.bgModel2)
314
315 # Persist camera-level image of calexp
316 image = makeCameraImage(camera, exposures)
317 expRef.put(image, "calexp_camera")
318
319 pool.mapToPrevious(self.writewrite, dataIdList)
320
321 def focalPlaneBackground(self, camera, pool, dataIdList, config):
322 """Perform full focal-plane background subtraction
323
324 This method runs on the master node.
325
326 Parameters
327 ----------
329 Camera description.
330 pool : `lsst.ctrl.pool.Pool`
331 Process pool.
332 dataIdList : iterable of `dict`
333 List of data identifiers for the CCDs.
335 Configuration to use for background subtraction.
336
337 Returns
338 -------
339 exposures : `list` of `lsst.afw.image.Image`
340 List of binned images, for creating focal plane image.
341 """
342 bgModel = FocalPlaneBackground.fromCamera(config, camera)
343 data = [pipeBase.Struct(dataId=dataId, bgModel=bgModel.clone()) for dataId in dataIdList]
344 bgModelList = pool.mapToPrevious(self.accumulateModelaccumulateModel, data)
345 for ii, bg in enumerate(bgModelList):
346 self.log.info("Background %d: %d pixels", ii, bg._numbers.array.sum())
347 bgModel.merge(bg)
348 return pool.mapToPrevious(self.subtractModelsubtractModel, dataIdList, bgModel)
349
350 def focalPlaneBackgroundRun(self, camera, cacheExposures, idList, config):
351 """Perform full focal-plane background subtraction
352
353 This method runs on the master node.
354
355 Parameters
356 ----------
358 Camera description.
359 cacheExposures : `list` of `lsst.afw.image.Exposures`
360 List of loaded and processed input calExp.
361 idList : `list` of `int`
362 List of detector ids to iterate over.
364 Configuration to use for background subtraction.
365
366 Returns
367 -------
368 exposures : `list` of `lsst.afw.image.Image`
369 List of binned images, for creating focal plane image.
370 newCacheBgList : `list` of `lsst.afwMath.backgroundList`
371 Background lists generated.
372 cacheBgModel : `FocalPlaneBackground`
373 Full focal plane background model.
374 """
375 bgModel = FocalPlaneBackground.fromCamera(config, camera)
376 data = [pipeBase.Struct(id=id, bgModel=bgModel.clone()) for id in idList]
377
378 bgModelList = []
379 for nodeData, cacheExp in zip(data, cacheExposures):
380 nodeData.bgModel.addCcd(cacheExp)
381 bgModelList.append(nodeData.bgModel)
382
383 for ii, bg in enumerate(bgModelList):
384 self.log.info("Background %d: %d pixels", ii, bg._numbers.getArray().sum())
385 bgModel.merge(bg)
386
387 exposures = []
388 newCacheBgList = []
389 cacheBgModel = []
390 for cacheExp in cacheExposures:
391 nodeExp, nodeBgModel, nodeBgList = self.subtractModelRunsubtractModelRun(cacheExp, bgModel)
392 exposures.append(afwMath.binImage(nodeExp.getMaskedImage(), self.config.binning))
393 cacheBgModel.append(nodeBgModel)
394 newCacheBgList.append(nodeBgList)
395
396 return exposures, newCacheBgList, cacheBgModel
397
398 def run(self, calExpArray, calBkgArray, skyCalibs, camera):
399 """Duplicate runDataRef method without ctrl_pool for Gen3.
400
401 Parameters
402 ----------
403 calExpArray : `list` of `lsst.afw.image.Exposure`
404 Array of detector input calExp images for the exposure to
405 process.
406 calBkgArray : `list` of `lsst.afw.math.BackgroundList`
407 Array of detector input background lists matching the
408 calExps to process.
409 skyCalibs : `list` of `lsst.afw.image.Exposure`
410 Array of SKY calibrations for the input detectors to be
411 processed.
413 Camera matching the input data to process.
414
415 Returns
416 -------
417 results : `pipeBase.Struct` containing
418 calExpCamera : `lsst.afw.image.Exposure`
419 Full camera image of the sky-corrected data.
420 skyCorr : `list` of `lsst.afw.math.BackgroundList`
421 Detector-level sky-corrected background lists.
422
423 See Also
424 --------
425 ~lsst.pipe.drivers.SkyCorrectionTask.runDataRef()
426 """
427 # To allow SkyCorrectionTask to run in the Gen3 butler
428 # environment, a new run() method was added that performs the
429 # same operations in a serial environment (pipetask processing
430 # does not support MPI processing as of 2019-05-03). Methods
431 # used in runDataRef() are used as appropriate in run(), but
432 # some have been rewritten in serial form. Please ensure that
433 # any updates to runDataRef() or the methods it calls with
434 # pool.mapToPrevious() are duplicated in run() and its
435 # methods.
436 #
437 # Variable names here should match those in runDataRef() as
438 # closely as possible. Variables matching data stored in the
439 # pool cache have a prefix indicating this. Variables that
440 # would be local to an MPI processing client have a prefix
441 # "node".
442 idList = [exp.getDetector().getId() for exp in calExpArray]
443
444 # Construct arrays that match the cache in self.runDataRef() after
445 # self.loadImage() is map/reduced.
446 cacheExposures = []
447 cacheBgList = []
448 exposures = []
449 for calExp, calBgModel in zip(calExpArray, calBkgArray):
450 nodeExp, nodeBgList = self.loadImageRunloadImageRun(calExp, calBgModel)
451 cacheExposures.append(nodeExp)
452 cacheBgList.append(nodeBgList)
453 exposures.append(afwMath.binImage(nodeExp.getMaskedImage(), self.config.binning))
454
455 if self.config.doBgModel:
456 # Generate focal plane background, updating backgrounds in the "cache".
457 exposures, newCacheBgList, cacheBgModel = self.focalPlaneBackgroundRunfocalPlaneBackgroundRun(
458 camera, cacheExposures, idList, self.config.bgModel
459 )
460 for cacheBg, newBg in zip(cacheBgList, newCacheBgList):
461 cacheBg.append(newBg)
462
463 if self.config.doSky:
464 # Measure the sky frame scale on all inputs. Results in
465 # values equal to self.measureSkyFrame() and
466 # self.sky.solveScales() in runDataRef().
467 cacheSky = []
468 measScales = []
469 for cacheExp, skyCalib in zip(cacheExposures, skyCalibs):
470 skyExp = self.sky.exposureToBackground(skyCalib)
471 cacheSky.append(skyExp)
472 scale = self.sky.measureScale(cacheExp.getMaskedImage(), skyExp)
473 measScales.append(scale)
474
475 scale = self.sky.solveScales(measScales)
476 self.log.info("Sky frame scale: %s" % (scale, ))
477
478 # Subtract sky frame, as in self.subtractSkyFrame(), with
479 # appropriate scale from the "cache".
480 exposures = []
481 newBgList = []
482 for cacheExp, nodeSky, nodeBgList in zip(cacheExposures, cacheSky, cacheBgList):
483 self.sky.subtractSkyFrame(cacheExp.getMaskedImage(), nodeSky, scale, nodeBgList)
484 exposures.append(afwMath.binImage(cacheExp.getMaskedImage(), self.config.binning))
485
486 if self.config.doBgModel2:
487 # As above, generate a focal plane background model and
488 # update the cache models.
489 exposures, newBgList, cacheBgModel = self.focalPlaneBackgroundRunfocalPlaneBackgroundRun(
490 camera, cacheExposures, idList, self.config.bgModel2
491 )
492 for cacheBg, newBg in zip(cacheBgList, newBgList):
493 cacheBg.append(newBg)
494
495 # Generate camera-level image of calexp and return it along
496 # with the list of sky corrected background models.
497 image = makeCameraImage(camera, zip(idList, exposures))
498
499 return pipeBase.Struct(
500 calExpCamera=image,
501 skyCorr=cacheBgList,
502 )
503
504 def loadImage(self, cache, dataId):
505 """Load original image and restore the sky
506
507 This method runs on the slave nodes.
508
509 Parameters
510 ----------
511 cache : `lsst.pipe.base.Struct`
512 Process pool cache.
513 dataId : `dict`
514 Data identifier.
515
516 Returns
517 -------
518 exposure : `lsst.afw.image.Exposure`
519 Resultant exposure.
520 """
521 cache.dataId = dataId
522 cache.exposure = cache.butler.get(self.config.calexpType, dataId, immediate=True).clone()
523 bgOld = cache.butler.get("calexpBackground", dataId, immediate=True)
524 image = cache.exposure.getMaskedImage()
525
526 # We're removing the old background, so change the sense of all its components
527 for bgData in bgOld:
528 statsImage = bgData[0].getStatsImage()
529 statsImage *= -1
530
531 image -= bgOld.getImage()
532 cache.bgList = afwMath.BackgroundList()
533 for bgData in bgOld:
534 cache.bgList.append(bgData)
535
536 if self.config.doMaskObjects:
537 self.maskObjects.findObjects(cache.exposure)
538
539 return self.collectcollect(cache)
540
541 def loadImageRun(self, calExp, calExpBkg):
542 """Serial implementation of self.loadImage() for Gen3.
543
544 Load and restore background to calExp and calExpBkg.
545
546 Parameters
547 ----------
548 calExp : `lsst.afw.image.Exposure`
549 Detector level calExp image to process.
550 calExpBkg : `lsst.afw.math.BackgroundList`
551 Detector level background list associated with the calExp.
552
553 Returns
554 -------
555 calExp : `lsst.afw.image.Exposure`
556 Background restored calExp.
557 bgList : `lsst.afw.math.BackgroundList`
558 New background list containing the restoration background.
559 """
560 image = calExp.getMaskedImage()
561
562 for bgOld in calExpBkg:
563 statsImage = bgOld[0].getStatsImage()
564 statsImage *= -1
565
566 image -= calExpBkg.getImage()
567 bgList = afwMath.BackgroundList()
568 for bgData in calExpBkg:
569 bgList.append(bgData)
570
571 if self.config.doMaskObjects:
572 self.maskObjects.findObjects(calExp)
573
574 return (calExp, bgList)
575
576 def measureSkyFrame(self, cache, dataId):
577 """Measure scale for sky frame
578
579 This method runs on the slave nodes.
580
581 Parameters
582 ----------
583 cache : `lsst.pipe.base.Struct`
584 Process pool cache.
585 dataId : `dict`
586 Data identifier.
587
588 Returns
589 -------
590 scale : `float`
591 Scale for sky frame.
592 """
593 assert cache.dataId == dataId
594 cache.sky = self.sky.getSkyData(cache.butler, dataId)
595 scale = self.sky.measureScale(cache.exposure.getMaskedImage(), cache.sky)
596 return scale
597
598 def subtractSkyFrame(self, cache, dataId, scale):
599 """Subtract sky frame
600
601 This method runs on the slave nodes.
602
603 Parameters
604 ----------
605 cache : `lsst.pipe.base.Struct`
606 Process pool cache.
607 dataId : `dict`
608 Data identifier.
609 scale : `float`
610 Scale for sky frame.
611
612 Returns
613 -------
614 exposure : `lsst.afw.image.Exposure`
615 Resultant exposure.
616 """
617 assert cache.dataId == dataId
618 self.sky.subtractSkyFrame(cache.exposure.getMaskedImage(), cache.sky, scale, cache.bgList)
619 return self.collectcollect(cache)
620
621 def accumulateModel(self, cache, data):
622 """Fit background model for CCD
623
624 This method runs on the slave nodes.
625
626 Parameters
627 ----------
628 cache : `lsst.pipe.base.Struct`
629 Process pool cache.
630 data : `lsst.pipe.base.Struct`
631 Data identifier, with `dataId` (data identifier) and `bgModel`
632 (background model) elements.
633
634 Returns
635 -------
637 Background model.
638 """
639 assert cache.dataId == data.dataId
640 data.bgModel.addCcd(cache.exposure)
641 return data.bgModel
642
643 def subtractModel(self, cache, dataId, bgModel):
644 """Subtract background model
645
646 This method runs on the slave nodes.
647
648 Parameters
649 ----------
650 cache : `lsst.pipe.base.Struct`
651 Process pool cache.
652 dataId : `dict`
653 Data identifier.
654 bgModel : `lsst.pipe.drivers.background.FocalPlaneBackround`
655 Background model.
656
657 Returns
658 -------
659 exposure : `lsst.afw.image.Exposure`
660 Resultant exposure.
661 """
662 assert cache.dataId == dataId
663 exposure = cache.exposure
664 image = exposure.getMaskedImage()
665 detector = exposure.getDetector()
666 bbox = image.getBBox()
667 try:
668 cache.bgModel = bgModel.toCcdBackground(detector, bbox)
669 image -= cache.bgModel.getImage()
670 except RuntimeError:
671 self.log.error(f"There was an error processing {dataId}, no calib file produced")
672 return
673 cache.bgList.append(cache.bgModel[0])
674 return self.collectcollect(cache)
675
676 def subtractModelRun(self, exposure, bgModel):
677 """Serial implementation of self.subtractModel() for Gen3.
678
679 Load and restore background to calExp and calExpBkg.
680
681 Parameters
682 ----------
683 exposure : `lsst.afw.image.Exposure`
684 Exposure to subtract the background model from.
686 Full camera level background model.
687
688 Returns
689 -------
690 exposure : `lsst.afw.image.Exposure`
691 Background subtracted input exposure.
692 bgModelCcd : `lsst.afw.math.BackgroundList`
693 Detector level realization of the full background model.
694 bgModelMaskedImage : `lsst.afw.image.MaskedImage`
695 Background model from the bgModelCcd realization.
696 """
697 image = exposure.getMaskedImage()
698 detector = exposure.getDetector()
699 bbox = image.getBBox()
700 bgModelCcd = bgModel.toCcdBackground(detector, bbox)
701 image -= bgModelCcd.getImage()
702
703 return (exposure, bgModelCcd, bgModelCcd[0])
704
705 def realiseModel(self, cache, dataId, bgModel):
706 """Generate an image of the background model for visualisation
707
708 Useful for debugging.
709
710 Parameters
711 ----------
712 cache : `lsst.pipe.base.Struct`
713 Process pool cache.
714 dataId : `dict`
715 Data identifier.
716 bgModel : `lsst.pipe.drivers.background.FocalPlaneBackround`
717 Background model.
718
719 Returns
720 -------
721 detId : `int`
722 Detector identifier.
724 Binned background model image.
725 """
726 assert cache.dataId == dataId
727 exposure = cache.exposure
728 detector = exposure.getDetector()
729 bbox = exposure.getMaskedImage().getBBox()
730 image = bgModel.toCcdBackground(detector, bbox).getImage()
731 return self.collectBinnedImagecollectBinnedImage(exposure, image)
732
733 def collectBinnedImage(self, exposure, image):
734 """Return the binned image required for visualization
735
736 This method just helps to cut down on boilerplate.
737
738 Parameters
739 ----------
741 Image to go into visualisation.
742
743 Returns
744 -------
745 detId : `int`
746 Detector identifier.
748 Binned image.
749 """
750 return (exposure.getDetector().getId(), afwMath.binImage(image, self.config.binning))
751
752 def collect(self, cache):
753 """Collect exposure for potential visualisation
754
755 This method runs on the slave nodes.
756
757 Parameters
758 ----------
759 cache : `lsst.pipe.base.Struct`
760 Process pool cache.
761
762 Returns
763 -------
764 detId : `int`
765 Detector identifier.
767 Binned image.
768 """
769 return self.collectBinnedImagecollectBinnedImage(cache.exposure, cache.exposure.maskedImage)
770
771 def collectOriginal(self, cache, dataId):
772 """Collect original image for visualisation
773
774 This method runs on the slave nodes.
775
776 Parameters
777 ----------
778 cache : `lsst.pipe.base.Struct`
779 Process pool cache.
780 dataId : `dict`
781 Data identifier.
782
783 Returns
784 -------
785 detId : `int`
786 Detector identifier.
788 Binned image.
789 """
790 exposure = cache.butler.get("calexp", dataId, immediate=True)
791 return self.collectBinnedImagecollectBinnedImage(exposure, exposure.maskedImage)
792
793 def collectSky(self, cache, dataId):
794 """Collect original image for visualisation
795
796 This method runs on the slave nodes.
797
798 Parameters
799 ----------
800 cache : `lsst.pipe.base.Struct`
801 Process pool cache.
802 dataId : `dict`
803 Data identifier.
804
805 Returns
806 -------
807 detId : `int`
808 Detector identifier.
810 Binned image.
811 """
812 return self.collectBinnedImagecollectBinnedImage(cache.exposure, cache.sky.getImage())
813
814 def collectMask(self, cache, dataId):
815 """Collect mask for visualisation
816
817 This method runs on the slave nodes.
818
819 Parameters
820 ----------
821 cache : `lsst.pipe.base.Struct`
822 Process pool cache.
823 dataId : `dict`
824 Data identifier.
825
826 Returns
827 -------
828 detId : `int`
829 Detector identifier.
830 image : `lsst.afw.image.Image`
831 Binned image.
832 """
833 # Convert Mask to floating-point image, because that's what's required for focal plane construction
834 image = afwImage.ImageF(cache.exposure.maskedImage.getBBox())
835 image.array[:] = cache.exposure.maskedImage.mask.array
836 return self.collectBinnedImagecollectBinnedImage(cache.exposure, image)
837
838 def write(self, cache, dataId):
839 """Write resultant background list
840
841 This method runs on the slave nodes.
842
843 Parameters
844 ----------
845 cache : `lsst.pipe.base.Struct`
846 Process pool cache.
847 dataId : `dict`
848 Data identifier.
849 """
850 cache.butler.put(cache.bgList, "skyCorr", dataId)
851
852 def _getMetadataName(self):
853 """There's no metadata to write out"""
854 return None
def logOperation(self, operation, catch=False, trace=True)
def run(self, calExpArray, calBkgArray, skyCalibs, camera)
def subtractModel(self, cache, dataId, bgModel)
def focalPlaneBackground(self, camera, pool, dataIdList, config)
def subtractSkyFrame(self, cache, dataId, scale)
def focalPlaneBackgroundRun(self, camera, cacheExposures, idList, config)
def runQuantum(self, butlerQC, inputRefs, outputRefs)
def batchWallTime(cls, time, parsedCmd, numCores)
def realiseModel(self, cache, dataId, bgModel)
def makeCameraImage(camera, exposures, filename=None, binning=8)
def reorderAndPadList(inputList, inputKeys, outputKeys, padWith=None)