lsst.pipe.drivers  14.0-15-gbcd3a35
skyCorrection.py
Go to the documentation of this file.
1 from __future__ import absolute_import, division, print_function
2 
3 import numpy
4 
5 import lsst.afw.math as afwMath
6 import lsst.afw.image as afwImage
7 import lsst.meas.algorithms as measAlg
8 
9 from lsst.afw.cameraGeom.utils import makeImageFromCamera
10 from lsst.pipe.base import ArgumentParser, Struct
11 from lsst.pex.config import Config, Field, ConfigurableField, ConfigField
12 from lsst.ctrl.pool.pool import Pool
13 from lsst.ctrl.pool.parallel import BatchPoolTask
14 from lsst.pipe.drivers.background import SkyMeasurementTask, FocalPlaneBackground, FocalPlaneBackgroundConfig
15 import lsst.pipe.drivers.visualizeVisit as visualizeVisit
16 
17 DEBUG = False # Debugging outputs?
18 BINNING = 8 # Binning factor for debugging outputs
19 
20 
21 def makeCameraImage(camera, exposures, filename=None, binning=8):
22  """Make and write an image of an entire focal plane
23 
24  Parameters
25  ----------
26  camera : `lsst.afw.cameraGeom.Camera`
27  Camera description.
28  exposures : `list` of `tuple` of `int` and `lsst.afw.image.Exposure`
29  List of detector ID and CCD exposures (binned by `binning`).
30  filename : `str`, optional
31  Output filename.
32  binning : `int`
33  Binning size that has been applied to images.
34  """
35  image = visualizeVisit.makeCameraImage(camera, dict(exp for exp in exposures if exp is not None), binning)
36  if filename is not None:
37  image.writeFits(filename)
38  return image
39 
40 
41 class SkyCorrectionConfig(Config):
42  """Configuration for SkyCorrectionTask"""
43  bgModel = ConfigField(dtype=FocalPlaneBackgroundConfig, doc="Background model")
44  sky = ConfigurableField(target=SkyMeasurementTask, doc="Sky measurement")
45  detection = ConfigurableField(target=measAlg.SourceDetectionTask, doc="Detection configuration")
46  detectSigma = Field(dtype=float, default=2.0, doc="Detection PSF gaussian sigma")
47  doBgModel = Field(dtype=bool, default=True, doc="Do background model subtraction?")
48  doSky = Field(dtype=bool, default=True, doc="Do sky frame subtraction?")
49 
50 
52  """Correct sky over entire focal plane"""
53  ConfigClass = SkyCorrectionConfig
54  _DefaultName = "skyCorr"
55 
56  def __init__(self, *args, **kwargs):
57  BatchPoolTask.__init__(self, *args, **kwargs)
58  self.makeSubtask("sky")
59  self.makeSubtask("detection")
60 
61  @classmethod
62  def _makeArgumentParser(cls, *args, **kwargs):
63  kwargs.pop("doBatch", False)
64  parser = ArgumentParser(name="skyCorr", *args, **kwargs)
65  parser.add_id_argument("--id", datasetType="calexp", level="visit",
66  help="data ID, e.g. --id visit=12345")
67  return parser
68 
69  @classmethod
70  def batchWallTime(cls, time, parsedCmd, numCores):
71  """Return walltime request for batch job
72 
73  Subclasses should override if the walltime should be calculated
74  differently (e.g., addition of some serial time).
75 
76  Parameters
77  ----------
78  time : `float`
79  Requested time per iteration.
80  parsedCmd : `argparse.Namespace`
81  Results of argument parsing.
82  numCores : `int`
83  Number of cores.
84  """
85  numTargets = len(cls.RunnerClass.getTargetList(parsedCmd))
86  return time*numTargets
87 
88  def run(self, expRef):
89  """Perform sky correction on an exposure
90 
91  We restore the original sky, and remove it again using multiple
92  algorithms. We optionally apply:
93 
94  1. A large-scale background model.
95  2. A sky frame.
96 
97  Only the master node executes this method. The data is held on
98  the slave nodes, which do all the hard work.
99 
100  Parameters
101  ----------
102  expRef : `lsst.daf.persistence.ButlerDataRef`
103  Data reference for exposure.
104  """
105  if DEBUG:
106  extension = "-%(visit)d.fits" % expRef.dataId
107 
108  with self.logOperation("processing %s" % (expRef.dataId,)):
109  pool = Pool()
110  pool.cacheClear()
111  pool.storeSet(butler=expRef.getButler())
112  camera = expRef.get("camera")
113 
114  dataIdList = [ccdRef.dataId for ccdRef in expRef.subItems("ccd") if
115  ccdRef.datasetExists("calexp")]
116 
117  exposures = pool.map(self.loadImage, dataIdList)
118  if DEBUG:
119  makeCameraImage(camera, exposures, "restored" + extension)
120  exposures = pool.mapToPrevious(self.collectOriginal, dataIdList)
121  makeCameraImage(camera, exposures, "original" + extension)
122 
123  if self.config.doBgModel:
124  bgModel = FocalPlaneBackground.fromCamera(self.config.bgModel, camera)
125  data = [Struct(dataId=dataId, bgModel=bgModel.clone()) for dataId in dataIdList]
126  bgModelList = pool.mapToPrevious(self.accumulateModel, data)
127  for ii, bg in enumerate(bgModelList):
128  self.log.info("Background %d: %d pixels", ii, bg._numbers.getArray().sum())
129  bgModel.merge(bg)
130 
131  if DEBUG:
132  bgModel.getStatsImage().writeFits("bgModel" + extension)
133  bgImages = pool.mapToPrevious(self.realiseModel, dataIdList, bgModel)
134  makeCameraImage(camera, bgImages, "bgModelCamera" + extension)
135 
136  exposures = pool.mapToPrevious(self.subtractModel, dataIdList, bgModel)
137  if DEBUG:
138  makeCameraImage(camera, exposures, "modelsub" + extension)
139 
140  if self.config.doSky:
141  measScales = pool.mapToPrevious(self.measureSkyFrame, dataIdList)
142  scale = self.sky.solveScales(measScales)
143  self.log.info("Sky frame scale: %s" % (scale,))
144  exposures = pool.mapToPrevious(self.subtractSkyFrame, dataIdList, scale)
145  if DEBUG:
146  makeCameraImage(camera, exposures, "skysub" + extension)
147  calibs = pool.mapToPrevious(self.collectSky, dataIdList)
148  makeCameraImage(camera, calibs, "sky" + extension)
149 
150  # Persist camera-level image of calexp
151  image = makeCameraImage(camera, exposures)
152  expRef.put(image, "calexp_camera")
153 
154  pool.mapToPrevious(self.write, dataIdList)
155 
156  def loadImage(self, cache, dataId):
157  """Load original image and restore the sky
158 
159  This method runs on the slave nodes.
160 
161  Parameters
162  ----------
163  cache : `lsst.pipe.base.Struct`
164  Process pool cache.
165  dataId : `dict`
166  Data identifier.
167 
168  Returns
169  -------
170  exposure : `lsst.afw.image.Exposure`
171  Resultant exposure.
172  """
173  cache.dataId = dataId
174  cache.exposure = cache.butler.get("calexp", dataId, immediate=True).clone()
175  bgOld = cache.butler.get("calexpBackground", dataId, immediate=True)
176  image = cache.exposure.getMaskedImage()
177 
178  # We're removing the old background, so change the sense of all its components
179  for bgData in bgOld:
180  statsImage = bgData[0].getStatsImage()
181  statsImage *= -1
182 
183  image -= bgOld.getImage()
184  cache.bgList = afwMath.BackgroundList()
185  for bgData in bgOld:
186  cache.bgList.append(bgData)
187 
188  return self.collect(cache)
189 
190  def measureSkyFrame(self, cache, dataId):
191  """Measure scale for sky frame
192 
193  This method runs on the slave nodes.
194 
195  Parameters
196  ----------
197  cache : `lsst.pipe.base.Struct`
198  Process pool cache.
199  dataId : `dict`
200  Data identifier.
201 
202  Returns
203  -------
204  scale : `float`
205  Scale for sky frame.
206  """
207  assert cache.dataId == dataId
208  cache.sky = self.sky.getSkyData(cache.butler, dataId)
209  scale = self.sky.measureScale(cache.exposure.getMaskedImage(), cache.sky)
210  return scale
211 
212  def subtractSkyFrame(self, cache, dataId, scale):
213  """Subtract sky frame
214 
215  This method runs on the slave nodes.
216 
217  Parameters
218  ----------
219  cache : `lsst.pipe.base.Struct`
220  Process pool cache.
221  dataId : `dict`
222  Data identifier.
223  scale : `float`
224  Scale for sky frame.
225 
226  Returns
227  -------
228  exposure : `lsst.afw.image.Exposure`
229  Resultant exposure.
230  """
231  assert cache.dataId == dataId
232  self.sky.subtractSkyFrame(cache.exposure.getMaskedImage(), cache.sky, scale, cache.bgList)
233  return self.collect(cache)
234 
235  def accumulateModel(self, cache, data):
236  """Fit background model for CCD
237 
238  This method runs on the slave nodes.
239 
240  Parameters
241  ----------
242  cache : `lsst.pipe.base.Struct`
243  Process pool cache.
244  data : `lsst.pipe.base.Struct`
245  Data identifier, with `dataId` (data identifier) and `bgModel`
246  (background model) elements.
247 
248  Returns
249  -------
250  bgModel : `lsst.pipe.drivers.background.FocalPlaneBackground`
251  Background model.
252  """
253  assert cache.dataId == data.dataId
254  data.bgModel.addCcd(cache.exposure)
255  return data.bgModel
256 
257  def subtractModel(self, cache, dataId, bgModel):
258  """Subtract background model
259 
260  This method runs on the slave nodes.
261 
262  Parameters
263  ----------
264  cache : `lsst.pipe.base.Struct`
265  Process pool cache.
266  dataId : `dict`
267  Data identifier.
268  bgModel : `lsst.pipe.drivers.background.FocalPlaneBackround`
269  Background model.
270 
271  Returns
272  -------
273  exposure : `lsst.afw.image.Exposure`
274  Resultant exposure.
275  """
276  assert cache.dataId == dataId
277  exposure = cache.exposure
278  image = exposure.getMaskedImage()
279  detector = exposure.getDetector()
280  bbox = image.getBBox()
281  cache.bgModel = bgModel.toCcdBackground(detector, bbox)
282  image -= cache.bgModel.getImage()
283  cache.bgList.append(cache.bgModel[0])
284  return self.collect(cache)
285 
286  def realiseModel(self, cache, dataId, bgModel):
287  """Generate an image of the background model for visualisation
288 
289  Useful for debugging.
290 
291  Parameters
292  ----------
293  cache : `lsst.pipe.base.Struct`
294  Process pool cache.
295  dataId : `dict`
296  Data identifier.
297  bgModel : `lsst.pipe.drivers.background.FocalPlaneBackround`
298  Background model.
299 
300  Returns
301  -------
302  detId : `int`
303  Detector identifier.
304  image : `lsst.afw.image.MaskedImage`
305  Binned background model image.
306  """
307  assert cache.dataId == dataId
308  exposure = cache.exposure
309  detector = exposure.getDetector()
310  bbox = exposure.getMaskedImage().getBBox()
311  image = bgModel.toCcdBackground(detector, bbox).getImage()
312  return (detector.getId(), afwMath.binImage(image, BINNING))
313 
314  def collect(self, cache):
315  """Collect exposure for potential visualisation
316 
317  This method runs on the slave nodes.
318 
319  Parameters
320  ----------
321  cache : `lsst.pipe.base.Struct`
322  Process pool cache.
323 
324  Returns
325  -------
326  detId : `int`
327  Detector identifier.
328  image : `lsst.afw.image.MaskedImage`
329  Binned image.
330  """
331  return (cache.exposure.getDetector().getId(),
332  afwMath.binImage(cache.exposure.getMaskedImage(), BINNING))
333 
334  def collectOriginal(self, cache, dataId):
335  """Collect original image for visualisation
336 
337  This method runs on the slave nodes.
338 
339  Parameters
340  ----------
341  cache : `lsst.pipe.base.Struct`
342  Process pool cache.
343  dataId : `dict`
344  Data identifier.
345 
346  Returns
347  -------
348  detId : `int`
349  Detector identifier.
350  image : `lsst.afw.image.MaskedImage`
351  Binned image.
352  """
353  exposure = cache.butler.get("calexp", dataId, immediate=True)
354  return (exposure.getDetector().getId(),
355  afwMath.binImage(exposure.getMaskedImage(), BINNING))
356 
357  def collectSky(self, cache, dataId):
358  """Collect original image for visualisation
359 
360  This method runs on the slave nodes.
361 
362  Parameters
363  ----------
364  cache : `lsst.pipe.base.Struct`
365  Process pool cache.
366  dataId : `dict`
367  Data identifier.
368 
369  Returns
370  -------
371  detId : `int`
372  Detector identifier.
373  image : `lsst.afw.image.MaskedImage`
374  Binned image.
375  """
376  return (cache.exposure.getDetector().getId(), afwMath.binImage(cache.sky.getImage(), BINNING))
377 
378  def write(self, cache, dataId):
379  """Write resultant background list
380 
381  This method runs on the slave nodes.
382 
383  Parameters
384  ----------
385  cache : `lsst.pipe.base.Struct`
386  Process pool cache.
387  dataId : `dict`
388  Data identifier.
389  """
390  cache.butler.put(cache.bgList, "skyCorr", dataId)
391 
392  def _getMetadataName(self):
393  """There's no metadata to write out"""
394  return None
def subtractModel(self, cache, dataId, bgModel)
def batchWallTime(cls, time, parsedCmd, numCores)
def makeCameraImage(camera, exposures, filename=None, binning=8)
def realiseModel(self, cache, dataId, bgModel)
def subtractSkyFrame(self, cache, dataId, scale)
def logOperation(self, operation, catch=False, trace=True)