lsst.pipe.drivers  17.0.1-3-g9730928+11
skyCorrection.py
Go to the documentation of this file.
1 from __future__ import absolute_import, division, print_function
2 
3 import lsst.afw.math as afwMath
4 import lsst.afw.image as afwImage
5 
6 from lsst.pipe.base import ArgumentParser, Struct
7 from lsst.pex.config import Config, Field, ConfigurableField, ConfigField
8 from lsst.ctrl.pool.pool import Pool
9 from lsst.ctrl.pool.parallel import BatchPoolTask
10 from lsst.pipe.drivers.background import (SkyMeasurementTask, FocalPlaneBackground,
11  FocalPlaneBackgroundConfig, MaskObjectsTask)
12 import lsst.pipe.drivers.visualizeVisit as visualizeVisit
13 
14 DEBUG = False # Debugging outputs?
15 
16 
17 def makeCameraImage(camera, exposures, filename=None, binning=8):
18  """Make and write an image of an entire focal plane
19 
20  Parameters
21  ----------
22  camera : `lsst.afw.cameraGeom.Camera`
23  Camera description.
24  exposures : `list` of `tuple` of `int` and `lsst.afw.image.Exposure`
25  List of detector ID and CCD exposures (binned by `binning`).
26  filename : `str`, optional
27  Output filename.
28  binning : `int`
29  Binning size that has been applied to images.
30  """
31  image = visualizeVisit.makeCameraImage(camera, dict(exp for exp in exposures if exp is not None), binning)
32  if filename is not None:
33  image.writeFits(filename)
34  return image
35 
36 
37 class SkyCorrectionConfig(Config):
38  """Configuration for SkyCorrectionTask"""
39  bgModel = ConfigField(dtype=FocalPlaneBackgroundConfig, doc="Background model")
40  bgModel2 = ConfigField(dtype=FocalPlaneBackgroundConfig, doc="2nd Background model")
41  sky = ConfigurableField(target=SkyMeasurementTask, doc="Sky measurement")
42  maskObjects = ConfigurableField(target=MaskObjectsTask, doc="Mask Objects")
43  doMaskObjects = Field(dtype=bool, default=True, doc="Mask objects to find good sky?")
44  doBgModel = Field(dtype=bool, default=True, doc="Do background model subtraction?")
45  doBgModel2 = Field(dtype=bool, default=True, doc="Do cleanup background model subtraction?")
46  doSky = Field(dtype=bool, default=True, doc="Do sky frame subtraction?")
47  binning = Field(dtype=int, default=8, doc="Binning factor for constructing focal-plane images")
48 
49  def setDefaults(self):
50  Config.setDefaults(self)
51  self.bgModel2.doSmooth = True
52  self.bgModel2.minFrac = 0.5
53  self.bgModel2.xSize = 256
54  self.bgModel2.ySize = 256
55  self.bgModel2.smoothScale = 1.0
56 
57 
59  """Correct sky over entire focal plane"""
60  ConfigClass = SkyCorrectionConfig
61  _DefaultName = "skyCorr"
62 
63  def __init__(self, *args, **kwargs):
64  BatchPoolTask.__init__(self, *args, **kwargs)
65  self.makeSubtask("maskObjects")
66  self.makeSubtask("sky")
67 
68  @classmethod
69  def _makeArgumentParser(cls, *args, **kwargs):
70  kwargs.pop("doBatch", False)
71  parser = ArgumentParser(name="skyCorr", *args, **kwargs)
72  parser.add_id_argument("--id", datasetType="calexp", level="visit",
73  help="data ID, e.g. --id visit=12345")
74  return parser
75 
76  @classmethod
77  def batchWallTime(cls, time, parsedCmd, numCores):
78  """Return walltime request for batch job
79 
80  Subclasses should override if the walltime should be calculated
81  differently (e.g., addition of some serial time).
82 
83  Parameters
84  ----------
85  time : `float`
86  Requested time per iteration.
87  parsedCmd : `argparse.Namespace`
88  Results of argument parsing.
89  numCores : `int`
90  Number of cores.
91  """
92  numTargets = len(cls.RunnerClass.getTargetList(parsedCmd))
93  return time*numTargets
94 
95  def runDataRef(self, expRef):
96  """Perform sky correction on an exposure
97 
98  We restore the original sky, and remove it again using multiple
99  algorithms. We optionally apply:
100 
101  1. A large-scale background model.
102  This step removes very-large-scale sky such as moonlight.
103  2. A sky frame.
104  3. A medium-scale background model.
105  This step removes residual sky (This is smooth on the focal plane).
106 
107  Only the master node executes this method. The data is held on
108  the slave nodes, which do all the hard work.
109 
110  Parameters
111  ----------
112  expRef : `lsst.daf.persistence.ButlerDataRef`
113  Data reference for exposure.
114  """
115  if DEBUG:
116  extension = "-%(visit)d.fits" % expRef.dataId
117 
118  with self.logOperation("processing %s" % (expRef.dataId,)):
119  pool = Pool()
120  pool.cacheClear()
121  pool.storeSet(butler=expRef.getButler())
122  camera = expRef.get("camera")
123 
124  dataIdList = [ccdRef.dataId for ccdRef in expRef.subItems("ccd") if
125  ccdRef.datasetExists("calexp")]
126 
127  exposures = pool.map(self.loadImage, dataIdList)
128  if DEBUG:
129  makeCameraImage(camera, exposures, "restored" + extension)
130  exposures = pool.mapToPrevious(self.collectOriginal, dataIdList)
131  makeCameraImage(camera, exposures, "original" + extension)
132  exposures = pool.mapToPrevious(self.collectMask, dataIdList)
133  makeCameraImage(camera, exposures, "mask" + extension)
134 
135  if self.config.doBgModel:
136  exposures = self.focalPlaneBackground(camera, pool, dataIdList, self.config.bgModel)
137 
138  if self.config.doSky:
139  measScales = pool.mapToPrevious(self.measureSkyFrame, dataIdList)
140  scale = self.sky.solveScales(measScales)
141  self.log.info("Sky frame scale: %s" % (scale,))
142  exposures = pool.mapToPrevious(self.subtractSkyFrame, dataIdList, scale)
143  if DEBUG:
144  makeCameraImage(camera, exposures, "skysub" + extension)
145  calibs = pool.mapToPrevious(self.collectSky, dataIdList)
146  makeCameraImage(camera, calibs, "sky" + extension)
147 
148  if self.config.doBgModel2:
149  exposures = self.focalPlaneBackground(camera, pool, dataIdList, self.config.bgModel2)
150 
151  # Persist camera-level image of calexp
152  image = makeCameraImage(camera, exposures)
153  expRef.put(image, "calexp_camera")
154 
155  pool.mapToPrevious(self.write, dataIdList)
156 
157  def focalPlaneBackground(self, camera, pool, dataIdList, config):
158  """Perform full focal-plane background subtraction
159 
160  This method runs on the master node.
161 
162  Parameters
163  ----------
164  camera : `lsst.afw.cameraGeom.Camera`
165  Camera description.
166  pool : `lsst.ctrl.pool.Pool`
167  Process pool.
168  dataIdList : iterable of `dict`
169  List of data identifiers for the CCDs.
170  config : `lsst.pipe.drivers.background.FocalPlaneBackgroundConfig`
171  Configuration to use for background subtraction.
172 
173  Returns
174  -------
175  exposures : `list` of `lsst.afw.image.Image`
176  List of binned images, for creating focal plane image.
177  """
178  bgModel = FocalPlaneBackground.fromCamera(config, camera)
179  data = [Struct(dataId=dataId, bgModel=bgModel.clone()) for dataId in dataIdList]
180  bgModelList = pool.mapToPrevious(self.accumulateModel, data)
181  for ii, bg in enumerate(bgModelList):
182  self.log.info("Background %d: %d pixels", ii, bg._numbers.array.sum())
183  bgModel.merge(bg)
184  return pool.mapToPrevious(self.subtractModel, dataIdList, bgModel)
185 
186  def loadImage(self, cache, dataId):
187  """Load original image and restore the sky
188 
189  This method runs on the slave nodes.
190 
191  Parameters
192  ----------
193  cache : `lsst.pipe.base.Struct`
194  Process pool cache.
195  dataId : `dict`
196  Data identifier.
197 
198  Returns
199  -------
200  exposure : `lsst.afw.image.Exposure`
201  Resultant exposure.
202  """
203  cache.dataId = dataId
204  cache.exposure = cache.butler.get("calexp", dataId, immediate=True).clone()
205  bgOld = cache.butler.get("calexpBackground", dataId, immediate=True)
206  image = cache.exposure.getMaskedImage()
207 
208  # We're removing the old background, so change the sense of all its components
209  for bgData in bgOld:
210  statsImage = bgData[0].getStatsImage()
211  statsImage *= -1
212 
213  image -= bgOld.getImage()
214  cache.bgList = afwMath.BackgroundList()
215  for bgData in bgOld:
216  cache.bgList.append(bgData)
217 
218  if self.config.doMaskObjects:
219  self.maskObjects.findObjects(cache.exposure)
220 
221  return self.collect(cache)
222 
223  def measureSkyFrame(self, cache, dataId):
224  """Measure scale for sky frame
225 
226  This method runs on the slave nodes.
227 
228  Parameters
229  ----------
230  cache : `lsst.pipe.base.Struct`
231  Process pool cache.
232  dataId : `dict`
233  Data identifier.
234 
235  Returns
236  -------
237  scale : `float`
238  Scale for sky frame.
239  """
240  assert cache.dataId == dataId
241  cache.sky = self.sky.getSkyData(cache.butler, dataId)
242  scale = self.sky.measureScale(cache.exposure.getMaskedImage(), cache.sky)
243  return scale
244 
245  def subtractSkyFrame(self, cache, dataId, scale):
246  """Subtract sky frame
247 
248  This method runs on the slave nodes.
249 
250  Parameters
251  ----------
252  cache : `lsst.pipe.base.Struct`
253  Process pool cache.
254  dataId : `dict`
255  Data identifier.
256  scale : `float`
257  Scale for sky frame.
258 
259  Returns
260  -------
261  exposure : `lsst.afw.image.Exposure`
262  Resultant exposure.
263  """
264  assert cache.dataId == dataId
265  self.sky.subtractSkyFrame(cache.exposure.getMaskedImage(), cache.sky, scale, cache.bgList)
266  return self.collect(cache)
267 
268  def accumulateModel(self, cache, data):
269  """Fit background model for CCD
270 
271  This method runs on the slave nodes.
272 
273  Parameters
274  ----------
275  cache : `lsst.pipe.base.Struct`
276  Process pool cache.
277  data : `lsst.pipe.base.Struct`
278  Data identifier, with `dataId` (data identifier) and `bgModel`
279  (background model) elements.
280 
281  Returns
282  -------
283  bgModel : `lsst.pipe.drivers.background.FocalPlaneBackground`
284  Background model.
285  """
286  assert cache.dataId == data.dataId
287  data.bgModel.addCcd(cache.exposure)
288  return data.bgModel
289 
290  def subtractModel(self, cache, dataId, bgModel):
291  """Subtract background model
292 
293  This method runs on the slave nodes.
294 
295  Parameters
296  ----------
297  cache : `lsst.pipe.base.Struct`
298  Process pool cache.
299  dataId : `dict`
300  Data identifier.
301  bgModel : `lsst.pipe.drivers.background.FocalPlaneBackround`
302  Background model.
303 
304  Returns
305  -------
306  exposure : `lsst.afw.image.Exposure`
307  Resultant exposure.
308  """
309  assert cache.dataId == dataId
310  exposure = cache.exposure
311  image = exposure.getMaskedImage()
312  detector = exposure.getDetector()
313  bbox = image.getBBox()
314  cache.bgModel = bgModel.toCcdBackground(detector, bbox)
315  image -= cache.bgModel.getImage()
316  cache.bgList.append(cache.bgModel[0])
317  return self.collect(cache)
318 
319  def realiseModel(self, cache, dataId, bgModel):
320  """Generate an image of the background model for visualisation
321 
322  Useful for debugging.
323 
324  Parameters
325  ----------
326  cache : `lsst.pipe.base.Struct`
327  Process pool cache.
328  dataId : `dict`
329  Data identifier.
330  bgModel : `lsst.pipe.drivers.background.FocalPlaneBackround`
331  Background model.
332 
333  Returns
334  -------
335  detId : `int`
336  Detector identifier.
337  image : `lsst.afw.image.MaskedImage`
338  Binned background model image.
339  """
340  assert cache.dataId == dataId
341  exposure = cache.exposure
342  detector = exposure.getDetector()
343  bbox = exposure.getMaskedImage().getBBox()
344  image = bgModel.toCcdBackground(detector, bbox).getImage()
345  return self.collectBinnedImage(exposure, image)
346 
347  def collectBinnedImage(self, exposure, image):
348  """Return the binned image required for visualization
349 
350  This method just helps to cut down on boilerplate.
351 
352  Parameters
353  ----------
354  image : `lsst.afw.image.MaskedImage`
355  Image to go into visualisation.
356 
357  Returns
358  -------
359  detId : `int`
360  Detector identifier.
361  image : `lsst.afw.image.MaskedImage`
362  Binned image.
363  """
364  return (exposure.getDetector().getId(), afwMath.binImage(image, self.config.binning))
365 
366  def collect(self, cache):
367  """Collect exposure for potential visualisation
368 
369  This method runs on the slave nodes.
370 
371  Parameters
372  ----------
373  cache : `lsst.pipe.base.Struct`
374  Process pool cache.
375 
376  Returns
377  -------
378  detId : `int`
379  Detector identifier.
380  image : `lsst.afw.image.MaskedImage`
381  Binned image.
382  """
383  return self.collectBinnedImage(cache.exposure, cache.exposure.maskedImage)
384 
385  def collectOriginal(self, cache, dataId):
386  """Collect original image for visualisation
387 
388  This method runs on the slave nodes.
389 
390  Parameters
391  ----------
392  cache : `lsst.pipe.base.Struct`
393  Process pool cache.
394  dataId : `dict`
395  Data identifier.
396 
397  Returns
398  -------
399  detId : `int`
400  Detector identifier.
401  image : `lsst.afw.image.MaskedImage`
402  Binned image.
403  """
404  exposure = cache.butler.get("calexp", dataId, immediate=True)
405  return self.collectBinnedImage(exposure, exposure.maskedImage)
406 
407  def collectSky(self, cache, dataId):
408  """Collect original image for visualisation
409 
410  This method runs on the slave nodes.
411 
412  Parameters
413  ----------
414  cache : `lsst.pipe.base.Struct`
415  Process pool cache.
416  dataId : `dict`
417  Data identifier.
418 
419  Returns
420  -------
421  detId : `int`
422  Detector identifier.
423  image : `lsst.afw.image.MaskedImage`
424  Binned image.
425  """
426  return self.collectBinnedImage(cache.exposure, cache.sky.getImage())
427 
428  def collectMask(self, cache, dataId):
429  """Collect mask for visualisation
430 
431  This method runs on the slave nodes.
432 
433  Parameters
434  ----------
435  cache : `lsst.pipe.base.Struct`
436  Process pool cache.
437  dataId : `dict`
438  Data identifier.
439 
440  Returns
441  -------
442  detId : `int`
443  Detector identifier.
444  image : `lsst.afw.image.Image`
445  Binned image.
446  """
447  # Convert Mask to floating-point image, because that's what's required for focal plane construction
448  image = afwImage.ImageF(cache.exposure.maskedImage.getBBox())
449  image.array[:] = cache.exposure.maskedImage.mask.array
450  return self.collectBinnedImage(cache.exposure, image)
451 
452  def write(self, cache, dataId):
453  """Write resultant background list
454 
455  This method runs on the slave nodes.
456 
457  Parameters
458  ----------
459  cache : `lsst.pipe.base.Struct`
460  Process pool cache.
461  dataId : `dict`
462  Data identifier.
463  """
464  cache.butler.put(cache.bgList, "skyCorr", dataId)
465 
466  def _getMetadataName(self):
467  """There's no metadata to write out"""
468  return None
def subtractModel(self, cache, dataId, bgModel)
def batchWallTime(cls, time, parsedCmd, numCores)
def makeCameraImage(camera, exposures, filename=None, binning=8)
def focalPlaneBackground(self, camera, pool, dataIdList, config)
def realiseModel(self, cache, dataId, bgModel)
def subtractSkyFrame(self, cache, dataId, scale)
def logOperation(self, operation, catch=False, trace=True)