lsst.pipe.drivers  15.0-3-ga03b4ca+23
skyCorrection.py
Go to the documentation of this file.
1 from __future__ import absolute_import, division, print_function
2 
3 import numpy
4 
5 import lsst.afw.math as afwMath
6 import lsst.afw.image as afwImage
7 import lsst.afw.table as afwTable
8 import lsst.meas.algorithms as measAlg
9 
10 from lsst.afw.cameraGeom.utils import makeImageFromCamera
11 from lsst.pipe.base import ArgumentParser, Struct
12 from lsst.pex.config import Config, Field, ConfigurableField, ConfigField
13 from lsst.ctrl.pool.pool import Pool
14 from lsst.ctrl.pool.parallel import BatchPoolTask
15 from lsst.pipe.drivers.background import SkyMeasurementTask, FocalPlaneBackground, FocalPlaneBackgroundConfig
16 import lsst.pipe.drivers.visualizeVisit as visualizeVisit
17 
18 DEBUG = False # Debugging outputs?
19 
20 
21 def makeCameraImage(camera, exposures, filename=None, binning=8):
22  """Make and write an image of an entire focal plane
23 
24  Parameters
25  ----------
26  camera : `lsst.afw.cameraGeom.Camera`
27  Camera description.
28  exposures : `list` of `tuple` of `int` and `lsst.afw.image.Exposure`
29  List of detector ID and CCD exposures (binned by `binning`).
30  filename : `str`, optional
31  Output filename.
32  binning : `int`
33  Binning size that has been applied to images.
34  """
35  image = visualizeVisit.makeCameraImage(camera, dict(exp for exp in exposures if exp is not None), binning)
36  if filename is not None:
37  image.writeFits(filename)
38  return image
39 
40 
41 class SkyCorrectionConfig(Config):
42  """Configuration for SkyCorrectionTask"""
43  bgModel = ConfigField(dtype=FocalPlaneBackgroundConfig, doc="Background model")
44  sky = ConfigurableField(target=SkyMeasurementTask, doc="Sky measurement")
45  detection = ConfigurableField(target=measAlg.SourceDetectionTask, doc="Detection configuration")
46  doDetection = Field(dtype=bool, default=True, doc="Detect sources (to find good sky)?")
47  detectSigma = Field(dtype=float, default=5.0, doc="Detection PSF gaussian sigma")
48  doBgModel = Field(dtype=bool, default=True, doc="Do background model subtraction?")
49  doSky = Field(dtype=bool, default=True, doc="Do sky frame subtraction?")
50  binning = Field(dtype=int, default=8, doc="Binning factor for constructing focal-plane images")
51 
52  def setDefaults(self):
53  Config.setDefaults(self)
54  self.detection.reEstimateBackground = False
55  self.detection.thresholdPolarity = "both"
56  self.detection.doTempLocalBackground = False
57  self.detection.thresholdType = "pixel_stdev"
58  self.detection.thresholdValue = 3.0
59 
61  """Correct sky over entire focal plane"""
62  ConfigClass = SkyCorrectionConfig
63  _DefaultName = "skyCorr"
64 
65  def __init__(self, *args, **kwargs):
66  BatchPoolTask.__init__(self, *args, **kwargs)
67  self.makeSubtask("sky")
68  # Disposable schema suppresses warning from SourceDetectionTask.__init__
69  self.makeSubtask("detection", schema=afwTable.Schema())
70 
71  @classmethod
72  def _makeArgumentParser(cls, *args, **kwargs):
73  kwargs.pop("doBatch", False)
74  parser = ArgumentParser(name="skyCorr", *args, **kwargs)
75  parser.add_id_argument("--id", datasetType="calexp", level="visit",
76  help="data ID, e.g. --id visit=12345")
77  return parser
78 
79  @classmethod
80  def batchWallTime(cls, time, parsedCmd, numCores):
81  """Return walltime request for batch job
82 
83  Subclasses should override if the walltime should be calculated
84  differently (e.g., addition of some serial time).
85 
86  Parameters
87  ----------
88  time : `float`
89  Requested time per iteration.
90  parsedCmd : `argparse.Namespace`
91  Results of argument parsing.
92  numCores : `int`
93  Number of cores.
94  """
95  numTargets = len(cls.RunnerClass.getTargetList(parsedCmd))
96  return time*numTargets
97 
98  def run(self, expRef):
99  """Perform sky correction on an exposure
100 
101  We restore the original sky, and remove it again using multiple
102  algorithms. We optionally apply:
103 
104  1. A large-scale background model.
105  2. A sky frame.
106 
107  Only the master node executes this method. The data is held on
108  the slave nodes, which do all the hard work.
109 
110  Parameters
111  ----------
112  expRef : `lsst.daf.persistence.ButlerDataRef`
113  Data reference for exposure.
114  """
115  if DEBUG:
116  extension = "-%(visit)d.fits" % expRef.dataId
117 
118  with self.logOperation("processing %s" % (expRef.dataId,)):
119  pool = Pool()
120  pool.cacheClear()
121  pool.storeSet(butler=expRef.getButler())
122  camera = expRef.get("camera")
123 
124  dataIdList = [ccdRef.dataId for ccdRef in expRef.subItems("ccd") if
125  ccdRef.datasetExists("calexp")]
126 
127  exposures = pool.map(self.loadImage, dataIdList)
128  if DEBUG:
129  makeCameraImage(camera, exposures, "restored" + extension)
130  exposures = pool.mapToPrevious(self.collectOriginal, dataIdList)
131  makeCameraImage(camera, exposures, "original" + extension)
132  exposures = pool.mapToPrevious(self.collectMask, dataIdList)
133  makeCameraImage(camera, exposures, "mask" + extension)
134 
135  if self.config.doBgModel:
136  bgModel = FocalPlaneBackground.fromCamera(self.config.bgModel, camera)
137  data = [Struct(dataId=dataId, bgModel=bgModel.clone()) for dataId in dataIdList]
138  bgModelList = pool.mapToPrevious(self.accumulateModel, data)
139  for ii, bg in enumerate(bgModelList):
140  self.log.info("Background %d: %d pixels", ii, bg._numbers.getArray().sum())
141  bgModel.merge(bg)
142 
143  if DEBUG:
144  bgModel.getStatsImage().writeFits("bgModel" + extension)
145  bgImages = pool.mapToPrevious(self.realiseModel, dataIdList, bgModel)
146  makeCameraImage(camera, bgImages, "bgModelCamera" + extension)
147 
148  exposures = pool.mapToPrevious(self.subtractModel, dataIdList, bgModel)
149  if DEBUG:
150  makeCameraImage(camera, exposures, "modelsub" + extension)
151 
152  if self.config.doSky:
153  measScales = pool.mapToPrevious(self.measureSkyFrame, dataIdList)
154  scale = self.sky.solveScales(measScales)
155  self.log.info("Sky frame scale: %s" % (scale,))
156  exposures = pool.mapToPrevious(self.subtractSkyFrame, dataIdList, scale)
157  if DEBUG:
158  makeCameraImage(camera, exposures, "skysub" + extension)
159  calibs = pool.mapToPrevious(self.collectSky, dataIdList)
160  makeCameraImage(camera, calibs, "sky" + extension)
161 
162  # Persist camera-level image of calexp
163  image = makeCameraImage(camera, exposures)
164  expRef.put(image, "calexp_camera")
165 
166  pool.mapToPrevious(self.write, dataIdList)
167 
168  def loadImage(self, cache, dataId):
169  """Load original image and restore the sky
170 
171  This method runs on the slave nodes.
172 
173  Parameters
174  ----------
175  cache : `lsst.pipe.base.Struct`
176  Process pool cache.
177  dataId : `dict`
178  Data identifier.
179 
180  Returns
181  -------
182  exposure : `lsst.afw.image.Exposure`
183  Resultant exposure.
184  """
185  cache.dataId = dataId
186  cache.exposure = cache.butler.get("calexp", dataId, immediate=True).clone()
187  bgOld = cache.butler.get("calexpBackground", dataId, immediate=True)
188  image = cache.exposure.getMaskedImage()
189 
190  if self.config.doDetection:
191  # We deliberately use the specified 'detectSigma' instead of the PSF, in order to better pick up
192  # the faint wings of objects.
193  results = self.detection.detectFootprints(cache.exposure, doSmooth=True,
194  sigma=self.config.detectSigma, clearMask=True)
195  if hasattr(results, "background") and results.background:
196  # Restore any background that was removed during detection
197  maskedImage += results.background.getImage()
198 
199  # We're removing the old background, so change the sense of all its components
200  for bgData in bgOld:
201  statsImage = bgData[0].getStatsImage()
202  statsImage *= -1
203 
204  image -= bgOld.getImage()
205  cache.bgList = afwMath.BackgroundList()
206  for bgData in bgOld:
207  cache.bgList.append(bgData)
208 
209  return self.collect(cache)
210 
211  def measureSkyFrame(self, cache, dataId):
212  """Measure scale for sky frame
213 
214  This method runs on the slave nodes.
215 
216  Parameters
217  ----------
218  cache : `lsst.pipe.base.Struct`
219  Process pool cache.
220  dataId : `dict`
221  Data identifier.
222 
223  Returns
224  -------
225  scale : `float`
226  Scale for sky frame.
227  """
228  assert cache.dataId == dataId
229  cache.sky = self.sky.getSkyData(cache.butler, dataId)
230  scale = self.sky.measureScale(cache.exposure.getMaskedImage(), cache.sky)
231  return scale
232 
233  def subtractSkyFrame(self, cache, dataId, scale):
234  """Subtract sky frame
235 
236  This method runs on the slave nodes.
237 
238  Parameters
239  ----------
240  cache : `lsst.pipe.base.Struct`
241  Process pool cache.
242  dataId : `dict`
243  Data identifier.
244  scale : `float`
245  Scale for sky frame.
246 
247  Returns
248  -------
249  exposure : `lsst.afw.image.Exposure`
250  Resultant exposure.
251  """
252  assert cache.dataId == dataId
253  self.sky.subtractSkyFrame(cache.exposure.getMaskedImage(), cache.sky, scale, cache.bgList)
254  return self.collect(cache)
255 
256  def accumulateModel(self, cache, data):
257  """Fit background model for CCD
258 
259  This method runs on the slave nodes.
260 
261  Parameters
262  ----------
263  cache : `lsst.pipe.base.Struct`
264  Process pool cache.
265  data : `lsst.pipe.base.Struct`
266  Data identifier, with `dataId` (data identifier) and `bgModel`
267  (background model) elements.
268 
269  Returns
270  -------
271  bgModel : `lsst.pipe.drivers.background.FocalPlaneBackground`
272  Background model.
273  """
274  assert cache.dataId == data.dataId
275  data.bgModel.addCcd(cache.exposure)
276  return data.bgModel
277 
278  def subtractModel(self, cache, dataId, bgModel):
279  """Subtract background model
280 
281  This method runs on the slave nodes.
282 
283  Parameters
284  ----------
285  cache : `lsst.pipe.base.Struct`
286  Process pool cache.
287  dataId : `dict`
288  Data identifier.
289  bgModel : `lsst.pipe.drivers.background.FocalPlaneBackround`
290  Background model.
291 
292  Returns
293  -------
294  exposure : `lsst.afw.image.Exposure`
295  Resultant exposure.
296  """
297  assert cache.dataId == dataId
298  exposure = cache.exposure
299  image = exposure.getMaskedImage()
300  detector = exposure.getDetector()
301  bbox = image.getBBox()
302  cache.bgModel = bgModel.toCcdBackground(detector, bbox)
303  image -= cache.bgModel.getImage()
304  cache.bgList.append(cache.bgModel[0])
305  return self.collect(cache)
306 
307  def realiseModel(self, cache, dataId, bgModel):
308  """Generate an image of the background model for visualisation
309 
310  Useful for debugging.
311 
312  Parameters
313  ----------
314  cache : `lsst.pipe.base.Struct`
315  Process pool cache.
316  dataId : `dict`
317  Data identifier.
318  bgModel : `lsst.pipe.drivers.background.FocalPlaneBackround`
319  Background model.
320 
321  Returns
322  -------
323  detId : `int`
324  Detector identifier.
325  image : `lsst.afw.image.MaskedImage`
326  Binned background model image.
327  """
328  assert cache.dataId == dataId
329  exposure = cache.exposure
330  detector = exposure.getDetector()
331  bbox = exposure.getMaskedImage().getBBox()
332  image = bgModel.toCcdBackground(detector, bbox).getImage()
333  return self.collectBinnedImage(exposure, image)
334 
335  def collectBinnedImage(self, exposure, image):
336  """Return the binned image required for visualization
337 
338  This method just helps to cut down on boilerplate.
339 
340  Parameters
341  ----------
342  image : `lsst.afw.image.MaskedImage`
343  Image to go into visualisation.
344 
345  Returns
346  -------
347  detId : `int`
348  Detector identifier.
349  image : `lsst.afw.image.MaskedImage`
350  Binned image.
351  """
352  return (exposure.getDetector().getId(), afwMath.binImage(image, self.config.binning))
353 
354  def collect(self, cache):
355  """Collect exposure for potential visualisation
356 
357  This method runs on the slave nodes.
358 
359  Parameters
360  ----------
361  cache : `lsst.pipe.base.Struct`
362  Process pool cache.
363 
364  Returns
365  -------
366  detId : `int`
367  Detector identifier.
368  image : `lsst.afw.image.MaskedImage`
369  Binned image.
370  """
371  return self.collectBinnedImage(cache.exposure, cache.exposure.maskedImage)
372 
373  def collectOriginal(self, cache, dataId):
374  """Collect original image for visualisation
375 
376  This method runs on the slave nodes.
377 
378  Parameters
379  ----------
380  cache : `lsst.pipe.base.Struct`
381  Process pool cache.
382  dataId : `dict`
383  Data identifier.
384 
385  Returns
386  -------
387  detId : `int`
388  Detector identifier.
389  image : `lsst.afw.image.MaskedImage`
390  Binned image.
391  """
392  exposure = cache.butler.get("calexp", dataId, immediate=True)
393  return self.collectBinnedImage(exposure, exposure.maskedImage)
394 
395  def collectSky(self, cache, dataId):
396  """Collect original image for visualisation
397 
398  This method runs on the slave nodes.
399 
400  Parameters
401  ----------
402  cache : `lsst.pipe.base.Struct`
403  Process pool cache.
404  dataId : `dict`
405  Data identifier.
406 
407  Returns
408  -------
409  detId : `int`
410  Detector identifier.
411  image : `lsst.afw.image.MaskedImage`
412  Binned image.
413  """
414  return self.collectBinnedImage(cache.exposure, cache.sky.getImage())
415 
416  def collectMask(self, cache, dataId):
417  """Collect mask for visualisation
418 
419  This method runs on the slave nodes.
420 
421  Parameters
422  ----------
423  cache : `lsst.pipe.base.Struct`
424  Process pool cache.
425  dataId : `dict`
426  Data identifier.
427 
428  Returns
429  -------
430  detId : `int`
431  Detector identifier.
432  image : `lsst.afw.image.Image`
433  Binned image.
434  """
435  # Convert Mask to floating-point image, because that's what's required for focal plane construction
436  image = afwImage.ImageF(cache.exposure.maskedImage.getBBox())
437  image.array[:] = cache.exposure.maskedImage.mask.array
438  return self.collectBinnedImage(cache.exposure, image)
439 
440  def write(self, cache, dataId):
441  """Write resultant background list
442 
443  This method runs on the slave nodes.
444 
445  Parameters
446  ----------
447  cache : `lsst.pipe.base.Struct`
448  Process pool cache.
449  dataId : `dict`
450  Data identifier.
451  """
452  cache.butler.put(cache.bgList, "skyCorr", dataId)
453 
454  def _getMetadataName(self):
455  """There's no metadata to write out"""
456  return None
def subtractModel(self, cache, dataId, bgModel)
def batchWallTime(cls, time, parsedCmd, numCores)
def makeCameraImage(camera, exposures, filename=None, binning=8)
def realiseModel(self, cache, dataId, bgModel)
def subtractSkyFrame(self, cache, dataId, scale)
def logOperation(self, operation, catch=False, trace=True)