lsst.pipe.drivers  14.0-13-g1010e0d+5
skyCorrection.py
Go to the documentation of this file.
1 from __future__ import absolute_import, division, print_function
2 
3 import numpy
4 
5 import lsst.afw.math as afwMath
6 import lsst.afw.image as afwImage
7 import lsst.meas.algorithms as measAlg
8 
9 from lsst.afw.cameraGeom.utils import makeImageFromCamera
10 from lsst.pipe.base import ArgumentParser, Struct
11 from lsst.pex.config import Config, Field, ConfigurableField, ConfigField
12 from lsst.ctrl.pool.pool import Pool
13 from lsst.ctrl.pool.parallel import BatchPoolTask
14 from lsst.pipe.drivers.background import SkyMeasurementTask, FocalPlaneBackground, FocalPlaneBackgroundConfig
15 
16 
17 DEBUG = False # Debugging outputs?
18 BINNING = 8 # Binning factor for debugging outputs
19 
20 
21 def makeCameraImage(camera, exposures, filename=None, binning=8):
22  """Make and write an image of an entire focal plane
23 
24  Parameters
25  ----------
26  camera : `lsst.afw.cameraGeom.Camera`
27  Camera description.
28  exposures : `dict` mapping detector ID to `lsst.afw.image.Exposure`
29  CCD exposures, binned by `binning`.
30  filename : `str`, optional
31  Output filename.
32  binning : `int`
33  Binning size that has been applied to images.
34  """
35  class ImageSource(object):
36  """Source of images for makeImageFromCamera"""
37  def __init__(self, exposures):
38  """Constructor
39 
40  Parameters
41  ----------
42  exposures : `dict` mapping detector ID to `lsst.afw.image.Exposure`
43  CCD exposures, already binned.
44  """
45  self.isTrimmed = True
46  self.exposures = exposures
47  self.background = numpy.nan
48 
49  def getCcdImage(self, detector, imageFactory, binSize):
50  """Provide image of CCD to makeImageFromCamera"""
51  if detector.getId() not in self.exposures:
52  return imageFactory(1, 1), detector
53  image = self.exposures[detector.getId()]
54  if hasattr(image, "getMaskedImage"):
55  image = image.getMaskedImage()
56  if hasattr(image, "getMask"):
57  mask = image.getMask()
58  isBad = mask.getArray() & mask.getPlaneBitMask("NO_DATA") > 0
59  image = image.clone()
60  image.getImage().getArray()[isBad] = numpy.nan
61  if hasattr(image, "getImage"):
62  image = image.getImage()
63  return image, detector
64 
65  image = makeImageFromCamera(
66  camera,
67  imageSource=ImageSource(dict(exp for exp in exposures if exp is not None)),
68  imageFactory=afwImage.ImageF,
69  binSize=binning
70  )
71  if filename is not None:
72  image.writeFits(filename)
73  return image
74 
75 
76 class SkyCorrectionConfig(Config):
77  """Configuration for SkyCorrectionTask"""
78  bgModel = ConfigField(dtype=FocalPlaneBackgroundConfig, doc="Background model")
79  sky = ConfigurableField(target=SkyMeasurementTask, doc="Sky measurement")
80  detection = ConfigurableField(target=measAlg.SourceDetectionTask, doc="Detection configuration")
81  detectSigma = Field(dtype=float, default=2.0, doc="Detection PSF gaussian sigma")
82  doBgModel = Field(dtype=bool, default=True, doc="Do background model subtraction?")
83  doSky = Field(dtype=bool, default=True, doc="Do sky frame subtraction?")
84 
85 
87  """Correct sky over entire focal plane"""
88  ConfigClass = SkyCorrectionConfig
89  _DefaultName = "skyCorr"
90 
91  def __init__(self, *args, **kwargs):
92  BatchPoolTask.__init__(self, *args, **kwargs)
93  self.makeSubtask("sky")
94  self.makeSubtask("detection")
95 
96  @classmethod
97  def _makeArgumentParser(cls, *args, **kwargs):
98  kwargs.pop("doBatch", False)
99  parser = ArgumentParser(name="skyCorr", *args, **kwargs)
100  parser.add_id_argument("--id", datasetType="calexp", level="visit",
101  help="data ID, e.g. --id visit=12345")
102  return parser
103 
104  @classmethod
105  def batchWallTime(cls, time, parsedCmd, numCores):
106  """Return walltime request for batch job
107 
108  Subclasses should override if the walltime should be calculated
109  differently (e.g., addition of some serial time).
110 
111  Parameters
112  ----------
113  time : `float`
114  Requested time per iteration.
115  parsedCmd : `argparse.Namespace`
116  Results of argument parsing.
117  numCores : `int`
118  Number of cores.
119  """
120  numTargets = len(cls.RunnerClass.getTargetList(parsedCmd))
121  return time*numTargets
122 
123  def run(self, expRef):
124  """Perform sky correction on an exposure
125 
126  We restore the original sky, and remove it again using multiple
127  algorithms. We optionally apply:
128 
129  1. A large-scale background model.
130  2. A sky frame.
131 
132  Only the master node executes this method. The data is held on
133  the slave nodes, which do all the hard work.
134 
135  Parameters
136  ----------
137  expRef : `lsst.daf.persistence.ButlerDataRef`
138  Data reference for exposure.
139  """
140  if DEBUG:
141  extension = "-%(visit)d.fits" % expRef.dataId
142 
143  with self.logOperation("processing %s" % (expRef.dataId,)):
144  pool = Pool()
145  pool.cacheClear()
146  pool.storeSet(butler=expRef.getButler())
147  camera = expRef.get("camera")
148 
149  dataIdList = [ccdRef.dataId for ccdRef in expRef.subItems("ccd") if
150  ccdRef.datasetExists("calexp")]
151 
152  exposures = pool.map(self.loadImage, dataIdList)
153  if DEBUG:
154  makeCameraImage(camera, exposures, "restored" + extension)
155  exposures = pool.mapToPrevious(self.collectOriginal, dataIdList)
156  makeCameraImage(camera, exposures, "original" + extension)
157 
158  if self.config.doBgModel:
159  bgModel = FocalPlaneBackground.fromCamera(self.config.bgModel, camera)
160  data = [Struct(dataId=dataId, bgModel=bgModel.clone()) for dataId in dataIdList]
161  bgModelList = pool.mapToPrevious(self.accumulateModel, data)
162  for ii, bg in enumerate(bgModelList):
163  self.log.info("Background %d: %d pixels", ii, bg._numbers.getArray().sum())
164  bgModel.merge(bg)
165 
166  if DEBUG:
167  bgModel.getStatsImage().writeFits("bgModel" + extension)
168  bgImages = pool.mapToPrevious(self.realiseModel, dataIdList, bgModel)
169  makeCameraImage(camera, bgImages, "bgModelCamera" + extension)
170 
171  exposures = pool.mapToPrevious(self.subtractModel, dataIdList, bgModel)
172  if DEBUG:
173  makeCameraImage(camera, exposures, "modelsub" + extension)
174 
175  if self.config.doSky:
176  measScales = pool.mapToPrevious(self.measureSkyFrame, dataIdList)
177  scale = self.sky.solveScales(measScales)
178  self.log.info("Sky frame scale: %s" % (scale,))
179  exposures = pool.mapToPrevious(self.subtractSkyFrame, dataIdList, scale)
180  if DEBUG:
181  makeCameraImage(camera, exposures, "skysub" + extension)
182  calibs = pool.mapToPrevious(self.collectSky, dataIdList)
183  makeCameraImage(camera, calibs, "sky" + extension)
184 
185  # Persist camera-level image of calexp
186  image = makeCameraImage(camera, exposures)
187  expRef.put(image, "calexp_camera")
188 
189  pool.mapToPrevious(self.write, dataIdList)
190 
191  def loadImage(self, cache, dataId):
192  """Load original image and restore the sky
193 
194  This method runs on the slave nodes.
195 
196  Parameters
197  ----------
198  cache : `lsst.pipe.base.Struct`
199  Process pool cache.
200  dataId : `dict`
201  Data identifier.
202 
203  Returns
204  -------
205  exposure : `lsst.afw.image.Exposure`
206  Resultant exposure.
207  """
208  cache.dataId = dataId
209  cache.exposure = cache.butler.get("calexp", dataId, immediate=True).clone()
210  bgOld = cache.butler.get("calexpBackground", dataId, immediate=True)
211  image = cache.exposure.getMaskedImage()
212 
213  # We're removing the old background, so change the sense of all its components
214  for bgData in bgOld:
215  statsImage = bgData[0].getStatsImage()
216  statsImage *= -1
217 
218  image -= bgOld.getImage()
219  cache.bgList = afwMath.BackgroundList()
220  for bgData in bgOld:
221  cache.bgList.append(bgData)
222 
223  return self.collect(cache)
224 
225  def measureSkyFrame(self, cache, dataId):
226  """Measure scale for sky frame
227 
228  This method runs on the slave nodes.
229 
230  Parameters
231  ----------
232  cache : `lsst.pipe.base.Struct`
233  Process pool cache.
234  dataId : `dict`
235  Data identifier.
236 
237  Returns
238  -------
239  scale : `float`
240  Scale for sky frame.
241  """
242  assert cache.dataId == dataId
243  cache.sky = self.sky.getSkyData(cache.butler, dataId)
244  scale = self.sky.measureScale(cache.exposure.getMaskedImage(), cache.sky)
245  return scale
246 
247  def subtractSkyFrame(self, cache, dataId, scale):
248  """Subtract sky frame
249 
250  This method runs on the slave nodes.
251 
252  Parameters
253  ----------
254  cache : `lsst.pipe.base.Struct`
255  Process pool cache.
256  dataId : `dict`
257  Data identifier.
258  scale : `float`
259  Scale for sky frame.
260 
261  Returns
262  -------
263  exposure : `lsst.afw.image.Exposure`
264  Resultant exposure.
265  """
266  assert cache.dataId == dataId
267  self.sky.subtractSkyFrame(cache.exposure.getMaskedImage(), cache.sky, scale, cache.bgList)
268  return self.collect(cache)
269 
270  def accumulateModel(self, cache, data):
271  """Fit background model for CCD
272 
273  This method runs on the slave nodes.
274 
275  Parameters
276  ----------
277  cache : `lsst.pipe.base.Struct`
278  Process pool cache.
279  data : `lsst.pipe.base.Struct`
280  Data identifier, with `dataId` (data identifier) and `bgModel`
281  (background model) elements.
282 
283  Returns
284  -------
285  bgModel : `lsst.pipe.drivers.background.FocalPlaneBackground`
286  Background model.
287  """
288  assert cache.dataId == data.dataId
289  data.bgModel.addCcd(cache.exposure)
290  return data.bgModel
291 
292  def subtractModel(self, cache, dataId, bgModel):
293  """Subtract background model
294 
295  This method runs on the slave nodes.
296 
297  Parameters
298  ----------
299  cache : `lsst.pipe.base.Struct`
300  Process pool cache.
301  dataId : `dict`
302  Data identifier.
303  bgModel : `lsst.pipe.drivers.background.FocalPlaneBackround`
304  Background model.
305 
306  Returns
307  -------
308  exposure : `lsst.afw.image.Exposure`
309  Resultant exposure.
310  """
311  assert cache.dataId == dataId
312  exposure = cache.exposure
313  image = exposure.getMaskedImage()
314  detector = exposure.getDetector()
315  bbox = image.getBBox()
316  cache.bgModel = bgModel.toCcdBackground(detector, bbox)
317  image -= cache.bgModel.getImage()
318  cache.bgList.append(cache.bgModel[0])
319  return self.collect(cache)
320 
321  def realiseModel(self, cache, dataId, bgModel):
322  """Generate an image of the background model for visualisation
323 
324  Useful for debugging.
325 
326  Parameters
327  ----------
328  cache : `lsst.pipe.base.Struct`
329  Process pool cache.
330  dataId : `dict`
331  Data identifier.
332  bgModel : `lsst.pipe.drivers.background.FocalPlaneBackround`
333  Background model.
334 
335  Returns
336  -------
337  detId : `int`
338  Detector identifier.
339  image : `lsst.afw.image.MaskedImage`
340  Binned background model image.
341  """
342  assert cache.dataId == dataId
343  exposure = cache.exposure
344  detector = exposure.getDetector()
345  bbox = exposure.getMaskedImage().getBBox()
346  image = bgModel.toCcdBackground(detector, bbox).getImage()
347  return (detector.getId(), afwMath.binImage(image, BINNING))
348 
349  def collect(self, cache):
350  """Collect exposure for potential visualisation
351 
352  This method runs on the slave nodes.
353 
354  Parameters
355  ----------
356  cache : `lsst.pipe.base.Struct`
357  Process pool cache.
358 
359  Returns
360  -------
361  detId : `int`
362  Detector identifier.
363  image : `lsst.afw.image.MaskedImage`
364  Binned image.
365  """
366  return (cache.exposure.getDetector().getId(),
367  afwMath.binImage(cache.exposure.getMaskedImage(), BINNING))
368 
369  def collectOriginal(self, cache, dataId):
370  """Collect original image for visualisation
371 
372  This method runs on the slave nodes.
373 
374  Parameters
375  ----------
376  cache : `lsst.pipe.base.Struct`
377  Process pool cache.
378  dataId : `dict`
379  Data identifier.
380 
381  Returns
382  -------
383  detId : `int`
384  Detector identifier.
385  image : `lsst.afw.image.MaskedImage`
386  Binned image.
387  """
388  exposure = cache.butler.get("calexp", dataId, immediate=True)
389  return (exposure.getDetector().getId(),
390  afwMath.binImage(exposure.getMaskedImage(), BINNING))
391 
392  def collectSky(self, cache, dataId):
393  """Collect original image for visualisation
394 
395  This method runs on the slave nodes.
396 
397  Parameters
398  ----------
399  cache : `lsst.pipe.base.Struct`
400  Process pool cache.
401  dataId : `dict`
402  Data identifier.
403 
404  Returns
405  -------
406  detId : `int`
407  Detector identifier.
408  image : `lsst.afw.image.MaskedImage`
409  Binned image.
410  """
411  return (cache.exposure.getDetector().getId(), afwMath.binImage(cache.sky.getImage(), BINNING))
412 
413  def write(self, cache, dataId):
414  """Write resultant background list
415 
416  This method runs on the slave nodes.
417 
418  Parameters
419  ----------
420  cache : `lsst.pipe.base.Struct`
421  Process pool cache.
422  dataId : `dict`
423  Data identifier.
424  """
425  cache.butler.put(cache.bgList, "skyCorr", dataId)
426 
427  def _getMetadataName(self):
428  """There's no metadata to write out"""
429  return None
def subtractModel(self, cache, dataId, bgModel)
def batchWallTime(cls, time, parsedCmd, numCores)
def makeCameraImage(camera, exposures, filename=None, binning=8)
def realiseModel(self, cache, dataId, bgModel)
def subtractSkyFrame(self, cache, dataId, scale)
def logOperation(self, operation, catch=False, trace=True)