lsst.meas.base  16.0-17-g7e0e4ff+12
noiseReplacer.py
Go to the documentation of this file.
1 #!/usr/bin/env python
2 #
3 # LSST Data Management System
4 # Copyright 2008-2016 AURA/LSST.
5 #
6 # This product includes software developed by the
7 # LSST Project (http://www.lsst.org/).
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the LSST License Statement and
20 # the GNU General Public License along with this program. If not,
21 # see <https://www.lsstcorp.org/LegalNotices/>.
22 #
23 
24 import math
25 
26 import lsst.afw.detection as afwDet
27 import lsst.afw.image as afwImage
28 import lsst.afw.math as afwMath
29 import lsst.pex.config
30 
31 __all__ = ("NoiseReplacerConfig", "NoiseReplacer", "DummyNoiseReplacer")
32 
33 
34 class NoiseReplacerConfig(lsst.pex.config.Config):
35  noiseSource = lsst.pex.config.ChoiceField(
36  doc='How to choose mean and variance of the Gaussian noise we generate?',
37  dtype=str,
38  allowed={
39  'measure': 'Measure clipped mean and variance from the whole image',
40  'meta': 'Mean = 0, variance = the "BGMEAN" metadata entry',
41  'variance': "Mean = 0, variance = the image's variance",
42  },
43  default='measure', optional=False
44  )
45  noiseOffset = lsst.pex.config.Field(
46  doc='Add ann offset to the generated noise.',
47  dtype=float, optional=False, default=0.0
48  )
49  noiseSeedMultiplier = lsst.pex.config.Field(
50  dtype=int, default=1,
51  doc="The seed multiplier value to use for random number generation\n"
52  " >= 1: set the seed deterministically based on exposureId\n"
53  " 0: fall back to the afw.math.Random default constructor (which uses a seed value of 1)"
54  )
55 
56 
58  """!
59  Class that handles replacing sources with noise during measurement.
60 
61  When measuring a source (or the children associated with a parent source), this class is used
62  to replace its neighbors with noise, using the deblender's definition of the sources as stored
63  in HeavyFootprints attached to the SourceRecords. The algorithm works as follows:
64  - We start by replacing all pixels that are in source Footprints with artificially
65  generated noise (__init__).
66  - When we are about to measure a particular source, we add it back in, by inserting that source's
67  HeavyFootprint (from the deblender) into the image.
68  - When we are done measuring that source, we again replace the HeavyFootprint with (the same)
69  artificial noise.
70  - After measuring all sources, we return the image to its original state.
71 
72  This is a functional copy of the code in the older ReplaceWithNoiseTask, but with a slightly different
73  API needed for the new measurement framework; note that it is not a Task, as the lifetime of a
74  NoiseReplacer now corresponds to a single exposure, not an entire processing run.
75  """
76 
77  ConfigClass = NoiseReplacerConfig
78 
79  def __init__(self, config, exposure, footprints, noiseImage=None, exposureId=None, log=None):
80  """!
81  Initialize the NoiseReplacer.
82 
83  @param[in] config instance of NoiseReplacerConfig
84  @param[in,out] exposure Exposure to be noise replaced. (All sources replaced on return)
85  @param[in] footprints dict of {id: (parent, footprint)};
86  @param[in] noiseImage an afw.image.ImageF used as a predictable noise replacement source
87  (for tests only)
88  @param[in] log Log object to use for status messages; no status messages
89  will be printed if None
90 
91  'footprints' is a dict of {id: (parent, footprint)}; when used in SFM, the ID will be the
92  source ID, but in forced photometry, this will be the reference ID, as that's what we used to
93  determine the deblend families. This routine should create HeavyFootprints for any non-Heavy
94  Footprints, and replace them in the dict. It should then create a dict of HeavyFootprints
95  containing noise, but only for parent objects, then replace all sources with noise.
96  This should ignore any footprints that lay outside the bounding box of the exposure,
97  and clip those that lie on the border.
98 
99  NOTE: as the code currently stands, the heavy footprint for a deblended object must be available
100  from the input catalog. If it is not, it cannot be reproduced here. In that case, the
101  topmost parent in the objects parent chain must be used. The heavy footprint for that source
102  is created in this class from the masked image.
103  """
104  noiseMeanVar = None
105  self.noiseSource = config.noiseSource
106  self.noiseOffset = config.noiseOffset
107  self.noiseSeedMultiplier = config.noiseSeedMultiplier
108  self.noiseGenMean = None
109  self.noiseGenStd = None
110  self.log = log
111 
112  # creates heavies, replaces all footprints with noise
113  # We need the source table to be sorted by ID to do the parent lookups
114  self.exposure = exposure
115  self.footprints = footprints
116  mi = exposure.getMaskedImage()
117  im = mi.getImage()
118  mask = mi.getMask()
119  # Add temporary Mask planes for THISDET and OTHERDET
120  self.removeplanes = []
121  bitmasks = []
122  for maskname in ['THISDET', 'OTHERDET']:
123  try:
124  # does it already exist?
125  plane = mask.getMaskPlane(maskname)
126  if self.log:
127  self.log.debug('Mask plane "%s" already existed', maskname)
128  except Exception:
129  # if not, add it; we should delete it when done.
130  plane = mask.addMaskPlane(maskname)
131  self.removeplanes.append(maskname)
132  mask.clearMaskPlane(plane)
133  bitmask = mask.getPlaneBitMask(maskname)
134  bitmasks.append(bitmask)
135  if self.log:
136  self.log.debug('Mask plane "%s": plane %i, bitmask %i = 0x%x',
137  maskname, plane, bitmask, bitmask)
138  self.thisbitmask, self.otherbitmask = bitmasks
139  del bitmasks
140  self.heavies = {}
141  # Start by creating HeavyFootprints for each source which has no parent
142  # and just use them for children which do not already have heavy footprints.
143  # If a heavy footprint is available for a child, we will use it. Otherwise,
144  # we use the first parent in the parent chain which has a heavy footprint,
145  # which with the one level deblender will alway be the topmost parent
146  # NOTE: heavy footprints get destroyed by the transform process in forcedPhotImage.py,
147  # so they are never available for forced measurements.
148 
149  # Create in the dict heavies = {id:heavyfootprint}
150  for id, fp in footprints.items():
151  if fp[1].isHeavy():
152  self.heavies[id] = fp[1]
153  elif fp[0] == 0:
154  self.heavies[id] = afwDet.makeHeavyFootprint(fp[1], mi)
155 
156  # ## FIXME: the heavy footprint includes the mask
157  # ## and variance planes, which we shouldn't need
158  # ## (I don't think we ever want to modify them in
159  # ## the input image). Copying them around is
160  # ## wasteful.
161 
162  # We now create a noise HeavyFootprint for each source with has a heavy footprint.
163  # We'll put the noise footprints in a dict heavyNoise = {id:heavyNoiseFootprint}
164  self.heavyNoise = {}
165  noisegen = self.getNoiseGenerator(exposure, noiseImage, noiseMeanVar, exposureId=exposureId)
166  # The noiseGenMean and Std are used by the unit tests
167  self.noiseGenMean = noisegen.mean
168  self.noiseGenStd = noisegen.std
169  if self.log:
170  self.log.debug('Using noise generator: %s', str(noisegen))
171  for id in self.heavies:
172  fp = footprints[id][1]
173  noiseFp = noisegen.getHeavyFootprint(fp)
174  self.heavyNoise[id] = noiseFp
175  # Also insert the noisy footprint into the image now.
176  # Notice that we're just inserting it into "im", ie,
177  # the Image, not the MaskedImage.
178  noiseFp.insert(im)
179  # Also set the OTHERDET bit
180  fp.spans.setMask(mask, self.otherbitmask)
181 
182  def insertSource(self, id):
183  """!
184  Insert the heavy footprint of a given source into the exposure
185 
186  @param[in] id id for current source to insert from original footprint dict
187 
188  Also adjusts the mask plane to show the source of this footprint.
189  """
190  # Copy this source's pixels into the image
191  mi = self.exposure.getMaskedImage()
192  im = mi.getImage()
193  mask = mi.getMask()
194  # usedid can point either to this source, or to the first parent in the
195  # parent chain which has a heavy footprint (or to the topmost parent,
196  # which always has one)
197  usedid = id
198  while self.footprints[usedid][0] != 0 and usedid not in self.heavies:
199  usedid = self.footprints[usedid][0]
200  fp = self.heavies[usedid]
201  fp.insert(im)
202  fp.spans.setMask(mask, self.thisbitmask)
203  fp.spans.clearMask(mask, self.otherbitmask)
204 
205  def removeSource(self, id):
206  """!
207  Remove the heavy footprint of a given source and replace with previous noise
208 
209  @param[in] id id for current source to insert from original footprint dict
210 
211  Also restore the mask plane.
212  """
213  # remove a single source
214  # (Replace this source's pixels by noise again.)
215  # Do this by finding the source's top-level ancestor
216  mi = self.exposure.getMaskedImage()
217  im = mi.getImage()
218  mask = mi.getMask()
219 
220  # use the same algorithm as in remove Source to find the heavy noise footprint
221  # which will undo what insertSource(id) does
222  usedid = id
223  while self.footprints[usedid][0] != 0 and usedid not in self.heavies:
224  usedid = self.footprints[usedid][0]
225  # Re-insert the noise pixels
226  fp = self.heavyNoise[usedid]
227  fp.insert(im)
228  # Clear the THISDET mask plane.
229  fp.spans.clearMask(mask, self.thisbitmask)
230  fp.spans.setMask(mask, self.otherbitmask)
231 
232  def end(self):
233  """!
234  End the NoiseReplacer.
235 
236  Restore original data to the exposure from the heavies dictionary
237  Restore the mask planes to their original state
238  """
239  # restores original image, cleans up temporaries
240  # (ie, replace all the top-level pixels)
241  mi = self.exposure.getMaskedImage()
242  im = mi.getImage()
243  mask = mi.getMask()
244  for id in self.footprints.keys():
245  if self.footprints[id][0] != 0:
246  continue
247  self.heavies[id].insert(im)
248  for maskname in self.removeplanes:
249  mask.removeAndClearMaskPlane(maskname, True)
250 
251  del self.removeplanes
252  del self.thisbitmask
253  del self.otherbitmask
254  del self.heavies
255  del self.heavyNoise
256 
257  def getNoiseGenerator(self, exposure, noiseImage, noiseMeanVar, exposureId=None):
258  """!
259  Generate noise image using parameters given
260  """
261  if noiseImage is not None:
262  return ImageNoiseGenerator(noiseImage)
263  rand = None
264  if self.noiseSeedMultiplier:
265  # default plugin, our seed
266  if exposureId is not None and exposureId != 0:
267  seed = exposureId*self.noiseSeedMultiplier
268  else:
269  seed = self.noiseSeedMultiplier
270  rand = afwMath.Random(afwMath.Random.MT19937, seed)
271  if noiseMeanVar is not None:
272  try:
273  # Assume noiseMeanVar is an iterable of floats
274  noiseMean, noiseVar = noiseMeanVar
275  noiseMean = float(noiseMean)
276  noiseVar = float(noiseVar)
277  noiseStd = math.sqrt(noiseVar)
278  if self.log:
279  self.log.debug('Using passed-in noise mean = %g, variance = %g -> stdev %g',
280  noiseMean, noiseVar, noiseStd)
281  return FixedGaussianNoiseGenerator(noiseMean, noiseStd, rand=rand)
282  except Exception:
283  if self.log:
284  self.log.debug('Failed to cast passed-in noiseMeanVar to floats: %s',
285  str(noiseMeanVar))
286  offset = self.noiseOffset
287  noiseSource = self.noiseSource
288 
289  if noiseSource == 'meta':
290  # check the exposure metadata
291  meta = exposure.getMetadata()
292  # this key name correspond to SubtractBackgroundTask() in meas_algorithms
293  try:
294  bgMean = meta.getAsDouble('BGMEAN')
295  # We would have to adjust for GAIN if ip_isr didn't make it 1.0
296  noiseStd = math.sqrt(bgMean)
297  if self.log:
298  self.log.debug('Using noise variance = (BGMEAN = %g) from exposure metadata',
299  bgMean)
300  return FixedGaussianNoiseGenerator(offset, noiseStd, rand=rand)
301  except Exception:
302  if self.log:
303  self.log.debug('Failed to get BGMEAN from exposure metadata')
304 
305  if noiseSource == 'variance':
306  if self.log:
307  self.log.debug('Will draw noise according to the variance plane.')
308  var = exposure.getMaskedImage().getVariance()
309  return VariancePlaneNoiseGenerator(var, mean=offset, rand=rand)
310 
311  # Compute an image-wide clipped variance.
312  im = exposure.getMaskedImage().getImage()
313  s = afwMath.makeStatistics(im, afwMath.MEANCLIP | afwMath.STDEVCLIP)
314  noiseMean = s.getValue(afwMath.MEANCLIP)
315  noiseStd = s.getValue(afwMath.STDEVCLIP)
316  if self.log:
317  self.log.debug("Measured from image: clipped mean = %g, stdev = %g",
318  noiseMean, noiseStd)
319  return FixedGaussianNoiseGenerator(noiseMean + offset, noiseStd, rand=rand)
320 
321 
322 class NoiseReplacerList(list):
323  """Syntactic sugar that makes a list of NoiseReplacers (for multiple exposures)
324  behave like a single one.
325 
326  This is only used in the multifit driver, but the logic there is already pretty
327  complex, so it's nice to have this to simplify it.
328  """
329 
330  def __init__(self, exposuresById, footprintsByExp):
331  # exposuresById --- dict of {exposureId: exposure} (possibly subimages)
332  # footprintsByExp --- nested dict of {exposureId: {objId: (parent, footprint)}}
333  list.__init__(self)
334  for expId, exposure in exposuresById.items():
335  self.append(NoiseReplacer(exposure, footprintsByExp[expId]), expId)
336 
337  def insertSource(self, id):
338  """Insert the original pixels for a given source (by id) into the original exposure.
339  """
340  for item in self:
341  self.insertSource(id)
342 
343  def removeSource(self, id):
344  """Insert the noise pixels for a given source (by id) into the original exposure.
345  """
346  for item in self:
347  self.removeSource(id)
348 
349  def end(self):
350  """Cleanup when the use of the Noise replacer is done.
351  """
352  for item in self:
353  self.end()
354 
355 
357  """!
358  Base class for noise generators used by the "doReplaceWithNoise" routine:
359  these produce HeavyFootprints filled with noise generated in various ways.
360 
361  This is an abstract base class.
362  """
363 
364  def getHeavyFootprint(self, fp):
365  bb = fp.getBBox()
366  mim = self.getMaskedImage(bb)
367  return afwDet.makeHeavyFootprint(fp, mim)
368 
369  def getMaskedImage(self, bb):
370  im = self.getImage(bb)
371  return afwImage.MaskedImageF(im)
372 
373  def getImage(self, bb):
374  return None
375 
376 
378  """
379  Generates noise by cutting out a subimage from a user-supplied noise Image.
380  """
381 
382  def __init__(self, img):
383  """!
384  @param[in] img an afwImage.ImageF
385  """
386  self.mim = afwImage.MaskedImageF(img)
387  self.mean = afwMath.makeStatistics(img, afwMath.MEAN)
388  self.std = afwMath.makeStatistics(img, afwMath.STDEV)
389 
390  def getMaskedImage(self, bb):
391  return self.mim
392 
393 
395  """!
396  Generates noise using the afwMath.Random() and afwMath.randomGaussianImage() routines.
397 
398  This is an abstract base class.
399  """
400 
401  def __init__(self, rand=None):
402  if rand is None:
403  rand = afwMath.Random()
404  self.rand = rand
405 
406  def getRandomImage(self, bb):
407  # Create an Image and fill it with Gaussian noise.
408  rim = afwImage.ImageF(bb.getWidth(), bb.getHeight())
409  rim.setXY0(bb.getMinX(), bb.getMinY())
410  afwMath.randomGaussianImage(rim, self.rand)
411  return rim
412 
413 
415  """!
416  Generates Gaussian noise with a fixed mean and standard deviation.
417  """
418 
419  def __init__(self, mean, std, rand=None):
420  super(FixedGaussianNoiseGenerator, self).__init__(rand=rand)
421  self.mean = mean
422  self.std = std
423 
424  def __str__(self):
425  return 'FixedGaussianNoiseGenerator: mean=%g, std=%g' % (self.mean, self.std)
426 
427  def getImage(self, bb):
428  rim = self.getRandomImage(bb)
429  rim *= self.std
430  rim += self.mean
431  return rim
432 
433 
435  """!
436  Generates Gaussian noise whose variance matches that of the variance plane of the image.
437  """
438 
439  def __init__(self, var, mean=None, rand=None):
440  """!
441  @param[in] var an afwImage.ImageF; the variance plane.
442  @param[in,out] mean floating-point or afwImage.Image
443  """
444  super(VariancePlaneNoiseGenerator, self).__init__(rand=rand)
445  self.var = var
446  if mean is not None and mean == 0.:
447  mean = None
448  self.mean = mean
449 
450  def __str__(self):
451  return 'VariancePlaneNoiseGenerator: mean=' + str(self.mean)
452 
453  def getImage(self, bb):
454  rim = self.getRandomImage(bb)
455  # Use the image's variance plane to scale the noise.
456  stdev = afwImage.ImageF(self.var, bb, afwImage.LOCAL, True)
457  stdev.sqrt()
458  rim *= stdev
459  if self.mean is not None:
460  rim += self.mean
461  return rim
462 
463 
465  """!
466  A do-nothing standin for NoiseReplacer, used when we want to disable NoiseReplacer
467 
468  DummyNoiseReplacer has all the public methods of NoiseReplacer, but none of them do anything.
469  """
470 
471  def insertSource(self, id):
472  pass
473 
474  def removeSource(self, id):
475  pass
476 
477  def end(self):
478  pass
def getNoiseGenerator(self, exposure, noiseImage, noiseMeanVar, exposureId=None)
Generate noise image using parameters given.
def end(self)
End the NoiseReplacer.
def removeSource(self, id)
Remove the heavy footprint of a given source and replace with previous noise.
Base class for noise generators used by the "doReplaceWithNoise" routine: these produce HeavyFootprin...
def __init__(self, exposuresById, footprintsByExp)
def __init__(self, config, exposure, footprints, noiseImage=None, exposureId=None, log=None)
Initialize the NoiseReplacer.
A do-nothing standin for NoiseReplacer, used when we want to disable NoiseReplacer.
Generates Gaussian noise whose variance matches that of the variance plane of the image...
Generates Gaussian noise with a fixed mean and standard deviation.
Class that handles replacing sources with noise during measurement.
def insertSource(self, id)
Insert the heavy footprint of a given source into the exposure.
Generates noise using the afwMath.Random() and afwMath.randomGaussianImage() routines.