lsst.meas.base  14.0-20-g83dc6f3
noiseReplacer.py
Go to the documentation of this file.
1 #!/usr/bin/env python
2 #
3 # LSST Data Management System
4 # Copyright 2008-2016 AURA/LSST.
5 #
6 # This product includes software developed by the
7 # LSST Project (http://www.lsst.org/).
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the LSST License Statement and
20 # the GNU General Public License along with this program. If not,
21 # see <https://www.lsstcorp.org/LegalNotices/>.
22 #
23 from builtins import str
24 from builtins import object
25 
26 import math
27 
28 import lsst.afw.detection as afwDet
29 import lsst.afw.image as afwImage
30 import lsst.afw.math as afwMath
31 import lsst.pex.config
32 
33 __all__ = ("NoiseReplacerConfig", "NoiseReplacer", "DummyNoiseReplacer")
34 
35 
36 class NoiseReplacerConfig(lsst.pex.config.Config):
37  noiseSource = lsst.pex.config.ChoiceField(
38  doc='How to choose mean and variance of the Gaussian noise we generate?',
39  dtype=str,
40  allowed={
41  'measure': 'Measure clipped mean and variance from the whole image',
42  'meta': 'Mean = 0, variance = the "BGMEAN" metadata entry',
43  'variance': "Mean = 0, variance = the image's variance",
44  },
45  default='measure', optional=False
46  )
47  noiseOffset = lsst.pex.config.Field(
48  doc='Add ann offset to the generated noise.',
49  dtype=float, optional=False, default=0.0
50  )
51  noiseSeedMultiplier = lsst.pex.config.Field(
52  dtype=int, default=1,
53  doc="The seed multiplier value to use for random number generation\n"
54  " >= 1: set the seed deterministically based on exposureId\n"
55  " 0: fall back to the afw.math.Random default constructor (which uses a seed value of 1)"
56  )
57 
58 
59 class NoiseReplacer(object):
60  """!
61  Class that handles replacing sources with noise during measurement.
62 
63  When measuring a source (or the children associated with a parent source), this class is used
64  to replace its neighbors with noise, using the deblender's definition of the sources as stored
65  in HeavyFootprints attached to the SourceRecords. The algorithm works as follows:
66  - We start by replacing all pixels that are in source Footprints with artificially
67  generated noise (__init__).
68  - When we are about to measure a particular source, we add it back in, by inserting that source's
69  HeavyFootprint (from the deblender) into the image.
70  - When we are done measuring that source, we again replace the HeavyFootprint with (the same)
71  artificial noise.
72  - After measuring all sources, we return the image to its original state.
73 
74  This is a functional copy of the code in the older ReplaceWithNoiseTask, but with a slightly different
75  API needed for the new measurement framework; note that it is not a Task, as the lifetime of a
76  NoiseReplacer now corresponds to a single exposure, not an entire processing run.
77  """
78 
79  ConfigClass = NoiseReplacerConfig
80 
81  def __init__(self, config, exposure, footprints, noiseImage=None, exposureId=None, log=None):
82  """!
83  Initialize the NoiseReplacer.
84 
85  @param[in] config instance of NoiseReplacerConfig
86  @param[in,out] exposure Exposure to be noise replaced. (All sources replaced on return)
87  @param[in] footprints dict of {id: (parent, footprint)};
88  @param[in] noiseImage an afw.image.ImageF used as a predictable noise replacement source
89  (for tests only)
90  @param[in] log Log object to use for status messages; no status messages
91  will be printed if None
92 
93  'footprints' is a dict of {id: (parent, footprint)}; when used in SFM, the ID will be the
94  source ID, but in forced photometry, this will be the reference ID, as that's what we used to
95  determine the deblend families. This routine should create HeavyFootprints for any non-Heavy
96  Footprints, and replace them in the dict. It should then create a dict of HeavyFootprints
97  containing noise, but only for parent objects, then replace all sources with noise.
98  This should ignore any footprints that lay outside the bounding box of the exposure,
99  and clip those that lie on the border.
100 
101  NOTE: as the code currently stands, the heavy footprint for a deblended object must be available
102  from the input catalog. If it is not, it cannot be reproduced here. In that case, the
103  topmost parent in the objects parent chain must be used. The heavy footprint for that source
104  is created in this class from the masked image.
105  """
106  noiseMeanVar = None
107  self.noiseSource = config.noiseSource
108  self.noiseOffset = config.noiseOffset
109  self.noiseSeedMultiplier = config.noiseSeedMultiplier
110  self.noiseGenMean = None
111  self.noiseGenStd = None
112  self.log = log
113 
114  # creates heavies, replaces all footprints with noise
115  # We need the source table to be sorted by ID to do the parent lookups
116  self.exposure = exposure
117  self.footprints = footprints
118  mi = exposure.getMaskedImage()
119  im = mi.getImage()
120  mask = mi.getMask()
121  # Add temporary Mask planes for THISDET and OTHERDET
122  self.removeplanes = []
123  bitmasks = []
124  for maskname in ['THISDET', 'OTHERDET']:
125  try:
126  # does it already exist?
127  plane = mask.getMaskPlane(maskname)
128  if self.log:
129  self.log.debug('Mask plane "%s" already existed', maskname)
130  except Exception:
131  # if not, add it; we should delete it when done.
132  plane = mask.addMaskPlane(maskname)
133  self.removeplanes.append(maskname)
134  mask.clearMaskPlane(plane)
135  bitmask = mask.getPlaneBitMask(maskname)
136  bitmasks.append(bitmask)
137  if self.log:
138  self.log.debug('Mask plane "%s": plane %i, bitmask %i = 0x%x',
139  maskname, plane, bitmask, bitmask)
140  self.thisbitmask, self.otherbitmask = bitmasks
141  del bitmasks
142  self.heavies = {}
143  # Start by creating HeavyFootprints for each source which has no parent
144  # and just use them for children which do not already have heavy footprints.
145  # If a heavy footprint is available for a child, we will use it. Otherwise,
146  # we use the first parent in the parent chain which has a heavy footprint,
147  # which with the one level deblender will alway be the topmost parent
148  # NOTE: heavy footprints get destroyed by the transform process in forcedPhotImage.py,
149  # so they are never available for forced measurements.
150 
151  # Create in the dict heavies = {id:heavyfootprint}
152  for id, fp in footprints.items():
153  if fp[1].isHeavy():
154  self.heavies[id] = fp[1]
155  elif fp[0] == 0:
156  self.heavies[id] = afwDet.makeHeavyFootprint(fp[1], mi)
157 
158  # ## FIXME: the heavy footprint includes the mask
159  # ## and variance planes, which we shouldn't need
160  # ## (I don't think we ever want to modify them in
161  # ## the input image). Copying them around is
162  # ## wasteful.
163 
164  # We now create a noise HeavyFootprint for each source with has a heavy footprint.
165  # We'll put the noise footprints in a dict heavyNoise = {id:heavyNoiseFootprint}
166  self.heavyNoise = {}
167  noisegen = self.getNoiseGenerator(exposure, noiseImage, noiseMeanVar, exposureId=exposureId)
168  # The noiseGenMean and Std are used by the unit tests
169  self.noiseGenMean = noisegen.mean
170  self.noiseGenStd = noisegen.std
171  if self.log:
172  self.log.debug('Using noise generator: %s', str(noisegen))
173  for id in self.heavies:
174  fp = footprints[id][1]
175  noiseFp = noisegen.getHeavyFootprint(fp)
176  self.heavyNoise[id] = noiseFp
177  # Also insert the noisy footprint into the image now.
178  # Notice that we're just inserting it into "im", ie,
179  # the Image, not the MaskedImage.
180  noiseFp.insert(im)
181  # Also set the OTHERDET bit
182  fp.spans.setMask(mask, self.otherbitmask)
183 
184  def insertSource(self, id):
185  """!
186  Insert the heavy footprint of a given source into the exposure
187 
188  @param[in] id id for current source to insert from original footprint dict
189 
190  Also adjusts the mask plane to show the source of this footprint.
191  """
192  # Copy this source's pixels into the image
193  mi = self.exposure.getMaskedImage()
194  im = mi.getImage()
195  mask = mi.getMask()
196  # usedid can point either to this source, or to the first parent in the
197  # parent chain which has a heavy footprint (or to the topmost parent,
198  # which always has one)
199  usedid = id
200  while self.footprints[usedid][0] != 0 and usedid not in self.heavies:
201  usedid = self.footprints[usedid][0]
202  fp = self.heavies[usedid]
203  fp.insert(im)
204  fp.spans.setMask(mask, self.thisbitmask)
205  fp.spans.clearMask(mask, self.otherbitmask)
206 
207  def removeSource(self, id):
208  """!
209  Remove the heavy footprint of a given source and replace with previous noise
210 
211  @param[in] id id for current source to insert from original footprint dict
212 
213  Also restore the mask plane.
214  """
215  # remove a single source
216  # (Replace this source's pixels by noise again.)
217  # Do this by finding the source's top-level ancestor
218  mi = self.exposure.getMaskedImage()
219  im = mi.getImage()
220  mask = mi.getMask()
221 
222  # use the same algorithm as in remove Source to find the heavy noise footprint
223  # which will undo what insertSource(id) does
224  usedid = id
225  while self.footprints[usedid][0] != 0 and usedid not in self.heavies:
226  usedid = self.footprints[usedid][0]
227  # Re-insert the noise pixels
228  fp = self.heavyNoise[usedid]
229  fp.insert(im)
230  # Clear the THISDET mask plane.
231  fp.spans.clearMask(mask, self.thisbitmask)
232  fp.spans.setMask(mask, self.otherbitmask)
233 
234  def end(self):
235  """!
236  End the NoiseReplacer.
237 
238  Restore original data to the exposure from the heavies dictionary
239  Restore the mask planes to their original state
240  """
241  # restores original image, cleans up temporaries
242  # (ie, replace all the top-level pixels)
243  mi = self.exposure.getMaskedImage()
244  im = mi.getImage()
245  mask = mi.getMask()
246  for id in self.footprints.keys():
247  if self.footprints[id][0] != 0:
248  continue
249  self.heavies[id].insert(im)
250  for maskname in self.removeplanes:
251  mask.removeAndClearMaskPlane(maskname, True)
252 
253  del self.removeplanes
254  del self.thisbitmask
255  del self.otherbitmask
256  del self.heavies
257  del self.heavyNoise
258 
259  def getNoiseGenerator(self, exposure, noiseImage, noiseMeanVar, exposureId=None):
260  """!
261  Generate noise image using parameters given
262  """
263  if noiseImage is not None:
264  return ImageNoiseGenerator(noiseImage)
265  rand = None
266  if self.noiseSeedMultiplier:
267  # default plugin, our seed
268  if exposureId is not None and exposureId != 0:
269  seed = exposureId*self.noiseSeedMultiplier
270  else:
271  seed = self.noiseSeedMultiplier
272  rand = afwMath.Random(afwMath.Random.MT19937, seed)
273  if noiseMeanVar is not None:
274  try:
275  # Assume noiseMeanVar is an iterable of floats
276  noiseMean, noiseVar = noiseMeanVar
277  noiseMean = float(noiseMean)
278  noiseVar = float(noiseVar)
279  noiseStd = math.sqrt(noiseVar)
280  if self.log:
281  self.log.debug('Using passed-in noise mean = %g, variance = %g -> stdev %g',
282  noiseMean, noiseVar, noiseStd)
283  return FixedGaussianNoiseGenerator(noiseMean, noiseStd, rand=rand)
284  except Exception:
285  if self.log:
286  self.log.debug('Failed to cast passed-in noiseMeanVar to floats: %s',
287  str(noiseMeanVar))
288  offset = self.noiseOffset
289  noiseSource = self.noiseSource
290 
291  if noiseSource == 'meta':
292  # check the exposure metadata
293  meta = exposure.getMetadata()
294  # this key name correspond to SubtractBackgroundTask() in meas_algorithms
295  try:
296  bgMean = meta.getAsDouble('BGMEAN')
297  # We would have to adjust for GAIN if ip_isr didn't make it 1.0
298  noiseStd = math.sqrt(bgMean)
299  if self.log:
300  self.log.debug('Using noise variance = (BGMEAN = %g) from exposure metadata',
301  bgMean)
302  return FixedGaussianNoiseGenerator(offset, noiseStd, rand=rand)
303  except Exception:
304  if self.log:
305  self.log.debug('Failed to get BGMEAN from exposure metadata')
306 
307  if noiseSource == 'variance':
308  if self.log:
309  self.log.debug('Will draw noise according to the variance plane.')
310  var = exposure.getMaskedImage().getVariance()
311  return VariancePlaneNoiseGenerator(var, mean=offset, rand=rand)
312 
313  # Compute an image-wide clipped variance.
314  im = exposure.getMaskedImage().getImage()
315  s = afwMath.makeStatistics(im, afwMath.MEANCLIP | afwMath.STDEVCLIP)
316  noiseMean = s.getValue(afwMath.MEANCLIP)
317  noiseStd = s.getValue(afwMath.STDEVCLIP)
318  if self.log:
319  self.log.debug("Measured from image: clipped mean = %g, stdev = %g",
320  noiseMean, noiseStd)
321  return FixedGaussianNoiseGenerator(noiseMean + offset, noiseStd, rand=rand)
322 
323 
324 class NoiseReplacerList(list):
325  """Syntactic sugar that makes a list of NoiseReplacers (for multiple exposures)
326  behave like a single one.
327 
328  This is only used in the multifit driver, but the logic there is already pretty
329  complex, so it's nice to have this to simplify it.
330  """
331 
332  def __init__(self, exposuresById, footprintsByExp):
333  # exposuresById --- dict of {exposureId: exposure} (possibly subimages)
334  # footprintsByExp --- nested dict of {exposureId: {objId: (parent, footprint)}}
335  list.__init__(self)
336  for expId, exposure in exposuresById.items():
337  self.append(NoiseReplacer(exposure, footprintsByExp[expId]), expId)
338 
339  def insertSource(self, id):
340  """Insert the original pixels for a given source (by id) into the original exposure.
341  """
342  for item in self:
343  self.insertSource(id)
344 
345  def removeSource(self, id):
346  """Insert the noise pixels for a given source (by id) into the original exposure.
347  """
348  for item in self:
349  self.removeSource(id)
350 
351  def end(self):
352  """Cleanup when the use of the Noise replacer is done.
353  """
354  for item in self:
355  self.end()
356 
357 
358 class NoiseGenerator(object):
359  """!
360  Base class for noise generators used by the "doReplaceWithNoise" routine:
361  these produce HeavyFootprints filled with noise generated in various ways.
362 
363  This is an abstract base class.
364  """
365 
366  def getHeavyFootprint(self, fp):
367  bb = fp.getBBox()
368  mim = self.getMaskedImage(bb)
369  return afwDet.makeHeavyFootprint(fp, mim)
370 
371  def getMaskedImage(self, bb):
372  im = self.getImage(bb)
373  return afwImage.MaskedImageF(im)
374 
375  def getImage(self, bb):
376  return None
377 
378 
380  """
381  Generates noise by cutting out a subimage from a user-supplied noise Image.
382  """
383 
384  def __init__(self, img):
385  """!
386  @param[in] img an afwImage.ImageF
387  """
388  self.mim = afwImage.MaskedImageF(img)
389  self.mean = afwMath.makeStatistics(img, afwMath.MEAN)
390  self.std = afwMath.makeStatistics(img, afwMath.STDEV)
391 
392  def getMaskedImage(self, bb):
393  return self.mim
394 
395 
397  """!
398  Generates noise using the afwMath.Random() and afwMath.randomGaussianImage() routines.
399 
400  This is an abstract base class.
401  """
402 
403  def __init__(self, rand=None):
404  if rand is None:
405  rand = afwMath.Random()
406  self.rand = rand
407 
408  def getRandomImage(self, bb):
409  # Create an Image and fill it with Gaussian noise.
410  rim = afwImage.ImageF(bb.getWidth(), bb.getHeight())
411  rim.setXY0(bb.getMinX(), bb.getMinY())
412  afwMath.randomGaussianImage(rim, self.rand)
413  return rim
414 
415 
417  """!
418  Generates Gaussian noise with a fixed mean and standard deviation.
419  """
420 
421  def __init__(self, mean, std, rand=None):
422  super(FixedGaussianNoiseGenerator, self).__init__(rand=rand)
423  self.mean = mean
424  self.std = std
425 
426  def __str__(self):
427  return 'FixedGaussianNoiseGenerator: mean=%g, std=%g' % (self.mean, self.std)
428 
429  def getImage(self, bb):
430  rim = self.getRandomImage(bb)
431  rim *= self.std
432  rim += self.mean
433  return rim
434 
435 
437  """!
438  Generates Gaussian noise whose variance matches that of the variance plane of the image.
439  """
440 
441  def __init__(self, var, mean=None, rand=None):
442  """!
443  @param[in] var an afwImage.ImageF; the variance plane.
444  @param[in,out] mean floating-point or afwImage.Image
445  """
446  super(VariancePlaneNoiseGenerator, self).__init__(rand=rand)
447  self.var = var
448  if mean is not None and mean == 0.:
449  mean = None
450  self.mean = mean
451 
452  def __str__(self):
453  return 'VariancePlaneNoiseGenerator: mean=' + str(self.mean)
454 
455  def getImage(self, bb):
456  rim = self.getRandomImage(bb)
457  # Use the image's variance plane to scale the noise.
458  stdev = afwImage.ImageF(self.var, bb, afwImage.LOCAL, True)
459  stdev.sqrt()
460  rim *= stdev
461  if self.mean is not None:
462  rim += self.mean
463  return rim
464 
465 
466 class DummyNoiseReplacer(object):
467  """!
468  A do-nothing standin for NoiseReplacer, used when we want to disable NoiseReplacer
469 
470  DummyNoiseReplacer has all the public methods of NoiseReplacer, but none of them do anything.
471  """
472 
473  def insertSource(self, id):
474  pass
475 
476  def removeSource(self, id):
477  pass
478 
479  def end(self):
480  pass
def getNoiseGenerator(self, exposure, noiseImage, noiseMeanVar, exposureId=None)
Generate noise image using parameters given.
def end(self)
End the NoiseReplacer.
def removeSource(self, id)
Remove the heavy footprint of a given source and replace with previous noise.
Base class for noise generators used by the "doReplaceWithNoise" routine: these produce HeavyFootprin...
def __init__(self, exposuresById, footprintsByExp)
def __init__(self, config, exposure, footprints, noiseImage=None, exposureId=None, log=None)
Initialize the NoiseReplacer.
A do-nothing standin for NoiseReplacer, used when we want to disable NoiseReplacer.
Generates Gaussian noise whose variance matches that of the variance plane of the image...
Generates Gaussian noise with a fixed mean and standard deviation.
Class that handles replacing sources with noise during measurement.
def insertSource(self, id)
Insert the heavy footprint of a given source into the exposure.
HeavyFootprint< ImagePixelT, MaskPixelT, VariancePixelT > makeHeavyFootprint(Footprint const &foot, lsst::afw::image::MaskedImage< ImagePixelT, MaskPixelT, VariancePixelT > const &img, HeavyFootprintCtrl const *ctrl=NULL)
Generates noise using the afwMath.Random() and afwMath.randomGaussianImage() routines.