lsst.pipe.tasks  15.0-7-g6bb3a066+2
scaleVariance.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 # Copyright 2018 AURA/LSST.
4 #
5 # This product includes software developed by the
6 # LSST Project (http://www.lsst.org/).
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <http://www.lsstcorp.org/LegalNotices/>.
21 #
22 from contextlib import contextmanager
23 import numpy as np
24 
25 from lsst.pex.config import Config, Field, ListField, ConfigurableField
26 from lsst.pipe.base import Task
27 from lsst.meas.algorithms import SubtractBackgroundTask
28 
29 
30 
31 class ScaleVarianceConfig(Config):
32  background = ConfigurableField(target=SubtractBackgroundTask, doc="Background subtraction")
33  maskPlanes = ListField(
34  dtype=str,
35  default=["DETECTED", "DETECTED_NEGATIVE", "BAD", "SAT", "NO_DATA", "INTRP"],
36  doc="Mask planes for pixels to ignore when scaling variance",
37  )
38  limit = Field(dtype=float, default=10.0, doc="Maximum variance scaling value to permit")
39 
40  def setDefaults(self):
41  self.background.binSize = 32
42  self.background.useApprox = False
43  self.background.undersampleStyle = "REDUCE_INTERP_ORDER"
44  self.background.ignoredPixelMask = ["DETECTED", "DETECTED_NEGATIVE", "BAD", "SAT", "NO_DATA", "INTRP"]
45 
46 
47 class ScaleVarianceTask(Task):
48  """Scale the variance in a MaskedImage
49 
50  The variance plane in a convolved or warped image (or a coadd derived
51  from warped images) does not accurately reflect the noise properties of
52  the image because variance has been lost to covariance. This Task
53  attempts to correct for this by scaling the variance plane to match
54  the observed variance in the image. This is not perfect (because we're
55  not tracking the covariance) but it's simple and is often good enough.
56  """
57  ConfigClass = ScaleVarianceConfig
58  _DefaultName = "scaleVariance"
59 
60  def __init__(self, *args, **kwargs):
61  Task.__init__(self, *args, **kwargs)
62  self.makeSubtask("background")
63 
64  @contextmanager
65  def subtractedBackground(self, maskedImage):
66  """Context manager for subtracting the background
67 
68  We need to subtract the background so that the entire image
69  (apart from objects, which should be clipped) will have the
70  image/sqrt(variance) distributed about zero.
71 
72  This context manager subtracts the background, and ensures it
73  is restored on exit.
74 
75  Parameters
76  ----------
77  maskedImage : `lsst.afw.image.MaskedImage`
78  Image+mask+variance to have background subtracted and restored.
79 
80  Returns
81  -------
82  context : context manager
83  Context manager that ensure the background is restored.
84  """
85  bg = self.background.fitBackground(maskedImage)
86  bgImage = bg.getImageF()
87  maskedImage -= bgImage
88  try:
89  yield
90  finally:
91  maskedImage += bgImage
92 
93  def run(self, maskedImage):
94  """Rescale the variance in a maskedImage
95 
96  Parameters
97  ----------
98  maskedImage : `lsst.afw.image.MaskedImage`
99  Image for which to determine the variance rescaling factor.
100 
101  Returns
102  -------
103  factor : `float`
104  Variance rescaling factor.
105 
106  Raises
107  ------
108  RuntimeError
109  If the estimated variance rescaling factor exceeds the
110  configured limit.
111  """
112  with self.subtractedBackground(maskedImage):
113  factor = self.pixelBased(maskedImage)
114  if np.isnan(factor) or factor > self.config.limit:
115  self.log.warn("Pixel-based variance rescaling factor (%f) exceeds configured limit (%f); "
116  "trying image-based method", factor, self.config.limit)
117  factor = self.imageBased(maskedImage)
118  if np.isnan(factor) or factor > self.config.limit:
119  raise RuntimeError("Variance rescaling factor (%f) exceeds configured limit (%f)" %
120  (factor, self.config.limit))
121  self.log.info("Renormalizing variance by %f" % (factor,))
122  maskedImage.variance *= factor
123  return factor
124 
125  def pixelBased(self, maskedImage):
126  """Determine the variance rescaling factor from pixel statistics
127 
128  We calculate SNR = image/sqrt(variance), and the distribution
129  for most of the background-subtracted image should have a standard
130  deviation of unity. The variance rescaling factor is the factor that
131  brings that distribution to have unit standard deviation.
132 
133  This may not work well if the image has a lot of structure in it, as
134  the assumptions are violated. In that case, use an alternate
135  method.
136 
137  Parameters
138  ----------
139  maskedImage : `lsst.afw.image.MaskedImage`
140  Image for which to determine the variance rescaling factor.
141 
142  Returns
143  -------
144  factor : `float`
145  Variance rescaling factor.
146  """
147  variance = maskedImage.variance
148  snr = maskedImage.image.array/np.sqrt(variance.array)
149  maskVal = maskedImage.mask.getPlaneBitMask(self.config.maskPlanes)
150  isGood = ((maskedImage.mask.array & maskVal) == 0) & (maskedImage.variance.array > 0)
151  # Robust measurement of stdev using inter-quartile range
152  q1, q3 = np.percentile(snr[isGood], (25, 75))
153  stdev = 0.74*(q3 - q1)
154  return stdev**2
155 
156  def imageBased(self, maskedImage):
157  """Determine the variance rescaling factor from image statistics
158 
159  We calculate average(SNR) = stdev(image)/median(variance), and
160  the value should be unity. The variance rescaling factor is the
161  factor that brings this value to unity.
162 
163  This may not work well if the pixels from which we measure the
164  standard deviation of the image are not effectively the same pixels
165  from which we measure the median of the variance. In that case, use
166  an alternate method.
167 
168  Parameters
169  ----------
170  maskedImage : `lsst.afw.image.MaskedImage`
171  Image for which to determine the variance rescaling factor.
172 
173  Returns
174  -------
175  factor : `float`
176  Variance rescaling factor.
177  """
178  maskVal = maskedImage.mask.getPlaneBitMask(self.config.maskPlanes)
179  isGood = ((maskedImage.mask.array & maskVal) == 0) & (maskedImage.variance.array > 0)
180  # Robust measurement of stdev
181  q1, q3 = np.percentile(maskedImage.image.array[isGood], (25, 75))
182  ratio = 0.74*(q3 - q1)/np.sqrt(np.median(maskedImage.variance.array[isGood]))
183  return ratio**2