lsst.pipe.tasks  15.0-13-g5992896b+1
scaleVariance.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 # Copyright 2018 AURA/LSST.
4 #
5 # This product includes software developed by the
6 # LSST Project (http://www.lsst.org/).
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <http://www.lsstcorp.org/LegalNotices/>.
21 #
22 from contextlib import contextmanager
23 import numpy as np
24 
25 from lsst.pex.config import Config, Field, ListField, ConfigurableField
26 from lsst.pipe.base import Task
27 from lsst.meas.algorithms import SubtractBackgroundTask
28 
29 
30 class ScaleVarianceConfig(Config):
31  background = ConfigurableField(target=SubtractBackgroundTask, doc="Background subtraction")
32  maskPlanes = ListField(
33  dtype=str,
34  default=["DETECTED", "DETECTED_NEGATIVE", "BAD", "SAT", "NO_DATA", "INTRP"],
35  doc="Mask planes for pixels to ignore when scaling variance",
36  )
37  limit = Field(dtype=float, default=10.0, doc="Maximum variance scaling value to permit")
38 
39  def setDefaults(self):
40  self.background.binSize = 32
41  self.background.useApprox = False
42  self.background.undersampleStyle = "REDUCE_INTERP_ORDER"
43  self.background.ignoredPixelMask = ["DETECTED", "DETECTED_NEGATIVE", "BAD", "SAT", "NO_DATA", "INTRP"]
44 
45 
46 class ScaleVarianceTask(Task):
47  """Scale the variance in a MaskedImage
48 
49  The variance plane in a convolved or warped image (or a coadd derived
50  from warped images) does not accurately reflect the noise properties of
51  the image because variance has been lost to covariance. This Task
52  attempts to correct for this by scaling the variance plane to match
53  the observed variance in the image. This is not perfect (because we're
54  not tracking the covariance) but it's simple and is often good enough.
55  """
56  ConfigClass = ScaleVarianceConfig
57  _DefaultName = "scaleVariance"
58 
59  def __init__(self, *args, **kwargs):
60  Task.__init__(self, *args, **kwargs)
61  self.makeSubtask("background")
62 
63  @contextmanager
64  def subtractedBackground(self, maskedImage):
65  """Context manager for subtracting the background
66 
67  We need to subtract the background so that the entire image
68  (apart from objects, which should be clipped) will have the
69  image/sqrt(variance) distributed about zero.
70 
71  This context manager subtracts the background, and ensures it
72  is restored on exit.
73 
74  Parameters
75  ----------
76  maskedImage : `lsst.afw.image.MaskedImage`
77  Image+mask+variance to have background subtracted and restored.
78 
79  Returns
80  -------
81  context : context manager
82  Context manager that ensure the background is restored.
83  """
84  bg = self.background.fitBackground(maskedImage)
85  bgImage = bg.getImageF()
86  maskedImage -= bgImage
87  try:
88  yield
89  finally:
90  maskedImage += bgImage
91 
92  def run(self, maskedImage):
93  """Rescale the variance in a maskedImage
94 
95  Parameters
96  ----------
97  maskedImage : `lsst.afw.image.MaskedImage`
98  Image for which to determine the variance rescaling factor.
99 
100  Returns
101  -------
102  factor : `float`
103  Variance rescaling factor.
104 
105  Raises
106  ------
107  RuntimeError
108  If the estimated variance rescaling factor exceeds the
109  configured limit.
110  """
111  with self.subtractedBackground(maskedImage):
112  factor = self.pixelBased(maskedImage)
113  if np.isnan(factor) or factor > self.config.limit:
114  self.log.warn("Pixel-based variance rescaling factor (%f) exceeds configured limit (%f); "
115  "trying image-based method", factor, self.config.limit)
116  factor = self.imageBased(maskedImage)
117  if np.isnan(factor) or factor > self.config.limit:
118  raise RuntimeError("Variance rescaling factor (%f) exceeds configured limit (%f)" %
119  (factor, self.config.limit))
120  self.log.info("Renormalizing variance by %f" % (factor,))
121  maskedImage.variance *= factor
122  return factor
123 
124  def pixelBased(self, maskedImage):
125  """Determine the variance rescaling factor from pixel statistics
126 
127  We calculate SNR = image/sqrt(variance), and the distribution
128  for most of the background-subtracted image should have a standard
129  deviation of unity. The variance rescaling factor is the factor that
130  brings that distribution to have unit standard deviation.
131 
132  This may not work well if the image has a lot of structure in it, as
133  the assumptions are violated. In that case, use an alternate
134  method.
135 
136  Parameters
137  ----------
138  maskedImage : `lsst.afw.image.MaskedImage`
139  Image for which to determine the variance rescaling factor.
140 
141  Returns
142  -------
143  factor : `float`
144  Variance rescaling factor.
145  """
146  variance = maskedImage.variance
147  snr = maskedImage.image.array/np.sqrt(variance.array)
148  maskVal = maskedImage.mask.getPlaneBitMask(self.config.maskPlanes)
149  isGood = ((maskedImage.mask.array & maskVal) == 0) & (maskedImage.variance.array > 0)
150  # Robust measurement of stdev using inter-quartile range
151  q1, q3 = np.percentile(snr[isGood], (25, 75))
152  stdev = 0.74*(q3 - q1)
153  return stdev**2
154 
155  def imageBased(self, maskedImage):
156  """Determine the variance rescaling factor from image statistics
157 
158  We calculate average(SNR) = stdev(image)/median(variance), and
159  the value should be unity. The variance rescaling factor is the
160  factor that brings this value to unity.
161 
162  This may not work well if the pixels from which we measure the
163  standard deviation of the image are not effectively the same pixels
164  from which we measure the median of the variance. In that case, use
165  an alternate method.
166 
167  Parameters
168  ----------
169  maskedImage : `lsst.afw.image.MaskedImage`
170  Image for which to determine the variance rescaling factor.
171 
172  Returns
173  -------
174  factor : `float`
175  Variance rescaling factor.
176  """
177  maskVal = maskedImage.mask.getPlaneBitMask(self.config.maskPlanes)
178  isGood = ((maskedImage.mask.array & maskVal) == 0) & (maskedImage.variance.array > 0)
179  # Robust measurement of stdev
180  q1, q3 = np.percentile(maskedImage.image.array[isGood], (25, 75))
181  ratio = 0.74*(q3 - q1)/np.sqrt(np.median(maskedImage.variance.array[isGood]))
182  return ratio**2