lsst.pipe.tasks  14.0-52-g00a5fa4c+2
scaleVariance.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 # Copyright 2018 AURA/LSST.
4 #
5 # This product includes software developed by the
6 # LSST Project (http://www.lsst.org/).
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <http://www.lsstcorp.org/LegalNotices/>.
21 #
22 
23 from __future__ import absolute_import, division, print_function
24 
25 from contextlib import contextmanager
26 import numpy as np
27 
28 from lsst.pex.config import Config, Field, ListField, ConfigurableField
29 from lsst.pipe.base import Task
30 from lsst.meas.algorithms import SubtractBackgroundTask
31 
32 
33 
34 class ScaleVarianceConfig(Config):
35  background = ConfigurableField(target=SubtractBackgroundTask, doc="Background subtraction")
36  maskPlanes = ListField(
37  dtype=str,
38  default=["DETECTED", "DETECTED_NEGATIVE", "BAD", "SAT", "NO_DATA", "INTRP"],
39  doc="Mask planes for pixels to ignore when scaling variance",
40  )
41  limit = Field(dtype=float, default=10.0, doc="Maximum variance scaling value to permit")
42 
43  def setDefaults(self):
44  self.background.binSize = 32
45  self.background.useApprox = False
46  self.background.undersampleStyle = "REDUCE_INTERP_ORDER"
47  self.background.ignoredPixelMask = ["DETECTED", "DETECTED_NEGATIVE", "BAD", "SAT", "NO_DATA", "INTRP"]
48 
49 
50 class ScaleVarianceTask(Task):
51  """Scale the variance in a MaskedImage
52 
53  The variance plane in a convolved or warped image (or a coadd derived
54  from warped images) does not accurately reflect the noise properties of
55  the image because variance has been lost to covariance. This Task
56  attempts to correct for this by scaling the variance plane to match
57  the observed variance in the image. This is not perfect (because we're
58  not tracking the covariance) but it's simple and is often good enough.
59  """
60  ConfigClass = ScaleVarianceConfig
61  _DefaultName = "scaleVariance"
62 
63  def __init__(self, *args, **kwargs):
64  Task.__init__(self, *args, **kwargs)
65  self.makeSubtask("background")
66 
67  @contextmanager
68  def subtractedBackground(self, maskedImage):
69  """Context manager for subtracting the background
70 
71  We need to subtract the background so that the entire image
72  (apart from objects, which should be clipped) will have the
73  image/sqrt(variance) distributed about zero.
74 
75  This context manager subtracts the background, and ensures it
76  is restored on exit.
77 
78  Parameters
79  ----------
80  maskedImage : `lsst.afw.image.MaskedImage`
81  Image+mask+variance to have background subtracted and restored.
82 
83  Returns
84  -------
85  context : context manager
86  Context manager that ensure the background is restored.
87  """
88  bg = self.background.fitBackground(maskedImage)
89  bgImage = bg.getImageF()
90  maskedImage -= bgImage
91  try:
92  yield
93  finally:
94  maskedImage += bgImage
95 
96  def run(self, maskedImage):
97  """Rescale the variance in a maskedImage
98 
99  Parameters
100  ----------
101  maskedImage : `lsst.afw.image.MaskedImage`
102  Image for which to determine the variance rescaling factor.
103 
104  Returns
105  -------
106  factor : `float`
107  Variance rescaling factor.
108 
109  Raises
110  ------
111  RuntimeError
112  If the estimated variance rescaling factor exceeds the
113  configured limit.
114  """
115  with self.subtractedBackground(maskedImage):
116  factor = self.pixelBased(maskedImage)
117  if factor > self.config.limit:
118  self.log.warn("Pixel-based variance rescaling factor (%f) exceeds configured limit (%f); "
119  "trying image-based method", factor, self.config.limit)
120  factor = self.imageBased(maskedImage)
121  if factor > self.config.limit:
122  raise RuntimeError("Variance rescaling factor (%f) exceeds configured limit (%f)" %
123  (factor, self.config.limit))
124  self.log.info("Renormalizing variance by %f" % (factor,))
125  maskedImage.variance *= factor
126  return factor
127 
128  def pixelBased(self, maskedImage):
129  """Determine the variance rescaling factor from pixel statistics
130 
131  We calculate SNR = image/sqrt(variance), and the distribution
132  for most of the background-subtracted image should have a standard
133  deviation of unity. The variance rescaling factor is the factor that
134  brings that distribution to have unit standard deviation.
135 
136  This may not work well if the image has a lot of structure in it, as
137  the assumptions are violated. In that case, use an alternate
138  method.
139 
140  Parameters
141  ----------
142  maskedImage : `lsst.afw.image.MaskedImage`
143  Image for which to determine the variance rescaling factor.
144 
145  Returns
146  -------
147  factor : `float`
148  Variance rescaling factor.
149  """
150  variance = maskedImage.variance
151  snr = maskedImage.image.array/np.sqrt(variance.array)
152  maskVal = maskedImage.mask.getPlaneBitMask(self.config.maskPlanes)
153  isGood = (maskedImage.mask.array & maskVal) == 0
154  # Robust measurement of stdev using inter-quartile range
155  q1, q3 = np.percentile(snr[isGood], (25, 75))
156  stdev = 0.74*(q3 - q1)
157  return stdev**2
158 
159  def imageBased(self, maskedImage):
160  """Determine the variance rescaling factor from image statistics
161 
162  We calculate average(SNR) = stdev(image)/median(variance), and
163  the value should be unity. The variance rescaling factor is the
164  factor that brings this value to unity.
165 
166  This may not work well if the pixels from which we measure the
167  standard deviation of the image are not effectively the same pixels
168  from which we measure the median of the variance. In that case, use
169  an alternate method.
170 
171  Parameters
172  ----------
173  maskedImage : `lsst.afw.image.MaskedImage`
174  Image for which to determine the variance rescaling factor.
175 
176  Returns
177  -------
178  factor : `float`
179  Variance rescaling factor.
180  """
181  isGood = (maskedImage.mask.array & maskVal) == 0
182  # Robust measurement of stdev
183  q1, q3 = np.percentile(maskedImage.image.array[isGood], (25, 75))
184  ratio = 0.74*(q3 - q1)/np.sqrt(np.median(maskedImage.variance.array[isGood]))
185  return ratio**2