Coverage for python/lsst/meas/algorithms/scaleVariance.py: 30%
66 statements
« prev ^ index » next coverage.py v6.4.1, created at 2022-06-11 03:05 -0700
« prev ^ index » next coverage.py v6.4.1, created at 2022-06-11 03:05 -0700
1#
2# LSST Data Management System
3# Copyright 2022 AURA/LSST.
4#
5# This product includes software developed by the
6# LSST Project (http://www.lsst.org/).
7#
8# This program is free software: you can redistribute it and/or modify
9# it under the terms of the GNU General Public License as published by
10# the Free Software Foundation, either version 3 of the License, or
11# (at your option) any later version.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the LSST License Statement and
19# the GNU General Public License along with this program. If not,
20# see <http://www.lsstcorp.org/LegalNotices/>.
21#
22from contextlib import contextmanager
23import numpy as np
25from lsst.pex.config import Config, Field, ListField, ConfigurableField
26from lsst.pipe.base import Task, Struct
27from . import SubtractBackgroundTask
29__all__ = ["ScaleVarianceConfig", "ScaleVarianceTask"]
32class ScaleVarianceConfig(Config):
33 background = ConfigurableField(target=SubtractBackgroundTask, doc="Background subtraction")
34 maskPlanes = ListField(
35 dtype=str,
36 default=["DETECTED", "DETECTED_NEGATIVE", "BAD", "SAT", "NO_DATA", "INTRP"],
37 doc="Mask planes for pixels to ignore when scaling variance",
38 )
39 limit = Field(dtype=float, default=10.0, doc="Maximum variance scaling value to permit")
41 def setDefaults(self):
42 self.background.binSize = 32
43 self.background.useApprox = False
44 self.background.undersampleStyle = "REDUCE_INTERP_ORDER"
45 self.background.ignoredPixelMask = ["DETECTED", "DETECTED_NEGATIVE", "BAD", "SAT", "NO_DATA", "INTRP"]
48class ScaleVarianceTask(Task):
49 """Scale the variance in a MaskedImage
51 The variance plane in a convolved or warped image (or a coadd derived
52 from warped images) does not accurately reflect the noise properties of
53 the image because variance has been lost to covariance. This Task
54 attempts to correct for this by scaling the variance plane to match
55 the observed variance in the image. This is not perfect (because we're
56 not tracking the covariance) but it's simple and is often good enough.
58 The task implements a pixel-based and an image-based correction estimator.
59 """
60 ConfigClass = ScaleVarianceConfig
61 _DefaultName = "scaleVariance"
63 def __init__(self, *args, **kwargs):
64 Task.__init__(self, *args, **kwargs)
65 self.makeSubtask("background")
67 @contextmanager
68 def subtractedBackground(self, maskedImage):
69 """Context manager for subtracting the background
71 We need to subtract the background so that the entire image
72 (apart from objects, which should be clipped) will have the
73 image/sqrt(variance) distributed about zero.
75 This context manager subtracts the background, and ensures it
76 is restored on exit.
78 Parameters
79 ----------
80 maskedImage : `lsst.afw.image.MaskedImage`
81 Image+mask+variance to have background subtracted and restored.
83 Returns
84 -------
85 context : context manager
86 Context manager that ensure the background is restored.
87 """
88 bg = self.background.fitBackground(maskedImage)
89 bgImage = bg.getImageF(self.background.config.algorithm, self.background.config.undersampleStyle)
90 maskedImage -= bgImage
91 try:
92 yield
93 finally:
94 maskedImage += bgImage
96 def run(self, maskedImage):
97 """Rescale the variance in a maskedImage in place.
99 Parameters
100 ----------
101 maskedImage : `lsst.afw.image.MaskedImage`
102 Image for which to determine the variance rescaling factor. The image
103 is modified in place.
105 Returns
106 -------
107 factor : `float`
108 Variance rescaling factor.
110 Raises
111 ------
112 RuntimeError
113 If the estimated variance rescaling factor by both methods exceed the
114 configured limit.
116 Notes
117 -----
118 The task calculates and applies the pixel-based correction unless
119 it is over the ``config.limit`` threshold. In this case, the image-based
120 method is applied.
121 """
122 with self.subtractedBackground(maskedImage):
123 factor = self.pixelBased(maskedImage)
124 if factor > self.config.limit:
125 self.log.warning("Pixel-based variance rescaling factor (%f) exceeds configured limit (%f); "
126 "trying image-based method", factor, self.config.limit)
127 factor = self.imageBased(maskedImage)
128 if factor > self.config.limit:
129 raise RuntimeError("Variance rescaling factor (%f) exceeds configured limit (%f)" %
130 (factor, self.config.limit))
131 self.log.info("Renormalizing variance by %f", factor)
132 maskedImage.variance *= factor
133 return factor
135 def computeScaleFactors(self, maskedImage):
136 """Calculate and return both variance scaling factors without modifying the image.
138 Parameters
139 ----------
140 maskedImage : `lsst.afw.image.MaskedImage`
141 Image for which to determine the variance rescaling factor.
143 Returns
144 -------
145 R : `lsst.pipe.base.Struct`
146 - ``pixelFactor`` : `float` The pixel based variance rescaling factor
147 or 1 if all pixels are masked or invalid.
148 - ``imageFactor`` : `float` The image based variance rescaling factor
149 or 1 if all pixels are masked or invalid.
150 """
151 with self.subtractedBackground(maskedImage):
152 pixelFactor = self.pixelBased(maskedImage)
153 imageFactor = self.imageBased(maskedImage)
154 return Struct(pixelFactor=pixelFactor, imageFactor=imageFactor)
156 def pixelBased(self, maskedImage):
157 """Determine the variance rescaling factor from pixel statistics
159 We calculate SNR = image/sqrt(variance), and the distribution
160 for most of the background-subtracted image should have a standard
161 deviation of unity. We use the interquartile range as a robust estimator
162 of the SNR standard deviation. The variance rescaling factor is the
163 factor that brings that distribution to have unit standard deviation.
165 This may not work well if the image has a lot of structure in it, as
166 the assumptions are violated. In that case, use an alternate
167 method.
169 Parameters
170 ----------
171 maskedImage : `lsst.afw.image.MaskedImage`
172 Image for which to determine the variance rescaling factor.
174 Returns
175 -------
176 factor : `float`
177 Variance rescaling factor or 1 if all pixels are masked or non-finite.
179 """
180 maskVal = maskedImage.mask.getPlaneBitMask(self.config.maskPlanes)
181 isGood = (((maskedImage.mask.array & maskVal) == 0)
182 & np.isfinite(maskedImage.image.array)
183 & np.isfinite(maskedImage.variance.array)
184 & (maskedImage.variance.array > 0))
186 nGood = np.sum(isGood)
187 self.log.debug("Number of selected background pixels: %d of %d.", nGood, isGood.size)
188 if nGood < 2:
189 # Not enough good data, np.percentile needs at least 2 points
190 # to estimate a range
191 return 1.0
192 # Robust measurement of stdev using inter-quartile range
193 snr = maskedImage.image.array[isGood]/np.sqrt(maskedImage.variance.array[isGood])
194 q1, q3 = np.percentile(snr, (25, 75))
195 stdev = 0.74*(q3 - q1)
196 return stdev**2
198 def imageBased(self, maskedImage):
199 """Determine the variance rescaling factor from image statistics
201 We calculate average(SNR) = stdev(image)/median(variance), and
202 the value should be unity. We use the interquartile range as a robust
203 estimator of the stdev. The variance rescaling factor is the
204 factor that brings this value to unity.
206 This may not work well if the pixels from which we measure the
207 standard deviation of the image are not effectively the same pixels
208 from which we measure the median of the variance. In that case, use
209 an alternate method.
211 Parameters
212 ----------
213 maskedImage : `lsst.afw.image.MaskedImage`
214 Image for which to determine the variance rescaling factor.
216 Returns
217 -------
218 factor : `float`
219 Variance rescaling factor or 1 if all pixels are masked or non-finite.
220 """
221 maskVal = maskedImage.mask.getPlaneBitMask(self.config.maskPlanes)
222 isGood = (((maskedImage.mask.array & maskVal) == 0)
223 & np.isfinite(maskedImage.image.array)
224 & np.isfinite(maskedImage.variance.array)
225 & (maskedImage.variance.array > 0))
226 nGood = np.sum(isGood)
227 self.log.debug("Number of selected background pixels: %d of %d.", nGood, isGood.size)
228 if nGood < 2:
229 # Not enough good data, np.percentile needs at least 2 points
230 # to estimate a range
231 return 1.0
232 # Robust measurement of stdev
233 q1, q3 = np.percentile(maskedImage.image.array[isGood], (25, 75))
234 ratio = 0.74*(q3 - q1)/np.sqrt(np.median(maskedImage.variance.array[isGood]))
235 return ratio**2