lsst.meas.algorithms gc655b1545f+78fc91a3d9
Loading...
Searching...
No Matches
scaleVariance.py
Go to the documentation of this file.
2# LSST Data Management System
3# Copyright 2022 AURA/LSST.
4#
5# This product includes software developed by the
6# LSST Project (http://www.lsst.org/).
7#
8# This program is free software: you can redistribute it and/or modify
9# it under the terms of the GNU General Public License as published by
10# the Free Software Foundation, either version 3 of the License, or
11# (at your option) any later version.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the LSST License Statement and
19# the GNU General Public License along with this program. If not,
20# see <http://www.lsstcorp.org/LegalNotices/>.
21#
22from contextlib import contextmanager
23import numpy as np
24
25from lsst.pex.config import Config, Field, ListField, ConfigurableField
26from lsst.pipe.base import Task, Struct
27from . import SubtractBackgroundTask
28
29__all__ = ["ScaleVarianceConfig", "ScaleVarianceTask"]
30
31
32class ScaleVarianceConfig(Config):
33 background = ConfigurableField(target=SubtractBackgroundTask, doc="Background subtraction")
34 maskPlanes = ListField(
35 dtype=str,
36 default=["DETECTED", "DETECTED_NEGATIVE", "BAD", "SAT", "NO_DATA", "INTRP"],
37 doc="Mask planes for pixels to ignore when scaling variance",
38 )
39 limit = Field(dtype=float, default=10.0, doc="Maximum variance scaling value to permit")
40
41 def setDefaults(self):
42 self.background.binSize = 32
43 self.background.useApprox = False
44 self.background.undersampleStyle = "REDUCE_INTERP_ORDER"
45 self.background.ignoredPixelMask = ["DETECTED", "DETECTED_NEGATIVE", "BAD", "SAT", "NO_DATA", "INTRP"]
46
47
49 """Scale the variance in a MaskedImage
50
51 The variance plane in a convolved or warped image (or a coadd derived
52 from warped images) does not accurately reflect the noise properties of
53 the image because variance has been lost to covariance. This Task
54 attempts to correct for this by scaling the variance plane to match
55 the observed variance in the image. This is not perfect (because we're
56 not tracking the covariance) but it's simple and is often good enough.
57
58 The task implements a pixel-based and an image-based correction estimator.
59 """
60 ConfigClass = ScaleVarianceConfig
61 _DefaultName = "scaleVariance"
62
63 def __init__(self, *args, **kwargs):
64 Task.__init__(self, *args, **kwargs)
65 self.makeSubtask("background")
66
67 @contextmanager
68 def subtractedBackground(self, maskedImage):
69 """Context manager for subtracting the background
70
71 We need to subtract the background so that the entire image
72 (apart from objects, which should be clipped) will have the
73 image/sqrt(variance) distributed about zero.
74
75 This context manager subtracts the background, and ensures it
76 is restored on exit.
77
78 Parameters
79 ----------
80 maskedImage : `lsst.afw.image.MaskedImage`
81 Image+mask+variance to have background subtracted and restored.
82
83 Returns
84 -------
85 context : context manager
86 Context manager that ensure the background is restored.
87 """
88 bg = self.background.fitBackground(maskedImage)
89 bgImage = bg.getImageF(self.background.config.algorithm, self.background.config.undersampleStyle)
90 maskedImage -= bgImage
91 try:
92 yield
93 finally:
94 maskedImage += bgImage
95
96 def run(self, maskedImage):
97 """Rescale the variance in a maskedImage in place.
98
99 Parameters
100 ----------
101 maskedImage : `lsst.afw.image.MaskedImage`
102 Image for which to determine the variance rescaling factor. The image
103 is modified in place.
104
105 Returns
106 -------
107 factor : `float`
108 Variance rescaling factor.
109
110 Raises
111 ------
112 RuntimeError
113 If the estimated variance rescaling factor by both methods exceed the
114 configured limit.
115
116 Notes
117 -----
118 The task calculates and applies the pixel-based correction unless
119 it is over the ``config.limit`` threshold. In this case, the image-based
120 method is applied.
121 """
122 with self.subtractedBackground(maskedImage):
123 factor = self.pixelBased(maskedImage)
124 if factor > self.config.limit:
125 self.log.warning("Pixel-based variance rescaling factor (%f) exceeds configured limit (%f); "
126 "trying image-based method", factor, self.config.limit)
127 factor = self.imageBased(maskedImage)
128 if factor > self.config.limit:
129 raise RuntimeError("Variance rescaling factor (%f) exceeds configured limit (%f)" %
130 (factor, self.config.limit))
131 self.log.info("Renormalizing variance by %f", factor)
132 maskedImage.variance *= factor
133 return factor
134
135 def computeScaleFactors(self, maskedImage):
136 """Calculate and return both variance scaling factors without modifying the image.
137
138 Parameters
139 ----------
140 maskedImage : `lsst.afw.image.MaskedImage`
141 Image for which to determine the variance rescaling factor.
142
143 Returns
144 -------
145 R : `lsst.pipe.base.Struct`
146 - ``pixelFactor`` : `float` The pixel based variance rescaling factor
147 or 1 if all pixels are masked or invalid.
148 - ``imageFactor`` : `float` The image based variance rescaling factor
149 or 1 if all pixels are masked or invalid.
150 """
151 with self.subtractedBackground(maskedImage):
152 pixelFactor = self.pixelBased(maskedImage)
153 imageFactor = self.imageBased(maskedImage)
154 return Struct(pixelFactor=pixelFactor, imageFactor=imageFactor)
155
156 def pixelBased(self, maskedImage):
157 """Determine the variance rescaling factor from pixel statistics
158
159 We calculate SNR = image/sqrt(variance), and the distribution
160 for most of the background-subtracted image should have a standard
161 deviation of unity. We use the interquartile range as a robust estimator
162 of the SNR standard deviation. The variance rescaling factor is the
163 factor that brings that distribution to have unit standard deviation.
164
165 This may not work well if the image has a lot of structure in it, as
166 the assumptions are violated. In that case, use an alternate
167 method.
168
169 Parameters
170 ----------
171 maskedImage : `lsst.afw.image.MaskedImage`
172 Image for which to determine the variance rescaling factor.
173
174 Returns
175 -------
176 factor : `float`
177 Variance rescaling factor or 1 if all pixels are masked or non-finite.
178
179 """
180 maskVal = maskedImage.mask.getPlaneBitMask(self.config.maskPlanes)
181 isGood = (((maskedImage.mask.array & maskVal) == 0)
182 & np.isfinite(maskedImage.image.array)
183 & np.isfinite(maskedImage.variance.array)
184 & (maskedImage.variance.array > 0))
185
186 nGood = np.sum(isGood)
187 self.log.debug("Number of selected background pixels: %d of %d.", nGood, isGood.size)
188 if nGood < 2:
189 # Not enough good data, np.percentile needs at least 2 points
190 # to estimate a range
191 return 1.0
192 # Robust measurement of stdev using inter-quartile range
193 snr = maskedImage.image.array[isGood]/np.sqrt(maskedImage.variance.array[isGood])
194 q1, q3 = np.percentile(snr, (25, 75))
195 stdev = 0.74*(q3 - q1)
196 return stdev**2
197
198 def imageBased(self, maskedImage):
199 """Determine the variance rescaling factor from image statistics
200
201 We calculate average(SNR) = stdev(image)/median(variance), and
202 the value should be unity. We use the interquartile range as a robust
203 estimator of the stdev. The variance rescaling factor is the
204 factor that brings this value to unity.
205
206 This may not work well if the pixels from which we measure the
207 standard deviation of the image are not effectively the same pixels
208 from which we measure the median of the variance. In that case, use
209 an alternate method.
210
211 Parameters
212 ----------
213 maskedImage : `lsst.afw.image.MaskedImage`
214 Image for which to determine the variance rescaling factor.
215
216 Returns
217 -------
218 factor : `float`
219 Variance rescaling factor or 1 if all pixels are masked or non-finite.
220 """
221 maskVal = maskedImage.mask.getPlaneBitMask(self.config.maskPlanes)
222 isGood = (((maskedImage.mask.array & maskVal) == 0)
223 & np.isfinite(maskedImage.image.array)
224 & np.isfinite(maskedImage.variance.array)
225 & (maskedImage.variance.array > 0))
226 nGood = np.sum(isGood)
227 self.log.debug("Number of selected background pixels: %d of %d.", nGood, isGood.size)
228 if nGood < 2:
229 # Not enough good data, np.percentile needs at least 2 points
230 # to estimate a range
231 return 1.0
232 # Robust measurement of stdev
233 q1, q3 = np.percentile(maskedImage.image.array[isGood], (25, 75))
234 ratio = 0.74*(q3 - q1)/np.sqrt(np.median(maskedImage.variance.array[isGood]))
235 return ratio**2