Coverage for python/lsst/pipe/tasks/scaleVariance.py : 33%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1#
2# LSST Data Management System
3# Copyright 2018 AURA/LSST.
4#
5# This product includes software developed by the
6# LSST Project (http://www.lsst.org/).
7#
8# This program is free software: you can redistribute it and/or modify
9# it under the terms of the GNU General Public License as published by
10# the Free Software Foundation, either version 3 of the License, or
11# (at your option) any later version.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the LSST License Statement and
19# the GNU General Public License along with this program. If not,
20# see <http://www.lsstcorp.org/LegalNotices/>.
21#
22from contextlib import contextmanager
23import numpy as np
25from lsst.pex.config import Config, Field, ListField, ConfigurableField
26from lsst.pipe.base import Task
27from lsst.meas.algorithms import SubtractBackgroundTask
29__all__ = ["ScaleVarianceConfig", "ScaleVarianceTask"]
32class ScaleVarianceConfig(Config):
33 background = ConfigurableField(target=SubtractBackgroundTask, doc="Background subtraction")
34 maskPlanes = ListField(
35 dtype=str,
36 default=["DETECTED", "DETECTED_NEGATIVE", "BAD", "SAT", "NO_DATA", "INTRP"],
37 doc="Mask planes for pixels to ignore when scaling variance",
38 )
39 limit = Field(dtype=float, default=10.0, doc="Maximum variance scaling value to permit")
41 def setDefaults(self):
42 self.background.binSize = 32
43 self.background.useApprox = False
44 self.background.undersampleStyle = "REDUCE_INTERP_ORDER"
45 self.background.ignoredPixelMask = ["DETECTED", "DETECTED_NEGATIVE", "BAD", "SAT", "NO_DATA", "INTRP"]
48class ScaleVarianceTask(Task):
49 """Scale the variance in a MaskedImage
51 The variance plane in a convolved or warped image (or a coadd derived
52 from warped images) does not accurately reflect the noise properties of
53 the image because variance has been lost to covariance. This Task
54 attempts to correct for this by scaling the variance plane to match
55 the observed variance in the image. This is not perfect (because we're
56 not tracking the covariance) but it's simple and is often good enough.
57 """
58 ConfigClass = ScaleVarianceConfig
59 _DefaultName = "scaleVariance"
61 def __init__(self, *args, **kwargs):
62 Task.__init__(self, *args, **kwargs)
63 self.makeSubtask("background")
65 @contextmanager
66 def subtractedBackground(self, maskedImage):
67 """Context manager for subtracting the background
69 We need to subtract the background so that the entire image
70 (apart from objects, which should be clipped) will have the
71 image/sqrt(variance) distributed about zero.
73 This context manager subtracts the background, and ensures it
74 is restored on exit.
76 Parameters
77 ----------
78 maskedImage : `lsst.afw.image.MaskedImage`
79 Image+mask+variance to have background subtracted and restored.
81 Returns
82 -------
83 context : context manager
84 Context manager that ensure the background is restored.
85 """
86 bg = self.background.fitBackground(maskedImage)
87 bgImage = bg.getImageF(self.background.config.algorithm, self.background.config.undersampleStyle)
88 maskedImage -= bgImage
89 try:
90 yield
91 finally:
92 maskedImage += bgImage
94 def run(self, maskedImage):
95 """Rescale the variance in a maskedImage
97 Parameters
98 ----------
99 maskedImage : `lsst.afw.image.MaskedImage`
100 Image for which to determine the variance rescaling factor.
102 Returns
103 -------
104 factor : `float`
105 Variance rescaling factor.
107 Raises
108 ------
109 RuntimeError
110 If the estimated variance rescaling factor exceeds the
111 configured limit.
112 """
113 with self.subtractedBackground(maskedImage):
114 factor = self.pixelBased(maskedImage)
115 if np.isnan(factor) or factor > self.config.limit:
116 self.log.warn("Pixel-based variance rescaling factor (%f) exceeds configured limit (%f); "
117 "trying image-based method", factor, self.config.limit)
118 factor = self.imageBased(maskedImage)
119 if np.isnan(factor) or factor > self.config.limit:
120 raise RuntimeError("Variance rescaling factor (%f) exceeds configured limit (%f)" %
121 (factor, self.config.limit))
122 self.log.info("Renormalizing variance by %f" % (factor,))
123 maskedImage.variance *= factor
124 return factor
126 def pixelBased(self, maskedImage):
127 """Determine the variance rescaling factor from pixel statistics
129 We calculate SNR = image/sqrt(variance), and the distribution
130 for most of the background-subtracted image should have a standard
131 deviation of unity. The variance rescaling factor is the factor that
132 brings that distribution to have unit standard deviation.
134 This may not work well if the image has a lot of structure in it, as
135 the assumptions are violated. In that case, use an alternate
136 method.
138 Parameters
139 ----------
140 maskedImage : `lsst.afw.image.MaskedImage`
141 Image for which to determine the variance rescaling factor.
143 Returns
144 -------
145 factor : `float`
146 Variance rescaling factor.
147 """
148 variance = maskedImage.variance
149 snr = maskedImage.image.array/np.sqrt(variance.array)
150 maskVal = maskedImage.mask.getPlaneBitMask(self.config.maskPlanes)
151 isGood = ((maskedImage.mask.array & maskVal) == 0) & (maskedImage.variance.array > 0)
152 # Robust measurement of stdev using inter-quartile range
153 try:
154 q1, q3 = np.percentile(snr[isGood], (25, 75))
155 except IndexError:
156 # This error is raised when all data is nan. Catch error so that it does not
157 # propagate any higher. Set the stdev to one will not fix the bad data, but
158 # it will allow the task to complete. It should be the job of higher level
159 # tasks to handle missing or bad data
160 return 1.0
161 stdev = 0.74*(q3 - q1)
162 return stdev**2
164 def imageBased(self, maskedImage):
165 """Determine the variance rescaling factor from image statistics
167 We calculate average(SNR) = stdev(image)/median(variance), and
168 the value should be unity. The variance rescaling factor is the
169 factor that brings this value to unity.
171 This may not work well if the pixels from which we measure the
172 standard deviation of the image are not effectively the same pixels
173 from which we measure the median of the variance. In that case, use
174 an alternate method.
176 Parameters
177 ----------
178 maskedImage : `lsst.afw.image.MaskedImage`
179 Image for which to determine the variance rescaling factor.
181 Returns
182 -------
183 factor : `float`
184 Variance rescaling factor.
185 """
186 maskVal = maskedImage.mask.getPlaneBitMask(self.config.maskPlanes)
187 isGood = ((maskedImage.mask.array & maskVal) == 0) & (maskedImage.variance.array > 0)
188 # Robust measurement of stdev
189 q1, q3 = np.percentile(maskedImage.image.array[isGood], (25, 75))
190 ratio = 0.74*(q3 - q1)/np.sqrt(np.median(maskedImage.variance.array[isGood]))
191 return ratio**2