Coverage for python/lsst/meas/algorithms/scaleVariance.py: 29%

66 statements  

« prev     ^ index     » next       coverage.py v7.4.4, created at 2024-03-22 02:38 -0700

1# 

2# LSST Data Management System 

3# Copyright 2022 AURA/LSST. 

4# 

5# This product includes software developed by the 

6# LSST Project (http://www.lsst.org/). 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <http://www.lsstcorp.org/LegalNotices/>. 

21# 

22from contextlib import contextmanager 

23import numpy as np 

24 

25from lsst.pex.config import Config, Field, ListField, ConfigurableField 

26from lsst.pipe.base import Task, Struct 

27from . import SubtractBackgroundTask 

28 

29__all__ = ["ScaleVarianceConfig", "ScaleVarianceTask"] 

30 

31 

32class ScaleVarianceConfig(Config): 

33 background = ConfigurableField(target=SubtractBackgroundTask, doc="Background subtraction") 

34 maskPlanes = ListField[str]( 

35 default=["DETECTED", "DETECTED_NEGATIVE", "BAD", "SAT", "NO_DATA", "INTRP"], 

36 doc="Mask planes for pixels to ignore when scaling variance", 

37 ) 

38 limit = Field[float](default=10.0, doc="Maximum variance scaling value to permit") 

39 

40 def setDefaults(self): 

41 self.background.binSize = 32 

42 self.background.useApprox = False 

43 self.background.undersampleStyle = "REDUCE_INTERP_ORDER" 

44 self.background.ignoredPixelMask = ["DETECTED", "DETECTED_NEGATIVE", "BAD", "SAT", "NO_DATA", "INTRP"] 

45 

46 

47class ScaleVarianceTask(Task): 

48 """Scale the variance in a MaskedImage 

49 

50 The variance plane in a convolved or warped image (or a coadd derived 

51 from warped images) does not accurately reflect the noise properties of 

52 the image because variance has been lost to covariance. This Task 

53 attempts to correct for this by scaling the variance plane to match 

54 the observed variance in the image. This is not perfect (because we're 

55 not tracking the covariance) but it's simple and is often good enough. 

56 

57 The task implements a pixel-based and an image-based correction estimator. 

58 """ 

59 ConfigClass = ScaleVarianceConfig 

60 _DefaultName = "scaleVariance" 

61 

62 def __init__(self, *args, **kwargs): 

63 Task.__init__(self, *args, **kwargs) 

64 self.makeSubtask("background") 

65 

66 @contextmanager 

67 def subtractedBackground(self, maskedImage): 

68 """Context manager for subtracting the background 

69 

70 We need to subtract the background so that the entire image 

71 (apart from objects, which should be clipped) will have the 

72 image/sqrt(variance) distributed about zero. 

73 

74 This context manager subtracts the background, and ensures it 

75 is restored on exit. 

76 

77 Parameters 

78 ---------- 

79 maskedImage : `lsst.afw.image.MaskedImage` 

80 Image+mask+variance to have background subtracted and restored. 

81 

82 Returns 

83 ------- 

84 context : context manager 

85 Context manager that ensure the background is restored. 

86 """ 

87 bg = self.background.fitBackground(maskedImage) 

88 bgImage = bg.getImageF(self.background.config.algorithm, self.background.config.undersampleStyle) 

89 maskedImage -= bgImage 

90 try: 

91 yield 

92 finally: 

93 maskedImage += bgImage 

94 

95 def run(self, maskedImage): 

96 """Rescale the variance in a maskedImage in place. 

97 

98 Parameters 

99 ---------- 

100 maskedImage : `lsst.afw.image.MaskedImage` 

101 Image for which to determine the variance rescaling factor. The image 

102 is modified in place. 

103 

104 Returns 

105 ------- 

106 factor : `float` 

107 Variance rescaling factor. 

108 

109 Raises 

110 ------ 

111 RuntimeError 

112 If the estimated variance rescaling factor by both methods exceed the 

113 configured limit. 

114 

115 Notes 

116 ----- 

117 The task calculates and applies the pixel-based correction unless 

118 it is over the ``config.limit`` threshold. In this case, the image-based 

119 method is applied. 

120 """ 

121 with self.subtractedBackground(maskedImage): 

122 factor = self.pixelBased(maskedImage) 

123 if factor > self.config.limit: 

124 self.log.warning("Pixel-based variance rescaling factor (%f) exceeds configured limit (%f); " 

125 "trying image-based method", factor, self.config.limit) 

126 factor = self.imageBased(maskedImage) 

127 if factor > self.config.limit: 

128 raise RuntimeError("Variance rescaling factor (%f) exceeds configured limit (%f)" % 

129 (factor, self.config.limit)) 

130 self.log.info("Renormalizing variance by %f", factor) 

131 maskedImage.variance *= factor 

132 return factor 

133 

134 def computeScaleFactors(self, maskedImage): 

135 """Calculate and return both variance scaling factors without modifying the image. 

136 

137 Parameters 

138 ---------- 

139 maskedImage : `lsst.afw.image.MaskedImage` 

140 Image for which to determine the variance rescaling factor. 

141 

142 Returns 

143 ------- 

144 R : `lsst.pipe.base.Struct` 

145 - ``pixelFactor`` : `float` The pixel based variance rescaling factor 

146 or 1 if all pixels are masked or invalid. 

147 - ``imageFactor`` : `float` The image based variance rescaling factor 

148 or 1 if all pixels are masked or invalid. 

149 """ 

150 with self.subtractedBackground(maskedImage): 

151 pixelFactor = self.pixelBased(maskedImage) 

152 imageFactor = self.imageBased(maskedImage) 

153 return Struct(pixelFactor=pixelFactor, imageFactor=imageFactor) 

154 

155 def pixelBased(self, maskedImage): 

156 """Determine the variance rescaling factor from pixel statistics 

157 

158 We calculate SNR = image/sqrt(variance), and the distribution 

159 for most of the background-subtracted image should have a standard 

160 deviation of unity. We use the interquartile range as a robust estimator 

161 of the SNR standard deviation. The variance rescaling factor is the 

162 factor that brings that distribution to have unit standard deviation. 

163 

164 This may not work well if the image has a lot of structure in it, as 

165 the assumptions are violated. In that case, use an alternate 

166 method. 

167 

168 Parameters 

169 ---------- 

170 maskedImage : `lsst.afw.image.MaskedImage` 

171 Image for which to determine the variance rescaling factor. 

172 

173 Returns 

174 ------- 

175 factor : `float` 

176 Variance rescaling factor or 1 if all pixels are masked or non-finite. 

177 

178 """ 

179 maskVal = maskedImage.mask.getPlaneBitMask(self.config.maskPlanes) 

180 isGood = (((maskedImage.mask.array & maskVal) == 0) 

181 & np.isfinite(maskedImage.image.array) 

182 & np.isfinite(maskedImage.variance.array) 

183 & (maskedImage.variance.array > 0)) 

184 

185 nGood = np.sum(isGood) 

186 self.log.debug("Number of selected background pixels: %d of %d.", nGood, isGood.size) 

187 if nGood < 2: 

188 # Not enough good data, np.percentile needs at least 2 points 

189 # to estimate a range 

190 return 1.0 

191 # Robust measurement of stdev using inter-quartile range 

192 snr = maskedImage.image.array[isGood]/np.sqrt(maskedImage.variance.array[isGood]) 

193 q1, q3 = np.percentile(snr, (25, 75)) 

194 stdev = 0.74*(q3 - q1) 

195 return stdev**2 

196 

197 def imageBased(self, maskedImage): 

198 """Determine the variance rescaling factor from image statistics 

199 

200 We calculate average(SNR) = stdev(image)/median(variance), and 

201 the value should be unity. We use the interquartile range as a robust 

202 estimator of the stdev. The variance rescaling factor is the 

203 factor that brings this value to unity. 

204 

205 This may not work well if the pixels from which we measure the 

206 standard deviation of the image are not effectively the same pixels 

207 from which we measure the median of the variance. In that case, use 

208 an alternate method. 

209 

210 Parameters 

211 ---------- 

212 maskedImage : `lsst.afw.image.MaskedImage` 

213 Image for which to determine the variance rescaling factor. 

214 

215 Returns 

216 ------- 

217 factor : `float` 

218 Variance rescaling factor or 1 if all pixels are masked or non-finite. 

219 """ 

220 maskVal = maskedImage.mask.getPlaneBitMask(self.config.maskPlanes) 

221 isGood = (((maskedImage.mask.array & maskVal) == 0) 

222 & np.isfinite(maskedImage.image.array) 

223 & np.isfinite(maskedImage.variance.array) 

224 & (maskedImage.variance.array > 0)) 

225 nGood = np.sum(isGood) 

226 self.log.debug("Number of selected background pixels: %d of %d.", nGood, isGood.size) 

227 if nGood < 2: 

228 # Not enough good data, np.percentile needs at least 2 points 

229 # to estimate a range 

230 return 1.0 

231 # Robust measurement of stdev 

232 q1, q3 = np.percentile(maskedImage.image.array[isGood], (25, 75)) 

233 ratio = 0.74*(q3 - q1)/np.sqrt(np.median(maskedImage.variance.array[isGood])) 

234 return ratio**2