Coverage for python/lsst/meas/algorithms/scaleVariance.py: 30%

66 statements  

« prev     ^ index     » next       coverage.py v6.4.2, created at 2022-07-20 03:06 -0700

1# 

2# LSST Data Management System 

3# Copyright 2022 AURA/LSST. 

4# 

5# This product includes software developed by the 

6# LSST Project (http://www.lsst.org/). 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <http://www.lsstcorp.org/LegalNotices/>. 

21# 

22from contextlib import contextmanager 

23import numpy as np 

24 

25from lsst.pex.config import Config, Field, ListField, ConfigurableField 

26from lsst.pipe.base import Task, Struct 

27from . import SubtractBackgroundTask 

28 

29__all__ = ["ScaleVarianceConfig", "ScaleVarianceTask"] 

30 

31 

32class ScaleVarianceConfig(Config): 

33 background = ConfigurableField(target=SubtractBackgroundTask, doc="Background subtraction") 

34 maskPlanes = ListField( 

35 dtype=str, 

36 default=["DETECTED", "DETECTED_NEGATIVE", "BAD", "SAT", "NO_DATA", "INTRP"], 

37 doc="Mask planes for pixels to ignore when scaling variance", 

38 ) 

39 limit = Field(dtype=float, default=10.0, doc="Maximum variance scaling value to permit") 

40 

41 def setDefaults(self): 

42 self.background.binSize = 32 

43 self.background.useApprox = False 

44 self.background.undersampleStyle = "REDUCE_INTERP_ORDER" 

45 self.background.ignoredPixelMask = ["DETECTED", "DETECTED_NEGATIVE", "BAD", "SAT", "NO_DATA", "INTRP"] 

46 

47 

48class ScaleVarianceTask(Task): 

49 """Scale the variance in a MaskedImage 

50 

51 The variance plane in a convolved or warped image (or a coadd derived 

52 from warped images) does not accurately reflect the noise properties of 

53 the image because variance has been lost to covariance. This Task 

54 attempts to correct for this by scaling the variance plane to match 

55 the observed variance in the image. This is not perfect (because we're 

56 not tracking the covariance) but it's simple and is often good enough. 

57 

58 The task implements a pixel-based and an image-based correction estimator. 

59 """ 

60 ConfigClass = ScaleVarianceConfig 

61 _DefaultName = "scaleVariance" 

62 

63 def __init__(self, *args, **kwargs): 

64 Task.__init__(self, *args, **kwargs) 

65 self.makeSubtask("background") 

66 

67 @contextmanager 

68 def subtractedBackground(self, maskedImage): 

69 """Context manager for subtracting the background 

70 

71 We need to subtract the background so that the entire image 

72 (apart from objects, which should be clipped) will have the 

73 image/sqrt(variance) distributed about zero. 

74 

75 This context manager subtracts the background, and ensures it 

76 is restored on exit. 

77 

78 Parameters 

79 ---------- 

80 maskedImage : `lsst.afw.image.MaskedImage` 

81 Image+mask+variance to have background subtracted and restored. 

82 

83 Returns 

84 ------- 

85 context : context manager 

86 Context manager that ensure the background is restored. 

87 """ 

88 bg = self.background.fitBackground(maskedImage) 

89 bgImage = bg.getImageF(self.background.config.algorithm, self.background.config.undersampleStyle) 

90 maskedImage -= bgImage 

91 try: 

92 yield 

93 finally: 

94 maskedImage += bgImage 

95 

96 def run(self, maskedImage): 

97 """Rescale the variance in a maskedImage in place. 

98 

99 Parameters 

100 ---------- 

101 maskedImage : `lsst.afw.image.MaskedImage` 

102 Image for which to determine the variance rescaling factor. The image 

103 is modified in place. 

104 

105 Returns 

106 ------- 

107 factor : `float` 

108 Variance rescaling factor. 

109 

110 Raises 

111 ------ 

112 RuntimeError 

113 If the estimated variance rescaling factor by both methods exceed the 

114 configured limit. 

115 

116 Notes 

117 ----- 

118 The task calculates and applies the pixel-based correction unless 

119 it is over the ``config.limit`` threshold. In this case, the image-based 

120 method is applied. 

121 """ 

122 with self.subtractedBackground(maskedImage): 

123 factor = self.pixelBased(maskedImage) 

124 if factor > self.config.limit: 

125 self.log.warning("Pixel-based variance rescaling factor (%f) exceeds configured limit (%f); " 

126 "trying image-based method", factor, self.config.limit) 

127 factor = self.imageBased(maskedImage) 

128 if factor > self.config.limit: 

129 raise RuntimeError("Variance rescaling factor (%f) exceeds configured limit (%f)" % 

130 (factor, self.config.limit)) 

131 self.log.info("Renormalizing variance by %f", factor) 

132 maskedImage.variance *= factor 

133 return factor 

134 

135 def computeScaleFactors(self, maskedImage): 

136 """Calculate and return both variance scaling factors without modifying the image. 

137 

138 Parameters 

139 ---------- 

140 maskedImage : `lsst.afw.image.MaskedImage` 

141 Image for which to determine the variance rescaling factor. 

142 

143 Returns 

144 ------- 

145 R : `lsst.pipe.base.Struct` 

146 - ``pixelFactor`` : `float` The pixel based variance rescaling factor 

147 or 1 if all pixels are masked or invalid. 

148 - ``imageFactor`` : `float` The image based variance rescaling factor 

149 or 1 if all pixels are masked or invalid. 

150 """ 

151 with self.subtractedBackground(maskedImage): 

152 pixelFactor = self.pixelBased(maskedImage) 

153 imageFactor = self.imageBased(maskedImage) 

154 return Struct(pixelFactor=pixelFactor, imageFactor=imageFactor) 

155 

156 def pixelBased(self, maskedImage): 

157 """Determine the variance rescaling factor from pixel statistics 

158 

159 We calculate SNR = image/sqrt(variance), and the distribution 

160 for most of the background-subtracted image should have a standard 

161 deviation of unity. We use the interquartile range as a robust estimator 

162 of the SNR standard deviation. The variance rescaling factor is the 

163 factor that brings that distribution to have unit standard deviation. 

164 

165 This may not work well if the image has a lot of structure in it, as 

166 the assumptions are violated. In that case, use an alternate 

167 method. 

168 

169 Parameters 

170 ---------- 

171 maskedImage : `lsst.afw.image.MaskedImage` 

172 Image for which to determine the variance rescaling factor. 

173 

174 Returns 

175 ------- 

176 factor : `float` 

177 Variance rescaling factor or 1 if all pixels are masked or non-finite. 

178 

179 """ 

180 maskVal = maskedImage.mask.getPlaneBitMask(self.config.maskPlanes) 

181 isGood = (((maskedImage.mask.array & maskVal) == 0) 

182 & np.isfinite(maskedImage.image.array) 

183 & np.isfinite(maskedImage.variance.array) 

184 & (maskedImage.variance.array > 0)) 

185 

186 nGood = np.sum(isGood) 

187 self.log.debug("Number of selected background pixels: %d of %d.", nGood, isGood.size) 

188 if nGood < 2: 

189 # Not enough good data, np.percentile needs at least 2 points 

190 # to estimate a range 

191 return 1.0 

192 # Robust measurement of stdev using inter-quartile range 

193 snr = maskedImage.image.array[isGood]/np.sqrt(maskedImage.variance.array[isGood]) 

194 q1, q3 = np.percentile(snr, (25, 75)) 

195 stdev = 0.74*(q3 - q1) 

196 return stdev**2 

197 

198 def imageBased(self, maskedImage): 

199 """Determine the variance rescaling factor from image statistics 

200 

201 We calculate average(SNR) = stdev(image)/median(variance), and 

202 the value should be unity. We use the interquartile range as a robust 

203 estimator of the stdev. The variance rescaling factor is the 

204 factor that brings this value to unity. 

205 

206 This may not work well if the pixels from which we measure the 

207 standard deviation of the image are not effectively the same pixels 

208 from which we measure the median of the variance. In that case, use 

209 an alternate method. 

210 

211 Parameters 

212 ---------- 

213 maskedImage : `lsst.afw.image.MaskedImage` 

214 Image for which to determine the variance rescaling factor. 

215 

216 Returns 

217 ------- 

218 factor : `float` 

219 Variance rescaling factor or 1 if all pixels are masked or non-finite. 

220 """ 

221 maskVal = maskedImage.mask.getPlaneBitMask(self.config.maskPlanes) 

222 isGood = (((maskedImage.mask.array & maskVal) == 0) 

223 & np.isfinite(maskedImage.image.array) 

224 & np.isfinite(maskedImage.variance.array) 

225 & (maskedImage.variance.array > 0)) 

226 nGood = np.sum(isGood) 

227 self.log.debug("Number of selected background pixels: %d of %d.", nGood, isGood.size) 

228 if nGood < 2: 

229 # Not enough good data, np.percentile needs at least 2 points 

230 # to estimate a range 

231 return 1.0 

232 # Robust measurement of stdev 

233 q1, q3 = np.percentile(maskedImage.image.array[isGood], (25, 75)) 

234 ratio = 0.74*(q3 - q1)/np.sqrt(np.median(maskedImage.variance.array[isGood])) 

235 return ratio**2