Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# 

2# LSST Data Management System 

3# Copyright 2018 AURA/LSST. 

4# 

5# This product includes software developed by the 

6# LSST Project (http://www.lsst.org/). 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <http://www.lsstcorp.org/LegalNotices/>. 

21# 

22from contextlib import contextmanager 

23import numpy as np 

24 

25from lsst.pex.config import Config, Field, ListField, ConfigurableField 

26from lsst.pipe.base import Task 

27from lsst.meas.algorithms import SubtractBackgroundTask 

28 

29__all__ = ["ScaleVarianceConfig", "ScaleVarianceTask"] 

30 

31 

32class ScaleVarianceConfig(Config): 

33 background = ConfigurableField(target=SubtractBackgroundTask, doc="Background subtraction") 

34 maskPlanes = ListField( 

35 dtype=str, 

36 default=["DETECTED", "DETECTED_NEGATIVE", "BAD", "SAT", "NO_DATA", "INTRP"], 

37 doc="Mask planes for pixels to ignore when scaling variance", 

38 ) 

39 limit = Field(dtype=float, default=10.0, doc="Maximum variance scaling value to permit") 

40 

41 def setDefaults(self): 

42 self.background.binSize = 32 

43 self.background.useApprox = False 

44 self.background.undersampleStyle = "REDUCE_INTERP_ORDER" 

45 self.background.ignoredPixelMask = ["DETECTED", "DETECTED_NEGATIVE", "BAD", "SAT", "NO_DATA", "INTRP"] 

46 

47 

48class ScaleVarianceTask(Task): 

49 """Scale the variance in a MaskedImage 

50 

51 The variance plane in a convolved or warped image (or a coadd derived 

52 from warped images) does not accurately reflect the noise properties of 

53 the image because variance has been lost to covariance. This Task 

54 attempts to correct for this by scaling the variance plane to match 

55 the observed variance in the image. This is not perfect (because we're 

56 not tracking the covariance) but it's simple and is often good enough. 

57 """ 

58 ConfigClass = ScaleVarianceConfig 

59 _DefaultName = "scaleVariance" 

60 

61 def __init__(self, *args, **kwargs): 

62 Task.__init__(self, *args, **kwargs) 

63 self.makeSubtask("background") 

64 

65 @contextmanager 

66 def subtractedBackground(self, maskedImage): 

67 """Context manager for subtracting the background 

68 

69 We need to subtract the background so that the entire image 

70 (apart from objects, which should be clipped) will have the 

71 image/sqrt(variance) distributed about zero. 

72 

73 This context manager subtracts the background, and ensures it 

74 is restored on exit. 

75 

76 Parameters 

77 ---------- 

78 maskedImage : `lsst.afw.image.MaskedImage` 

79 Image+mask+variance to have background subtracted and restored. 

80 

81 Returns 

82 ------- 

83 context : context manager 

84 Context manager that ensure the background is restored. 

85 """ 

86 bg = self.background.fitBackground(maskedImage) 

87 bgImage = bg.getImageF(self.background.config.algorithm, self.background.config.undersampleStyle) 

88 maskedImage -= bgImage 

89 try: 

90 yield 

91 finally: 

92 maskedImage += bgImage 

93 

94 def run(self, maskedImage): 

95 """Rescale the variance in a maskedImage 

96 

97 Parameters 

98 ---------- 

99 maskedImage : `lsst.afw.image.MaskedImage` 

100 Image for which to determine the variance rescaling factor. 

101 

102 Returns 

103 ------- 

104 factor : `float` 

105 Variance rescaling factor. 

106 

107 Raises 

108 ------ 

109 RuntimeError 

110 If the estimated variance rescaling factor exceeds the 

111 configured limit. 

112 """ 

113 with self.subtractedBackground(maskedImage): 

114 factor = self.pixelBased(maskedImage) 

115 if np.isnan(factor) or factor > self.config.limit: 

116 self.log.warn("Pixel-based variance rescaling factor (%f) exceeds configured limit (%f); " 

117 "trying image-based method", factor, self.config.limit) 

118 factor = self.imageBased(maskedImage) 

119 if np.isnan(factor) or factor > self.config.limit: 

120 raise RuntimeError("Variance rescaling factor (%f) exceeds configured limit (%f)" % 

121 (factor, self.config.limit)) 

122 self.log.info("Renormalizing variance by %f" % (factor,)) 

123 maskedImage.variance *= factor 

124 return factor 

125 

126 def pixelBased(self, maskedImage): 

127 """Determine the variance rescaling factor from pixel statistics 

128 

129 We calculate SNR = image/sqrt(variance), and the distribution 

130 for most of the background-subtracted image should have a standard 

131 deviation of unity. The variance rescaling factor is the factor that 

132 brings that distribution to have unit standard deviation. 

133 

134 This may not work well if the image has a lot of structure in it, as 

135 the assumptions are violated. In that case, use an alternate 

136 method. 

137 

138 Parameters 

139 ---------- 

140 maskedImage : `lsst.afw.image.MaskedImage` 

141 Image for which to determine the variance rescaling factor. 

142 

143 Returns 

144 ------- 

145 factor : `float` 

146 Variance rescaling factor. 

147 """ 

148 variance = maskedImage.variance 

149 snr = maskedImage.image.array/np.sqrt(variance.array) 

150 maskVal = maskedImage.mask.getPlaneBitMask(self.config.maskPlanes) 

151 isGood = ((maskedImage.mask.array & maskVal) == 0) & (maskedImage.variance.array > 0) 

152 # Robust measurement of stdev using inter-quartile range 

153 try: 

154 q1, q3 = np.percentile(snr[isGood], (25, 75)) 

155 except IndexError: 

156 # This error is raised when all data is nan. Catch error so that it does not 

157 # propagate any higher. Set the stdev to one will not fix the bad data, but 

158 # it will allow the task to complete. It should be the job of higher level 

159 # tasks to handle missing or bad data 

160 return 1.0 

161 stdev = 0.74*(q3 - q1) 

162 return stdev**2 

163 

164 def imageBased(self, maskedImage): 

165 """Determine the variance rescaling factor from image statistics 

166 

167 We calculate average(SNR) = stdev(image)/median(variance), and 

168 the value should be unity. The variance rescaling factor is the 

169 factor that brings this value to unity. 

170 

171 This may not work well if the pixels from which we measure the 

172 standard deviation of the image are not effectively the same pixels 

173 from which we measure the median of the variance. In that case, use 

174 an alternate method. 

175 

176 Parameters 

177 ---------- 

178 maskedImage : `lsst.afw.image.MaskedImage` 

179 Image for which to determine the variance rescaling factor. 

180 

181 Returns 

182 ------- 

183 factor : `float` 

184 Variance rescaling factor. 

185 """ 

186 maskVal = maskedImage.mask.getPlaneBitMask(self.config.maskPlanes) 

187 isGood = ((maskedImage.mask.array & maskVal) == 0) & (maskedImage.variance.array > 0) 

188 # Robust measurement of stdev 

189 q1, q3 = np.percentile(maskedImage.image.array[isGood], (25, 75)) 

190 ratio = 0.74*(q3 - q1)/np.sqrt(np.median(maskedImage.variance.array[isGood])) 

191 return ratio**2