Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

23

24

25

26

27

28

29

30

31

32

33

34

35

36

37

38

39

40

41

42

43

44

45

46

47

48

49

50

51

52

53

54

55

56

57

58

59

60

61

62

63

64

65

66

67

68

69

70

71

72

73

74

75

76

77

78

79

80

81

82

83

84

85

86

87

88

89

90

91

92

93

94

95

96

97

98

99

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

158

159

160

161

162

163

164

165

166

167

168

169

170

171

172

173

174

175

176

177

178

179

180

181

182

183

184

185

186

187

188

189

190

191

# 

# LSST Data Management System 

# Copyright 2018 AURA/LSST. 

# 

# This product includes software developed by the 

# LSST Project (http://www.lsst.org/). 

# 

# This program is free software: you can redistribute it and/or modify 

# it under the terms of the GNU General Public License as published by 

# the Free Software Foundation, either version 3 of the License, or 

# (at your option) any later version. 

# 

# This program is distributed in the hope that it will be useful, 

# but WITHOUT ANY WARRANTY; without even the implied warranty of 

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

# GNU General Public License for more details. 

# 

# You should have received a copy of the LSST License Statement and 

# the GNU General Public License along with this program. If not, 

# see <http://www.lsstcorp.org/LegalNotices/>. 

# 

from contextlib import contextmanager 

import numpy as np 

 

from lsst.pex.config import Config, Field, ListField, ConfigurableField 

from lsst.pipe.base import Task 

from lsst.meas.algorithms import SubtractBackgroundTask 

 

__all__ = ["ScaleVarianceConfig", "ScaleVarianceTask"] 

 

 

class ScaleVarianceConfig(Config): 

background = ConfigurableField(target=SubtractBackgroundTask, doc="Background subtraction") 

maskPlanes = ListField( 

dtype=str, 

default=["DETECTED", "DETECTED_NEGATIVE", "BAD", "SAT", "NO_DATA", "INTRP"], 

doc="Mask planes for pixels to ignore when scaling variance", 

) 

limit = Field(dtype=float, default=10.0, doc="Maximum variance scaling value to permit") 

 

def setDefaults(self): 

self.background.binSize = 32 

self.background.useApprox = False 

self.background.undersampleStyle = "REDUCE_INTERP_ORDER" 

self.background.ignoredPixelMask = ["DETECTED", "DETECTED_NEGATIVE", "BAD", "SAT", "NO_DATA", "INTRP"] 

 

 

class ScaleVarianceTask(Task): 

"""Scale the variance in a MaskedImage 

 

The variance plane in a convolved or warped image (or a coadd derived 

from warped images) does not accurately reflect the noise properties of 

the image because variance has been lost to covariance. This Task 

attempts to correct for this by scaling the variance plane to match 

the observed variance in the image. This is not perfect (because we're 

not tracking the covariance) but it's simple and is often good enough. 

""" 

ConfigClass = ScaleVarianceConfig 

_DefaultName = "scaleVariance" 

 

def __init__(self, *args, **kwargs): 

Task.__init__(self, *args, **kwargs) 

self.makeSubtask("background") 

 

@contextmanager 

def subtractedBackground(self, maskedImage): 

"""Context manager for subtracting the background 

 

We need to subtract the background so that the entire image 

(apart from objects, which should be clipped) will have the 

image/sqrt(variance) distributed about zero. 

 

This context manager subtracts the background, and ensures it 

is restored on exit. 

 

Parameters 

---------- 

maskedImage : `lsst.afw.image.MaskedImage` 

Image+mask+variance to have background subtracted and restored. 

 

Returns 

------- 

context : context manager 

Context manager that ensure the background is restored. 

""" 

bg = self.background.fitBackground(maskedImage) 

bgImage = bg.getImageF() 

maskedImage -= bgImage 

try: 

yield 

finally: 

maskedImage += bgImage 

 

def run(self, maskedImage): 

"""Rescale the variance in a maskedImage 

 

Parameters 

---------- 

maskedImage : `lsst.afw.image.MaskedImage` 

Image for which to determine the variance rescaling factor. 

 

Returns 

------- 

factor : `float` 

Variance rescaling factor. 

 

Raises 

------ 

RuntimeError 

If the estimated variance rescaling factor exceeds the 

configured limit. 

""" 

with self.subtractedBackground(maskedImage): 

factor = self.pixelBased(maskedImage) 

if np.isnan(factor) or factor > self.config.limit: 

self.log.warn("Pixel-based variance rescaling factor (%f) exceeds configured limit (%f); " 

"trying image-based method", factor, self.config.limit) 

factor = self.imageBased(maskedImage) 

if np.isnan(factor) or factor > self.config.limit: 

raise RuntimeError("Variance rescaling factor (%f) exceeds configured limit (%f)" % 

(factor, self.config.limit)) 

self.log.info("Renormalizing variance by %f" % (factor,)) 

maskedImage.variance *= factor 

return factor 

 

def pixelBased(self, maskedImage): 

"""Determine the variance rescaling factor from pixel statistics 

 

We calculate SNR = image/sqrt(variance), and the distribution 

for most of the background-subtracted image should have a standard 

deviation of unity. The variance rescaling factor is the factor that 

brings that distribution to have unit standard deviation. 

 

This may not work well if the image has a lot of structure in it, as 

the assumptions are violated. In that case, use an alternate 

method. 

 

Parameters 

---------- 

maskedImage : `lsst.afw.image.MaskedImage` 

Image for which to determine the variance rescaling factor. 

 

Returns 

------- 

factor : `float` 

Variance rescaling factor. 

""" 

variance = maskedImage.variance 

snr = maskedImage.image.array/np.sqrt(variance.array) 

maskVal = maskedImage.mask.getPlaneBitMask(self.config.maskPlanes) 

isGood = ((maskedImage.mask.array & maskVal) == 0) & (maskedImage.variance.array > 0) 

# Robust measurement of stdev using inter-quartile range 

try: 

q1, q3 = np.percentile(snr[isGood], (25, 75)) 

except IndexError: 

# This error is raised when all data is nan. Catch error so that it does not 

# propagate any higher. Set the stdev to one will not fix the bad data, but 

# it will allow the task to complete. It should be the job of higher level 

# tasks to handle missing or bad data 

return 1.0 

stdev = 0.74*(q3 - q1) 

return stdev**2 

 

def imageBased(self, maskedImage): 

"""Determine the variance rescaling factor from image statistics 

 

We calculate average(SNR) = stdev(image)/median(variance), and 

the value should be unity. The variance rescaling factor is the 

factor that brings this value to unity. 

 

This may not work well if the pixels from which we measure the 

standard deviation of the image are not effectively the same pixels 

from which we measure the median of the variance. In that case, use 

an alternate method. 

 

Parameters 

---------- 

maskedImage : `lsst.afw.image.MaskedImage` 

Image for which to determine the variance rescaling factor. 

 

Returns 

------- 

factor : `float` 

Variance rescaling factor. 

""" 

maskVal = maskedImage.mask.getPlaneBitMask(self.config.maskPlanes) 

isGood = ((maskedImage.mask.array & maskVal) == 0) & (maskedImage.variance.array > 0) 

# Robust measurement of stdev 

q1, q3 = np.percentile(maskedImage.image.array[isGood], (25, 75)) 

ratio = 0.74*(q3 - q1)/np.sqrt(np.median(maskedImage.variance.array[isGood])) 

return ratio**2