Coverage for python/lsst/cp/verify/verifyDark.py: 10%

65 statements  

« prev     ^ index     » next       coverage.py v7.5.1, created at 2024-05-10 11:36 +0000

1# This file is part of cp_verify. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21import numpy as np 

22 

23from .verifyStats import CpVerifyStatsConfig, CpVerifyStatsTask, CpVerifyStatsConnections 

24 

25 

26__all__ = ['CpVerifyDarkConfig', 'CpVerifyDarkTask'] 

27 

28 

29class CpVerifyDarkConfig(CpVerifyStatsConfig, 

30 pipelineConnections=CpVerifyStatsConnections): 

31 """Inherits from base CpVerifyStatsConfig. 

32 """ 

33 

34 def setDefaults(self): 

35 super().setDefaults() 

36 self.stageName = 'DARK' 

37 self.imageStatKeywords = {'MEAN': 'MEAN', # noqa F841 

38 'NOISE': 'STDEVCLIP', } 

39 self.crImageStatKeywords = {'CR_NOISE': 'STDEV', } # noqa F841 

40 self.metadataStatKeywords = {'RESIDUAL STDEV': 'AMP', } # noqa F841 

41 

42 

43class CpVerifyDarkTask(CpVerifyStatsTask): 

44 """Dark verification sub-class, implementing the verify method. 

45 """ 

46 ConfigClass = CpVerifyDarkConfig 

47 _DefaultName = 'cpVerifyDark' 

48 

49 def verify(self, exposure, statisticsDict): 

50 """Verify that the measured statistics meet the verification criteria. 

51 

52 Parameters 

53 ---------- 

54 exposure : `lsst.afw.image.Exposure` 

55 The exposure the statistics are from. 

56 statisticsDictionary : `dict` [`str`, `dict` [`str`, scalar]], 

57 Dictionary of measured statistics. The inner dictionary 

58 should have keys that are statistic names (`str`) with 

59 values that are some sort of scalar (`int` or `float` are 

60 the mostly likely types). 

61 

62 Returns 

63 ------- 

64 outputStatistics : `dict` [`str`, `dict` [`str`, `bool`]] 

65 A dictionary indexed by the amplifier name, containing 

66 dictionaries of the verification criteria. 

67 success : `bool` 

68 A boolean indicating if all tests have passed. 

69 """ 

70 detector = exposure.getDetector() 

71 ampStats = statisticsDict['AMP'] 

72 metadataStats = statisticsDict['METADATA'] 

73 

74 verifyStats = {} 

75 success = True 

76 for ampName, stats in ampStats.items(): 

77 verify = {} 

78 

79 # DMTN-101 Test 5.2: Mean is 0.0: 

80 verify['MEAN'] = bool(np.abs(stats['MEAN']) < stats['NOISE']) 

81 

82 # DMTN-101 Test 5.3: Clipped mean matches readNoise. This 

83 # test should use the nominal detector read noise. The 

84 # f"RESIDUAL STDEV {ampName}" metadata entry contains the 

85 # measured dispersion in the overscan-corrected overscan 

86 # region, which should provide an estimate of the read 

87 # noise. However, directly using this value will cause 

88 # some fraction of verification runs to fail if the 

89 # scatter in read noise values is comparable to the test 

90 # threshold, as the overscan residual measured may be 

91 # sampling from the low end tail of the distribution. 

92 # This measurement is also likely to be smaller than that 

93 # measured on the bulk of the image as the overscan 

94 # correction should be an optimal fit to the overscan 

95 # region, but not necessarily for the image region. 

96 readNoise = detector[ampName].getReadNoise() 

97 verify['NOISE'] = bool((stats['NOISE'] - readNoise)/readNoise <= 0.05) 

98 

99 # DMTN-101 Test 5.4: CR rejection matches clipped mean 

100 verify['CR_NOISE'] = bool(np.abs(stats['NOISE'] - stats['CR_NOISE'])/stats['CR_NOISE'] <= 0.05) 

101 

102 # Confirm this hasn't triggered a raise condition. 

103 if 'FORCE_FAILURE' in stats: 

104 verify['PROCESSING'] = False 

105 

106 verify['SUCCESS'] = bool(np.all(list(verify.values()))) 

107 if verify['SUCCESS'] is False: 

108 success = False 

109 

110 # After determining the verification status for this 

111 # exposure, we can also check to see how well the read 

112 # noise measured from the overscan residual matches the 

113 # nominal value used above in Test 5.3. If these disagree 

114 # consistently and significantly, then the assumptions 

115 # used in that test may be incorrect, and the nominal read 

116 # noise may need recalculation. Only perform this check 

117 # if the metadataStats contain the required entry. 

118 if 'RESIDUAL STDEV' in metadataStats and ampName in metadataStats['RESIDUAL STDEV']: 

119 verify['READ_NOISE_CONSISTENT'] = True 

120 overscanReadNoise = metadataStats['RESIDUAL STDEV'][ampName] 

121 if overscanReadNoise: 

122 if ((overscanReadNoise - readNoise)/readNoise > 0.05): 

123 verify['READ_NOISE_CONSISTENT'] = False 

124 

125 verifyStats[ampName] = verify 

126 

127 return {'AMP': verifyStats}, bool(success) 

128 

129 def repackStats(self, statisticsDict, dimensions): 

130 # docstring inherited 

131 rows = {} 

132 rowList = [] 

133 matrixRowList = None 

134 

135 if self.config.useIsrStatistics: 

136 mjd = statisticsDict["ISR"]["MJD"] 

137 else: 

138 mjd = np.nan 

139 

140 rowBase = { 

141 "instrument": dimensions["instrument"], 

142 "exposure": dimensions["exposure"], 

143 "detector": dimensions["detector"], 

144 "mjd": mjd, 

145 } 

146 

147 # AMP results: 

148 for ampName, stats in statisticsDict["AMP"].items(): 

149 rows[ampName] = {} 

150 rows[ampName].update(rowBase) 

151 rows[ampName]["amplifier"] = ampName 

152 for key, value in stats.items(): 

153 rows[ampName][f"{self.config.stageName}_{key}"] = value 

154 

155 # VERIFY results 

156 for ampName, stats in statisticsDict["VERIFY"]["AMP"].items(): 

157 for key, value in stats.items(): 

158 rows[ampName][f"{self.config.stageName}_VERIFY_{key}"] = value 

159 

160 # METADATA results 

161 for ampName, value in statisticsDict["METADATA"]["RESIDUAL STDEV"].items(): 

162 rows[ampName][f"{self.config.stageName}_READ_NOISE"] = value 

163 

164 # ISR results 

165 if self.config.useIsrStatistics and "ISR" in statisticsDict: 

166 for ampName, stats in statisticsDict["ISR"]["CALIBDIST"].items(): 

167 for level in self.config.expectedDistributionLevels: 

168 key = f"LSST CALIB {self.config.stageName.upper()} {ampName} DISTRIBUTION {level}-PCT" 

169 rows[ampName][f"{self.config.stageName}_DARK_DIST_{level}_PCT"] = stats[key] 

170 

171 # pack final list 

172 for ampName, stats in rows.items(): 

173 rowList.append(stats) 

174 

175 return rowList, matrixRowList