Coverage for python/lsst/cp/verify/verifyDefects.py: 18%

113 statements  

« prev     ^ index     » next       coverage.py v7.5.1, created at 2024-05-10 03:51 -0700

1# This file is part of cp_verify. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21import numpy as np 

22import scipy.stats 

23import lsst.pipe.base.connectionTypes as cT 

24from lsst.ip.isr.isrFunctions import countMaskedPixels 

25 

26from .verifyStats import ( 

27 CpVerifyStatsConfig, 

28 CpVerifyStatsTask, 

29 CpVerifyStatsConnections, 

30) 

31 

32 

33__all__ = ["CpVerifyDefectsConfig", "CpVerifyDefectsTask"] 

34 

35 

36class CpVerifyDefectsConnections( 

37 CpVerifyStatsConnections, dimensions={"instrument", "visit", "detector"} 

38): 

39 inputExp = cT.Input( 

40 name="icExp", 

41 doc="Input exposure to calculate statistics for.", 

42 storageClass="ExposureF", 

43 dimensions=["instrument", "visit", "detector"], 

44 ) 

45 uncorrectedExp = cT.Input( 

46 name="uncorrectedExp", 

47 doc="Uncorrected input exposure to calculate statistics for.", 

48 storageClass="ExposureF", 

49 dimensions=["instrument", "visit", "detector"], 

50 ) 

51 inputCatalog = cT.Input( 

52 name="icSrc", 

53 doc="Input catalog to calculate statistics from.", 

54 storageClass="SourceCatalog", 

55 dimensions=["instrument", "visit", "detector"], 

56 ) 

57 uncorrectedCatalog = cT.Input( 

58 name="uncorrectedSrc", 

59 doc="Input catalog without correction applied.", 

60 storageClass="SourceCatalog", 

61 dimensions=["instrument", "visit", "detector"], 

62 ) 

63 camera = cT.PrerequisiteInput( 

64 name="camera", 

65 storageClass="Camera", 

66 doc="Input camera.", 

67 dimensions=["instrument", ], 

68 isCalibration=True, 

69 ) 

70 outputStats = cT.Output( 

71 name="detectorStats", 

72 doc="Output statistics from cp_verify.", 

73 storageClass="StructuredDataDict", 

74 dimensions=["instrument", "visit", "detector"], 

75 ) 

76 outputResults = cT.Output( 

77 name="detectorResults", 

78 doc="Output results from cp_verify.", 

79 storageClass="ArrowAstropy", 

80 dimensions=["instrument", "visit", "detector"], 

81 ) 

82 outputMatrix = cT.Output( 

83 name="detectorMatrix", 

84 doc="Output matrix results from cp_verify.", 

85 storageClass="ArrowAstropy", 

86 dimensions=["instrument", "visit", "detector"], 

87 ) 

88 

89 

90class CpVerifyDefectsConfig( 

91 CpVerifyStatsConfig, pipelineConnections=CpVerifyDefectsConnections 

92): 

93 """Inherits from base CpVerifyStatsConfig.""" 

94 

95 def setDefaults(self): 

96 super().setDefaults() 

97 self.stageName = 'DEFECTS' 

98 

99 self.maskNameList = ["BAD"] # noqa F821 

100 

101 self.imageStatKeywords = { 

102 "DEFECT_PIXELS": "NMASKED", # noqa F821 

103 "OUTLIERS": "NCLIPPED", 

104 "MEDIAN": "MEDIAN", 

105 "STDEV": "STDEVCLIP", 

106 "MIN": "MIN", 

107 "MAX": "MAX", 

108 } 

109 self.unmaskedImageStatKeywords = { 

110 "UNMASKED_MIN": "MIN", # noqa F821 

111 "UNMASKED_MAX": "MAX", 

112 "UNMASKED_STDEV": "STDEVCLIP", 

113 "UNMASKED_OUTLIERS": "NCLIPPED", 

114 } 

115 self.uncorrectedImageStatKeywords = { 

116 "UNC_DEFECT_PIXELS": "NMASKED", # noqa F821 

117 "UNC_OUTLIERS": "NCLIPPED", 

118 "UNC_MEDIAN": "MEDIAN", 

119 "UNC_STDEV": "STDEVCLIP", 

120 "UNC_MIN": "MIN", 

121 "UNC_MAX": "MAX", 

122 } 

123 # These config options need to have a key/value pair 

124 # to run verification analysis, but the contents of 

125 # that pair are not used. 

126 self.catalogStatKeywords = {"empty": "dictionary"} 

127 self.detectorStatKeywords = {"empty": "dictionary"} 

128 

129 

130class CpVerifyDefectsTask(CpVerifyStatsTask): 

131 """Defects verification sub-class, implementing the verify method. 

132 

133 This also applies additional image processing statistics. 

134 """ 

135 

136 ConfigClass = CpVerifyDefectsConfig 

137 _DefaultName = "cpVerifyDefects" 

138 

139 def catalogStatistics(self, exposure, catalog, uncorrectedCatalog, statControl): 

140 """Measure the catalog statistics. 

141 

142 Parameters 

143 ---------- 

144 exposure : `lsst.afw.image.Exposure` 

145 The exposure to measure. 

146 catalog : `lsst.afw.table.Table` 

147 The catalog to measure. 

148 uncorrectedCatalog : `lsst.afw.table.Table` 

149 The uncorrected catalog to measure. 

150 statControl : `lsst.afw.math.StatisticsControl` 

151 Statistics control object with parameters defined by 

152 the config. 

153 

154 Returns 

155 ------- 

156 outputStatistics : `dict` [`str`, `dict` [`str`, scalar]] 

157 A dictionary indexed by the amplifier name, containing 

158 dictionaries of the statistics measured and their values. 

159 

160 Notes 

161 ----- 

162 Number of detections test: running with defects would have fewer 

163 detections 

164 """ 

165 outputStatistics = {} 

166 

167 # Number of detections test 

168 outputStatistics["NUM_OBJECTS_BEFORE"] = len(uncorrectedCatalog) 

169 outputStatistics["NUM_OBJECTS_AFTER"] = len(catalog) 

170 

171 return outputStatistics 

172 

173 def detectorStatistics(self, statisticsDict, statControl, exposure=None, uncorrectedExposure=None): 

174 """Measure the detector statistics. 

175 

176 Parameters 

177 ---------- 

178 statisticsDict : `dict` [`str`, scalar] 

179 Dictionary with detector tests. 

180 statControl : `lsst.afw.math.StatControl` 

181 Statistics control object with parameters defined by 

182 the config. 

183 exposure : `lsst.afw.image.Exposure`, optional 

184 Exposure containing the ISR-processed data to measure. 

185 uncorrectedExposure : `lsst.afw.image.Exposure`, optional 

186 uncorrected esposure (no defects) containing the 

187 ISR-processed data to measure. 

188 

189 Returns 

190 ------- 

191 outputStatistics : `dict` [`str`, scalar] 

192 A dictionary containing statistics measured and their values. 

193 

194 Notes 

195 ----- 

196 Number of cosmic rays test: If there are defects in our data that 

197 we didn't properly identify and cover, they might appear similar 

198 to cosmic rays because they have sharp edges compared to the point 

199 spread function (PSF). When we process the data, if these defects 

200 aren't marked in our defect mask, the software might mistakenly think 

201 they are cosmic rays and try to remove them. However, if we've already 

202 included these defects in the defect mask, the software won't treat 

203 them as cosmic rays, so we'll have fewer pixels that are falsely 

204 identified and removed as cosmic rays when we compare two sets of 

205 data reductions. 

206 """ 

207 outputStatistics = {} 

208 # Cosmic Rays test: Count number of cosmic rays before 

209 # and after masking with defects 

210 nCosmicsBefore = countMaskedPixels(uncorrectedExposure, ["CR"]) 

211 nCosmicsAfter = countMaskedPixels(exposure, ["CR"]) 

212 

213 outputStatistics["NUM_COSMICS_BEFORE"] = nCosmicsBefore 

214 outputStatistics["NUM_COSMICS_AFTER"] = nCosmicsAfter 

215 

216 return outputStatistics 

217 

218 def imageStatistics(self, exposure, uncorrectedExposure, statControl): 

219 """Measure additional defect statistics. 

220 

221 This calls the parent class method first, then adds additional 

222 measurements. 

223 

224 Parameters 

225 ---------- 

226 exposure : `lsst.afw.image.Exposure` 

227 Exposure containing the ISR processed data to measure. 

228 statControl : `lsst.afw.math.StatControl` 

229 Statistics control object with parameters defined by 

230 the config. 

231 

232 Returns 

233 ------- 

234 outputStatistics : `dict` [`str`, `dict` [`str`, scalar]] 

235 A dictionary indexed by the amplifier name, containing 

236 dictionaries of the statistics measured and their values. 

237 

238 Notes 

239 ----- 

240 Number of cosmic rays test: If there are defects in our data that 

241 we didn't properly identify and cover, they might appear similar 

242 to cosmic rays because they have sharp edges compared to the point 

243 spread function (PSF). When we process the data, if these defects 

244 aren't marked in our defect mask, the software might mistakenly think 

245 they are cosmic rays and try to remove them. However, if we've already 

246 included these defects in the defect mask, the software won't treat 

247 them as cosmic rays, so we'll have fewer pixels that are falsely 

248 identified and removed as cosmic rays when we compare two sets of 

249 data reductions. 

250 """ 

251 outputStatistics = super().imageStatistics( 

252 exposure, uncorrectedExposure, statControl 

253 ) 

254 

255 # Is this a useful test? It saves having to do chi^2 fits, 

256 # which are going to be biased by the bulk of points. 

257 for amp in exposure.getDetector(): 

258 ampName = amp.getName() 

259 ampExp = exposure.Factory(exposure, amp.getBBox()) 

260 

261 normImage = ampExp.getImage() 

262 normArray = normImage.getArray() 

263 

264 normArray -= outputStatistics[ampName]["MEDIAN"] 

265 normArray /= outputStatistics[ampName]["STDEV"] 

266 

267 probability = scipy.stats.norm.pdf(normArray) 

268 outliers = np.where(probability < 1.0 / probability.size, 1.0, 0.0) 

269 outputStatistics[ampName]["STAT_OUTLIERS"] = int(np.sum(outliers)) 

270 

271 return outputStatistics 

272 

273 def verify(self, exposure, statisticsDict): 

274 """Verify that the measured statistics meet the verification criteria. 

275 

276 Parameters 

277 ---------- 

278 exposure : `lsst.afw.image.Exposure` 

279 The exposure the statistics are from. 

280 statisticsDictionary : `dict` [`str`, `dict` [`str`, scalar]], 

281 Dictionary of measured statistics. The inner dictionary 

282 should have keys that are statistic names (`str`) with 

283 values that are some sort of scalar (`int` or `float` are 

284 the mostly likely types). 

285 

286 Returns 

287 ------- 

288 outputStatistics : `dict` [`str`, `dict` [`str`, `bool`]] 

289 A dictionary indexed by the amplifier name, containing 

290 dictionaries of the verification criteria. 

291 success : `bool` 

292 A boolean indicating if all tests have passed. 

293 """ 

294 # Amplifier statistics 

295 ampStats = statisticsDict["AMP"] 

296 verifyStats = {} 

297 successAmp = True 

298 for ampName, stats in ampStats.items(): 

299 verify = {} 

300 

301 # These are not defined in DMTN-101 yet. 

302 verify["OUTLIERS"] = bool(stats["UNMASKED_OUTLIERS"] >= stats["OUTLIERS"]) 

303 verify["STDEV"] = bool(stats["UNMASKED_STDEV"] >= stats["STDEV"]) 

304 verify["MIN"] = bool(stats["UNMASKED_MIN"] <= stats["MIN"]) 

305 verify["MAX"] = bool(stats["UNMASKED_MAX"] >= stats["MAX"]) 

306 

307 # This test is bad, and should be made not bad. 

308 verify["PROB_TEST"] = bool(stats["STAT_OUTLIERS"] == stats["DEFECT_PIXELS"]) 

309 

310 verify["SUCCESS"] = bool(np.all(list(verify.values()))) 

311 if verify["SUCCESS"] is False: 

312 successAmp = False 

313 

314 verifyStats[ampName] = verify 

315 

316 # Detector statistics 

317 detStats = statisticsDict["DET"] 

318 verifyStatsDet = {} 

319 successDet = True 

320 # Cosmic rays test from DM-38563, before and after defects. 

321 verifyStatsDet["NUMBER_COSMIC_RAYS"] = bool( 

322 detStats["NUM_COSMICS_BEFORE"] > detStats["NUM_COSMICS_AFTER"] 

323 ) 

324 

325 verifyStatsDet["SUCCESS"] = bool(np.all(list(verifyStatsDet.values()))) 

326 if verifyStatsDet["SUCCESS"] is False: 

327 successDet = False 

328 

329 # Catalog statistics 

330 catStats = statisticsDict["CATALOG"] 

331 verifyStatsCat = {} 

332 successCat = True 

333 # Detection tests from DM-38563, before and after defects. 

334 verifyStatsCat["NUMBER_DETECTIONS"] = bool( 

335 catStats["NUM_OBJECTS_BEFORE"] > catStats["NUM_OBJECTS_AFTER"] 

336 ) 

337 

338 verifyStatsCat["SUCCESS"] = bool(np.all(list(verifyStatsCat.values()))) 

339 if verifyStatsCat["SUCCESS"] is False: 

340 successCat = False 

341 

342 success = successDet & successAmp & successCat 

343 return { 

344 "AMP": verifyStats, 

345 "DET": verifyStatsDet, 

346 "CATALOG": verifyStatsCat, 

347 }, bool(success) 

348 

349 def repackStats(self, statisticsDict, dimensions): 

350 # docstring inherited 

351 rows = {} 

352 rowList = [] 

353 matrixRowList = None 

354 

355 if self.config.useIsrStatistics: 

356 mjd = statisticsDict["ISR"]["MJD"] 

357 else: 

358 mjd = np.nan 

359 

360 rowBase = { 

361 "instrument": dimensions["instrument"], 

362 "exposure": dimensions["visit"], # ensure an exposure dimension for downstream. 

363 "visit": dimensions["visit"], 

364 "detector": dimensions["detector"], 

365 "mjd": mjd, 

366 } 

367 

368 # AMP results 

369 for ampName, stats in statisticsDict["AMP"].items(): 

370 rows[ampName] = {} 

371 rows[ampName].update(rowBase) 

372 rows[ampName]["amplifier"] = ampName 

373 for key, value in stats.items(): 

374 rows[ampName][f"{self.config.stageName}_{key}"] = value 

375 

376 # ISR results 

377 nBadColumns = np.nan 

378 if self.config.useIsrStatistics and "ISR" in statisticsDict: 

379 for ampName, stats in statisticsDict["ISR"]["CALIBDIST"].items(): 

380 if ampName == "detector": 

381 nBadColumns = stats[ampName].get("LSST CALIB DEFECTS N_BAD_COLUMNS", np.nan) 

382 elif ampName in stats.keys(): 

383 key = f"LSST CALIB DEFECTS {ampName} N_HOT" 

384 rows[ampName][f"{self.config.stageName}_N_HOT"] = stats[ampName].get(key, np.nan) 

385 key = f"LSST CALIB DEFECTS {ampName} N_COLD" 

386 rows[ampName][f"{self.config.stageName}_N_COLD"] = stats[ampName].get(key, np.nan) 

387 

388 # DET results 

389 rows["detector"] = rowBase 

390 rows["detector"][f"{self.config.stageName}_N_BAD_COLUMNS"] = nBadColumns 

391 

392 for ampName, stats in rows.items(): 

393 rowList.append(stats) 

394 

395 return rowList, matrixRowList