Coverage for tests/test_defects.py: 10%

330 statements  

« prev     ^ index     » next       coverage.py v7.3.2, created at 2023-10-04 09:08 +0000

1# 

2# LSST Data Management System 

3# 

4# Copyright 2008-2017 AURA/LSST. 

5# 

6# This product includes software developed by the 

7# LSST Project (http://www.lsst.org/). 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the LSST License Statement and 

20# the GNU General Public License along with this program. If not, 

21# see <https://www.lsstcorp.org/LegalNotices/>. 

22# 

23"""Test cases for lsst.cp.pipe.defects.MeasureDefectsTask.""" 

24 

25import unittest 

26import numpy as np 

27import copy 

28 

29import lsst.utils 

30import lsst.utils.tests 

31 

32import lsst.ip.isr as ipIsr 

33import lsst.cp.pipe as cpPipe 

34from lsst.ip.isr import isrMock, countMaskedPixels 

35from lsst.geom import Box2I, Point2I, Extent2I 

36from lsst.daf.base import PropertyList 

37 

38 

39class MeasureDefectsTaskTestCase(lsst.utils.tests.TestCase): 

40 """A test case for the defect finding task.""" 

41 

42 def setUp(self): 

43 self.defaultConfig = cpPipe.defects.MeasureDefectsTask.ConfigClass() 

44 

45 self.flatMean = 2000 

46 self.darkMean = 1 

47 self.readNoiseAdu = 10 

48 self.nSigmaBright = 8 

49 self.nSigmaDark = 8 

50 

51 mockImageConfig = isrMock.IsrMock.ConfigClass() 

52 

53 # flatDrop is not really relevant as we replace the data 

54 # but good to note it in case we change how this image is made 

55 mockImageConfig.flatDrop = 0.99999 

56 mockImageConfig.isTrimmed = True 

57 

58 self.flatExp = isrMock.FlatMock(config=mockImageConfig).run() 

59 (shapeY, shapeX) = self.flatExp.getDimensions() 

60 # x, y, size tuples 

61 # always put edge defects at the start and change the value of nEdge 

62 

63 self.brightDefects = [(0, 15, 3, 3), (100, 123, 1, 1)] 

64 

65 self.darkDefects = [(5, 0, 1, 1), (7, 62, 2, 2)] 

66 

67 nEdge = 1 # NOTE: update if more edge defects are included 

68 self.noEdges = slice(nEdge, None) 

69 self.onlyEdges = slice(0, nEdge) 

70 

71 self.darkBBoxes = [Box2I(Point2I(x, y), Extent2I(sx, sy)) for (x, y, sx, sy) in self.darkDefects] 

72 self.brightBBoxes = [Box2I(Point2I(x, y), Extent2I(sx, sy)) for (x, y, sx, sy) in self.brightDefects] 

73 

74 flatWidth = np.sqrt(self.flatMean) + self.readNoiseAdu 

75 darkWidth = self.readNoiseAdu 

76 self.rng = np.random.RandomState(0) 

77 flatData = self.rng.normal(self.flatMean, flatWidth, (shapeX, shapeY)) 

78 darkData = self.rng.normal(self.darkMean, darkWidth, (shapeX, shapeY)) 

79 

80 # NOTE: darks and flats have same defects applied deliberately to both 

81 for defect in self.brightDefects: 

82 y, x, sy, sx = defect 

83 # are these actually the numbers we want? 

84 flatData[x:x+sx, y:y+sy] += self.nSigmaBright * flatWidth 

85 darkData[x:x+sx, y:y+sy] += self.nSigmaBright * darkWidth 

86 

87 for defect in self.darkDefects: 

88 y, x, sy, sx = defect 

89 # are these actually the numbers we want? 

90 flatData[x:x+sx, y:y+sy] -= self.nSigmaDark * flatWidth 

91 darkData[x:x+sx, y:y+sy] -= self.nSigmaDark * darkWidth 

92 

93 self.darkExp = self.flatExp.clone() 

94 self.spareImage = self.flatExp.clone() # for testing edge bits and misc 

95 

96 self.flatExp.image.array[:] = flatData 

97 self.darkExp.image.array[:] = darkData 

98 

99 self.defaultTask = cpPipe.defects.MeasureDefectsTask() 

100 

101 self.allDefectsList = ipIsr.Defects() 

102 self.brightDefectsList = ipIsr.Defects() 

103 self.darkDefectsList = ipIsr.Defects() 

104 

105 # Set image types, the defects code will use them. 

106 metaDataFlat = PropertyList() 

107 metaDataFlat["IMGTYPE"] = "FLAT" 

108 self.flatExp.setMetadata(metaDataFlat) 

109 

110 metaDataDark = PropertyList() 

111 metaDataDark["IMGTYPE"] = "DARK" 

112 self.darkExp.setMetadata(metaDataDark) 

113 

114 with self.allDefectsList.bulk_update(): 

115 with self.brightDefectsList.bulk_update(): 

116 for d in self.brightBBoxes: 

117 self.brightDefectsList.append(d) 

118 self.allDefectsList.append(d) 

119 

120 with self.darkDefectsList.bulk_update(): 

121 for d in self.darkBBoxes: 

122 self.darkDefectsList.append(d) 

123 self.allDefectsList.append(d) 

124 

125 def check_maskBlocks(self, inputDefects, expectedDefects): 

126 """A helper function for the tests of 

127 maskBlocksIfIntermitentBadPixelsInColumn. 

128 

129 """ 

130 config = copy.copy(self.defaultConfig) 

131 config.badOnAndOffPixelColumnThreshold = 10 

132 config.goodPixelColumnGapThreshold = 5 

133 config.nPixBorderUpDown = 0 

134 config.nPixBorderLeftRight = 0 

135 

136 task = self.defaultTask 

137 task.config = config 

138 

139 defectsWithColumns = task.maskBlocksIfIntermitentBadPixelsInColumn(inputDefects) 

140 boxesMeasured = [] 

141 for defect in defectsWithColumns: 

142 boxesMeasured.append(defect.getBBox()) 

143 

144 for boxInput in expectedDefects: 

145 self.assertIn(boxInput, boxesMeasured) 

146 

147 # Check that the code did not mask anything extra by 

148 # looking in both the input list and "expanded-column" list. 

149 unionInputExpectedBoxes = [] 

150 for defect in inputDefects: 

151 unionInputExpectedBoxes.append(defect.getBBox()) 

152 for defect in expectedDefects: 

153 unionInputExpectedBoxes.append(defect) 

154 

155 # Check that code doesn't mask more than it is supposed to. 

156 for boxMeas in boxesMeasured: 

157 self.assertIn(boxMeas, unionInputExpectedBoxes) 

158 

159 def test_maskBlocks_full_column(self): 

160 """A test for maskBlocksIfIntermitentBadPixelsInColumn. 

161 

162 Tests that a contigous bad column does not get split by the 

163 code. 

164 

165 The mock flat has a size of 200X204 pixels. This column has a 

166 maximum length of 50 pixels, otherwise there would be a split 

167 along the mock amp boundary. 

168 

169 Plots can be found in DM-19903 on Jira. 

170 

171 """ 

172 

173 defects = self.allDefectsList 

174 defects.append(Box2I(corner=Point2I(15, 1), dimensions=Extent2I(1, 50))) 

175 expectedDefects = [Box2I(corner=Point2I(15, 1), dimensions=Extent2I(1, 50))] 

176 

177 self.check_maskBlocks(defects, expectedDefects) 

178 

179 def test_maskBlocks_long_column(self): 

180 """A test for maskBlocksIfIntermitentBadPixelsInColumn. 

181 

182 Tests that a contigous bad column with Npix >= 

183 badOnAndOffPixelColumnThreshold (10) does not get split by the 

184 code. 

185 

186 Plots can be found in DM-19903 on Jira. 

187 

188 """ 

189 

190 expectedDefects = [Box2I(corner=Point2I(20, 1), dimensions=Extent2I(1, 25))] 

191 defects = self.allDefectsList 

192 defects.append(Box2I(corner=Point2I(20, 1), dimensions=Extent2I(1, 25))) 

193 

194 self.check_maskBlocks(defects, expectedDefects) 

195 

196 def test_maskBlocks_short_column(self): 

197 """A test for maskBlocksIfIntermitentBadPixelsInColumn. 

198 

199 Tests that a contigous bad column Npix < 

200 badOnAndOffPixelColumnThreshold (10) does not get split by the 

201 code. 

202 

203 Plots can be found in DM-19903 on Jira. 

204 

205 """ 

206 

207 expectedDefects = [Box2I(corner=Point2I(25, 1), dimensions=Extent2I(1, 8))] 

208 defects = self.allDefectsList 

209 defects.append(Box2I(corner=Point2I(25, 1), dimensions=Extent2I(1, 8))) 

210 

211 self.check_maskBlocks(defects, expectedDefects) 

212 

213 def test_maskBlocks_discontigous_to_single_block(self): 

214 """A test for maskBlocksIfIntermitentBadPixelsInColumn. 

215 

216 Npix discontiguous bad pixels in a column where Npix >= 

217 badOnAndOffPixelColumnThreshold (10) and gaps of good pixels < 

218 goodPixelColumnGapThreshold (5). Under these conditions, the 

219 whole block of bad pixels (including good gaps) should be 

220 masked. 

221 

222 Plots can be found in DM-19903 on Jira. 

223 

224 """ 

225 

226 expectedDefects = [Box2I(corner=Point2I(30, 1), dimensions=Extent2I(1, 48))] 

227 defects = self.allDefectsList 

228 badPixels = [Box2I(corner=Point2I(30, 1), dimensions=Extent2I(1, 2)), 

229 Box2I(corner=Point2I(30, 5), dimensions=Extent2I(1, 3)), 

230 Box2I(corner=Point2I(30, 11), dimensions=Extent2I(1, 5)), 

231 Box2I(corner=Point2I(30, 19), dimensions=Extent2I(1, 5)), 

232 Box2I(corner=Point2I(30, 27), dimensions=Extent2I(1, 4)), 

233 Box2I(corner=Point2I(30, 34), dimensions=Extent2I(1, 15))] 

234 with defects.bulk_update(): 

235 for badBox in badPixels: 

236 defects.append(badBox) 

237 

238 self.check_maskBlocks(defects, expectedDefects) 

239 

240 def test_maskBlocks_discontigous_less_than_thresholds(self): 

241 """A test for maskBlocksIfIntermitentBadPixelsInColumn. 

242 

243 Npix discontiguous bad pixels in a column where Npix < 

244 badOnAndOffPixelColumnThreshold (10) and gaps of good pixels < 

245 goodPixelColumnGapThreshold (5). Under these conditions, the 

246 expected defect boxes should be the same as the input boxes. 

247 

248 Plots can be found in DM-19903 on Jira. 

249 

250 """ 

251 

252 expectedDefects = [Box2I(corner=Point2I(35, 1), dimensions=Extent2I(1, 2)), 

253 Box2I(corner=Point2I(35, 5), dimensions=Extent2I(1, 3)), 

254 Box2I(corner=Point2I(35, 11), dimensions=Extent2I(1, 2))] 

255 defects = self.allDefectsList 

256 badPixels = [Box2I(corner=Point2I(35, 1), dimensions=Extent2I(1, 2)), 

257 Box2I(corner=Point2I(35, 5), dimensions=Extent2I(1, 3)), 

258 Box2I(corner=Point2I(35, 11), dimensions=Extent2I(1, 2))] 

259 with defects.bulk_update(): 

260 for badBox in badPixels: 

261 defects.append(badBox) 

262 

263 self.check_maskBlocks(defects, expectedDefects) 

264 

265 def test_maskBlocks_more_than_thresholds(self): 

266 """A test for maskBlocksIfIntermitentBadPixelsInColumn. 

267 

268 Npix discontiguous bad pixels in a column where Npix < 

269 badOnAndOffPixelColumnThreshold (10) and gaps of good pixels < 

270 goodPixelColumnGapThreshold (5). Npix=34 (> 10) bad pixels 

271 total, 1 "good" gap with 13 pixels big enough (13 >= 5 good 

272 pixels, from y=6 (1+5) to y=19). 

273 

274 Plots can be found in DM-19903 on Jira. 

275 

276 """ 

277 

278 expectedDefects = [Box2I(corner=Point2I(40, 1), dimensions=Extent2I(1, 7)), 

279 Box2I(corner=Point2I(40, 19), dimensions=Extent2I(1, 30))] 

280 defects = self.allDefectsList 

281 badPixels = [Box2I(corner=Point2I(40, 1), dimensions=Extent2I(1, 2)), 

282 Box2I(corner=Point2I(40, 5), dimensions=Extent2I(1, 3)), 

283 Box2I(corner=Point2I(40, 19), dimensions=Extent2I(1, 5)), 

284 Box2I(corner=Point2I(40, 27), dimensions=Extent2I(1, 4)), 

285 Box2I(corner=Point2I(40, 34), dimensions=Extent2I(1, 15))] 

286 with defects.bulk_update(): 

287 for badBox in badPixels: 

288 defects.append(badBox) 

289 

290 self.check_maskBlocks(defects, expectedDefects) 

291 

292 def test_maskBlocks_not_enough_bad_pixels_in_column(self): 

293 """A test for maskBlocksIfIntermitentBadPixelsInColumn. 

294 

295 Npix discontiguous bad pixels in a column where Npix < 

296 badOnAndOffPixelColumnThreshold (10) and and gaps of good 

297 pixels > goodPixelColumnGapThreshold (5). Since Npix < 

298 badOnAndOffPixelColumnThreshold, then it doesn't matter that 

299 the number of good pixels in gap > 

300 goodPixelColumnGapThreshold. 5<10 bad pixels total, 1 "good" 

301 gap big enough (29>=5 good pixels, from y =12 (10+2) to y=30) 

302 

303 Plots can be found in DM-19903 on Jira. 

304 

305 """ 

306 

307 expectedDefects = [Box2I(corner=Point2I(45, 10), dimensions=Extent2I(1, 2)), 

308 Box2I(corner=Point2I(45, 30), dimensions=Extent2I(1, 3))] 

309 defects = self.allDefectsList 

310 badPixels = [Box2I(corner=Point2I(45, 10), dimensions=Extent2I(1, 2)), 

311 Box2I(corner=Point2I(45, 30), dimensions=Extent2I(1, 3))] 

312 with defects.bulk_update(): 

313 for badBox in badPixels: 

314 defects.append(badBox) 

315 

316 self.check_maskBlocks(defects, expectedDefects) 

317 

318 def test_maskBlocks_every_other_pixel_bad_greater_than_threshold(self): 

319 """A test for maskBlocksIfIntermitentBadPixelsInColumn. 

320 

321 Npix discontiguous bad pixels in a column where Npix > 

322 badOnAndOffPixelColumnThreshold (10) and every other pixel is 

323 bad. 

324 

325 Plots can be found in DM-19903 on Jira. 

326 

327 """ 

328 

329 expectedDefects = [Box2I(corner=Point2I(50, 10), dimensions=Extent2I(1, 31))] 

330 defects = self.allDefectsList 

331 badPixels = [Box2I(corner=Point2I(50, 10), dimensions=Extent2I(1, 1)), 

332 Box2I(corner=Point2I(50, 12), dimensions=Extent2I(1, 1)), 

333 Box2I(corner=Point2I(50, 14), dimensions=Extent2I(1, 1)), 

334 Box2I(corner=Point2I(50, 16), dimensions=Extent2I(1, 1)), 

335 Box2I(corner=Point2I(50, 18), dimensions=Extent2I(1, 1)), 

336 Box2I(corner=Point2I(50, 20), dimensions=Extent2I(1, 1)), 

337 Box2I(corner=Point2I(50, 22), dimensions=Extent2I(1, 1)), 

338 Box2I(corner=Point2I(50, 24), dimensions=Extent2I(1, 1)), 

339 Box2I(corner=Point2I(50, 26), dimensions=Extent2I(1, 1)), 

340 Box2I(corner=Point2I(50, 28), dimensions=Extent2I(1, 1)), 

341 Box2I(corner=Point2I(50, 30), dimensions=Extent2I(1, 1)), 

342 Box2I(corner=Point2I(50, 32), dimensions=Extent2I(1, 1)), 

343 Box2I(corner=Point2I(50, 34), dimensions=Extent2I(1, 1)), 

344 Box2I(corner=Point2I(50, 36), dimensions=Extent2I(1, 1)), 

345 Box2I(corner=Point2I(50, 38), dimensions=Extent2I(1, 1)), 

346 Box2I(corner=Point2I(50, 40), dimensions=Extent2I(1, 1))] 

347 with defects.bulk_update(): 

348 for badBox in badPixels: 

349 defects.append(badBox) 

350 

351 self.check_maskBlocks(defects, expectedDefects) 

352 

353 def test_maskBlocks_every_other_pixel_bad_less_than_threshold(self): 

354 """A test for maskBlocksIfIntermitentBadPixelsInColumn. 

355 

356 Npix discontiguous bad pixels in a column where Npix > 

357 badOnAndOffPixelColumnThreshold (10) and every other pixel is 

358 bad. 

359 

360 Plots can be found in DM-19903 on Jira. 

361 

362 """ 

363 

364 expectedDefects = [Box2I(corner=Point2I(55, 20), dimensions=Extent2I(1, 1)), 

365 Box2I(corner=Point2I(55, 22), dimensions=Extent2I(1, 1)), 

366 Box2I(corner=Point2I(55, 24), dimensions=Extent2I(1, 1)), 

367 Box2I(corner=Point2I(55, 26), dimensions=Extent2I(1, 1)), 

368 Box2I(corner=Point2I(55, 28), dimensions=Extent2I(1, 1)), 

369 Box2I(corner=Point2I(55, 30), dimensions=Extent2I(1, 1))] 

370 defects = self.allDefectsList 

371 badPixels = [Box2I(corner=Point2I(55, 20), dimensions=Extent2I(1, 1)), 

372 Box2I(corner=Point2I(55, 22), dimensions=Extent2I(1, 1)), 

373 Box2I(corner=Point2I(55, 24), dimensions=Extent2I(1, 1)), 

374 Box2I(corner=Point2I(55, 26), dimensions=Extent2I(1, 1)), 

375 Box2I(corner=Point2I(55, 28), dimensions=Extent2I(1, 1)), 

376 Box2I(corner=Point2I(55, 30), dimensions=Extent2I(1, 1))] 

377 with defects.bulk_update(): 

378 for badBox in badPixels: 

379 defects.append(badBox) 

380 

381 self.check_maskBlocks(defects, expectedDefects) 

382 

383 def test_maskBlocks_blobs_one_side_good_less_than_threshold(self): 

384 """A test for maskBlocksIfIntermitentBadPixelsInColumn. 

385 

386 Npix discontiguous bad pixels in column with "blobs" of "m" 

387 bad pixels to one side, m > badOnAndOffPixelColumnThreshold 

388 (10), number of good pixel in gaps between blobs < 

389 goodPixelColumnGapThreshold (5). 

390 

391 Plots can be found in DM-19903 on Jira. 

392 

393 """ 

394 

395 expectedDefects = [Box2I(corner=Point2I(60, 1), dimensions=Extent2I(1, 29)), 

396 Box2I(corner=Point2I(61, 2), dimensions=Extent2I(2, 12))] 

397 defects = self.allDefectsList 

398 badPixels = [Box2I(corner=Point2I(60, 1), dimensions=Extent2I(1, 18)), 

399 Box2I(corner=Point2I(60, 20), dimensions=Extent2I(1, 10)), 

400 Box2I(corner=Point2I(61, 2), dimensions=Extent2I(2, 2)), 

401 Box2I(corner=Point2I(61, 6), dimensions=Extent2I(2, 8))] 

402 with defects.bulk_update(): 

403 for badBox in badPixels: 

404 defects.append(badBox) 

405 

406 self.check_maskBlocks(defects, expectedDefects) 

407 

408 def test_maskBlocks_blobs_other_side_good_less_than_threshold(self): 

409 """A test for maskBlocksIfIntermitentBadPixelsInColumn. 

410 

411 Npix discontiguous bad pixels in column with "blobs" of "m" 

412 bad pixels to the other side, m > 

413 badOnAndOffPixelColumnThreshold (10), number of good pixel in 

414 gaps between blobs < goodPixelColumnGapThreshold (5). 

415 

416 Plots can be found in DM-19903 on Jira. 

417 

418 """ 

419 

420 expectedDefects = [Box2I(corner=Point2I(70, 1), dimensions=Extent2I(1, 29)), 

421 Box2I(corner=Point2I(68, 2), dimensions=Extent2I(2, 12))] 

422 defects = self.allDefectsList 

423 badPixels = [Box2I(corner=Point2I(70, 1), dimensions=Extent2I(1, 18)), 

424 Box2I(corner=Point2I(70, 20), dimensions=Extent2I(1, 10)), 

425 Box2I(corner=Point2I(68, 2), dimensions=Extent2I(2, 2)), 

426 Box2I(corner=Point2I(68, 6), dimensions=Extent2I(2, 8))] 

427 with defects.bulk_update(): 

428 for badBox in badPixels: 

429 defects.append(badBox) 

430 

431 self.check_maskBlocks(defects, expectedDefects) 

432 

433 def test_maskBlocks_blob_both_sides_good_less_than_threshold(self): 

434 """A test for maskBlocksIfIntermitentBadPixelsInColumn. 

435 

436 Npix discontiguous bad pixels in column with "blobs" of "m" 

437 bad pixels to both sides, m > badOnAndOffPixelColumnThreshold 

438 (10), number of good pixel in gaps between blobs < 

439 goodPixelColumnGapThreshold (5). 

440 

441 Plots can be found in DM-19903 on Jira. 

442 

443 """ 

444 

445 expectedDefects = [Box2I(corner=Point2I(75, 1), dimensions=Extent2I(1, 29)), 

446 Box2I(corner=Point2I(73, 2), dimensions=Extent2I(2, 12)), 

447 Box2I(corner=Point2I(76, 2), dimensions=Extent2I(2, 12))] 

448 defects = self.allDefectsList 

449 badPixels = [Box2I(corner=Point2I(75, 1), dimensions=Extent2I(1, 18)), 

450 Box2I(corner=Point2I(75, 20), dimensions=Extent2I(1, 10)), 

451 Box2I(corner=Point2I(73, 2), dimensions=Extent2I(2, 2)), 

452 Box2I(corner=Point2I(73, 6), dimensions=Extent2I(2, 8)), 

453 Box2I(corner=Point2I(76, 2), dimensions=Extent2I(2, 2)), 

454 Box2I(corner=Point2I(76, 6), dimensions=Extent2I(2, 8))] 

455 with defects.bulk_update(): 

456 for badBox in badPixels: 

457 defects.append(badBox) 

458 

459 self.check_maskBlocks(defects, expectedDefects) 

460 

461 def test_maskBlocks_blob_one_side_good_greater_than_threshold(self): 

462 """A test for maskBlocksIfIntermitentBadPixelsInColumn. 

463 

464 Npix discontiguous bad pixels in column with "blobs" of "m" 

465 bad pixels to one side, m > badOnAndOffPixelColumnThreshold 

466 (10), number of good pixel in gaps between blobs > 

467 goodPixelColumnGapThreshold (5). 

468 

469 Plots can be found in DM-19903 on Jira. 

470 

471 """ 

472 

473 expectedDefects = [Box2I(corner=Point2I(80, 1), dimensions=Extent2I(1, 29)), 

474 Box2I(corner=Point2I(81, 2), dimensions=Extent2I(2, 2)), 

475 Box2I(corner=Point2I(81, 8), dimensions=Extent2I(2, 8))] 

476 defects = self.allDefectsList 

477 badPixels = [Box2I(corner=Point2I(80, 1), dimensions=Extent2I(1, 18)), 

478 Box2I(corner=Point2I(80, 20), dimensions=Extent2I(1, 10)), 

479 Box2I(corner=Point2I(81, 2), dimensions=Extent2I(2, 2)), 

480 Box2I(corner=Point2I(81, 8), dimensions=Extent2I(2, 8))] 

481 with defects.bulk_update(): 

482 for badBox in badPixels: 

483 defects.append(badBox) 

484 

485 self.check_maskBlocks(defects, expectedDefects) 

486 

487 def test_maskBlocks_other_side_good_greater_than_threshold(self): 

488 """A test for maskBlocksIfIntermitentBadPixelsInColumn. 

489 

490 Npix discontiguous bad pixels in column with "blobs" of "m" 

491 bad pixels to the other side, m > 

492 badOnAndOffPixelColumnThreshold (10), number of good pixel in 

493 gaps between blobs > goodPixelColumnGapThreshold (5). 

494 

495 Plots can be found in DM-19903 on Jira. 

496 

497 """ 

498 

499 expectedDefects = [Box2I(corner=Point2I(87, 1), dimensions=Extent2I(1, 29)), 

500 Box2I(corner=Point2I(85, 2), dimensions=Extent2I(2, 2)), 

501 Box2I(corner=Point2I(85, 8), dimensions=Extent2I(2, 8))] 

502 defects = self.allDefectsList 

503 badPixels = [Box2I(corner=Point2I(87, 1), dimensions=Extent2I(1, 18)), 

504 Box2I(corner=Point2I(87, 20), dimensions=Extent2I(1, 10)), 

505 Box2I(corner=Point2I(85, 2), dimensions=Extent2I(2, 2)), 

506 Box2I(corner=Point2I(85, 8), dimensions=Extent2I(2, 8))] 

507 with defects.bulk_update(): 

508 for badBox in badPixels: 

509 defects.append(badBox) 

510 

511 self.check_maskBlocks(defects, expectedDefects) 

512 

513 def test_maskBlocks_both_sides_good_greater_than_threshold(self): 

514 """A test for maskBlocksIfIntermitentBadPixelsInColumn. 

515 

516 Npix discontiguous bad pixels in column with "blobs" of "m" 

517 bad pixels to both sides, m > badOnAndOffPixelColumnThreshold 

518 (10), number of good pixel in gaps between blobs > 

519 goodPixelColumnGapThreshold (5). 

520 

521 Plots can be found in DM-19903 on Jira. 

522 

523 """ 

524 

525 expectedDefects = [Box2I(corner=Point2I(93, 1), dimensions=Extent2I(1, 34)), 

526 Box2I(corner=Point2I(91, 2), dimensions=Extent2I(2, 7)), 

527 Box2I(corner=Point2I(91, 18), dimensions=Extent2I(2, 9)), 

528 Box2I(corner=Point2I(94, 2), dimensions=Extent2I(2, 7)), 

529 Box2I(corner=Point2I(94, 18), dimensions=Extent2I(2, 9))] 

530 defects = self.allDefectsList 

531 badPixels = [Box2I(corner=Point2I(93, 1), dimensions=Extent2I(1, 12)), 

532 Box2I(corner=Point2I(93, 15), dimensions=Extent2I(1, 20)), 

533 Box2I(corner=Point2I(91, 2), dimensions=Extent2I(2, 2)), 

534 Box2I(corner=Point2I(91, 7), dimensions=Extent2I(2, 2)), 

535 Box2I(corner=Point2I(94, 2), dimensions=Extent2I(2, 2)), 

536 Box2I(corner=Point2I(94, 7), dimensions=Extent2I(2, 2)), 

537 Box2I(corner=Point2I(91, 18), dimensions=Extent2I(2, 3)), 

538 Box2I(corner=Point2I(91, 24), dimensions=Extent2I(2, 3)), 

539 Box2I(corner=Point2I(94, 18), dimensions=Extent2I(2, 3)), 

540 Box2I(corner=Point2I(94, 24), dimensions=Extent2I(2, 3))] 

541 with defects.bulk_update(): 

542 for badBox in badPixels: 

543 defects.append(badBox) 

544 

545 self.check_maskBlocks(defects, expectedDefects) 

546 

547 def test_maskBlocks_y_out_of_order_dm38103(self): 

548 """A test for maskBlocksIfIntermitentBadPixelsInColumn, y out of order. 

549 

550 This test is a variant of 

551 notest_maskBlocks_every_other_pixel_bad_greater_than_threshold with 

552 an extra out-of-y-order bad pixel to trigger DM-38103. 

553 """ 

554 expectedDefects = [Box2I(corner=Point2I(50, 110), dimensions=Extent2I(1, 31))] 

555 defects = self.allDefectsList 

556 badPixels = [Box2I(corner=Point2I(50, 110), dimensions=Extent2I(1, 1)), 

557 Box2I(corner=Point2I(50, 112), dimensions=Extent2I(1, 1)), 

558 Box2I(corner=Point2I(50, 114), dimensions=Extent2I(1, 1)), 

559 Box2I(corner=Point2I(50, 116), dimensions=Extent2I(1, 1)), 

560 Box2I(corner=Point2I(50, 118), dimensions=Extent2I(1, 1)), 

561 Box2I(corner=Point2I(50, 120), dimensions=Extent2I(1, 1)), 

562 Box2I(corner=Point2I(50, 122), dimensions=Extent2I(1, 1)), 

563 Box2I(corner=Point2I(50, 124), dimensions=Extent2I(1, 1)), 

564 Box2I(corner=Point2I(50, 126), dimensions=Extent2I(1, 1)), 

565 Box2I(corner=Point2I(50, 128), dimensions=Extent2I(1, 1)), 

566 Box2I(corner=Point2I(50, 130), dimensions=Extent2I(1, 1)), 

567 Box2I(corner=Point2I(50, 132), dimensions=Extent2I(1, 1)), 

568 Box2I(corner=Point2I(50, 134), dimensions=Extent2I(1, 1)), 

569 Box2I(corner=Point2I(50, 136), dimensions=Extent2I(1, 1)), 

570 Box2I(corner=Point2I(50, 138), dimensions=Extent2I(1, 1)), 

571 Box2I(corner=Point2I(50, 140), dimensions=Extent2I(1, 1)), 

572 # This last point is out of order in y. 

573 Box2I(corner=Point2I(50, 100), dimensions=Extent2I(1, 1))] 

574 

575 # Force defect normalization off in order to trigger DM-38301, because 

576 # defects.fromFootprintList() which is called by _findHotAndColdPixels 

577 # does not do normalization. 

578 defects._bulk_update = True 

579 for badBox in badPixels: 

580 defects.append(badBox) 

581 defects._bulk_update = False 

582 

583 self.check_maskBlocks(defects, expectedDefects) 

584 

585 def test_defectFindingAllSensor(self): 

586 config = copy.copy(self.defaultConfig) 

587 config.nPixBorderLeftRight = 0 

588 config.nPixBorderUpDown = 0 

589 

590 task = self.defaultTask 

591 task.config = config 

592 

593 defects = task._findHotAndColdPixels(self.flatExp) 

594 

595 allBBoxes = self.darkBBoxes + self.brightBBoxes 

596 

597 boxesMeasured = [] 

598 for defect in defects: 

599 boxesMeasured.append(defect.getBBox()) 

600 

601 for expectedBBox in allBBoxes: 

602 self.assertIn(expectedBBox, boxesMeasured) 

603 

604 def test_defectFindingEdgeIgnore(self): 

605 config = copy.copy(self.defaultConfig) 

606 config.nPixBorderUpDown = 0 

607 task = self.defaultTask 

608 task.config = config 

609 defects = task._findHotAndColdPixels(self.flatExp) 

610 

611 shouldBeFound = self.darkBBoxes[self.noEdges] + self.brightBBoxes[self.noEdges] 

612 

613 boxesMeasured = [] 

614 for defect in defects: 

615 boxesMeasured.append(defect.getBBox()) 

616 

617 for expectedBBox in shouldBeFound: 

618 self.assertIn(expectedBBox, boxesMeasured) 

619 

620 shouldBeMissed = self.darkBBoxes[self.onlyEdges] + self.brightBBoxes[self.onlyEdges] 

621 for boxMissed in shouldBeMissed: 

622 self.assertNotIn(boxMissed, boxesMeasured) 

623 

624 def valueThreshold(self, fileType, saturateAmpInFlat=False): 

625 """Helper function to loop over flats and darks 

626 to test thresholdType = 'VALUE'.""" 

627 config = copy.copy(self.defaultConfig) 

628 config.thresholdType = 'VALUE' 

629 task = self.defaultTask 

630 task.config = config 

631 

632 for amp in self.flatExp.getDetector(): 

633 if amp.getName() == 'C:0,0': 

634 regionC00 = amp.getBBox() 

635 

636 if fileType == 'dark': 

637 exp = self.darkExp 

638 shouldBeFound = self.brightBBoxes[self.noEdges] 

639 else: 

640 exp = self.flatExp 

641 if saturateAmpInFlat: 

642 exp.maskedImage[regionC00].image.array[:] = 0.0 

643 # Amp C:0,0: minimum=(0, 0), maximum=(99, 50) 

644 x = self.defaultConfig.nPixBorderUpDown 

645 y = self.defaultConfig.nPixBorderLeftRight 

646 width, height = regionC00.getEndX() - x, regionC00.getEndY() - y 

647 # Defects code will mark whole saturated amp as defect box. 

648 shouldBeFound = [Box2I(corner=Point2I(x, y), dimensions=Extent2I(width, height))] 

649 else: 

650 shouldBeFound = self.darkBBoxes[self.noEdges] 

651 # Change the default a bit so it works for the 

652 # existing simulated defects. 

653 task.config.fracThresholdFlat = 0.9 

654 

655 defects = task._findHotAndColdPixels(exp) 

656 

657 boxesMeasured = [] 

658 for defect in defects: 

659 boxesMeasured.append(defect.getBBox()) 

660 

661 for expectedBBox in shouldBeFound: 

662 self.assertIn(expectedBBox, boxesMeasured) 

663 

664 def test_valueThreshold(self): 

665 for fileType in ['dark', 'flat']: 

666 self.valueThreshold(fileType) 

667 # stdDev = 0.0 

668 self.valueThreshold('flat', saturateAmpInFlat=True) 

669 

670 def test_pixelCounting(self): 

671 """Test that the number of defective pixels identified is as expected. 

672 """ 

673 config = copy.copy(self.defaultConfig) 

674 config.nPixBorderUpDown = 0 

675 config.nPixBorderLeftRight = 0 

676 task = self.defaultTask 

677 task.config = config 

678 defects = task._findHotAndColdPixels(self.flatExp) 

679 

680 defectArea = 0 

681 for defect in defects: 

682 defectArea += defect.getBBox().getArea() 

683 

684 # The columnar code will cover blocks of a column with 

685 # on-and-off pixels, thus creating more bad pixels that what 

686 # initially placed in self.brightDefects and self.darkDefects. 

687 # Thus, defectArea should be >= crossCheck. 

688 crossCheck = 0 

689 for x, y, sx, sy in self.brightDefects: 

690 crossCheck += sx*sy 

691 for x, y, sx, sy in self.darkDefects: 

692 crossCheck += sx*sy 

693 

694 # Test the result of _nPixFromDefects() 

695 # via two different ways of calculating area. 

696 self.assertEqual(defectArea, task._nPixFromDefects(defects)) 

697 # defectArea should be >= crossCheck 

698 self.assertGreaterEqual(defectArea, crossCheck) 

699 

700 def test_getNumGoodPixels(self): 

701 """Test the the number of pixels in the image not masked is as 

702 expected. 

703 """ 

704 testImage = self.flatExp.clone() 

705 mi = testImage.maskedImage 

706 

707 imageSize = testImage.getBBox().getArea() 

708 nGood = self.defaultTask._getNumGoodPixels(mi) 

709 

710 self.assertEqual(imageSize, nGood) 

711 

712 NODATABIT = mi.mask.getPlaneBitMask("NO_DATA") 

713 

714 noDataBox = Box2I(Point2I(31, 49), Extent2I(3, 6)) 

715 testImage.mask[noDataBox] |= NODATABIT 

716 

717 self.assertEqual(imageSize - noDataBox.getArea(), self.defaultTask._getNumGoodPixels(mi)) 

718 # check for misfire; we're setting NO_DATA here, not BAD 

719 self.assertEqual(imageSize, self.defaultTask._getNumGoodPixels(mi, 'BAD')) 

720 

721 testImage.mask[noDataBox] ^= NODATABIT # XOR to reset what we did 

722 self.assertEqual(imageSize, nGood) 

723 

724 BADBIT = mi.mask.getPlaneBitMask("BAD") 

725 badBox = Box2I(Point2I(85, 98), Extent2I(4, 7)) 

726 testImage.mask[badBox] |= BADBIT 

727 

728 self.assertEqual(imageSize - badBox.getArea(), self.defaultTask._getNumGoodPixels(mi, 'BAD')) 

729 

730 def test_edgeMasking(self): 

731 """Check that the right number of edge pixels are masked by 

732 _setEdgeBits(). 

733 """ 

734 testImage = self.flatExp.clone() 

735 mi = testImage.maskedImage 

736 

737 self.assertEqual(countMaskedPixels(mi, 'EDGE'), 0) 

738 self.defaultTask._setEdgeBits(mi) 

739 

740 hEdge = self.defaultConfig.nPixBorderLeftRight 

741 vEdge = self.defaultConfig.nPixBorderUpDown 

742 xSize, ySize = mi.getDimensions() 

743 

744 nEdge = xSize*vEdge*2 + ySize*hEdge*2 - hEdge*vEdge*4 

745 

746 self.assertEqual(countMaskedPixels(mi, 'EDGE'), nEdge) 

747 

748 def test_badImage(self): 

749 """Check that fully-bad images do not fail. 

750 """ 

751 testImage = self.flatExp.clone() 

752 testImage.image.array[:, :] = 125000 

753 

754 config = copy.copy(self.defaultConfig) 

755 # Do not exclude any pixels, so the areas match. 

756 config.nPixBorderUpDown = 0 

757 config.nPixBorderLeftRight = 0 

758 

759 task = self.defaultTask 

760 task.config = config 

761 defects = task._findHotAndColdPixels(testImage) 

762 

763 defectArea = 0 

764 for defect in defects: 

765 defectArea += defect.getBBox().getArea() 

766 self.assertEqual(defectArea, testImage.getBBox().getArea()) 

767 

768 

769class TestMemory(lsst.utils.tests.MemoryTestCase): 

770 pass 

771 

772 

773def setup_module(module): 

774 lsst.utils.tests.init() 

775 

776 

777if __name__ == "__main__": 777 ↛ 778line 777 didn't jump to line 778, because the condition on line 777 was never true

778 lsst.utils.tests.init() 

779 unittest.main()