Coverage for tests/test_defects.py: 10%

320 statements  

« prev     ^ index     » next       coverage.py v7.2.3, created at 2023-04-25 04:39 -0700

1# 

2# LSST Data Management System 

3# 

4# Copyright 2008-2017 AURA/LSST. 

5# 

6# This product includes software developed by the 

7# LSST Project (http://www.lsst.org/). 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the LSST License Statement and 

20# the GNU General Public License along with this program. If not, 

21# see <https://www.lsstcorp.org/LegalNotices/>. 

22# 

23"""Test cases for lsst.cp.pipe.defects.MeasureDefectsTask.""" 

24 

25import unittest 

26import numpy as np 

27import copy 

28 

29import lsst.utils 

30import lsst.utils.tests 

31 

32import lsst.ip.isr as ipIsr 

33import lsst.cp.pipe as cpPipe 

34from lsst.ip.isr import isrMock, countMaskedPixels 

35from lsst.geom import Box2I, Point2I, Extent2I 

36from lsst.daf.base import PropertyList 

37 

38 

39class MeasureDefectsTaskTestCase(lsst.utils.tests.TestCase): 

40 """A test case for the defect finding task.""" 

41 

42 def setUp(self): 

43 self.defaultConfig = cpPipe.defects.MeasureDefectsTask.ConfigClass() 

44 

45 self.flatMean = 2000 

46 self.darkMean = 1 

47 self.readNoiseAdu = 10 

48 self.nSigmaBright = 8 

49 self.nSigmaDark = 8 

50 

51 mockImageConfig = isrMock.IsrMock.ConfigClass() 

52 

53 # flatDrop is not really relevant as we replace the data 

54 # but good to note it in case we change how this image is made 

55 mockImageConfig.flatDrop = 0.99999 

56 mockImageConfig.isTrimmed = True 

57 

58 self.flatExp = isrMock.FlatMock(config=mockImageConfig).run() 

59 (shapeY, shapeX) = self.flatExp.getDimensions() 

60 

61 # x, y, size tuples 

62 # always put edge defects at the start and change the value of nEdge 

63 

64 self.brightDefects = [(0, 15, 3, 3), (100, 123, 1, 1)] 

65 

66 self.darkDefects = [(5, 0, 1, 1), (7, 62, 2, 2)] 

67 

68 nEdge = 1 # NOTE: update if more edge defects are included 

69 self.noEdges = slice(nEdge, None) 

70 self.onlyEdges = slice(0, nEdge) 

71 

72 self.darkBBoxes = [Box2I(Point2I(x, y), Extent2I(sx, sy)) for (x, y, sx, sy) in self.darkDefects] 

73 self.brightBBoxes = [Box2I(Point2I(x, y), Extent2I(sx, sy)) for (x, y, sx, sy) in self.brightDefects] 

74 

75 flatWidth = np.sqrt(self.flatMean) + self.readNoiseAdu 

76 darkWidth = self.readNoiseAdu 

77 self.rng = np.random.RandomState(0) 

78 flatData = self.rng.normal(self.flatMean, flatWidth, (shapeX, shapeY)) 

79 darkData = self.rng.normal(self.darkMean, darkWidth, (shapeX, shapeY)) 

80 

81 # NOTE: darks and flats have same defects applied deliberately to both 

82 for defect in self.brightDefects: 

83 y, x, sy, sx = defect 

84 # are these actually the numbers we want? 

85 flatData[x:x+sx, y:y+sy] += self.nSigmaBright * flatWidth 

86 darkData[x:x+sx, y:y+sy] += self.nSigmaBright * darkWidth 

87 

88 for defect in self.darkDefects: 

89 y, x, sy, sx = defect 

90 # are these actually the numbers we want? 

91 flatData[x:x+sx, y:y+sy] -= self.nSigmaDark * flatWidth 

92 darkData[x:x+sx, y:y+sy] -= self.nSigmaDark * darkWidth 

93 

94 self.darkExp = self.flatExp.clone() 

95 self.spareImage = self.flatExp.clone() # for testing edge bits and misc 

96 

97 self.flatExp.image.array[:] = flatData 

98 self.darkExp.image.array[:] = darkData 

99 

100 self.defaultTask = cpPipe.defects.MeasureDefectsTask() 

101 

102 self.allDefectsList = ipIsr.Defects() 

103 self.brightDefectsList = ipIsr.Defects() 

104 self.darkDefectsList = ipIsr.Defects() 

105 

106 # Set image types, the defects code will use them. 

107 metaDataFlat = PropertyList() 

108 metaDataFlat["IMGTYPE"] = "FLAT" 

109 self.flatExp.setMetadata(metaDataFlat) 

110 

111 metaDataDark = PropertyList() 

112 metaDataDark["IMGTYPE"] = "DARK" 

113 self.darkExp.setMetadata(metaDataDark) 

114 

115 with self.allDefectsList.bulk_update(): 

116 with self.brightDefectsList.bulk_update(): 

117 for d in self.brightBBoxes: 

118 self.brightDefectsList.append(d) 

119 self.allDefectsList.append(d) 

120 

121 with self.darkDefectsList.bulk_update(): 

122 for d in self.darkBBoxes: 

123 self.darkDefectsList.append(d) 

124 self.allDefectsList.append(d) 

125 

126 def check_maskBlocks(self, inputDefects, expectedDefects): 

127 """A helper function for the tests of 

128 maskBlocksIfIntermitentBadPixelsInColumn. 

129 

130 """ 

131 config = copy.copy(self.defaultConfig) 

132 config.badOnAndOffPixelColumnThreshold = 10 

133 config.goodPixelColumnGapThreshold = 5 

134 config.nPixBorderUpDown = 0 

135 config.nPixBorderLeftRight = 0 

136 

137 task = self.defaultTask 

138 task.config = config 

139 

140 defectsWithColumns = task.maskBlocksIfIntermitentBadPixelsInColumn(inputDefects) 

141 boxesMeasured = [] 

142 for defect in defectsWithColumns: 

143 boxesMeasured.append(defect.getBBox()) 

144 

145 for boxInput in expectedDefects: 

146 self.assertIn(boxInput, boxesMeasured) 

147 

148 # Check that the code did not mask anything extra by 

149 # looking in both the input list and "expanded-column" list. 

150 unionInputExpectedBoxes = [] 

151 for defect in inputDefects: 

152 unionInputExpectedBoxes.append(defect.getBBox()) 

153 for defect in expectedDefects: 

154 unionInputExpectedBoxes.append(defect) 

155 

156 # Check that code doesn't mask more than it is supposed to. 

157 for boxMeas in boxesMeasured: 

158 self.assertIn(boxMeas, unionInputExpectedBoxes) 

159 

160 def test_maskBlocks_full_column(self): 

161 """A test for maskBlocksIfIntermitentBadPixelsInColumn. 

162 

163 Tests that a contigous bad column does not get split by the 

164 code. 

165 

166 The mock flat has a size of 200X204 pixels. This column has a 

167 maximum length of 50 pixels, otherwise there would be a split 

168 along the mock amp boundary. 

169 

170 Plots can be found in DM-19903 on Jira. 

171 

172 """ 

173 

174 defects = self.allDefectsList 

175 defects.append(Box2I(corner=Point2I(15, 1), dimensions=Extent2I(1, 50))) 

176 expectedDefects = [Box2I(corner=Point2I(15, 1), dimensions=Extent2I(1, 50))] 

177 

178 self.check_maskBlocks(defects, expectedDefects) 

179 

180 def test_maskBlocks_long_column(self): 

181 """A test for maskBlocksIfIntermitentBadPixelsInColumn. 

182 

183 Tests that a contigous bad column with Npix >= 

184 badOnAndOffPixelColumnThreshold (10) does not get split by the 

185 code. 

186 

187 Plots can be found in DM-19903 on Jira. 

188 

189 """ 

190 

191 expectedDefects = [Box2I(corner=Point2I(20, 1), dimensions=Extent2I(1, 25))] 

192 defects = self.allDefectsList 

193 defects.append(Box2I(corner=Point2I(20, 1), dimensions=Extent2I(1, 25))) 

194 

195 self.check_maskBlocks(defects, expectedDefects) 

196 

197 def test_maskBlocks_short_column(self): 

198 """A test for maskBlocksIfIntermitentBadPixelsInColumn. 

199 

200 Tests that a contigous bad column Npix < 

201 badOnAndOffPixelColumnThreshold (10) does not get split by the 

202 code. 

203 

204 Plots can be found in DM-19903 on Jira. 

205 

206 """ 

207 

208 expectedDefects = [Box2I(corner=Point2I(25, 1), dimensions=Extent2I(1, 8))] 

209 defects = self.allDefectsList 

210 defects.append(Box2I(corner=Point2I(25, 1), dimensions=Extent2I(1, 8))) 

211 

212 self.check_maskBlocks(defects, expectedDefects) 

213 

214 def test_maskBlocks_discontigous_to_single_block(self): 

215 """A test for maskBlocksIfIntermitentBadPixelsInColumn. 

216 

217 Npix discontiguous bad pixels in a column where Npix >= 

218 badOnAndOffPixelColumnThreshold (10) and gaps of good pixels < 

219 goodPixelColumnGapThreshold (5). Under these conditions, the 

220 whole block of bad pixels (including good gaps) should be 

221 masked. 

222 

223 Plots can be found in DM-19903 on Jira. 

224 

225 """ 

226 

227 expectedDefects = [Box2I(corner=Point2I(30, 1), dimensions=Extent2I(1, 48))] 

228 defects = self.allDefectsList 

229 badPixels = [Box2I(corner=Point2I(30, 1), dimensions=Extent2I(1, 2)), 

230 Box2I(corner=Point2I(30, 5), dimensions=Extent2I(1, 3)), 

231 Box2I(corner=Point2I(30, 11), dimensions=Extent2I(1, 5)), 

232 Box2I(corner=Point2I(30, 19), dimensions=Extent2I(1, 5)), 

233 Box2I(corner=Point2I(30, 27), dimensions=Extent2I(1, 4)), 

234 Box2I(corner=Point2I(30, 34), dimensions=Extent2I(1, 15))] 

235 with defects.bulk_update(): 

236 for badBox in badPixels: 

237 defects.append(badBox) 

238 

239 self.check_maskBlocks(defects, expectedDefects) 

240 

241 def test_maskBlocks_discontigous_less_than_thresholds(self): 

242 """A test for maskBlocksIfIntermitentBadPixelsInColumn. 

243 

244 Npix discontiguous bad pixels in a column where Npix < 

245 badOnAndOffPixelColumnThreshold (10) and gaps of good pixels < 

246 goodPixelColumnGapThreshold (5). Under these conditions, the 

247 expected defect boxes should be the same as the input boxes. 

248 

249 Plots can be found in DM-19903 on Jira. 

250 

251 """ 

252 

253 expectedDefects = [Box2I(corner=Point2I(35, 1), dimensions=Extent2I(1, 2)), 

254 Box2I(corner=Point2I(35, 5), dimensions=Extent2I(1, 3)), 

255 Box2I(corner=Point2I(35, 11), dimensions=Extent2I(1, 2))] 

256 defects = self.allDefectsList 

257 badPixels = [Box2I(corner=Point2I(35, 1), dimensions=Extent2I(1, 2)), 

258 Box2I(corner=Point2I(35, 5), dimensions=Extent2I(1, 3)), 

259 Box2I(corner=Point2I(35, 11), dimensions=Extent2I(1, 2))] 

260 with defects.bulk_update(): 

261 for badBox in badPixels: 

262 defects.append(badBox) 

263 

264 self.check_maskBlocks(defects, expectedDefects) 

265 

266 def test_maskBlocks_more_than_thresholds(self): 

267 """A test for maskBlocksIfIntermitentBadPixelsInColumn. 

268 

269 Npix discontiguous bad pixels in a column where Npix < 

270 badOnAndOffPixelColumnThreshold (10) and gaps of good pixels < 

271 goodPixelColumnGapThreshold (5). Npix=34 (> 10) bad pixels 

272 total, 1 "good" gap with 13 pixels big enough (13 >= 5 good 

273 pixels, from y=6 (1+5) to y=19). 

274 

275 Plots can be found in DM-19903 on Jira. 

276 

277 """ 

278 

279 expectedDefects = [Box2I(corner=Point2I(40, 1), dimensions=Extent2I(1, 7)), 

280 Box2I(corner=Point2I(40, 19), dimensions=Extent2I(1, 30))] 

281 defects = self.allDefectsList 

282 badPixels = [Box2I(corner=Point2I(40, 1), dimensions=Extent2I(1, 2)), 

283 Box2I(corner=Point2I(40, 5), dimensions=Extent2I(1, 3)), 

284 Box2I(corner=Point2I(40, 19), dimensions=Extent2I(1, 5)), 

285 Box2I(corner=Point2I(40, 27), dimensions=Extent2I(1, 4)), 

286 Box2I(corner=Point2I(40, 34), dimensions=Extent2I(1, 15))] 

287 with defects.bulk_update(): 

288 for badBox in badPixels: 

289 defects.append(badBox) 

290 

291 self.check_maskBlocks(defects, expectedDefects) 

292 

293 def test_maskBlocks_not_enough_bad_pixels_in_column(self): 

294 """A test for maskBlocksIfIntermitentBadPixelsInColumn. 

295 

296 Npix discontiguous bad pixels in a column where Npix < 

297 badOnAndOffPixelColumnThreshold (10) and and gaps of good 

298 pixels > goodPixelColumnGapThreshold (5). Since Npix < 

299 badOnAndOffPixelColumnThreshold, then it doesn't matter that 

300 the number of good pixels in gap > 

301 goodPixelColumnGapThreshold. 5<10 bad pixels total, 1 "good" 

302 gap big enough (29>=5 good pixels, from y =12 (10+2) to y=30) 

303 

304 Plots can be found in DM-19903 on Jira. 

305 

306 """ 

307 

308 expectedDefects = [Box2I(corner=Point2I(45, 10), dimensions=Extent2I(1, 2)), 

309 Box2I(corner=Point2I(45, 30), dimensions=Extent2I(1, 3))] 

310 defects = self.allDefectsList 

311 badPixels = [Box2I(corner=Point2I(45, 10), dimensions=Extent2I(1, 2)), 

312 Box2I(corner=Point2I(45, 30), dimensions=Extent2I(1, 3))] 

313 with defects.bulk_update(): 

314 for badBox in badPixels: 

315 defects.append(badBox) 

316 

317 self.check_maskBlocks(defects, expectedDefects) 

318 

319 def test_maskBlocks_every_other_pixel_bad_greater_than_threshold(self): 

320 """A test for maskBlocksIfIntermitentBadPixelsInColumn. 

321 

322 Npix discontiguous bad pixels in a column where Npix > 

323 badOnAndOffPixelColumnThreshold (10) and every other pixel is 

324 bad. 

325 

326 Plots can be found in DM-19903 on Jira. 

327 

328 """ 

329 

330 expectedDefects = [Box2I(corner=Point2I(50, 10), dimensions=Extent2I(1, 31))] 

331 defects = self.allDefectsList 

332 badPixels = [Box2I(corner=Point2I(50, 10), dimensions=Extent2I(1, 1)), 

333 Box2I(corner=Point2I(50, 12), dimensions=Extent2I(1, 1)), 

334 Box2I(corner=Point2I(50, 14), dimensions=Extent2I(1, 1)), 

335 Box2I(corner=Point2I(50, 16), dimensions=Extent2I(1, 1)), 

336 Box2I(corner=Point2I(50, 18), dimensions=Extent2I(1, 1)), 

337 Box2I(corner=Point2I(50, 20), dimensions=Extent2I(1, 1)), 

338 Box2I(corner=Point2I(50, 22), dimensions=Extent2I(1, 1)), 

339 Box2I(corner=Point2I(50, 24), dimensions=Extent2I(1, 1)), 

340 Box2I(corner=Point2I(50, 26), dimensions=Extent2I(1, 1)), 

341 Box2I(corner=Point2I(50, 28), dimensions=Extent2I(1, 1)), 

342 Box2I(corner=Point2I(50, 30), dimensions=Extent2I(1, 1)), 

343 Box2I(corner=Point2I(50, 32), dimensions=Extent2I(1, 1)), 

344 Box2I(corner=Point2I(50, 34), dimensions=Extent2I(1, 1)), 

345 Box2I(corner=Point2I(50, 36), dimensions=Extent2I(1, 1)), 

346 Box2I(corner=Point2I(50, 38), dimensions=Extent2I(1, 1)), 

347 Box2I(corner=Point2I(50, 40), dimensions=Extent2I(1, 1))] 

348 with defects.bulk_update(): 

349 for badBox in badPixels: 

350 defects.append(badBox) 

351 

352 self.check_maskBlocks(defects, expectedDefects) 

353 

354 def test_maskBlocks_every_other_pixel_bad_less_than_threshold(self): 

355 """A test for maskBlocksIfIntermitentBadPixelsInColumn. 

356 

357 Npix discontiguous bad pixels in a column where Npix > 

358 badOnAndOffPixelColumnThreshold (10) and every other pixel is 

359 bad. 

360 

361 Plots can be found in DM-19903 on Jira. 

362 

363 """ 

364 

365 expectedDefects = [Box2I(corner=Point2I(55, 20), dimensions=Extent2I(1, 1)), 

366 Box2I(corner=Point2I(55, 22), dimensions=Extent2I(1, 1)), 

367 Box2I(corner=Point2I(55, 24), dimensions=Extent2I(1, 1)), 

368 Box2I(corner=Point2I(55, 26), dimensions=Extent2I(1, 1)), 

369 Box2I(corner=Point2I(55, 28), dimensions=Extent2I(1, 1)), 

370 Box2I(corner=Point2I(55, 30), dimensions=Extent2I(1, 1))] 

371 defects = self.allDefectsList 

372 badPixels = [Box2I(corner=Point2I(55, 20), dimensions=Extent2I(1, 1)), 

373 Box2I(corner=Point2I(55, 22), dimensions=Extent2I(1, 1)), 

374 Box2I(corner=Point2I(55, 24), dimensions=Extent2I(1, 1)), 

375 Box2I(corner=Point2I(55, 26), dimensions=Extent2I(1, 1)), 

376 Box2I(corner=Point2I(55, 28), dimensions=Extent2I(1, 1)), 

377 Box2I(corner=Point2I(55, 30), dimensions=Extent2I(1, 1))] 

378 with defects.bulk_update(): 

379 for badBox in badPixels: 

380 defects.append(badBox) 

381 

382 self.check_maskBlocks(defects, expectedDefects) 

383 

384 def test_maskBlocks_blobs_one_side_good_less_than_threshold(self): 

385 """A test for maskBlocksIfIntermitentBadPixelsInColumn. 

386 

387 Npix discontiguous bad pixels in column with "blobs" of "m" 

388 bad pixels to one side, m > badOnAndOffPixelColumnThreshold 

389 (10), number of good pixel in gaps between blobs < 

390 goodPixelColumnGapThreshold (5). 

391 

392 Plots can be found in DM-19903 on Jira. 

393 

394 """ 

395 

396 expectedDefects = [Box2I(corner=Point2I(60, 1), dimensions=Extent2I(1, 29)), 

397 Box2I(corner=Point2I(61, 2), dimensions=Extent2I(2, 12))] 

398 defects = self.allDefectsList 

399 badPixels = [Box2I(corner=Point2I(60, 1), dimensions=Extent2I(1, 18)), 

400 Box2I(corner=Point2I(60, 20), dimensions=Extent2I(1, 10)), 

401 Box2I(corner=Point2I(61, 2), dimensions=Extent2I(2, 2)), 

402 Box2I(corner=Point2I(61, 6), dimensions=Extent2I(2, 8))] 

403 with defects.bulk_update(): 

404 for badBox in badPixels: 

405 defects.append(badBox) 

406 

407 self.check_maskBlocks(defects, expectedDefects) 

408 

409 def test_maskBlocks_blobs_other_side_good_less_than_threshold(self): 

410 """A test for maskBlocksIfIntermitentBadPixelsInColumn. 

411 

412 Npix discontiguous bad pixels in column with "blobs" of "m" 

413 bad pixels to the other side, m > 

414 badOnAndOffPixelColumnThreshold (10), number of good pixel in 

415 gaps between blobs < goodPixelColumnGapThreshold (5). 

416 

417 Plots can be found in DM-19903 on Jira. 

418 

419 """ 

420 

421 expectedDefects = [Box2I(corner=Point2I(70, 1), dimensions=Extent2I(1, 29)), 

422 Box2I(corner=Point2I(68, 2), dimensions=Extent2I(2, 12))] 

423 defects = self.allDefectsList 

424 badPixels = [Box2I(corner=Point2I(70, 1), dimensions=Extent2I(1, 18)), 

425 Box2I(corner=Point2I(70, 20), dimensions=Extent2I(1, 10)), 

426 Box2I(corner=Point2I(68, 2), dimensions=Extent2I(2, 2)), 

427 Box2I(corner=Point2I(68, 6), dimensions=Extent2I(2, 8))] 

428 with defects.bulk_update(): 

429 for badBox in badPixels: 

430 defects.append(badBox) 

431 

432 self.check_maskBlocks(defects, expectedDefects) 

433 

434 def test_maskBlocks_blob_both_sides_good_less_than_threshold(self): 

435 """A test for maskBlocksIfIntermitentBadPixelsInColumn. 

436 

437 Npix discontiguous bad pixels in column with "blobs" of "m" 

438 bad pixels to both sides, m > badOnAndOffPixelColumnThreshold 

439 (10), number of good pixel in gaps between blobs < 

440 goodPixelColumnGapThreshold (5). 

441 

442 Plots can be found in DM-19903 on Jira. 

443 

444 """ 

445 

446 expectedDefects = [Box2I(corner=Point2I(75, 1), dimensions=Extent2I(1, 29)), 

447 Box2I(corner=Point2I(73, 2), dimensions=Extent2I(2, 12)), 

448 Box2I(corner=Point2I(76, 2), dimensions=Extent2I(2, 12))] 

449 defects = self.allDefectsList 

450 badPixels = [Box2I(corner=Point2I(75, 1), dimensions=Extent2I(1, 18)), 

451 Box2I(corner=Point2I(75, 20), dimensions=Extent2I(1, 10)), 

452 Box2I(corner=Point2I(73, 2), dimensions=Extent2I(2, 2)), 

453 Box2I(corner=Point2I(73, 6), dimensions=Extent2I(2, 8)), 

454 Box2I(corner=Point2I(76, 2), dimensions=Extent2I(2, 2)), 

455 Box2I(corner=Point2I(76, 6), dimensions=Extent2I(2, 8))] 

456 with defects.bulk_update(): 

457 for badBox in badPixels: 

458 defects.append(badBox) 

459 

460 self.check_maskBlocks(defects, expectedDefects) 

461 

462 def test_maskBlocks_blob_one_side_good_greater_than_threshold(self): 

463 """A test for maskBlocksIfIntermitentBadPixelsInColumn. 

464 

465 Npix discontiguous bad pixels in column with "blobs" of "m" 

466 bad pixels to one side, m > badOnAndOffPixelColumnThreshold 

467 (10), number of good pixel in gaps between blobs > 

468 goodPixelColumnGapThreshold (5). 

469 

470 Plots can be found in DM-19903 on Jira. 

471 

472 """ 

473 

474 expectedDefects = [Box2I(corner=Point2I(80, 1), dimensions=Extent2I(1, 29)), 

475 Box2I(corner=Point2I(81, 2), dimensions=Extent2I(2, 2)), 

476 Box2I(corner=Point2I(81, 8), dimensions=Extent2I(2, 8))] 

477 defects = self.allDefectsList 

478 badPixels = [Box2I(corner=Point2I(80, 1), dimensions=Extent2I(1, 18)), 

479 Box2I(corner=Point2I(80, 20), dimensions=Extent2I(1, 10)), 

480 Box2I(corner=Point2I(81, 2), dimensions=Extent2I(2, 2)), 

481 Box2I(corner=Point2I(81, 8), dimensions=Extent2I(2, 8))] 

482 with defects.bulk_update(): 

483 for badBox in badPixels: 

484 defects.append(badBox) 

485 

486 self.check_maskBlocks(defects, expectedDefects) 

487 

488 def test_maskBlocks_other_side_good_greater_than_threshold(self): 

489 """A test for maskBlocksIfIntermitentBadPixelsInColumn. 

490 

491 Npix discontiguous bad pixels in column with "blobs" of "m" 

492 bad pixels to the other side, m > 

493 badOnAndOffPixelColumnThreshold (10), number of good pixel in 

494 gaps between blobs > goodPixelColumnGapThreshold (5). 

495 

496 Plots can be found in DM-19903 on Jira. 

497 

498 """ 

499 

500 expectedDefects = [Box2I(corner=Point2I(87, 1), dimensions=Extent2I(1, 29)), 

501 Box2I(corner=Point2I(85, 2), dimensions=Extent2I(2, 2)), 

502 Box2I(corner=Point2I(85, 8), dimensions=Extent2I(2, 8))] 

503 defects = self.allDefectsList 

504 badPixels = [Box2I(corner=Point2I(87, 1), dimensions=Extent2I(1, 18)), 

505 Box2I(corner=Point2I(87, 20), dimensions=Extent2I(1, 10)), 

506 Box2I(corner=Point2I(85, 2), dimensions=Extent2I(2, 2)), 

507 Box2I(corner=Point2I(85, 8), dimensions=Extent2I(2, 8))] 

508 with defects.bulk_update(): 

509 for badBox in badPixels: 

510 defects.append(badBox) 

511 

512 self.check_maskBlocks(defects, expectedDefects) 

513 

514 def test_maskBlocks_both_sides_good_greater_than_threshold(self): 

515 """A test for maskBlocksIfIntermitentBadPixelsInColumn. 

516 

517 Npix discontiguous bad pixels in column with "blobs" of "m" 

518 bad pixels to both sides, m > badOnAndOffPixelColumnThreshold 

519 (10), number of good pixel in gaps between blobs > 

520 goodPixelColumnGapThreshold (5). 

521 

522 Plots can be found in DM-19903 on Jira. 

523 

524 """ 

525 

526 expectedDefects = [Box2I(corner=Point2I(93, 1), dimensions=Extent2I(1, 34)), 

527 Box2I(corner=Point2I(91, 2), dimensions=Extent2I(2, 7)), 

528 Box2I(corner=Point2I(91, 18), dimensions=Extent2I(2, 9)), 

529 Box2I(corner=Point2I(94, 2), dimensions=Extent2I(2, 7)), 

530 Box2I(corner=Point2I(94, 18), dimensions=Extent2I(2, 9))] 

531 defects = self.allDefectsList 

532 badPixels = [Box2I(corner=Point2I(93, 1), dimensions=Extent2I(1, 12)), 

533 Box2I(corner=Point2I(93, 15), dimensions=Extent2I(1, 20)), 

534 Box2I(corner=Point2I(91, 2), dimensions=Extent2I(2, 2)), 

535 Box2I(corner=Point2I(91, 7), dimensions=Extent2I(2, 2)), 

536 Box2I(corner=Point2I(94, 2), dimensions=Extent2I(2, 2)), 

537 Box2I(corner=Point2I(94, 7), dimensions=Extent2I(2, 2)), 

538 Box2I(corner=Point2I(91, 18), dimensions=Extent2I(2, 3)), 

539 Box2I(corner=Point2I(91, 24), dimensions=Extent2I(2, 3)), 

540 Box2I(corner=Point2I(94, 18), dimensions=Extent2I(2, 3)), 

541 Box2I(corner=Point2I(94, 24), dimensions=Extent2I(2, 3))] 

542 with defects.bulk_update(): 

543 for badBox in badPixels: 

544 defects.append(badBox) 

545 

546 self.check_maskBlocks(defects, expectedDefects) 

547 

548 def test_maskBlocks_y_out_of_order_dm38103(self): 

549 """A test for maskBlocksIfIntermitentBadPixelsInColumn, y out of order. 

550 

551 This test is a variant of 

552 notest_maskBlocks_every_other_pixel_bad_greater_than_threshold with 

553 an extra out-of-y-order bad pixel to trigger DM-38103. 

554 """ 

555 expectedDefects = [Box2I(corner=Point2I(50, 110), dimensions=Extent2I(1, 31))] 

556 defects = self.allDefectsList 

557 badPixels = [Box2I(corner=Point2I(50, 110), dimensions=Extent2I(1, 1)), 

558 Box2I(corner=Point2I(50, 112), dimensions=Extent2I(1, 1)), 

559 Box2I(corner=Point2I(50, 114), dimensions=Extent2I(1, 1)), 

560 Box2I(corner=Point2I(50, 116), dimensions=Extent2I(1, 1)), 

561 Box2I(corner=Point2I(50, 118), dimensions=Extent2I(1, 1)), 

562 Box2I(corner=Point2I(50, 120), dimensions=Extent2I(1, 1)), 

563 Box2I(corner=Point2I(50, 122), dimensions=Extent2I(1, 1)), 

564 Box2I(corner=Point2I(50, 124), dimensions=Extent2I(1, 1)), 

565 Box2I(corner=Point2I(50, 126), dimensions=Extent2I(1, 1)), 

566 Box2I(corner=Point2I(50, 128), dimensions=Extent2I(1, 1)), 

567 Box2I(corner=Point2I(50, 130), dimensions=Extent2I(1, 1)), 

568 Box2I(corner=Point2I(50, 132), dimensions=Extent2I(1, 1)), 

569 Box2I(corner=Point2I(50, 134), dimensions=Extent2I(1, 1)), 

570 Box2I(corner=Point2I(50, 136), dimensions=Extent2I(1, 1)), 

571 Box2I(corner=Point2I(50, 138), dimensions=Extent2I(1, 1)), 

572 Box2I(corner=Point2I(50, 140), dimensions=Extent2I(1, 1)), 

573 # This last point is out of order in y. 

574 Box2I(corner=Point2I(50, 100), dimensions=Extent2I(1, 1))] 

575 

576 # Force defect normalization off in order to trigger DM-38301, because 

577 # defects.fromFootprintList() which is called by _findHotAndColdPixels 

578 # does not do normalization. 

579 defects._bulk_update = True 

580 for badBox in badPixels: 

581 defects.append(badBox) 

582 defects._bulk_update = False 

583 

584 self.check_maskBlocks(defects, expectedDefects) 

585 

586 def test_defectFindingAllSensor(self): 

587 config = copy.copy(self.defaultConfig) 

588 config.nPixBorderLeftRight = 0 

589 config.nPixBorderUpDown = 0 

590 

591 task = self.defaultTask 

592 task.config = config 

593 

594 defects = task._findHotAndColdPixels(self.flatExp) 

595 

596 allBBoxes = self.darkBBoxes + self.brightBBoxes 

597 

598 boxesMeasured = [] 

599 for defect in defects: 

600 boxesMeasured.append(defect.getBBox()) 

601 

602 for expectedBBox in allBBoxes: 

603 self.assertIn(expectedBBox, boxesMeasured) 

604 

605 def test_defectFindingEdgeIgnore(self): 

606 config = copy.copy(self.defaultConfig) 

607 config.nPixBorderUpDown = 0 

608 task = self.defaultTask 

609 task.config = config 

610 defects = task._findHotAndColdPixels(self.flatExp) 

611 

612 shouldBeFound = self.darkBBoxes[self.noEdges] + self.brightBBoxes[self.noEdges] 

613 

614 boxesMeasured = [] 

615 for defect in defects: 

616 boxesMeasured.append(defect.getBBox()) 

617 

618 for expectedBBox in shouldBeFound: 

619 self.assertIn(expectedBBox, boxesMeasured) 

620 

621 shouldBeMissed = self.darkBBoxes[self.onlyEdges] + self.brightBBoxes[self.onlyEdges] 

622 for boxMissed in shouldBeMissed: 

623 self.assertNotIn(boxMissed, boxesMeasured) 

624 

625 def valueThreshold(self, fileType): 

626 """Helper function to loop over flats and darks 

627 to test thresholdType = 'VALUE'.""" 

628 config = copy.copy(self.defaultConfig) 

629 config.thresholdType = 'VALUE' 

630 task = self.defaultTask 

631 task.config = config 

632 

633 if fileType == 'dark': 

634 exp = self.darkExp 

635 shouldBeFound = self.brightBBoxes[self.noEdges] 

636 else: 

637 exp = self.flatExp 

638 shouldBeFound = self.darkBBoxes[self.noEdges] 

639 # Change the default a bit so it works for the 

640 # existing simulated defects. 

641 task.config.fracThresholdFlat = 0.9 

642 

643 defects = task._findHotAndColdPixels(exp) 

644 

645 boxesMeasured = [] 

646 for defect in defects: 

647 boxesMeasured.append(defect.getBBox()) 

648 

649 for expectedBBox in shouldBeFound: 

650 self.assertIn(expectedBBox, boxesMeasured) 

651 

652 def test_valueThreshold(self): 

653 for fileType in ['flat', 'flat']: 

654 self.valueThreshold(fileType) 

655 

656 def test_pixelCounting(self): 

657 """Test that the number of defective pixels identified is as expected. 

658 """ 

659 config = copy.copy(self.defaultConfig) 

660 config.nPixBorderUpDown = 0 

661 config.nPixBorderLeftRight = 0 

662 task = self.defaultTask 

663 task.config = config 

664 defects = task._findHotAndColdPixels(self.flatExp) 

665 

666 defectArea = 0 

667 for defect in defects: 

668 defectArea += defect.getBBox().getArea() 

669 

670 # The columnar code will cover blocks of a column with 

671 # on-and-off pixels, thus creating more bad pixels that what 

672 # initially placed in self.brightDefects and self.darkDefects. 

673 # Thus, defectArea should be >= crossCheck. 

674 crossCheck = 0 

675 for x, y, sx, sy in self.brightDefects: 

676 crossCheck += sx*sy 

677 for x, y, sx, sy in self.darkDefects: 

678 crossCheck += sx*sy 

679 

680 # Test the result of _nPixFromDefects() 

681 # via two different ways of calculating area. 

682 self.assertEqual(defectArea, task._nPixFromDefects(defects)) 

683 # defectArea should be >= crossCheck 

684 self.assertGreaterEqual(defectArea, crossCheck) 

685 

686 def test_getNumGoodPixels(self): 

687 """Test the the number of pixels in the image not masked is as 

688 expected. 

689 """ 

690 testImage = self.flatExp.clone() 

691 mi = testImage.maskedImage 

692 

693 imageSize = testImage.getBBox().getArea() 

694 nGood = self.defaultTask._getNumGoodPixels(mi) 

695 

696 self.assertEqual(imageSize, nGood) 

697 

698 NODATABIT = mi.mask.getPlaneBitMask("NO_DATA") 

699 

700 noDataBox = Box2I(Point2I(31, 49), Extent2I(3, 6)) 

701 testImage.mask[noDataBox] |= NODATABIT 

702 

703 self.assertEqual(imageSize - noDataBox.getArea(), self.defaultTask._getNumGoodPixels(mi)) 

704 # check for misfire; we're setting NO_DATA here, not BAD 

705 self.assertEqual(imageSize, self.defaultTask._getNumGoodPixels(mi, 'BAD')) 

706 

707 testImage.mask[noDataBox] ^= NODATABIT # XOR to reset what we did 

708 self.assertEqual(imageSize, nGood) 

709 

710 BADBIT = mi.mask.getPlaneBitMask("BAD") 

711 badBox = Box2I(Point2I(85, 98), Extent2I(4, 7)) 

712 testImage.mask[badBox] |= BADBIT 

713 

714 self.assertEqual(imageSize - badBox.getArea(), self.defaultTask._getNumGoodPixels(mi, 'BAD')) 

715 

716 def test_edgeMasking(self): 

717 """Check that the right number of edge pixels are masked by 

718 _setEdgeBits(). 

719 """ 

720 testImage = self.flatExp.clone() 

721 mi = testImage.maskedImage 

722 

723 self.assertEqual(countMaskedPixels(mi, 'EDGE'), 0) 

724 self.defaultTask._setEdgeBits(mi) 

725 

726 hEdge = self.defaultConfig.nPixBorderLeftRight 

727 vEdge = self.defaultConfig.nPixBorderUpDown 

728 xSize, ySize = mi.getDimensions() 

729 

730 nEdge = xSize*vEdge*2 + ySize*hEdge*2 - hEdge*vEdge*4 

731 

732 self.assertEqual(countMaskedPixels(mi, 'EDGE'), nEdge) 

733 

734 def test_badImage(self): 

735 """Check that fully-bad images do not fail. 

736 """ 

737 testImage = self.flatExp.clone() 

738 testImage.image.array[:, :] = 125000 

739 

740 config = copy.copy(self.defaultConfig) 

741 # Do not exclude any pixels, so the areas match. 

742 config.nPixBorderUpDown = 0 

743 config.nPixBorderLeftRight = 0 

744 

745 task = self.defaultTask 

746 task.config = config 

747 defects = task._findHotAndColdPixels(testImage) 

748 

749 defectArea = 0 

750 for defect in defects: 

751 defectArea += defect.getBBox().getArea() 

752 self.assertEqual(defectArea, testImage.getBBox().getArea()) 

753 

754 

755class TestMemory(lsst.utils.tests.MemoryTestCase): 

756 pass 

757 

758 

759def setup_module(module): 

760 lsst.utils.tests.init() 

761 

762 

763if __name__ == "__main__": 763 ↛ 764line 763 didn't jump to line 764, because the condition on line 763 was never true

764 lsst.utils.tests.init() 

765 unittest.main()