Coverage for tests/test_defects.py: 9%

414 statements  

« prev     ^ index     » next       coverage.py v7.5.1, created at 2024-05-15 02:24 -0700

1# 

2# LSST Data Management System 

3# 

4# Copyright 2008-2017 AURA/LSST. 

5# 

6# This product includes software developed by the 

7# LSST Project (http://www.lsst.org/). 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the LSST License Statement and 

20# the GNU General Public License along with this program. If not, 

21# see <https://www.lsstcorp.org/LegalNotices/>. 

22# 

23"""Test cases for lsst.cp.pipe.defects.MeasureDefectsTask.""" 

24 

25import unittest 

26import numpy as np 

27import copy 

28 

29import lsst.utils 

30import lsst.utils.tests 

31 

32import lsst.afw.image as afwImage 

33import lsst.ip.isr as ipIsr 

34import lsst.cp.pipe as cpPipe 

35from lsst.ip.isr import isrMock, countMaskedPixels 

36from lsst.geom import Box2I, Point2I, Extent2I 

37from lsst.daf.base import PropertyList 

38 

39 

40class MeasureDefectsTaskTestCase(lsst.utils.tests.TestCase): 

41 """A test case for the defect finding task.""" 

42 

43 def setUp(self): 

44 self.defaultConfig = cpPipe.defects.MeasureDefectsTask.ConfigClass() 

45 

46 self.flatMean = 2000 

47 self.darkMean = 1 

48 self.readNoiseAdu = 10 

49 self.nSigmaBright = 8 

50 self.nSigmaDark = 8 

51 

52 mockImageConfig = isrMock.IsrMock.ConfigClass() 

53 

54 # flatDrop is not really relevant as we replace the data 

55 # but good to note it in case we change how this image is made 

56 mockImageConfig.flatDrop = 0.99999 

57 mockImageConfig.isTrimmed = True 

58 

59 self.flatExp = isrMock.FlatMock(config=mockImageConfig).run() 

60 (shapeY, shapeX) = self.flatExp.getDimensions() 

61 # x, y, size tuples 

62 # always put edge defects at the start and change the value of nEdge 

63 

64 self.brightDefects = [(0, 15, 3, 3), (100, 123, 1, 1)] 

65 

66 self.darkDefects = [(5, 0, 1, 1), (7, 62, 2, 2)] 

67 

68 nEdge = 1 # NOTE: update if more edge defects are included 

69 self.noEdges = slice(nEdge, None) 

70 self.onlyEdges = slice(0, nEdge) 

71 

72 self.darkBBoxes = [Box2I(Point2I(x, y), Extent2I(sx, sy)) for (x, y, sx, sy) in self.darkDefects] 

73 self.brightBBoxes = [Box2I(Point2I(x, y), Extent2I(sx, sy)) for (x, y, sx, sy) in self.brightDefects] 

74 

75 flatWidth = np.sqrt(self.flatMean) + self.readNoiseAdu 

76 darkWidth = self.readNoiseAdu 

77 self.rng = np.random.RandomState(0) 

78 flatData = self.rng.normal(self.flatMean, flatWidth, (shapeX, shapeY)) 

79 darkData = self.rng.normal(self.darkMean, darkWidth, (shapeX, shapeY)) 

80 

81 # NOTE: darks and flats have same defects applied deliberately to both 

82 for defect in self.brightDefects: 

83 y, x, sy, sx = defect 

84 # are these actually the numbers we want? 

85 flatData[x:x+sx, y:y+sy] += self.nSigmaBright * flatWidth 

86 darkData[x:x+sx, y:y+sy] += self.nSigmaBright * darkWidth 

87 

88 for defect in self.darkDefects: 

89 y, x, sy, sx = defect 

90 # are these actually the numbers we want? 

91 flatData[x:x+sx, y:y+sy] -= self.nSigmaDark * flatWidth 

92 darkData[x:x+sx, y:y+sy] -= self.nSigmaDark * darkWidth 

93 

94 self.darkExp = self.flatExp.clone() 

95 self.spareImage = self.flatExp.clone() # for testing edge bits and misc 

96 

97 self.flatExp.image.array[:] = flatData 

98 self.darkExp.image.array[:] = darkData 

99 

100 self.defaultTask = cpPipe.defects.MeasureDefectsTask() 

101 

102 self.allDefectsList = ipIsr.Defects() 

103 self.brightDefectsList = ipIsr.Defects() 

104 self.darkDefectsList = ipIsr.Defects() 

105 

106 # Set image types, the defects code will use them. 

107 metaDataFlat = PropertyList() 

108 metaDataFlat["IMGTYPE"] = "FLAT" 

109 self.flatExp.setMetadata(metaDataFlat) 

110 

111 metaDataDark = PropertyList() 

112 metaDataDark["IMGTYPE"] = "DARK" 

113 self.darkExp.setMetadata(metaDataDark) 

114 

115 with self.allDefectsList.bulk_update(): 

116 with self.brightDefectsList.bulk_update(): 

117 for d in self.brightBBoxes: 

118 self.brightDefectsList.append(d) 

119 self.allDefectsList.append(d) 

120 

121 with self.darkDefectsList.bulk_update(): 

122 for d in self.darkBBoxes: 

123 self.darkDefectsList.append(d) 

124 self.allDefectsList.append(d) 

125 

126 def check_maskBlocks(self, inputDefects, expectedDefects): 

127 """A helper function for the tests of 

128 maskBlocksIfIntermitentBadPixelsInColumn. 

129 

130 """ 

131 config = copy.copy(self.defaultConfig) 

132 config.badOnAndOffPixelColumnThreshold = 10 

133 config.goodPixelColumnGapThreshold = 5 

134 config.nPixBorderUpDown = 0 

135 config.nPixBorderLeftRight = 0 

136 

137 task = self.defaultTask 

138 task.config = config 

139 

140 defectsWithColumns, count = task.maskBlocksIfIntermitentBadPixelsInColumn(inputDefects) 

141 boxesMeasured = [] 

142 for defect in defectsWithColumns: 

143 boxesMeasured.append(defect.getBBox()) 

144 

145 for boxInput in expectedDefects: 

146 self.assertIn(boxInput, boxesMeasured) 

147 

148 # Check that the code did not mask anything extra by 

149 # looking in both the input list and "expanded-column" list. 

150 unionInputExpectedBoxes = [] 

151 for defect in inputDefects: 

152 unionInputExpectedBoxes.append(defect.getBBox()) 

153 for defect in expectedDefects: 

154 unionInputExpectedBoxes.append(defect) 

155 

156 # Check that code doesn't mask more than it is supposed to. 

157 for boxMeas in boxesMeasured: 

158 self.assertIn(boxMeas, unionInputExpectedBoxes) 

159 

160 def test_maskBlocks_full_column(self): 

161 """A test for maskBlocksIfIntermitentBadPixelsInColumn. 

162 

163 Tests that a contigous bad column does not get split by the 

164 code. 

165 

166 The mock flat has a size of 200X204 pixels. This column has a 

167 maximum length of 50 pixels, otherwise there would be a split 

168 along the mock amp boundary. 

169 

170 Plots can be found in DM-19903 on Jira. 

171 

172 """ 

173 

174 defects = self.allDefectsList 

175 defects.append(Box2I(corner=Point2I(15, 1), dimensions=Extent2I(1, 50))) 

176 expectedDefects = [Box2I(corner=Point2I(15, 1), dimensions=Extent2I(1, 50))] 

177 

178 self.check_maskBlocks(defects, expectedDefects) 

179 

180 def test_maskBlocks_long_column(self): 

181 """A test for maskBlocksIfIntermitentBadPixelsInColumn. 

182 

183 Tests that a contigous bad column with Npix >= 

184 badOnAndOffPixelColumnThreshold (10) does not get split by the 

185 code. 

186 

187 Plots can be found in DM-19903 on Jira. 

188 

189 """ 

190 

191 expectedDefects = [Box2I(corner=Point2I(20, 1), dimensions=Extent2I(1, 25))] 

192 defects = self.allDefectsList 

193 defects.append(Box2I(corner=Point2I(20, 1), dimensions=Extent2I(1, 25))) 

194 

195 self.check_maskBlocks(defects, expectedDefects) 

196 

197 def test_maskBlocks_short_column(self): 

198 """A test for maskBlocksIfIntermitentBadPixelsInColumn. 

199 

200 Tests that a contigous bad column Npix < 

201 badOnAndOffPixelColumnThreshold (10) does not get split by the 

202 code. 

203 

204 Plots can be found in DM-19903 on Jira. 

205 

206 """ 

207 

208 expectedDefects = [Box2I(corner=Point2I(25, 1), dimensions=Extent2I(1, 8))] 

209 defects = self.allDefectsList 

210 defects.append(Box2I(corner=Point2I(25, 1), dimensions=Extent2I(1, 8))) 

211 

212 self.check_maskBlocks(defects, expectedDefects) 

213 

214 def test_maskBlocks_discontigous_to_single_block(self): 

215 """A test for maskBlocksIfIntermitentBadPixelsInColumn. 

216 

217 Npix discontiguous bad pixels in a column where Npix >= 

218 badOnAndOffPixelColumnThreshold (10) and gaps of good pixels < 

219 goodPixelColumnGapThreshold (5). Under these conditions, the 

220 whole block of bad pixels (including good gaps) should be 

221 masked. 

222 

223 Plots can be found in DM-19903 on Jira. 

224 

225 """ 

226 

227 expectedDefects = [Box2I(corner=Point2I(30, 1), dimensions=Extent2I(1, 48))] 

228 defects = self.allDefectsList 

229 badPixels = [Box2I(corner=Point2I(30, 1), dimensions=Extent2I(1, 2)), 

230 Box2I(corner=Point2I(30, 5), dimensions=Extent2I(1, 3)), 

231 Box2I(corner=Point2I(30, 11), dimensions=Extent2I(1, 5)), 

232 Box2I(corner=Point2I(30, 19), dimensions=Extent2I(1, 5)), 

233 Box2I(corner=Point2I(30, 27), dimensions=Extent2I(1, 4)), 

234 Box2I(corner=Point2I(30, 34), dimensions=Extent2I(1, 15))] 

235 with defects.bulk_update(): 

236 for badBox in badPixels: 

237 defects.append(badBox) 

238 

239 self.check_maskBlocks(defects, expectedDefects) 

240 

241 def test_maskBlocks_discontigous_less_than_thresholds(self): 

242 """A test for maskBlocksIfIntermitentBadPixelsInColumn. 

243 

244 Npix discontiguous bad pixels in a column where Npix < 

245 badOnAndOffPixelColumnThreshold (10) and gaps of good pixels < 

246 goodPixelColumnGapThreshold (5). Under these conditions, the 

247 expected defect boxes should be the same as the input boxes. 

248 

249 Plots can be found in DM-19903 on Jira. 

250 

251 """ 

252 

253 expectedDefects = [Box2I(corner=Point2I(35, 1), dimensions=Extent2I(1, 2)), 

254 Box2I(corner=Point2I(35, 5), dimensions=Extent2I(1, 3)), 

255 Box2I(corner=Point2I(35, 11), dimensions=Extent2I(1, 2))] 

256 defects = self.allDefectsList 

257 badPixels = [Box2I(corner=Point2I(35, 1), dimensions=Extent2I(1, 2)), 

258 Box2I(corner=Point2I(35, 5), dimensions=Extent2I(1, 3)), 

259 Box2I(corner=Point2I(35, 11), dimensions=Extent2I(1, 2))] 

260 with defects.bulk_update(): 

261 for badBox in badPixels: 

262 defects.append(badBox) 

263 

264 self.check_maskBlocks(defects, expectedDefects) 

265 

266 def test_maskBlocks_more_than_thresholds(self): 

267 """A test for maskBlocksIfIntermitentBadPixelsInColumn. 

268 

269 Npix discontiguous bad pixels in a column where Npix < 

270 badOnAndOffPixelColumnThreshold (10) and gaps of good pixels < 

271 goodPixelColumnGapThreshold (5). Npix=34 (> 10) bad pixels 

272 total, 1 "good" gap with 13 pixels big enough (13 >= 5 good 

273 pixels, from y=6 (1+5) to y=19). 

274 

275 Plots can be found in DM-19903 on Jira. 

276 

277 """ 

278 

279 expectedDefects = [Box2I(corner=Point2I(40, 1), dimensions=Extent2I(1, 7)), 

280 Box2I(corner=Point2I(40, 19), dimensions=Extent2I(1, 30))] 

281 defects = self.allDefectsList 

282 badPixels = [Box2I(corner=Point2I(40, 1), dimensions=Extent2I(1, 2)), 

283 Box2I(corner=Point2I(40, 5), dimensions=Extent2I(1, 3)), 

284 Box2I(corner=Point2I(40, 19), dimensions=Extent2I(1, 5)), 

285 Box2I(corner=Point2I(40, 27), dimensions=Extent2I(1, 4)), 

286 Box2I(corner=Point2I(40, 34), dimensions=Extent2I(1, 15))] 

287 with defects.bulk_update(): 

288 for badBox in badPixels: 

289 defects.append(badBox) 

290 

291 self.check_maskBlocks(defects, expectedDefects) 

292 

293 def test_maskBlocks_not_enough_bad_pixels_in_column(self): 

294 """A test for maskBlocksIfIntermitentBadPixelsInColumn. 

295 

296 Npix discontiguous bad pixels in a column where Npix < 

297 badOnAndOffPixelColumnThreshold (10) and and gaps of good 

298 pixels > goodPixelColumnGapThreshold (5). Since Npix < 

299 badOnAndOffPixelColumnThreshold, then it doesn't matter that 

300 the number of good pixels in gap > 

301 goodPixelColumnGapThreshold. 5<10 bad pixels total, 1 "good" 

302 gap big enough (29>=5 good pixels, from y =12 (10+2) to y=30) 

303 

304 Plots can be found in DM-19903 on Jira. 

305 

306 """ 

307 

308 expectedDefects = [Box2I(corner=Point2I(45, 10), dimensions=Extent2I(1, 2)), 

309 Box2I(corner=Point2I(45, 30), dimensions=Extent2I(1, 3))] 

310 defects = self.allDefectsList 

311 badPixels = [Box2I(corner=Point2I(45, 10), dimensions=Extent2I(1, 2)), 

312 Box2I(corner=Point2I(45, 30), dimensions=Extent2I(1, 3))] 

313 with defects.bulk_update(): 

314 for badBox in badPixels: 

315 defects.append(badBox) 

316 

317 self.check_maskBlocks(defects, expectedDefects) 

318 

319 def test_maskBlocks_every_other_pixel_bad_greater_than_threshold(self): 

320 """A test for maskBlocksIfIntermitentBadPixelsInColumn. 

321 

322 Npix discontiguous bad pixels in a column where Npix > 

323 badOnAndOffPixelColumnThreshold (10) and every other pixel is 

324 bad. 

325 

326 Plots can be found in DM-19903 on Jira. 

327 

328 """ 

329 

330 expectedDefects = [Box2I(corner=Point2I(50, 10), dimensions=Extent2I(1, 31))] 

331 defects = self.allDefectsList 

332 badPixels = [Box2I(corner=Point2I(50, 10), dimensions=Extent2I(1, 1)), 

333 Box2I(corner=Point2I(50, 12), dimensions=Extent2I(1, 1)), 

334 Box2I(corner=Point2I(50, 14), dimensions=Extent2I(1, 1)), 

335 Box2I(corner=Point2I(50, 16), dimensions=Extent2I(1, 1)), 

336 Box2I(corner=Point2I(50, 18), dimensions=Extent2I(1, 1)), 

337 Box2I(corner=Point2I(50, 20), dimensions=Extent2I(1, 1)), 

338 Box2I(corner=Point2I(50, 22), dimensions=Extent2I(1, 1)), 

339 Box2I(corner=Point2I(50, 24), dimensions=Extent2I(1, 1)), 

340 Box2I(corner=Point2I(50, 26), dimensions=Extent2I(1, 1)), 

341 Box2I(corner=Point2I(50, 28), dimensions=Extent2I(1, 1)), 

342 Box2I(corner=Point2I(50, 30), dimensions=Extent2I(1, 1)), 

343 Box2I(corner=Point2I(50, 32), dimensions=Extent2I(1, 1)), 

344 Box2I(corner=Point2I(50, 34), dimensions=Extent2I(1, 1)), 

345 Box2I(corner=Point2I(50, 36), dimensions=Extent2I(1, 1)), 

346 Box2I(corner=Point2I(50, 38), dimensions=Extent2I(1, 1)), 

347 Box2I(corner=Point2I(50, 40), dimensions=Extent2I(1, 1))] 

348 with defects.bulk_update(): 

349 for badBox in badPixels: 

350 defects.append(badBox) 

351 

352 self.check_maskBlocks(defects, expectedDefects) 

353 

354 def test_maskBlocks_every_other_pixel_bad_less_than_threshold(self): 

355 """A test for maskBlocksIfIntermitentBadPixelsInColumn. 

356 

357 Npix discontiguous bad pixels in a column where Npix > 

358 badOnAndOffPixelColumnThreshold (10) and every other pixel is 

359 bad. 

360 

361 Plots can be found in DM-19903 on Jira. 

362 

363 """ 

364 

365 expectedDefects = [Box2I(corner=Point2I(55, 20), dimensions=Extent2I(1, 1)), 

366 Box2I(corner=Point2I(55, 22), dimensions=Extent2I(1, 1)), 

367 Box2I(corner=Point2I(55, 24), dimensions=Extent2I(1, 1)), 

368 Box2I(corner=Point2I(55, 26), dimensions=Extent2I(1, 1)), 

369 Box2I(corner=Point2I(55, 28), dimensions=Extent2I(1, 1)), 

370 Box2I(corner=Point2I(55, 30), dimensions=Extent2I(1, 1))] 

371 defects = self.allDefectsList 

372 badPixels = [Box2I(corner=Point2I(55, 20), dimensions=Extent2I(1, 1)), 

373 Box2I(corner=Point2I(55, 22), dimensions=Extent2I(1, 1)), 

374 Box2I(corner=Point2I(55, 24), dimensions=Extent2I(1, 1)), 

375 Box2I(corner=Point2I(55, 26), dimensions=Extent2I(1, 1)), 

376 Box2I(corner=Point2I(55, 28), dimensions=Extent2I(1, 1)), 

377 Box2I(corner=Point2I(55, 30), dimensions=Extent2I(1, 1))] 

378 with defects.bulk_update(): 

379 for badBox in badPixels: 

380 defects.append(badBox) 

381 

382 self.check_maskBlocks(defects, expectedDefects) 

383 

384 def test_maskBlocks_blobs_one_side_good_less_than_threshold(self): 

385 """A test for maskBlocksIfIntermitentBadPixelsInColumn. 

386 

387 Npix discontiguous bad pixels in column with "blobs" of "m" 

388 bad pixels to one side, m > badOnAndOffPixelColumnThreshold 

389 (10), number of good pixel in gaps between blobs < 

390 goodPixelColumnGapThreshold (5). 

391 

392 Plots can be found in DM-19903 on Jira. 

393 

394 """ 

395 

396 expectedDefects = [Box2I(corner=Point2I(60, 1), dimensions=Extent2I(1, 29)), 

397 Box2I(corner=Point2I(61, 2), dimensions=Extent2I(2, 12))] 

398 defects = self.allDefectsList 

399 badPixels = [Box2I(corner=Point2I(60, 1), dimensions=Extent2I(1, 18)), 

400 Box2I(corner=Point2I(60, 20), dimensions=Extent2I(1, 10)), 

401 Box2I(corner=Point2I(61, 2), dimensions=Extent2I(2, 2)), 

402 Box2I(corner=Point2I(61, 6), dimensions=Extent2I(2, 8))] 

403 with defects.bulk_update(): 

404 for badBox in badPixels: 

405 defects.append(badBox) 

406 

407 self.check_maskBlocks(defects, expectedDefects) 

408 

409 def test_maskBlocks_blobs_other_side_good_less_than_threshold(self): 

410 """A test for maskBlocksIfIntermitentBadPixelsInColumn. 

411 

412 Npix discontiguous bad pixels in column with "blobs" of "m" 

413 bad pixels to the other side, m > 

414 badOnAndOffPixelColumnThreshold (10), number of good pixel in 

415 gaps between blobs < goodPixelColumnGapThreshold (5). 

416 

417 Plots can be found in DM-19903 on Jira. 

418 

419 """ 

420 

421 expectedDefects = [Box2I(corner=Point2I(70, 1), dimensions=Extent2I(1, 29)), 

422 Box2I(corner=Point2I(68, 2), dimensions=Extent2I(2, 12))] 

423 defects = self.allDefectsList 

424 badPixels = [Box2I(corner=Point2I(70, 1), dimensions=Extent2I(1, 18)), 

425 Box2I(corner=Point2I(70, 20), dimensions=Extent2I(1, 10)), 

426 Box2I(corner=Point2I(68, 2), dimensions=Extent2I(2, 2)), 

427 Box2I(corner=Point2I(68, 6), dimensions=Extent2I(2, 8))] 

428 with defects.bulk_update(): 

429 for badBox in badPixels: 

430 defects.append(badBox) 

431 

432 self.check_maskBlocks(defects, expectedDefects) 

433 

434 def test_maskBlocks_blob_both_sides_good_less_than_threshold(self): 

435 """A test for maskBlocksIfIntermitentBadPixelsInColumn. 

436 

437 Npix discontiguous bad pixels in column with "blobs" of "m" 

438 bad pixels to both sides, m > badOnAndOffPixelColumnThreshold 

439 (10), number of good pixel in gaps between blobs < 

440 goodPixelColumnGapThreshold (5). 

441 

442 Plots can be found in DM-19903 on Jira. 

443 

444 """ 

445 

446 expectedDefects = [Box2I(corner=Point2I(75, 1), dimensions=Extent2I(1, 29)), 

447 Box2I(corner=Point2I(73, 2), dimensions=Extent2I(2, 12)), 

448 Box2I(corner=Point2I(76, 2), dimensions=Extent2I(2, 12))] 

449 defects = self.allDefectsList 

450 badPixels = [Box2I(corner=Point2I(75, 1), dimensions=Extent2I(1, 18)), 

451 Box2I(corner=Point2I(75, 20), dimensions=Extent2I(1, 10)), 

452 Box2I(corner=Point2I(73, 2), dimensions=Extent2I(2, 2)), 

453 Box2I(corner=Point2I(73, 6), dimensions=Extent2I(2, 8)), 

454 Box2I(corner=Point2I(76, 2), dimensions=Extent2I(2, 2)), 

455 Box2I(corner=Point2I(76, 6), dimensions=Extent2I(2, 8))] 

456 with defects.bulk_update(): 

457 for badBox in badPixels: 

458 defects.append(badBox) 

459 

460 self.check_maskBlocks(defects, expectedDefects) 

461 

462 def test_maskBlocks_blob_one_side_good_greater_than_threshold(self): 

463 """A test for maskBlocksIfIntermitentBadPixelsInColumn. 

464 

465 Npix discontiguous bad pixels in column with "blobs" of "m" 

466 bad pixels to one side, m > badOnAndOffPixelColumnThreshold 

467 (10), number of good pixel in gaps between blobs > 

468 goodPixelColumnGapThreshold (5). 

469 

470 Plots can be found in DM-19903 on Jira. 

471 

472 """ 

473 

474 expectedDefects = [Box2I(corner=Point2I(80, 1), dimensions=Extent2I(1, 29)), 

475 Box2I(corner=Point2I(81, 2), dimensions=Extent2I(2, 2)), 

476 Box2I(corner=Point2I(81, 8), dimensions=Extent2I(2, 8))] 

477 defects = self.allDefectsList 

478 badPixels = [Box2I(corner=Point2I(80, 1), dimensions=Extent2I(1, 18)), 

479 Box2I(corner=Point2I(80, 20), dimensions=Extent2I(1, 10)), 

480 Box2I(corner=Point2I(81, 2), dimensions=Extent2I(2, 2)), 

481 Box2I(corner=Point2I(81, 8), dimensions=Extent2I(2, 8))] 

482 with defects.bulk_update(): 

483 for badBox in badPixels: 

484 defects.append(badBox) 

485 

486 self.check_maskBlocks(defects, expectedDefects) 

487 

488 def test_maskBlocks_other_side_good_greater_than_threshold(self): 

489 """A test for maskBlocksIfIntermitentBadPixelsInColumn. 

490 

491 Npix discontiguous bad pixels in column with "blobs" of "m" 

492 bad pixels to the other side, m > 

493 badOnAndOffPixelColumnThreshold (10), number of good pixel in 

494 gaps between blobs > goodPixelColumnGapThreshold (5). 

495 

496 Plots can be found in DM-19903 on Jira. 

497 

498 """ 

499 

500 expectedDefects = [Box2I(corner=Point2I(87, 1), dimensions=Extent2I(1, 29)), 

501 Box2I(corner=Point2I(85, 2), dimensions=Extent2I(2, 2)), 

502 Box2I(corner=Point2I(85, 8), dimensions=Extent2I(2, 8))] 

503 defects = self.allDefectsList 

504 badPixels = [Box2I(corner=Point2I(87, 1), dimensions=Extent2I(1, 18)), 

505 Box2I(corner=Point2I(87, 20), dimensions=Extent2I(1, 10)), 

506 Box2I(corner=Point2I(85, 2), dimensions=Extent2I(2, 2)), 

507 Box2I(corner=Point2I(85, 8), dimensions=Extent2I(2, 8))] 

508 with defects.bulk_update(): 

509 for badBox in badPixels: 

510 defects.append(badBox) 

511 

512 self.check_maskBlocks(defects, expectedDefects) 

513 

514 def test_maskBlocks_both_sides_good_greater_than_threshold(self): 

515 """A test for maskBlocksIfIntermitentBadPixelsInColumn. 

516 

517 Npix discontiguous bad pixels in column with "blobs" of "m" 

518 bad pixels to both sides, m > badOnAndOffPixelColumnThreshold 

519 (10), number of good pixel in gaps between blobs > 

520 goodPixelColumnGapThreshold (5). 

521 

522 Plots can be found in DM-19903 on Jira. 

523 

524 """ 

525 

526 expectedDefects = [Box2I(corner=Point2I(93, 1), dimensions=Extent2I(1, 34)), 

527 Box2I(corner=Point2I(91, 2), dimensions=Extent2I(2, 7)), 

528 Box2I(corner=Point2I(91, 18), dimensions=Extent2I(2, 9)), 

529 Box2I(corner=Point2I(94, 2), dimensions=Extent2I(2, 7)), 

530 Box2I(corner=Point2I(94, 18), dimensions=Extent2I(2, 9))] 

531 defects = self.allDefectsList 

532 badPixels = [Box2I(corner=Point2I(93, 1), dimensions=Extent2I(1, 12)), 

533 Box2I(corner=Point2I(93, 15), dimensions=Extent2I(1, 20)), 

534 Box2I(corner=Point2I(91, 2), dimensions=Extent2I(2, 2)), 

535 Box2I(corner=Point2I(91, 7), dimensions=Extent2I(2, 2)), 

536 Box2I(corner=Point2I(94, 2), dimensions=Extent2I(2, 2)), 

537 Box2I(corner=Point2I(94, 7), dimensions=Extent2I(2, 2)), 

538 Box2I(corner=Point2I(91, 18), dimensions=Extent2I(2, 3)), 

539 Box2I(corner=Point2I(91, 24), dimensions=Extent2I(2, 3)), 

540 Box2I(corner=Point2I(94, 18), dimensions=Extent2I(2, 3)), 

541 Box2I(corner=Point2I(94, 24), dimensions=Extent2I(2, 3))] 

542 with defects.bulk_update(): 

543 for badBox in badPixels: 

544 defects.append(badBox) 

545 

546 self.check_maskBlocks(defects, expectedDefects) 

547 

548 def test_maskBlocks_y_out_of_order_dm38103(self): 

549 """A test for maskBlocksIfIntermitentBadPixelsInColumn, y out of order. 

550 

551 This test is a variant of 

552 notest_maskBlocks_every_other_pixel_bad_greater_than_threshold with 

553 an extra out-of-y-order bad pixel to trigger DM-38103. 

554 """ 

555 expectedDefects = [Box2I(corner=Point2I(50, 110), dimensions=Extent2I(1, 31))] 

556 defects = self.allDefectsList 

557 badPixels = [Box2I(corner=Point2I(50, 110), dimensions=Extent2I(1, 1)), 

558 Box2I(corner=Point2I(50, 112), dimensions=Extent2I(1, 1)), 

559 Box2I(corner=Point2I(50, 114), dimensions=Extent2I(1, 1)), 

560 Box2I(corner=Point2I(50, 116), dimensions=Extent2I(1, 1)), 

561 Box2I(corner=Point2I(50, 118), dimensions=Extent2I(1, 1)), 

562 Box2I(corner=Point2I(50, 120), dimensions=Extent2I(1, 1)), 

563 Box2I(corner=Point2I(50, 122), dimensions=Extent2I(1, 1)), 

564 Box2I(corner=Point2I(50, 124), dimensions=Extent2I(1, 1)), 

565 Box2I(corner=Point2I(50, 126), dimensions=Extent2I(1, 1)), 

566 Box2I(corner=Point2I(50, 128), dimensions=Extent2I(1, 1)), 

567 Box2I(corner=Point2I(50, 130), dimensions=Extent2I(1, 1)), 

568 Box2I(corner=Point2I(50, 132), dimensions=Extent2I(1, 1)), 

569 Box2I(corner=Point2I(50, 134), dimensions=Extent2I(1, 1)), 

570 Box2I(corner=Point2I(50, 136), dimensions=Extent2I(1, 1)), 

571 Box2I(corner=Point2I(50, 138), dimensions=Extent2I(1, 1)), 

572 Box2I(corner=Point2I(50, 140), dimensions=Extent2I(1, 1)), 

573 # This last point is out of order in y. 

574 Box2I(corner=Point2I(50, 100), dimensions=Extent2I(1, 1))] 

575 

576 # Force defect normalization off in order to trigger DM-38301, because 

577 # defects.fromFootprintList() which is called by _findHotAndColdPixels 

578 # does not do normalization. 

579 defects._bulk_update = True 

580 for badBox in badPixels: 

581 defects.append(badBox) 

582 defects._bulk_update = False 

583 

584 self.check_maskBlocks(defects, expectedDefects) 

585 

586 def check_maskBadColumns(self, exp, inputDefects, expectedDefects): 

587 """A helper function for the tests of 

588 maskBadColumns. 

589 

590 """ 

591 config = copy.copy(self.defaultConfig) 

592 config.badPixelsToFillColumnThreshold = 25 

593 config.saturatedPixelsToFillColumnThreshold = 5 

594 

595 task = self.defaultTask 

596 task.config = config 

597 

598 defectsWithColumns, count = task.maskBadColumns(exp, inputDefects) 

599 

600 self.assertEqual(count, len(expectedDefects)) 

601 

602 boxesMeasured = [] 

603 for defect in defectsWithColumns: 

604 boxesMeasured.append(defect.getBBox()) 

605 

606 for boxInput in expectedDefects: 

607 self.assertIn(boxInput, boxesMeasured) 

608 

609 # Check that the code did not mask anything extra by 

610 # looking in both the input list and "expanded-column" list. 

611 unionInputExpectedBoxes = [] 

612 for defect in inputDefects: 

613 unionInputExpectedBoxes.append(defect.getBBox()) 

614 for defect in expectedDefects: 

615 unionInputExpectedBoxes.append(defect) 

616 

617 # Check that code doesn't mask more than it is supposed to. 

618 for boxMeas in boxesMeasured: 

619 self.assertIn(boxMeas, unionInputExpectedBoxes) 

620 

621 def test_maskBadColumns_extend_full_columns(self): 

622 """Test maskBadColumns, extend to full column. 

623 """ 

624 expectedDefects = [Box2I(corner=Point2I(20, 0), dimensions=Extent2I(1, 51)), 

625 Box2I(corner=Point2I(150, 0), dimensions=Extent2I(1, 51)), 

626 Box2I(corner=Point2I(50, 153), dimensions=Extent2I(1, 51))] 

627 defects = self.allDefectsList 

628 defects.append(Box2I(corner=Point2I(20, 10), dimensions=Extent2I(1, 30))) 

629 defects.append(Box2I(corner=Point2I(150, 5), dimensions=Extent2I(1, 25))) 

630 defects.append(Box2I(corner=Point2I(50, 170), dimensions=Extent2I(1, 30))) 

631 

632 self.check_maskBadColumns(self.flatExp, defects, expectedDefects) 

633 

634 def test_maskBadColumns_no_extend_partial_columns(self): 

635 """Test maskBadColumns, do not extend to full column. 

636 """ 

637 expectedDefects = [] 

638 defects = self.allDefectsList 

639 defects.append(Box2I(corner=Point2I(20, 10), dimensions=Extent2I(1, 20))) 

640 defects.append(Box2I(corner=Point2I(150, 5), dimensions=Extent2I(1, 22))) 

641 defects.append(Box2I(corner=Point2I(50, 170), dimensions=Extent2I(1, 24))) 

642 

643 self.check_maskBadColumns(self.flatExp, defects, expectedDefects) 

644 

645 def test_maskBadColumns_extend_saturated_columns(self): 

646 """Test maskBadColumns, extend saturation to full column. 

647 """ 

648 exp = self.flatExp.clone() 

649 

650 mask = afwImage.Mask.getPlaneBitMask("SAT") 

651 

652 expectedDefects = [Box2I(corner=Point2I(20, 0), dimensions=Extent2I(1, 51)), 

653 Box2I(corner=Point2I(150, 0), dimensions=Extent2I(1, 51)), 

654 Box2I(corner=Point2I(50, 153), dimensions=Extent2I(1, 51))] 

655 defects = self.allDefectsList 

656 

657 # These defects are too small to trigger the former column extension 

658 # (as tested in test_maskBadColumns_no_extend_partial_columns) but 

659 # should still trigger the saturation extension code. 

660 satColumns = [Box2I(corner=Point2I(20, 10), dimensions=Extent2I(1, 5)), 

661 Box2I(corner=Point2I(150, 5), dimensions=Extent2I(1, 5)), 

662 Box2I(corner=Point2I(50, 170), dimensions=Extent2I(1, 5))] 

663 for satColumn in satColumns: 

664 exp.mask[satColumn] |= mask 

665 defects.append(satColumn) 

666 

667 self.check_maskBadColumns(exp, defects, expectedDefects) 

668 

669 def check_dilateSaturatedColumns(self, exp, inputDefects, expectedDefects): 

670 config = copy.copy(self.defaultConfig) 

671 config.saturatedColumnDilationRadius = 2 

672 

673 task = self.defaultTask 

674 task.config = config 

675 

676 defectsDilated = task.dilateSaturatedColumns(exp, inputDefects) 

677 

678 boxesMeasured = [] 

679 for defect in defectsDilated: 

680 boxesMeasured.append(defect.getBBox()) 

681 

682 for boxInput in expectedDefects: 

683 self.assertIn(boxInput, boxesMeasured) 

684 

685 # Check that the code did not mask anything extra by 

686 # looking in both the input list and "expanded-column" list. 

687 unionInputExpectedBoxes = [] 

688 for defect in inputDefects: 

689 unionInputExpectedBoxes.append(defect.getBBox()) 

690 for defect in expectedDefects: 

691 unionInputExpectedBoxes.append(defect) 

692 

693 # Check that code doesn't mask more than it is supposed to. 

694 for boxMeas in boxesMeasured: 

695 self.assertIn(boxMeas, unionInputExpectedBoxes) 

696 

697 def test_dilateSaturatedColumns_saturated_column(self): 

698 exp = self.flatExp.clone() 

699 

700 mask = afwImage.Mask.getPlaneBitMask("SAT") 

701 

702 # We include saturated defects hitting the side to ensure we do not 

703 # have any overflow. 

704 # The dilation radius is set to 2 pixels. 

705 expectedDefects = [Box2I(corner=Point2I(20, 5), dimensions=Extent2I(5, 20)), 

706 Box2I(corner=Point2I(197, 5), dimensions=Extent2I(3, 10)), 

707 Box2I(corner=Point2I(0, 160), dimensions=Extent2I(4, 15))] 

708 defects = self.allDefectsList 

709 

710 satColumns = [Box2I(corner=Point2I(22, 5), dimensions=Extent2I(1, 20)), 

711 Box2I(corner=Point2I(199, 5), dimensions=Extent2I(1, 10)), 

712 Box2I(corner=Point2I(1, 160), dimensions=Extent2I(1, 15))] 

713 for satColumn in satColumns: 

714 exp.mask[satColumn] |= mask 

715 defects.append(satColumn) 

716 

717 self.check_dilateSaturatedColumns(exp, defects, expectedDefects) 

718 

719 def test_dilateSaturatedColumns_no_saturated_column(self): 

720 exp = self.flatExp.clone() 

721 

722 # These are marked BAD but not saturated. 

723 mask = afwImage.Mask.getPlaneBitMask("BAD") 

724 

725 expectedDefects = [] 

726 defects = self.allDefectsList 

727 

728 satColumns = [Box2I(corner=Point2I(22, 5), dimensions=Extent2I(1, 20)), 

729 Box2I(corner=Point2I(199, 5), dimensions=Extent2I(1, 10)), 

730 Box2I(corner=Point2I(1, 160), dimensions=Extent2I(1, 15))] 

731 for satColumn in satColumns: 

732 exp.mask[satColumn] |= mask 

733 defects.append(satColumn) 

734 

735 self.check_dilateSaturatedColumns(exp, defects, expectedDefects) 

736 

737 def test_defectFindingAllSensor(self): 

738 config = copy.copy(self.defaultConfig) 

739 config.nPixBorderLeftRight = 0 

740 config.nPixBorderUpDown = 0 

741 

742 task = self.defaultTask 

743 task.config = config 

744 

745 defects = task._findHotAndColdPixels(self.flatExp) 

746 

747 allBBoxes = self.darkBBoxes + self.brightBBoxes 

748 

749 boxesMeasured = [] 

750 for defect in defects: 

751 boxesMeasured.append(defect.getBBox()) 

752 

753 for expectedBBox in allBBoxes: 

754 self.assertIn(expectedBBox, boxesMeasured) 

755 

756 def test_defectFindingEdgeIgnore(self): 

757 config = copy.copy(self.defaultConfig) 

758 config.nPixBorderUpDown = 0 

759 config.nPixBorderLeftRight = 7 

760 task = self.defaultTask 

761 task.config = config 

762 defects = task._findHotAndColdPixels(self.flatExp) 

763 

764 shouldBeFound = self.darkBBoxes[self.noEdges] + self.brightBBoxes[self.noEdges] 

765 

766 boxesMeasured = [] 

767 for defect in defects: 

768 boxesMeasured.append(defect.getBBox()) 

769 

770 for expectedBBox in shouldBeFound: 

771 self.assertIn(expectedBBox, boxesMeasured) 

772 

773 shouldBeMissed = self.darkBBoxes[self.onlyEdges] + self.brightBBoxes[self.onlyEdges] 

774 for boxMissed in shouldBeMissed: 

775 self.assertNotIn(boxMissed, boxesMeasured) 

776 

777 def valueThreshold(self, fileType, saturateAmpInFlat=False): 

778 """Helper function to loop over flats and darks 

779 to test thresholdType = 'VALUE'.""" 

780 config = copy.copy(self.defaultConfig) 

781 config.thresholdType = 'VALUE' 

782 task = self.defaultTask 

783 task.config = config 

784 

785 for amp in self.flatExp.getDetector(): 

786 if amp.getName() == 'C:0,0': 

787 regionC00 = amp.getBBox() 

788 

789 if fileType == 'dark': 

790 exp = self.darkExp 

791 shouldBeFound = self.brightBBoxes[self.noEdges] 

792 else: 

793 exp = self.flatExp 

794 if saturateAmpInFlat: 

795 exp.maskedImage[regionC00].image.array[:] = 0.0 

796 # Amp C:0,0: minimum=(0, 0), maximum=(99, 50) 

797 x = self.defaultConfig.nPixBorderUpDown 

798 y = self.defaultConfig.nPixBorderLeftRight 

799 width, height = regionC00.getEndX() - x, regionC00.getEndY() - y 

800 # Defects code will mark whole saturated amp as defect box. 

801 shouldBeFound = [Box2I(corner=Point2I(x, y), dimensions=Extent2I(width, height))] 

802 else: 

803 shouldBeFound = self.darkBBoxes[self.noEdges] 

804 # Change the default a bit so it works for the 

805 # existing simulated defects. 

806 task.config.fracThresholdFlat = 0.9 

807 

808 defects = task._findHotAndColdPixels(exp) 

809 

810 boxesMeasured = [] 

811 for defect in defects: 

812 boxesMeasured.append(defect.getBBox()) 

813 

814 for expectedBBox in shouldBeFound: 

815 self.assertIn(expectedBBox, boxesMeasured) 

816 

817 def test_valueThreshold(self): 

818 for fileType in ['dark', 'flat']: 

819 self.valueThreshold(fileType) 

820 # stdDev = 0.0 

821 self.valueThreshold('flat', saturateAmpInFlat=True) 

822 

823 def test_pixelCounting(self): 

824 """Test that the number of defective pixels identified is as expected. 

825 """ 

826 config = copy.copy(self.defaultConfig) 

827 config.nPixBorderUpDown = 0 

828 config.nPixBorderLeftRight = 0 

829 task = self.defaultTask 

830 task.config = config 

831 defects = task._findHotAndColdPixels(self.flatExp) 

832 

833 defectArea = 0 

834 for defect in defects: 

835 defectArea += defect.getBBox().getArea() 

836 

837 # The columnar code will cover blocks of a column with 

838 # on-and-off pixels, thus creating more bad pixels that what 

839 # initially placed in self.brightDefects and self.darkDefects. 

840 # Thus, defectArea should be >= crossCheck. 

841 crossCheck = 0 

842 for x, y, sx, sy in self.brightDefects: 

843 crossCheck += sx*sy 

844 for x, y, sx, sy in self.darkDefects: 

845 crossCheck += sx*sy 

846 

847 # Test the result of _nPixFromDefects() 

848 # via two different ways of calculating area. 

849 self.assertEqual(defectArea, task._nPixFromDefects(defects)) 

850 # defectArea should be >= crossCheck 

851 self.assertGreaterEqual(defectArea, crossCheck) 

852 

853 def test_getNumGoodPixels(self): 

854 """Test the the number of pixels in the image not masked is as 

855 expected. 

856 """ 

857 testImage = self.flatExp.clone() 

858 mi = testImage.maskedImage 

859 

860 imageSize = testImage.getBBox().getArea() 

861 nGood = self.defaultTask._getNumGoodPixels(mi) 

862 

863 self.assertEqual(imageSize, nGood) 

864 

865 NODATABIT = mi.mask.getPlaneBitMask("NO_DATA") 

866 

867 noDataBox = Box2I(Point2I(31, 49), Extent2I(3, 6)) 

868 testImage.mask[noDataBox] |= NODATABIT 

869 

870 self.assertEqual(imageSize - noDataBox.getArea(), self.defaultTask._getNumGoodPixels(mi)) 

871 # check for misfire; we're setting NO_DATA here, not BAD 

872 self.assertEqual(imageSize, self.defaultTask._getNumGoodPixels(mi, 'BAD')) 

873 

874 testImage.mask[noDataBox] ^= NODATABIT # XOR to reset what we did 

875 self.assertEqual(imageSize, nGood) 

876 

877 BADBIT = mi.mask.getPlaneBitMask("BAD") 

878 badBox = Box2I(Point2I(85, 98), Extent2I(4, 7)) 

879 testImage.mask[badBox] |= BADBIT 

880 

881 self.assertEqual(imageSize - badBox.getArea(), self.defaultTask._getNumGoodPixels(mi, 'BAD')) 

882 

883 def test_edgeMasking(self): 

884 """Check that the right number of edge pixels are masked by 

885 _setEdgeBits(). 

886 """ 

887 testImage = self.flatExp.clone() 

888 mi = testImage.maskedImage 

889 

890 self.assertEqual(countMaskedPixels(mi, 'EDGE'), 0) 

891 self.defaultTask._setEdgeBits(mi) 

892 

893 hEdge = self.defaultConfig.nPixBorderLeftRight 

894 vEdge = self.defaultConfig.nPixBorderUpDown 

895 xSize, ySize = mi.getDimensions() 

896 

897 nEdge = xSize*vEdge*2 + ySize*hEdge*2 - hEdge*vEdge*4 

898 

899 self.assertEqual(countMaskedPixels(mi, 'EDGE'), nEdge) 

900 

901 def test_badImage(self): 

902 """Check that fully-bad images do not fail. 

903 """ 

904 testImage = self.flatExp.clone() 

905 testImage.image.array[:, :] = 125000 

906 

907 config = copy.copy(self.defaultConfig) 

908 # Do not exclude any pixels, so the areas match. 

909 config.nPixBorderUpDown = 0 

910 config.nPixBorderLeftRight = 0 

911 

912 task = self.defaultTask 

913 task.config = config 

914 defects = task._findHotAndColdPixels(testImage) 

915 

916 defectArea = 0 

917 for defect in defects: 

918 defectArea += defect.getBBox().getArea() 

919 self.assertEqual(defectArea, testImage.getBBox().getArea()) 

920 

921 

922class TestMemory(lsst.utils.tests.MemoryTestCase): 

923 pass 

924 

925 

926def setup_module(module): 

927 lsst.utils.tests.init() 

928 

929 

930if __name__ == "__main__": 930 ↛ 931line 930 didn't jump to line 931, because the condition on line 930 was never true

931 lsst.utils.tests.init() 

932 unittest.main()