Coverage for tests/test

2# LSST Data Management System

6# This product includes software developed by the

7# LSST Project (http://www.lsst.org/).

9# This program is free software: you can redistribute it and/or modify

10# it under the terms of the GNU General Public License as published by

11# the Free Software Foundation, either version 3 of the License, or

12# (at your option) any later version.

13#

14# This program is distributed in the hope that it will be useful,

15# but WITHOUT ANY WARRANTY; without even the implied warranty of

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the

17# GNU General Public License for more details.

18#

19# You should have received a copy of the LSST License Statement and

20# the GNU General Public License along with this program. If not,

21# see <https://www.lsstcorp.org/LegalNotices/>.

22#

23"""Test cases for lsst.cp.pipe.MeasureDefectsTask."""

25import unittest

26import numpy as np

27import copy

29import lsst.utils

30import lsst.utils.tests

32import lsst.afw.image as afwImage

33import lsst.ip.isr as ipIsr

34import lsst.cp.pipe as cpPipe

35from lsst.ip.isr import isrMock, countMaskedPixels

36from lsst.geom import Box2I, Point2I, Extent2I

37from lsst.daf.base import PropertyList

40class MeasureDefectsTaskTestCase(lsst.utils.tests.TestCase):

41 """A test case for the defect finding task."""

43 def setUp(self):

44 self.defaultConfig = cpPipe.MeasureDefectsTask.ConfigClass()

46 self.flatMean = 2000

47 self.darkMean = 1

48 self.readNoiseAdu = 10

49 self.nSigmaBright = 8

50 self.nSigmaDark = 8

52 mockImageConfig = isrMock.IsrMock.ConfigClass()

54 # flatDrop is not really relevant as we replace the data

55 # but good to note it in case we change how this image is made

56 mockImageConfig.flatDrop = 0.99999

57 mockImageConfig.isTrimmed = True

59 self.flatExp = isrMock.FlatMock(config=mockImageConfig).run()

60 (shapeY, shapeX) = self.flatExp.getDimensions()

61 # x, y, size tuples

62 # always put edge defects at the start and change the value of nEdge

64 self.brightDefects = [(0, 15, 3, 3), (100, 123, 1, 1)]

66 self.darkDefects = [(5, 0, 1, 1), (7, 62, 2, 2)]

68 nEdge = 1 # NOTE: update if more edge defects are included

69 self.noEdges = slice(nEdge, None)

70 self.onlyEdges = slice(0, nEdge)

72 self.darkBBoxes = [Box2I(Point2I(x, y), Extent2I(sx, sy)) for (x, y, sx, sy) in self.darkDefects]

73 self.brightBBoxes = [Box2I(Point2I(x, y), Extent2I(sx, sy)) for (x, y, sx, sy) in self.brightDefects]

75 flatWidth = np.sqrt(self.flatMean) + self.readNoiseAdu

76 darkWidth = self.readNoiseAdu

77 self.rng = np.random.RandomState(0)

78 flatData = self.rng.normal(self.flatMean, flatWidth, (shapeX, shapeY))

79 darkData = self.rng.normal(self.darkMean, darkWidth, (shapeX, shapeY))

81 # NOTE: darks and flats have same defects applied deliberately to both

82 for defect in self.brightDefects:

83 y, x, sy, sx = defect

84 # are these actually the numbers we want?

85 flatData[x:x+sx, y:y+sy] += self.nSigmaBright * flatWidth

86 darkData[x:x+sx, y:y+sy] += self.nSigmaBright * darkWidth

88 for defect in self.darkDefects:

89 y, x, sy, sx = defect

90 # are these actually the numbers we want?

91 flatData[x:x+sx, y:y+sy] -= self.nSigmaDark * flatWidth

92 darkData[x:x+sx, y:y+sy] -= self.nSigmaDark * darkWidth

94 self.darkExp = self.flatExp.clone()

95 self.spareImage = self.flatExp.clone() # for testing edge bits and misc

97 self.flatExp.image.array[:] = flatData

98 self.darkExp.image.array[:] = darkData

100 self.defaultTask = cpPipe.MeasureDefectsTask()

101

102 self.allDefectsList = ipIsr.Defects()

103 self.brightDefectsList = ipIsr.Defects()

104 self.darkDefectsList = ipIsr.Defects()

105

106 # Set image types, the defects code will use them.

107 metaDataFlat = PropertyList()

108 metaDataFlat["IMGTYPE"] = "FLAT"

109 self.flatExp.setMetadata(metaDataFlat)

110

111 metaDataDark = PropertyList()

112 metaDataDark["IMGTYPE"] = "DARK"

113 self.darkExp.setMetadata(metaDataDark)

114

115 with self.allDefectsList.bulk_update():

116 with self.brightDefectsList.bulk_update():

117 for d in self.brightBBoxes:

118 self.brightDefectsList.append(d)

119 self.allDefectsList.append(d)

120

121 with self.darkDefectsList.bulk_update():

122 for d in self.darkBBoxes:

123 self.darkDefectsList.append(d)

124 self.allDefectsList.append(d)

125

126 def check_maskBlocks(self, inputDefects, expectedDefects):

127 """A helper function for the tests of

128 maskBlocksIfIntermitentBadPixelsInColumn.

129

130 """

131 config = copy.copy(self.defaultConfig)

132 config.badOnAndOffPixelColumnThreshold = 10

133 config.goodPixelColumnGapThreshold = 5

134 config.nPixBorderUpDown = 0

135 config.nPixBorderLeftRight = 0

136

137 task = self.defaultTask

138 task.config = config

139

140 defectsWithColumns, count = task.maskBlocksIfIntermitentBadPixelsInColumn(inputDefects)

141 boxesMeasured = []

142 for defect in defectsWithColumns:

143 boxesMeasured.append(defect.getBBox())

144

145 for boxInput in expectedDefects:

146 self.assertIn(boxInput, boxesMeasured)

147

148 # Check that the code did not mask anything extra by

149 # looking in both the input list and "expanded-column" list.

150 unionInputExpectedBoxes = []

151 for defect in inputDefects:

152 unionInputExpectedBoxes.append(defect.getBBox())

153 for defect in expectedDefects:

154 unionInputExpectedBoxes.append(defect)

155

156 # Check that code doesn't mask more than it is supposed to.

157 for boxMeas in boxesMeasured:

158 self.assertIn(boxMeas, unionInputExpectedBoxes)

159

160 def test_maskBlocks_full_column(self):

161 """A test for maskBlocksIfIntermitentBadPixelsInColumn.

162

163 Tests that a contigous bad column does not get split by the

164 code.

165

166 The mock flat has a size of 200X204 pixels. This column has a

167 maximum length of 50 pixels, otherwise there would be a split

168 along the mock amp boundary.

169

170 Plots can be found in DM-19903 on Jira.

171

172 """

173

174 defects = self.allDefectsList

175 defects.append(Box2I(corner=Point2I(15, 1), dimensions=Extent2I(1, 50)))

176 expectedDefects = [Box2I(corner=Point2I(15, 1), dimensions=Extent2I(1, 50))]

177

178 self.check_maskBlocks(defects, expectedDefects)

179

180 def test_maskBlocks_long_column(self):

181 """A test for maskBlocksIfIntermitentBadPixelsInColumn.

182

183 Tests that a contigous bad column with Npix >=

184 badOnAndOffPixelColumnThreshold (10) does not get split by the

185 code.

186

187 Plots can be found in DM-19903 on Jira.

188

189 """

190

191 expectedDefects = [Box2I(corner=Point2I(20, 1), dimensions=Extent2I(1, 25))]

192 defects = self.allDefectsList

193 defects.append(Box2I(corner=Point2I(20, 1), dimensions=Extent2I(1, 25)))

194

195 self.check_maskBlocks(defects, expectedDefects)

196

197 def test_maskBlocks_short_column(self):

198 """A test for maskBlocksIfIntermitentBadPixelsInColumn.

199

200 Tests that a contigous bad column Npix <

201 badOnAndOffPixelColumnThreshold (10) does not get split by the

202 code.

203

204 Plots can be found in DM-19903 on Jira.

205

206 """

207

208 expectedDefects = [Box2I(corner=Point2I(25, 1), dimensions=Extent2I(1, 8))]

209 defects = self.allDefectsList

210 defects.append(Box2I(corner=Point2I(25, 1), dimensions=Extent2I(1, 8)))

211

212 self.check_maskBlocks(defects, expectedDefects)

213

214 def test_maskBlocks_discontigous_to_single_block(self):

215 """A test for maskBlocksIfIntermitentBadPixelsInColumn.

216

217 Npix discontiguous bad pixels in a column where Npix >=

218 badOnAndOffPixelColumnThreshold (10) and gaps of good pixels <

219 goodPixelColumnGapThreshold (5). Under these conditions, the

220 whole block of bad pixels (including good gaps) should be

221 masked.

222

223 Plots can be found in DM-19903 on Jira.

224

225 """

226

227 expectedDefects = [Box2I(corner=Point2I(30, 1), dimensions=Extent2I(1, 48))]

228 defects = self.allDefectsList

229 badPixels = [Box2I(corner=Point2I(30, 1), dimensions=Extent2I(1, 2)),

230 Box2I(corner=Point2I(30, 5), dimensions=Extent2I(1, 3)),

231 Box2I(corner=Point2I(30, 11), dimensions=Extent2I(1, 5)),

232 Box2I(corner=Point2I(30, 19), dimensions=Extent2I(1, 5)),

233 Box2I(corner=Point2I(30, 27), dimensions=Extent2I(1, 4)),

234 Box2I(corner=Point2I(30, 34), dimensions=Extent2I(1, 15))]

235 with defects.bulk_update():

236 for badBox in badPixels:

237 defects.append(badBox)

238

239 self.check_maskBlocks(defects, expectedDefects)

240

241 def test_maskBlocks_discontigous_less_than_thresholds(self):

242 """A test for maskBlocksIfIntermitentBadPixelsInColumn.

243

244 Npix discontiguous bad pixels in a column where Npix <

245 badOnAndOffPixelColumnThreshold (10) and gaps of good pixels <

246 goodPixelColumnGapThreshold (5). Under these conditions, the

247 expected defect boxes should be the same as the input boxes.

248

249 Plots can be found in DM-19903 on Jira.

250

251 """

252

253 expectedDefects = [Box2I(corner=Point2I(35, 1), dimensions=Extent2I(1, 2)),

254 Box2I(corner=Point2I(35, 5), dimensions=Extent2I(1, 3)),

255 Box2I(corner=Point2I(35, 11), dimensions=Extent2I(1, 2))]

256 defects = self.allDefectsList

257 badPixels = [Box2I(corner=Point2I(35, 1), dimensions=Extent2I(1, 2)),

258 Box2I(corner=Point2I(35, 5), dimensions=Extent2I(1, 3)),

259 Box2I(corner=Point2I(35, 11), dimensions=Extent2I(1, 2))]

260 with defects.bulk_update():

261 for badBox in badPixels:

262 defects.append(badBox)

263

264 self.check_maskBlocks(defects, expectedDefects)

265

266 def test_maskBlocks_more_than_thresholds(self):

267 """A test for maskBlocksIfIntermitentBadPixelsInColumn.

268

269 Npix discontiguous bad pixels in a column where Npix <

270 badOnAndOffPixelColumnThreshold (10) and gaps of good pixels <

271 goodPixelColumnGapThreshold (5). Npix=34 (> 10) bad pixels

272 total, 1 "good" gap with 13 pixels big enough (13 >= 5 good

273 pixels, from y=6 (1+5) to y=19).

274

275 Plots can be found in DM-19903 on Jira.

276

277 """

278

279 expectedDefects = [Box2I(corner=Point2I(40, 1), dimensions=Extent2I(1, 7)),

280 Box2I(corner=Point2I(40, 19), dimensions=Extent2I(1, 30))]

281 defects = self.allDefectsList

282 badPixels = [Box2I(corner=Point2I(40, 1), dimensions=Extent2I(1, 2)),

283 Box2I(corner=Point2I(40, 5), dimensions=Extent2I(1, 3)),

284 Box2I(corner=Point2I(40, 19), dimensions=Extent2I(1, 5)),

285 Box2I(corner=Point2I(40, 27), dimensions=Extent2I(1, 4)),

286 Box2I(corner=Point2I(40, 34), dimensions=Extent2I(1, 15))]

287 with defects.bulk_update():

288 for badBox in badPixels:

289 defects.append(badBox)

290

291 self.check_maskBlocks(defects, expectedDefects)

292

293 def test_maskBlocks_not_enough_bad_pixels_in_column(self):

294 """A test for maskBlocksIfIntermitentBadPixelsInColumn.

295

296 Npix discontiguous bad pixels in a column where Npix <

297 badOnAndOffPixelColumnThreshold (10) and and gaps of good

298 pixels > goodPixelColumnGapThreshold (5). Since Npix <

299 badOnAndOffPixelColumnThreshold, then it doesn't matter that

300 the number of good pixels in gap >

301 goodPixelColumnGapThreshold. 5<10 bad pixels total, 1 "good"

302 gap big enough (29>=5 good pixels, from y =12 (10+2) to y=30)

303

304 Plots can be found in DM-19903 on Jira.

305

306 """

307

308 expectedDefects = [Box2I(corner=Point2I(45, 10), dimensions=Extent2I(1, 2)),

309 Box2I(corner=Point2I(45, 30), dimensions=Extent2I(1, 3))]

310 defects = self.allDefectsList

311 badPixels = [Box2I(corner=Point2I(45, 10), dimensions=Extent2I(1, 2)),

312 Box2I(corner=Point2I(45, 30), dimensions=Extent2I(1, 3))]

313 with defects.bulk_update():

314 for badBox in badPixels:

315 defects.append(badBox)

316

317 self.check_maskBlocks(defects, expectedDefects)

318

319 def test_maskBlocks_every_other_pixel_bad_greater_than_threshold(self):

320 """A test for maskBlocksIfIntermitentBadPixelsInColumn.

321

322 Npix discontiguous bad pixels in a column where Npix >

323 badOnAndOffPixelColumnThreshold (10) and every other pixel is

324 bad.

325

326 Plots can be found in DM-19903 on Jira.

327

328 """

329

330 expectedDefects = [Box2I(corner=Point2I(50, 10), dimensions=Extent2I(1, 31))]

331 defects = self.allDefectsList

332 badPixels = [Box2I(corner=Point2I(50, 10), dimensions=Extent2I(1, 1)),

333 Box2I(corner=Point2I(50, 12), dimensions=Extent2I(1, 1)),

334 Box2I(corner=Point2I(50, 14), dimensions=Extent2I(1, 1)),

335 Box2I(corner=Point2I(50, 16), dimensions=Extent2I(1, 1)),

336 Box2I(corner=Point2I(50, 18), dimensions=Extent2I(1, 1)),

337 Box2I(corner=Point2I(50, 20), dimensions=Extent2I(1, 1)),

338 Box2I(corner=Point2I(50, 22), dimensions=Extent2I(1, 1)),

339 Box2I(corner=Point2I(50, 24), dimensions=Extent2I(1, 1)),

340 Box2I(corner=Point2I(50, 26), dimensions=Extent2I(1, 1)),

341 Box2I(corner=Point2I(50, 28), dimensions=Extent2I(1, 1)),

342 Box2I(corner=Point2I(50, 30), dimensions=Extent2I(1, 1)),

343 Box2I(corner=Point2I(50, 32), dimensions=Extent2I(1, 1)),

344 Box2I(corner=Point2I(50, 34), dimensions=Extent2I(1, 1)),

345 Box2I(corner=Point2I(50, 36), dimensions=Extent2I(1, 1)),

346 Box2I(corner=Point2I(50, 38), dimensions=Extent2I(1, 1)),

347 Box2I(corner=Point2I(50, 40), dimensions=Extent2I(1, 1))]

348 with defects.bulk_update():

349 for badBox in badPixels:

350 defects.append(badBox)

351

352 self.check_maskBlocks(defects, expectedDefects)

353

354 def test_maskBlocks_every_other_pixel_bad_less_than_threshold(self):

355 """A test for maskBlocksIfIntermitentBadPixelsInColumn.

356

357 Npix discontiguous bad pixels in a column where Npix >

358 badOnAndOffPixelColumnThreshold (10) and every other pixel is

359 bad.

360

361 Plots can be found in DM-19903 on Jira.

362

363 """

364

365 expectedDefects = [Box2I(corner=Point2I(55, 20), dimensions=Extent2I(1, 1)),

366 Box2I(corner=Point2I(55, 22), dimensions=Extent2I(1, 1)),

367 Box2I(corner=Point2I(55, 24), dimensions=Extent2I(1, 1)),

368 Box2I(corner=Point2I(55, 26), dimensions=Extent2I(1, 1)),

369 Box2I(corner=Point2I(55, 28), dimensions=Extent2I(1, 1)),

370 Box2I(corner=Point2I(55, 30), dimensions=Extent2I(1, 1))]

371 defects = self.allDefectsList

372 badPixels = [Box2I(corner=Point2I(55, 20), dimensions=Extent2I(1, 1)),

373 Box2I(corner=Point2I(55, 22), dimensions=Extent2I(1, 1)),

374 Box2I(corner=Point2I(55, 24), dimensions=Extent2I(1, 1)),

375 Box2I(corner=Point2I(55, 26), dimensions=Extent2I(1, 1)),

376 Box2I(corner=Point2I(55, 28), dimensions=Extent2I(1, 1)),

377 Box2I(corner=Point2I(55, 30), dimensions=Extent2I(1, 1))]

378 with defects.bulk_update():

379 for badBox in badPixels:

380 defects.append(badBox)

381

382 self.check_maskBlocks(defects, expectedDefects)

383

384 def test_maskBlocks_blobs_one_side_good_less_than_threshold(self):

385 """A test for maskBlocksIfIntermitentBadPixelsInColumn.

386

387 Npix discontiguous bad pixels in column with "blobs" of "m"

388 bad pixels to one side, m > badOnAndOffPixelColumnThreshold

389 (10), number of good pixel in gaps between blobs <

390 goodPixelColumnGapThreshold (5).

391

392 Plots can be found in DM-19903 on Jira.

393

394 """

395

396 expectedDefects = [Box2I(corner=Point2I(60, 1), dimensions=Extent2I(1, 29)),

397 Box2I(corner=Point2I(61, 2), dimensions=Extent2I(2, 12))]

398 defects = self.allDefectsList

399 badPixels = [Box2I(corner=Point2I(60, 1), dimensions=Extent2I(1, 18)),

400 Box2I(corner=Point2I(60, 20), dimensions=Extent2I(1, 10)),

401 Box2I(corner=Point2I(61, 2), dimensions=Extent2I(2, 2)),

402 Box2I(corner=Point2I(61, 6), dimensions=Extent2I(2, 8))]

403 with defects.bulk_update():

404 for badBox in badPixels:

405 defects.append(badBox)

406

407 self.check_maskBlocks(defects, expectedDefects)

408

409 def test_maskBlocks_blobs_other_side_good_less_than_threshold(self):

410 """A test for maskBlocksIfIntermitentBadPixelsInColumn.

411

412 Npix discontiguous bad pixels in column with "blobs" of "m"

413 bad pixels to the other side, m >

414 badOnAndOffPixelColumnThreshold (10), number of good pixel in

415 gaps between blobs < goodPixelColumnGapThreshold (5).

416

417 Plots can be found in DM-19903 on Jira.

418

419 """

420

421 expectedDefects = [Box2I(corner=Point2I(70, 1), dimensions=Extent2I(1, 29)),

422 Box2I(corner=Point2I(68, 2), dimensions=Extent2I(2, 12))]

423 defects = self.allDefectsList

424 badPixels = [Box2I(corner=Point2I(70, 1), dimensions=Extent2I(1, 18)),

425 Box2I(corner=Point2I(70, 20), dimensions=Extent2I(1, 10)),

426 Box2I(corner=Point2I(68, 2), dimensions=Extent2I(2, 2)),

427 Box2I(corner=Point2I(68, 6), dimensions=Extent2I(2, 8))]

428 with defects.bulk_update():

429 for badBox in badPixels:

430 defects.append(badBox)

431

432 self.check_maskBlocks(defects, expectedDefects)

433

434 def test_maskBlocks_blob_both_sides_good_less_than_threshold(self):

435 """A test for maskBlocksIfIntermitentBadPixelsInColumn.

436

437 Npix discontiguous bad pixels in column with "blobs" of "m"

438 bad pixels to both sides, m > badOnAndOffPixelColumnThreshold

439 (10), number of good pixel in gaps between blobs <

440 goodPixelColumnGapThreshold (5).

441

442 Plots can be found in DM-19903 on Jira.

443

444 """

445

446 expectedDefects = [Box2I(corner=Point2I(75, 1), dimensions=Extent2I(1, 29)),

447 Box2I(corner=Point2I(73, 2), dimensions=Extent2I(2, 12)),

448 Box2I(corner=Point2I(76, 2), dimensions=Extent2I(2, 12))]

449 defects = self.allDefectsList

450 badPixels = [Box2I(corner=Point2I(75, 1), dimensions=Extent2I(1, 18)),

451 Box2I(corner=Point2I(75, 20), dimensions=Extent2I(1, 10)),

452 Box2I(corner=Point2I(73, 2), dimensions=Extent2I(2, 2)),

453 Box2I(corner=Point2I(73, 6), dimensions=Extent2I(2, 8)),

454 Box2I(corner=Point2I(76, 2), dimensions=Extent2I(2, 2)),

455 Box2I(corner=Point2I(76, 6), dimensions=Extent2I(2, 8))]

456 with defects.bulk_update():

457 for badBox in badPixels:

458 defects.append(badBox)

459

460 self.check_maskBlocks(defects, expectedDefects)

461

462 def test_maskBlocks_blob_one_side_good_greater_than_threshold(self):

463 """A test for maskBlocksIfIntermitentBadPixelsInColumn.

464

465 Npix discontiguous bad pixels in column with "blobs" of "m"

466 bad pixels to one side, m > badOnAndOffPixelColumnThreshold

467 (10), number of good pixel in gaps between blobs >

468 goodPixelColumnGapThreshold (5).

469

470 Plots can be found in DM-19903 on Jira.

471

472 """

473

474 expectedDefects = [Box2I(corner=Point2I(80, 1), dimensions=Extent2I(1, 29)),

475 Box2I(corner=Point2I(81, 2), dimensions=Extent2I(2, 2)),

476 Box2I(corner=Point2I(81, 8), dimensions=Extent2I(2, 8))]

477 defects = self.allDefectsList

478 badPixels = [Box2I(corner=Point2I(80, 1), dimensions=Extent2I(1, 18)),

479 Box2I(corner=Point2I(80, 20), dimensions=Extent2I(1, 10)),

480 Box2I(corner=Point2I(81, 2), dimensions=Extent2I(2, 2)),

481 Box2I(corner=Point2I(81, 8), dimensions=Extent2I(2, 8))]

482 with defects.bulk_update():

483 for badBox in badPixels:

484 defects.append(badBox)

485

486 self.check_maskBlocks(defects, expectedDefects)

487

488 def test_maskBlocks_other_side_good_greater_than_threshold(self):

489 """A test for maskBlocksIfIntermitentBadPixelsInColumn.

490

491 Npix discontiguous bad pixels in column with "blobs" of "m"

492 bad pixels to the other side, m >

493 badOnAndOffPixelColumnThreshold (10), number of good pixel in

494 gaps between blobs > goodPixelColumnGapThreshold (5).

495

496 Plots can be found in DM-19903 on Jira.

497

498 """

499

500 expectedDefects = [Box2I(corner=Point2I(87, 1), dimensions=Extent2I(1, 29)),

501 Box2I(corner=Point2I(85, 2), dimensions=Extent2I(2, 2)),

502 Box2I(corner=Point2I(85, 8), dimensions=Extent2I(2, 8))]

503 defects = self.allDefectsList

504 badPixels = [Box2I(corner=Point2I(87, 1), dimensions=Extent2I(1, 18)),

505 Box2I(corner=Point2I(87, 20), dimensions=Extent2I(1, 10)),

506 Box2I(corner=Point2I(85, 2), dimensions=Extent2I(2, 2)),

507 Box2I(corner=Point2I(85, 8), dimensions=Extent2I(2, 8))]

508 with defects.bulk_update():

509 for badBox in badPixels:

510 defects.append(badBox)

511

512 self.check_maskBlocks(defects, expectedDefects)

513

514 def test_maskBlocks_both_sides_good_greater_than_threshold(self):

515 """A test for maskBlocksIfIntermitentBadPixelsInColumn.

516

517 Npix discontiguous bad pixels in column with "blobs" of "m"

518 bad pixels to both sides, m > badOnAndOffPixelColumnThreshold

519 (10), number of good pixel in gaps between blobs >

520 goodPixelColumnGapThreshold (5).

521

522 Plots can be found in DM-19903 on Jira.

523

524 """

525

526 expectedDefects = [Box2I(corner=Point2I(93, 1), dimensions=Extent2I(1, 34)),

527 Box2I(corner=Point2I(91, 2), dimensions=Extent2I(2, 7)),

528 Box2I(corner=Point2I(91, 18), dimensions=Extent2I(2, 9)),

529 Box2I(corner=Point2I(94, 2), dimensions=Extent2I(2, 7)),

530 Box2I(corner=Point2I(94, 18), dimensions=Extent2I(2, 9))]

531 defects = self.allDefectsList

532 badPixels = [Box2I(corner=Point2I(93, 1), dimensions=Extent2I(1, 12)),

533 Box2I(corner=Point2I(93, 15), dimensions=Extent2I(1, 20)),

534 Box2I(corner=Point2I(91, 2), dimensions=Extent2I(2, 2)),

535 Box2I(corner=Point2I(91, 7), dimensions=Extent2I(2, 2)),

536 Box2I(corner=Point2I(94, 2), dimensions=Extent2I(2, 2)),

537 Box2I(corner=Point2I(94, 7), dimensions=Extent2I(2, 2)),

538 Box2I(corner=Point2I(91, 18), dimensions=Extent2I(2, 3)),

539 Box2I(corner=Point2I(91, 24), dimensions=Extent2I(2, 3)),

540 Box2I(corner=Point2I(94, 18), dimensions=Extent2I(2, 3)),

541 Box2I(corner=Point2I(94, 24), dimensions=Extent2I(2, 3))]

542 with defects.bulk_update():

543 for badBox in badPixels:

544 defects.append(badBox)

545

546 self.check_maskBlocks(defects, expectedDefects)

547

548 def test_maskBlocks_y_out_of_order_dm38103(self):

549 """A test for maskBlocksIfIntermitentBadPixelsInColumn, y out of order.

550

551 This test is a variant of

552 test_maskBlocks_every_other_pixel_bad_greater_than_threshold with

553 an extra out-of-y-order bad pixel to trigger DM-38103.

554 """

555 expectedDefects = [Box2I(corner=Point2I(50, 110), dimensions=Extent2I(1, 31))]

556 defects = self.allDefectsList

557 badPixels = [Box2I(corner=Point2I(50, 110), dimensions=Extent2I(1, 1)),

558 Box2I(corner=Point2I(50, 112), dimensions=Extent2I(1, 1)),

559 Box2I(corner=Point2I(50, 114), dimensions=Extent2I(1, 1)),

560 Box2I(corner=Point2I(50, 116), dimensions=Extent2I(1, 1)),

561 Box2I(corner=Point2I(50, 118), dimensions=Extent2I(1, 1)),

562 Box2I(corner=Point2I(50, 120), dimensions=Extent2I(1, 1)),

563 Box2I(corner=Point2I(50, 122), dimensions=Extent2I(1, 1)),

564 Box2I(corner=Point2I(50, 124), dimensions=Extent2I(1, 1)),

565 Box2I(corner=Point2I(50, 126), dimensions=Extent2I(1, 1)),

566 Box2I(corner=Point2I(50, 128), dimensions=Extent2I(1, 1)),

567 Box2I(corner=Point2I(50, 130), dimensions=Extent2I(1, 1)),

568 Box2I(corner=Point2I(50, 132), dimensions=Extent2I(1, 1)),

569 Box2I(corner=Point2I(50, 134), dimensions=Extent2I(1, 1)),

570 Box2I(corner=Point2I(50, 136), dimensions=Extent2I(1, 1)),

571 Box2I(corner=Point2I(50, 138), dimensions=Extent2I(1, 1)),

572 Box2I(corner=Point2I(50, 140), dimensions=Extent2I(1, 1)),

573 # This last point is out of order in y.

574 Box2I(corner=Point2I(50, 100), dimensions=Extent2I(1, 1))]

575

576 # Force defect normalization off in order to trigger DM-38301, because

577 # defects.fromFootprintList() which is called by _findHotAndColdPixels

578 # does not do normalization.

579 defects._bulk_update = True

580 for badBox in badPixels:

581 defects.append(badBox)

582 defects._bulk_update = False

583

584 self.check_maskBlocks(defects, expectedDefects)

585

586 def check_maskBadColumns(self, exp, inputDefects, expectedDefects):

587 """A helper function for the tests of

588 maskBadColumns.

589

590 """

591 config = copy.copy(self.defaultConfig)

592 config.badPixelsToFillColumnThreshold = 25

593 config.saturatedPixelsToFillColumnThreshold = 5

594

595 task = self.defaultTask

596 task.config = config

597

598 defectsWithColumns, count = task.maskBadColumns(exp, inputDefects)

599

600 self.assertEqual(count, len(expectedDefects))

601

602 boxesMeasured = []

603 for defect in defectsWithColumns:

604 boxesMeasured.append(defect.getBBox())

605

606 for boxInput in expectedDefects:

607 self.assertIn(boxInput, boxesMeasured)

608

609 # Check that the code did not mask anything extra by

610 # looking in both the input list and "expanded-column" list.

611 unionInputExpectedBoxes = []

612 for defect in inputDefects:

613 unionInputExpectedBoxes.append(defect.getBBox())

614 for defect in expectedDefects:

615 unionInputExpectedBoxes.append(defect)

616

617 # Check that code doesn't mask more than it is supposed to.

618 for boxMeas in boxesMeasured:

619 self.assertIn(boxMeas, unionInputExpectedBoxes)

620

621 def test_maskBadColumns_extend_full_columns(self):

622 """Test maskBadColumns, extend to full column.

623 """

624 expectedDefects = [Box2I(corner=Point2I(20, 0), dimensions=Extent2I(1, 51)),

625 Box2I(corner=Point2I(150, 0), dimensions=Extent2I(1, 51)),

626 Box2I(corner=Point2I(50, 153), dimensions=Extent2I(1, 51))]

627 defects = self.allDefectsList

628 defects.append(Box2I(corner=Point2I(20, 10), dimensions=Extent2I(1, 30)))

629 defects.append(Box2I(corner=Point2I(150, 5), dimensions=Extent2I(1, 25)))

630 defects.append(Box2I(corner=Point2I(50, 170), dimensions=Extent2I(1, 30)))

631

632 self.check_maskBadColumns(self.flatExp, defects, expectedDefects)

633

634 def test_maskBadColumns_no_extend_partial_columns(self):

635 """Test maskBadColumns, do not extend to full column.

636 """

637 expectedDefects = []

638 defects = self.allDefectsList

639 defects.append(Box2I(corner=Point2I(20, 10), dimensions=Extent2I(1, 20)))

640 defects.append(Box2I(corner=Point2I(150, 5), dimensions=Extent2I(1, 22)))

641 defects.append(Box2I(corner=Point2I(50, 170), dimensions=Extent2I(1, 24)))

642

643 self.check_maskBadColumns(self.flatExp, defects, expectedDefects)

644

645 def test_maskBadColumns_extend_saturated_columns(self):

646 """Test maskBadColumns, extend saturation to full column.

647 """

648 exp = self.flatExp.clone()

649

650 mask = afwImage.Mask.getPlaneBitMask("SAT")

651

652 expectedDefects = [Box2I(corner=Point2I(20, 0), dimensions=Extent2I(1, 51)),

653 Box2I(corner=Point2I(150, 0), dimensions=Extent2I(1, 51)),

654 Box2I(corner=Point2I(50, 153), dimensions=Extent2I(1, 51))]

655 defects = self.allDefectsList

656

657 # These defects are too small to trigger the former column extension

658 # (as tested in test_maskBadColumns_no_extend_partial_columns) but

659 # should still trigger the saturation extension code.

660 satColumns = [Box2I(corner=Point2I(20, 10), dimensions=Extent2I(1, 5)),

661 Box2I(corner=Point2I(150, 5), dimensions=Extent2I(1, 5)),

662 Box2I(corner=Point2I(50, 170), dimensions=Extent2I(1, 5))]

663 for satColumn in satColumns:

664 exp.mask[satColumn] |= mask

665 defects.append(satColumn)

666

667 self.check_maskBadColumns(exp, defects, expectedDefects)

668

669 def check_dilateSaturatedColumns(self, exp, inputDefects, expectedDefects):

670 config = copy.copy(self.defaultConfig)

671 config.saturatedColumnDilationRadius = 2

672

673 task = self.defaultTask

674 task.config = config

675

676 defectsDilated = task.dilateSaturatedColumns(exp, inputDefects)

677

678 boxesMeasured = []

679 for defect in defectsDilated:

680 boxesMeasured.append(defect.getBBox())

681

682 for boxInput in expectedDefects:

683 self.assertIn(boxInput, boxesMeasured)

684

685 # Check that the code did not mask anything extra by

686 # looking in both the input list and "expanded-column" list.

687 unionInputExpectedBoxes = []

688 for defect in inputDefects:

689 unionInputExpectedBoxes.append(defect.getBBox())

690 for defect in expectedDefects:

691 unionInputExpectedBoxes.append(defect)

692

693 # Check that code doesn't mask more than it is supposed to.

694 for boxMeas in boxesMeasured:

695 self.assertIn(boxMeas, unionInputExpectedBoxes)

696

697 def test_dilateSaturatedColumns_saturated_column(self):

698 exp = self.flatExp.clone()

699

700 mask = afwImage.Mask.getPlaneBitMask("SAT")

701

702 # We include saturated defects hitting the side to ensure we do not

703 # have any overflow.

704 # The dilation radius is set to 2 pixels.

705 expectedDefects = [Box2I(corner=Point2I(20, 5), dimensions=Extent2I(5, 20)),

706 Box2I(corner=Point2I(197, 5), dimensions=Extent2I(3, 10)),

707 Box2I(corner=Point2I(0, 160), dimensions=Extent2I(4, 15))]

708 defects = self.allDefectsList

709

710 satColumns = [Box2I(corner=Point2I(22, 5), dimensions=Extent2I(1, 20)),

711 Box2I(corner=Point2I(199, 5), dimensions=Extent2I(1, 10)),

712 Box2I(corner=Point2I(1, 160), dimensions=Extent2I(1, 15))]

713 for satColumn in satColumns:

714 exp.mask[satColumn] |= mask

715 defects.append(satColumn)

716

717 self.check_dilateSaturatedColumns(exp, defects, expectedDefects)

718

719 def test_dilateSaturatedColumns_no_saturated_column(self):

720 exp = self.flatExp.clone()

721

722 # These are marked BAD but not saturated.

723 mask = afwImage.Mask.getPlaneBitMask("BAD")

724

725 expectedDefects = []

726 defects = self.allDefectsList

727

728 satColumns = [Box2I(corner=Point2I(22, 5), dimensions=Extent2I(1, 20)),

729 Box2I(corner=Point2I(199, 5), dimensions=Extent2I(1, 10)),

730 Box2I(corner=Point2I(1, 160), dimensions=Extent2I(1, 15))]

731 for satColumn in satColumns:

732 exp.mask[satColumn] |= mask

733 defects.append(satColumn)

734

735 self.check_dilateSaturatedColumns(exp, defects, expectedDefects)

736

737 def test_defectVampirePixels(self):

738 config = copy.copy(self.defaultConfig)

739 # We set configs to mask vampire pixels

740 config.doVampirePixels = True

741 # We choose small radius to get a only 3 bbox to check

742 config.radiusVampirePixels = 1

743 # This is the threshold type that is actually used

744 # for defects search in flats

745 config.thresholdType = 'VALUE'

746 # Flat mock has higher pixel values than real flatBootstrap

747 config.thresholdVampirePixels = 2550.

748

749 task = self.defaultTask

750 task.config = config

751

752 exp = self.flatExp.clone()

753

754 # We set one bright pixel to test the vampire pixel masking

755 yVampirePixel = 130

756 xVampirePixel = 50

757 exp.image.array[yVampirePixel, xVampirePixel] = 2600.

758

759 # Find usual hot and pixels as well as the bright defect

760 defects = task._findHotAndColdPixels(exp)

761

762 # Make bright defect BBox

763 vampireBBox = [Box2I(corner=Point2I(50, 129), dimensions=Extent2I(1, 3)),

764 Box2I(corner=Point2I(49, 130), dimensions=Extent2I(1, 1)),

765 Box2I(corner=Point2I(51, 130), dimensions=Extent2I(1, 1))]

766 # Test that the BBox is within the defects measured

767 boxesMeasured = []

768 for defect in defects:

769 boxesMeasured.append(defect.getBBox())

770 for expectedBBox in vampireBBox:

771 self.assertIn(expectedBBox, boxesMeasured)

772

773 def test_defectFindingAllSensor(self):

774 config = copy.copy(self.defaultConfig)

775 config.nPixBorderLeftRight = 0

776 config.nPixBorderUpDown = 0

777

778 task = self.defaultTask

779 task.config = config

780

781 defects = task._findHotAndColdPixels(self.flatExp)

782

783 allBBoxes = self.darkBBoxes + self.brightBBoxes

784

785 boxesMeasured = []

786 for defect in defects:

787 boxesMeasured.append(defect.getBBox())

788

789 for expectedBBox in allBBoxes:

790 self.assertIn(expectedBBox, boxesMeasured)

791

792 def test_defectFindingEdgeIgnore(self):

793 config = copy.copy(self.defaultConfig)

794 config.nPixBorderUpDown = 0

795 config.nPixBorderLeftRight = 7

796 task = self.defaultTask

797 task.config = config

798 defects = task._findHotAndColdPixels(self.flatExp)

799

800 shouldBeFound = self.darkBBoxes[self.noEdges] + self.brightBBoxes[self.noEdges]

801

802 boxesMeasured = []

803 for defect in defects:

804 boxesMeasured.append(defect.getBBox())

805

806 for expectedBBox in shouldBeFound:

807 self.assertIn(expectedBBox, boxesMeasured)

808

809 shouldBeMissed = self.darkBBoxes[self.onlyEdges] + self.brightBBoxes[self.onlyEdges]

810 for boxMissed in shouldBeMissed:

811 self.assertNotIn(boxMissed, boxesMeasured)

812

813 def valueThreshold(self, fileType, saturateAmpInFlat=False):

814 """Helper function to loop over flats and darks

815 to test thresholdType = 'VALUE'."""

816 config = copy.copy(self.defaultConfig)

817 config.thresholdType = 'VALUE'

818 task = self.defaultTask

819 task.config = config

820

821 for amp in self.flatExp.getDetector():

822 if amp.getName() == 'C:0,0':

823 regionC00 = amp.getBBox()

824

825 if fileType == 'dark':

826 exp = self.darkExp

827 shouldBeFound = self.brightBBoxes[self.noEdges]

828 else:

829 exp = self.flatExp

830 if saturateAmpInFlat:

831 exp.maskedImage[regionC00].image.array[:] = 0.0

832 # Amp C:0,0: minimum=(0, 0), maximum=(99, 50)

833 x = self.defaultConfig.nPixBorderUpDown

834 y = self.defaultConfig.nPixBorderLeftRight

835 width, height = regionC00.getEndX() - x, regionC00.getEndY() - y

836 # Defects code will mark whole saturated amp as defect box.

837 shouldBeFound = [Box2I(corner=Point2I(x, y), dimensions=Extent2I(width, height))]

838 else:

839 shouldBeFound = self.darkBBoxes[self.noEdges]

840 # Change the default a bit so it works for the

841 # existing simulated defects.

842 task.config.fracThresholdFlat = 0.9

843

844 defects = task._findHotAndColdPixels(exp)

845

846 boxesMeasured = []

847 for defect in defects:

848 boxesMeasured.append(defect.getBBox())

849

850 for expectedBBox in shouldBeFound:

851 self.assertIn(expectedBBox, boxesMeasured)

852

853 def test_valueThreshold(self):

854 for fileType in ['dark', 'flat']:

855 self.valueThreshold(fileType)

856 # stdDev = 0.0

857 self.valueThreshold('flat', saturateAmpInFlat=True)

858

859 def test_pixelCounting(self):

860 """Test that the number of defective pixels identified is as expected.

861 """

862 config = copy.copy(self.defaultConfig)

863 config.nPixBorderUpDown = 0

864 config.nPixBorderLeftRight = 0

865 task = self.defaultTask

866 task.config = config

867 defects = task._findHotAndColdPixels(self.flatExp)

868

869 defectArea = 0

870 for defect in defects:

871 defectArea += defect.getBBox().getArea()

872

873 # The columnar code will cover blocks of a column with

874 # on-and-off pixels, thus creating more bad pixels that what

875 # initially placed in self.brightDefects and self.darkDefects.

876 # Thus, defectArea should be >= crossCheck.

877 crossCheck = 0

878 for x, y, sx, sy in self.brightDefects:

879 crossCheck += sx*sy

880 for x, y, sx, sy in self.darkDefects:

881 crossCheck += sx*sy

882

883 # Test the result of _nPixFromDefects()

884 # via two different ways of calculating area.

885 self.assertEqual(defectArea, task._nPixFromDefects(defects))

886 # defectArea should be >= crossCheck

887 self.assertGreaterEqual(defectArea, crossCheck)

888

889 def test_getNumGoodPixels(self):

890 """Test the the number of pixels in the image not masked is as

891 expected.

892 """

893 testImage = self.flatExp.clone()

894 mi = testImage.maskedImage

895

896 imageSize = testImage.getBBox().getArea()

897 nGood = self.defaultTask._getNumGoodPixels(mi)

898

899 self.assertEqual(imageSize, nGood)

900

901 NODATABIT = mi.mask.getPlaneBitMask("NO_DATA")

902

903 noDataBox = Box2I(Point2I(31, 49), Extent2I(3, 6))

904 testImage.mask[noDataBox] |= NODATABIT

905

906 self.assertEqual(imageSize - noDataBox.getArea(), self.defaultTask._getNumGoodPixels(mi))

907 # check for misfire; we're setting NO_DATA here, not BAD

908 self.assertEqual(imageSize, self.defaultTask._getNumGoodPixels(mi, 'BAD'))

909

910 testImage.mask[noDataBox] ^= NODATABIT # XOR to reset what we did

911 self.assertEqual(imageSize, nGood)

912

913 BADBIT = mi.mask.getPlaneBitMask("BAD")

914 badBox = Box2I(Point2I(85, 98), Extent2I(4, 7))

915 testImage.mask[badBox] |= BADBIT

916

917 self.assertEqual(imageSize - badBox.getArea(), self.defaultTask._getNumGoodPixels(mi, 'BAD'))

918

919 def test_edgeMasking(self):

920 """Check that the right number of edge pixels are masked by

921 _setEdgeBits().

922 """

923 testImage = self.flatExp.clone()

924 mi = testImage.maskedImage

925

926 self.assertEqual(countMaskedPixels(mi, 'EDGE'), 0)

927 self.defaultTask._setEdgeBits(mi)

928

929 hEdge = self.defaultConfig.nPixBorderLeftRight

930 vEdge = self.defaultConfig.nPixBorderUpDown

931 xSize, ySize = mi.getDimensions()

932

933 nEdge = xSize*vEdge*2 + ySize*hEdge*2 - hEdge*vEdge*4

934

935 self.assertEqual(countMaskedPixels(mi, 'EDGE'), nEdge)

936

937 def test_badImage(self):

938 """Check that fully-bad images do not fail.

939 """

940 testImage = self.flatExp.clone()

941 testImage.image.array[:, :] = 125000

942

943 config = copy.copy(self.defaultConfig)

944 # Do not exclude any pixels, so the areas match.

945 config.nPixBorderUpDown = 0

946 config.nPixBorderLeftRight = 0

947

948 task = self.defaultTask

949 task.config = config

950 defects = task._findHotAndColdPixels(testImage)

951

952 defectArea = 0

953 for defect in defects:

954 defectArea += defect.getBBox().getArea()

955 self.assertEqual(defectArea, testImage.getBBox().getArea())

956

957 def test_e2vMidline(self):

958 """Test handling of E2V midline."""

959

960 class MockE2VAmp:

961 def __init__(self, name, bbox):

962 self._name = name

963 self._bbox = bbox

964

965 def getName(self):

966 return self._name

967

968 def getBBox(self):

969 return self._bbox

970

971 def __repr__(self):

972 return f"MockE2VAmp({self._name})"

973

974 class MockE2VDetector(list):

975 def __init__(self):

976 amps = []

977 for i in range(8):

978 name = f"C1{i}"

979 bbox = Box2I(corner=Point2I(i*512, 2002), dimensions=Extent2I(512, 2002))

980 amps.append(MockE2VAmp(name, bbox))

981 for i in reversed(range(8)):

982 name = f"C0{i}"

983 bbox = Box2I(corner=Point2I(i*512, 0), dimensions=Extent2I(512, 2002))

984 amps.append(MockE2VAmp(name, bbox))

985

986 super().__init__(amps)

987

988 def getBBox(self):

989 return Box2I(corner=Point2I(0, 0), dimensions=Extent2I(4096, 4004))

990

991 def getPhysicalType(self):

992 return "E2V"

993

994 class MockE2VExposure(afwImage.ExposureF):

995 def __init__(self, *args, **kwargs):

996 super().__init__(*args, **kwargs)

997

998 def setDetector(self, detector):

999 self._detector = detector

1000

1001 def getDetector(self):

1002 return self._detector

1003

1004 detector = MockE2VDetector()

1005 flat = MockE2VExposure(detector.getBBox())

1006 flat.setDetector(detector)

1007 flat.metadata["IMGTYPE"] = "FLAT"

1008 visitInfo = afwImage.VisitInfo(exposureTime=10.0, darkTime=10.0)

1009 flat.info.setVisitInfo(visitInfo)

1010

1011 flat.mask.array[:, :] = 0

1012

1013 rng = np.random.RandomState(0)

1014

1015 # Make a flat, and make a very dark midline.

1016 flat.image.array[:, :] = rng.normal(1.0, 0.01, flat.image.array.shape)

1017 flat.image.array[2001, :] = 0.0

1018 flat.image.array[2002, :] = 0.0

1019

1020 # First test that with midline checking off it is masked.

1021 config = cpPipe.MeasureDefectsTask.ConfigClass()

1022 config.thresholdType = "VALUE"

1023 config.fracThresholdFlat = 0.9

1024 config.e2vMidlineBreakNRow = 0

1025 task = cpPipe.MeasureDefectsTask(config=config)

1026

1027 defects = task._findHotAndColdPixels(flat)

1028

1029 # There should be 16 defects (one for each amp).

1030 self.assertEqual(len(defects), len(detector))

1031

1032 flat.mask.array[:, :] = 0

1033 for defect in defects:

1034 flat.mask[defect.getBBox()].array |= 1

1035

1036 np.testing.assert_array_equal(flat.mask.array[2001, :], 1)

1037 np.testing.assert_array_equal(flat.mask.array[2002, :], 1)

1038

1039 # Test again *ignoring* the midline.

1040 config = cpPipe.MeasureDefectsTask.ConfigClass()

1041 config.thresholdType = "VALUE"

1042 config.fracThresholdFlat = 0.9

1043 config.e2vMidlineBreakNRow = 1

1044 task = cpPipe.MeasureDefectsTask(config=config)

1045

1046 flat.mask.array[:, :] = 0

1047 defects = task._findHotAndColdPixels(flat)

1048

1049 self.assertEqual(len(defects), 0)

1050

1051 # Test again *requiring* the midline mask.

1052 config = cpPipe.MeasureDefectsTask.ConfigClass()

1053 config.thresholdType = "VALUE"

1054 config.fracThresholdFlat = 0.9

1055 config.e2vMidlineBreakNRow = 1

1056 config.e2vMidlineBreakOption = "MASK"

1057 task = cpPipe.MeasureDefectsTask(config=config)

1058

1059 flat.mask.array[:, :] = 0

1060 flat.image.array[:, :] = rng.normal(1.0, 0.01, flat.image.array.shape)

1061

1062 defects = task._findHotAndColdPixels(flat)

1063

1064 # There should be 16 defects (one for each amp).

1065 self.assertEqual(len(defects), len(detector))

1066

1067 flat.mask.array[:, :] = 0

1068 for defect in defects:

1069 flat.mask[defect.getBBox()].array |= 1

1070

1071 np.testing.assert_array_equal(flat.mask.array[2001, :], 1)

1072 np.testing.assert_array_equal(flat.mask.array[2002, :], 1)

1073

1074

1075class TestMemory(lsst.utils.tests.MemoryTestCase):

1076 pass

1077

1078

1079def setup_module(module):

1080 lsst.utils.tests.init()

1081

1082

1083if __name__ == "__main__": 1083 ↛ 1084line 1083 didn't jump to line 1084 because the condition on line 1083 was never true

1084 lsst.utils.tests.init()

1085 unittest.main()

Coverage for tests/test_defects.py: 8%

512 statements