Coverage for tests/test_isrTask.py: 12%

328 statements  

« prev     ^ index     » next       coverage.py v7.3.0, created at 2023-08-19 11:10 +0000

1# 

2# LSST Data Management System 

3# Copyright 2008-2017 AURA/LSST. 

4# 

5# This product includes software developed by the 

6# LSST Project (http://www.lsst.org/). 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <https://www.lsstcorp.org/LegalNotices/>. 

21# 

22 

23import unittest 

24import numpy as np 

25 

26import lsst.afw.image as afwImage 

27import lsst.ip.isr.isrMock as isrMock 

28import lsst.utils.tests 

29from lsst.ip.isr.isrTask import (IsrTask, IsrTaskConfig) 

30from lsst.ip.isr.isrQa import IsrQaConfig 

31from lsst.pipe.base import Struct 

32 

33 

34def countMaskedPixels(maskedImage, maskPlane): 

35 """Function to count the number of masked pixels of a given type. 

36 

37 Parameters 

38 ---------- 

39 maskedImage : `lsst.afw.image.MaskedImage` 

40 Image to measure the mask on. 

41 maskPlane : `str` 

42 Name of the mask plane to count 

43 

44 Returns 

45 ------- 

46 nMask : `int` 

47 Number of masked pixels. 

48 """ 

49 bitMask = maskedImage.getMask().getPlaneBitMask(maskPlane) 

50 isBit = maskedImage.getMask().getArray() & bitMask > 0 

51 numBit = np.sum(isBit) 

52 return numBit 

53 

54 

55def computeImageMedianAndStd(image): 

56 """Function to calculate median and std of image data. 

57 

58 Parameters 

59 ---------- 

60 image : `lsst.afw.image.Image` 

61 Image to measure statistics on. 

62 

63 Returns 

64 ------- 

65 median : `float` 

66 Image median. 

67 std : `float` 

68 Image stddev. 

69 """ 

70 median = np.nanmedian(image.getArray()) 

71 std = np.nanstd(image.getArray()) 

72 return (median, std) 

73 

74 

75class IsrTaskTestCases(lsst.utils.tests.TestCase): 

76 """Test IsrTask methods with trimmed raw data. 

77 """ 

78 def setUp(self): 

79 self.config = IsrTaskConfig() 

80 self.config.overscan.doParallelOverscan = True 

81 self.config.qa = IsrQaConfig() 

82 self.task = IsrTask(config=self.config) 

83 self.camera = isrMock.IsrMock().getCamera() 

84 

85 self.inputExp = isrMock.TrimmedRawMock().run() 

86 self.amp = self.inputExp.getDetector()[0] 

87 self.mi = self.inputExp.getMaskedImage() 

88 

89 def validateIsrData(self, results): 

90 """results should be a struct with components that are 

91 not None if included in the configuration file. 

92 """ 

93 self.assertIsInstance(results, Struct) 

94 if self.config.doBias is True: 

95 self.assertIsNotNone(results.bias) 

96 if self.config.doDark is True: 

97 self.assertIsNotNone(results.dark) 

98 if self.config.doFlat is True: 

99 self.assertIsNotNone(results.flat) 

100 if self.config.doFringe is True: 

101 self.assertIsNotNone(results.fringes) 

102 if self.config.doDefect is True: 

103 self.assertIsNotNone(results.defects) 

104 if self.config.doBrighterFatter is True: 

105 self.assertIsNotNone(results.bfKernel) 

106 if self.config.doAttachTransmissionCurve is True: 

107 self.assertIsNotNone(results.opticsTransmission) 

108 self.assertIsNotNone(results.filterTransmission) 

109 self.assertIsNotNone(results.sensorTransmission) 

110 self.assertIsNotNone(results.atmosphereTransmission) 

111 

112 def test_ensureExposure(self): 

113 """Test that an exposure has a usable instance class. 

114 """ 

115 self.assertIsInstance(self.task.ensureExposure(self.inputExp, self.camera, 0), 

116 afwImage.Exposure) 

117 

118 def test_convertItoF(self): 

119 """Test conversion from integer to floating point pixels. 

120 """ 

121 result = self.task.convertIntToFloat(self.inputExp) 

122 self.assertEqual(result.getImage().getArray().dtype, np.dtype("float32")) 

123 self.assertEqual(result, self.inputExp) 

124 

125 def test_updateVariance(self): 

126 """Expect The variance image should have a larger median value after 

127 this operation. 

128 """ 

129 statBefore = computeImageMedianAndStd(self.inputExp.variance[self.amp.getBBox()]) 

130 self.task.updateVariance(self.inputExp, self.amp) 

131 statAfter = computeImageMedianAndStd(self.inputExp.variance[self.amp.getBBox()]) 

132 self.assertGreater(statAfter[0], statBefore[0]) 

133 self.assertFloatsAlmostEqual(statBefore[0], 0.0, atol=1e-2) 

134 self.assertFloatsAlmostEqual(statAfter[0], 8170.0195, atol=1e-2) 

135 

136 def test_darkCorrection(self): 

137 """Expect the median image value should decrease after this operation. 

138 """ 

139 darkIm = isrMock.DarkMock().run() 

140 

141 statBefore = computeImageMedianAndStd(self.inputExp.image[self.amp.getBBox()]) 

142 self.task.darkCorrection(self.inputExp, darkIm) 

143 statAfter = computeImageMedianAndStd(self.inputExp.image[self.amp.getBBox()]) 

144 self.assertLess(statAfter[0], statBefore[0]) 

145 self.assertFloatsAlmostEqual(statBefore[0], 8070.0195, atol=1e-2) 

146 self.assertFloatsAlmostEqual(statAfter[0], 8045.7773, atol=1e-2) 

147 

148 def test_darkCorrection_noVisitInfo(self): 

149 """Expect the median image value should decrease after this operation. 

150 """ 

151 darkIm = isrMock.DarkMock().run() 

152 darkIm.getInfo().setVisitInfo(None) 

153 

154 statBefore = computeImageMedianAndStd(self.inputExp.image[self.amp.getBBox()]) 

155 self.task.darkCorrection(self.inputExp, darkIm) 

156 statAfter = computeImageMedianAndStd(self.inputExp.image[self.amp.getBBox()]) 

157 self.assertLess(statAfter[0], statBefore[0]) 

158 self.assertFloatsAlmostEqual(statBefore[0], 8070.0195, atol=1e-2) 

159 self.assertFloatsAlmostEqual(statAfter[0], 8045.7773, atol=1e-2) 

160 

161 def test_flatCorrection(self): 

162 """Expect the image median should increase (divide by < 1). 

163 """ 

164 flatIm = isrMock.FlatMock().run() 

165 

166 statBefore = computeImageMedianAndStd(self.inputExp.image[self.amp.getBBox()]) 

167 self.task.flatCorrection(self.inputExp, flatIm) 

168 statAfter = computeImageMedianAndStd(self.inputExp.image[self.amp.getBBox()]) 

169 self.assertGreater(statAfter[1], statBefore[1]) 

170 self.assertFloatsAlmostEqual(statAfter[1], 147407.02, atol=1e-2) 

171 self.assertFloatsAlmostEqual(statBefore[1], 147.55304, atol=1e-2) 

172 

173 def test_saturationDetection(self): 

174 """Expect the saturation level detection/masking to scale with 

175 threshold. 

176 """ 

177 ampB = self.amp.rebuild() 

178 ampB.setSaturation(9000.0) 

179 self.task.saturationDetection(self.inputExp, ampB.finish()) 

180 countBefore = countMaskedPixels(self.mi, "SAT") 

181 

182 ampB.setSaturation(8250.0) 

183 self.task.saturationDetection(self.inputExp, ampB.finish()) 

184 countAfter = countMaskedPixels(self.mi, "SAT") 

185 

186 self.assertLessEqual(countBefore, countAfter) 

187 self.assertEqual(countBefore, 43) 

188 self.assertEqual(countAfter, 136) 

189 

190 def test_measureBackground(self): 

191 """Expect the background measurement runs successfully and to save 

192 metadata values. 

193 """ 

194 self.config.qa.flatness.meshX = 20 

195 self.config.qa.flatness.meshY = 20 

196 self.task.measureBackground(self.inputExp, self.config.qa) 

197 self.assertIsNotNone(self.inputExp.getMetadata().getScalar('SKYLEVEL')) 

198 

199 def test_flatContext(self): 

200 """Expect the flat context manager runs successfully (applying both 

201 flat and dark within the context), and results in the same 

202 image data after completion. 

203 """ 

204 darkExp = isrMock.DarkMock().run() 

205 flatExp = isrMock.FlatMock().run() 

206 

207 mi = self.inputExp.getMaskedImage().clone() 

208 with self.task.flatContext(self.inputExp, flatExp, darkExp): 

209 contextStat = computeImageMedianAndStd(self.inputExp.getMaskedImage().getImage()) 

210 self.assertFloatsAlmostEqual(contextStat[0], 37165.594, atol=1e-2) 

211 

212 self.assertMaskedImagesAlmostEqual(mi, self.inputExp.getMaskedImage()) 

213 

214 

215class IsrTaskUnTrimmedTestCases(lsst.utils.tests.TestCase): 

216 """Test IsrTask methods using untrimmed raw data. 

217 """ 

218 def setUp(self): 

219 self.config = IsrTaskConfig() 

220 self.config.overscan.doParallelOverscan = True 

221 self.config.qa = IsrQaConfig() 

222 self.task = IsrTask(config=self.config) 

223 

224 self.mockConfig = isrMock.IsrMockConfig() 

225 self.mockConfig.isTrimmed = False 

226 self.doGenerateImage = True 

227 self.dataContainer = isrMock.MockDataContainer(config=self.mockConfig) 

228 self.camera = isrMock.IsrMock(config=self.mockConfig).getCamera() 

229 

230 self.inputExp = isrMock.RawMock(config=self.mockConfig).run() 

231 self.amp = self.inputExp.getDetector()[0] 

232 self.mi = self.inputExp.getMaskedImage() 

233 

234 def batchSetConfiguration(self, value): 

235 """Set the configuration state to a consistent value. 

236 

237 Disable options we do not need as well. 

238 

239 Parameters 

240 ---------- 

241 value : `bool` 

242 Value to switch common ISR configuration options to. 

243 """ 

244 self.config.qa.flatness.meshX = 20 

245 self.config.qa.flatness.meshY = 20 

246 self.config.doWrite = False 

247 self.config.doLinearize = False 

248 self.config.doCrosstalk = False 

249 

250 self.config.doConvertIntToFloat = value 

251 self.config.doSaturation = value 

252 self.config.doSuspect = value 

253 self.config.doSetBadRegions = value 

254 self.config.doOverscan = value 

255 self.config.doBias = value 

256 self.config.doVariance = value 

257 self.config.doWidenSaturationTrails = value 

258 self.config.doBrighterFatter = value 

259 self.config.doDefect = value 

260 self.config.doSaturationInterpolation = value 

261 self.config.doDark = value 

262 self.config.doStrayLight = value 

263 self.config.doFlat = value 

264 self.config.doFringe = value 

265 self.config.doMeasureBackground = value 

266 self.config.doVignette = value 

267 self.config.doAttachTransmissionCurve = value 

268 self.config.doUseOpticsTransmission = value 

269 self.config.doUseFilterTransmission = value 

270 self.config.doUseSensorTransmission = value 

271 self.config.doUseAtmosphereTransmission = value 

272 self.config.qa.saveStats = value 

273 self.config.qa.doThumbnailOss = value 

274 self.config.qa.doThumbnailFlattened = value 

275 

276 self.config.doApplyGains = not value 

277 self.config.doCameraSpecificMasking = value 

278 

279 def validateIsrResults(self): 

280 """results should be a struct with components that are 

281 not None if included in the configuration file. 

282 

283 Returns 

284 ------- 

285 results : `pipeBase.Struct` 

286 Results struct generated from the current ISR configuration. 

287 """ 

288 self.task = IsrTask(config=self.config) 

289 results = self.task.run(self.inputExp, 

290 camera=self.camera, 

291 bias=self.dataContainer.get("bias"), 

292 dark=self.dataContainer.get("dark"), 

293 flat=self.dataContainer.get("flat"), 

294 bfKernel=self.dataContainer.get("bfKernel"), 

295 defects=self.dataContainer.get("defects"), 

296 fringes=Struct(fringes=self.dataContainer.get("fringe"), seed=1234), 

297 opticsTransmission=self.dataContainer.get("transmission_"), 

298 filterTransmission=self.dataContainer.get("transmission_"), 

299 sensorTransmission=self.dataContainer.get("transmission_"), 

300 atmosphereTransmission=self.dataContainer.get("transmission_") 

301 ) 

302 

303 self.assertIsInstance(results, Struct) 

304 self.assertIsInstance(results.exposure, afwImage.Exposure) 

305 return results 

306 

307 def test_run_allTrue(self): 

308 """Expect successful run with expected outputs when all non-exclusive 

309 configuration options are on. 

310 

311 Output results should be tested more precisely by the 

312 individual function tests. 

313 

314 """ 

315 self.batchSetConfiguration(True) 

316 self.validateIsrResults() 

317 

318 def test_run_allFalse(self): 

319 """Expect successful run with expected outputs when all non-exclusive 

320 configuration options are off. 

321 

322 Output results should be tested more precisely by the 

323 individual function tests. 

324 

325 """ 

326 self.batchSetConfiguration(False) 

327 self.validateIsrResults() 

328 

329 def test_failCases(self): 

330 """Expect failure with crosstalk enabled. 

331 

332 Output results should be tested more precisely by the 

333 individual function tests. 

334 """ 

335 self.batchSetConfiguration(True) 

336 

337 # This breaks it 

338 self.config.doCrosstalk = True 

339 

340 with self.assertRaises(RuntimeError): 

341 self.validateIsrResults() 

342 

343 def test_maskingCase_negativeVariance(self): 

344 """Test masking cases of configuration parameters. 

345 """ 

346 self.batchSetConfiguration(True) 

347 self.config.overscan.doParallelOverscan = False 

348 self.config.overscan.fitType = "POLY" 

349 self.config.overscan.order = 1 

350 

351 self.config.doSaturation = False 

352 self.config.doWidenSaturationTrails = False 

353 self.config.doSaturationInterpolation = False 

354 self.config.doSuspect = False 

355 self.config.doSetBadRegions = False 

356 self.config.doDefect = False 

357 self.config.doBrighterFatter = False 

358 

359 self.config.maskNegativeVariance = True 

360 self.config.doInterpolate = False 

361 

362 results = self.validateIsrResults() 

363 

364 self.assertEqual(countMaskedPixels(results.exposure, "SAT"), 0) 

365 self.assertEqual(countMaskedPixels(results.exposure, "INTRP"), 0) 

366 self.assertEqual(countMaskedPixels(results.exposure, "SUSPECT"), 0) 

367 self.assertEqual(countMaskedPixels(results.exposure, "BAD"), 40800) 

368 

369 def test_maskingCase_noMasking(self): 

370 """Test masking cases of configuration parameters. 

371 """ 

372 self.batchSetConfiguration(True) 

373 self.config.overscan.fitType = "POLY" 

374 self.config.overscan.order = 1 

375 

376 self.config.doSaturation = False 

377 self.config.doWidenSaturationTrails = False 

378 self.config.doSaturationInterpolation = False 

379 self.config.doSuspect = False 

380 self.config.doSetBadRegions = False 

381 self.config.doDefect = False 

382 self.config.doBrighterFatter = False 

383 

384 self.config.maskNegativeVariance = False 

385 self.config.doInterpolate = False 

386 

387 results = self.validateIsrResults() 

388 

389 self.assertEqual(countMaskedPixels(results.exposure, "SAT"), 0) 

390 self.assertEqual(countMaskedPixels(results.exposure, "INTRP"), 0) 

391 self.assertEqual(countMaskedPixels(results.exposure, "SUSPECT"), 0) 

392 self.assertEqual(countMaskedPixels(results.exposure, "BAD"), 0) 

393 

394 def test_maskingCase_satMasking(self): 

395 """Test masking cases of configuration parameters. 

396 """ 

397 self.batchSetConfiguration(True) 

398 self.config.overscan.fitType = "POLY" 

399 self.config.overscan.order = 1 

400 

401 self.config.saturation = 20000.0 

402 self.config.doSaturation = True 

403 self.config.doWidenSaturationTrails = True 

404 

405 self.config.doSaturationInterpolation = False 

406 self.config.doSuspect = False 

407 self.config.doSetBadRegions = False 

408 self.config.doDefect = False 

409 self.config.doBrighterFatter = False 

410 

411 self.config.maskNegativeVariance = False # These are mock images. 

412 

413 results = self.validateIsrResults() 

414 

415 self.assertEqual(countMaskedPixels(results.exposure, "SAT"), 0) 

416 self.assertEqual(countMaskedPixels(results.exposure, "INTRP"), 0) 

417 self.assertEqual(countMaskedPixels(results.exposure, "SUSPECT"), 0) 

418 self.assertEqual(countMaskedPixels(results.exposure, "BAD"), 0) 

419 

420 def test_maskingCase_satMaskingAndInterp(self): 

421 """Test masking cases of configuration parameters. 

422 """ 

423 self.batchSetConfiguration(True) 

424 self.config.overscan.fitType = "POLY" 

425 self.config.overscan.order = 1 

426 

427 self.config.saturation = 20000.0 

428 self.config.doSaturation = True 

429 self.config.doWidenSaturationTrails = True 

430 self.config.doSaturationInterpolation = True 

431 

432 self.config.doSuspect = False 

433 self.config.doSetBadRegions = False 

434 self.config.doDefect = False 

435 self.config.doBrighterFatter = False 

436 

437 self.config.maskNegativeVariance = False # These are mock images. 

438 

439 results = self.validateIsrResults() 

440 

441 self.assertEqual(countMaskedPixels(results.exposure, "SAT"), 0) 

442 self.assertEqual(countMaskedPixels(results.exposure, "INTRP"), 0) 

443 self.assertEqual(countMaskedPixels(results.exposure, "SUSPECT"), 0) 

444 self.assertEqual(countMaskedPixels(results.exposure, "BAD"), 0) 

445 

446 def test_maskingCase_throughEdge(self): 

447 """Test masking cases of configuration parameters. 

448 """ 

449 self.batchSetConfiguration(True) 

450 self.config.overscan.fitType = "POLY" 

451 self.config.overscan.order = 1 

452 

453 self.config.saturation = 20000.0 

454 self.config.doSaturation = True 

455 self.config.doWidenSaturationTrails = True 

456 self.config.doSaturationInterpolation = True 

457 self.config.numEdgeSuspect = 5 

458 self.config.doSuspect = True 

459 

460 self.config.doSetBadRegions = False 

461 self.config.doDefect = False 

462 self.config.doBrighterFatter = False 

463 

464 self.config.maskNegativeVariance = False # These are mock images. 

465 

466 results = self.validateIsrResults() 

467 

468 self.assertEqual(countMaskedPixels(results.exposure, "SAT"), 0) 

469 self.assertEqual(countMaskedPixels(results.exposure, "INTRP"), 0) 

470 self.assertEqual(countMaskedPixels(results.exposure, "SUSPECT"), 0) 

471 self.assertEqual(countMaskedPixels(results.exposure, "BAD"), 0) 

472 

473 def test_maskingCase_throughDefects(self): 

474 """Test masking cases of configuration parameters. 

475 """ 

476 self.batchSetConfiguration(True) 

477 self.config.overscan.fitType = "POLY" 

478 self.config.overscan.order = 1 

479 

480 self.config.saturation = 20000.0 

481 self.config.doSaturation = True 

482 self.config.doWidenSaturationTrails = True 

483 self.config.doSaturationInterpolation = True 

484 self.config.numEdgeSuspect = 5 

485 self.config.doSuspect = True 

486 self.config.doDefect = True 

487 

488 self.config.doSetBadRegions = False 

489 self.config.doBrighterFatter = False 

490 

491 self.config.maskNegativeVariance = False # These are mock images. 

492 

493 results = self.validateIsrResults() 

494 

495 self.assertEqual(countMaskedPixels(results.exposure, "SAT"), 0) 

496 self.assertEqual(countMaskedPixels(results.exposure, "INTRP"), 2000) 

497 self.assertEqual(countMaskedPixels(results.exposure, "SUSPECT"), 3940) 

498 self.assertEqual(countMaskedPixels(results.exposure, "BAD"), 2000) 

499 

500 def test_maskingCase_throughDefectsAmpEdges(self): 

501 """Test masking cases of configuration parameters. 

502 """ 

503 self.batchSetConfiguration(True) 

504 self.config.overscan.fitType = "POLY" 

505 self.config.overscan.order = 1 

506 

507 self.config.saturation = 20000.0 

508 self.config.doSaturation = True 

509 self.config.doWidenSaturationTrails = True 

510 self.config.doSaturationInterpolation = True 

511 self.config.numEdgeSuspect = 5 

512 self.config.doSuspect = True 

513 self.config.doDefect = True 

514 self.config.edgeMaskLevel = 'AMP' 

515 

516 self.config.doSetBadRegions = False 

517 self.config.doBrighterFatter = False 

518 

519 self.config.maskNegativeVariance = False # These are mock images. 

520 

521 results = self.validateIsrResults() 

522 

523 self.assertEqual(countMaskedPixels(results.exposure, "SAT"), 0) 

524 self.assertEqual(countMaskedPixels(results.exposure, "INTRP"), 2000) 

525 self.assertEqual(countMaskedPixels(results.exposure, "SUSPECT"), 11280) 

526 self.assertEqual(countMaskedPixels(results.exposure, "BAD"), 2000) 

527 

528 def test_maskingCase_throughBad(self): 

529 """Test masking cases of configuration parameters. 

530 """ 

531 self.batchSetConfiguration(True) 

532 self.config.overscan.fitType = "POLY" 

533 self.config.overscan.order = 1 

534 

535 self.config.saturation = 20000.0 

536 self.config.doSaturation = True 

537 self.config.doWidenSaturationTrails = True 

538 self.config.doSaturationInterpolation = True 

539 

540 self.config.doSuspect = True 

541 self.config.doDefect = True 

542 self.config.doSetBadRegions = True 

543 self.config.doBrighterFatter = False 

544 

545 self.config.maskNegativeVariance = False # These are mock images. 

546 

547 results = self.validateIsrResults() 

548 

549 self.assertEqual(countMaskedPixels(results.exposure, "SAT"), 0) 

550 self.assertEqual(countMaskedPixels(results.exposure, "INTRP"), 2000) 

551 self.assertEqual(countMaskedPixels(results.exposure, "SUSPECT"), 0) 

552 self.assertEqual(countMaskedPixels(results.exposure, "BAD"), 2000) 

553 

554 

555class MemoryTester(lsst.utils.tests.MemoryTestCase): 

556 pass 

557 

558 

559def setup_module(module): 

560 lsst.utils.tests.init() 

561 

562 

563if __name__ == "__main__": 563 ↛ 564line 563 didn't jump to line 564, because the condition on line 563 was never true

564 lsst.utils.tests.init() 

565 unittest.main(failfast=True)