Coverage for tests/test_isrTask.py: 12%

325 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2023-02-04 03:29 -0800

1# 

2# LSST Data Management System 

3# Copyright 2008-2017 AURA/LSST. 

4# 

5# This product includes software developed by the 

6# LSST Project (http://www.lsst.org/). 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <https://www.lsstcorp.org/LegalNotices/>. 

21# 

22 

23import unittest 

24import numpy as np 

25 

26import lsst.afw.image as afwImage 

27import lsst.ip.isr.isrMock as isrMock 

28import lsst.utils.tests 

29from lsst.ip.isr.isrTask import (IsrTask, IsrTaskConfig) 

30from lsst.ip.isr.isrQa import IsrQaConfig 

31from lsst.pipe.base import Struct 

32 

33 

34def countMaskedPixels(maskedImage, maskPlane): 

35 """Function to count the number of masked pixels of a given type. 

36 

37 Parameters 

38 ---------- 

39 maskedImage : `lsst.afw.image.MaskedImage` 

40 Image to measure the mask on. 

41 maskPlane : `str` 

42 Name of the mask plane to count 

43 

44 Returns 

45 ------- 

46 nMask : `int` 

47 Number of masked pixels. 

48 """ 

49 bitMask = maskedImage.getMask().getPlaneBitMask(maskPlane) 

50 isBit = maskedImage.getMask().getArray() & bitMask > 0 

51 numBit = np.sum(isBit) 

52 return numBit 

53 

54 

55def computeImageMedianAndStd(image): 

56 """Function to calculate median and std of image data. 

57 

58 Parameters 

59 ---------- 

60 image : `lsst.afw.image.Image` 

61 Image to measure statistics on. 

62 

63 Returns 

64 ------- 

65 median : `float` 

66 Image median. 

67 std : `float` 

68 Image stddev. 

69 """ 

70 median = np.nanmedian(image.getArray()) 

71 std = np.nanstd(image.getArray()) 

72 return (median, std) 

73 

74 

75class IsrTaskTestCases(lsst.utils.tests.TestCase): 

76 """Test IsrTask methods with trimmed raw data. 

77 """ 

78 def setUp(self): 

79 self.config = IsrTaskConfig() 

80 self.config.qa = IsrQaConfig() 

81 self.task = IsrTask(config=self.config) 

82 self.camera = isrMock.IsrMock().getCamera() 

83 

84 self.inputExp = isrMock.TrimmedRawMock().run() 

85 self.amp = self.inputExp.getDetector()[0] 

86 self.mi = self.inputExp.getMaskedImage() 

87 

88 def validateIsrData(self, results): 

89 """results should be a struct with components that are 

90 not None if included in the configuration file. 

91 """ 

92 self.assertIsInstance(results, Struct) 

93 if self.config.doBias is True: 

94 self.assertIsNotNone(results.bias) 

95 if self.config.doDark is True: 

96 self.assertIsNotNone(results.dark) 

97 if self.config.doFlat is True: 

98 self.assertIsNotNone(results.flat) 

99 if self.config.doFringe is True: 

100 self.assertIsNotNone(results.fringes) 

101 if self.config.doDefect is True: 

102 self.assertIsNotNone(results.defects) 

103 if self.config.doBrighterFatter is True: 

104 self.assertIsNotNone(results.bfKernel) 

105 if self.config.doAttachTransmissionCurve is True: 

106 self.assertIsNotNone(results.opticsTransmission) 

107 self.assertIsNotNone(results.filterTransmission) 

108 self.assertIsNotNone(results.sensorTransmission) 

109 self.assertIsNotNone(results.atmosphereTransmission) 

110 

111 def test_ensureExposure(self): 

112 """Test that an exposure has a usable instance class. 

113 """ 

114 self.assertIsInstance(self.task.ensureExposure(self.inputExp, self.camera, 0), 

115 afwImage.Exposure) 

116 

117 def test_convertItoF(self): 

118 """Test conversion from integer to floating point pixels. 

119 """ 

120 result = self.task.convertIntToFloat(self.inputExp) 

121 self.assertEqual(result.getImage().getArray().dtype, np.dtype("float32")) 

122 self.assertEqual(result, self.inputExp) 

123 

124 def test_updateVariance(self): 

125 """Expect The variance image should have a larger median value after 

126 this operation. 

127 """ 

128 statBefore = computeImageMedianAndStd(self.inputExp.variance[self.amp.getBBox()]) 

129 self.task.updateVariance(self.inputExp, self.amp) 

130 statAfter = computeImageMedianAndStd(self.inputExp.variance[self.amp.getBBox()]) 

131 self.assertGreater(statAfter[0], statBefore[0]) 

132 self.assertFloatsAlmostEqual(statBefore[0], 0.0, atol=1e-2) 

133 self.assertFloatsAlmostEqual(statAfter[0], 8170.0195, atol=1e-2) 

134 

135 def test_darkCorrection(self): 

136 """Expect the median image value should decrease after this operation. 

137 """ 

138 darkIm = isrMock.DarkMock().run() 

139 

140 statBefore = computeImageMedianAndStd(self.inputExp.image[self.amp.getBBox()]) 

141 self.task.darkCorrection(self.inputExp, darkIm) 

142 statAfter = computeImageMedianAndStd(self.inputExp.image[self.amp.getBBox()]) 

143 self.assertLess(statAfter[0], statBefore[0]) 

144 self.assertFloatsAlmostEqual(statBefore[0], 8070.0195, atol=1e-2) 

145 self.assertFloatsAlmostEqual(statAfter[0], 8045.7773, atol=1e-2) 

146 

147 def test_darkCorrection_noVisitInfo(self): 

148 """Expect the median image value should decrease after this operation. 

149 """ 

150 darkIm = isrMock.DarkMock().run() 

151 darkIm.getInfo().setVisitInfo(None) 

152 

153 statBefore = computeImageMedianAndStd(self.inputExp.image[self.amp.getBBox()]) 

154 self.task.darkCorrection(self.inputExp, darkIm) 

155 statAfter = computeImageMedianAndStd(self.inputExp.image[self.amp.getBBox()]) 

156 self.assertLess(statAfter[0], statBefore[0]) 

157 self.assertFloatsAlmostEqual(statBefore[0], 8070.0195, atol=1e-2) 

158 self.assertFloatsAlmostEqual(statAfter[0], 8045.7773, atol=1e-2) 

159 

160 def test_flatCorrection(self): 

161 """Expect the image median should increase (divide by < 1). 

162 """ 

163 flatIm = isrMock.FlatMock().run() 

164 

165 statBefore = computeImageMedianAndStd(self.inputExp.image[self.amp.getBBox()]) 

166 self.task.flatCorrection(self.inputExp, flatIm) 

167 statAfter = computeImageMedianAndStd(self.inputExp.image[self.amp.getBBox()]) 

168 self.assertGreater(statAfter[1], statBefore[1]) 

169 self.assertFloatsAlmostEqual(statAfter[1], 147407.02, atol=1e-2) 

170 self.assertFloatsAlmostEqual(statBefore[1], 147.55304, atol=1e-2) 

171 

172 def test_saturationDetection(self): 

173 """Expect the saturation level detection/masking to scale with 

174 threshold. 

175 """ 

176 ampB = self.amp.rebuild() 

177 ampB.setSaturation(9000.0) 

178 self.task.saturationDetection(self.inputExp, ampB.finish()) 

179 countBefore = countMaskedPixels(self.mi, "SAT") 

180 

181 ampB.setSaturation(8250.0) 

182 self.task.saturationDetection(self.inputExp, ampB.finish()) 

183 countAfter = countMaskedPixels(self.mi, "SAT") 

184 

185 self.assertLessEqual(countBefore, countAfter) 

186 self.assertEqual(countBefore, 43) 

187 self.assertEqual(countAfter, 136) 

188 

189 def test_measureBackground(self): 

190 """Expect the background measurement runs successfully and to save 

191 metadata values. 

192 """ 

193 self.config.qa.flatness.meshX = 20 

194 self.config.qa.flatness.meshY = 20 

195 self.task.measureBackground(self.inputExp, self.config.qa) 

196 self.assertIsNotNone(self.inputExp.getMetadata().getScalar('SKYLEVEL')) 

197 

198 def test_flatContext(self): 

199 """Expect the flat context manager runs successfully (applying both 

200 flat and dark within the context), and results in the same 

201 image data after completion. 

202 """ 

203 darkExp = isrMock.DarkMock().run() 

204 flatExp = isrMock.FlatMock().run() 

205 

206 mi = self.inputExp.getMaskedImage().clone() 

207 with self.task.flatContext(self.inputExp, flatExp, darkExp): 

208 contextStat = computeImageMedianAndStd(self.inputExp.getMaskedImage().getImage()) 

209 self.assertFloatsAlmostEqual(contextStat[0], 37165.594, atol=1e-2) 

210 

211 self.assertMaskedImagesAlmostEqual(mi, self.inputExp.getMaskedImage()) 

212 

213 

214class IsrTaskUnTrimmedTestCases(lsst.utils.tests.TestCase): 

215 """Test IsrTask methods using untrimmed raw data. 

216 """ 

217 def setUp(self): 

218 self.config = IsrTaskConfig() 

219 self.config.qa = IsrQaConfig() 

220 self.task = IsrTask(config=self.config) 

221 

222 self.mockConfig = isrMock.IsrMockConfig() 

223 self.mockConfig.isTrimmed = False 

224 self.doGenerateImage = True 

225 self.dataContainer = isrMock.MockDataContainer(config=self.mockConfig) 

226 self.camera = isrMock.IsrMock(config=self.mockConfig).getCamera() 

227 

228 self.inputExp = isrMock.RawMock(config=self.mockConfig).run() 

229 self.amp = self.inputExp.getDetector()[0] 

230 self.mi = self.inputExp.getMaskedImage() 

231 

232 def batchSetConfiguration(self, value): 

233 """Set the configuration state to a consistent value. 

234 

235 Disable options we do not need as well. 

236 

237 Parameters 

238 ---------- 

239 value : `bool` 

240 Value to switch common ISR configuration options to. 

241 """ 

242 self.config.qa.flatness.meshX = 20 

243 self.config.qa.flatness.meshY = 20 

244 self.config.doWrite = False 

245 self.config.doLinearize = False 

246 self.config.doCrosstalk = False 

247 

248 self.config.doConvertIntToFloat = value 

249 self.config.doSaturation = value 

250 self.config.doSuspect = value 

251 self.config.doSetBadRegions = value 

252 self.config.doOverscan = value 

253 self.config.doBias = value 

254 self.config.doVariance = value 

255 self.config.doWidenSaturationTrails = value 

256 self.config.doBrighterFatter = value 

257 self.config.doDefect = value 

258 self.config.doSaturationInterpolation = value 

259 self.config.doDark = value 

260 self.config.doStrayLight = value 

261 self.config.doFlat = value 

262 self.config.doFringe = value 

263 self.config.doMeasureBackground = value 

264 self.config.doVignette = value 

265 self.config.doAttachTransmissionCurve = value 

266 self.config.doUseOpticsTransmission = value 

267 self.config.doUseFilterTransmission = value 

268 self.config.doUseSensorTransmission = value 

269 self.config.doUseAtmosphereTransmission = value 

270 self.config.qa.saveStats = value 

271 self.config.qa.doThumbnailOss = value 

272 self.config.qa.doThumbnailFlattened = value 

273 

274 self.config.doApplyGains = not value 

275 self.config.doCameraSpecificMasking = value 

276 

277 def validateIsrResults(self): 

278 """results should be a struct with components that are 

279 not None if included in the configuration file. 

280 

281 Returns 

282 ------- 

283 results : `pipeBase.Struct` 

284 Results struct generated from the current ISR configuration. 

285 """ 

286 self.task = IsrTask(config=self.config) 

287 results = self.task.run(self.inputExp, 

288 camera=self.camera, 

289 bias=self.dataContainer.get("bias"), 

290 dark=self.dataContainer.get("dark"), 

291 flat=self.dataContainer.get("flat"), 

292 bfKernel=self.dataContainer.get("bfKernel"), 

293 defects=self.dataContainer.get("defects"), 

294 fringes=Struct(fringes=self.dataContainer.get("fringe"), seed=1234), 

295 opticsTransmission=self.dataContainer.get("transmission_"), 

296 filterTransmission=self.dataContainer.get("transmission_"), 

297 sensorTransmission=self.dataContainer.get("transmission_"), 

298 atmosphereTransmission=self.dataContainer.get("transmission_") 

299 ) 

300 

301 self.assertIsInstance(results, Struct) 

302 self.assertIsInstance(results.exposure, afwImage.Exposure) 

303 return results 

304 

305 def test_run_allTrue(self): 

306 """Expect successful run with expected outputs when all non-exclusive 

307 configuration options are on. 

308 

309 Output results should be tested more precisely by the 

310 individual function tests. 

311 

312 """ 

313 self.batchSetConfiguration(True) 

314 self.validateIsrResults() 

315 

316 def test_run_allFalse(self): 

317 """Expect successful run with expected outputs when all non-exclusive 

318 configuration options are off. 

319 

320 Output results should be tested more precisely by the 

321 individual function tests. 

322 

323 """ 

324 self.batchSetConfiguration(False) 

325 self.validateIsrResults() 

326 

327 def test_failCases(self): 

328 """Expect failure with crosstalk enabled. 

329 

330 Output results should be tested more precisely by the 

331 individual function tests. 

332 """ 

333 self.batchSetConfiguration(True) 

334 

335 # This breaks it 

336 self.config.doCrosstalk = True 

337 

338 with self.assertRaises(RuntimeError): 

339 self.validateIsrResults() 

340 

341 def test_maskingCase_negativeVariance(self): 

342 """Test masking cases of configuration parameters. 

343 """ 

344 self.batchSetConfiguration(True) 

345 self.config.overscan.fitType = "POLY" 

346 self.config.overscan.order = 1 

347 

348 self.config.doSaturation = False 

349 self.config.doWidenSaturationTrails = False 

350 self.config.doSaturationInterpolation = False 

351 self.config.doSuspect = False 

352 self.config.doSetBadRegions = False 

353 self.config.doDefect = False 

354 self.config.doBrighterFatter = False 

355 

356 self.config.maskNegativeVariance = True 

357 self.config.doInterpolate = False 

358 

359 results = self.validateIsrResults() 

360 

361 self.assertEqual(countMaskedPixels(results.exposure, "SAT"), 0) 

362 self.assertEqual(countMaskedPixels(results.exposure, "INTRP"), 0) 

363 self.assertEqual(countMaskedPixels(results.exposure, "SUSPECT"), 0) 

364 self.assertEqual(countMaskedPixels(results.exposure, "BAD"), 40800) 

365 

366 def test_maskingCase_noMasking(self): 

367 """Test masking cases of configuration parameters. 

368 """ 

369 self.batchSetConfiguration(True) 

370 self.config.overscan.fitType = "POLY" 

371 self.config.overscan.order = 1 

372 

373 self.config.doSaturation = False 

374 self.config.doWidenSaturationTrails = False 

375 self.config.doSaturationInterpolation = False 

376 self.config.doSuspect = False 

377 self.config.doSetBadRegions = False 

378 self.config.doDefect = False 

379 self.config.doBrighterFatter = False 

380 

381 self.config.maskNegativeVariance = False 

382 self.config.doInterpolate = False 

383 

384 results = self.validateIsrResults() 

385 

386 self.assertEqual(countMaskedPixels(results.exposure, "SAT"), 0) 

387 self.assertEqual(countMaskedPixels(results.exposure, "INTRP"), 0) 

388 self.assertEqual(countMaskedPixels(results.exposure, "SUSPECT"), 0) 

389 self.assertEqual(countMaskedPixels(results.exposure, "BAD"), 0) 

390 

391 def test_maskingCase_satMasking(self): 

392 """Test masking cases of configuration parameters. 

393 """ 

394 self.batchSetConfiguration(True) 

395 self.config.overscan.fitType = "POLY" 

396 self.config.overscan.order = 1 

397 

398 self.config.saturation = 20000.0 

399 self.config.doSaturation = True 

400 self.config.doWidenSaturationTrails = True 

401 

402 self.config.doSaturationInterpolation = False 

403 self.config.doSuspect = False 

404 self.config.doSetBadRegions = False 

405 self.config.doDefect = False 

406 self.config.doBrighterFatter = False 

407 

408 self.config.maskNegativeVariance = False # These are mock images. 

409 

410 results = self.validateIsrResults() 

411 

412 self.assertEqual(countMaskedPixels(results.exposure, "SAT"), 0) 

413 self.assertEqual(countMaskedPixels(results.exposure, "INTRP"), 0) 

414 self.assertEqual(countMaskedPixels(results.exposure, "SUSPECT"), 0) 

415 self.assertEqual(countMaskedPixels(results.exposure, "BAD"), 0) 

416 

417 def test_maskingCase_satMaskingAndInterp(self): 

418 """Test masking cases of configuration parameters. 

419 """ 

420 self.batchSetConfiguration(True) 

421 self.config.overscan.fitType = "POLY" 

422 self.config.overscan.order = 1 

423 

424 self.config.saturation = 20000.0 

425 self.config.doSaturation = True 

426 self.config.doWidenSaturationTrails = True 

427 self.config.doSaturationInterpolation = True 

428 

429 self.config.doSuspect = False 

430 self.config.doSetBadRegions = False 

431 self.config.doDefect = False 

432 self.config.doBrighterFatter = False 

433 

434 self.config.maskNegativeVariance = False # These are mock images. 

435 

436 results = self.validateIsrResults() 

437 

438 self.assertEqual(countMaskedPixels(results.exposure, "SAT"), 0) 

439 self.assertEqual(countMaskedPixels(results.exposure, "INTRP"), 0) 

440 self.assertEqual(countMaskedPixels(results.exposure, "SUSPECT"), 0) 

441 self.assertEqual(countMaskedPixels(results.exposure, "BAD"), 0) 

442 

443 def test_maskingCase_throughEdge(self): 

444 """Test masking cases of configuration parameters. 

445 """ 

446 self.batchSetConfiguration(True) 

447 self.config.overscan.fitType = "POLY" 

448 self.config.overscan.order = 1 

449 

450 self.config.saturation = 20000.0 

451 self.config.doSaturation = True 

452 self.config.doWidenSaturationTrails = True 

453 self.config.doSaturationInterpolation = True 

454 self.config.numEdgeSuspect = 5 

455 self.config.doSuspect = True 

456 

457 self.config.doSetBadRegions = False 

458 self.config.doDefect = False 

459 self.config.doBrighterFatter = False 

460 

461 self.config.maskNegativeVariance = False # These are mock images. 

462 

463 results = self.validateIsrResults() 

464 

465 self.assertEqual(countMaskedPixels(results.exposure, "SAT"), 0) 

466 self.assertEqual(countMaskedPixels(results.exposure, "INTRP"), 0) 

467 self.assertEqual(countMaskedPixels(results.exposure, "SUSPECT"), 0) 

468 self.assertEqual(countMaskedPixels(results.exposure, "BAD"), 0) 

469 

470 def test_maskingCase_throughDefects(self): 

471 """Test masking cases of configuration parameters. 

472 """ 

473 self.batchSetConfiguration(True) 

474 self.config.overscan.fitType = "POLY" 

475 self.config.overscan.order = 1 

476 

477 self.config.saturation = 20000.0 

478 self.config.doSaturation = True 

479 self.config.doWidenSaturationTrails = True 

480 self.config.doSaturationInterpolation = True 

481 self.config.numEdgeSuspect = 5 

482 self.config.doSuspect = True 

483 self.config.doDefect = True 

484 

485 self.config.doSetBadRegions = False 

486 self.config.doBrighterFatter = False 

487 

488 self.config.maskNegativeVariance = False # These are mock images. 

489 

490 results = self.validateIsrResults() 

491 

492 self.assertEqual(countMaskedPixels(results.exposure, "SAT"), 0) 

493 self.assertEqual(countMaskedPixels(results.exposure, "INTRP"), 2000) 

494 self.assertEqual(countMaskedPixels(results.exposure, "SUSPECT"), 3940) 

495 self.assertEqual(countMaskedPixels(results.exposure, "BAD"), 2000) 

496 

497 def test_maskingCase_throughDefectsAmpEdges(self): 

498 """Test masking cases of configuration parameters. 

499 """ 

500 self.batchSetConfiguration(True) 

501 self.config.overscan.fitType = "POLY" 

502 self.config.overscan.order = 1 

503 

504 self.config.saturation = 20000.0 

505 self.config.doSaturation = True 

506 self.config.doWidenSaturationTrails = True 

507 self.config.doSaturationInterpolation = True 

508 self.config.numEdgeSuspect = 5 

509 self.config.doSuspect = True 

510 self.config.doDefect = True 

511 self.config.edgeMaskLevel = 'AMP' 

512 

513 self.config.doSetBadRegions = False 

514 self.config.doBrighterFatter = False 

515 

516 self.config.maskNegativeVariance = False # These are mock images. 

517 

518 results = self.validateIsrResults() 

519 

520 self.assertEqual(countMaskedPixels(results.exposure, "SAT"), 0) 

521 self.assertEqual(countMaskedPixels(results.exposure, "INTRP"), 2000) 

522 self.assertEqual(countMaskedPixels(results.exposure, "SUSPECT"), 11280) 

523 self.assertEqual(countMaskedPixels(results.exposure, "BAD"), 2000) 

524 

525 def test_maskingCase_throughBad(self): 

526 """Test masking cases of configuration parameters. 

527 """ 

528 self.batchSetConfiguration(True) 

529 self.config.overscan.fitType = "POLY" 

530 self.config.overscan.order = 1 

531 

532 self.config.saturation = 20000.0 

533 self.config.doSaturation = True 

534 self.config.doWidenSaturationTrails = True 

535 self.config.doSaturationInterpolation = True 

536 

537 self.config.doSuspect = True 

538 self.config.doDefect = True 

539 self.config.doSetBadRegions = True 

540 self.config.doBrighterFatter = False 

541 

542 self.config.maskNegativeVariance = False # These are mock images. 

543 

544 results = self.validateIsrResults() 

545 

546 self.assertEqual(countMaskedPixels(results.exposure, "SAT"), 0) 

547 self.assertEqual(countMaskedPixels(results.exposure, "INTRP"), 2000) 

548 self.assertEqual(countMaskedPixels(results.exposure, "SUSPECT"), 0) 

549 self.assertEqual(countMaskedPixels(results.exposure, "BAD"), 2000) 

550 

551 

552class MemoryTester(lsst.utils.tests.MemoryTestCase): 

553 pass 

554 

555 

556def setup_module(module): 

557 lsst.utils.tests.init() 

558 

559 

560if __name__ == "__main__": 560 ↛ 561line 560 didn't jump to line 561, because the condition on line 560 was never true

561 lsst.utils.tests.init() 

562 unittest.main(failfast=True)