Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# 

2# LSST Data Management System 

3# Copyright 2008-2017 AURA/LSST. 

4# 

5# This product includes software developed by the 

6# LSST Project (http://www.lsst.org/). 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <https://www.lsstcorp.org/LegalNotices/>. 

21# 

22 

23import unittest 

24import numpy as np 

25 

26import lsst.afw.image as afwImage 

27import lsst.ip.isr.isrMock as isrMock 

28import lsst.utils.tests 

29from lsst.ip.isr.isrTask import (IsrTask, IsrTaskConfig) 

30from lsst.ip.isr.isrQa import IsrQaConfig 

31from lsst.pipe.base import Struct 

32 

33 

34def countMaskedPixels(maskedImage, maskPlane): 

35 """Function to count the number of masked pixels of a given type. 

36 

37 Parameters 

38 ---------- 

39 maskedImage : `lsst.afw.image.MaskedImage` 

40 Image to measure the mask on. 

41 maskPlane : `str` 

42 Name of the mask plane to count 

43 

44 Returns 

45 ------- 

46 nMask : `int` 

47 Number of masked pixels. 

48 """ 

49 bitMask = maskedImage.getMask().getPlaneBitMask(maskPlane) 

50 isBit = maskedImage.getMask().getArray() & bitMask > 0 

51 numBit = np.sum(isBit) 

52 return numBit 

53 

54 

55def computeImageMedianAndStd(image): 

56 """Function to calculate median and std of image data. 

57 

58 Parameters 

59 ---------- 

60 image : `lsst.afw.image.Image` 

61 Image to measure statistics on. 

62 

63 Returns 

64 ------- 

65 median : `float` 

66 Image median. 

67 std : `float` 

68 Image stddev. 

69 """ 

70 median = np.nanmedian(image.getArray()) 

71 std = np.nanstd(image.getArray()) 

72 return (median, std) 

73 

74 

75class IsrTaskTestCases(lsst.utils.tests.TestCase): 

76 """Test IsrTask methods with trimmed raw data. 

77 """ 

78 def setUp(self): 

79 self.config = IsrTaskConfig() 

80 self.config.qa = IsrQaConfig() 

81 self.task = IsrTask(config=self.config) 

82 self.dataRef = isrMock.DataRefMock() 

83 self.camera = isrMock.IsrMock().getCamera() 

84 

85 self.inputExp = isrMock.TrimmedRawMock().run() 

86 self.amp = self.inputExp.getDetector()[0] 

87 self.mi = self.inputExp.getMaskedImage() 

88 

89 def validateIsrData(self, results): 

90 """results should be a struct with components that are 

91 not None if included in the configuration file. 

92 """ 

93 self.assertIsInstance(results, Struct) 

94 if self.config.doBias is True: 

95 self.assertIsNotNone(results.bias) 

96 if self.config.doDark is True: 

97 self.assertIsNotNone(results.dark) 

98 if self.config.doFlat is True: 

99 self.assertIsNotNone(results.flat) 

100 if self.config.doFringe is True: 

101 self.assertIsNotNone(results.fringes) 

102 if self.config.doDefect is True: 

103 self.assertIsNotNone(results.defects) 

104 if self.config.doBrighterFatter is True: 

105 self.assertIsNotNone(results.bfKernel) 

106 if self.config.doAttachTransmissionCurve is True: 

107 self.assertIsNotNone(results.opticsTransmission) 

108 self.assertIsNotNone(results.filterTransmission) 

109 self.assertIsNotNone(results.sensorTransmission) 

110 self.assertIsNotNone(results.atmosphereTransmission) 

111 

112 def test_readIsrData_noTrans(self): 

113 """Test that all necessary calibration frames are retrieved. 

114 """ 

115 self.config.doAttachTransmissionCurve = False 

116 self.task = IsrTask(config=self.config) 

117 results = self.task.readIsrData(self.dataRef, self.inputExp) 

118 self.validateIsrData(results) 

119 

120 def test_readIsrData_withTrans(self): 

121 """Test that all necessary calibration frames are retrieved. 

122 """ 

123 self.config.doAttachTransmissionCurve = True 

124 self.task = IsrTask(config=self.config) 

125 results = self.task.readIsrData(self.dataRef, self.inputExp) 

126 self.validateIsrData(results) 

127 

128 def test_ensureExposure(self): 

129 """Test that an exposure has a usable instance class. 

130 """ 

131 self.assertIsInstance(self.task.ensureExposure(self.inputExp, self.camera, 0), 

132 afwImage.Exposure) 

133 

134 def test_convertItoF(self): 

135 """Test conversion from integer to floating point pixels. 

136 """ 

137 result = self.task.convertIntToFloat(self.inputExp) 

138 self.assertEqual(result.getImage().getArray().dtype, np.dtype("float32")) 

139 self.assertEqual(result, self.inputExp) 

140 

141 def test_updateVariance(self): 

142 """Expect The variance image should have a larger median value after 

143 this operation. 

144 """ 

145 statBefore = computeImageMedianAndStd(self.inputExp.variance[self.amp.getBBox()]) 

146 self.task.updateVariance(self.inputExp, self.amp) 

147 statAfter = computeImageMedianAndStd(self.inputExp.variance[self.amp.getBBox()]) 

148 self.assertGreater(statAfter[0], statBefore[0]) 

149 self.assertFloatsAlmostEqual(statBefore[0], 0.0, atol=1e-2) 

150 self.assertFloatsAlmostEqual(statAfter[0], 8170.0195, atol=1e-2) 

151 

152 def test_darkCorrection(self): 

153 """Expect the median image value should decrease after this operation. 

154 """ 

155 darkIm = isrMock.DarkMock().run() 

156 

157 statBefore = computeImageMedianAndStd(self.inputExp.image[self.amp.getBBox()]) 

158 self.task.darkCorrection(self.inputExp, darkIm) 

159 statAfter = computeImageMedianAndStd(self.inputExp.image[self.amp.getBBox()]) 

160 self.assertLess(statAfter[0], statBefore[0]) 

161 self.assertFloatsAlmostEqual(statBefore[0], 8070.0195, atol=1e-2) 

162 self.assertFloatsAlmostEqual(statAfter[0], 8045.7773, atol=1e-2) 

163 

164 def test_darkCorrection_noVisitInfo(self): 

165 """Expect the median image value should decrease after this operation. 

166 """ 

167 darkIm = isrMock.DarkMock().run() 

168 darkIm.getInfo().setVisitInfo(None) 

169 

170 statBefore = computeImageMedianAndStd(self.inputExp.image[self.amp.getBBox()]) 

171 self.task.darkCorrection(self.inputExp, darkIm) 

172 statAfter = computeImageMedianAndStd(self.inputExp.image[self.amp.getBBox()]) 

173 self.assertLess(statAfter[0], statBefore[0]) 

174 self.assertFloatsAlmostEqual(statBefore[0], 8070.0195, atol=1e-2) 

175 self.assertFloatsAlmostEqual(statAfter[0], 8045.7773, atol=1e-2) 

176 

177 def test_flatCorrection(self): 

178 """Expect the image median should increase (divide by < 1). 

179 """ 

180 flatIm = isrMock.FlatMock().run() 

181 

182 statBefore = computeImageMedianAndStd(self.inputExp.image[self.amp.getBBox()]) 

183 self.task.flatCorrection(self.inputExp, flatIm) 

184 statAfter = computeImageMedianAndStd(self.inputExp.image[self.amp.getBBox()]) 

185 self.assertGreater(statAfter[1], statBefore[1]) 

186 self.assertFloatsAlmostEqual(statAfter[1], 147407.02, atol=1e-2) 

187 self.assertFloatsAlmostEqual(statBefore[1], 147.55304, atol=1e-2) 

188 

189 def test_saturationDetection(self): 

190 """Expect the saturation level detection/masking to scale with 

191 threshold. 

192 """ 

193 ampB = self.amp.rebuild() 

194 ampB.setSaturation(9000.0) 

195 self.task.saturationDetection(self.inputExp, ampB.finish()) 

196 countBefore = countMaskedPixels(self.mi, "SAT") 

197 

198 ampB.setSaturation(8250.0) 

199 self.task.saturationDetection(self.inputExp, ampB.finish()) 

200 countAfter = countMaskedPixels(self.mi, "SAT") 

201 

202 self.assertLessEqual(countBefore, countAfter) 

203 self.assertEqual(countBefore, 43) 

204 self.assertEqual(countAfter, 136) 

205 

206 def test_measureBackground(self): 

207 """Expect the background measurement runs successfully and to save 

208 metadata values. 

209 """ 

210 self.config.qa.flatness.meshX = 20 

211 self.config.qa.flatness.meshY = 20 

212 self.task.measureBackground(self.inputExp, self.config.qa) 

213 self.assertIsNotNone(self.inputExp.getMetadata().getScalar('SKYLEVEL')) 

214 

215 def test_flatContext(self): 

216 """Expect the flat context manager runs successfully (applying both 

217 flat and dark within the context), and results in the same 

218 image data after completion. 

219 """ 

220 darkExp = isrMock.DarkMock().run() 

221 flatExp = isrMock.FlatMock().run() 

222 

223 mi = self.inputExp.getMaskedImage().clone() 

224 with self.task.flatContext(self.inputExp, flatExp, darkExp): 

225 contextStat = computeImageMedianAndStd(self.inputExp.getMaskedImage().getImage()) 

226 self.assertFloatsAlmostEqual(contextStat[0], 37165.594, atol=1e-2) 

227 

228 self.assertMaskedImagesAlmostEqual(mi, self.inputExp.getMaskedImage()) 

229 

230 

231class IsrTaskUnTrimmedTestCases(lsst.utils.tests.TestCase): 

232 """Test IsrTask methods using untrimmed raw data. 

233 """ 

234 def setUp(self): 

235 self.config = IsrTaskConfig() 

236 self.config.qa = IsrQaConfig() 

237 self.task = IsrTask(config=self.config) 

238 

239 self.mockConfig = isrMock.IsrMockConfig() 

240 self.mockConfig.isTrimmed = False 

241 self.doGenerateImage = True 

242 self.dataRef = isrMock.DataRefMock(config=self.mockConfig) 

243 self.camera = isrMock.IsrMock(config=self.mockConfig).getCamera() 

244 

245 self.inputExp = isrMock.RawMock(config=self.mockConfig).run() 

246 self.amp = self.inputExp.getDetector()[0] 

247 self.mi = self.inputExp.getMaskedImage() 

248 

249 def batchSetConfiguration(self, value): 

250 """Set the configuration state to a consistent value. 

251 

252 Disable options we do not need as well. 

253 

254 Parameters 

255 ---------- 

256 value : `bool` 

257 Value to switch common ISR configuration options to. 

258 """ 

259 self.config.qa.flatness.meshX = 20 

260 self.config.qa.flatness.meshY = 20 

261 self.config.doWrite = False 

262 self.config.doLinearize = False 

263 self.config.doCrosstalk = False 

264 

265 self.config.doConvertIntToFloat = value 

266 self.config.doSaturation = value 

267 self.config.doSuspect = value 

268 self.config.doSetBadRegions = value 

269 self.config.doOverscan = value 

270 self.config.doBias = value 

271 self.config.doVariance = value 

272 self.config.doWidenSaturationTrails = value 

273 self.config.doBrighterFatter = value 

274 self.config.doDefect = value 

275 self.config.doSaturationInterpolation = value 

276 self.config.doDark = value 

277 self.config.doStrayLight = value 

278 self.config.doFlat = value 

279 self.config.doFringe = value 

280 self.config.doAddDistortionModel = value 

281 self.config.doMeasureBackground = value 

282 self.config.doVignette = value 

283 self.config.doAttachTransmissionCurve = value 

284 self.config.doUseOpticsTransmission = value 

285 self.config.doUseFilterTransmission = value 

286 self.config.doUseSensorTransmission = value 

287 self.config.doUseAtmosphereTransmission = value 

288 self.config.qa.saveStats = value 

289 self.config.qa.doThumbnailOss = value 

290 self.config.qa.doThumbnailFlattened = value 

291 

292 self.config.doApplyGains = not value 

293 self.config.doCameraSpecificMasking = value 

294 self.config.vignette.doWriteVignettePolygon = value 

295 

296 def validateIsrResults(self): 

297 """results should be a struct with components that are 

298 not None if included in the configuration file. 

299 

300 Returns 

301 ------- 

302 results : `pipeBase.Struct` 

303 Results struct generated from the current ISR configuration. 

304 """ 

305 self.task = IsrTask(config=self.config) 

306 results = self.task.run(self.inputExp, 

307 camera=self.camera, 

308 bias=self.dataRef.get("bias"), 

309 dark=self.dataRef.get("dark"), 

310 flat=self.dataRef.get("flat"), 

311 bfKernel=self.dataRef.get("bfKernel"), 

312 defects=self.dataRef.get("defects"), 

313 fringes=Struct(fringes=self.dataRef.get("fringe"), seed=1234), 

314 opticsTransmission=self.dataRef.get("transmission_"), 

315 filterTransmission=self.dataRef.get("transmission_"), 

316 sensorTransmission=self.dataRef.get("transmission_"), 

317 atmosphereTransmission=self.dataRef.get("transmission_") 

318 ) 

319 

320 self.assertIsInstance(results, Struct) 

321 self.assertIsInstance(results.exposure, afwImage.Exposure) 

322 return results 

323 

324 def test_overscanCorrection(self): 

325 """Expect that this should reduce the image variance with a full fit. 

326 The default fitType of MEDIAN will reduce the median value. 

327 

328 This needs to operate on a RawMock() to have overscan data to use. 

329 

330 The output types may be different when fitType != MEDIAN. 

331 """ 

332 statBefore = computeImageMedianAndStd(self.inputExp.image[self.amp.getRawDataBBox()]) 

333 oscanResults = self.task.overscanCorrection(self.inputExp, self.amp) 

334 self.assertIsInstance(oscanResults, Struct) 

335 self.assertIsInstance(oscanResults.imageFit, float) 

336 self.assertIsInstance(oscanResults.overscanFit, float) 

337 self.assertIsInstance(oscanResults.overscanImage, afwImage.MaskedImageF) 

338 

339 statAfter = computeImageMedianAndStd(self.inputExp.image[self.amp.getRawDataBBox()]) 

340 self.assertLess(statAfter[0], statBefore[0]) 

341 

342 def test_overscanCorrectionMedianPerRow(self): 

343 """Expect that this should reduce the image variance with a full fit. 

344 fitType of MEDIAN_PER_ROW will reduce the median value. 

345 

346 This needs to operate on a RawMock() to have overscan data to use. 

347 

348 The output types may be different when fitType != MEDIAN_PER_ROW. 

349 """ 

350 self.config.overscan.fitType = 'MEDIAN_PER_ROW' 

351 statBefore = computeImageMedianAndStd(self.inputExp.image[self.amp.getRawDataBBox()]) 

352 oscanResults = self.task.overscanCorrection(self.inputExp, self.amp) 

353 self.assertIsInstance(oscanResults, Struct) 

354 self.assertIsInstance(oscanResults.imageFit, afwImage.ImageF) 

355 self.assertIsInstance(oscanResults.overscanFit, afwImage.ImageF) 

356 self.assertIsInstance(oscanResults.overscanImage, afwImage.MaskedImageF) 

357 

358 statAfter = computeImageMedianAndStd(self.inputExp.image[self.amp.getRawDataBBox()]) 

359 self.assertLess(statAfter[0], statBefore[0]) 

360 

361 def test_runDataRef(self): 

362 """Expect a dataRef to be handled correctly. 

363 """ 

364 self.config.doLinearize = False 

365 self.config.doWrite = False 

366 self.task = IsrTask(config=self.config) 

367 results = self.task.runDataRef(self.dataRef) 

368 

369 self.assertIsInstance(results, Struct) 

370 self.assertIsInstance(results.exposure, afwImage.Exposure) 

371 

372 def test_run_allTrue(self): 

373 """Expect successful run with expected outputs when all non-exclusive 

374 configuration options are on. 

375 

376 Output results should be tested more precisely by the 

377 individual function tests. 

378 

379 """ 

380 self.batchSetConfiguration(True) 

381 self.validateIsrResults() 

382 

383 def test_run_allFalse(self): 

384 """Expect successful run with expected outputs when all non-exclusive 

385 configuration options are off. 

386 

387 Output results should be tested more precisely by the 

388 individual function tests. 

389 

390 """ 

391 self.batchSetConfiguration(False) 

392 self.validateIsrResults() 

393 

394 def test_failCases(self): 

395 """Expect failure with crosstalk enabled. 

396 

397 Output results should be tested more precisely by the 

398 individual function tests. 

399 """ 

400 self.batchSetConfiguration(True) 

401 

402 # This breaks it 

403 self.config.doCrosstalk = True 

404 

405 with self.assertRaises(RuntimeError): 

406 self.validateIsrResults() 

407 

408 def test_maskingCase_noMasking(self): 

409 """Test masking cases of configuration parameters. 

410 """ 

411 self.batchSetConfiguration(True) 

412 self.config.overscanFitType = "POLY" 

413 self.config.overscanOrder = 1 

414 

415 self.config.doSaturation = False 

416 self.config.doWidenSaturationTrails = False 

417 self.config.doSaturationInterpolation = False 

418 self.config.doSuspect = False 

419 self.config.doSetBadRegions = False 

420 self.config.doDefect = False 

421 self.config.doBrighterFatter = False 

422 

423 results = self.validateIsrResults() 

424 

425 self.assertEqual(countMaskedPixels(results.exposure, "SAT"), 0) 

426 self.assertEqual(countMaskedPixels(results.exposure, "INTRP"), 0) 

427 self.assertEqual(countMaskedPixels(results.exposure, "SUSPECT"), 0) 

428 self.assertEqual(countMaskedPixels(results.exposure, "BAD"), 0) 

429 

430 def test_maskingCase_satMasking(self): 

431 """Test masking cases of configuration parameters. 

432 """ 

433 self.batchSetConfiguration(True) 

434 self.config.overscanFitType = "POLY" 

435 self.config.overscanOrder = 1 

436 

437 self.config.saturation = 20000.0 

438 self.config.doSaturation = True 

439 self.config.doWidenSaturationTrails = True 

440 

441 self.config.doSaturationInterpolation = False 

442 self.config.doSuspect = False 

443 self.config.doSetBadRegions = False 

444 self.config.doDefect = False 

445 self.config.doBrighterFatter = False 

446 

447 results = self.validateIsrResults() 

448 

449 self.assertEqual(countMaskedPixels(results.exposure, "SAT"), 0) 

450 self.assertEqual(countMaskedPixels(results.exposure, "INTRP"), 0) 

451 self.assertEqual(countMaskedPixels(results.exposure, "SUSPECT"), 0) 

452 self.assertEqual(countMaskedPixels(results.exposure, "BAD"), 0) 

453 

454 def test_maskingCase_satMaskingAndInterp(self): 

455 """Test masking cases of configuration parameters. 

456 """ 

457 self.batchSetConfiguration(True) 

458 self.config.overscanFitType = "POLY" 

459 self.config.overscanOrder = 1 

460 

461 self.config.saturation = 20000.0 

462 self.config.doSaturation = True 

463 self.config.doWidenSaturationTrails = True 

464 self.config.doSaturationInterpolation = True 

465 

466 self.config.doSuspect = False 

467 self.config.doSetBadRegions = False 

468 self.config.doDefect = False 

469 self.config.doBrighterFatter = False 

470 

471 results = self.validateIsrResults() 

472 

473 self.assertEqual(countMaskedPixels(results.exposure, "SAT"), 0) 

474 self.assertEqual(countMaskedPixels(results.exposure, "INTRP"), 0) 

475 self.assertEqual(countMaskedPixels(results.exposure, "SUSPECT"), 0) 

476 self.assertEqual(countMaskedPixels(results.exposure, "BAD"), 0) 

477 

478 def test_maskingCase_throughEdge(self): 

479 """Test masking cases of configuration parameters. 

480 """ 

481 self.batchSetConfiguration(True) 

482 self.config.overscanFitType = "POLY" 

483 self.config.overscanOrder = 1 

484 

485 self.config.saturation = 20000.0 

486 self.config.doSaturation = True 

487 self.config.doWidenSaturationTrails = True 

488 self.config.doSaturationInterpolation = True 

489 self.config.numEdgeSuspect = 5 

490 self.config.doSuspect = True 

491 

492 self.config.doSetBadRegions = False 

493 self.config.doDefect = False 

494 self.config.doBrighterFatter = False 

495 

496 results = self.validateIsrResults() 

497 

498 self.assertEqual(countMaskedPixels(results.exposure, "SAT"), 0) 

499 self.assertEqual(countMaskedPixels(results.exposure, "INTRP"), 0) 

500 self.assertEqual(countMaskedPixels(results.exposure, "SUSPECT"), 0) 

501 self.assertEqual(countMaskedPixels(results.exposure, "BAD"), 0) 

502 

503 def test_maskingCase_throughDefects(self): 

504 """Test masking cases of configuration parameters. 

505 """ 

506 self.batchSetConfiguration(True) 

507 self.config.overscanFitType = "POLY" 

508 self.config.overscanOrder = 1 

509 

510 self.config.saturation = 20000.0 

511 self.config.doSaturation = True 

512 self.config.doWidenSaturationTrails = True 

513 self.config.doSaturationInterpolation = True 

514 self.config.numEdgeSuspect = 5 

515 self.config.doSuspect = True 

516 self.config.doDefect = True 

517 

518 self.config.doSetBadRegions = False 

519 self.config.doBrighterFatter = False 

520 

521 results = self.validateIsrResults() 

522 

523 self.assertEqual(countMaskedPixels(results.exposure, "SAT"), 0) 

524 self.assertEqual(countMaskedPixels(results.exposure, "INTRP"), 2000) 

525 self.assertEqual(countMaskedPixels(results.exposure, "SUSPECT"), 3940) 

526 self.assertEqual(countMaskedPixels(results.exposure, "BAD"), 2000) 

527 

528 def test_maskingCase_throughBad(self): 

529 """Test masking cases of configuration parameters. 

530 """ 

531 self.batchSetConfiguration(True) 

532 self.config.overscanFitType = "POLY" 

533 self.config.overscanOrder = 1 

534 

535 self.config.saturation = 20000.0 

536 self.config.doSaturation = True 

537 self.config.doWidenSaturationTrails = True 

538 self.config.doSaturationInterpolation = True 

539 

540 self.config.doSuspect = True 

541 self.config.doDefect = True 

542 self.config.doSetBadRegions = True 

543 self.config.doBrighterFatter = False 

544 

545 results = self.validateIsrResults() 

546 

547 self.assertEqual(countMaskedPixels(results.exposure, "SAT"), 0) 

548 self.assertEqual(countMaskedPixels(results.exposure, "INTRP"), 2000) 

549 self.assertEqual(countMaskedPixels(results.exposure, "SUSPECT"), 0) 

550 self.assertEqual(countMaskedPixels(results.exposure, "BAD"), 2000) 

551 

552 

553class MemoryTester(lsst.utils.tests.MemoryTestCase): 

554 pass 

555 

556 

557def setup_module(module): 

558 lsst.utils.tests.init() 

559 

560 

561if __name__ == "__main__": 561 ↛ 562line 561 didn't jump to line 562, because the condition on line 561 was never true

562 lsst.utils.tests.init() 

563 unittest.main()