Coverage for tests/test_isrTaskLSST.py: 11%

265 statements  

« prev     ^ index     » next       coverage.py v7.5.1, created at 2024-05-15 02:10 -0700

1# 

2# LSST Data Management System 

3# Copyright 2008-2017 AURA/LSST. 

4# 

5# This product includes software developed by the 

6# LSST Project (http://www.lsst.org/). 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <https://www.lsstcorp.org/LegalNotices/>. 

21# 

22 

23import unittest 

24import numpy as np 

25 

26import lsst.afw.image as afwImage 

27import lsst.ip.isr.isrMockLSST as isrMockLSST 

28import lsst.utils.tests 

29from lsst.ip.isr.isrTaskLSST import (IsrTaskLSST, IsrTaskLSSTConfig) 

30from lsst.ip.isr.crosstalk import CrosstalkCalib 

31from lsst.pipe.base import Struct 

32from lsst.ip.isr import PhotonTransferCurveDataset 

33 

34 

35class IsrTaskLSSTTestCases(lsst.utils.tests.TestCase): 

36 """Test IsrTaskLSST step-by-step 

37 to produce mock calibrations and apply ISR correction on a mock image. 

38 """ 

39 def doSetUp_mock(self, config): 

40 """Set up a mock image and calibration products with specified configs. 

41 

42 Parameters 

43 ---------- 

44 config : `lsst.ip.isr.IsrMockLSSTConfig` 

45 Configs to produce the mock image and calibration products. 

46 """ 

47 # Create mock image 

48 self.mock = isrMockLSST.IsrMockLSST(config=config) 

49 self.inputExp = self.mock.run() 

50 self.camera = self.mock.getCamera() 

51 self.detector = self.inputExp.getDetector() 

52 

53 # Get number of amps 

54 self.namp = len(self.detector.getAmplifiers()) 

55 

56 # Create mock bias 

57 self.bias = isrMockLSST.BiasMockLSST().run() 

58 

59 # Create mock CTI, by instatianting the class with default settings 

60 # so we can test the pipeline would run 

61 # TODO: Update with some mock CTI data DM-???? 

62 # self.cti = DeferredChargeCalib() 

63 

64 # Create mock flat 

65 self.flat = isrMockLSST.FlatMockLSST().run() 

66 

67 # Create mock dark 

68 self.dark = isrMockLSST.DarkMockLSST().run() 

69 

70 # Create mock brighter-fatter kernel 

71 self.bfkernel = isrMockLSST.BfKernelMockLSST().run() 

72 

73 # Create crosstalk calib with default coefficients matrix 

74 self.crosstalk = CrosstalkCalib(nAmp=self.namp) 

75 self.crosstalk.hasCrosstalk = True 

76 self.crosstalk.coeffs = isrMockLSST.CrosstalkCoeffMockLSST().run() 

77 

78 # Create mock defects 

79 self.defect = isrMockLSST.DefectMockLSST().run() 

80 

81 # Create mock PTC 

82 ampNames = [x.getName() for x in self.detector.getAmplifiers()] 

83 self.ptc = PhotonTransferCurveDataset(ampNames, 

84 ptcFitType='DUMMY_PTC', 

85 covMatrixSide=1) 

86 for ampName in ampNames: 

87 self.ptc.gain[ampName] = 3.5 # gain in e-/ADU 

88 self.ptc.noise[ampName] = 8.5 # read noise in ADU 

89 

90 def setMockConfigFalse(self): 

91 """Set all configs to produce mocks to False. 

92 """ 

93 self.mockConfig.isTrimmed = False 

94 self.mockConfig.doGenerateImage = True 

95 self.mockConfig.doGenerateData = False 

96 self.mockConfig.doGenerateAmpDict = False 

97 

98 self.mockConfig.doAddSky = True 

99 self.mockConfig.doAddSource = True 

100 

101 self.mockConfig.doAddBias = False 

102 self.mockConfig.doAddFringe = False 

103 self.mockConfig.doAddFlat = False 

104 self.mockConfig.doAddDark = False 

105 self.mockConfig.doApplyGain = False 

106 self.mockConfig.doAddCrosstalk = False 

107 self.mockConfig.doAddParallelOverscan = False 

108 self.mockConfig.doAddSerialOverscan = False 

109 

110 def setIsrConfig(self): 

111 """Set all configs corresponding to ISR steps to False. 

112 """ 

113 self.defaultAmpConfig = self.config.overscanCamera.\ 

114 getOverscanDetectorConfig(self.detector).defaultAmpConfig 

115 self.defaultAmpConfig.doSerialOverscan = False 

116 self.defaultAmpConfig.doParallelOverscanCrosstalk = False 

117 self.defaultAmpConfig.doParallelOverscan = False 

118 self.config.doDiffNonLinearCorrection = False 

119 self.config.doAssembleCcd = False 

120 self.config.doLinearize = False 

121 self.config.doCrosstalk = False 

122 self.config.doBias = False 

123 self.config.doGainsCorrection = False 

124 self.config.doApplyGains = False 

125 self.config.doDeferredCharge = False 

126 self.config.doVariance = False 

127 self.config.doDefect = False 

128 self.config.doNanMasking = False 

129 self.config.doWidenSaturationTrails = False 

130 self.config.doSaveInterpPixels = False 

131 self.config.doSetBadRegions = False 

132 self.config.doInterpolate = False 

133 self.config.doDark = False 

134 self.config.doBrighterFatter = False 

135 self.config.doFlat = False 

136 

137 def validateIsrResults(self): 

138 """Validate the ISR LSST pipeline by running it and checking the 

139 results format and compare the mean before and after ISR correction. 

140 

141 Returns 

142 ------- 

143 results : `pipeBase.Struct` 

144 Results struct generated from the current ISR configuration. 

145 """ 

146 self.task = IsrTaskLSST(config=self.config) 

147 

148 mockMean = 0. 

149 for amp in self.inputExp.getDetector(): 

150 bbox = amp.getRawDataBBox() 

151 ampData = self.inputExp.image[bbox] 

152 mockMean += np.nanmean(ampData.array) 

153 

154 # Not testing dnlLUT (not existant yet), deferred Charge, 

155 # linearizer, bfgains 

156 results = self.task.run(self.inputExp, 

157 camera=self.camera, 

158 bias=self.bias, 

159 ptc=self.ptc, 

160 crosstalk=self.crosstalk, 

161 # deferredChargeCalib=self.cti, 

162 defects=self.defect, 

163 bfKernel=self.bfkernel, 

164 dark=self.dark, 

165 flat=self.flat 

166 ) 

167 

168 outputMean = np.nanmean(results.outputExposure.image.array) 

169 # Test that the output has a smaller mean than the input mock 

170 self.assertLess(outputMean, mockMean) 

171 # Test that the output is a struct 

172 self.assertIsInstance(results, Struct) 

173 # Test that the output has an exposure with expected format 

174 self.assertIsInstance(results.exposure, afwImage.Exposure) 

175 

176 def test_run_serialOverscanCorrection(self): 

177 """Test up to serial overscan correction. 

178 """ 

179 self.mockConfig = isrMockLSST.IsrMockLSSTConfig() 

180 self.setMockConfigFalse() 

181 self.mockConfig.doAddSerialOverscan = True 

182 self.doSetUp_mock(config=self.mockConfig) 

183 

184 self.config = IsrTaskLSSTConfig() 

185 self.setIsrConfig() 

186 self.defaultAmpConfig.doSerialOverscan = True 

187 

188 self.validateIsrResults() 

189 

190 def test_run_parallelOverscanCrosstalkCorrection(self): 

191 """Test up to parallel overscan crosstalk correction. 

192 """ 

193 # TODO: DM-43286 

194 pass 

195 

196 def test_run_parallelOverscanCorrection(self): 

197 """Test up to parallel overscan correction. 

198 """ 

199 

200 self.mockConfig = isrMockLSST.IsrMockLSSTConfig() 

201 self.setMockConfigFalse() 

202 self.mockConfig.doAddSerialOverscan = True 

203 self.mockConfig.doAddParallelOverscan = True 

204 self.doSetUp_mock(config=self.mockConfig) 

205 

206 self.config = IsrTaskLSSTConfig() 

207 self.setIsrConfig() 

208 self.defaultAmpConfig.doSerialOverscan = True 

209 self.defaultAmpConfig.doParallelOverscan = True 

210 

211 self.validateIsrResults() 

212 

213 def test_run_linearize(self): 

214 """Test up to linearizer. 

215 """ 

216 # TODO DM-44314 

217 

218 pass 

219 

220 def test_run_crosstalkCorrection(self): 

221 """Test up to crosstalk correction. 

222 """ 

223 self.mockConfig = isrMockLSST.IsrMockLSSTConfig() 

224 self.setMockConfigFalse() 

225 self.mockConfig.doAddSerialOverscan = True 

226 self.mockConfig.doAddParallelOverscan = True 

227 self.mockConfig.doAddCrosstalk = True 

228 self.doSetUp_mock(config=self.mockConfig) 

229 

230 self.config = IsrTaskLSSTConfig() 

231 self.setIsrConfig() 

232 self.defaultAmpConfig.doSerialOverscan = True 

233 self.defaultAmpConfig.doParallelOverscan = True 

234 self.config.doAssembleCcd = True 

235 self.config.doCrosstalk = True 

236 

237 self.validateIsrResults() 

238 

239 def test_run_biasCorrection(self): 

240 """Test up to bias correction. 

241 """ 

242 self.mockConfig = isrMockLSST.IsrMockLSSTConfig() 

243 self.setMockConfigFalse() 

244 self.mockConfig.doAddSerialOverscan = True 

245 self.mockConfig.doAddParallelOverscan = True 

246 self.mockConfig.doAddCrosstalk = True 

247 self.mockConfig.doAddBias = True 

248 self.doSetUp_mock(config=self.mockConfig) 

249 

250 self.config = IsrTaskLSSTConfig() 

251 self.setIsrConfig() 

252 self.defaultAmpConfig.doSerialOverscan = True 

253 self.defaultAmpConfig.doParallelOverscan = True 

254 self.config.doAssembleCcd = True 

255 self.config.doCrosstalk = True 

256 self.config.doBias = True 

257 

258 self.validateIsrResults() 

259 

260 def test_run_applyGains(self): 

261 """Test up to gain correction. 

262 """ 

263 self.mockConfig = isrMockLSST.IsrMockLSSTConfig() 

264 self.setMockConfigFalse() 

265 self.mockConfig.doAddSerialOverscan = True 

266 self.mockConfig.doAddParallelOverscan = True 

267 self.mockConfig.doAddCrosstalk = True 

268 self.mockConfig.doAddBias = True 

269 self.mockConfig.doApplyGain = True 

270 self.mockConfig.gain = 3.5 

271 self.doSetUp_mock(config=self.mockConfig) 

272 

273 self.config = IsrTaskLSSTConfig() 

274 self.setIsrConfig() 

275 self.defaultAmpConfig.doSerialOverscan = True 

276 self.defaultAmpConfig.doParallelOverscan = True 

277 self.config.doAssembleCcd = True 

278 self.config.doCrosstalk = True 

279 self.config.doBias = True 

280 self.config.doApplyGains = True 

281 

282 self.validateIsrResults() 

283 

284 def test_run_doVarianceDefectsMasking(self): 

285 """Test up to masking of bad pixels. 

286 """ 

287 self.mockConfig = isrMockLSST.IsrMockLSSTConfig() 

288 self.setMockConfigFalse() 

289 self.mockConfig.doAddSerialOverscan = True 

290 self.mockConfig.doAddParallelOverscan = True 

291 self.mockConfig.doAddCrosstalk = True 

292 self.mockConfig.doAddBias = True 

293 self.mockConfig.doApplyGain = True 

294 self.mockConfig.gain = 3.5 

295 self.doSetUp_mock(config=self.mockConfig) 

296 

297 self.config = IsrTaskLSSTConfig() 

298 self.setIsrConfig() 

299 self.defaultAmpConfig.doSerialOverscan = True 

300 self.defaultAmpConfig.doParallelOverscan = True 

301 self.config.doAssembleCcd = True 

302 self.config.doCrosstalk = True 

303 self.config.doBias = True 

304 self.config.doApplyGains = True 

305 self.config.doVariance = True 

306 self.config.doDefect = True 

307 self.config.doNanMasking = True 

308 self.config.doWidenSaturationTrails = True 

309 

310 self.validateIsrResults() 

311 

312 def test_run_doDark(self): 

313 """Test up to dark correction. 

314 """ 

315 self.mockConfig = isrMockLSST.IsrMockLSSTConfig() 

316 self.setMockConfigFalse() 

317 self.mockConfig.doAddSerialOverscan = True 

318 self.mockConfig.doAddParallelOverscan = True 

319 self.mockConfig.doAddCrosstalk = True 

320 self.mockConfig.doAddBias = True 

321 self.mockConfig.doApplyGain = True 

322 self.mockConfig.gain = 3.5 

323 self.mockConfig.doAddDark = True 

324 self.doSetUp_mock(config=self.mockConfig) 

325 

326 self.config = IsrTaskLSSTConfig() 

327 self.setIsrConfig() 

328 self.defaultAmpConfig.doSerialOverscan = True 

329 self.defaultAmpConfig.doParallelOverscan = True 

330 self.config.doAssembleCcd = True 

331 self.config.doCrosstalk = True 

332 self.config.doBias = True 

333 self.config.doApplyGains = True 

334 self.config.doVariance = True 

335 self.config.doDefect = True 

336 self.config.doNanMasking = True 

337 self.config.doWidenSaturationTrails = True 

338 self.config.doDark = True 

339 

340 self.validateIsrResults() 

341 

342 def test_run_doBFcorrection(self): 

343 """Test up to do BF correction 

344 # TODO DM-44315 

345 """ 

346 pass 

347 

348 def test_run_doFlat(self): 

349 """Test up to flat correction 

350 """ 

351 self.mockConfig = isrMockLSST.IsrMockLSSTConfig() 

352 self.setMockConfigFalse() 

353 self.mockConfig.doAddSerialOverscan = True 

354 self.mockConfig.doAddParallelOverscan = True 

355 self.mockConfig.doAddCrosstalk = True 

356 self.mockConfig.doAddBias = True 

357 self.mockConfig.doApplyGain = True 

358 self.mockConfig.gain = 3.5 

359 self.mockConfig.doAddDark = True 

360 self.mockConfig.doAddFlat = True 

361 self.doSetUp_mock(config=self.mockConfig) 

362 

363 self.config = IsrTaskLSSTConfig() 

364 self.setIsrConfig() 

365 self.defaultAmpConfig.doSerialOverscan = True 

366 self.defaultAmpConfig.doParallelOverscan = True 

367 self.config.doAssembleCcd = True 

368 self.config.doCrosstalk = True 

369 self.config.doBias = True 

370 self.config.doApplyGains = True 

371 self.config.doVariance = True 

372 self.config.doDefect = True 

373 self.config.doNanMasking = True 

374 self.config.doWidenSaturationTrails = True 

375 self.config.doDark = True 

376 self.config.doFlat = True 

377 

378 self.validateIsrResults() 

379 

380 def test_run_setPixelValues(self): 

381 """Test up to flat correction 

382 """ 

383 self.mockConfig = isrMockLSST.IsrMockLSSTConfig() 

384 self.setMockConfigFalse() 

385 self.mockConfig.doAddSerialOverscan = True 

386 self.mockConfig.doAddParallelOverscan = True 

387 self.mockConfig.doAddCrosstalk = True 

388 self.mockConfig.doAddBias = True 

389 self.mockConfig.doApplyGain = True 

390 self.mockConfig.gain = 3.5 

391 self.mockConfig.doAddDark = True 

392 self.mockConfig.doAddFlat = True 

393 self.doSetUp_mock(config=self.mockConfig) 

394 

395 self.config = IsrTaskLSSTConfig() 

396 self.setIsrConfig() 

397 self.defaultAmpConfig.doSerialOverscan = True 

398 self.defaultAmpConfig.doParallelOverscan = True 

399 self.config.doAssembleCcd = True 

400 self.config.doCrosstalk = True 

401 self.config.doBias = True 

402 self.config.doApplyGains = True 

403 self.config.doVariance = True 

404 self.config.doDefect = True 

405 self.config.doNanMasking = True 

406 self.config.doWidenSaturationTrails = True 

407 self.config.doDark = True 

408 self.config.doFlat = True 

409 self.config.doSetBadRegions = True 

410 self.config.doInterpolate = True 

411 

412 self.validateIsrResults() 

413 

414 

415class MemoryTester(lsst.utils.tests.MemoryTestCase): 

416 pass 

417 

418 

419def setup_module(module): 

420 lsst.utils.tests.init() 

421 

422 

423if __name__ == "__main__": 423 ↛ 424line 423 didn't jump to line 424, because the condition on line 423 was never true

424 lsst.utils.tests.init() 

425 unittest.main(failfast=True)