Coverage for tests/test_ptcDataset.py: 8%

234 statements  

« prev     ^ index     » next       coverage.py v7.5.0, created at 2024-05-01 04:11 -0700

1# This file is part of ip_isr. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21import unittest 

22import tempfile 

23import copy 

24import logging 

25 

26import numpy as np 

27 

28import lsst.utils.tests 

29 

30from lsst.ip.isr import PhotonTransferCurveDataset 

31import lsst.ip.isr.isrMock as isrMock 

32 

33 

34class PtcDatasetCases(lsst.utils.tests.TestCase): 

35 """Test that write/read methods of PhotonTransferCurveDataset work 

36 """ 

37 def setUp(self): 

38 

39 self.flatMean = 2000 

40 self.readNoiseAdu = 10 

41 mockImageConfig = isrMock.IsrMock.ConfigClass() 

42 

43 # flatDrop is not really relevant as we replace the data 

44 # but good to note it in case we change how this image is made 

45 mockImageConfig.flatDrop = 0.99999 

46 mockImageConfig.isTrimmed = True 

47 

48 self.flatExp1 = isrMock.FlatMock(config=mockImageConfig).run() 

49 self.flatExp2 = self.flatExp1.clone() 

50 (shapeY, shapeX) = self.flatExp1.getDimensions() 

51 

52 self.flatWidth = np.sqrt(self.flatMean) + self.readNoiseAdu 

53 

54 self.rng1 = np.random.RandomState(1984) 

55 flatData1 = self.rng1.normal(self.flatMean, self.flatWidth, (shapeX, shapeY)) 

56 self.rng2 = np.random.RandomState(666) 

57 flatData2 = self.rng2.normal(self.flatMean, self.flatWidth, (shapeX, shapeY)) 

58 

59 self.flatExp1.image.array[:] = flatData1 

60 self.flatExp2.image.array[:] = flatData2 

61 

62 self.flux = 1000. # ADU/sec 

63 self.gain = 1.5 # e-/ADU 

64 self.noiseSq = 5*self.gain # 7.5 (e-)^2 

65 self.c1 = 1./self.gain 

66 self.timeVec = np.arange(1., 101., 5) 

67 self.k2NonLinearity = -5e-6 

68 # quadratic signal-chain non-linearity 

69 muVec = self.flux*self.timeVec + self.k2NonLinearity*self.timeVec**2 

70 

71 self.ampNames = [amp.getName() for amp in self.flatExp1.getDetector().getAmplifiers()] 

72 self.dataset = PhotonTransferCurveDataset(self.ampNames, " ") # pack raw data for fitting 

73 self.covariancesSqrtWeights = {} 

74 for ampName in self.ampNames: # just the expTimes and means here - vars vary per function 

75 self.dataset.rawExpTimes[ampName] = self.timeVec 

76 self.dataset.rawMeans[ampName] = muVec 

77 self.covariancesSqrtWeights[ampName] = [] 

78 

79 def _checkTypes(self, ptcDataset): 

80 """Check that all the types are correct for a ptc dataset.""" 

81 for ampName in ptcDataset.ampNames: 

82 self.assertIsInstance(ptcDataset.expIdMask[ampName], np.ndarray) 

83 self.assertEqual(ptcDataset.expIdMask[ampName].dtype, bool) 

84 self.assertIsInstance(ptcDataset.rawExpTimes[ampName], np.ndarray) 

85 self.assertEqual(ptcDataset.rawExpTimes[ampName].dtype, np.float64) 

86 self.assertIsInstance(ptcDataset.rawMeans[ampName], np.ndarray) 

87 self.assertEqual(ptcDataset.rawMeans[ampName].dtype, np.float64) 

88 self.assertIsInstance(ptcDataset.rawVars[ampName], np.ndarray) 

89 self.assertEqual(ptcDataset.rawVars[ampName].dtype, np.float64) 

90 self.assertEqual(ptcDataset.rowMeanVariance[ampName].dtype, np.float64) 

91 self.assertIsInstance(ptcDataset.noiseList[ampName], np.ndarray) 

92 self.assertEqual(ptcDataset.noiseList[ampName].dtype, np.float64) 

93 self.assertIsInstance(ptcDataset.gain[ampName], float) 

94 self.assertIsInstance(ptcDataset.gainErr[ampName], float) 

95 self.assertIsInstance(ptcDataset.noise[ampName], float) 

96 self.assertIsInstance(ptcDataset.noiseErr[ampName], float) 

97 self.assertIsInstance(ptcDataset.histVars[ampName], np.ndarray) 

98 self.assertEqual(ptcDataset.histVars[ampName].dtype, np.float64) 

99 self.assertIsInstance(ptcDataset.histChi2Dofs[ampName], np.ndarray) 

100 self.assertEqual(ptcDataset.histChi2Dofs[ampName].dtype, np.float64) 

101 self.assertIsInstance(ptcDataset.kspValues[ampName], np.ndarray) 

102 self.assertEqual(ptcDataset.kspValues[ampName].dtype, np.float64) 

103 self.assertIsInstance(ptcDataset.ptcFitPars[ampName], np.ndarray) 

104 self.assertEqual(ptcDataset.ptcFitPars[ampName].dtype, np.float64) 

105 self.assertIsInstance(ptcDataset.ptcFitParsError[ampName], np.ndarray) 

106 self.assertEqual(ptcDataset.ptcFitParsError[ampName].dtype, np.float64) 

107 self.assertIsInstance(ptcDataset.ptcFitChiSq[ampName], float) 

108 self.assertIsInstance(ptcDataset.ptcTurnoff[ampName], float) 

109 self.assertIsInstance(ptcDataset.ptcTurnoffSamplingError[ampName], float) 

110 self.assertIsInstance(ptcDataset.covariances[ampName], np.ndarray) 

111 self.assertEqual(ptcDataset.covariances[ampName].dtype, np.float64) 

112 self.assertIsInstance(ptcDataset.covariancesModel[ampName], np.ndarray) 

113 self.assertEqual(ptcDataset.covariancesModel[ampName].dtype, np.float64) 

114 self.assertIsInstance(ptcDataset.covariancesSqrtWeights[ampName], np.ndarray) 

115 self.assertEqual(ptcDataset.covariancesSqrtWeights[ampName].dtype, np.float64) 

116 self.assertIsInstance(ptcDataset.aMatrix[ampName], np.ndarray) 

117 self.assertEqual(ptcDataset.aMatrix[ampName].dtype, np.float64) 

118 self.assertIsInstance(ptcDataset.bMatrix[ampName], np.ndarray) 

119 self.assertEqual(ptcDataset.bMatrix[ampName].dtype, np.float64) 

120 self.assertIsInstance(ptcDataset.noiseMatrix[ampName], np.ndarray) 

121 self.assertEqual(ptcDataset.noiseMatrix[ampName].dtype, np.float64) 

122 self.assertIsInstance(ptcDataset.covariancesModelNoB[ampName], np.ndarray) 

123 self.assertEqual(ptcDataset.covariancesModelNoB[ampName].dtype, np.float64) 

124 self.assertIsInstance(ptcDataset.aMatrixNoB[ampName], np.ndarray) 

125 self.assertEqual(ptcDataset.aMatrixNoB[ampName].dtype, np.float64) 

126 self.assertIsInstance(ptcDataset.noiseMatrixNoB[ampName], np.ndarray) 

127 self.assertEqual(ptcDataset.noiseMatrixNoB[ampName].dtype, np.float64) 

128 self.assertIsInstance(ptcDataset.finalVars[ampName], np.ndarray) 

129 self.assertEqual(ptcDataset.finalVars[ampName].dtype, np.float64) 

130 self.assertIsInstance(ptcDataset.finalModelVars[ampName], np.ndarray) 

131 self.assertEqual(ptcDataset.finalModelVars[ampName].dtype, np.float64) 

132 self.assertIsInstance(ptcDataset.finalMeans[ampName], np.ndarray) 

133 self.assertEqual(ptcDataset.finalMeans[ampName].dtype, np.float64) 

134 self.assertIsInstance(ptcDataset.photoCharges[ampName], np.ndarray) 

135 self.assertEqual(ptcDataset.photoCharges[ampName].dtype, np.float64) 

136 

137 for key, value in ptcDataset.auxValues.items(): 

138 self.assertIsInstance(value, np.ndarray) 

139 self.assertEqual(value.dtype, np.float64) 

140 

141 def test_emptyPtcDataset(self): 

142 """Test an empty PTC dataset.""" 

143 emptyDataset = PhotonTransferCurveDataset( 

144 self.ampNames, 

145 ptcFitType="PARTIAL", 

146 ) 

147 self._checkTypes(emptyDataset) 

148 

149 with tempfile.NamedTemporaryFile(suffix=".yaml") as f: 

150 usedFilename = emptyDataset.writeText(f.name) 

151 fromText = PhotonTransferCurveDataset.readText(usedFilename) 

152 self.assertEqual(emptyDataset, fromText) 

153 self._checkTypes(emptyDataset) 

154 

155 with tempfile.NamedTemporaryFile(suffix=".fits") as f: 

156 usedFilename = emptyDataset.writeFits(f.name) 

157 fromFits = PhotonTransferCurveDataset.readFits(usedFilename) 

158 self.assertEqual(emptyDataset, fromFits) 

159 self._checkTypes(emptyDataset) 

160 

161 def test_partialPtcDataset(self): 

162 """Test of a partial PTC dataset.""" 

163 # Fill the dataset with made up data. 

164 nSideCovMatrix = 2 

165 nSideCovMatrixFullCovFit = 2 

166 

167 partialDataset = PhotonTransferCurveDataset( 

168 self.ampNames, 

169 ptcFitType="PARTIAL", 

170 covMatrixSide=nSideCovMatrix, 

171 covMatrixSideFullCovFit=nSideCovMatrixFullCovFit 

172 ) 

173 self._checkTypes(partialDataset) 

174 

175 for ampName in partialDataset.ampNames: 

176 partialDataset.setAmpValuesPartialDataset( 

177 ampName, 

178 inputExpIdPair=(10, 11), 

179 rawExpTime=10.0, 

180 rawMean=10.0, 

181 rawVar=10.0, 

182 ) 

183 

184 for useAuxValues in [False, True]: 

185 if useAuxValues: 

186 partialDataset.setAuxValuesPartialDataset( 

187 { 

188 "CCOBCURR": 1.0, 

189 "CCDTEMP": 0.0, 

190 } 

191 ) 

192 self._checkTypes(partialDataset) 

193 

194 with tempfile.NamedTemporaryFile(suffix=".yaml") as f: 

195 usedFilename = partialDataset.writeText(f.name) 

196 fromText = PhotonTransferCurveDataset.readText(usedFilename) 

197 self.assertEqual(fromText, partialDataset) 

198 self._checkTypes(fromText) 

199 

200 with tempfile.NamedTemporaryFile(suffix=".fits") as f: 

201 usedFilename = partialDataset.writeFits(f.name) 

202 fromFits = PhotonTransferCurveDataset.readFits(usedFilename) 

203 self.assertEqual(fromFits, partialDataset) 

204 self._checkTypes(fromFits) 

205 

206 def test_ptcDatset(self): 

207 """Test of a full PTC dataset.""" 

208 # Fill the dataset with made up data. 

209 nSignalPoints = 5 

210 nSideCovMatrixInput = 3 # Size of measured covariances 

211 

212 for nSideCovMatrixFullCovFitInput in np.arange(1, nSideCovMatrixInput + 2): 

213 for fitType in ['POLYNOMIAL', 'EXPAPPROXIMATION', 'FULLCOVARIANCE']: 

214 localDataset = PhotonTransferCurveDataset( 

215 self.ampNames, 

216 ptcFitType=fitType, 

217 covMatrixSide=nSideCovMatrixInput, 

218 covMatrixSideFullCovFit=nSideCovMatrixFullCovFitInput, 

219 ) 

220 nSideCovMatrix = localDataset.covMatrixSide 

221 nSideCovMatrixFullCovFit = localDataset.covMatrixSideFullCovFit 

222 localDataset.badAmps = [localDataset.ampNames[0], localDataset.ampNames[1]] 

223 for ampName in localDataset.ampNames: 

224 

225 localDataset.inputExpIdPairs[ampName] = [(1, 2)]*nSignalPoints 

226 localDataset.expIdMask[ampName] = np.ones(nSignalPoints, dtype=bool) 

227 localDataset.expIdMask[ampName][1] = False 

228 localDataset.rawExpTimes[ampName] = np.arange(nSignalPoints, dtype=np.float64) 

229 localDataset.rawMeans[ampName] = self.flux*np.arange(nSignalPoints) 

230 localDataset.rawVars[ampName] = self.c1*self.flux*np.arange(nSignalPoints) 

231 localDataset.photoCharges[ampName] = np.full(nSignalPoints, np.nan) 

232 localDataset.gain[ampName] = self.gain 

233 localDataset.gainErr[ampName] = 0.1 

234 localDataset.noise[ampName] = self.noiseSq 

235 localDataset.noiseErr[ampName] = 2.0 

236 localDataset.histVars[ampName] = localDataset.rawVars[ampName] 

237 localDataset.histChi2Dofs[ampName] = np.full(nSignalPoints, 1.0) 

238 localDataset.kspValues[ampName] = np.full(nSignalPoints, 0.5) 

239 

240 localDataset.finalVars[ampName] = self.c1*self.flux*np.arange(nSignalPoints) 

241 localDataset.finalModelVars[ampName] = np.full(nSignalPoints, 100.0) 

242 localDataset.finalMeans[ampName] = self.flux*np.arange(nSignalPoints) 

243 

244 if fitType in ['POLYNOMIAL', 'EXPAPPROXIMATION', ]: 

245 localDataset.ptcFitPars[ampName] = np.array([10.0, 1.5, 1e-6]) 

246 localDataset.ptcFitParsError[ampName] = np.array([1.0, 0.2, 1e-7]) 

247 localDataset.ptcFitChiSq[ampName] = 1.0 

248 localDataset.ptcTurnoff[ampName] = localDataset.rawMeans[ampName][-1] 

249 localDataset.ptcTurnoffSamplingError[ampName] = localDataset.ptcTurnoff[ampName]/100. 

250 

251 localDataset.covariances[ampName] = np.full( 

252 (nSignalPoints, nSideCovMatrix, nSideCovMatrix), 105.0) 

253 localDataset.covariancesModel[ampName] = np.full( 

254 (nSignalPoints, nSideCovMatrixFullCovFit, nSideCovMatrixFullCovFit), np.nan) 

255 localDataset.covariancesSqrtWeights[ampName] = np.full((nSignalPoints, nSideCovMatrix, 

256 nSideCovMatrix), 10.0) 

257 localDataset.aMatrix[ampName] = np.full((nSideCovMatrixFullCovFit, 

258 nSideCovMatrixFullCovFit), np.nan) 

259 localDataset.bMatrix[ampName] = np.full((nSideCovMatrixFullCovFit, 

260 nSideCovMatrixFullCovFit), np.nan) 

261 localDataset.noiseMatrix[ampName] = np.full((nSideCovMatrixFullCovFit, 

262 nSideCovMatrixFullCovFit), np.nan) 

263 localDataset.covariancesModelNoB[ampName] = np.full((nSignalPoints, 

264 nSideCovMatrixFullCovFit, 

265 nSideCovMatrixFullCovFit), np.nan) 

266 localDataset.aMatrixNoB[ampName] = np.full( 

267 (nSideCovMatrixFullCovFit, nSideCovMatrixFullCovFit), np.nan) 

268 localDataset.noiseMatrixNoB[ampName] = np.full( 

269 (nSideCovMatrixFullCovFit, nSideCovMatrixFullCovFit), np.nan) 

270 

271 if localDataset.ptcFitType in ['FULLCOVARIANCE', ]: 

272 localDataset.ptcFitPars[ampName] = np.array([np.nan, np.nan]) 

273 localDataset.ptcFitParsError[ampName] = np.array([np.nan, np.nan]) 

274 localDataset.ptcFitChiSq[ampName] = np.nan 

275 localDataset.ptcTurnoff[ampName] = np.nan 

276 localDataset.ptcTurnoffSamplingError[ampName] = np.nan 

277 

278 localDataset.covariances[ampName] = np.full( 

279 (nSignalPoints, nSideCovMatrix, nSideCovMatrix), 105.0) 

280 localDataset.covariancesModel[ampName] = np.full( 

281 (nSignalPoints, nSideCovMatrixFullCovFit, nSideCovMatrixFullCovFit), 100.0) 

282 localDataset.covariancesSqrtWeights[ampName] = np.full((nSignalPoints, nSideCovMatrix, 

283 nSideCovMatrix), 10.0) 

284 localDataset.aMatrix[ampName] = np.full((nSideCovMatrixFullCovFit, 

285 nSideCovMatrixFullCovFit), 1e-6) 

286 localDataset.bMatrix[ampName] = np.full((nSideCovMatrixFullCovFit, 

287 nSideCovMatrixFullCovFit), 1e-7) 

288 localDataset.noiseMatrix[ampName] = np.full((nSideCovMatrixFullCovFit, 

289 nSideCovMatrixFullCovFit), 3.0) 

290 localDataset.covariancesModelNoB[ampName] = np.full((nSignalPoints, 

291 nSideCovMatrixFullCovFit, 

292 nSideCovMatrixFullCovFit), 15.0) 

293 localDataset.aMatrixNoB[ampName] = np.full( 

294 (nSideCovMatrixFullCovFit, nSideCovMatrixFullCovFit), 2e-6) 

295 localDataset.noiseMatrixNoB[ampName] = np.full( 

296 (nSideCovMatrixFullCovFit, nSideCovMatrixFullCovFit), 3.0) 

297 

298 for useAuxValues in [False, True]: 

299 if useAuxValues: 

300 localDataset.auxValues = { 

301 "CCOBCURR": np.ones(nSignalPoints), 

302 "CCDTEMP": np.zeros(nSignalPoints), 

303 } 

304 

305 self._checkTypes(localDataset) 

306 with tempfile.NamedTemporaryFile(suffix=".yaml") as f: 

307 usedFilename = localDataset.writeText(f.name) 

308 fromText = PhotonTransferCurveDataset.readText(usedFilename) 

309 self.assertEqual(fromText, localDataset) 

310 self._checkTypes(fromText) 

311 

312 with tempfile.NamedTemporaryFile(suffix=".fits") as f: 

313 usedFilename = localDataset.writeFits(f.name) 

314 fromFits = PhotonTransferCurveDataset.readFits(usedFilename) 

315 self.assertEqual(fromFits, localDataset) 

316 self._checkTypes(fromFits) 

317 

318 def test_getExpIdsUsed(self): 

319 localDataset = copy.copy(self.dataset) 

320 

321 for pair in [(12, 34), (56, 78), (90, 10)]: 

322 localDataset.inputExpIdPairs["C:0,0"].append(pair) 

323 localDataset.expIdMask["C:0,0"] = np.array([True, False, True]) 

324 self.assertTrue(np.all(localDataset.getExpIdsUsed("C:0,0") == [(12, 34), (90, 10)])) 

325 

326 localDataset.expIdMask["C:0,0"] = np.array([True, False, True, True]) # wrong length now 

327 with self.assertRaises(AssertionError): 

328 localDataset.getExpIdsUsed("C:0,0") 

329 

330 def test_getGoodAmps(self): 

331 dataset = self.dataset 

332 

333 self.assertTrue(dataset.ampNames == self.ampNames) 

334 dataset.badAmps.append("C:0,1") 

335 self.assertTrue(dataset.getGoodAmps() == [amp for amp in self.ampNames if amp != "C:0,1"]) 

336 

337 def test_ptcDataset_pre_dm38309(self): 

338 """Test for PTC datasets created by cpSolvePtcTask prior to DM-38309. 

339 """ 

340 localDataset = copy.copy(self.dataset) 

341 

342 for pair in [[(12, 34)], [(56, 78)], [(90, 10)]]: 

343 localDataset.inputExpIdPairs["C:0,0"].append(pair) 

344 localDataset.expIdMask["C:0,0"] = np.array([True, False, True]) 

345 

346 with self.assertLogs("lsst.ip.isr.calibType", logging.WARNING) as cm: 

347 used = localDataset.getExpIdsUsed("C:0,0") 

348 self.assertIn("PTC file was written incorrectly", cm.output[0]) 

349 

350 self.assertTrue(np.all(used == [(12, 34), (90, 10)])) 

351 

352 

353class MemoryTester(lsst.utils.tests.MemoryTestCase): 

354 pass 

355 

356 

357def setup_module(module): 

358 lsst.utils.tests.init() 

359 

360 

361if __name__ == "__main__": 361 ↛ 362line 361 didn't jump to line 362, because the condition on line 361 was never true

362 import sys 

363 setup_module(sys.modules[__name__]) 

364 unittest.main()