Coverage for tests/test_ptcDataset.py: 6%

1# This file is part of ip_isr.

3# Developed for the LSST Data Management System.

4# This product includes software developed by the LSST Project

5# (https://www.lsst.org).

6# See the COPYRIGHT file at the top-level directory of this distribution

7# for details of code ownership.

9# This program is free software: you can redistribute it and/or modify

10# it under the terms of the GNU General Public License as published by

11# the Free Software Foundation, either version 3 of the License, or

12# (at your option) any later version.

13#

14# This program is distributed in the hope that it will be useful,

15# but WITHOUT ANY WARRANTY; without even the implied warranty of

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the

17# GNU General Public License for more details.

18#

19# You should have received a copy of the GNU General Public License

20# along with this program. If not, see <https://www.gnu.org/licenses/>.

21import unittest

22import tempfile

23import copy

24import logging

26import numpy as np

28import lsst.utils.tests

30from lsst.ip.isr import PhotonTransferCurveDataset

31import lsst.ip.isr.isrMock as isrMock

34class PtcDatasetCases(lsst.utils.tests.TestCase):

35 """Test that write/read methods of PhotonTransferCurveDataset work

36 """

37 def setUp(self):

39 self.flatMean = 2000

40 self.readNoiseAdu = 10

41 mockImageConfig = isrMock.IsrMock.ConfigClass()

43 # flatDrop is not really relevant as we replace the data

44 # but good to note it in case we change how this image is made

45 mockImageConfig.flatDrop = 0.99999

46 mockImageConfig.isTrimmed = True

48 self.flatExp1 = isrMock.FlatMock(config=mockImageConfig).run()

49 self.flatExp2 = self.flatExp1.clone()

50 (shapeY, shapeX) = self.flatExp1.getDimensions()

52 self.flatWidth = np.sqrt(self.flatMean) + self.readNoiseAdu

54 self.rng1 = np.random.RandomState(1984)

55 flatData1 = self.rng1.normal(self.flatMean, self.flatWidth, (shapeX, shapeY))

56 self.rng2 = np.random.RandomState(666)

57 flatData2 = self.rng2.normal(self.flatMean, self.flatWidth, (shapeX, shapeY))

59 self.flatExp1.image.array[:] = flatData1

60 self.flatExp2.image.array[:] = flatData2

62 self.flux = 1000. # ADU/sec

63 self.gain = 1.5 # e-/ADU

64 self.noiseSq = 5*self.gain # 7.5 (e-)^2

65 self.c1 = 1./self.gain

66 self.timeVec = np.arange(1., 101., 5)

67 self.k2NonLinearity = -5e-6

68 # quadratic signal-chain non-linearity

69 muVec = self.flux*self.timeVec + self.k2NonLinearity*self.timeVec**2

71 self.ampNames = [amp.getName() for amp in self.flatExp1.getDetector().getAmplifiers()]

72 self.dataset = PhotonTransferCurveDataset(self.ampNames, " ") # pack raw data for fitting

73 self.covariancesSqrtWeights = {}

74 for ampName in self.ampNames: # just the expTimes and means here - vars vary per function

75 self.dataset.rawExpTimes[ampName] = self.timeVec

76 self.dataset.rawMeans[ampName] = muVec

77 self.covariancesSqrtWeights[ampName] = []

79 def _checkTypes(self, ptcDataset):

80 """Check that all the types are correct for a ptc dataset."""

81 for ampName in ptcDataset.ampNames:

82 self.assertIsInstance(ptcDataset.expIdMask[ampName], np.ndarray)

83 self.assertEqual(ptcDataset.expIdMask[ampName].dtype, bool)

84 self.assertIsInstance(ptcDataset.expIdRolloffMask[ampName], np.ndarray)

85 self.assertEqual(ptcDataset.expIdRolloffMask[ampName].dtype, bool)

86 self.assertIsInstance(ptcDataset.inputExpPairMjdStartList[ampName], np.ndarray)

87 self.assertIsInstance(ptcDataset.overscanMedianLevelList[ampName], np.ndarray)

88 self.assertEqual(ptcDataset.overscanMedianLevelList[ampName].dtype, float)

89 self.assertIsInstance(ptcDataset.overscanMedian[ampName], float)

90 self.assertIsInstance(ptcDataset.overscanMedianSigma[ampName], float)

91 self.assertIsInstance(ptcDataset.rawExpTimes[ampName], np.ndarray)

92 self.assertEqual(ptcDataset.rawExpTimes[ampName].dtype, np.float64)

93 self.assertIsInstance(ptcDataset.rawMeans[ampName], np.ndarray)

94 self.assertEqual(ptcDataset.rawMeans[ampName].dtype, np.float64)

95 self.assertIsInstance(ptcDataset.rawVars[ampName], np.ndarray)

96 self.assertEqual(ptcDataset.rawVars[ampName].dtype, np.float64)

97 self.assertIsInstance(ptcDataset.rawDeltas[ampName], np.ndarray)

98 self.assertEqual(ptcDataset.rawDeltas[ampName].dtype, np.float64)

99 self.assertEqual(ptcDataset.rowMeanVariance[ampName].dtype, np.float64)

100 self.assertIsInstance(ptcDataset.noiseList[ampName], np.ndarray)

101 self.assertEqual(ptcDataset.noiseList[ampName].dtype, np.float64)

102 self.assertIsInstance(ptcDataset.gain[ampName], float)

103 self.assertIsInstance(ptcDataset.gainErr[ampName], float)

104 self.assertIsInstance(ptcDataset.noise[ampName], float)

105 self.assertIsInstance(ptcDataset.noiseErr[ampName], float)

106 self.assertIsInstance(ptcDataset.histVars[ampName], np.ndarray)

107 self.assertEqual(ptcDataset.histVars[ampName].dtype, np.float64)

108 self.assertIsInstance(ptcDataset.histChi2Dofs[ampName], np.ndarray)

109 self.assertEqual(ptcDataset.histChi2Dofs[ampName].dtype, np.float64)

110 self.assertIsInstance(ptcDataset.kspValues[ampName], np.ndarray)

111 self.assertEqual(ptcDataset.kspValues[ampName].dtype, np.float64)

112 self.assertIsInstance(ptcDataset.ptcFitPars[ampName], np.ndarray)

113 self.assertEqual(ptcDataset.ptcFitPars[ampName].dtype, np.float64)

114 self.assertIsInstance(ptcDataset.ptcFitParsError[ampName], np.ndarray)

115 self.assertEqual(ptcDataset.ptcFitParsError[ampName].dtype, np.float64)

116 self.assertIsInstance(ptcDataset.ptcFitChiSq[ampName], float)

117 self.assertIsInstance(ptcDataset.ptcTurnoff[ampName], float)

118 self.assertIsInstance(ptcDataset.ptcTurnoffSamplingError[ampName], float)

119 self.assertIsInstance(ptcDataset.covariances[ampName], np.ndarray)

120 self.assertEqual(ptcDataset.covariances[ampName].dtype, np.float64)

121 self.assertIsInstance(ptcDataset.covariancesModel[ampName], np.ndarray)

122 self.assertEqual(ptcDataset.covariancesModel[ampName].dtype, np.float64)

123 self.assertIsInstance(ptcDataset.covariancesSqrtWeights[ampName], np.ndarray)

124 self.assertEqual(ptcDataset.covariancesSqrtWeights[ampName].dtype, np.float64)

125 self.assertIsInstance(ptcDataset.aMatrix[ampName], np.ndarray)

126 self.assertEqual(ptcDataset.aMatrix[ampName].dtype, np.float64)

127 self.assertIsInstance(ptcDataset.bMatrix[ampName], np.ndarray)

128 self.assertEqual(ptcDataset.bMatrix[ampName].dtype, np.float64)

129 self.assertIsInstance(ptcDataset.noiseMatrix[ampName], np.ndarray)

130 self.assertEqual(ptcDataset.noiseMatrix[ampName].dtype, np.float64)

131 self.assertIsInstance(ptcDataset.finalVars[ampName], np.ndarray)

132 self.assertEqual(ptcDataset.finalVars[ampName].dtype, np.float64)

133 self.assertIsInstance(ptcDataset.finalModelVars[ampName], np.ndarray)

134 self.assertEqual(ptcDataset.finalModelVars[ampName].dtype, np.float64)

135 self.assertIsInstance(ptcDataset.finalMeans[ampName], np.ndarray)

136 self.assertEqual(ptcDataset.finalMeans[ampName].dtype, np.float64)

137 self.assertIsInstance(ptcDataset.photoCharges[ampName], np.ndarray)

138 self.assertEqual(ptcDataset.photoCharges[ampName].dtype, np.float64)

139

140 for key, value in ptcDataset.auxValues.items():

141 self.assertIsInstance(value, np.ndarray)

142 # This key is explicitly camelCase to ensure that this dataset

143 # can handle mixed-case names.

144 if key == "intVal":

145 self.assertEqual(value.dtype, np.int64)

146 elif key == "PROGRAM":

147 self.assertIsInstance(value[0], np.str_)

148 else:

149 self.assertEqual(value.dtype, np.float64)

150

151 def test_emptyPtcDataset(self):

152 """Test an empty PTC dataset."""

153 emptyDataset = PhotonTransferCurveDataset(

154 self.ampNames,

155 ptcFitType="PARTIAL",

156 )

157 self._checkTypes(emptyDataset)

158

159 with tempfile.NamedTemporaryFile(suffix=".yaml") as f:

160 usedFilename = emptyDataset.writeText(f.name)

161 fromText = PhotonTransferCurveDataset.readText(usedFilename)

162 self.assertEqual(emptyDataset, fromText)

163 self._checkTypes(emptyDataset)

164

165 with tempfile.NamedTemporaryFile(suffix=".fits") as f:

166 usedFilename = emptyDataset.writeFits(f.name)

167 fromFits = PhotonTransferCurveDataset.readFits(usedFilename)

168 self.assertEqual(emptyDataset, fromFits)

169 self._checkTypes(emptyDataset)

170

171 def test_partialPtcDataset(self):

172 """Test of a partial PTC dataset."""

173 # Fill the dataset with made up data.

174 nSideCovMatrix = 2

175 nSideCovMatrixFullCovFit = 2

176

177 partialDataset = PhotonTransferCurveDataset(

178 self.ampNames,

179 ptcFitType="PARTIAL",

180 covMatrixSide=nSideCovMatrix,

181 covMatrixSideFullCovFit=nSideCovMatrixFullCovFit

182 )

183 self._checkTypes(partialDataset)

184

185 for ampName in partialDataset.ampNames:

186 partialDataset.setAmpValuesPartialDataset(

187 ampName,

188 inputExpIdPair=(10, 11),

189 inputExpPairMjdStart=60775.38958333,

190 rawExpTime=10.0,

191 rawMean=10.0,

192 rawVar=10.0,

193 rawDelta=0.01,

194 )

195

196 for useAuxValues in [False, True]:

197 if useAuxValues:

198 # This key is explicitly camelCase to ensure that this dataset

199 # can handle mixed-case names.

200 partialDataset.setAuxValuesPartialDataset(

201 {

202 "CCOBCURR": 1.0,

203 "CCDTEMP": 0.0,

204 "PROGRAM": "BLOCK-000",

205 # This key is explicitly camelCase to ensure that this

206 # dataset can handle mixed-case names.

207 "intVal": 7,

208 }

209 )

210 self._checkTypes(partialDataset)

211

212 with tempfile.NamedTemporaryFile(suffix=".yaml") as f:

213 usedFilename = partialDataset.writeText(f.name)

214 fromText = PhotonTransferCurveDataset.readText(usedFilename)

215 self.assertEqual(fromText, partialDataset)

216 self._checkTypes(fromText)

217

218 with tempfile.NamedTemporaryFile(suffix=".fits") as f:

219 usedFilename = partialDataset.writeFits(f.name)

220 fromFits = PhotonTransferCurveDataset.readFits(usedFilename)

221 self.assertEqual(fromFits, partialDataset)

222 self._checkTypes(fromFits)

223

224 def test_ptcDataset(self):

225 """Test of a full PTC dataset."""

226 # Fill the dataset with made up data.

227 nSignalPoints = 5

228 nSideCovMatrixInput = 3 # Size of measured covariances

229

230 for nSideCovMatrixFullCovFitInput in np.arange(1, nSideCovMatrixInput + 2):

231 for fitType in ['EXPAPPROXIMATION', 'FULLCOVARIANCE']:

232 localDataset = PhotonTransferCurveDataset(

233 self.ampNames,

234 ptcFitType=fitType,

235 covMatrixSide=nSideCovMatrixInput,

236 covMatrixSideFullCovFit=nSideCovMatrixFullCovFitInput,

237 )

238 nSideCovMatrix = localDataset.covMatrixSide

239 nSideCovMatrixFullCovFit = localDataset.covMatrixSideFullCovFit

240 localDataset.badAmps = [localDataset.ampNames[0], localDataset.ampNames[1]]

241 for ampName in localDataset.ampNames:

242

243 localDataset.inputExpIdPairs[ampName] = [(1, 2)]*nSignalPoints

244 localDataset.inputExpPairMjdStartList[ampName] = np.full(nSignalPoints, 60775.39)

245 localDataset.overscanMedianLevelList[ampName] = np.full(nSignalPoints, 25000.0)

246 localDataset.expIdMask[ampName] = np.ones(nSignalPoints, dtype=bool)

247 localDataset.expIdMask[ampName][1] = False

248 localDataset.expIdRolloffMask[ampName] = np.ones(nSignalPoints, dtype=bool)

249 localDataset.rawExpTimes[ampName] = np.arange(nSignalPoints, dtype=np.float64)

250 localDataset.rawMeans[ampName] = self.flux*np.arange(nSignalPoints)

251 localDataset.rawVars[ampName] = self.c1*self.flux*np.arange(nSignalPoints)

252 localDataset.rawDeltas[ampName] = 0.01*self.flux*np.arange(nSignalPoints)

253 localDataset.photoCharges[ampName] = np.full(nSignalPoints, np.nan)

254 localDataset.photoChargeDeltas[ampName] = np.full(nSignalPoints, np.nan)

255 localDataset.gain[ampName] = self.gain

256 localDataset.gainErr[ampName] = 0.1

257 localDataset.noise[ampName] = self.noiseSq

258 localDataset.noiseErr[ampName] = 2.0

259 localDataset.histVars[ampName] = localDataset.rawVars[ampName]

260 localDataset.histChi2Dofs[ampName] = np.full(nSignalPoints, 1.0)

261 localDataset.kspValues[ampName] = np.full(nSignalPoints, 0.5)

262

263 localDataset.finalVars[ampName] = self.c1*self.flux*np.arange(nSignalPoints)

264 localDataset.finalModelVars[ampName] = np.full(nSignalPoints, 100.0)

265 localDataset.finalMeans[ampName] = self.flux*np.arange(nSignalPoints)

266

267 if fitType == 'EXPAPPROXIMATION':

268 localDataset.ptcFitPars[ampName] = np.array([10.0, 1.5, 1e-6])

269 localDataset.ptcFitParsError[ampName] = np.array([1.0, 0.2, 1e-7])

270 localDataset.ptcFitChiSq[ampName] = 1.0

271 localDataset.ptcTurnoff[ampName] = localDataset.rawMeans[ampName][-1]

272 localDataset.ptcTurnoffSamplingError[ampName] = localDataset.ptcTurnoff[ampName]/100.

273

274 localDataset.covariances[ampName] = np.full(

275 (nSignalPoints, nSideCovMatrix, nSideCovMatrix), 105.0)

276 localDataset.covariancesModel[ampName] = np.full(

277 (nSignalPoints, nSideCovMatrixFullCovFit, nSideCovMatrixFullCovFit), np.nan)

278 localDataset.covariancesSqrtWeights[ampName] = np.full((nSignalPoints, nSideCovMatrix,

279 nSideCovMatrix), 10.0)

280 localDataset.aMatrix[ampName] = np.full((nSideCovMatrixFullCovFit,

281 nSideCovMatrixFullCovFit), np.nan)

282 localDataset.bMatrix[ampName] = np.full((nSideCovMatrixFullCovFit,

283 nSideCovMatrixFullCovFit), np.nan)

284 localDataset.noiseMatrix[ampName] = np.full((nSideCovMatrixFullCovFit,

285 nSideCovMatrixFullCovFit), np.nan)

286

287 if localDataset.ptcFitType in ['FULLCOVARIANCE', ]:

288 localDataset.ptcFitPars[ampName] = np.array([np.nan, np.nan])

289 localDataset.ptcFitParsError[ampName] = np.array([np.nan, np.nan])

290 localDataset.ptcFitChiSq[ampName] = np.nan

291 localDataset.ptcTurnoff[ampName] = np.nan

292 localDataset.ptcTurnoffSamplingError[ampName] = np.nan

293

294 localDataset.covariances[ampName] = np.full(

295 (nSignalPoints, nSideCovMatrix, nSideCovMatrix), 105.0)

296 localDataset.covariancesModel[ampName] = np.full(

297 (nSignalPoints, nSideCovMatrixFullCovFit, nSideCovMatrixFullCovFit), 100.0)

298 localDataset.covariancesSqrtWeights[ampName] = np.full((nSignalPoints, nSideCovMatrix,

299 nSideCovMatrix), 10.0)

300 localDataset.aMatrix[ampName] = np.full((nSideCovMatrixFullCovFit,

301 nSideCovMatrixFullCovFit), 1e-6)

302 localDataset.bMatrix[ampName] = np.full((nSideCovMatrixFullCovFit,

303 nSideCovMatrixFullCovFit), 1e-7)

304 localDataset.noiseMatrix[ampName] = np.full((nSideCovMatrixFullCovFit,

305 nSideCovMatrixFullCovFit), 3.0)

306

307 for useAuxValues in [False, True]:

308 if useAuxValues:

309 localDataset.auxValues = {

310 "CCOBCURR": np.ones(nSignalPoints),

311 "CCDTEMP": np.zeros(nSignalPoints),

312 "PROGRAM": np.full(nSignalPoints, "BLOCK-000"),

313 # This key is explicitly camelCase to ensure that

314 # this dataset can handle mixed-case names.

315 "intVal": np.ones(nSignalPoints, dtype=np.int64),

316 }

317

318 self._checkTypes(localDataset)

319 with tempfile.NamedTemporaryFile(suffix=".yaml") as f:

320 usedFilename = localDataset.writeText(f.name)

321 fromText = PhotonTransferCurveDataset.readText(usedFilename)

322 self.assertEqual(fromText, localDataset)

323 self._checkTypes(fromText)

324

325 with tempfile.NamedTemporaryFile(suffix=".fits") as f:

326 usedFilename = localDataset.writeFits(f.name)

327 fromFits = PhotonTransferCurveDataset.readFits(usedFilename)

328 self.assertEqual(fromFits, localDataset)

329 self._checkTypes(fromFits)

330

331 def test_getExpIdsUsed(self):

332 localDataset = copy.copy(self.dataset)

333

334 for pair in [(12, 34), (56, 78), (90, 10)]:

335 localDataset.inputExpIdPairs["C:0,0"].append(pair)

336 localDataset.expIdMask["C:0,0"] = np.array([True, False, True])

337 self.assertTrue(np.all(localDataset.getExpIdsUsed("C:0,0") == [(12, 34), (90, 10)]))

338

339 localDataset.expIdMask["C:0,0"] = np.array([True, False, True, True]) # wrong length now

340 with self.assertRaises(AssertionError):

341 localDataset.getExpIdsUsed("C:0,0")

342

343 def test_getGoodAmps(self):

344 dataset = self.dataset

345

346 self.assertTrue(dataset.ampNames == self.ampNames)

347 dataset.badAmps.append("C:0,1")

348 self.assertTrue(dataset.getGoodAmps() == [amp for amp in self.ampNames if amp != "C:0,1"])

349

350 def test_ptcDataset_pre_dm38309(self):

351 """Test for PTC datasets created by cpSolvePtcTask prior to DM-38309.

352 """

353 localDataset = copy.copy(self.dataset)

354

355 for pair in [[(12, 34)], [(56, 78)], [(90, 10)]]:

356 localDataset.inputExpIdPairs["C:0,0"].append(pair)

357 localDataset.expIdMask["C:0,0"] = np.array([True, False, True])

358

359 with self.assertLogs("lsst.ip.isr.calibType", logging.WARNING) as cm:

360 used = localDataset.getExpIdsUsed("C:0,0")

361 self.assertIn("PTC file was written incorrectly", cm.output[0])

362

363 self.assertTrue(np.all(used == [(12, 34), (90, 10)]))

364

365 def test_ptcDatasetSort(self):

366 """Test the sorting function for the PTC dataset.

367 """

368 localDataset = copy.copy(self.dataset)

369

370 testCov = np.zeros((3, 4, 4))

371 for i in range(3):

372 testCov[i, :, :] = np.identity(4)*(2-i)

373

374 testArr = np.array([2.0, 1.0, 0.0])

375

376 for ampName in self.ampNames:

377 localDataset.inputExpIdPairs[ampName] = [(12, 34), (56, 78), (90, 10)]

378 localDataset.inputExpPairMjdStartList[ampName] = np.array(

379 [60775.39027778, 60775.39201389, 60775.39270833],

380 )

381 localDataset.overscanMedianLevelList[ampName] = np.full(3, 25000.0)

382 localDataset.expIdMask[ampName] = np.ones(3, dtype=np.bool_)

383 localDataset.expIdMask[ampName][1] = False

384 localDataset.expIdRolloffMask[ampName] = np.ones(3, dtype=np.bool_)

385 localDataset.rawExpTimes[ampName] = testArr.copy()

386 localDataset.rawMeans[ampName] = testArr.copy()

387 localDataset.rawVars[ampName] = testArr.copy()

388 localDataset.rawDeltas[ampName] = testArr.copy()

389 localDataset.rowMeanVariance[ampName] = testArr.copy()

390 localDataset.photoCharges[ampName] = testArr.copy()

391 localDataset.photoChargeDeltas[ampName] = testArr.copy()

392 localDataset.ampOffsets[ampName] = testArr.copy()

393 localDataset.gainList[ampName] = testArr.copy()

394 localDataset.noiseList[ampName] = testArr.copy()

395 localDataset.histVars[ampName] = testArr.copy()

396 localDataset.histChi2Dofs[ampName] = testArr.copy()

397 localDataset.kspValues[ampName] = testArr.copy()

398

399 localDataset.covariances[ampName] = testCov.copy()

400 localDataset.covariancesSqrtWeights[ampName] = testCov.copy()

401 localDataset.covariancesModel[ampName] = testCov.copy()

402

403 localDataset.finalMeans[ampName] = testArr.copy()

404 localDataset.finalMeans[ampName][~localDataset.expIdMask[ampName]] = np.nan

405 localDataset.finalVars[ampName] = testArr.copy()

406 localDataset.finalVars[ampName][~localDataset.expIdMask[ampName]] = np.nan

407 localDataset.finalModelVars[ampName] = testArr.copy()

408 localDataset.finalModelVars[ampName][~localDataset.expIdMask[ampName]] = np.nan

409

410 localDataset.auxValues["TEST1"] = testArr

411 localDataset.auxValues["TEST2"] = np.array(["two", "one", "zero"])

412

413 # We know this should be the sort order.

414 sortIndex = np.argsort(testArr)

415 localDataset.sort(sortIndex)

416

417 testArrSorted = testArr[sortIndex]

418 # Do the covariance sorting the "slow way" by hand to

419 # ensure that we have an independent check on the sort method itself.

420 testCovSorted = np.zeros_like(testCov)

421 for i in range(3):

422 testCovSorted[i, :, :] = testCov[sortIndex[i], :, :]

423

424 testArrSortedMasked = testArrSorted.copy()

425 testArrSortedMasked[1] = np.nan

426

427 for ampName in self.ampNames:

428 np.testing.assert_array_equal(

429 np.asarray(localDataset.inputExpIdPairs[ampName]),

430 np.asarray([(90, 10), (56, 78), (12, 34)]),

431 )

432 np.testing.assert_array_equal(

433 np.asarray(localDataset.inputExpPairMjdStartList[ampName]),

434 np.asarray([60775.39270833, 60775.39201389, 60775.39027778]),

435 )

436 np.testing.assert_array_equal(localDataset.expIdMask[ampName], np.array([True, False, True]))

437 np.testing.assert_array_equal(localDataset.expIdRolloffMask[ampName],

438 np.array([True, True, True]))

439 np.testing.assert_array_equal(localDataset.rawExpTimes[ampName], testArrSorted)

440 np.testing.assert_array_equal(localDataset.rawMeans[ampName], testArrSorted)

441 np.testing.assert_array_equal(localDataset.rawVars[ampName], testArrSorted)

442 np.testing.assert_array_equal(localDataset.rawDeltas[ampName], testArrSorted)

443 np.testing.assert_array_equal(localDataset.rowMeanVariance[ampName], testArrSorted)

444 np.testing.assert_array_equal(localDataset.photoCharges[ampName], testArrSorted)

445 np.testing.assert_array_equal(localDataset.photoChargeDeltas[ampName], testArrSorted)

446 np.testing.assert_array_equal(localDataset.ampOffsets[ampName], testArrSorted)

447 np.testing.assert_array_equal(localDataset.gainList[ampName], testArrSorted)

448 np.testing.assert_array_equal(localDataset.noiseList[ampName], testArrSorted)

449 np.testing.assert_array_equal(localDataset.histVars[ampName], testArrSorted)

450 np.testing.assert_array_equal(localDataset.histChi2Dofs[ampName], testArrSorted)

451 np.testing.assert_array_equal(localDataset.kspValues[ampName], testArrSorted)

452 np.testing.assert_array_equal(localDataset.covariances[ampName], testCovSorted)

453 np.testing.assert_array_equal(localDataset.covariancesSqrtWeights[ampName], testCovSorted)

454 np.testing.assert_array_equal(localDataset.covariancesModel[ampName], testCovSorted)

455 np.testing.assert_array_equal(localDataset.finalVars[ampName], testArrSortedMasked)

456 np.testing.assert_array_equal(localDataset.finalModelVars[ampName], testArrSortedMasked)

457 np.testing.assert_array_equal(localDataset.finalMeans[ampName], testArrSortedMasked)

458

459 np.testing.assert_array_equal(localDataset.auxValues["TEST1"], testArrSorted)

460 np.testing.assert_array_equal(localDataset.auxValues["TEST2"], np.array(["zero", "one", "two"]))

461

462 def test_ptcDatasetAppend(self):

463 """Test the append function for the PTC dataset.

464 """

465 testCov = np.zeros((3, 4, 4))

466 for i in range(3):

467 testCov[i, :, :] = np.identity(4)*(i)

468

469 testArr = np.array([0.0, 1.0, 2.0])

470

471 testPairs = [(12, 34), (56, 78), (90, 10)]

472

473 testMjds = [60775.39027778, 60775.39201389, 60775.39270833]

474

475 testStrValues = ["zero", "one", "two"]

476

477 partials = []

478 for i in range(3):

479 partial = PhotonTransferCurveDataset(self.ampNames, "PARTIAL", covMatrixSide=4)

480

481 for ampName in self.ampNames:

482 partial.setAmpValuesPartialDataset(

483 ampName,

484 inputExpIdPair=testPairs[i],

485 inputExpPairMjdStart=testMjds[i],

486 rawExpTime=testArr[i],

487 rawMean=testArr[i],

488 rawVar=testArr[i],

489 rawDelta=testArr[i],

490 rowMeanVariance=testArr[i],

491 photoCharge=testArr[i],

492 photoChargeDelta=testArr[i],

493 ampOffset=testArr[i],

494 expIdMask=True,

495 expIdRolloffMask=True,

496 nPixelCovariance=100_000,

497 covariance=testCov[i, :, :].reshape(4, 4),

498 covSqrtWeights=testCov[i, :, :].reshape(4, 4),

499 gain=testArr[i],

500 noise=testArr[i],

501 histVar=testArr[i],

502 histChi2Dof=testArr[i],

503 kspValue=testArr[i],

504 )

505 partial.setAuxValuesPartialDataset({"TEST1": float(i),

506 "TEST2": testStrValues[i]})

507

508 partials.append(partial)

509

510 ptc = PhotonTransferCurveDataset(

511 ampNames=self.ampNames,

512 ptcFitType="FULLCOVARIANCE",

513 covMatrixSide=4,

514 )

515

516 for partial in partials:

517 ptc.appendPartialPtc(partial)

518

519 for ampName in self.ampNames:

520 np.testing.assert_array_equal(ptc.inputExpIdPairs[ampName], testPairs)

521 np.testing.assert_array_equal(ptc.inputExpPairMjdStartList[ampName], testMjds)

522 np.testing.assert_array_equal(ptc.expIdMask[ampName], np.array([True, True, True]))

523 np.testing.assert_array_equal(ptc.expIdRolloffMask[ampName],

524 np.array([True, True, True]))

525 np.testing.assert_array_equal(ptc.rawExpTimes[ampName], testArr)

526 np.testing.assert_array_equal(ptc.rawMeans[ampName], testArr)

527 np.testing.assert_array_equal(ptc.rawVars[ampName], testArr)

528 np.testing.assert_array_equal(ptc.rawDeltas[ampName], testArr)

529 np.testing.assert_array_equal(ptc.rowMeanVariance[ampName], testArr)

530 np.testing.assert_array_equal(ptc.photoCharges[ampName], testArr)

531 np.testing.assert_array_equal(ptc.photoChargeDeltas[ampName], testArr)

532 np.testing.assert_array_equal(ptc.ampOffsets[ampName], testArr)

533 np.testing.assert_array_equal(ptc.gainList[ampName], testArr)

534 np.testing.assert_array_equal(ptc.noiseList[ampName], testArr)

535 np.testing.assert_array_equal(ptc.histVars[ampName], testArr)

536 np.testing.assert_array_equal(ptc.histChi2Dofs[ampName], testArr)

537 np.testing.assert_array_equal(ptc.kspValues[ampName], testArr)

538 np.testing.assert_array_equal(ptc.nPixelCovariances[ampName], 100_000)

539 np.testing.assert_array_equal(ptc.covariances[ampName], testCov)

540 np.testing.assert_array_equal(ptc.covariancesSqrtWeights[ampName], testCov)

541 # These two should have the same shape, but no useful values.

542 self.assertEqual(ptc.covariancesModel[ampName].shape, testCov.shape)

543 self.assertEqual(ptc.finalVars[ampName].shape, testArr.shape)

544 self.assertEqual(ptc.finalModelVars[ampName].shape, testArr.shape)

545 self.assertEqual(ptc.finalMeans[ampName].shape, testArr.shape)

546

547 np.testing.assert_array_equal(ptc.auxValues["TEST1"], testArr)

548 np.testing.assert_array_equal(ptc.auxValues["TEST2"], np.array(["zero", "one", "two"]))

549

550 # And test illegal inputs

551 with self.assertRaises(ValueError):

552 ptc.appendPartialPtc(ptc)

553

554 badPartial = partials[0]

555 badPartial.ptcFitType = "FULLCOVARIANCE"

556 with self.assertRaises(ValueError):

557 ptc.appendPartialPtc(badPartial)

558

559 ptc.ampNames = ["A", "B"]

560 with self.assertRaises(ValueError):

561 ptc.appendPartialPtc(partials[1])

562

563

564class MemoryTester(lsst.utils.tests.MemoryTestCase):

565 pass

566

567

568def setup_module(module):

569 lsst.utils.tests.init()

570

571

572if __name__ == "__main__": 572 ↛ 573line 572 didn't jump to line 573 because the condition on line 572 was never true

573 import sys

574 setup_module(sys.modules[__name__])

575 unittest.main()

Coverage for tests / test_ptcDataset.py: 6%

365 statements