Coverage for tests/test_ptc.py: 9%

371 statements  

« prev     ^ index     » next       coverage.py v7.4.4, created at 2024-04-23 02:19 -0700

1#!/usr/bin/env python 

2 

3# 

4# LSST Data Management System 

5# 

6# Copyright 2008-2017 AURA/LSST. 

7# 

8# This product includes software developed by the 

9# LSST Project (http://www.lsst.org/). 

10# 

11# This program is free software: you can redistribute it and/or modify 

12# it under the terms of the GNU General Public License as published by 

13# the Free Software Foundation, either version 3 of the License, or 

14# (at your option) any later version. 

15# 

16# This program is distributed in the hope that it will be useful, 

17# but WITHOUT ANY WARRANTY; without even the implied warranty of 

18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

19# GNU General Public License for more details. 

20# 

21# You should have received a copy of the LSST License Statement and 

22# the GNU General Public License along with this program. If not, 

23# see <https://www.lsstcorp.org/LegalNotices/>. 

24# 

25"""Test cases for cp_pipe.""" 

26 

27import unittest 

28import numpy as np 

29import copy 

30import tempfile 

31import logging 

32 

33import lsst.utils 

34import lsst.utils.tests 

35 

36import lsst.cp.pipe as cpPipe 

37import lsst.ip.isr.isrMock as isrMock 

38from lsst.ip.isr import PhotonTransferCurveDataset, PhotodiodeCalib 

39from lsst.cp.pipe.utils import makeMockFlats 

40 

41from lsst.pipe.base import InMemoryDatasetHandle, TaskMetadata 

42 

43 

44class FakeCamera(list): 

45 def getName(self): 

46 return "FakeCam" 

47 

48 

49class PretendRef: 

50 "A class to act as a mock exposure reference" 

51 

52 def __init__(self, exposure): 

53 self.exp = exposure 

54 

55 def get(self, component=None): 

56 if component == "visitInfo": 

57 return self.exp.getVisitInfo() 

58 elif component == "detector": 

59 return self.exp.getDetector() 

60 elif component == "metadata": 

61 return self.exp.getMetadata() 

62 else: 

63 return self.exp 

64 

65 

66class MeasurePhotonTransferCurveTaskTestCase(lsst.utils.tests.TestCase): 

67 """A test case for the PTC tasks.""" 

68 

69 def setUp(self): 

70 self.defaultConfigExtract = ( 

71 cpPipe.ptc.PhotonTransferCurveExtractTask.ConfigClass() 

72 ) 

73 self.defaultTaskExtract = cpPipe.ptc.PhotonTransferCurveExtractTask( 

74 config=self.defaultConfigExtract 

75 ) 

76 

77 self.defaultConfigSolve = cpPipe.ptc.PhotonTransferCurveSolveTask.ConfigClass() 

78 self.defaultTaskSolve = cpPipe.ptc.PhotonTransferCurveSolveTask( 

79 config=self.defaultConfigSolve 

80 ) 

81 

82 self.flatMean = 2000 

83 self.readNoiseAdu = 10 

84 mockImageConfig = isrMock.IsrMock.ConfigClass() 

85 

86 # flatDrop is not really relevant as we replace the data 

87 # but good to note it in case we change how this image is made 

88 mockImageConfig.flatDrop = 0.99999 

89 mockImageConfig.isTrimmed = True 

90 

91 self.flatExp1 = isrMock.FlatMock(config=mockImageConfig).run() 

92 self.flatExp2 = self.flatExp1.clone() 

93 (shapeY, shapeX) = self.flatExp1.getDimensions() 

94 

95 self.flatWidth = np.sqrt(self.flatMean) + self.readNoiseAdu 

96 

97 self.rng1 = np.random.RandomState(1984) 

98 flatData1 = self.rng1.normal(self.flatMean, self.flatWidth, (shapeX, shapeY)) 

99 self.rng2 = np.random.RandomState(666) 

100 flatData2 = self.rng2.normal(self.flatMean, self.flatWidth, (shapeX, shapeY)) 

101 

102 self.flatExp1.image.array[:] = flatData1 

103 self.flatExp2.image.array[:] = flatData2 

104 

105 # create fake PTC data to see if fit works, for one amp ('amp') 

106 self.flux = 1000.0 # ADU/sec 

107 self.timeVec = np.arange(1.0, 101.0, 5) 

108 self.k2NonLinearity = -5e-6 

109 # quadratic signal-chain non-linearity 

110 muVec = self.flux * self.timeVec + self.k2NonLinearity * self.timeVec**2 

111 self.gain = 0.75 # e-/ADU 

112 self.c1 = 1.0 / self.gain 

113 self.noiseSq = 2 * self.gain # 7.5 (e-)^2 

114 self.a00 = -1.2e-6 

115 self.c2 = -1.5e-6 

116 self.c3 = -4.7e-12 # tuned so that it turns over for 200k mean 

117 self.photoCharges = np.linspace(1e-8, 1e-5, len(self.timeVec)) 

118 

119 self.ampNames = [ 

120 amp.getName() for amp in self.flatExp1.getDetector().getAmplifiers() 

121 ] 

122 self.dataset = PhotonTransferCurveDataset(self.ampNames, ptcFitType="PARTIAL") 

123 self.covariancesSqrtWeights = {} 

124 for ( 

125 ampName 

126 ) in self.ampNames: # just the expTimes and means here - vars vary per function 

127 self.dataset.rawExpTimes[ampName] = self.timeVec 

128 self.dataset.rawMeans[ampName] = muVec 

129 self.dataset.covariancesSqrtWeights[ampName] = np.zeros( 

130 (1, self.dataset.covMatrixSide, self.dataset.covMatrixSide) 

131 ) 

132 

133 # ISR metadata 

134 self.metadataContents = TaskMetadata() 

135 self.metadataContents["isr"] = {} 

136 # Overscan readout noise [in ADU] 

137 for amp in self.ampNames: 

138 self.metadataContents["isr"][f"RESIDUAL STDEV {amp}"] = ( 

139 np.sqrt(self.noiseSq) / self.gain 

140 ) 

141 

142 def test_covAstier(self): 

143 """Test to check getCovariancesAstier 

144 

145 We check that the gain is the same as the imput gain from the 

146 mock data, that the covariances via FFT (as it is in 

147 MeasurePhotonTransferCurveTask when doCovariancesAstier=True) 

148 are the same as calculated in real space, and that Cov[0, 0] 

149 (i.e., the variances) are similar to the variances calculated 

150 with the standard method (when doCovariancesAstier=false), 

151 

152 """ 

153 extractConfig = self.defaultConfigExtract 

154 extractConfig.minNumberGoodPixelsForCovariance = 5000 

155 extractConfig.detectorMeasurementRegion = "FULL" 

156 extractConfig.doExtractPhotodiodeData = True 

157 extractConfig.auxiliaryHeaderKeys = ["CCOBCURR", "CCDTEMP"] 

158 extractTask = cpPipe.ptc.PhotonTransferCurveExtractTask(config=extractConfig) 

159 

160 solveConfig = self.defaultConfigSolve 

161 solveConfig.ptcFitType = "FULLCOVARIANCE" 

162 # Cut off the low-flux point which is a bad fit, and this 

163 # also exercises this functionality and makes the tests 

164 # run a lot faster. 

165 solveConfig.minMeanSignal["ALL_AMPS"] = 2000.0 

166 # Set the outlier fit threshold higher than the default appropriate 

167 # for this test dataset. 

168 solveConfig.maxSignalInitialPtcOutlierFit = 90000.0 

169 solveTask = cpPipe.ptc.PhotonTransferCurveSolveTask(config=solveConfig) 

170 

171 inputGain = self.gain 

172 

173 muStandard, varStandard = {}, {} 

174 expDict = {} 

175 expIds = [] 

176 pdHandles = [] 

177 idCounter = 0 

178 for i, expTime in enumerate(self.timeVec): 

179 mockExp1, mockExp2 = makeMockFlats( 

180 expTime, 

181 gain=inputGain, 

182 readNoiseElectrons=3, 

183 expId1=idCounter, 

184 expId2=idCounter + 1, 

185 ) 

186 for mockExp in [mockExp1, mockExp2]: 

187 md = mockExp.getMetadata() 

188 # These values are chosen to be easily compared after 

189 # processing for correct ordering. 

190 md['CCOBCURR'] = float(idCounter) 

191 md['CCDTEMP'] = float(idCounter + 1) 

192 mockExp.setMetadata(md) 

193 

194 mockExpRef1 = PretendRef(mockExp1) 

195 mockExpRef2 = PretendRef(mockExp2) 

196 expDict[expTime] = ((mockExpRef1, idCounter), (mockExpRef2, idCounter + 1)) 

197 expIds.append(idCounter) 

198 expIds.append(idCounter + 1) 

199 for ampNumber, ampName in enumerate(self.ampNames): 

200 # cov has (i, j, var, cov, npix) 

201 ( 

202 im1Area, 

203 im2Area, 

204 imStatsCtrl, 

205 mu1, 

206 mu2, 

207 ) = extractTask.getImageAreasMasksStats(mockExp1, mockExp2) 

208 muDiff, varDiff, covAstier, rowMeanVariance = extractTask.measureMeanVarCov( 

209 im1Area, im2Area, imStatsCtrl, mu1, mu2 

210 ) 

211 muStandard.setdefault(ampName, []).append(muDiff) 

212 varStandard.setdefault(ampName, []).append(varDiff) 

213 

214 # Make a photodiode dataset to integrate. 

215 timeSamples = np.linspace(0, 20.0, 100) 

216 currentSamples = np.zeros(100) 

217 currentSamples[50] = -1.0*self.photoCharges[i] 

218 

219 pdCalib = PhotodiodeCalib(timeSamples=timeSamples, currentSamples=currentSamples) 

220 pdCalib.currentScale = -1.0 

221 pdCalib.integrationMethod = "CHARGE_SUM" 

222 

223 pdHandles.append( 

224 InMemoryDatasetHandle( 

225 pdCalib, 

226 dataId={"exposure": idCounter}, 

227 ) 

228 ) 

229 pdHandles.append( 

230 InMemoryDatasetHandle( 

231 pdCalib, 

232 dataId={"exposure": idCounter + 1}, 

233 ) 

234 ) 

235 idCounter += 2 

236 

237 resultsExtract = extractTask.run( 

238 inputExp=expDict, 

239 inputDims=expIds, 

240 taskMetadata=[self.metadataContents for x in expIds], 

241 inputPhotodiodeData=pdHandles, 

242 ) 

243 

244 # Force the last PTC dataset to have a NaN, and ensure that the 

245 # task runs (DM-38029). This is a minor perturbation and does not 

246 # affect the output comparison. Note that we use index -2 because 

247 # these datasets are in pairs of [real, dummy] to match the inputs 

248 # to the extract task. 

249 resultsExtract.outputCovariances[-2].rawMeans["C:0,0"] = np.array([np.nan]) 

250 resultsExtract.outputCovariances[-2].rawVars["C:0,0"] = np.array([np.nan]) 

251 

252 # Force the next-to-last PTC dataset to have a decreased variance to 

253 # ensure that the outlier fit rejection works. Note that we use 

254 # index -4 because these datasets are in pairs of [real, dummy] to 

255 # match the inputs to the extract task. 

256 rawVar = resultsExtract.outputCovariances[-4].rawVars["C:0,0"] 

257 resultsExtract.outputCovariances[-4].rawVars["C:0,0"] = rawVar * 0.9 

258 

259 # Reorganize the outputCovariances so we can confirm they come 

260 # out sorted afterwards. 

261 outputCovariancesRev = resultsExtract.outputCovariances[::-1] 

262 

263 resultsSolve = solveTask.run( 

264 outputCovariancesRev, camera=FakeCamera([self.flatExp1.getDetector()]) 

265 ) 

266 

267 ptc = resultsSolve.outputPtcDataset 

268 

269 # Some expected values for noise matrix, just to check that 

270 # it was calculated. 

271 noiseMatrixNoBExpected = { 

272 (0, 0): 6.53126505, 

273 (1, 1): -23.20924747, 

274 (2, 2): 35.69834113, 

275 } 

276 noiseMatrixExpected = { 

277 (0, 0): 29.37146918, 

278 (1, 1): -14.6849025, 

279 (2, 2): 24.7328517, 

280 } 

281 

282 noiseMatrixExpected = np.array( 

283 [ 

284 [ 

285 29.37146918, 

286 9.2760363, 

287 -29.08907932, 

288 33.65818827, 

289 -52.65710984, 

290 -18.5821773, 

291 -46.26896286, 

292 65.01049736, 

293 ], 

294 [ 

295 -3.62427987, 

296 -14.6849025, 

297 -46.55230305, 

298 -1.30410627, 

299 6.44903599, 

300 18.11796075, 

301 -22.72874074, 

302 20.90219857, 

303 ], 

304 [ 

305 5.09203058, 

306 -4.40097862, 

307 24.7328517, 

308 39.2847586, 

309 -21.46132351, 

310 8.12179783, 

311 6.23585617, 

312 -2.09949622, 

313 ], 

314 [ 

315 35.79204016, 

316 -6.50205005, 

317 3.37910363, 

318 15.22335662, 

319 -19.29035067, 

320 9.66065941, 

321 7.47510934, 

322 20.25962845, 

323 ], 

324 [ 

325 -36.23187633, 

326 -22.72307472, 

327 16.29140749, 

328 -13.09493835, 

329 3.32091085, 

330 52.4380977, 

331 -8.06428902, 

332 -22.66669839, 

333 ], 

334 [ 

335 -27.93122896, 

336 15.37016686, 

337 9.18835073, 

338 -24.48892946, 

339 8.14480304, 

340 22.38983222, 

341 22.36866891, 

342 -0.38803439, 

343 ], 

344 [ 

345 17.13962665, 

346 -28.33153763, 

347 -17.79744334, 

348 -18.57064463, 

349 7.69408833, 

350 8.48265396, 

351 18.0447022, 

352 -16.97496022, 

353 ], 

354 [ 

355 10.09078383, 

356 -26.61613002, 

357 10.48504889, 

358 15.33196998, 

359 -23.35165517, 

360 -24.53098643, 

361 -18.21201067, 

362 17.40755051, 

363 ], 

364 ] 

365 ) 

366 

367 noiseMatrixNoBExpected = np.array( 

368 [ 

369 [ 

370 6.53126505, 

371 12.14827594, 

372 -37.11919923, 

373 41.18675353, 

374 -85.1613845, 

375 -28.45801954, 

376 -61.24442999, 

377 88.76480122, 

378 ], 

379 [ 

380 -4.64541165, 

381 -23.20924747, 

382 -66.08733987, 

383 -0.87558055, 

384 12.20111853, 

385 24.84795549, 

386 -34.92458788, 

387 24.42745014, 

388 ], 

389 [ 

390 7.66734507, 

391 -4.51403645, 

392 35.69834113, 

393 52.73693356, 

394 -30.85044089, 

395 10.86761771, 

396 10.8503068, 

397 -2.18908327, 

398 ], 

399 [ 

400 50.9901156, 

401 -7.34803977, 

402 5.33443765, 

403 21.60899396, 

404 -25.06129827, 

405 15.14015505, 

406 10.94263771, 

407 29.23975515, 

408 ], 

409 [ 

410 -48.66912069, 

411 -31.58003774, 

412 21.81305735, 

413 -13.08993444, 

414 8.17275394, 

415 74.85293723, 

416 -11.18403252, 

417 -31.7799437, 

418 ], 

419 [ 

420 -38.55206382, 

421 22.92982676, 

422 13.39861008, 

423 -33.3307362, 

424 8.65362238, 

425 29.18775548, 

426 31.78433947, 

427 1.27923706, 

428 ], 

429 [ 

430 23.33663918, 

431 -41.74105625, 

432 -26.55920751, 

433 -24.71611677, 

434 12.13343146, 

435 11.25763907, 

436 21.79131019, 

437 -26.579393, 

438 ], 

439 [ 

440 11.44334226, 

441 -34.9759641, 

442 13.96449509, 

443 19.64121933, 

444 -36.09794843, 

445 -34.27205933, 

446 -25.16574105, 

447 23.80460972, 

448 ], 

449 ] 

450 ) 

451 

452 for amp in self.ampNames: 

453 self.assertAlmostEqual(ptc.gain[amp], inputGain, places=2) 

454 for v1, v2 in zip(varStandard[amp], ptc.finalVars[amp]): 

455 self.assertAlmostEqual(v1 / v2, 1.0, places=1) 

456 

457 # Check that the PTC turnoff is correctly computed. 

458 # This will be different for the C:0,0 amp. 

459 if amp == "C:0,0": 

460 self.assertAlmostEqual(ptc.ptcTurnoff[amp], ptc.rawMeans[ampName][-3]) 

461 else: 

462 self.assertAlmostEqual(ptc.ptcTurnoff[amp], ptc.rawMeans[ampName][-1]) 

463 

464 # Test that all the quantities are correctly ordered and have 

465 # not accidentally been masked. We check every other output ([::2]) 

466 # because these datasets are in pairs of [real, dummy] to 

467 # match the inputs to the extract task. 

468 for i, extractPtc in enumerate(resultsExtract.outputCovariances[::2]): 

469 self.assertFloatsAlmostEqual( 

470 extractPtc.rawExpTimes[ampName][0], 

471 ptc.rawExpTimes[ampName][i], 

472 ) 

473 self.assertFloatsAlmostEqual( 

474 extractPtc.rawMeans[ampName][0], 

475 ptc.rawMeans[ampName][i], 

476 ) 

477 self.assertFloatsAlmostEqual( 

478 extractPtc.rawVars[ampName][0], 

479 ptc.rawVars[ampName][i], 

480 ) 

481 self.assertFloatsAlmostEqual( 

482 extractPtc.photoCharges[ampName][0], 

483 ptc.photoCharges[ampName][i], 

484 ) 

485 self.assertFloatsAlmostEqual( 

486 extractPtc.histVars[ampName][0], 

487 ptc.histVars[ampName][i], 

488 ) 

489 self.assertFloatsAlmostEqual( 

490 extractPtc.histChi2Dofs[ampName][0], 

491 ptc.histChi2Dofs[ampName][i], 

492 ) 

493 self.assertFloatsAlmostEqual( 

494 extractPtc.kspValues[ampName][0], 

495 ptc.kspValues[ampName][i], 

496 ) 

497 self.assertFloatsAlmostEqual( 

498 extractPtc.covariances[ampName][0], 

499 ptc.covariances[ampName][i], 

500 ) 

501 self.assertFloatsAlmostEqual( 

502 extractPtc.covariancesSqrtWeights[ampName][0], 

503 ptc.covariancesSqrtWeights[ampName][i], 

504 ) 

505 self.assertFloatsAlmostEqual( 

506 ptc.noiseMatrix[ampName], noiseMatrixExpected, atol=1e-8, rtol=None 

507 ) 

508 self.assertFloatsAlmostEqual( 

509 ptc.noiseMatrixNoB[ampName], 

510 noiseMatrixNoBExpected, 

511 atol=1e-8, 

512 rtol=None, 

513 ) 

514 

515 mask = ptc.getGoodPoints(amp) 

516 

517 values = ( 

518 ptc.covariancesModel[amp][mask, 0, 0] - ptc.covariances[amp][mask, 0, 0] 

519 ) / ptc.covariancesModel[amp][mask, 0, 0] 

520 np.testing.assert_array_less(np.abs(values), 2e-3) 

521 

522 values = ( 

523 ptc.covariancesModel[amp][mask, 1, 1] - ptc.covariances[amp][mask, 1, 1] 

524 ) / ptc.covariancesModel[amp][mask, 1, 1] 

525 np.testing.assert_array_less(np.abs(values), 0.2) 

526 

527 values = ( 

528 ptc.covariancesModel[amp][mask, 1, 2] - ptc.covariances[amp][mask, 1, 2] 

529 ) / ptc.covariancesModel[amp][mask, 1, 2] 

530 np.testing.assert_array_less(np.abs(values), 0.2) 

531 

532 # And test that the auxiliary values are there and correctly ordered. 

533 self.assertIn('CCOBCURR', ptc.auxValues) 

534 self.assertIn('CCDTEMP', ptc.auxValues) 

535 firstExpIds = np.array([i for i, _ in ptc.inputExpIdPairs['C:0,0']], dtype=np.float64) 

536 self.assertFloatsAlmostEqual(ptc.auxValues['CCOBCURR'], firstExpIds) 

537 self.assertFloatsAlmostEqual(ptc.auxValues['CCDTEMP'], firstExpIds + 1) 

538 

539 expIdsUsed = ptc.getExpIdsUsed("C:0,0") 

540 # Check that these are the same as the inputs, paired up, with the 

541 # first two (low flux) and final four (outliers, nans) removed. 

542 self.assertTrue( 

543 np.all(expIdsUsed == np.array(expIds).reshape(len(expIds) // 2, 2)[1:-2]) 

544 ) 

545 

546 goodAmps = ptc.getGoodAmps() 

547 self.assertEqual(goodAmps, self.ampNames) 

548 

549 # Check that every possibly modified field has the same length. 

550 covShape = None 

551 covSqrtShape = None 

552 covModelShape = None 

553 covModelNoBShape = None 

554 

555 for ampName in self.ampNames: 

556 if covShape is None: 

557 covShape = ptc.covariances[ampName].shape 

558 covSqrtShape = ptc.covariancesSqrtWeights[ampName].shape 

559 covModelShape = ptc.covariancesModel[ampName].shape 

560 covModelNoBShape = ptc.covariancesModelNoB[ampName].shape 

561 else: 

562 self.assertEqual(ptc.covariances[ampName].shape, covShape) 

563 self.assertEqual( 

564 ptc.covariancesSqrtWeights[ampName].shape, covSqrtShape 

565 ) 

566 self.assertEqual(ptc.covariancesModel[ampName].shape, covModelShape) 

567 self.assertEqual( 

568 ptc.covariancesModelNoB[ampName].shape, covModelNoBShape 

569 ) 

570 

571 # And check that this is serializable 

572 with tempfile.NamedTemporaryFile(suffix=".fits") as f: 

573 usedFilename = ptc.writeFits(f.name) 

574 fromFits = PhotonTransferCurveDataset.readFits(usedFilename) 

575 self.assertEqual(fromFits, ptc) 

576 

577 def ptcFitAndCheckPtc( 

578 self, 

579 order=None, 

580 fitType=None, 

581 doFitBootstrap=False, 

582 doLegacy=False, 

583 ): 

584 localDataset = copy.deepcopy(self.dataset) 

585 localDataset.ptcFitType = fitType 

586 configSolve = copy.copy(self.defaultConfigSolve) 

587 if doFitBootstrap: 

588 configSolve.doFitBootstrap = True 

589 

590 configSolve.doLegacyTurnoffSelection = doLegacy 

591 

592 if fitType == "POLYNOMIAL": 

593 if order not in [2, 3]: 

594 RuntimeError("Enter a valid polynomial order for this test: 2 or 3") 

595 if order == 2: 

596 for ampName in self.ampNames: 

597 localDataset.rawVars[ampName] = [ 

598 self.noiseSq + self.c1 * mu + self.c2 * mu**2 

599 for mu in localDataset.rawMeans[ampName] 

600 ] 

601 configSolve.polynomialFitDegree = 2 

602 if order == 3: 

603 for ampName in self.ampNames: 

604 localDataset.rawVars[ampName] = [ 

605 self.noiseSq 

606 + self.c1 * mu 

607 + self.c2 * mu**2 

608 + self.c3 * mu**3 

609 for mu in localDataset.rawMeans[ampName] 

610 ] 

611 configSolve.polynomialFitDegree = 3 

612 elif fitType == "EXPAPPROXIMATION": 

613 g = self.gain 

614 for ampName in self.ampNames: 

615 localDataset.rawVars[ampName] = [ 

616 ( 

617 0.5 / (self.a00 * g**2) * (np.exp(2 * self.a00 * mu * g) - 1) 

618 + self.noiseSq / (g * g) 

619 ) 

620 for mu in localDataset.rawMeans[ampName] 

621 ] 

622 else: 

623 raise RuntimeError( 

624 "Enter a fit function type: 'POLYNOMIAL' or 'EXPAPPROXIMATION'" 

625 ) 

626 

627 # Initialize mask and covariance weights that will be used in fits. 

628 # Covariance weights values empirically determined from one of 

629 # the cases in test_covAstier. 

630 matrixSize = localDataset.covMatrixSide 

631 maskLength = len(localDataset.rawMeans[ampName]) 

632 for ampName in self.ampNames: 

633 localDataset.expIdMask[ampName] = np.repeat(True, maskLength) 

634 localDataset.covariancesSqrtWeights[ampName] = np.repeat( 

635 np.ones((matrixSize, matrixSize)), maskLength 

636 ).reshape((maskLength, matrixSize, matrixSize)) 

637 localDataset.covariancesSqrtWeights[ampName][:, 0, 0] = [ 

638 0.07980188, 

639 0.01339653, 

640 0.0073118, 

641 0.00502802, 

642 0.00383132, 

643 0.00309475, 

644 0.00259572, 

645 0.00223528, 

646 0.00196273, 

647 0.00174943, 

648 0.00157794, 

649 0.00143707, 

650 0.00131929, 

651 0.00121935, 

652 0.0011334, 

653 0.00105893, 

654 0.00099357, 

655 0.0009358, 

656 0.00088439, 

657 0.00083833, 

658 ] 

659 

660 solveTask = cpPipe.ptc.PhotonTransferCurveSolveTask(config=configSolve) 

661 

662 localDataset = solveTask.fitMeasurementsToModel(localDataset) 

663 

664 # check entries in localDataset, which was modified by the function 

665 for ampName in self.ampNames: 

666 self.assertEqual(fitType, localDataset.ptcFitType) 

667 self.assertAlmostEqual(self.gain, localDataset.gain[ampName]) 

668 if fitType == "POLYNOMIAL": 

669 self.assertAlmostEqual(self.c1, localDataset.ptcFitPars[ampName][1]) 

670 self.assertAlmostEqual( 

671 np.sqrt(self.noiseSq) * self.gain, localDataset.noise[ampName] 

672 ) 

673 if fitType == "EXPAPPROXIMATION": 

674 self.assertAlmostEqual( 

675 self.a00, localDataset.ptcFitPars[ampName][0] 

676 ) 

677 # noise already in electrons for 'EXPAPPROXIMATION' fit 

678 self.assertAlmostEqual( 

679 np.sqrt(self.noiseSq), localDataset.noise[ampName] 

680 ) 

681 

682 def test_ptcFit(self): 

683 for doLegacy in [False, True]: 

684 for fitType, order in [ 

685 ("POLYNOMIAL", 2), 

686 ("POLYNOMIAL", 3), 

687 ("EXPAPPROXIMATION", None), 

688 ]: 

689 self.ptcFitAndCheckPtc( 

690 fitType=fitType, 

691 order=order, 

692 doLegacy=doLegacy, 

693 ) 

694 

695 def test_meanVarMeasurement(self): 

696 task = self.defaultTaskExtract 

697 im1Area, im2Area, imStatsCtrl, mu1, mu2 = task.getImageAreasMasksStats( 

698 self.flatExp1, self.flatExp2 

699 ) 

700 mu, varDiff, _, _ = task.measureMeanVarCov(im1Area, im2Area, imStatsCtrl, mu1, mu2) 

701 

702 self.assertLess(self.flatWidth - np.sqrt(varDiff), 1) 

703 self.assertLess(self.flatMean - mu, 1) 

704 

705 def test_meanVarMeasurementWithNans(self): 

706 task = self.defaultTaskExtract 

707 

708 flatExp1 = self.flatExp1.clone() 

709 flatExp2 = self.flatExp2.clone() 

710 

711 flatExp1.image.array[20:30, :] = np.nan 

712 flatExp2.image.array[20:30, :] = np.nan 

713 

714 im1Area, im2Area, imStatsCtrl, mu1, mu2 = task.getImageAreasMasksStats( 

715 flatExp1, flatExp2 

716 ) 

717 mu, varDiff, _, _ = task.measureMeanVarCov(im1Area, im2Area, imStatsCtrl, mu1, mu2) 

718 

719 expectedMu1 = np.nanmean(flatExp1.image.array) 

720 expectedMu2 = np.nanmean(flatExp2.image.array) 

721 expectedMu = 0.5 * (expectedMu1 + expectedMu2) 

722 

723 # Now the variance of the difference. First, create the diff image. 

724 im1 = flatExp1.maskedImage 

725 im2 = flatExp2.maskedImage 

726 

727 temp = im2.clone() 

728 temp *= expectedMu1 

729 diffIm = im1.clone() 

730 diffIm *= expectedMu2 

731 diffIm -= temp 

732 diffIm /= expectedMu 

733 

734 # Divide by two as it is what measureMeanVarCov returns 

735 # (variance of difference) 

736 expectedVar = 0.5 * np.nanvar(diffIm.image.array) 

737 

738 # Check that the standard deviations and the emans agree to 

739 # less than 1 ADU 

740 self.assertLess(np.sqrt(expectedVar) - np.sqrt(varDiff), 1) 

741 self.assertLess(expectedMu - mu, 1) 

742 

743 def test_meanVarMeasurementAllNan(self): 

744 task = self.defaultTaskExtract 

745 flatExp1 = self.flatExp1.clone() 

746 flatExp2 = self.flatExp2.clone() 

747 

748 flatExp1.image.array[:, :] = np.nan 

749 flatExp2.image.array[:, :] = np.nan 

750 

751 im1Area, im2Area, imStatsCtrl, mu1, mu2 = task.getImageAreasMasksStats( 

752 flatExp1, flatExp2 

753 ) 

754 mu, varDiff, covDiff, rowMeanVariance = task.measureMeanVarCov( 

755 im1Area, im2Area, imStatsCtrl, mu1, mu2 

756 ) 

757 

758 self.assertTrue(np.isnan(mu)) 

759 self.assertTrue(np.isnan(varDiff)) 

760 self.assertTrue(covDiff is None) 

761 self.assertTrue(np.isnan(rowMeanVariance)) 

762 

763 def test_meanVarMeasurementTooFewPixels(self): 

764 task = self.defaultTaskExtract 

765 flatExp1 = self.flatExp1.clone() 

766 flatExp2 = self.flatExp2.clone() 

767 

768 flatExp1.image.array[0:190, :] = np.nan 

769 flatExp2.image.array[0:190, :] = np.nan 

770 

771 bit = flatExp1.mask.getMaskPlaneDict()["NO_DATA"] 

772 flatExp1.mask.array[0:190, :] &= 2**bit 

773 flatExp2.mask.array[0:190, :] &= 2**bit 

774 

775 im1Area, im2Area, imStatsCtrl, mu1, mu2 = task.getImageAreasMasksStats( 

776 flatExp1, flatExp2 

777 ) 

778 with self.assertLogs(level=logging.WARNING) as cm: 

779 mu, varDiff, covDiff, rowMeanVariance = task.measureMeanVarCov( 

780 im1Area, im2Area, imStatsCtrl, mu1, mu2 

781 ) 

782 self.assertIn("Number of good points", cm.output[0]) 

783 

784 self.assertTrue(np.isnan(mu)) 

785 self.assertTrue(np.isnan(varDiff)) 

786 self.assertTrue(covDiff is None) 

787 self.assertTrue(np.isnan(rowMeanVariance)) 

788 

789 def test_meanVarMeasurementTooNarrowStrip(self): 

790 # We need a new config to make sure the second covariance cut is 

791 # triggered. 

792 config = cpPipe.ptc.PhotonTransferCurveExtractTask.ConfigClass() 

793 config.minNumberGoodPixelsForCovariance = 10 

794 task = cpPipe.ptc.PhotonTransferCurveExtractTask(config=config) 

795 flatExp1 = self.flatExp1.clone() 

796 flatExp2 = self.flatExp2.clone() 

797 

798 flatExp1.image.array[0:195, :] = np.nan 

799 flatExp2.image.array[0:195, :] = np.nan 

800 flatExp1.image.array[:, 0:195] = np.nan 

801 flatExp2.image.array[:, 0:195] = np.nan 

802 

803 bit = flatExp1.mask.getMaskPlaneDict()["NO_DATA"] 

804 flatExp1.mask.array[0:195, :] &= 2**bit 

805 flatExp2.mask.array[0:195, :] &= 2**bit 

806 flatExp1.mask.array[:, 0:195] &= 2**bit 

807 flatExp2.mask.array[:, 0:195] &= 2**bit 

808 

809 im1Area, im2Area, imStatsCtrl, mu1, mu2 = task.getImageAreasMasksStats( 

810 flatExp1, flatExp2 

811 ) 

812 with self.assertLogs(level=logging.WARNING) as cm: 

813 mu, varDiff, covDiff, rowMeanVariance = task.measureMeanVarCov( 

814 im1Area, im2Area, imStatsCtrl, mu1, mu2 

815 ) 

816 self.assertIn("Not enough pixels", cm.output[0]) 

817 

818 self.assertTrue(np.isnan(mu)) 

819 self.assertTrue(np.isnan(varDiff)) 

820 self.assertTrue(covDiff is None) 

821 self.assertTrue(np.isnan(rowMeanVariance)) 

822 

823 def test_makeZeroSafe(self): 

824 noZerosArray = [1.0, 20, -35, 45578.98, 90.0, 897, 659.8] 

825 someZerosArray = [1.0, 20, 0, 0, 90, 879, 0] 

826 allZerosArray = [0.0, 0.0, 0, 0, 0.0, 0, 0] 

827 

828 substituteValue = 1e-10 

829 

830 expectedSomeZerosArray = [ 

831 1.0, 

832 20, 

833 substituteValue, 

834 substituteValue, 

835 90, 

836 879, 

837 substituteValue, 

838 ] 

839 expectedAllZerosArray = np.repeat(substituteValue, len(allZerosArray)) 

840 

841 measuredSomeZerosArray = self.defaultTaskSolve._makeZeroSafe( 

842 someZerosArray, substituteValue=substituteValue 

843 ) 

844 measuredAllZerosArray = self.defaultTaskSolve._makeZeroSafe( 

845 allZerosArray, substituteValue=substituteValue 

846 ) 

847 measuredNoZerosArray = self.defaultTaskSolve._makeZeroSafe( 

848 noZerosArray, substituteValue=substituteValue 

849 ) 

850 

851 for exp, meas in zip(expectedSomeZerosArray, measuredSomeZerosArray): 

852 self.assertEqual(exp, meas) 

853 for exp, meas in zip(expectedAllZerosArray, measuredAllZerosArray): 

854 self.assertEqual(exp, meas) 

855 for exp, meas in zip(noZerosArray, measuredNoZerosArray): 

856 self.assertEqual(exp, meas) 

857 

858 def test_getInitialGoodPoints(self): 

859 xs = [1, 2, 3, 4, 5, 6] 

860 ys = [2 * x for x in xs] 

861 points = self.defaultTaskSolve._getInitialGoodPoints( 

862 xs, ys, minVarPivotSearch=0.0, consecutivePointsVarDecreases=2 

863 ) 

864 assert np.all(points) == np.all(np.array([True for x in xs])) 

865 

866 ys[4] = 7 # Variance decreases in two consecutive points after ys[3]=8 

867 ys[5] = 6 

868 points = self.defaultTaskSolve._getInitialGoodPoints( 

869 xs, ys, minVarPivotSearch=0.0, consecutivePointsVarDecreases=2 

870 ) 

871 assert np.all(points) == np.all(np.array([True, True, True, True, False])) 

872 

873 def runGetGainFromFlatPair(self, correctionType="NONE"): 

874 extractConfig = self.defaultConfigExtract 

875 extractConfig.gainCorrectionType = correctionType 

876 extractConfig.minNumberGoodPixelsForCovariance = 5000 

877 extractTask = cpPipe.ptc.PhotonTransferCurveExtractTask(config=extractConfig) 

878 

879 expDict = {} 

880 expIds = [] 

881 idCounter = 0 

882 inputGain = self.gain # 1.5 e/ADU 

883 for expTime in self.timeVec: 

884 # Approximation works better at low flux, e.g., < 10000 ADU 

885 mockExp1, mockExp2 = makeMockFlats( 

886 expTime, 

887 gain=inputGain, 

888 readNoiseElectrons=np.sqrt(self.noiseSq), 

889 fluxElectrons=100, 

890 expId1=idCounter, 

891 expId2=idCounter + 1, 

892 ) 

893 mockExpRef1 = PretendRef(mockExp1) 

894 mockExpRef2 = PretendRef(mockExp2) 

895 expDict[expTime] = ((mockExpRef1, idCounter), (mockExpRef2, idCounter + 1)) 

896 expIds.append(idCounter) 

897 expIds.append(idCounter + 1) 

898 idCounter += 2 

899 

900 resultsExtract = extractTask.run( 

901 inputExp=expDict, 

902 inputDims=expIds, 

903 taskMetadata=[self.metadataContents for x in expIds], 

904 ) 

905 for exposurePair in resultsExtract.outputCovariances: 

906 for ampName in self.ampNames: 

907 if exposurePair.gain[ampName] is np.nan: 

908 continue 

909 self.assertAlmostEqual( 

910 exposurePair.gain[ampName], inputGain, delta=0.04 

911 ) 

912 

913 def test_getGainFromFlatPair(self): 

914 for gainCorrectionType in [ 

915 "NONE", 

916 "SIMPLE", 

917 "FULL", 

918 ]: 

919 self.runGetGainFromFlatPair(gainCorrectionType) 

920 

921 def test_ptcFitBootstrap(self): 

922 """Test the bootstrap fit option for the PTC""" 

923 for (fitType, order) in [('POLYNOMIAL', 2), ('POLYNOMIAL', 3), ('EXPAPPROXIMATION', None)]: 

924 self.ptcFitAndCheckPtc(fitType=fitType, order=order, doFitBootstrap=True) 

925 

926 

927class MeasurePhotonTransferCurveDatasetTestCase(lsst.utils.tests.TestCase): 

928 def setUp(self): 

929 self.ptcData = PhotonTransferCurveDataset(["C00", "C01"], " ") 

930 self.ptcData.inputExpIdPairs = { 

931 "C00": [(123, 234), (345, 456), (567, 678)], 

932 "C01": [(123, 234), (345, 456), (567, 678)], 

933 } 

934 

935 def test_generalBehaviour(self): 

936 test = PhotonTransferCurveDataset(["C00", "C01"], " ") 

937 test.inputExpIdPairs = { 

938 "C00": [(123, 234), (345, 456), (567, 678)], 

939 "C01": [(123, 234), (345, 456), (567, 678)], 

940 } 

941 

942 

943class TestMemory(lsst.utils.tests.MemoryTestCase): 

944 pass 

945 

946 

947def setup_module(module): 

948 lsst.utils.tests.init() 

949 

950 

951if __name__ == "__main__": 951 ↛ 952line 951 didn't jump to line 952, because the condition on line 951 was never true

952 lsst.utils.tests.init() 

953 unittest.main()