Coverage for tests/test_ptc.py: 8%

391 statements  

« prev     ^ index     » next       coverage.py v7.2.7, created at 2023-06-26 01:39 -0700

1#!/usr/bin/env python 

2 

3# 

4# LSST Data Management System 

5# 

6# Copyright 2008-2017 AURA/LSST. 

7# 

8# This product includes software developed by the 

9# LSST Project (http://www.lsst.org/). 

10# 

11# This program is free software: you can redistribute it and/or modify 

12# it under the terms of the GNU General Public License as published by 

13# the Free Software Foundation, either version 3 of the License, or 

14# (at your option) any later version. 

15# 

16# This program is distributed in the hope that it will be useful, 

17# but WITHOUT ANY WARRANTY; without even the implied warranty of 

18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

19# GNU General Public License for more details. 

20# 

21# You should have received a copy of the LSST License Statement and 

22# the GNU General Public License along with this program. If not, 

23# see <https://www.lsstcorp.org/LegalNotices/>. 

24# 

25"""Test cases for cp_pipe.""" 

26 

27import unittest 

28import numpy as np 

29import copy 

30import tempfile 

31import logging 

32 

33import lsst.utils 

34import lsst.utils.tests 

35 

36import lsst.cp.pipe as cpPipe 

37import lsst.ip.isr.isrMock as isrMock 

38from lsst.ip.isr import PhotonTransferCurveDataset 

39from lsst.cp.pipe.utils import funcPolynomial, makeMockFlats 

40 

41from lsst.pipe.base import TaskMetadata 

42 

43 

44class FakeCamera(list): 

45 def getName(self): 

46 return "FakeCam" 

47 

48 

49class PretendRef: 

50 "A class to act as a mock exposure reference" 

51 

52 def __init__(self, exposure): 

53 self.exp = exposure 

54 

55 def get(self, component=None): 

56 if component == "visitInfo": 

57 return self.exp.getVisitInfo() 

58 elif component == "detector": 

59 return self.exp.getDetector() 

60 elif component == "metadata": 

61 return self.exp.getMetadata() 

62 else: 

63 return self.exp 

64 

65 

66class MeasurePhotonTransferCurveTaskTestCase(lsst.utils.tests.TestCase): 

67 """A test case for the PTC tasks.""" 

68 

69 def setUp(self): 

70 self.defaultConfigExtract = ( 

71 cpPipe.ptc.PhotonTransferCurveExtractTask.ConfigClass() 

72 ) 

73 self.defaultTaskExtract = cpPipe.ptc.PhotonTransferCurveExtractTask( 

74 config=self.defaultConfigExtract 

75 ) 

76 

77 self.defaultConfigSolve = cpPipe.ptc.PhotonTransferCurveSolveTask.ConfigClass() 

78 self.defaultTaskSolve = cpPipe.ptc.PhotonTransferCurveSolveTask( 

79 config=self.defaultConfigSolve 

80 ) 

81 

82 self.flatMean = 2000 

83 self.readNoiseAdu = 10 

84 mockImageConfig = isrMock.IsrMock.ConfigClass() 

85 

86 # flatDrop is not really relevant as we replace the data 

87 # but good to note it in case we change how this image is made 

88 mockImageConfig.flatDrop = 0.99999 

89 mockImageConfig.isTrimmed = True 

90 

91 self.flatExp1 = isrMock.FlatMock(config=mockImageConfig).run() 

92 self.flatExp2 = self.flatExp1.clone() 

93 (shapeY, shapeX) = self.flatExp1.getDimensions() 

94 

95 self.flatWidth = np.sqrt(self.flatMean) + self.readNoiseAdu 

96 

97 self.rng1 = np.random.RandomState(1984) 

98 flatData1 = self.rng1.normal(self.flatMean, self.flatWidth, (shapeX, shapeY)) 

99 self.rng2 = np.random.RandomState(666) 

100 flatData2 = self.rng2.normal(self.flatMean, self.flatWidth, (shapeX, shapeY)) 

101 

102 self.flatExp1.image.array[:] = flatData1 

103 self.flatExp2.image.array[:] = flatData2 

104 

105 # create fake PTC data to see if fit works, for one amp ('amp') 

106 self.flux = 1000.0 # ADU/sec 

107 self.timeVec = np.arange(1.0, 101.0, 5) 

108 self.k2NonLinearity = -5e-6 

109 # quadratic signal-chain non-linearity 

110 muVec = self.flux * self.timeVec + self.k2NonLinearity * self.timeVec**2 

111 self.gain = 0.75 # e-/ADU 

112 self.c1 = 1.0 / self.gain 

113 self.noiseSq = 2 * self.gain # 7.5 (e-)^2 

114 self.a00 = -1.2e-6 

115 self.c2 = -1.5e-6 

116 self.c3 = -4.7e-12 # tuned so that it turns over for 200k mean 

117 

118 self.ampNames = [ 

119 amp.getName() for amp in self.flatExp1.getDetector().getAmplifiers() 

120 ] 

121 self.dataset = PhotonTransferCurveDataset(self.ampNames, ptcFitType="PARTIAL") 

122 self.covariancesSqrtWeights = {} 

123 for ( 

124 ampName 

125 ) in self.ampNames: # just the expTimes and means here - vars vary per function 

126 self.dataset.rawExpTimes[ampName] = self.timeVec 

127 self.dataset.rawMeans[ampName] = muVec 

128 self.dataset.covariancesSqrtWeights[ampName] = np.zeros( 

129 (1, self.dataset.covMatrixSide, self.dataset.covMatrixSide) 

130 ) 

131 

132 # ISR metadata 

133 self.metadataContents = TaskMetadata() 

134 self.metadataContents["isr"] = {} 

135 # Overscan readout noise [in ADU] 

136 for amp in self.ampNames: 

137 self.metadataContents["isr"][f"RESIDUAL STDEV {amp}"] = ( 

138 np.sqrt(self.noiseSq) / self.gain 

139 ) 

140 

141 def test_covAstier(self): 

142 """Test to check getCovariancesAstier 

143 

144 We check that the gain is the same as the imput gain from the 

145 mock data, that the covariances via FFT (as it is in 

146 MeasurePhotonTransferCurveTask when doCovariancesAstier=True) 

147 are the same as calculated in real space, and that Cov[0, 0] 

148 (i.e., the variances) are similar to the variances calculated 

149 with the standard method (when doCovariancesAstier=false), 

150 

151 """ 

152 extractConfig = self.defaultConfigExtract 

153 extractConfig.minNumberGoodPixelsForCovariance = 5000 

154 extractConfig.detectorMeasurementRegion = "FULL" 

155 extractTask = cpPipe.ptc.PhotonTransferCurveExtractTask(config=extractConfig) 

156 

157 solveConfig = self.defaultConfigSolve 

158 solveConfig.ptcFitType = "FULLCOVARIANCE" 

159 # Cut off the low-flux point which is a bad fit, and this 

160 # also exercises this functionality and makes the tests 

161 # run a lot faster. 

162 solveConfig.minMeanSignal["ALL_AMPS"] = 2000.0 

163 # Set the outlier fit threshold higher than the default appropriate 

164 # for this test dataset. 

165 solveConfig.maxSignalInitialPtcOutlierFit = 90000.0 

166 solveTask = cpPipe.ptc.PhotonTransferCurveSolveTask(config=solveConfig) 

167 

168 inputGain = self.gain 

169 

170 muStandard, varStandard = {}, {} 

171 expDict = {} 

172 expIds = [] 

173 idCounter = 0 

174 for expTime in self.timeVec: 

175 mockExp1, mockExp2 = makeMockFlats( 

176 expTime, 

177 gain=inputGain, 

178 readNoiseElectrons=3, 

179 expId1=idCounter, 

180 expId2=idCounter + 1, 

181 ) 

182 mockExpRef1 = PretendRef(mockExp1) 

183 mockExpRef2 = PretendRef(mockExp2) 

184 expDict[expTime] = ((mockExpRef1, idCounter), (mockExpRef2, idCounter + 1)) 

185 expIds.append(idCounter) 

186 expIds.append(idCounter + 1) 

187 for ampNumber, ampName in enumerate(self.ampNames): 

188 # cov has (i, j, var, cov, npix) 

189 ( 

190 im1Area, 

191 im2Area, 

192 imStatsCtrl, 

193 mu1, 

194 mu2, 

195 ) = extractTask.getImageAreasMasksStats(mockExp1, mockExp2) 

196 muDiff, varDiff, covAstier = extractTask.measureMeanVarCov( 

197 im1Area, im2Area, imStatsCtrl, mu1, mu2 

198 ) 

199 muStandard.setdefault(ampName, []).append(muDiff) 

200 varStandard.setdefault(ampName, []).append(varDiff) 

201 idCounter += 2 

202 

203 resultsExtract = extractTask.run( 

204 inputExp=expDict, 

205 inputDims=expIds, 

206 taskMetadata=[self.metadataContents for x in expIds], 

207 ) 

208 

209 # Force the last PTC dataset to have a NaN, and ensure that the 

210 # task runs (DM-38029). This is a minor perturbation and does not 

211 # affect the output comparison. Note that we use index -2 because 

212 # these datasets are in pairs of [real, dummy] to match the inputs 

213 # to the extract task. 

214 resultsExtract.outputCovariances[-2].rawMeans["C:0,0"] = np.array([np.nan]) 

215 resultsExtract.outputCovariances[-2].rawVars["C:0,0"] = np.array([np.nan]) 

216 

217 # Force the next-to-last PTC dataset to have a decreased variance to 

218 # ensure that the outlier fit rejection works. Note that we use 

219 # index -4 because these datasets are in pairs of [real, dummy] to 

220 # match the inputs to the extract task. 

221 rawVar = resultsExtract.outputCovariances[-4].rawVars["C:0,0"] 

222 resultsExtract.outputCovariances[-4].rawVars["C:0,0"] = rawVar * 0.9 

223 

224 # Reorganize the outputCovariances so we can confirm they come 

225 # out sorted afterwards. 

226 outputCovariancesRev = resultsExtract.outputCovariances[::-1] 

227 

228 resultsSolve = solveTask.run( 

229 outputCovariancesRev, camera=FakeCamera([self.flatExp1.getDetector()]) 

230 ) 

231 

232 ptc = resultsSolve.outputPtcDataset 

233 

234 # Some expected values for noise matrix, just to check that 

235 # it was calculated. 

236 noiseMatrixNoBExpected = { 

237 (0, 0): 6.53126505, 

238 (1, 1): -23.20924747, 

239 (2, 2): 35.69834113, 

240 } 

241 noiseMatrixExpected = { 

242 (0, 0): 29.37146918, 

243 (1, 1): -14.6849025, 

244 (2, 2): 24.7328517, 

245 } 

246 

247 noiseMatrixExpected = np.array( 

248 [ 

249 [ 

250 29.37146918, 

251 9.2760363, 

252 -29.08907932, 

253 33.65818827, 

254 -52.65710984, 

255 -18.5821773, 

256 -46.26896286, 

257 65.01049736, 

258 ], 

259 [ 

260 -3.62427987, 

261 -14.6849025, 

262 -46.55230305, 

263 -1.30410627, 

264 6.44903599, 

265 18.11796075, 

266 -22.72874074, 

267 20.90219857, 

268 ], 

269 [ 

270 5.09203058, 

271 -4.40097862, 

272 24.7328517, 

273 39.2847586, 

274 -21.46132351, 

275 8.12179783, 

276 6.23585617, 

277 -2.09949622, 

278 ], 

279 [ 

280 35.79204016, 

281 -6.50205005, 

282 3.37910363, 

283 15.22335662, 

284 -19.29035067, 

285 9.66065941, 

286 7.47510934, 

287 20.25962845, 

288 ], 

289 [ 

290 -36.23187633, 

291 -22.72307472, 

292 16.29140749, 

293 -13.09493835, 

294 3.32091085, 

295 52.4380977, 

296 -8.06428902, 

297 -22.66669839, 

298 ], 

299 [ 

300 -27.93122896, 

301 15.37016686, 

302 9.18835073, 

303 -24.48892946, 

304 8.14480304, 

305 22.38983222, 

306 22.36866891, 

307 -0.38803439, 

308 ], 

309 [ 

310 17.13962665, 

311 -28.33153763, 

312 -17.79744334, 

313 -18.57064463, 

314 7.69408833, 

315 8.48265396, 

316 18.0447022, 

317 -16.97496022, 

318 ], 

319 [ 

320 10.09078383, 

321 -26.61613002, 

322 10.48504889, 

323 15.33196998, 

324 -23.35165517, 

325 -24.53098643, 

326 -18.21201067, 

327 17.40755051, 

328 ], 

329 ] 

330 ) 

331 

332 noiseMatrixNoBExpected = np.array( 

333 [ 

334 [ 

335 6.53126505, 

336 12.14827594, 

337 -37.11919923, 

338 41.18675353, 

339 -85.1613845, 

340 -28.45801954, 

341 -61.24442999, 

342 88.76480122, 

343 ], 

344 [ 

345 -4.64541165, 

346 -23.20924747, 

347 -66.08733987, 

348 -0.87558055, 

349 12.20111853, 

350 24.84795549, 

351 -34.92458788, 

352 24.42745014, 

353 ], 

354 [ 

355 7.66734507, 

356 -4.51403645, 

357 35.69834113, 

358 52.73693356, 

359 -30.85044089, 

360 10.86761771, 

361 10.8503068, 

362 -2.18908327, 

363 ], 

364 [ 

365 50.9901156, 

366 -7.34803977, 

367 5.33443765, 

368 21.60899396, 

369 -25.06129827, 

370 15.14015505, 

371 10.94263771, 

372 29.23975515, 

373 ], 

374 [ 

375 -48.66912069, 

376 -31.58003774, 

377 21.81305735, 

378 -13.08993444, 

379 8.17275394, 

380 74.85293723, 

381 -11.18403252, 

382 -31.7799437, 

383 ], 

384 [ 

385 -38.55206382, 

386 22.92982676, 

387 13.39861008, 

388 -33.3307362, 

389 8.65362238, 

390 29.18775548, 

391 31.78433947, 

392 1.27923706, 

393 ], 

394 [ 

395 23.33663918, 

396 -41.74105625, 

397 -26.55920751, 

398 -24.71611677, 

399 12.13343146, 

400 11.25763907, 

401 21.79131019, 

402 -26.579393, 

403 ], 

404 [ 

405 11.44334226, 

406 -34.9759641, 

407 13.96449509, 

408 19.64121933, 

409 -36.09794843, 

410 -34.27205933, 

411 -25.16574105, 

412 23.80460972, 

413 ], 

414 ] 

415 ) 

416 

417 for amp in self.ampNames: 

418 self.assertAlmostEqual(ptc.gain[amp], inputGain, places=2) 

419 for v1, v2 in zip(varStandard[amp], ptc.finalVars[amp]): 

420 self.assertAlmostEqual(v1 / v2, 1.0, places=1) 

421 

422 # Check that the PTC turnoff is correctly computed. 

423 # This will be different for the C:0,0 amp. 

424 if amp == "C:0,0": 

425 self.assertAlmostEqual(ptc.ptcTurnoff[amp], ptc.rawMeans[ampName][-3]) 

426 else: 

427 self.assertAlmostEqual(ptc.ptcTurnoff[amp], ptc.rawMeans[ampName][-1]) 

428 

429 # Test that all the quantities are correctly ordered and have 

430 # not accidentally been masked. We check every other output ([::2]) 

431 # because these datasets are in pairs of [real, dummy] to 

432 # match the inputs to the extract task. 

433 for i, extractPtc in enumerate(resultsExtract.outputCovariances[::2]): 

434 self.assertFloatsAlmostEqual( 

435 extractPtc.rawExpTimes[ampName][0], 

436 ptc.rawExpTimes[ampName][i], 

437 ) 

438 self.assertFloatsAlmostEqual( 

439 extractPtc.rawMeans[ampName][0], 

440 ptc.rawMeans[ampName][i], 

441 ) 

442 self.assertFloatsAlmostEqual( 

443 extractPtc.rawVars[ampName][0], 

444 ptc.rawVars[ampName][i], 

445 ) 

446 self.assertFloatsAlmostEqual( 

447 extractPtc.histVars[ampName][0], 

448 ptc.histVars[ampName][i], 

449 ) 

450 self.assertFloatsAlmostEqual( 

451 extractPtc.histChi2Dofs[ampName][0], 

452 ptc.histChi2Dofs[ampName][i], 

453 ) 

454 self.assertFloatsAlmostEqual( 

455 extractPtc.kspValues[ampName][0], 

456 ptc.kspValues[ampName][i], 

457 ) 

458 self.assertFloatsAlmostEqual( 

459 extractPtc.covariances[ampName][0], 

460 ptc.covariances[ampName][i], 

461 ) 

462 self.assertFloatsAlmostEqual( 

463 extractPtc.covariancesSqrtWeights[ampName][0], 

464 ptc.covariancesSqrtWeights[ampName][i], 

465 ) 

466 self.assertFloatsAlmostEqual( 

467 ptc.noiseMatrix[ampName], noiseMatrixExpected, atol=1e-8, rtol=None 

468 ) 

469 self.assertFloatsAlmostEqual( 

470 ptc.noiseMatrixNoB[ampName], 

471 noiseMatrixNoBExpected, 

472 atol=1e-8, 

473 rtol=None, 

474 ) 

475 

476 mask = ptc.getGoodPoints(amp) 

477 

478 values = ( 

479 ptc.covariancesModel[amp][mask, 0, 0] - ptc.covariances[amp][mask, 0, 0] 

480 ) / ptc.covariancesModel[amp][mask, 0, 0] 

481 np.testing.assert_array_less(np.abs(values), 2e-3) 

482 

483 values = ( 

484 ptc.covariancesModel[amp][mask, 1, 1] - ptc.covariances[amp][mask, 1, 1] 

485 ) / ptc.covariancesModel[amp][mask, 1, 1] 

486 np.testing.assert_array_less(np.abs(values), 0.2) 

487 

488 values = ( 

489 ptc.covariancesModel[amp][mask, 1, 2] - ptc.covariances[amp][mask, 1, 2] 

490 ) / ptc.covariancesModel[amp][mask, 1, 2] 

491 np.testing.assert_array_less(np.abs(values), 0.2) 

492 

493 expIdsUsed = ptc.getExpIdsUsed("C:0,0") 

494 # Check that these are the same as the inputs, paired up, with the 

495 # first two (low flux) and final four (outliers, nans) removed. 

496 self.assertTrue( 

497 np.all(expIdsUsed == np.array(expIds).reshape(len(expIds) // 2, 2)[1:-2]) 

498 ) 

499 

500 goodAmps = ptc.getGoodAmps() 

501 self.assertEqual(goodAmps, self.ampNames) 

502 

503 # Check that every possibly modified field has the same length. 

504 covShape = None 

505 covSqrtShape = None 

506 covModelShape = None 

507 covModelNoBShape = None 

508 

509 for ampName in self.ampNames: 

510 if covShape is None: 

511 covShape = ptc.covariances[ampName].shape 

512 covSqrtShape = ptc.covariancesSqrtWeights[ampName].shape 

513 covModelShape = ptc.covariancesModel[ampName].shape 

514 covModelNoBShape = ptc.covariancesModelNoB[ampName].shape 

515 else: 

516 self.assertEqual(ptc.covariances[ampName].shape, covShape) 

517 self.assertEqual( 

518 ptc.covariancesSqrtWeights[ampName].shape, covSqrtShape 

519 ) 

520 self.assertEqual(ptc.covariancesModel[ampName].shape, covModelShape) 

521 self.assertEqual( 

522 ptc.covariancesModelNoB[ampName].shape, covModelNoBShape 

523 ) 

524 

525 # And check that this is serializable 

526 with tempfile.NamedTemporaryFile(suffix=".fits") as f: 

527 usedFilename = ptc.writeFits(f.name) 

528 fromFits = PhotonTransferCurveDataset.readFits(usedFilename) 

529 self.assertEqual(fromFits, ptc) 

530 

531 def ptcFitAndCheckPtc( 

532 self, 

533 order=None, 

534 fitType=None, 

535 doTableArray=False, 

536 doFitBootstrap=False, 

537 doLegacy=False, 

538 ): 

539 localDataset = copy.deepcopy(self.dataset) 

540 localDataset.ptcFitType = fitType 

541 configSolve = copy.copy(self.defaultConfigSolve) 

542 configLin = cpPipe.linearity.LinearitySolveTask.ConfigClass() 

543 placesTests = 6 

544 if doFitBootstrap: 

545 configSolve.doFitBootstrap = True 

546 # Bootstrap method in cp_pipe/utils.py does multiple fits 

547 # in the precense of noise. Allow for more margin of 

548 # error. 

549 placesTests = 3 

550 

551 configSolve.doLegacyTurnoffSelection = doLegacy 

552 

553 if fitType == "POLYNOMIAL": 

554 if order not in [2, 3]: 

555 RuntimeError("Enter a valid polynomial order for this test: 2 or 3") 

556 if order == 2: 

557 for ampName in self.ampNames: 

558 localDataset.rawVars[ampName] = [ 

559 self.noiseSq + self.c1 * mu + self.c2 * mu**2 

560 for mu in localDataset.rawMeans[ampName] 

561 ] 

562 configSolve.polynomialFitDegree = 2 

563 if order == 3: 

564 for ampName in self.ampNames: 

565 localDataset.rawVars[ampName] = [ 

566 self.noiseSq 

567 + self.c1 * mu 

568 + self.c2 * mu**2 

569 + self.c3 * mu**3 

570 for mu in localDataset.rawMeans[ampName] 

571 ] 

572 configSolve.polynomialFitDegree = 3 

573 elif fitType == "EXPAPPROXIMATION": 

574 g = self.gain 

575 for ampName in self.ampNames: 

576 localDataset.rawVars[ampName] = [ 

577 ( 

578 0.5 / (self.a00 * g**2) * (np.exp(2 * self.a00 * mu * g) - 1) 

579 + self.noiseSq / (g * g) 

580 ) 

581 for mu in localDataset.rawMeans[ampName] 

582 ] 

583 else: 

584 raise RuntimeError( 

585 "Enter a fit function type: 'POLYNOMIAL' or 'EXPAPPROXIMATION'" 

586 ) 

587 

588 # Initialize mask and covariance weights that will be used in fits. 

589 # Covariance weights values empirically determined from one of 

590 # the cases in test_covAstier. 

591 matrixSize = localDataset.covMatrixSide 

592 maskLength = len(localDataset.rawMeans[ampName]) 

593 for ampName in self.ampNames: 

594 localDataset.expIdMask[ampName] = np.repeat(True, maskLength) 

595 localDataset.covariancesSqrtWeights[ampName] = np.repeat( 

596 np.ones((matrixSize, matrixSize)), maskLength 

597 ).reshape((maskLength, matrixSize, matrixSize)) 

598 localDataset.covariancesSqrtWeights[ampName][:, 0, 0] = [ 

599 0.07980188, 

600 0.01339653, 

601 0.0073118, 

602 0.00502802, 

603 0.00383132, 

604 0.00309475, 

605 0.00259572, 

606 0.00223528, 

607 0.00196273, 

608 0.00174943, 

609 0.00157794, 

610 0.00143707, 

611 0.00131929, 

612 0.00121935, 

613 0.0011334, 

614 0.00105893, 

615 0.00099357, 

616 0.0009358, 

617 0.00088439, 

618 0.00083833, 

619 ] 

620 

621 configLin.maxLookupTableAdu = 200000 # Max ADU in input mock flats 

622 configLin.maxLinearAdu = 100000 

623 configLin.minLinearAdu = 50000 

624 if doTableArray: 

625 configLin.linearityType = "LookupTable" 

626 else: 

627 configLin.linearityType = "Polynomial" 

628 solveTask = cpPipe.ptc.PhotonTransferCurveSolveTask(config=configSolve) 

629 linearityTask = cpPipe.linearity.LinearitySolveTask(config=configLin) 

630 

631 if doTableArray: 

632 # Non-linearity 

633 numberAmps = len(self.ampNames) 

634 # localDataset: PTC dataset 

635 # (`lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset`) 

636 localDataset = solveTask.fitMeasurementsToModel(localDataset) 

637 # linDataset here is a lsst.pipe.base.Struct 

638 linDataset = linearityTask.run( 

639 localDataset, 

640 dummy=[1.0], 

641 camera=FakeCamera([self.flatExp1.getDetector()]), 

642 inputPhotodiodeData={}, 

643 inputDims={"detector": 0}, 

644 ) 

645 linDataset = linDataset.outputLinearizer 

646 else: 

647 localDataset = solveTask.fitMeasurementsToModel(localDataset) 

648 linDataset = linearityTask.run( 

649 localDataset, 

650 dummy=[1.0], 

651 camera=FakeCamera([self.flatExp1.getDetector()]), 

652 inputPhotodiodeData={}, 

653 inputDims={"detector": 0}, 

654 ) 

655 linDataset = linDataset.outputLinearizer 

656 if doTableArray: 

657 # check that the linearizer table has been filled out properly 

658 for i in np.arange(numberAmps): 

659 tMax = (configLin.maxLookupTableAdu) / self.flux 

660 timeRange = np.linspace(0.0, tMax, configLin.maxLookupTableAdu) 

661 signalIdeal = timeRange * self.flux 

662 signalUncorrected = funcPolynomial( 

663 np.array([0.0, self.flux, self.k2NonLinearity]), timeRange 

664 ) 

665 linearizerTableRow = signalIdeal - signalUncorrected 

666 self.assertEqual( 

667 len(linearizerTableRow), len(linDataset.tableData[i, :]) 

668 ) 

669 for j in np.arange(len(linearizerTableRow)): 

670 self.assertAlmostEqual( 

671 linearizerTableRow[j], 

672 linDataset.tableData[i, :][j], 

673 places=placesTests, 

674 ) 

675 else: 

676 # check entries in localDataset, which was modified by the function 

677 for ampName in self.ampNames: 

678 maskAmp = localDataset.expIdMask[ampName] 

679 finalMuVec = localDataset.rawMeans[ampName][maskAmp] 

680 finalTimeVec = localDataset.rawExpTimes[ampName][maskAmp] 

681 linearPart = self.flux * finalTimeVec 

682 inputFracNonLinearityResiduals = ( 

683 100 * (linearPart - finalMuVec) / linearPart 

684 ) 

685 self.assertEqual(fitType, localDataset.ptcFitType) 

686 self.assertAlmostEqual(self.gain, localDataset.gain[ampName]) 

687 if fitType == "POLYNOMIAL": 

688 self.assertAlmostEqual(self.c1, localDataset.ptcFitPars[ampName][1]) 

689 self.assertAlmostEqual( 

690 np.sqrt(self.noiseSq) * self.gain, localDataset.noise[ampName] 

691 ) 

692 if fitType == "EXPAPPROXIMATION": 

693 self.assertAlmostEqual( 

694 self.a00, localDataset.ptcFitPars[ampName][0] 

695 ) 

696 # noise already in electrons for 'EXPAPPROXIMATION' fit 

697 self.assertAlmostEqual( 

698 np.sqrt(self.noiseSq), localDataset.noise[ampName] 

699 ) 

700 

701 # check entries in returned dataset (a dict of , for nonlinearity) 

702 for ampName in self.ampNames: 

703 maskAmp = localDataset.expIdMask[ampName] 

704 finalMuVec = localDataset.rawMeans[ampName][maskAmp] 

705 finalTimeVec = localDataset.rawExpTimes[ampName][maskAmp] 

706 linearPart = self.flux * finalTimeVec 

707 inputFracNonLinearityResiduals = ( 

708 100 * (linearPart - finalMuVec) / linearPart 

709 ) 

710 

711 # Nonlinearity fit parameters 

712 # Polynomial fits are now normalized to unit flux scaling 

713 self.assertAlmostEqual(0.0, linDataset.fitParams[ampName][0], places=1) 

714 self.assertAlmostEqual(1.0, linDataset.fitParams[ampName][1], places=5) 

715 

716 # Non-linearity coefficient for linearizer 

717 squaredCoeff = self.k2NonLinearity / (self.flux**2) 

718 self.assertAlmostEqual( 

719 squaredCoeff, linDataset.fitParams[ampName][2], places=placesTests 

720 ) 

721 self.assertAlmostEqual( 

722 -squaredCoeff, 

723 linDataset.linearityCoeffs[ampName][2], 

724 places=placesTests, 

725 ) 

726 

727 linearPartModel = ( 

728 linDataset.fitParams[ampName][1] * finalTimeVec * self.flux 

729 ) 

730 outputFracNonLinearityResiduals = ( 

731 100 * (linearPartModel - finalMuVec) / linearPartModel 

732 ) 

733 # Fractional nonlinearity residuals 

734 self.assertEqual( 

735 len(outputFracNonLinearityResiduals), 

736 len(inputFracNonLinearityResiduals), 

737 ) 

738 for calc, truth in zip( 

739 outputFracNonLinearityResiduals, inputFracNonLinearityResiduals 

740 ): 

741 self.assertAlmostEqual(calc, truth, places=3) 

742 

743 def test_ptcFit(self): 

744 for createArray in [True, False]: 

745 for doLegacy in [False, True]: 

746 for fitType, order in [ 

747 ("POLYNOMIAL", 2), 

748 ("POLYNOMIAL", 3), 

749 ("EXPAPPROXIMATION", None), 

750 ]: 

751 self.ptcFitAndCheckPtc( 

752 fitType=fitType, 

753 order=order, 

754 doTableArray=createArray, 

755 doLegacy=doLegacy, 

756 ) 

757 

758 def test_meanVarMeasurement(self): 

759 task = self.defaultTaskExtract 

760 im1Area, im2Area, imStatsCtrl, mu1, mu2 = task.getImageAreasMasksStats( 

761 self.flatExp1, self.flatExp2 

762 ) 

763 mu, varDiff, _ = task.measureMeanVarCov(im1Area, im2Area, imStatsCtrl, mu1, mu2) 

764 

765 self.assertLess(self.flatWidth - np.sqrt(varDiff), 1) 

766 self.assertLess(self.flatMean - mu, 1) 

767 

768 def test_meanVarMeasurementWithNans(self): 

769 task = self.defaultTaskExtract 

770 

771 flatExp1 = self.flatExp1.clone() 

772 flatExp2 = self.flatExp2.clone() 

773 

774 flatExp1.image.array[20:30, :] = np.nan 

775 flatExp2.image.array[20:30, :] = np.nan 

776 

777 im1Area, im2Area, imStatsCtrl, mu1, mu2 = task.getImageAreasMasksStats( 

778 flatExp1, flatExp2 

779 ) 

780 mu, varDiff, _ = task.measureMeanVarCov(im1Area, im2Area, imStatsCtrl, mu1, mu2) 

781 

782 expectedMu1 = np.nanmean(flatExp1.image.array) 

783 expectedMu2 = np.nanmean(flatExp2.image.array) 

784 expectedMu = 0.5 * (expectedMu1 + expectedMu2) 

785 

786 # Now the variance of the difference. First, create the diff image. 

787 im1 = flatExp1.maskedImage 

788 im2 = flatExp2.maskedImage 

789 

790 temp = im2.clone() 

791 temp *= expectedMu1 

792 diffIm = im1.clone() 

793 diffIm *= expectedMu2 

794 diffIm -= temp 

795 diffIm /= expectedMu 

796 

797 # Divide by two as it is what measureMeanVarCov returns 

798 # (variance of difference) 

799 expectedVar = 0.5 * np.nanvar(diffIm.image.array) 

800 

801 # Check that the standard deviations and the emans agree to 

802 # less than 1 ADU 

803 self.assertLess(np.sqrt(expectedVar) - np.sqrt(varDiff), 1) 

804 self.assertLess(expectedMu - mu, 1) 

805 

806 def test_meanVarMeasurementAllNan(self): 

807 task = self.defaultTaskExtract 

808 flatExp1 = self.flatExp1.clone() 

809 flatExp2 = self.flatExp2.clone() 

810 

811 flatExp1.image.array[:, :] = np.nan 

812 flatExp2.image.array[:, :] = np.nan 

813 

814 im1Area, im2Area, imStatsCtrl, mu1, mu2 = task.getImageAreasMasksStats( 

815 flatExp1, flatExp2 

816 ) 

817 mu, varDiff, covDiff = task.measureMeanVarCov( 

818 im1Area, im2Area, imStatsCtrl, mu1, mu2 

819 ) 

820 

821 self.assertTrue(np.isnan(mu)) 

822 self.assertTrue(np.isnan(varDiff)) 

823 self.assertTrue(covDiff is None) 

824 

825 def test_meanVarMeasurementTooFewPixels(self): 

826 task = self.defaultTaskExtract 

827 flatExp1 = self.flatExp1.clone() 

828 flatExp2 = self.flatExp2.clone() 

829 

830 flatExp1.image.array[0:190, :] = np.nan 

831 flatExp2.image.array[0:190, :] = np.nan 

832 

833 bit = flatExp1.mask.getMaskPlaneDict()["NO_DATA"] 

834 flatExp1.mask.array[0:190, :] &= 2**bit 

835 flatExp2.mask.array[0:190, :] &= 2**bit 

836 

837 im1Area, im2Area, imStatsCtrl, mu1, mu2 = task.getImageAreasMasksStats( 

838 flatExp1, flatExp2 

839 ) 

840 with self.assertLogs(level=logging.WARNING) as cm: 

841 mu, varDiff, covDiff = task.measureMeanVarCov( 

842 im1Area, im2Area, imStatsCtrl, mu1, mu2 

843 ) 

844 self.assertIn("Number of good points", cm.output[0]) 

845 

846 self.assertTrue(np.isnan(mu)) 

847 self.assertTrue(np.isnan(varDiff)) 

848 self.assertTrue(covDiff is None) 

849 

850 def test_meanVarMeasurementTooNarrowStrip(self): 

851 # We need a new config to make sure the second covariance cut is 

852 # triggered. 

853 config = cpPipe.ptc.PhotonTransferCurveExtractTask.ConfigClass() 

854 config.minNumberGoodPixelsForCovariance = 10 

855 task = cpPipe.ptc.PhotonTransferCurveExtractTask(config=config) 

856 flatExp1 = self.flatExp1.clone() 

857 flatExp2 = self.flatExp2.clone() 

858 

859 flatExp1.image.array[0:195, :] = np.nan 

860 flatExp2.image.array[0:195, :] = np.nan 

861 flatExp1.image.array[:, 0:195] = np.nan 

862 flatExp2.image.array[:, 0:195] = np.nan 

863 

864 bit = flatExp1.mask.getMaskPlaneDict()["NO_DATA"] 

865 flatExp1.mask.array[0:195, :] &= 2**bit 

866 flatExp2.mask.array[0:195, :] &= 2**bit 

867 flatExp1.mask.array[:, 0:195] &= 2**bit 

868 flatExp2.mask.array[:, 0:195] &= 2**bit 

869 

870 im1Area, im2Area, imStatsCtrl, mu1, mu2 = task.getImageAreasMasksStats( 

871 flatExp1, flatExp2 

872 ) 

873 with self.assertLogs(level=logging.WARNING) as cm: 

874 mu, varDiff, covDiff = task.measureMeanVarCov( 

875 im1Area, im2Area, imStatsCtrl, mu1, mu2 

876 ) 

877 self.assertIn("Not enough pixels", cm.output[0]) 

878 

879 self.assertTrue(np.isnan(mu)) 

880 self.assertTrue(np.isnan(varDiff)) 

881 self.assertTrue(covDiff is None) 

882 

883 def test_makeZeroSafe(self): 

884 noZerosArray = [1.0, 20, -35, 45578.98, 90.0, 897, 659.8] 

885 someZerosArray = [1.0, 20, 0, 0, 90, 879, 0] 

886 allZerosArray = [0.0, 0.0, 0, 0, 0.0, 0, 0] 

887 

888 substituteValue = 1e-10 

889 

890 expectedSomeZerosArray = [ 

891 1.0, 

892 20, 

893 substituteValue, 

894 substituteValue, 

895 90, 

896 879, 

897 substituteValue, 

898 ] 

899 expectedAllZerosArray = np.repeat(substituteValue, len(allZerosArray)) 

900 

901 measuredSomeZerosArray = self.defaultTaskSolve._makeZeroSafe( 

902 someZerosArray, substituteValue=substituteValue 

903 ) 

904 measuredAllZerosArray = self.defaultTaskSolve._makeZeroSafe( 

905 allZerosArray, substituteValue=substituteValue 

906 ) 

907 measuredNoZerosArray = self.defaultTaskSolve._makeZeroSafe( 

908 noZerosArray, substituteValue=substituteValue 

909 ) 

910 

911 for exp, meas in zip(expectedSomeZerosArray, measuredSomeZerosArray): 

912 self.assertEqual(exp, meas) 

913 for exp, meas in zip(expectedAllZerosArray, measuredAllZerosArray): 

914 self.assertEqual(exp, meas) 

915 for exp, meas in zip(noZerosArray, measuredNoZerosArray): 

916 self.assertEqual(exp, meas) 

917 

918 def test_getInitialGoodPoints(self): 

919 xs = [1, 2, 3, 4, 5, 6] 

920 ys = [2 * x for x in xs] 

921 points = self.defaultTaskSolve._getInitialGoodPoints( 

922 xs, ys, minVarPivotSearch=0.0, consecutivePointsVarDecreases=2 

923 ) 

924 assert np.all(points) == np.all(np.array([True for x in xs])) 

925 

926 ys[4] = 7 # Variance decreases in two consecutive points after ys[3]=8 

927 ys[5] = 6 

928 points = self.defaultTaskSolve._getInitialGoodPoints( 

929 xs, ys, minVarPivotSearch=0.0, consecutivePointsVarDecreases=2 

930 ) 

931 assert np.all(points) == np.all(np.array([True, True, True, True, False])) 

932 

933 def runGetGainFromFlatPair(self, correctionType="NONE"): 

934 extractConfig = self.defaultConfigExtract 

935 extractConfig.gainCorrectionType = correctionType 

936 extractConfig.minNumberGoodPixelsForCovariance = 5000 

937 extractTask = cpPipe.ptc.PhotonTransferCurveExtractTask(config=extractConfig) 

938 

939 expDict = {} 

940 expIds = [] 

941 idCounter = 0 

942 inputGain = self.gain # 1.5 e/ADU 

943 for expTime in self.timeVec: 

944 # Approximation works better at low flux, e.g., < 10000 ADU 

945 mockExp1, mockExp2 = makeMockFlats( 

946 expTime, 

947 gain=inputGain, 

948 readNoiseElectrons=np.sqrt(self.noiseSq), 

949 fluxElectrons=100, 

950 expId1=idCounter, 

951 expId2=idCounter + 1, 

952 ) 

953 mockExpRef1 = PretendRef(mockExp1) 

954 mockExpRef2 = PretendRef(mockExp2) 

955 expDict[expTime] = ((mockExpRef1, idCounter), (mockExpRef2, idCounter + 1)) 

956 expIds.append(idCounter) 

957 expIds.append(idCounter + 1) 

958 idCounter += 2 

959 

960 resultsExtract = extractTask.run( 

961 inputExp=expDict, 

962 inputDims=expIds, 

963 taskMetadata=[self.metadataContents for x in expIds], 

964 ) 

965 for exposurePair in resultsExtract.outputCovariances: 

966 for ampName in self.ampNames: 

967 if exposurePair.gain[ampName] is np.nan: 

968 continue 

969 self.assertAlmostEqual( 

970 exposurePair.gain[ampName], inputGain, delta=0.04 

971 ) 

972 

973 def test_getGainFromFlatPair(self): 

974 for gainCorrectionType in [ 

975 "NONE", 

976 "SIMPLE", 

977 "FULL", 

978 ]: 

979 self.runGetGainFromFlatPair(gainCorrectionType) 

980 

981 

982class MeasurePhotonTransferCurveDatasetTestCase(lsst.utils.tests.TestCase): 

983 def setUp(self): 

984 self.ptcData = PhotonTransferCurveDataset(["C00", "C01"], " ") 

985 self.ptcData.inputExpIdPairs = { 

986 "C00": [(123, 234), (345, 456), (567, 678)], 

987 "C01": [(123, 234), (345, 456), (567, 678)], 

988 } 

989 

990 def test_generalBehaviour(self): 

991 test = PhotonTransferCurveDataset(["C00", "C01"], " ") 

992 test.inputExpIdPairs = { 

993 "C00": [(123, 234), (345, 456), (567, 678)], 

994 "C01": [(123, 234), (345, 456), (567, 678)], 

995 } 

996 

997 

998class TestMemory(lsst.utils.tests.MemoryTestCase): 

999 pass 

1000 

1001 

1002def setup_module(module): 

1003 lsst.utils.tests.init() 

1004 

1005 

1006if __name__ == "__main__": 1006 ↛ 1007line 1006 didn't jump to line 1007, because the condition on line 1006 was never true

1007 lsst.utils.tests.init() 

1008 unittest.main()