Coverage for python/lsst/ip/isr/ptcDataset.py: 6%

363 statements  

« prev     ^ index     » next       coverage.py v7.4.1, created at 2024-01-27 10:05 +0000

1# 

2# LSST Data Management System 

3# Copyright 2008-2017 AURA/LSST. 

4# 

5# This product includes software developed by the 

6# LSST Project (http://www.lsst.org/). 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <https://www.lsstcorp.org/LegalNotices/>. 

21# 

22""" 

23Define dataset class for MeasurePhotonTransferCurve task 

24""" 

25 

26__all__ = ['PhotonTransferCurveDataset'] 

27 

28import numpy as np 

29import math 

30from astropy.table import Table 

31 

32from lsst.ip.isr import IsrCalib 

33 

34 

35class PhotonTransferCurveDataset(IsrCalib): 

36 """A simple class to hold the output data from the PTC task. 

37 

38 The dataset is made up of a dictionary for each item, keyed by the 

39 amplifiers' names, which much be supplied at construction time. 

40 New items cannot be added to the class to save accidentally saving to the 

41 wrong property, and the class can be frozen if desired. 

42 inputExpIdPairs records the exposures used to produce the data. 

43 When fitPtc() or fitCovariancesAstier() is run, a mask is built up, which 

44 is by definition always the same length as inputExpIdPairs, rawExpTimes, 

45 rawMeans and rawVars, and is a list of bools, which are incrementally set 

46 to False as points are discarded from the fits. 

47 PTC fit parameters for polynomials are stored in a list in ascending order 

48 of polynomial term, i.e. par[0]*x^0 + par[1]*x + par[2]*x^2 etc 

49 with the length of the list corresponding to the order of the polynomial 

50 plus one. 

51 

52 Parameters 

53 ---------- 

54 ampNames : `list` 

55 List with the names of the amplifiers of the detector at hand. 

56 ptcFitType : `str`, optional 

57 Type of model fitted to the PTC: "POLYNOMIAL", "EXPAPPROXIMATION", 

58 or "FULLCOVARIANCE". 

59 covMatrixSide : `int`, optional 

60 Maximum lag of measured covariances (size of square covariance 

61 matrices). 

62 covMatrixSideFullCovFit : `int, optional 

63 Maximum covariances lag for FULLCOVARIANCE fit. It should be less or 

64 equal than covMatrixSide. 

65 kwargs : `dict`, optional 

66 Other keyword arguments to pass to the parent init. 

67 

68 Notes 

69 ----- 

70 The stored attributes are: 

71 

72 badAmps : `list` [`str`] 

73 List with bad amplifiers names. 

74 inputExpIdPairs : `dict`, [`str`, `list`] 

75 Dictionary keyed by amp names containing the input exposures IDs. 

76 expIdMask : `dict`, [`str`, `np.ndarray`] 

77 Dictionary keyed by amp names containing the mask produced after 

78 outlier rejection. The mask produced by the "FULLCOVARIANCE" 

79 option may differ from the one produced in the other two PTC 

80 fit types. 

81 rawExpTimes : `dict`, [`str`, `np.ndarray`] 

82 Dictionary keyed by amp names containing the unmasked exposure times. 

83 rawMeans : `dict`, [`str`, `np.ndarray`] 

84 Dictionary keyed by amp names containing the unmasked average of the 

85 means of the exposures in each flat pair. 

86 rawVars : `dict`, [`str`, `np.ndarray`] 

87 Dictionary keyed by amp names containing the variance of the 

88 difference image of the exposures in each flat pair. 

89 rowMeanVariance : `dict`, [`str`, `np.ndarray`] 

90 Dictionary keyed by amp names containing the variance of the 

91 means of the rows of the difference image of the exposures 

92 in each flat pair. 

93 histVars : `dict`, [`str`, `np.ndarray`] 

94 Dictionary keyed by amp names containing the variance of the 

95 difference image of the exposures in each flat pair estimated 

96 by fitting a Gaussian model. 

97 histChi2Dofs : `dict`, [`str`, `np.ndarray`] 

98 Dictionary keyed by amp names containing the chi-squared per degree 

99 of freedom fitting the difference image to a Gaussian model. 

100 kspValues : `dict`, [`str`, `np.ndarray`] 

101 Dictionary keyed by amp names containing the KS test p-value from 

102 fitting the difference image to a Gaussian model. 

103 gain : `dict`, [`str`, `float`] 

104 Dictionary keyed by amp names containing the fitted gains. 

105 gainErr : `dict`, [`str`, `float`] 

106 Dictionary keyed by amp names containing the errors on the 

107 fitted gains. 

108 noise : `dict`, [`str`, `float`] 

109 Dictionary keyed by amp names containing the fitted noise. 

110 noiseErr : `dict`, [`str`, `float`] 

111 Dictionary keyed by amp names containing the errors on the fitted 

112 noise. 

113 ptcFitPars : `dict`, [`str`, `np.ndarray`] 

114 Dictionary keyed by amp names containing the fitted parameters of the 

115 PTC model for ptcFitTye in ["POLYNOMIAL", "EXPAPPROXIMATION"]. 

116 ptcFitParsError : `dict`, [`str`, `np.ndarray`] 

117 Dictionary keyed by amp names containing the errors on the fitted 

118 parameters of the PTC model for ptcFitTye in 

119 ["POLYNOMIAL", "EXPAPPROXIMATION"]. 

120 ptcFitChiSq : `dict`, [`str`, `float`] 

121 Dictionary keyed by amp names containing the reduced chi squared 

122 of the fit for ptcFitTye in ["POLYNOMIAL", "EXPAPPROXIMATION"]. 

123 ptcTurnoff : `dict` [`str, `float`] 

124 Flux value (in ADU) where the variance of the PTC curve starts 

125 decreasing consistently. 

126 covariances : `dict`, [`str`, `np.ndarray`] 

127 Dictionary keyed by amp names containing a list of measured 

128 covariances per mean flux. 

129 covariancesModel : `dict`, [`str`, `np.ndarray`] 

130 Dictionary keyed by amp names containinging covariances model 

131 (Eq. 20 of Astier+19) per mean flux. 

132 covariancesSqrtWeights : `dict`, [`str`, `np.ndarray`] 

133 Dictionary keyed by amp names containinging sqrt. of covariances 

134 weights. 

135 aMatrix : `dict`, [`str`, `np.ndarray`] 

136 Dictionary keyed by amp names containing the "a" parameters from 

137 the model in Eq. 20 of Astier+19. 

138 bMatrix : `dict`, [`str`, `np.ndarray`] 

139 Dictionary keyed by amp names containing the "b" parameters from 

140 the model in Eq. 20 of Astier+19. 

141 noiseMatrix : `dict`, [`str`, `np.ndarray`] 

142 Dictionary keyed by amp names containing the "noise" parameters from 

143 the model in Eq. 20 of Astier+19. 

144 covariancesModelNoB : `dict`, [`str`, `np.ndarray`] 

145 Dictionary keyed by amp names containing covariances model 

146 (with 'b'=0 in Eq. 20 of Astier+19) 

147 per mean flux. 

148 aMatrixNoB : `dict`, [`str`, `np.ndarray`] 

149 Dictionary keyed by amp names containing the "a" parameters from the 

150 model in Eq. 20 of Astier+19 

151 (and 'b' = 0). 

152 noiseMatrixNoB : `dict`, [`str`, `np.ndarray`] 

153 Dictionary keyed by amp names containing the "noise" parameters from 

154 the model in Eq. 20 of Astier+19, with 'b' = 0. 

155 finalVars : `dict`, [`str`, `np.ndarray`] 

156 Dictionary keyed by amp names containing the masked variance of the 

157 difference image of each flat 

158 pair. If needed, each array will be right-padded with 

159 np.nan to match the length of rawExpTimes. 

160 finalModelVars : `dict`, [`str`, `np.ndarray`] 

161 Dictionary keyed by amp names containing the masked modeled 

162 variance of the difference image of each flat pair. If needed, each 

163 array will be right-padded with np.nan to match the length of 

164 rawExpTimes. 

165 finalMeans : `dict`, [`str`, `np.ndarray`] 

166 Dictionary keyed by amp names containing the masked average of the 

167 means of the exposures in each flat pair. If needed, each array 

168 will be right-padded with np.nan to match the length of 

169 rawExpTimes. 

170 photoCharges : `dict`, [`str`, `np.ndarray`] 

171 Dictionary keyed by amp names containing the integrated photocharge 

172 for linearity calibration. 

173 auxValues : `dict`, [`str`, `np.ndarray`] 

174 Dictionary of per-detector auxiliary header values that can be used 

175 for PTC, linearity computation. 

176 

177 Version 1.1 adds the `ptcTurnoff` attribute. 

178 Version 1.2 adds the `histVars`, `histChi2Dofs`, and `kspValues` 

179 attributes. 

180 Version 1.3 adds the `noiseMatrix` and `noiseMatrixNoB` attributes. 

181 Version 1.4 adds the `auxValues` attribute. 

182 Version 1.5 adds the `covMatrixSideFullCovFit` attribute. 

183 Version 1.6 adds the `rowMeanVariance` attribute. 

184 """ 

185 

186 _OBSTYPE = 'PTC' 

187 _SCHEMA = 'Gen3 Photon Transfer Curve' 

188 _VERSION = 1.6 

189 

190 def __init__(self, ampNames=[], ptcFitType=None, covMatrixSide=1, 

191 covMatrixSideFullCovFit=None, **kwargs): 

192 self.ptcFitType = ptcFitType 

193 self.ampNames = ampNames 

194 self.covMatrixSide = covMatrixSide 

195 if covMatrixSideFullCovFit is None: 

196 self.covMatrixSideFullCovFit = covMatrixSide 

197 else: 

198 self.covMatrixSideFullCovFit = covMatrixSideFullCovFit 

199 

200 self.badAmps = [] 

201 

202 self.inputExpIdPairs = {ampName: [] for ampName in ampNames} 

203 self.expIdMask = {ampName: np.array([], dtype=bool) for ampName in ampNames} 

204 self.rawExpTimes = {ampName: np.array([]) for ampName in ampNames} 

205 self.rawMeans = {ampName: np.array([]) for ampName in ampNames} 

206 self.rawVars = {ampName: np.array([]) for ampName in ampNames} 

207 self.rowMeanVariance = {ampName: np.array([]) for ampName in ampNames} 

208 self.photoCharges = {ampName: np.array([]) for ampName in ampNames} 

209 

210 self.gain = {ampName: np.nan for ampName in ampNames} 

211 self.gainErr = {ampName: np.nan for ampName in ampNames} 

212 self.noise = {ampName: np.nan for ampName in ampNames} 

213 self.noiseErr = {ampName: np.nan for ampName in ampNames} 

214 

215 self.histVars = {ampName: np.array([]) for ampName in ampNames} 

216 self.histChi2Dofs = {ampName: np.array([]) for ampName in ampNames} 

217 self.kspValues = {ampName: np.array([]) for ampName in ampNames} 

218 

219 self.ptcFitPars = {ampName: np.array([]) for ampName in ampNames} 

220 self.ptcFitParsError = {ampName: np.array([]) for ampName in ampNames} 

221 self.ptcFitChiSq = {ampName: np.nan for ampName in ampNames} 

222 self.ptcTurnoff = {ampName: np.nan for ampName in ampNames} 

223 

224 self.covariances = {ampName: np.array([]) for ampName in ampNames} 

225 self.covariancesModel = {ampName: np.array([]) for ampName in ampNames} 

226 self.covariancesSqrtWeights = {ampName: np.array([]) for ampName in ampNames} 

227 self.aMatrix = {ampName: np.array([]) for ampName in ampNames} 

228 self.bMatrix = {ampName: np.array([]) for ampName in ampNames} 

229 self.noiseMatrix = {ampName: np.array([]) for ampName in ampNames} 

230 self.covariancesModelNoB = {ampName: np.array([]) for ampName in ampNames} 

231 self.aMatrixNoB = {ampName: np.array([]) for ampName in ampNames} 

232 self.noiseMatrixNoB = {ampName: np.array([]) for ampName in ampNames} 

233 

234 self.finalVars = {ampName: np.array([]) for ampName in ampNames} 

235 self.finalModelVars = {ampName: np.array([]) for ampName in ampNames} 

236 self.finalMeans = {ampName: np.array([]) for ampName in ampNames} 

237 

238 # Try this as a dict of arrays. 

239 self.auxValues = {} 

240 

241 super().__init__(**kwargs) 

242 self.requiredAttributes.update(['badAmps', 'inputExpIdPairs', 'expIdMask', 'rawExpTimes', 

243 'rawMeans', 'rawVars', 'rowMeanVariance', 'gain', 

244 'gainErr', 'noise', 'noiseErr', 

245 'ptcFitPars', 'ptcFitParsError', 'ptcFitChiSq', 'ptcTurnoff', 

246 'aMatrixNoB', 'covariances', 'covariancesModel', 

247 'covariancesSqrtWeights', 'covariancesModelNoB', 

248 'aMatrix', 'bMatrix', 'noiseMatrix', 'noiseMatrixNoB', 'finalVars', 

249 'finalModelVars', 'finalMeans', 'photoCharges', 'histVars', 

250 'histChi2Dofs', 'kspValues', 'auxValues']) 

251 

252 self.updateMetadata(setCalibInfo=True, setCalibId=True, **kwargs) 

253 self._validateCovarianceMatrizSizes() 

254 

255 def setAmpValuesPartialDataset( 

256 self, 

257 ampName, 

258 inputExpIdPair=(-1, -1), 

259 rawExpTime=np.nan, 

260 rawMean=np.nan, 

261 rawVar=np.nan, 

262 rowMeanVariance=np.nan, 

263 photoCharge=np.nan, 

264 expIdMask=False, 

265 covariance=None, 

266 covSqrtWeights=None, 

267 gain=np.nan, 

268 noise=np.nan, 

269 histVar=np.nan, 

270 histChi2Dof=np.nan, 

271 kspValue=0.0, 

272 auxValues=None, 

273 ): 

274 """ 

275 Set the amp values for a partial PTC Dataset (from cpExtractPtcTask). 

276 

277 Parameters 

278 ---------- 

279 ampName : `str` 

280 Name of the amp to set the values. 

281 inputExpIdPair : `tuple` [`int`] 

282 Exposure IDs of input pair. 

283 rawExpTime : `float`, optional 

284 Exposure time for this exposure pair. 

285 rawMean : `float`, optional 

286 Average of the means of the exposures in this pair. 

287 rawVar : `float`, optional 

288 Variance of the difference of the exposures in this pair. 

289 rowMeanVariance : `float`, optional 

290 Variance of the means of the rows in the difference image 

291 of the exposures in this pair. 

292 photoCharge : `float`, optional 

293 Integrated photocharge for flat pair for linearity calibration. 

294 expIdMask : `bool`, optional 

295 Flag setting if this exposure pair should be used (True) 

296 or not used (False). 

297 covariance : `np.ndarray` or None, optional 

298 Measured covariance for this exposure pair. 

299 covSqrtWeights : `np.ndarray` or None, optional 

300 Measured sqrt of covariance weights in this exposure pair. 

301 gain : `float`, optional 

302 Estimated gain for this exposure pair. 

303 noise : `float`, optional 

304 Estimated read noise for this exposure pair. 

305 histVar : `float`, optional 

306 Variance estimated from fitting a histogram with a Gaussian model. 

307 histChi2Dof : `float`, optional 

308 Chi-squared per degree of freedom from Gaussian histogram fit. 

309 kspValue : `float`, optional 

310 KS test p-value from the Gaussian histogram fit. 

311 """ 

312 nanMatrix = np.full((self.covMatrixSide, self.covMatrixSide), np.nan) 

313 nanMatrixFit = np.full((self.covMatrixSideFullCovFit, 

314 self.covMatrixSideFullCovFit), np.nan) 

315 if covariance is None: 

316 covariance = nanMatrix 

317 if covSqrtWeights is None: 

318 covSqrtWeights = nanMatrix 

319 

320 self.inputExpIdPairs[ampName] = [inputExpIdPair] 

321 self.rawExpTimes[ampName] = np.array([rawExpTime]) 

322 self.rawMeans[ampName] = np.array([rawMean]) 

323 self.rawVars[ampName] = np.array([rawVar]) 

324 self.rowMeanVariance[ampName] = np.array([rowMeanVariance]) 

325 self.photoCharges[ampName] = np.array([photoCharge]) 

326 self.expIdMask[ampName] = np.array([expIdMask]) 

327 self.covariances[ampName] = np.array([covariance]) 

328 self.covariancesSqrtWeights[ampName] = np.array([covSqrtWeights]) 

329 self.gain[ampName] = gain 

330 self.noise[ampName] = noise 

331 self.histVars[ampName] = np.array([histVar]) 

332 self.histChi2Dofs[ampName] = np.array([histChi2Dof]) 

333 self.kspValues[ampName] = np.array([kspValue]) 

334 

335 # From FULLCOVARIANCE model 

336 self.covariancesModel[ampName] = np.array([nanMatrixFit]) 

337 self.covariancesModelNoB[ampName] = np.array([nanMatrixFit]) 

338 self.aMatrix[ampName] = nanMatrixFit 

339 self.bMatrix[ampName] = nanMatrixFit 

340 self.aMatrixNoB[ampName] = nanMatrixFit 

341 self.noiseMatrix[ampName] = nanMatrixFit 

342 self.noiseMatrixNoB[ampName] = nanMatrixFit 

343 

344 def setAuxValuesPartialDataset(self, auxDict): 

345 """ 

346 Set a dictionary of auxiliary values for a partial dataset. 

347 

348 Parameters 

349 ---------- 

350 auxDict : `dict` [`str`, `float`] 

351 Dictionary of float values. 

352 """ 

353 for key, value in auxDict.items(): 

354 self.auxValues[key] = np.atleast_1d(np.array(value, dtype=np.float64)) 

355 

356 def updateMetadata(self, **kwargs): 

357 """Update calibration metadata. 

358 This calls the base class's method after ensuring the required 

359 calibration keywords will be saved. 

360 

361 Parameters 

362 ---------- 

363 setDate : `bool`, optional 

364 Update the CALIBDATE fields in the metadata to the current 

365 time. Defaults to False. 

366 kwargs : 

367 Other keyword parameters to set in the metadata. 

368 """ 

369 super().updateMetadata(PTC_FIT_TYPE=self.ptcFitType, **kwargs) 

370 

371 @classmethod 

372 def fromDict(cls, dictionary): 

373 """Construct a calibration from a dictionary of properties. 

374 Must be implemented by the specific calibration subclasses. 

375 

376 Parameters 

377 ---------- 

378 dictionary : `dict` 

379 Dictionary of properties. 

380 

381 Returns 

382 ------- 

383 calib : `lsst.ip.isr.PhotonTransferCurveDataset` 

384 Constructed calibration. 

385 

386 Raises 

387 ------ 

388 RuntimeError 

389 Raised if the supplied dictionary is for a different 

390 calibration. 

391 """ 

392 calib = cls() 

393 if calib._OBSTYPE != dictionary['metadata']['OBSTYPE']: 

394 raise RuntimeError(f"Incorrect Photon Transfer Curve dataset supplied. " 

395 f"Expected {calib._OBSTYPE}, found {dictionary['metadata']['OBSTYPE']}") 

396 calib.setMetadata(dictionary['metadata']) 

397 calib.ptcFitType = dictionary['ptcFitType'] 

398 calib.covMatrixSide = dictionary['covMatrixSide'] 

399 calib.covMatrixSideFullCovFit = dictionary['covMatrixSideFullCovFit'] 

400 calib.badAmps = np.array(dictionary['badAmps'], 'str').tolist() 

401 calib.ampNames = [] 

402 

403 # The cov matrices are square 

404 covMatrixSide = calib.covMatrixSide 

405 covMatrixSideFullCovFit = calib.covMatrixSideFullCovFit 

406 # Number of final signal levels 

407 covDimensionsProduct = len(np.array(list(dictionary['covariances'].values())[0]).ravel()) 

408 nSignalPoints = int(covDimensionsProduct/(covMatrixSide*covMatrixSide)) 

409 

410 for ampName in dictionary['ampNames']: 

411 calib.ampNames.append(ampName) 

412 calib.inputExpIdPairs[ampName] = dictionary['inputExpIdPairs'][ampName] 

413 calib.expIdMask[ampName] = np.array(dictionary['expIdMask'][ampName]) 

414 calib.rawExpTimes[ampName] = np.array(dictionary['rawExpTimes'][ampName], dtype=np.float64) 

415 calib.rawMeans[ampName] = np.array(dictionary['rawMeans'][ampName], dtype=np.float64) 

416 calib.rawVars[ampName] = np.array(dictionary['rawVars'][ampName], dtype=np.float64) 

417 calib.rowMeanVariance[ampName] = np.array(dictionary['rowMeanVariance'][ampName], 

418 dtype=np.float64) 

419 calib.gain[ampName] = float(dictionary['gain'][ampName]) 

420 calib.gainErr[ampName] = float(dictionary['gainErr'][ampName]) 

421 calib.noise[ampName] = float(dictionary['noise'][ampName]) 

422 calib.noiseErr[ampName] = float(dictionary['noiseErr'][ampName]) 

423 calib.histVars[ampName] = np.array(dictionary['histVars'][ampName], dtype=np.float64) 

424 calib.histChi2Dofs[ampName] = np.array(dictionary['histChi2Dofs'][ampName], dtype=np.float64) 

425 calib.kspValues[ampName] = np.array(dictionary['kspValues'][ampName], dtype=np.float64) 

426 calib.ptcFitPars[ampName] = np.array(dictionary['ptcFitPars'][ampName], dtype=np.float64) 

427 calib.ptcFitParsError[ampName] = np.array(dictionary['ptcFitParsError'][ampName], 

428 dtype=np.float64) 

429 calib.ptcFitChiSq[ampName] = float(dictionary['ptcFitChiSq'][ampName]) 

430 calib.ptcTurnoff[ampName] = float(dictionary['ptcTurnoff'][ampName]) 

431 if nSignalPoints > 0: 

432 # Regular dataset 

433 calib.covariances[ampName] = np.array(dictionary['covariances'][ampName], 

434 dtype=np.float64).reshape( 

435 (nSignalPoints, covMatrixSide, covMatrixSide)) 

436 calib.covariancesModel[ampName] = np.array( 

437 dictionary['covariancesModel'][ampName], 

438 dtype=np.float64).reshape( 

439 (nSignalPoints, covMatrixSideFullCovFit, covMatrixSideFullCovFit)) 

440 calib.covariancesSqrtWeights[ampName] = np.array( 

441 dictionary['covariancesSqrtWeights'][ampName], 

442 dtype=np.float64).reshape( 

443 (nSignalPoints, covMatrixSide, covMatrixSide)) 

444 calib.aMatrix[ampName] = np.array(dictionary['aMatrix'][ampName], 

445 dtype=np.float64).reshape( 

446 (covMatrixSideFullCovFit, covMatrixSideFullCovFit)) 

447 calib.bMatrix[ampName] = np.array(dictionary['bMatrix'][ampName], 

448 dtype=np.float64).reshape( 

449 (covMatrixSideFullCovFit, covMatrixSideFullCovFit)) 

450 calib.covariancesModelNoB[ampName] = np.array( 

451 dictionary['covariancesModelNoB'][ampName], dtype=np.float64).reshape( 

452 (nSignalPoints, covMatrixSideFullCovFit, covMatrixSideFullCovFit)) 

453 calib.aMatrixNoB[ampName] = np.array( 

454 dictionary['aMatrixNoB'][ampName], 

455 dtype=np.float64).reshape((covMatrixSideFullCovFit, covMatrixSideFullCovFit)) 

456 calib.noiseMatrix[ampName] = np.array( 

457 dictionary['noiseMatrix'][ampName], 

458 dtype=np.float64).reshape((covMatrixSideFullCovFit, covMatrixSideFullCovFit)) 

459 calib.noiseMatrixNoB[ampName] = np.array( 

460 dictionary['noiseMatrixNoB'][ampName], 

461 dtype=np.float64).reshape((covMatrixSideFullCovFit, covMatrixSideFullCovFit)) 

462 else: 

463 # Empty dataset 

464 calib.covariances[ampName] = np.array([], dtype=np.float64) 

465 calib.covariancesModel[ampName] = np.array([], dtype=np.float64) 

466 calib.covariancesSqrtWeights[ampName] = np.array([], dtype=np.float64) 

467 calib.aMatrix[ampName] = np.array([], dtype=np.float64) 

468 calib.bMatrix[ampName] = np.array([], dtype=np.float64) 

469 calib.covariancesModelNoB[ampName] = np.array([], dtype=np.float64) 

470 calib.aMatrixNoB[ampName] = np.array([], dtype=np.float64) 

471 calib.noiseMatrix[ampName] = np.array([], dtype=np.float64) 

472 calib.noiseMatrixNoB[ampName] = np.array([], dtype=np.float64) 

473 

474 calib.finalVars[ampName] = np.array(dictionary['finalVars'][ampName], dtype=np.float64) 

475 calib.finalModelVars[ampName] = np.array(dictionary['finalModelVars'][ampName], dtype=np.float64) 

476 calib.finalMeans[ampName] = np.array(dictionary['finalMeans'][ampName], dtype=np.float64) 

477 calib.photoCharges[ampName] = np.array(dictionary['photoCharges'][ampName], dtype=np.float64) 

478 

479 for key, value in dictionary['auxValues'].items(): 

480 calib.auxValues[key] = np.atleast_1d(np.array(value, dtype=np.float64)) 

481 

482 calib.updateMetadata() 

483 return calib 

484 

485 def toDict(self): 

486 """Return a dictionary containing the calibration properties. 

487 The dictionary should be able to be round-tripped through 

488 `fromDict`. 

489 

490 Returns 

491 ------- 

492 dictionary : `dict` 

493 Dictionary of properties. 

494 """ 

495 self.updateMetadata() 

496 

497 outDict = dict() 

498 metadata = self.getMetadata() 

499 outDict['metadata'] = metadata 

500 

501 def _dictOfArraysToDictOfLists(dictOfArrays): 

502 dictOfLists = {} 

503 for key, value in dictOfArrays.items(): 

504 dictOfLists[key] = value.ravel().tolist() 

505 

506 return dictOfLists 

507 

508 outDict['ptcFitType'] = self.ptcFitType 

509 outDict['covMatrixSide'] = self.covMatrixSide 

510 outDict['covMatrixSideFullCovFit'] = self.covMatrixSideFullCovFit 

511 outDict['ampNames'] = self.ampNames 

512 outDict['badAmps'] = self.badAmps 

513 outDict['inputExpIdPairs'] = self.inputExpIdPairs 

514 outDict['expIdMask'] = _dictOfArraysToDictOfLists(self.expIdMask) 

515 outDict['rawExpTimes'] = _dictOfArraysToDictOfLists(self.rawExpTimes) 

516 outDict['rawMeans'] = _dictOfArraysToDictOfLists(self.rawMeans) 

517 outDict['rawVars'] = _dictOfArraysToDictOfLists(self.rawVars) 

518 outDict['rowMeanVariance'] = _dictOfArraysToDictOfLists(self.rowMeanVariance) 

519 outDict['gain'] = self.gain 

520 outDict['gainErr'] = self.gainErr 

521 outDict['noise'] = self.noise 

522 outDict['noiseErr'] = self.noiseErr 

523 outDict['histVars'] = self.histVars 

524 outDict['histChi2Dofs'] = self.histChi2Dofs 

525 outDict['kspValues'] = self.kspValues 

526 outDict['ptcFitPars'] = _dictOfArraysToDictOfLists(self.ptcFitPars) 

527 outDict['ptcFitParsError'] = _dictOfArraysToDictOfLists(self.ptcFitParsError) 

528 outDict['ptcFitChiSq'] = self.ptcFitChiSq 

529 outDict['ptcTurnoff'] = self.ptcTurnoff 

530 outDict['covariances'] = _dictOfArraysToDictOfLists(self.covariances) 

531 outDict['covariancesModel'] = _dictOfArraysToDictOfLists(self.covariancesModel) 

532 outDict['covariancesSqrtWeights'] = _dictOfArraysToDictOfLists(self.covariancesSqrtWeights) 

533 outDict['aMatrix'] = _dictOfArraysToDictOfLists(self.aMatrix) 

534 outDict['bMatrix'] = _dictOfArraysToDictOfLists(self.bMatrix) 

535 outDict['noiseMatrix'] = _dictOfArraysToDictOfLists(self.noiseMatrix) 

536 outDict['covariancesModelNoB'] = _dictOfArraysToDictOfLists(self.covariancesModelNoB) 

537 outDict['aMatrixNoB'] = _dictOfArraysToDictOfLists(self.aMatrixNoB) 

538 outDict['noiseMatrixNoB'] = _dictOfArraysToDictOfLists(self.noiseMatrixNoB) 

539 outDict['finalVars'] = _dictOfArraysToDictOfLists(self.finalVars) 

540 outDict['finalModelVars'] = _dictOfArraysToDictOfLists(self.finalModelVars) 

541 outDict['finalMeans'] = _dictOfArraysToDictOfLists(self.finalMeans) 

542 outDict['photoCharges'] = _dictOfArraysToDictOfLists(self.photoCharges) 

543 outDict['auxValues'] = _dictOfArraysToDictOfLists(self.auxValues) 

544 

545 return outDict 

546 

547 @classmethod 

548 def fromTable(cls, tableList): 

549 """Construct calibration from a list of tables. 

550 This method uses the `fromDict` method to create the 

551 calibration, after constructing an appropriate dictionary from 

552 the input tables. 

553 

554 Parameters 

555 ---------- 

556 tableList : `list` [`lsst.afw.table.Table`] 

557 List of tables to use to construct the datasetPtc. 

558 

559 Returns 

560 ------- 

561 calib : `lsst.ip.isr.PhotonTransferCurveDataset` 

562 The calibration defined in the tables. 

563 """ 

564 ptcTable = tableList[0] 

565 

566 metadata = ptcTable.meta 

567 inDict = dict() 

568 inDict['metadata'] = metadata 

569 inDict['ampNames'] = [] 

570 inDict['ptcFitType'] = [] 

571 inDict['covMatrixSide'] = [] 

572 inDict['covMatrixSideFullCovFit'] = [] 

573 inDict['inputExpIdPairs'] = dict() 

574 inDict['expIdMask'] = dict() 

575 inDict['rawExpTimes'] = dict() 

576 inDict['rawMeans'] = dict() 

577 inDict['rawVars'] = dict() 

578 inDict['rowMeanVariance'] = dict() 

579 inDict['gain'] = dict() 

580 inDict['gainErr'] = dict() 

581 inDict['noise'] = dict() 

582 inDict['noiseErr'] = dict() 

583 inDict['histVars'] = dict() 

584 inDict['histChi2Dofs'] = dict() 

585 inDict['kspValues'] = dict() 

586 inDict['ptcFitPars'] = dict() 

587 inDict['ptcFitParsError'] = dict() 

588 inDict['ptcFitChiSq'] = dict() 

589 inDict['ptcTurnoff'] = dict() 

590 inDict['covariances'] = dict() 

591 inDict['covariancesModel'] = dict() 

592 inDict['covariancesSqrtWeights'] = dict() 

593 inDict['aMatrix'] = dict() 

594 inDict['bMatrix'] = dict() 

595 inDict['noiseMatrix'] = dict() 

596 inDict['covariancesModelNoB'] = dict() 

597 inDict['aMatrixNoB'] = dict() 

598 inDict['noiseMatrixNoB'] = dict() 

599 inDict['finalVars'] = dict() 

600 inDict['finalModelVars'] = dict() 

601 inDict['finalMeans'] = dict() 

602 inDict['badAmps'] = [] 

603 inDict['photoCharges'] = dict() 

604 

605 calibVersion = metadata['PTC_VERSION'] 

606 if calibVersion == 1.0: 

607 cls().log.warning(f"Previous version found for PTC dataset: {calibVersion}. " 

608 f"Setting 'ptcTurnoff' in all amps to last value in 'finalMeans'.") 

609 for record in ptcTable: 

610 ampName = record['AMPLIFIER_NAME'] 

611 

612 inDict['ptcFitType'] = record['PTC_FIT_TYPE'] 

613 inDict['covMatrixSide'] = record['COV_MATRIX_SIDE'] 

614 inDict['ampNames'].append(ampName) 

615 inDict['inputExpIdPairs'][ampName] = record['INPUT_EXP_ID_PAIRS'].tolist() 

616 inDict['expIdMask'][ampName] = record['EXP_ID_MASK'] 

617 inDict['rawExpTimes'][ampName] = record['RAW_EXP_TIMES'] 

618 inDict['rawMeans'][ampName] = record['RAW_MEANS'] 

619 inDict['rawVars'][ampName] = record['RAW_VARS'] 

620 inDict['rowMeanVariance'][ampName] = record['ROW_MEAN_VARIANCE'] 

621 inDict['gain'][ampName] = record['GAIN'] 

622 inDict['gainErr'][ampName] = record['GAIN_ERR'] 

623 inDict['noise'][ampName] = record['NOISE'] 

624 inDict['noiseErr'][ampName] = record['NOISE_ERR'] 

625 inDict['ptcFitPars'][ampName] = record['PTC_FIT_PARS'] 

626 inDict['ptcFitParsError'][ampName] = record['PTC_FIT_PARS_ERROR'] 

627 inDict['ptcFitChiSq'][ampName] = record['PTC_FIT_CHI_SQ'] 

628 inDict['covariances'][ampName] = record['COVARIANCES'] 

629 inDict['covariancesModel'][ampName] = record['COVARIANCES_MODEL'] 

630 inDict['covariancesSqrtWeights'][ampName] = record['COVARIANCES_SQRT_WEIGHTS'] 

631 inDict['aMatrix'][ampName] = record['A_MATRIX'] 

632 inDict['bMatrix'][ampName] = record['B_MATRIX'] 

633 inDict['covariancesModelNoB'][ampName] = record['COVARIANCES_MODEL_NO_B'] 

634 inDict['aMatrixNoB'][ampName] = record['A_MATRIX_NO_B'] 

635 inDict['finalVars'][ampName] = record['FINAL_VARS'] 

636 inDict['finalModelVars'][ampName] = record['FINAL_MODEL_VARS'] 

637 inDict['finalMeans'][ampName] = record['FINAL_MEANS'] 

638 inDict['badAmps'] = record['BAD_AMPS'].tolist() 

639 inDict['photoCharges'][ampName] = record['PHOTO_CHARGE'] 

640 if calibVersion == 1.0: 

641 mask = record['FINAL_MEANS'].mask 

642 array = record['FINAL_MEANS'][~mask] 

643 if len(array) > 0: 

644 inDict['ptcTurnoff'][ampName] = record['FINAL_MEANS'][~mask][-1] 

645 else: 

646 inDict['ptcTurnoff'][ampName] = np.nan 

647 else: 

648 inDict['ptcTurnoff'][ampName] = record['PTC_TURNOFF'] 

649 if calibVersion < 1.2: 

650 inDict['histVars'][ampName] = np.array([np.nan]) 

651 inDict['histChi2Dofs'][ampName] = np.array([np.nan]) 

652 inDict['kspValues'][ampName] = np.array([0.0]) 

653 else: 

654 inDict['histVars'][ampName] = record['HIST_VARS'] 

655 inDict['histChi2Dofs'][ampName] = record['HIST_CHI2_DOFS'] 

656 inDict['kspValues'][ampName] = record['KS_PVALUES'] 

657 if calibVersion < 1.3: 

658 nanMatrix = np.full_like(inDict['aMatrix'][ampName], np.nan) 

659 inDict['noiseMatrix'][ampName] = nanMatrix 

660 inDict['noiseMatrixNoB'][ampName] = nanMatrix 

661 else: 

662 inDict['noiseMatrix'][ampName] = record['NOISE_MATRIX'] 

663 inDict['noiseMatrixNoB'][ampName] = record['NOISE_MATRIX_NO_B'] 

664 if calibVersion < 1.5: 

665 # Matched to `COV_MATRIX_SIDE`. Same for all amps. 

666 inDict['covMatrixSideFullCovFit'] = inDict['covMatrixSide'] 

667 else: 

668 inDict['covMatrixSideFullCovFit'] = record['COV_MATRIX_SIDE_FULL_COV_FIT'] 

669 if calibVersion < 1.6: 

670 inDict['rowMeanVariance'][ampName] = np.array([np.nan]) 

671 else: 

672 inDict['rowMeanVariance'][ampName] = record['ROW_MEAN_VARIANCE'] 

673 

674 inDict['auxValues'] = {} 

675 record = ptcTable[0] 

676 for col in record.columns.keys(): 

677 if col.startswith('PTCAUX_'): 

678 parts = col.split('PTCAUX_') 

679 inDict['auxValues'][parts[1]] = record[col] 

680 

681 return cls().fromDict(inDict) 

682 

683 def toTable(self): 

684 """Construct a list of tables containing the information in this 

685 calibration. 

686 

687 The list of tables should create an identical calibration 

688 after being passed to this class's fromTable method. 

689 

690 Returns 

691 ------- 

692 tableList : `list` [`astropy.table.Table`] 

693 List of tables containing the linearity calibration 

694 information. 

695 """ 

696 tableList = [] 

697 self.updateMetadata() 

698 

699 badAmps = np.array(self.badAmps) if len(self.badAmps) else np.array([], dtype="U3") 

700 

701 catalogList = [] 

702 for ampName in self.ampNames: 

703 ampDict = { 

704 'AMPLIFIER_NAME': ampName, 

705 'PTC_FIT_TYPE': self.ptcFitType, 

706 'COV_MATRIX_SIDE': self.covMatrixSide, 

707 'COV_MATRIX_SIDE_FULL_COV_FIT': self.covMatrixSideFullCovFit, 

708 'INPUT_EXP_ID_PAIRS': self.inputExpIdPairs[ampName], 

709 'EXP_ID_MASK': self.expIdMask[ampName], 

710 'RAW_EXP_TIMES': self.rawExpTimes[ampName], 

711 'RAW_MEANS': self.rawMeans[ampName], 

712 'RAW_VARS': self.rawVars[ampName], 

713 'ROW_MEAN_VARIANCE': self.rowMeanVariance[ampName], 

714 'GAIN': self.gain[ampName], 

715 'GAIN_ERR': self.gainErr[ampName], 

716 'NOISE': self.noise[ampName], 

717 'NOISE_ERR': self.noiseErr[ampName], 

718 'HIST_VARS': self.histVars[ampName], 

719 'HIST_CHI2_DOFS': self.histChi2Dofs[ampName], 

720 'KS_PVALUES': self.kspValues[ampName], 

721 'PTC_FIT_PARS': np.array(self.ptcFitPars[ampName]), 

722 'PTC_FIT_PARS_ERROR': np.array(self.ptcFitParsError[ampName]), 

723 'PTC_FIT_CHI_SQ': self.ptcFitChiSq[ampName], 

724 'PTC_TURNOFF': self.ptcTurnoff[ampName], 

725 'A_MATRIX': self.aMatrix[ampName].ravel(), 

726 'B_MATRIX': self.bMatrix[ampName].ravel(), 

727 'A_MATRIX_NO_B': self.aMatrixNoB[ampName].ravel(), 

728 'NOISE_MATRIX': self.noiseMatrix[ampName].ravel(), 

729 'NOISE_MATRIX_NO_B': self.noiseMatrixNoB[ampName].ravel(), 

730 'BAD_AMPS': badAmps, 

731 'PHOTO_CHARGE': self.photoCharges[ampName], 

732 'COVARIANCES': self.covariances[ampName].ravel(), 

733 'COVARIANCES_MODEL': self.covariancesModel[ampName].ravel(), 

734 'COVARIANCES_SQRT_WEIGHTS': self.covariancesSqrtWeights[ampName].ravel(), 

735 'COVARIANCES_MODEL_NO_B': self.covariancesModelNoB[ampName].ravel(), 

736 'FINAL_VARS': self.finalVars[ampName], 

737 'FINAL_MODEL_VARS': self.finalModelVars[ampName], 

738 'FINAL_MEANS': self.finalMeans[ampName], 

739 } 

740 

741 if self.auxValues: 

742 for key, value in self.auxValues.items(): 

743 ampDict[f"PTCAUX_{key}"] = value 

744 

745 catalogList.append(ampDict) 

746 

747 catalog = Table(catalogList) 

748 

749 inMeta = self.getMetadata().toDict() 

750 outMeta = {k: v for k, v in inMeta.items() if v is not None} 

751 outMeta.update({k: "" for k, v in inMeta.items() if v is None}) 

752 catalog.meta = outMeta 

753 tableList.append(catalog) 

754 

755 return tableList 

756 

757 def fromDetector(self, detector): 

758 """Read metadata parameters from a detector. 

759 

760 Parameters 

761 ---------- 

762 detector : `lsst.afw.cameraGeom.detector` 

763 Input detector with parameters to use. 

764 

765 Returns 

766 ------- 

767 calib : `lsst.ip.isr.PhotonTransferCurveDataset` 

768 The calibration constructed from the detector. 

769 """ 

770 

771 pass 

772 

773 def getExpIdsUsed(self, ampName): 

774 """Get the exposures used, i.e. not discarded, for a given amp. 

775 If no mask has been created yet, all exposures are returned. 

776 

777 Parameters 

778 ---------- 

779 ampName : `str` 

780 

781 Returns 

782 ------- 

783 expIdsUsed : `list` [`tuple`] 

784 List of pairs of exposure ids used in PTC. 

785 """ 

786 if len(self.expIdMask[ampName]) == 0: 

787 return self.inputExpIdPairs[ampName] 

788 

789 # if the mask exists it had better be the same length as the expIdPairs 

790 assert len(self.expIdMask[ampName]) == len(self.inputExpIdPairs[ampName]) 

791 

792 pairs = self.inputExpIdPairs[ampName] 

793 mask = self.expIdMask[ampName] 

794 # cast to bool required because numpy 

795 try: 

796 expIdsUsed = [(exp1, exp2) for ((exp1, exp2), m) in zip(pairs, mask) if m] 

797 except ValueError: 

798 self.log.warning("The PTC file was written incorrectly; you should rerun the " 

799 "PTC solve task if possible.") 

800 expIdsUsed = [] 

801 for pairList, m in zip(pairs, mask): 

802 if m: 

803 expIdsUsed.append(pairList[0]) 

804 

805 return expIdsUsed 

806 

807 def getGoodAmps(self): 

808 """Get the good amps from this PTC.""" 

809 return [amp for amp in self.ampNames if amp not in self.badAmps] 

810 

811 def getGoodPoints(self, ampName): 

812 """Get the good points used for a given amp in the PTC. 

813 

814 Parameters 

815 ---------- 

816 ampName : `str` 

817 Amplifier's name. 

818 

819 Returns 

820 ------- 

821 goodPoints : `np.ndarray` 

822 Boolean array of good points used in PTC. 

823 """ 

824 return self.expIdMask[ampName] 

825 

826 def validateGainNoiseTurnoffValues(self, ampName, doWarn=False): 

827 """Ensure the gain, read noise, and PTC turnoff have 

828 sensible values. 

829 

830 Parameters 

831 ---------- 

832 ampName : `str` 

833 Amplifier's name. 

834 """ 

835 

836 gain = self.gain[ampName] 

837 noise = self.noise[ampName] 

838 ptcTurnoff = self.ptcTurnoff[ampName] 

839 

840 # Check if gain is not positive or is np.nan 

841 if not (isinstance(gain, (int, float)) and gain > 0) or math.isnan(gain): 

842 if doWarn: 

843 self.log.warning(f"Invalid gain value {gain}" 

844 " Setting to default: Gain=1") 

845 gain = 1 

846 

847 # Check if noise is not positive or is np.nan 

848 if not (isinstance(noise, (int, float)) and noise > 0) or math.isnan(noise): 

849 if doWarn: 

850 self.log.warning(f"Invalid noise value: {noise}" 

851 " Setting to default: Noise=1") 

852 noise = 1 

853 

854 # Check if ptcTurnoff is not positive or is np.nan 

855 if not (isinstance(ptcTurnoff, (int, float)) and ptcTurnoff > 0) or math.isnan(ptcTurnoff): 

856 if doWarn: 

857 self.log.warning(f"Invalid PTC turnoff value: {ptcTurnoff}" 

858 " Setting to default: PTC Turnoff=2e19") 

859 ptcTurnoff = 2e19 

860 

861 self.gain[ampName] = gain 

862 self.noise[ampName] = noise 

863 self.ptcTurnoff[ampName] = ptcTurnoff 

864 

865 def _validateCovarianceMatrizSizes(self): 

866 """Ensure covMatrixSideFullCovFit <= covMatrixSide.""" 

867 if self.covMatrixSideFullCovFit > self.covMatrixSide: 

868 self.log.warning("covMatrixSideFullCovFit > covMatrixSide " 

869 f"({self.covMatrixSideFullCovFit} > {self.covMatrixSide})." 

870 "Setting the former to the latter.") 

871 self.covMatrixSideFullCovFit = self.covMatrixSide