Coverage for python/lsst/ip/isr/ptcDataset.py: 6%

264 statements  

« prev     ^ index     » next       coverage.py v6.4.1, created at 2022-07-03 01:44 -0700

1# 

2# LSST Data Management System 

3# Copyright 2008-2017 AURA/LSST. 

4# 

5# This product includes software developed by the 

6# LSST Project (http://www.lsst.org/). 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <https://www.lsstcorp.org/LegalNotices/>. 

21# 

22""" 

23Define dataset class for MeasurePhotonTransferCurve task 

24""" 

25import numpy as np 

26from astropy.table import Table 

27 

28from lsst.ip.isr import IsrCalib 

29 

30__all__ = ['PhotonTransferCurveDataset'] 

31 

32 

33class PhotonTransferCurveDataset(IsrCalib): 

34 """A simple class to hold the output data from the PTC task. 

35 The dataset is made up of a dictionary for each item, keyed by the 

36 amplifiers' names, which much be supplied at construction time. 

37 New items cannot be added to the class to save accidentally saving to the 

38 wrong property, and the class can be frozen if desired. 

39 inputExpIdPairs records the exposures used to produce the data. 

40 When fitPtc() or fitCovariancesAstier() is run, a mask is built up, which 

41 is by definition always the same length as inputExpIdPairs, rawExpTimes, 

42 rawMeans and rawVars, and is a list of bools, which are incrementally set 

43 to False as points are discarded from the fits. 

44 PTC fit parameters for polynomials are stored in a list in ascending order 

45 of polynomial term, i.e. par[0]*x^0 + par[1]*x + par[2]*x^2 etc 

46 with the length of the list corresponding to the order of the polynomial 

47 plus one. 

48 

49 Parameters 

50 ---------- 

51 ampNames : `list` 

52 List with the names of the amplifiers of the detector at hand. 

53 

54 ptcFitType : `str` 

55 Type of model fitted to the PTC: "POLYNOMIAL", "EXPAPPROXIMATION", 

56 or "FULLCOVARIANCE". 

57 

58 covMatrixSide : `int` 

59 Maximum lag of covariances (size of square covariance matrices). 

60 

61 kwargs : `dict`, optional 

62 Other keyword arguments to pass to the parent init. 

63 

64 Notes 

65 ----- 

66 The stored attributes are: 

67 badAmps : `list` 

68 List with bad amplifiers names. 

69 inputExpIdPairs : `dict`, [`str`, `list`] 

70 Dictionary keyed by amp names containing the input exposures IDs. 

71 expIdMask : `dict`, [`str`, `list`] 

72 Dictionary keyed by amp names containing the mask produced after 

73 outlier rejection. The mask produced by the "FULLCOVARIANCE" 

74 option may differ from the one produced in the other two PTC 

75 fit types. 

76 rawExpTimes : `dict`, [`str`, `list`] 

77 Dictionary keyed by amp names containing the unmasked exposure times. 

78 rawMeans : `dict`, [`str`, `list`] 

79 Dictionary keyed by amp namescontaining the unmasked average of the 

80 means of the exposures in each flat pair. 

81 rawVars : `dict`, [`str`, `list`] 

82 Dictionary keyed by amp names containing the variance of the 

83 difference image of the exposures in each flat pair. 

84 gain : `dict`, [`str`, `list`] 

85 Dictionary keyed by amp names containing the fitted gains. 

86 gainErr : `dict`, [`str`, `list`] 

87 Dictionary keyed by amp names containing the errors on the 

88 fitted gains. 

89 noise : `dict`, [`str`, `list`] 

90 Dictionary keyed by amp names containing the fitted noise. 

91 noiseErr : `dict`, [`str`, `list`] 

92 Dictionary keyed by amp names containing the errors on the fitted 

93 noise. 

94 ptcFitPars : `dict`, [`str`, `list`] 

95 Dictionary keyed by amp names containing the fitted parameters of the 

96 PTC model for ptcFitTye in ["POLYNOMIAL", "EXPAPPROXIMATION"]. 

97 ptcFitParsError : `dict`, [`str`, `list`] 

98 Dictionary keyed by amp names containing the errors on the fitted 

99 parameters of the PTC model for ptcFitTye in 

100 ["POLYNOMIAL", "EXPAPPROXIMATION"]. 

101 ptcFitChiSq : `dict`, [`str`, `list`] 

102 Dictionary keyed by amp names containing the reduced chi squared 

103 of the fit for ptcFitTye in ["POLYNOMIAL", "EXPAPPROXIMATION"]. 

104 ptcTurnoff : `float` 

105 Flux value (in ADU) where the variance of the PTC curve starts 

106 decreasing consistently. 

107 covariances : `dict`, [`str`, `list`] 

108 Dictionary keyed by amp names containing a list of measured 

109 covariances per mean flux. 

110 covariancesModel : `dict`, [`str`, `list`] 

111 Dictionary keyed by amp names containinging covariances model 

112 (Eq. 20 of Astier+19) per mean flux. 

113 covariancesSqrtWeights : `dict`, [`str`, `list`] 

114 Dictionary keyed by amp names containinging sqrt. of covariances 

115 weights. 

116 aMatrix : `dict`, [`str`, `list`] 

117 Dictionary keyed by amp names containing the "a" parameters from 

118 the model in Eq. 20 of Astier+19. 

119 bMatrix : `dict`, [`str`, `list`] 

120 Dictionary keyed by amp names containing the "b" parameters from 

121 the model in Eq. 20 of Astier+19. 

122 covariancesModelNoB : `dict`, [`str`, `list`] 

123 Dictionary keyed by amp names containing covariances model 

124 (with 'b'=0 in Eq. 20 of Astier+19) 

125 per mean flux. 

126 aMatrixNoB : `dict`, [`str`, `list`] 

127 Dictionary keyed by amp names containing the "a" parameters from the 

128 model in Eq. 20 of Astier+19 

129 (and 'b' = 0). 

130 finalVars : `dict`, [`str`, `list`] 

131 Dictionary keyed by amp names containing the masked variance of the 

132 difference image of each flat 

133 pair. If needed, each array will be right-padded with 

134 np.nan to match the length of rawExpTimes. 

135 finalModelVars : `dict`, [`str`, `list`] 

136 Dictionary keyed by amp names containing the masked modeled 

137 variance of the difference image of each flat pair. If needed, each 

138 array will be right-padded with np.nan to match the length of 

139 rawExpTimes. 

140 finalMeans : `dict`, [`str`, `list`] 

141 Dictionary keyed by amp names containing the masked average of the 

142 means of the exposures in each flat pair. If needed, each array 

143 will be right-padded with np.nan to match the length of 

144 rawExpTimes. 

145 photoCharge : `dict`, [`str`, `list`] 

146 Dictionary keyed by amp names containing the integrated photocharge 

147 for linearity calibration. 

148 

149 Returns 

150 ------- 

151 `lsst.cp.pipe.ptc.PhotonTransferCurveDataset` 

152 Output dataset from MeasurePhotonTransferCurveTask. 

153 

154 Notes 

155 ----- 

156 Version 1.1 adds the `ptcTurnoff` attribute. 

157 """ 

158 

159 _OBSTYPE = 'PTC' 

160 _SCHEMA = 'Gen3 Photon Transfer Curve' 

161 _VERSION = 1.1 

162 

163 def __init__(self, ampNames=[], ptcFitType=None, covMatrixSide=1, **kwargs): 

164 

165 self.ptcFitType = ptcFitType 

166 self.ampNames = ampNames 

167 self.covMatrixSide = covMatrixSide 

168 

169 self.badAmps = [np.nan] 

170 

171 self.inputExpIdPairs = {ampName: [] for ampName in ampNames} 

172 self.expIdMask = {ampName: [] for ampName in ampNames} 

173 self.rawExpTimes = {ampName: [] for ampName in ampNames} 

174 self.rawMeans = {ampName: [] for ampName in ampNames} 

175 self.rawVars = {ampName: [] for ampName in ampNames} 

176 self.photoCharge = {ampName: [] for ampName in ampNames} 

177 

178 self.gain = {ampName: np.nan for ampName in ampNames} 

179 self.gainErr = {ampName: np.nan for ampName in ampNames} 

180 self.noise = {ampName: np.nan for ampName in ampNames} 

181 self.noiseErr = {ampName: np.nan for ampName in ampNames} 

182 

183 self.ptcFitPars = {ampName: [] for ampName in ampNames} 

184 self.ptcFitParsError = {ampName: [] for ampName in ampNames} 

185 self.ptcFitChiSq = {ampName: np.nan for ampName in ampNames} 

186 self.ptcTurnoff = {ampName: np.nan for ampName in ampNames} 

187 

188 self.covariances = {ampName: [] for ampName in ampNames} 

189 self.covariancesModel = {ampName: [] for ampName in ampNames} 

190 self.covariancesSqrtWeights = {ampName: [] for ampName in ampNames} 

191 self.aMatrix = {ampName: np.nan for ampName in ampNames} 

192 self.bMatrix = {ampName: np.nan for ampName in ampNames} 

193 self.covariancesModelNoB = {ampName: [] for ampName in ampNames} 

194 self.aMatrixNoB = {ampName: np.nan for ampName in ampNames} 

195 

196 self.finalVars = {ampName: [] for ampName in ampNames} 

197 self.finalModelVars = {ampName: [] for ampName in ampNames} 

198 self.finalMeans = {ampName: [] for ampName in ampNames} 

199 

200 super().__init__(**kwargs) 

201 self.requiredAttributes.update(['badAmps', 'inputExpIdPairs', 'expIdMask', 'rawExpTimes', 

202 'rawMeans', 'rawVars', 'gain', 'gainErr', 'noise', 'noiseErr', 

203 'ptcFitPars', 'ptcFitParsError', 'ptcFitChiSq', 'ptcTurnoff', 

204 'aMatrixNoB', 'covariances', 'covariancesModel', 

205 'covariancesSqrtWeights', 'covariancesModelNoB', 

206 'aMatrix', 'bMatrix', 'finalVars', 'finalModelVars', 'finalMeans', 

207 'photoCharge']) 

208 

209 def setAmpValues(self, ampName, inputExpIdPair=[(np.nan, np.nan)], expIdMask=[np.nan], 

210 rawExpTime=[np.nan], rawMean=[np.nan], rawVar=[np.nan], photoCharge=[np.nan], 

211 gain=np.nan, gainErr=np.nan, noise=np.nan, noiseErr=np.nan, ptcFitPars=[np.nan], 

212 ptcFitParsError=[np.nan], ptcFitChiSq=np.nan, ptcTurnoff=np.nan, covArray=[], 

213 covArrayModel=[], covSqrtWeights=[], aMatrix=[], bMatrix=[], covArrayModelNoB=[], 

214 aMatrixNoB=[], finalVar=[np.nan], finalModelVar=[np.nan], finalMean=[np.nan]): 

215 """Function to initialize an amp of a PhotonTransferCurveDataset. 

216 

217 Notes 

218 ----- 

219 The parameters are all documented in `init`. 

220 """ 

221 nanMatrix = np.full((self.covMatrixSide, self.covMatrixSide), np.nan) 

222 if len(covArray) == 0: 

223 covArray = [nanMatrix] 

224 if len(covArrayModel) == 0: 

225 covArrayModel = [nanMatrix] 

226 if len(covSqrtWeights) == 0: 

227 covSqrtWeights = [nanMatrix] 

228 if len(covArrayModelNoB) == 0: 

229 covArrayModelNoB = [nanMatrix] 

230 if len(aMatrix) == 0: 

231 aMatrix = nanMatrix 

232 if len(bMatrix) == 0: 

233 bMatrix = nanMatrix 

234 if len(aMatrixNoB) == 0: 

235 aMatrixNoB = nanMatrix 

236 

237 self.inputExpIdPairs[ampName] = inputExpIdPair 

238 self.expIdMask[ampName] = expIdMask 

239 self.rawExpTimes[ampName] = rawExpTime 

240 self.rawMeans[ampName] = rawMean 

241 self.rawVars[ampName] = rawVar 

242 self.photoCharge[ampName] = photoCharge 

243 self.gain[ampName] = gain 

244 self.gainErr[ampName] = gainErr 

245 self.noise[ampName] = noise 

246 self.noiseErr[ampName] = noiseErr 

247 self.ptcFitPars[ampName] = ptcFitPars 

248 self.ptcFitParsError[ampName] = ptcFitParsError 

249 self.ptcFitChiSq[ampName] = ptcFitChiSq 

250 self.ptcTurnoff[ampName] = ptcTurnoff 

251 self.covariances[ampName] = covArray 

252 self.covariancesSqrtWeights[ampName] = covSqrtWeights 

253 self.covariancesModel[ampName] = covArrayModel 

254 self.covariancesModelNoB[ampName] = covArrayModelNoB 

255 self.aMatrix[ampName] = aMatrix 

256 self.bMatrix[ampName] = bMatrix 

257 self.aMatrixNoB[ampName] = aMatrixNoB 

258 self.ptcFitPars[ampName] = ptcFitPars 

259 self.ptcFitParsError[ampName] = ptcFitParsError 

260 self.ptcFitChiSq[ampName] = ptcFitChiSq 

261 self.finalVars[ampName] = finalVar 

262 self.finalModelVars[ampName] = finalModelVar 

263 self.finalMeans[ampName] = finalMean 

264 

265 def updateMetadata(self, setDate=False, **kwargs): 

266 """Update calibration metadata. 

267 This calls the base class's method after ensuring the required 

268 calibration keywords will be saved. 

269 Parameters 

270 ---------- 

271 setDate : `bool`, optional 

272 Update the CALIBDATE fields in the metadata to the current 

273 time. Defaults to False. 

274 kwargs : 

275 Other keyword parameters to set in the metadata. 

276 """ 

277 kwargs['PTC_FIT_TYPE'] = self.ptcFitType 

278 

279 super().updateMetadata(setDate=setDate, **kwargs) 

280 

281 @classmethod 

282 def fromDict(cls, dictionary): 

283 """Construct a calibration from a dictionary of properties. 

284 Must be implemented by the specific calibration subclasses. 

285 Parameters 

286 ---------- 

287 dictionary : `dict` 

288 Dictionary of properties. 

289 Returns 

290 ------- 

291 calib : `lsst.ip.isr.CalibType` 

292 Constructed calibration. 

293 Raises 

294 ------ 

295 RuntimeError : 

296 Raised if the supplied dictionary is for a different 

297 calibration. 

298 """ 

299 calib = cls() 

300 if calib._OBSTYPE != dictionary['metadata']['OBSTYPE']: 

301 raise RuntimeError(f"Incorrect Photon Transfer Curve dataset supplied. " 

302 f"Expected {calib._OBSTYPE}, found {dictionary['metadata']['OBSTYPE']}") 

303 calib.setMetadata(dictionary['metadata']) 

304 calib.ptcFitType = dictionary['ptcFitType'] 

305 calib.covMatrixSide = dictionary['covMatrixSide'] 

306 calib.badAmps = np.array(dictionary['badAmps'], 'str').tolist() 

307 # The cov matrices are square 

308 covMatrixSide = calib.covMatrixSide 

309 # Number of final signal levels 

310 covDimensionsProduct = len(np.array(list(dictionary['covariances'].values())[0]).ravel()) 

311 nSignalPoints = int(covDimensionsProduct/(covMatrixSide*covMatrixSide)) 

312 

313 for ampName in dictionary['ampNames']: 

314 covsAmp = np.array(dictionary['covariances'][ampName]).reshape((nSignalPoints, covMatrixSide, 

315 covMatrixSide)) 

316 

317 # After cpPtcExtract runs in the PTC pipeline, the datasets 

318 # created ('PARTIAL' and 'DUMMY') have a single measurement. 

319 # Apply the maskign to the final ptcDataset, after running 

320 # cpPtcSolve. 

321 if len(covsAmp) > 1: 

322 # Masks for covariances padding in `toTable` 

323 maskCovsAmp = np.array([~np.isnan(entry).all() for entry in covsAmp]) 

324 maskAmp = ~np.isnan(np.array(dictionary['finalMeans'][ampName])) 

325 else: 

326 maskCovsAmp = np.array([True]) 

327 maskAmp = np.array([True]) 

328 

329 calib.ampNames.append(ampName) 

330 calib.inputExpIdPairs[ampName] = np.array(dictionary['inputExpIdPairs'][ampName]).tolist() 

331 calib.expIdMask[ampName] = np.array(dictionary['expIdMask'][ampName]).tolist() 

332 calib.rawExpTimes[ampName] = np.array(dictionary['rawExpTimes'][ampName]).tolist() 

333 calib.rawMeans[ampName] = np.array(dictionary['rawMeans'][ampName]).tolist() 

334 calib.rawVars[ampName] = np.array(dictionary['rawVars'][ampName]).tolist() 

335 calib.gain[ampName] = np.array(dictionary['gain'][ampName]).tolist() 

336 calib.gainErr[ampName] = np.array(dictionary['gainErr'][ampName]).tolist() 

337 calib.noise[ampName] = np.array(dictionary['noise'][ampName]).tolist() 

338 calib.noiseErr[ampName] = np.array(dictionary['noiseErr'][ampName]).tolist() 

339 calib.ptcFitPars[ampName] = np.array(dictionary['ptcFitPars'][ampName]).tolist() 

340 calib.ptcFitParsError[ampName] = np.array(dictionary['ptcFitParsError'][ampName]).tolist() 

341 calib.ptcFitChiSq[ampName] = np.array(dictionary['ptcFitChiSq'][ampName]).tolist() 

342 calib.ptcTurnoff[ampName] = np.array(dictionary['ptcTurnoff'][ampName]).tolist() 

343 calib.covariances[ampName] = covsAmp[maskCovsAmp].tolist() 

344 calib.covariancesModel[ampName] = np.array( 

345 dictionary['covariancesModel'][ampName]).reshape( 

346 (nSignalPoints, covMatrixSide, covMatrixSide))[maskCovsAmp].tolist() 

347 calib.covariancesSqrtWeights[ampName] = np.array( 

348 dictionary['covariancesSqrtWeights'][ampName]).reshape( 

349 (nSignalPoints, covMatrixSide, covMatrixSide))[maskCovsAmp].tolist() 

350 calib.aMatrix[ampName] = np.array(dictionary['aMatrix'][ampName]).reshape( 

351 (covMatrixSide, covMatrixSide)).tolist() 

352 calib.bMatrix[ampName] = np.array(dictionary['bMatrix'][ampName]).reshape( 

353 (covMatrixSide, covMatrixSide)).tolist() 

354 calib.covariancesModelNoB[ampName] = np.array( 

355 dictionary['covariancesModelNoB'][ampName]).reshape( 

356 (nSignalPoints, covMatrixSide, covMatrixSide))[maskCovsAmp].tolist() 

357 calib.aMatrixNoB[ampName] = np.array( 

358 dictionary['aMatrixNoB'][ampName]).reshape((covMatrixSide, covMatrixSide)).tolist() 

359 calib.finalVars[ampName] = np.array(dictionary['finalVars'][ampName])[maskAmp].tolist() 

360 calib.finalModelVars[ampName] = np.array(dictionary['finalModelVars'][ampName])[maskAmp].tolist() 

361 calib.finalMeans[ampName] = np.array(dictionary['finalMeans'][ampName])[maskAmp].tolist() 

362 calib.photoCharge[ampName] = np.array(dictionary['photoCharge'][ampName]).tolist() 

363 calib.updateMetadata() 

364 return calib 

365 

366 def toDict(self): 

367 """Return a dictionary containing the calibration properties. 

368 The dictionary should be able to be round-tripped through 

369 `fromDict`. 

370 Returns 

371 ------- 

372 dictionary : `dict` 

373 Dictionary of properties. 

374 """ 

375 self.updateMetadata() 

376 

377 outDict = dict() 

378 metadata = self.getMetadata() 

379 outDict['metadata'] = metadata 

380 

381 outDict['ptcFitType'] = self.ptcFitType 

382 outDict['covMatrixSide'] = self.covMatrixSide 

383 outDict['ampNames'] = self.ampNames 

384 outDict['badAmps'] = self.badAmps 

385 outDict['inputExpIdPairs'] = self.inputExpIdPairs 

386 outDict['expIdMask'] = self.expIdMask 

387 outDict['rawExpTimes'] = self.rawExpTimes 

388 outDict['rawMeans'] = self.rawMeans 

389 outDict['rawVars'] = self.rawVars 

390 outDict['gain'] = self.gain 

391 outDict['gainErr'] = self.gainErr 

392 outDict['noise'] = self.noise 

393 outDict['noiseErr'] = self.noiseErr 

394 outDict['ptcFitPars'] = self.ptcFitPars 

395 outDict['ptcFitParsError'] = self.ptcFitParsError 

396 outDict['ptcFitChiSq'] = self.ptcFitChiSq 

397 outDict['ptcTurnoff'] = self.ptcTurnoff 

398 outDict['covariances'] = self.covariances 

399 outDict['covariancesModel'] = self.covariancesModel 

400 outDict['covariancesSqrtWeights'] = self.covariancesSqrtWeights 

401 outDict['aMatrix'] = self.aMatrix 

402 outDict['bMatrix'] = self.bMatrix 

403 outDict['covariancesModelNoB'] = self.covariancesModelNoB 

404 outDict['aMatrixNoB'] = self.aMatrixNoB 

405 outDict['finalVars'] = self.finalVars 

406 outDict['finalModelVars'] = self.finalModelVars 

407 outDict['finalMeans'] = self.finalMeans 

408 outDict['photoCharge'] = self.photoCharge 

409 

410 return outDict 

411 

412 @classmethod 

413 def fromTable(cls, tableList): 

414 """Construct calibration from a list of tables. 

415 This method uses the `fromDict` method to create the 

416 calibration, after constructing an appropriate dictionary from 

417 the input tables. 

418 Parameters 

419 ---------- 

420 tableList : `list` [`lsst.afw.table.Table`] 

421 List of tables to use to construct the datasetPtc. 

422 Returns 

423 ------- 

424 calib : `lsst.cp.pipe.` 

425 The calibration defined in the tables. 

426 """ 

427 ptcTable = tableList[0] 

428 

429 metadata = ptcTable.meta 

430 inDict = dict() 

431 inDict['metadata'] = metadata 

432 inDict['ampNames'] = [] 

433 inDict['ptcFitType'] = [] 

434 inDict['covMatrixSide'] = [] 

435 inDict['inputExpIdPairs'] = dict() 

436 inDict['expIdMask'] = dict() 

437 inDict['rawExpTimes'] = dict() 

438 inDict['rawMeans'] = dict() 

439 inDict['rawVars'] = dict() 

440 inDict['gain'] = dict() 

441 inDict['gainErr'] = dict() 

442 inDict['noise'] = dict() 

443 inDict['noiseErr'] = dict() 

444 inDict['ptcFitPars'] = dict() 

445 inDict['ptcFitParsError'] = dict() 

446 inDict['ptcFitChiSq'] = dict() 

447 inDict['ptcTurnoff'] = dict() 

448 inDict['covariances'] = dict() 

449 inDict['covariancesModel'] = dict() 

450 inDict['covariancesSqrtWeights'] = dict() 

451 inDict['aMatrix'] = dict() 

452 inDict['bMatrix'] = dict() 

453 inDict['covariancesModelNoB'] = dict() 

454 inDict['aMatrixNoB'] = dict() 

455 inDict['finalVars'] = dict() 

456 inDict['finalModelVars'] = dict() 

457 inDict['finalMeans'] = dict() 

458 inDict['badAmps'] = [] 

459 inDict['photoCharge'] = dict() 

460 

461 calibVersion = metadata['PTC_VERSION'] 

462 if calibVersion == 1.0: 

463 cls().log.warning(f"Previous version found for PTC dataset: {calibVersion}. " 

464 f"Setting 'ptcTurnoff' in all amps to last value in 'finalMeans'.") 

465 for record in ptcTable: 

466 ampName = record['AMPLIFIER_NAME'] 

467 

468 inDict['ptcFitType'] = record['PTC_FIT_TYPE'] 

469 inDict['covMatrixSide'] = record['COV_MATRIX_SIDE'] 

470 inDict['ampNames'].append(ampName) 

471 inDict['inputExpIdPairs'][ampName] = record['INPUT_EXP_ID_PAIRS'] 

472 inDict['expIdMask'][ampName] = record['EXP_ID_MASK'] 

473 inDict['rawExpTimes'][ampName] = record['RAW_EXP_TIMES'] 

474 inDict['rawMeans'][ampName] = record['RAW_MEANS'] 

475 inDict['rawVars'][ampName] = record['RAW_VARS'] 

476 inDict['gain'][ampName] = record['GAIN'] 

477 inDict['gainErr'][ampName] = record['GAIN_ERR'] 

478 inDict['noise'][ampName] = record['NOISE'] 

479 inDict['noiseErr'][ampName] = record['NOISE_ERR'] 

480 inDict['ptcFitPars'][ampName] = record['PTC_FIT_PARS'] 

481 inDict['ptcFitParsError'][ampName] = record['PTC_FIT_PARS_ERROR'] 

482 inDict['ptcFitChiSq'][ampName] = record['PTC_FIT_CHI_SQ'] 

483 inDict['covariances'][ampName] = record['COVARIANCES'] 

484 inDict['covariancesModel'][ampName] = record['COVARIANCES_MODEL'] 

485 inDict['covariancesSqrtWeights'][ampName] = record['COVARIANCES_SQRT_WEIGHTS'] 

486 inDict['aMatrix'][ampName] = record['A_MATRIX'] 

487 inDict['bMatrix'][ampName] = record['B_MATRIX'] 

488 inDict['covariancesModelNoB'][ampName] = record['COVARIANCES_MODEL_NO_B'] 

489 inDict['aMatrixNoB'][ampName] = record['A_MATRIX_NO_B'] 

490 inDict['finalVars'][ampName] = record['FINAL_VARS'] 

491 inDict['finalModelVars'][ampName] = record['FINAL_MODEL_VARS'] 

492 inDict['finalMeans'][ampName] = record['FINAL_MEANS'] 

493 inDict['badAmps'] = record['BAD_AMPS'] 

494 inDict['photoCharge'][ampName] = record['PHOTO_CHARGE'] 

495 if calibVersion == 1.0: 

496 mask = record['FINAL_MEANS'].mask 

497 inDict['ptcTurnoff'][ampName] = record['FINAL_MEANS'][~mask][-1] 

498 else: 

499 inDict['ptcTurnoff'][ampName] = record['PTC_TURNOFF'] 

500 return cls().fromDict(inDict) 

501 

502 def toTable(self): 

503 """Construct a list of tables containing the information in this 

504 calibration. 

505 

506 The list of tables should create an identical calibration 

507 after being passed to this class's fromTable method. 

508 Returns 

509 ------- 

510 tableList : `list` [`astropy.table.Table`] 

511 List of tables containing the linearity calibration 

512 information. 

513 """ 

514 tableList = [] 

515 self.updateMetadata() 

516 nPoints = [] 

517 for i, ampName in enumerate(self.ampNames): 

518 nPoints.append(len(list(self.covariances.values())[i])) 

519 nSignalPoints = max(nPoints) 

520 nPadPoints = {} 

521 for i, ampName in enumerate(self.ampNames): 

522 nPadPoints[ampName] = nSignalPoints - len(list(self.covariances.values())[i]) 

523 covMatrixSide = self.covMatrixSide 

524 

525 catalog = Table([{'AMPLIFIER_NAME': ampName, 

526 'PTC_FIT_TYPE': self.ptcFitType, 

527 'COV_MATRIX_SIDE': self.covMatrixSide, 

528 'INPUT_EXP_ID_PAIRS': self.inputExpIdPairs[ampName] 

529 if len(self.expIdMask[ampName]) else np.nan, 

530 'EXP_ID_MASK': self.expIdMask[ampName] 

531 if len(self.expIdMask[ampName]) else np.nan, 

532 'RAW_EXP_TIMES': np.array(self.rawExpTimes[ampName]).tolist() 

533 if len(self.rawExpTimes[ampName]) else np.nan, 

534 'RAW_MEANS': np.array(self.rawMeans[ampName]).tolist() 

535 if len(self.rawMeans[ampName]) else np.nan, 

536 'RAW_VARS': np.array(self.rawVars[ampName]).tolist() 

537 if len(self.rawVars[ampName]) else np.nan, 

538 'GAIN': self.gain[ampName], 

539 'GAIN_ERR': self.gainErr[ampName], 

540 'NOISE': self.noise[ampName], 

541 'NOISE_ERR': self.noiseErr[ampName], 

542 'PTC_FIT_PARS': np.array(self.ptcFitPars[ampName]).tolist(), 

543 'PTC_FIT_PARS_ERROR': np.array(self.ptcFitParsError[ampName]).tolist(), 

544 'PTC_FIT_CHI_SQ': self.ptcFitChiSq[ampName], 

545 'PTC_TURNOFF': self.ptcTurnoff[ampName], 

546 'COVARIANCES': np.pad(np.array(self.covariances[ampName]), 

547 ((0, nPadPoints[ampName]), (0, 0), (0, 0)), 

548 'constant', constant_values=np.nan).reshape( 

549 nSignalPoints*covMatrixSide**2).tolist(), 

550 'COVARIANCES_MODEL': np.pad(np.array(self.covariancesModel[ampName]), 

551 ((0, nPadPoints[ampName]), (0, 0), (0, 0)), 

552 'constant', constant_values=np.nan).reshape( 

553 nSignalPoints*covMatrixSide**2).tolist(), 

554 'COVARIANCES_SQRT_WEIGHTS': np.pad(np.array(self.covariancesSqrtWeights[ampName]), 

555 ((0, nPadPoints[ampName]), (0, 0), (0, 0)), 

556 'constant', constant_values=0.0).reshape( 

557 nSignalPoints*covMatrixSide**2).tolist(), 

558 'A_MATRIX': np.array(self.aMatrix[ampName]).reshape(covMatrixSide**2).tolist(), 

559 'B_MATRIX': np.array(self.bMatrix[ampName]).reshape(covMatrixSide**2).tolist(), 

560 'COVARIANCES_MODEL_NO_B': 

561 np.pad(np.array(self.covariancesModelNoB[ampName]), 

562 ((0, nPadPoints[ampName]), (0, 0), (0, 0)), 

563 'constant', constant_values=np.nan).reshape( 

564 nSignalPoints*covMatrixSide**2).tolist(), 

565 'A_MATRIX_NO_B': np.array(self.aMatrixNoB[ampName]).reshape( 

566 covMatrixSide**2).tolist(), 

567 'FINAL_VARS': np.pad(np.array(self.finalVars[ampName]), (0, nPadPoints[ampName]), 

568 'constant', constant_values=np.nan).tolist(), 

569 'FINAL_MODEL_VARS': np.pad(np.array(self.finalModelVars[ampName]), 

570 (0, nPadPoints[ampName]), 

571 'constant', constant_values=np.nan).tolist(), 

572 'FINAL_MEANS': np.pad(np.array(self.finalMeans[ampName]), 

573 (0, nPadPoints[ampName]), 

574 'constant', constant_values=np.nan).tolist(), 

575 'BAD_AMPS': np.array(self.badAmps).tolist() if len(self.badAmps) else np.nan, 

576 'PHOTO_CHARGE': np.array(self.photoCharge[ampName]).tolist(), 

577 } for ampName in self.ampNames]) 

578 inMeta = self.getMetadata().toDict() 

579 outMeta = {k: v for k, v in inMeta.items() if v is not None} 

580 outMeta.update({k: "" for k, v in inMeta.items() if v is None}) 

581 catalog.meta = outMeta 

582 tableList.append(catalog) 

583 

584 return(tableList) 

585 

586 def getExpIdsUsed(self, ampName): 

587 """Get the exposures used, i.e. not discarded, for a given amp. 

588 If no mask has been created yet, all exposures are returned. 

589 """ 

590 if len(self.expIdMask[ampName]) == 0: 

591 return self.inputExpIdPairs[ampName] 

592 

593 # if the mask exists it had better be the same length as the expIdPairs 

594 assert len(self.expIdMask[ampName]) == len(self.inputExpIdPairs[ampName]) 

595 

596 pairs = self.inputExpIdPairs[ampName] 

597 mask = self.expIdMask[ampName] 

598 # cast to bool required because numpy 

599 return [(exp1, exp2) for ((exp1, exp2), m) in zip(pairs, mask) if bool(m) is True] 

600 

601 def getGoodAmps(self): 

602 return [amp for amp in self.ampNames if amp not in self.badAmps]