Coverage for python/lsst/ip/isr/ptcDataset.py: 6%

264 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2022-12-08 15:00 -0800

1# 

2# LSST Data Management System 

3# Copyright 2008-2017 AURA/LSST. 

4# 

5# This product includes software developed by the 

6# LSST Project (http://www.lsst.org/). 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <https://www.lsstcorp.org/LegalNotices/>. 

21# 

22""" 

23Define dataset class for MeasurePhotonTransferCurve task 

24""" 

25import numpy as np 

26from astropy.table import Table 

27 

28from lsst.ip.isr import IsrCalib 

29 

30__all__ = ['PhotonTransferCurveDataset'] 

31 

32 

33class PhotonTransferCurveDataset(IsrCalib): 

34 """A simple class to hold the output data from the PTC task. 

35 The dataset is made up of a dictionary for each item, keyed by the 

36 amplifiers' names, which much be supplied at construction time. 

37 New items cannot be added to the class to save accidentally saving to the 

38 wrong property, and the class can be frozen if desired. 

39 inputExpIdPairs records the exposures used to produce the data. 

40 When fitPtc() or fitCovariancesAstier() is run, a mask is built up, which 

41 is by definition always the same length as inputExpIdPairs, rawExpTimes, 

42 rawMeans and rawVars, and is a list of bools, which are incrementally set 

43 to False as points are discarded from the fits. 

44 PTC fit parameters for polynomials are stored in a list in ascending order 

45 of polynomial term, i.e. par[0]*x^0 + par[1]*x + par[2]*x^2 etc 

46 with the length of the list corresponding to the order of the polynomial 

47 plus one. 

48 

49 Parameters 

50 ---------- 

51 ampNames : `list` 

52 List with the names of the amplifiers of the detector at hand. 

53 

54 ptcFitType : `str` 

55 Type of model fitted to the PTC: "POLYNOMIAL", "EXPAPPROXIMATION", 

56 or "FULLCOVARIANCE". 

57 

58 covMatrixSide : `int` 

59 Maximum lag of covariances (size of square covariance matrices). 

60 

61 kwargs : `dict`, optional 

62 Other keyword arguments to pass to the parent init. 

63 

64 Notes 

65 ----- 

66 The stored attributes are: 

67 badAmps : `list` 

68 List with bad amplifiers names. 

69 inputExpIdPairs : `dict`, [`str`, `list`] 

70 Dictionary keyed by amp names containing the input exposures IDs. 

71 expIdMask : `dict`, [`str`, `list`] 

72 Dictionary keyed by amp names containing the mask produced after 

73 outlier rejection. The mask produced by the "FULLCOVARIANCE" 

74 option may differ from the one produced in the other two PTC 

75 fit types. 

76 rawExpTimes : `dict`, [`str`, `list`] 

77 Dictionary keyed by amp names containing the unmasked exposure times. 

78 rawMeans : `dict`, [`str`, `list`] 

79 Dictionary keyed by amp namescontaining the unmasked average of the 

80 means of the exposures in each flat pair. 

81 rawVars : `dict`, [`str`, `list`] 

82 Dictionary keyed by amp names containing the variance of the 

83 difference image of the exposures in each flat pair. 

84 gain : `dict`, [`str`, `list`] 

85 Dictionary keyed by amp names containing the fitted gains. 

86 gainErr : `dict`, [`str`, `list`] 

87 Dictionary keyed by amp names containing the errors on the 

88 fitted gains. 

89 noise : `dict`, [`str`, `list`] 

90 Dictionary keyed by amp names containing the fitted noise. 

91 noiseErr : `dict`, [`str`, `list`] 

92 Dictionary keyed by amp names containing the errors on the fitted 

93 noise. 

94 ptcFitPars : `dict`, [`str`, `list`] 

95 Dictionary keyed by amp names containing the fitted parameters of the 

96 PTC model for ptcFitTye in ["POLYNOMIAL", "EXPAPPROXIMATION"]. 

97 ptcFitParsError : `dict`, [`str`, `list`] 

98 Dictionary keyed by amp names containing the errors on the fitted 

99 parameters of the PTC model for ptcFitTye in 

100 ["POLYNOMIAL", "EXPAPPROXIMATION"]. 

101 ptcFitChiSq : `dict`, [`str`, `list`] 

102 Dictionary keyed by amp names containing the reduced chi squared 

103 of the fit for ptcFitTye in ["POLYNOMIAL", "EXPAPPROXIMATION"]. 

104 ptcTurnoff : `float` 

105 Flux value (in ADU) where the variance of the PTC curve starts 

106 decreasing consistently. 

107 covariances : `dict`, [`str`, `list`] 

108 Dictionary keyed by amp names containing a list of measured 

109 covariances per mean flux. 

110 covariancesModel : `dict`, [`str`, `list`] 

111 Dictionary keyed by amp names containinging covariances model 

112 (Eq. 20 of Astier+19) per mean flux. 

113 covariancesSqrtWeights : `dict`, [`str`, `list`] 

114 Dictionary keyed by amp names containinging sqrt. of covariances 

115 weights. 

116 aMatrix : `dict`, [`str`, `list`] 

117 Dictionary keyed by amp names containing the "a" parameters from 

118 the model in Eq. 20 of Astier+19. 

119 bMatrix : `dict`, [`str`, `list`] 

120 Dictionary keyed by amp names containing the "b" parameters from 

121 the model in Eq. 20 of Astier+19. 

122 covariancesModelNoB : `dict`, [`str`, `list`] 

123 Dictionary keyed by amp names containing covariances model 

124 (with 'b'=0 in Eq. 20 of Astier+19) 

125 per mean flux. 

126 aMatrixNoB : `dict`, [`str`, `list`] 

127 Dictionary keyed by amp names containing the "a" parameters from the 

128 model in Eq. 20 of Astier+19 

129 (and 'b' = 0). 

130 finalVars : `dict`, [`str`, `list`] 

131 Dictionary keyed by amp names containing the masked variance of the 

132 difference image of each flat 

133 pair. If needed, each array will be right-padded with 

134 np.nan to match the length of rawExpTimes. 

135 finalModelVars : `dict`, [`str`, `list`] 

136 Dictionary keyed by amp names containing the masked modeled 

137 variance of the difference image of each flat pair. If needed, each 

138 array will be right-padded with np.nan to match the length of 

139 rawExpTimes. 

140 finalMeans : `dict`, [`str`, `list`] 

141 Dictionary keyed by amp names containing the masked average of the 

142 means of the exposures in each flat pair. If needed, each array 

143 will be right-padded with np.nan to match the length of 

144 rawExpTimes. 

145 photoCharge : `dict`, [`str`, `list`] 

146 Dictionary keyed by amp names containing the integrated photocharge 

147 for linearity calibration. 

148 

149 Version 1.1 adds the `ptcTurnoff` attribute. 

150 """ 

151 

152 _OBSTYPE = 'PTC' 

153 _SCHEMA = 'Gen3 Photon Transfer Curve' 

154 _VERSION = 1.1 

155 

156 def __init__(self, ampNames=[], ptcFitType=None, covMatrixSide=1, **kwargs): 

157 

158 self.ptcFitType = ptcFitType 

159 self.ampNames = ampNames 

160 self.covMatrixSide = covMatrixSide 

161 

162 self.badAmps = [np.nan] 

163 

164 self.inputExpIdPairs = {ampName: [] for ampName in ampNames} 

165 self.expIdMask = {ampName: [] for ampName in ampNames} 

166 self.rawExpTimes = {ampName: [] for ampName in ampNames} 

167 self.rawMeans = {ampName: [] for ampName in ampNames} 

168 self.rawVars = {ampName: [] for ampName in ampNames} 

169 self.photoCharge = {ampName: [] for ampName in ampNames} 

170 

171 self.gain = {ampName: np.nan for ampName in ampNames} 

172 self.gainErr = {ampName: np.nan for ampName in ampNames} 

173 self.noise = {ampName: np.nan for ampName in ampNames} 

174 self.noiseErr = {ampName: np.nan for ampName in ampNames} 

175 

176 self.ptcFitPars = {ampName: [] for ampName in ampNames} 

177 self.ptcFitParsError = {ampName: [] for ampName in ampNames} 

178 self.ptcFitChiSq = {ampName: np.nan for ampName in ampNames} 

179 self.ptcTurnoff = {ampName: np.nan for ampName in ampNames} 

180 

181 self.covariances = {ampName: [] for ampName in ampNames} 

182 self.covariancesModel = {ampName: [] for ampName in ampNames} 

183 self.covariancesSqrtWeights = {ampName: [] for ampName in ampNames} 

184 self.aMatrix = {ampName: np.nan for ampName in ampNames} 

185 self.bMatrix = {ampName: np.nan for ampName in ampNames} 

186 self.covariancesModelNoB = {ampName: [] for ampName in ampNames} 

187 self.aMatrixNoB = {ampName: np.nan for ampName in ampNames} 

188 

189 self.finalVars = {ampName: [] for ampName in ampNames} 

190 self.finalModelVars = {ampName: [] for ampName in ampNames} 

191 self.finalMeans = {ampName: [] for ampName in ampNames} 

192 

193 super().__init__(**kwargs) 

194 self.requiredAttributes.update(['badAmps', 'inputExpIdPairs', 'expIdMask', 'rawExpTimes', 

195 'rawMeans', 'rawVars', 'gain', 'gainErr', 'noise', 'noiseErr', 

196 'ptcFitPars', 'ptcFitParsError', 'ptcFitChiSq', 'ptcTurnoff', 

197 'aMatrixNoB', 'covariances', 'covariancesModel', 

198 'covariancesSqrtWeights', 'covariancesModelNoB', 

199 'aMatrix', 'bMatrix', 'finalVars', 'finalModelVars', 'finalMeans', 

200 'photoCharge']) 

201 

202 def setAmpValues(self, ampName, inputExpIdPair=[(np.nan, np.nan)], expIdMask=[np.nan], 

203 rawExpTime=[np.nan], rawMean=[np.nan], rawVar=[np.nan], photoCharge=[np.nan], 

204 gain=np.nan, gainErr=np.nan, noise=np.nan, noiseErr=np.nan, ptcFitPars=[np.nan], 

205 ptcFitParsError=[np.nan], ptcFitChiSq=np.nan, ptcTurnoff=np.nan, covArray=[], 

206 covArrayModel=[], covSqrtWeights=[], aMatrix=[], bMatrix=[], covArrayModelNoB=[], 

207 aMatrixNoB=[], finalVar=[np.nan], finalModelVar=[np.nan], finalMean=[np.nan]): 

208 """Function to initialize an amp of a PhotonTransferCurveDataset. 

209 

210 Notes 

211 ----- 

212 The parameters are all documented in `init`. 

213 """ 

214 nanMatrix = np.full((self.covMatrixSide, self.covMatrixSide), np.nan) 

215 if len(covArray) == 0: 

216 covArray = [nanMatrix] 

217 if len(covArrayModel) == 0: 

218 covArrayModel = [nanMatrix] 

219 if len(covSqrtWeights) == 0: 

220 covSqrtWeights = [nanMatrix] 

221 if len(covArrayModelNoB) == 0: 

222 covArrayModelNoB = [nanMatrix] 

223 if len(aMatrix) == 0: 

224 aMatrix = nanMatrix 

225 if len(bMatrix) == 0: 

226 bMatrix = nanMatrix 

227 if len(aMatrixNoB) == 0: 

228 aMatrixNoB = nanMatrix 

229 

230 self.inputExpIdPairs[ampName] = inputExpIdPair 

231 self.expIdMask[ampName] = expIdMask 

232 self.rawExpTimes[ampName] = rawExpTime 

233 self.rawMeans[ampName] = rawMean 

234 self.rawVars[ampName] = rawVar 

235 self.photoCharge[ampName] = photoCharge 

236 self.gain[ampName] = gain 

237 self.gainErr[ampName] = gainErr 

238 self.noise[ampName] = noise 

239 self.noiseErr[ampName] = noiseErr 

240 self.ptcFitPars[ampName] = ptcFitPars 

241 self.ptcFitParsError[ampName] = ptcFitParsError 

242 self.ptcFitChiSq[ampName] = ptcFitChiSq 

243 self.ptcTurnoff[ampName] = ptcTurnoff 

244 self.covariances[ampName] = covArray 

245 self.covariancesSqrtWeights[ampName] = covSqrtWeights 

246 self.covariancesModel[ampName] = covArrayModel 

247 self.covariancesModelNoB[ampName] = covArrayModelNoB 

248 self.aMatrix[ampName] = aMatrix 

249 self.bMatrix[ampName] = bMatrix 

250 self.aMatrixNoB[ampName] = aMatrixNoB 

251 self.ptcFitPars[ampName] = ptcFitPars 

252 self.ptcFitParsError[ampName] = ptcFitParsError 

253 self.ptcFitChiSq[ampName] = ptcFitChiSq 

254 self.finalVars[ampName] = finalVar 

255 self.finalModelVars[ampName] = finalModelVar 

256 self.finalMeans[ampName] = finalMean 

257 

258 def updateMetadata(self, setDate=False, **kwargs): 

259 """Update calibration metadata. 

260 This calls the base class's method after ensuring the required 

261 calibration keywords will be saved. 

262 Parameters 

263 ---------- 

264 setDate : `bool`, optional 

265 Update the CALIBDATE fields in the metadata to the current 

266 time. Defaults to False. 

267 kwargs : 

268 Other keyword parameters to set in the metadata. 

269 """ 

270 kwargs['PTC_FIT_TYPE'] = self.ptcFitType 

271 

272 super().updateMetadata(setDate=setDate, **kwargs) 

273 

274 @classmethod 

275 def fromDict(cls, dictionary): 

276 """Construct a calibration from a dictionary of properties. 

277 Must be implemented by the specific calibration subclasses. 

278 Parameters 

279 ---------- 

280 dictionary : `dict` 

281 Dictionary of properties. 

282 Returns 

283 ------- 

284 calib : `lsst.ip.isr.CalibType` 

285 Constructed calibration. 

286 Raises 

287 ------ 

288 RuntimeError : 

289 Raised if the supplied dictionary is for a different 

290 calibration. 

291 """ 

292 calib = cls() 

293 if calib._OBSTYPE != dictionary['metadata']['OBSTYPE']: 

294 raise RuntimeError(f"Incorrect Photon Transfer Curve dataset supplied. " 

295 f"Expected {calib._OBSTYPE}, found {dictionary['metadata']['OBSTYPE']}") 

296 calib.setMetadata(dictionary['metadata']) 

297 calib.ptcFitType = dictionary['ptcFitType'] 

298 calib.covMatrixSide = dictionary['covMatrixSide'] 

299 calib.badAmps = np.array(dictionary['badAmps'], 'str').tolist() 

300 # The cov matrices are square 

301 covMatrixSide = calib.covMatrixSide 

302 # Number of final signal levels 

303 covDimensionsProduct = len(np.array(list(dictionary['covariances'].values())[0]).ravel()) 

304 nSignalPoints = int(covDimensionsProduct/(covMatrixSide*covMatrixSide)) 

305 

306 for ampName in dictionary['ampNames']: 

307 covsAmp = np.array(dictionary['covariances'][ampName]).reshape((nSignalPoints, covMatrixSide, 

308 covMatrixSide)) 

309 

310 # After cpPtcExtract runs in the PTC pipeline, the datasets 

311 # created ('PARTIAL' and 'DUMMY') have a single measurement. 

312 # Apply the maskign to the final ptcDataset, after running 

313 # cpPtcSolve. 

314 if len(covsAmp) > 1: 

315 # Masks for covariances padding in `toTable` 

316 maskCovsAmp = np.array([~np.isnan(entry).all() for entry in covsAmp]) 

317 maskAmp = ~np.isnan(np.array(dictionary['finalMeans'][ampName])) 

318 else: 

319 maskCovsAmp = np.array([True]) 

320 maskAmp = np.array([True]) 

321 

322 calib.ampNames.append(ampName) 

323 calib.inputExpIdPairs[ampName] = np.array(dictionary['inputExpIdPairs'][ampName]).tolist() 

324 calib.expIdMask[ampName] = np.array(dictionary['expIdMask'][ampName]).tolist() 

325 calib.rawExpTimes[ampName] = np.array(dictionary['rawExpTimes'][ampName]).tolist() 

326 calib.rawMeans[ampName] = np.array(dictionary['rawMeans'][ampName]).tolist() 

327 calib.rawVars[ampName] = np.array(dictionary['rawVars'][ampName]).tolist() 

328 calib.gain[ampName] = np.array(dictionary['gain'][ampName]).tolist() 

329 calib.gainErr[ampName] = np.array(dictionary['gainErr'][ampName]).tolist() 

330 calib.noise[ampName] = np.array(dictionary['noise'][ampName]).tolist() 

331 calib.noiseErr[ampName] = np.array(dictionary['noiseErr'][ampName]).tolist() 

332 calib.ptcFitPars[ampName] = np.array(dictionary['ptcFitPars'][ampName]).tolist() 

333 calib.ptcFitParsError[ampName] = np.array(dictionary['ptcFitParsError'][ampName]).tolist() 

334 calib.ptcFitChiSq[ampName] = np.array(dictionary['ptcFitChiSq'][ampName]).tolist() 

335 calib.ptcTurnoff[ampName] = np.array(dictionary['ptcTurnoff'][ampName]).tolist() 

336 calib.covariances[ampName] = covsAmp[maskCovsAmp].tolist() 

337 calib.covariancesModel[ampName] = np.array( 

338 dictionary['covariancesModel'][ampName]).reshape( 

339 (nSignalPoints, covMatrixSide, covMatrixSide))[maskCovsAmp].tolist() 

340 calib.covariancesSqrtWeights[ampName] = np.array( 

341 dictionary['covariancesSqrtWeights'][ampName]).reshape( 

342 (nSignalPoints, covMatrixSide, covMatrixSide))[maskCovsAmp].tolist() 

343 calib.aMatrix[ampName] = np.array(dictionary['aMatrix'][ampName]).reshape( 

344 (covMatrixSide, covMatrixSide)).tolist() 

345 calib.bMatrix[ampName] = np.array(dictionary['bMatrix'][ampName]).reshape( 

346 (covMatrixSide, covMatrixSide)).tolist() 

347 calib.covariancesModelNoB[ampName] = np.array( 

348 dictionary['covariancesModelNoB'][ampName]).reshape( 

349 (nSignalPoints, covMatrixSide, covMatrixSide))[maskCovsAmp].tolist() 

350 calib.aMatrixNoB[ampName] = np.array( 

351 dictionary['aMatrixNoB'][ampName]).reshape((covMatrixSide, covMatrixSide)).tolist() 

352 calib.finalVars[ampName] = np.array(dictionary['finalVars'][ampName])[maskAmp].tolist() 

353 calib.finalModelVars[ampName] = np.array(dictionary['finalModelVars'][ampName])[maskAmp].tolist() 

354 calib.finalMeans[ampName] = np.array(dictionary['finalMeans'][ampName])[maskAmp].tolist() 

355 calib.photoCharge[ampName] = np.array(dictionary['photoCharge'][ampName]).tolist() 

356 calib.updateMetadata() 

357 return calib 

358 

359 def toDict(self): 

360 """Return a dictionary containing the calibration properties. 

361 The dictionary should be able to be round-tripped through 

362 `fromDict`. 

363 Returns 

364 ------- 

365 dictionary : `dict` 

366 Dictionary of properties. 

367 """ 

368 self.updateMetadata() 

369 

370 outDict = dict() 

371 metadata = self.getMetadata() 

372 outDict['metadata'] = metadata 

373 

374 outDict['ptcFitType'] = self.ptcFitType 

375 outDict['covMatrixSide'] = self.covMatrixSide 

376 outDict['ampNames'] = self.ampNames 

377 outDict['badAmps'] = self.badAmps 

378 outDict['inputExpIdPairs'] = self.inputExpIdPairs 

379 outDict['expIdMask'] = self.expIdMask 

380 outDict['rawExpTimes'] = self.rawExpTimes 

381 outDict['rawMeans'] = self.rawMeans 

382 outDict['rawVars'] = self.rawVars 

383 outDict['gain'] = self.gain 

384 outDict['gainErr'] = self.gainErr 

385 outDict['noise'] = self.noise 

386 outDict['noiseErr'] = self.noiseErr 

387 outDict['ptcFitPars'] = self.ptcFitPars 

388 outDict['ptcFitParsError'] = self.ptcFitParsError 

389 outDict['ptcFitChiSq'] = self.ptcFitChiSq 

390 outDict['ptcTurnoff'] = self.ptcTurnoff 

391 outDict['covariances'] = self.covariances 

392 outDict['covariancesModel'] = self.covariancesModel 

393 outDict['covariancesSqrtWeights'] = self.covariancesSqrtWeights 

394 outDict['aMatrix'] = self.aMatrix 

395 outDict['bMatrix'] = self.bMatrix 

396 outDict['covariancesModelNoB'] = self.covariancesModelNoB 

397 outDict['aMatrixNoB'] = self.aMatrixNoB 

398 outDict['finalVars'] = self.finalVars 

399 outDict['finalModelVars'] = self.finalModelVars 

400 outDict['finalMeans'] = self.finalMeans 

401 outDict['photoCharge'] = self.photoCharge 

402 

403 return outDict 

404 

405 @classmethod 

406 def fromTable(cls, tableList): 

407 """Construct calibration from a list of tables. 

408 This method uses the `fromDict` method to create the 

409 calibration, after constructing an appropriate dictionary from 

410 the input tables. 

411 Parameters 

412 ---------- 

413 tableList : `list` [`lsst.afw.table.Table`] 

414 List of tables to use to construct the datasetPtc. 

415 Returns 

416 ------- 

417 calib : `lsst.cp.pipe.` 

418 The calibration defined in the tables. 

419 """ 

420 ptcTable = tableList[0] 

421 

422 metadata = ptcTable.meta 

423 inDict = dict() 

424 inDict['metadata'] = metadata 

425 inDict['ampNames'] = [] 

426 inDict['ptcFitType'] = [] 

427 inDict['covMatrixSide'] = [] 

428 inDict['inputExpIdPairs'] = dict() 

429 inDict['expIdMask'] = dict() 

430 inDict['rawExpTimes'] = dict() 

431 inDict['rawMeans'] = dict() 

432 inDict['rawVars'] = dict() 

433 inDict['gain'] = dict() 

434 inDict['gainErr'] = dict() 

435 inDict['noise'] = dict() 

436 inDict['noiseErr'] = dict() 

437 inDict['ptcFitPars'] = dict() 

438 inDict['ptcFitParsError'] = dict() 

439 inDict['ptcFitChiSq'] = dict() 

440 inDict['ptcTurnoff'] = dict() 

441 inDict['covariances'] = dict() 

442 inDict['covariancesModel'] = dict() 

443 inDict['covariancesSqrtWeights'] = dict() 

444 inDict['aMatrix'] = dict() 

445 inDict['bMatrix'] = dict() 

446 inDict['covariancesModelNoB'] = dict() 

447 inDict['aMatrixNoB'] = dict() 

448 inDict['finalVars'] = dict() 

449 inDict['finalModelVars'] = dict() 

450 inDict['finalMeans'] = dict() 

451 inDict['badAmps'] = [] 

452 inDict['photoCharge'] = dict() 

453 

454 calibVersion = metadata['PTC_VERSION'] 

455 if calibVersion == 1.0: 

456 cls().log.warning(f"Previous version found for PTC dataset: {calibVersion}. " 

457 f"Setting 'ptcTurnoff' in all amps to last value in 'finalMeans'.") 

458 for record in ptcTable: 

459 ampName = record['AMPLIFIER_NAME'] 

460 

461 inDict['ptcFitType'] = record['PTC_FIT_TYPE'] 

462 inDict['covMatrixSide'] = record['COV_MATRIX_SIDE'] 

463 inDict['ampNames'].append(ampName) 

464 inDict['inputExpIdPairs'][ampName] = record['INPUT_EXP_ID_PAIRS'] 

465 inDict['expIdMask'][ampName] = record['EXP_ID_MASK'] 

466 inDict['rawExpTimes'][ampName] = record['RAW_EXP_TIMES'] 

467 inDict['rawMeans'][ampName] = record['RAW_MEANS'] 

468 inDict['rawVars'][ampName] = record['RAW_VARS'] 

469 inDict['gain'][ampName] = record['GAIN'] 

470 inDict['gainErr'][ampName] = record['GAIN_ERR'] 

471 inDict['noise'][ampName] = record['NOISE'] 

472 inDict['noiseErr'][ampName] = record['NOISE_ERR'] 

473 inDict['ptcFitPars'][ampName] = record['PTC_FIT_PARS'] 

474 inDict['ptcFitParsError'][ampName] = record['PTC_FIT_PARS_ERROR'] 

475 inDict['ptcFitChiSq'][ampName] = record['PTC_FIT_CHI_SQ'] 

476 inDict['covariances'][ampName] = record['COVARIANCES'] 

477 inDict['covariancesModel'][ampName] = record['COVARIANCES_MODEL'] 

478 inDict['covariancesSqrtWeights'][ampName] = record['COVARIANCES_SQRT_WEIGHTS'] 

479 inDict['aMatrix'][ampName] = record['A_MATRIX'] 

480 inDict['bMatrix'][ampName] = record['B_MATRIX'] 

481 inDict['covariancesModelNoB'][ampName] = record['COVARIANCES_MODEL_NO_B'] 

482 inDict['aMatrixNoB'][ampName] = record['A_MATRIX_NO_B'] 

483 inDict['finalVars'][ampName] = record['FINAL_VARS'] 

484 inDict['finalModelVars'][ampName] = record['FINAL_MODEL_VARS'] 

485 inDict['finalMeans'][ampName] = record['FINAL_MEANS'] 

486 inDict['badAmps'] = record['BAD_AMPS'] 

487 inDict['photoCharge'][ampName] = record['PHOTO_CHARGE'] 

488 if calibVersion == 1.0: 

489 mask = record['FINAL_MEANS'].mask 

490 inDict['ptcTurnoff'][ampName] = record['FINAL_MEANS'][~mask][-1] 

491 else: 

492 inDict['ptcTurnoff'][ampName] = record['PTC_TURNOFF'] 

493 return cls().fromDict(inDict) 

494 

495 def toTable(self): 

496 """Construct a list of tables containing the information in this 

497 calibration. 

498 

499 The list of tables should create an identical calibration 

500 after being passed to this class's fromTable method. 

501 Returns 

502 ------- 

503 tableList : `list` [`astropy.table.Table`] 

504 List of tables containing the linearity calibration 

505 information. 

506 """ 

507 tableList = [] 

508 self.updateMetadata() 

509 nPoints = [] 

510 for i, ampName in enumerate(self.ampNames): 

511 nPoints.append(len(list(self.covariances.values())[i])) 

512 nSignalPoints = max(nPoints) 

513 nPadPoints = {} 

514 for i, ampName in enumerate(self.ampNames): 

515 nPadPoints[ampName] = nSignalPoints - len(list(self.covariances.values())[i]) 

516 covMatrixSide = self.covMatrixSide 

517 

518 catalog = Table([{'AMPLIFIER_NAME': ampName, 

519 'PTC_FIT_TYPE': self.ptcFitType, 

520 'COV_MATRIX_SIDE': self.covMatrixSide, 

521 'INPUT_EXP_ID_PAIRS': self.inputExpIdPairs[ampName] 

522 if len(self.expIdMask[ampName]) else np.nan, 

523 'EXP_ID_MASK': self.expIdMask[ampName] 

524 if len(self.expIdMask[ampName]) else np.nan, 

525 'RAW_EXP_TIMES': np.array(self.rawExpTimes[ampName]).tolist() 

526 if len(self.rawExpTimes[ampName]) else np.nan, 

527 'RAW_MEANS': np.array(self.rawMeans[ampName]).tolist() 

528 if len(self.rawMeans[ampName]) else np.nan, 

529 'RAW_VARS': np.array(self.rawVars[ampName]).tolist() 

530 if len(self.rawVars[ampName]) else np.nan, 

531 'GAIN': self.gain[ampName], 

532 'GAIN_ERR': self.gainErr[ampName], 

533 'NOISE': self.noise[ampName], 

534 'NOISE_ERR': self.noiseErr[ampName], 

535 'PTC_FIT_PARS': np.array(self.ptcFitPars[ampName]).tolist(), 

536 'PTC_FIT_PARS_ERROR': np.array(self.ptcFitParsError[ampName]).tolist(), 

537 'PTC_FIT_CHI_SQ': self.ptcFitChiSq[ampName], 

538 'PTC_TURNOFF': self.ptcTurnoff[ampName], 

539 'COVARIANCES': np.pad(np.array(self.covariances[ampName]), 

540 ((0, nPadPoints[ampName]), (0, 0), (0, 0)), 

541 'constant', constant_values=np.nan).reshape( 

542 nSignalPoints*covMatrixSide**2).tolist(), 

543 'COVARIANCES_MODEL': np.pad(np.array(self.covariancesModel[ampName]), 

544 ((0, nPadPoints[ampName]), (0, 0), (0, 0)), 

545 'constant', constant_values=np.nan).reshape( 

546 nSignalPoints*covMatrixSide**2).tolist(), 

547 'COVARIANCES_SQRT_WEIGHTS': np.pad(np.array(self.covariancesSqrtWeights[ampName]), 

548 ((0, nPadPoints[ampName]), (0, 0), (0, 0)), 

549 'constant', constant_values=0.0).reshape( 

550 nSignalPoints*covMatrixSide**2).tolist(), 

551 'A_MATRIX': np.array(self.aMatrix[ampName]).reshape(covMatrixSide**2).tolist(), 

552 'B_MATRIX': np.array(self.bMatrix[ampName]).reshape(covMatrixSide**2).tolist(), 

553 'COVARIANCES_MODEL_NO_B': 

554 np.pad(np.array(self.covariancesModelNoB[ampName]), 

555 ((0, nPadPoints[ampName]), (0, 0), (0, 0)), 

556 'constant', constant_values=np.nan).reshape( 

557 nSignalPoints*covMatrixSide**2).tolist(), 

558 'A_MATRIX_NO_B': np.array(self.aMatrixNoB[ampName]).reshape( 

559 covMatrixSide**2).tolist(), 

560 'FINAL_VARS': np.pad(np.array(self.finalVars[ampName]), (0, nPadPoints[ampName]), 

561 'constant', constant_values=np.nan).tolist(), 

562 'FINAL_MODEL_VARS': np.pad(np.array(self.finalModelVars[ampName]), 

563 (0, nPadPoints[ampName]), 

564 'constant', constant_values=np.nan).tolist(), 

565 'FINAL_MEANS': np.pad(np.array(self.finalMeans[ampName]), 

566 (0, nPadPoints[ampName]), 

567 'constant', constant_values=np.nan).tolist(), 

568 'BAD_AMPS': np.array(self.badAmps).tolist() if len(self.badAmps) else np.nan, 

569 'PHOTO_CHARGE': np.array(self.photoCharge[ampName]).tolist(), 

570 } for ampName in self.ampNames]) 

571 inMeta = self.getMetadata().toDict() 

572 outMeta = {k: v for k, v in inMeta.items() if v is not None} 

573 outMeta.update({k: "" for k, v in inMeta.items() if v is None}) 

574 catalog.meta = outMeta 

575 tableList.append(catalog) 

576 

577 return(tableList) 

578 

579 def getExpIdsUsed(self, ampName): 

580 """Get the exposures used, i.e. not discarded, for a given amp. 

581 If no mask has been created yet, all exposures are returned. 

582 """ 

583 if len(self.expIdMask[ampName]) == 0: 

584 return self.inputExpIdPairs[ampName] 

585 

586 # if the mask exists it had better be the same length as the expIdPairs 

587 assert len(self.expIdMask[ampName]) == len(self.inputExpIdPairs[ampName]) 

588 

589 pairs = self.inputExpIdPairs[ampName] 

590 mask = self.expIdMask[ampName] 

591 # cast to bool required because numpy 

592 return [(exp1, exp2) for ((exp1, exp2), m) in zip(pairs, mask) if bool(m) is True] 

593 

594 def getGoodAmps(self): 

595 return [amp for amp in self.ampNames if amp not in self.badAmps]