Coverage for python/lsst/ip/isr/ptcDataset.py: 6%

305 statements  

« prev     ^ index     » next       coverage.py v7.2.7, created at 2023-06-26 08:34 +0000

1# 

2# LSST Data Management System 

3# Copyright 2008-2017 AURA/LSST. 

4# 

5# This product includes software developed by the 

6# LSST Project (http://www.lsst.org/). 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <https://www.lsstcorp.org/LegalNotices/>. 

21# 

22""" 

23Define dataset class for MeasurePhotonTransferCurve task 

24""" 

25 

26__all__ = ['PhotonTransferCurveDataset'] 

27 

28import numpy as np 

29from astropy.table import Table 

30import warnings 

31 

32from lsst.ip.isr import IsrCalib 

33 

34 

35class PhotonTransferCurveDataset(IsrCalib): 

36 """A simple class to hold the output data from the PTC task. 

37 

38 The dataset is made up of a dictionary for each item, keyed by the 

39 amplifiers' names, which much be supplied at construction time. 

40 New items cannot be added to the class to save accidentally saving to the 

41 wrong property, and the class can be frozen if desired. 

42 inputExpIdPairs records the exposures used to produce the data. 

43 When fitPtc() or fitCovariancesAstier() is run, a mask is built up, which 

44 is by definition always the same length as inputExpIdPairs, rawExpTimes, 

45 rawMeans and rawVars, and is a list of bools, which are incrementally set 

46 to False as points are discarded from the fits. 

47 PTC fit parameters for polynomials are stored in a list in ascending order 

48 of polynomial term, i.e. par[0]*x^0 + par[1]*x + par[2]*x^2 etc 

49 with the length of the list corresponding to the order of the polynomial 

50 plus one. 

51 

52 Parameters 

53 ---------- 

54 ampNames : `list` 

55 List with the names of the amplifiers of the detector at hand. 

56 ptcFitType : `str` 

57 Type of model fitted to the PTC: "POLYNOMIAL", "EXPAPPROXIMATION", 

58 or "FULLCOVARIANCE". 

59 covMatrixSide : `int` 

60 Maximum lag of covariances (size of square covariance matrices). 

61 kwargs : `dict`, optional 

62 Other keyword arguments to pass to the parent init. 

63 

64 Notes 

65 ----- 

66 The stored attributes are: 

67 

68 badAmps : `list` [`str`] 

69 List with bad amplifiers names. 

70 inputExpIdPairs : `dict`, [`str`, `list`] 

71 Dictionary keyed by amp names containing the input exposures IDs. 

72 expIdMask : `dict`, [`str`, `np.ndarray`] 

73 Dictionary keyed by amp names containing the mask produced after 

74 outlier rejection. The mask produced by the "FULLCOVARIANCE" 

75 option may differ from the one produced in the other two PTC 

76 fit types. 

77 rawExpTimes : `dict`, [`str`, `np.ndarray`] 

78 Dictionary keyed by amp names containing the unmasked exposure times. 

79 rawMeans : `dict`, [`str`, `np.ndarray`] 

80 Dictionary keyed by amp names containing the unmasked average of the 

81 means of the exposures in each flat pair. 

82 rawVars : `dict`, [`str`, `np.ndarray`] 

83 Dictionary keyed by amp names containing the variance of the 

84 difference image of the exposures in each flat pair. 

85 histVars : `dict`, [`str`, `np.ndarray`] 

86 Dictionary keyed by amp names containing the variance of the 

87 difference image of the exposures in each flat pair estimated 

88 by fitting a Gaussian model. 

89 histChi2Dofs : `dict`, [`str`, `np.ndarray`] 

90 Dictionary keyed by amp names containing the chi-squared per degree 

91 of freedom fitting the difference image to a Gaussian model. 

92 kspValues : `dict`, [`str`, `np.ndarray`] 

93 Dictionary keyed by amp names containing the KS test p-value from 

94 fitting the difference image to a Gaussian model. 

95 gain : `dict`, [`str`, `float`] 

96 Dictionary keyed by amp names containing the fitted gains. 

97 gainErr : `dict`, [`str`, `float`] 

98 Dictionary keyed by amp names containing the errors on the 

99 fitted gains. 

100 noise : `dict`, [`str`, `float`] 

101 Dictionary keyed by amp names containing the fitted noise. 

102 noiseErr : `dict`, [`str`, `float`] 

103 Dictionary keyed by amp names containing the errors on the fitted 

104 noise. 

105 ptcFitPars : `dict`, [`str`, `np.ndarray`] 

106 Dictionary keyed by amp names containing the fitted parameters of the 

107 PTC model for ptcFitTye in ["POLYNOMIAL", "EXPAPPROXIMATION"]. 

108 ptcFitParsError : `dict`, [`str`, `np.ndarray`] 

109 Dictionary keyed by amp names containing the errors on the fitted 

110 parameters of the PTC model for ptcFitTye in 

111 ["POLYNOMIAL", "EXPAPPROXIMATION"]. 

112 ptcFitChiSq : `dict`, [`str`, `float`] 

113 Dictionary keyed by amp names containing the reduced chi squared 

114 of the fit for ptcFitTye in ["POLYNOMIAL", "EXPAPPROXIMATION"]. 

115 ptcTurnoff : `dict` [`str, `float`] 

116 Flux value (in ADU) where the variance of the PTC curve starts 

117 decreasing consistently. 

118 covariances : `dict`, [`str`, `np.ndarray`] 

119 Dictionary keyed by amp names containing a list of measured 

120 covariances per mean flux. 

121 covariancesModel : `dict`, [`str`, `np.ndarray`] 

122 Dictionary keyed by amp names containinging covariances model 

123 (Eq. 20 of Astier+19) per mean flux. 

124 covariancesSqrtWeights : `dict`, [`str`, `np.ndarray`] 

125 Dictionary keyed by amp names containinging sqrt. of covariances 

126 weights. 

127 aMatrix : `dict`, [`str`, `np.ndarray`] 

128 Dictionary keyed by amp names containing the "a" parameters from 

129 the model in Eq. 20 of Astier+19. 

130 bMatrix : `dict`, [`str`, `np.ndarray`] 

131 Dictionary keyed by amp names containing the "b" parameters from 

132 the model in Eq. 20 of Astier+19. 

133 noiseMatrix : `dict`, [`str`, `np.ndarray`] 

134 Dictionary keyed by amp names containing the "noise" parameters from 

135 the model in Eq. 20 of Astier+19. 

136 covariancesModelNoB : `dict`, [`str`, `np.ndarray`] 

137 Dictionary keyed by amp names containing covariances model 

138 (with 'b'=0 in Eq. 20 of Astier+19) 

139 per mean flux. 

140 aMatrixNoB : `dict`, [`str`, `np.ndarray`] 

141 Dictionary keyed by amp names containing the "a" parameters from the 

142 model in Eq. 20 of Astier+19 

143 (and 'b' = 0). 

144 noiseMatrixNoB : `dict`, [`str`, `np.ndarray`] 

145 Dictionary keyed by amp names containing the "noise" parameters from 

146 the model in Eq. 20 of Astier+19, with 'b' = 0. 

147 finalVars : `dict`, [`str`, `np.ndarray`] 

148 Dictionary keyed by amp names containing the masked variance of the 

149 difference image of each flat 

150 pair. If needed, each array will be right-padded with 

151 np.nan to match the length of rawExpTimes. 

152 finalModelVars : `dict`, [`str`, `np.ndarray`] 

153 Dictionary keyed by amp names containing the masked modeled 

154 variance of the difference image of each flat pair. If needed, each 

155 array will be right-padded with np.nan to match the length of 

156 rawExpTimes. 

157 finalMeans : `dict`, [`str`, `np.ndarray`] 

158 Dictionary keyed by amp names containing the masked average of the 

159 means of the exposures in each flat pair. If needed, each array 

160 will be right-padded with np.nan to match the length of 

161 rawExpTimes. 

162 photoCharges : `dict`, [`str`, `np.ndarray`] 

163 Dictionary keyed by amp names containing the integrated photocharge 

164 for linearity calibration. 

165 

166 Version 1.1 adds the `ptcTurnoff` attribute. 

167 Version 1.2 adds the `histVars`, `histChi2Dofs`, and `kspValues` 

168 attributes. 

169 Version 1.3 adds the `noiseMatrix` and `noiseMatrixNoB` attributes. 

170 """ 

171 

172 _OBSTYPE = 'PTC' 

173 _SCHEMA = 'Gen3 Photon Transfer Curve' 

174 _VERSION = 1.3 

175 

176 def __init__(self, ampNames=[], ptcFitType=None, covMatrixSide=1, **kwargs): 

177 self.ptcFitType = ptcFitType 

178 self.ampNames = ampNames 

179 self.covMatrixSide = covMatrixSide 

180 

181 self.badAmps = [] 

182 

183 self.inputExpIdPairs = {ampName: [] for ampName in ampNames} 

184 self.expIdMask = {ampName: np.array([], dtype=bool) for ampName in ampNames} 

185 self.rawExpTimes = {ampName: np.array([]) for ampName in ampNames} 

186 self.rawMeans = {ampName: np.array([]) for ampName in ampNames} 

187 self.rawVars = {ampName: np.array([]) for ampName in ampNames} 

188 self.photoCharges = {ampName: np.array([]) for ampName in ampNames} 

189 

190 self.gain = {ampName: np.nan for ampName in ampNames} 

191 self.gainErr = {ampName: np.nan for ampName in ampNames} 

192 self.noise = {ampName: np.nan for ampName in ampNames} 

193 self.noiseErr = {ampName: np.nan for ampName in ampNames} 

194 

195 self.histVars = {ampName: np.array([]) for ampName in ampNames} 

196 self.histChi2Dofs = {ampName: np.array([]) for ampName in ampNames} 

197 self.kspValues = {ampName: np.array([]) for ampName in ampNames} 

198 

199 self.ptcFitPars = {ampName: np.array([]) for ampName in ampNames} 

200 self.ptcFitParsError = {ampName: np.array([]) for ampName in ampNames} 

201 self.ptcFitChiSq = {ampName: np.nan for ampName in ampNames} 

202 self.ptcTurnoff = {ampName: np.nan for ampName in ampNames} 

203 

204 self.covariances = {ampName: np.array([]) for ampName in ampNames} 

205 self.covariancesModel = {ampName: np.array([]) for ampName in ampNames} 

206 self.covariancesSqrtWeights = {ampName: np.array([]) for ampName in ampNames} 

207 self.aMatrix = {ampName: np.array([]) for ampName in ampNames} 

208 self.bMatrix = {ampName: np.array([]) for ampName in ampNames} 

209 self.noiseMatrix = {ampName: np.array([]) for ampName in ampNames} 

210 self.covariancesModelNoB = {ampName: np.array([]) for ampName in ampNames} 

211 self.aMatrixNoB = {ampName: np.array([]) for ampName in ampNames} 

212 self.noiseMatrixNoB = {ampName: np.array([]) for ampName in ampNames} 

213 

214 self.finalVars = {ampName: np.array([]) for ampName in ampNames} 

215 self.finalModelVars = {ampName: np.array([]) for ampName in ampNames} 

216 self.finalMeans = {ampName: np.array([]) for ampName in ampNames} 

217 

218 super().__init__(**kwargs) 

219 self.requiredAttributes.update(['badAmps', 'inputExpIdPairs', 'expIdMask', 'rawExpTimes', 

220 'rawMeans', 'rawVars', 'gain', 'gainErr', 'noise', 'noiseErr', 

221 'ptcFitPars', 'ptcFitParsError', 'ptcFitChiSq', 'ptcTurnoff', 

222 'aMatrixNoB', 'covariances', 'covariancesModel', 

223 'covariancesSqrtWeights', 'covariancesModelNoB', 

224 'aMatrix', 'bMatrix', 'noiseMatrix', 'noiseMatrixNoB', 'finalVars', 

225 'finalModelVars', 'finalMeans', 'photoCharges', 'histVars', 

226 'histChi2Dofs', 'kspValues']) 

227 

228 self.updateMetadata(setCalibInfo=True, setCalibId=True, **kwargs) 

229 

230 def setAmpValuesPartialDataset( 

231 self, 

232 ampName, 

233 inputExpIdPair=(-1, -1), 

234 rawExpTime=np.nan, 

235 rawMean=np.nan, 

236 rawVar=np.nan, 

237 photoCharge=np.nan, 

238 expIdMask=False, 

239 covariance=None, 

240 covSqrtWeights=None, 

241 gain=np.nan, 

242 noise=np.nan, 

243 histVar=np.nan, 

244 histChi2Dof=np.nan, 

245 kspValue=0.0, 

246 ): 

247 """ 

248 Set the amp values for a partial PTC Dataset (from cpExtractPtcTask). 

249 

250 Parameters 

251 ---------- 

252 ampName : `str` 

253 Name of the amp to set the values. 

254 inputExpIdPair : `tuple` [`int`] 

255 Exposure IDs of input pair. 

256 rawExpTime : `float`, optional 

257 Exposure time for this exposure pair. 

258 rawMean : `float`, optional 

259 Average of the means of the exposures in this pair. 

260 rawVar : `float`, optional 

261 Variance of the difference of the exposures in this pair. 

262 photoCharge : `float`, optional 

263 Integrated photocharge for flat pair for linearity calibration. 

264 expIdMask : `bool`, optional 

265 Flag setting if this exposure pair should be used (True) 

266 or not used (False). 

267 covariance : `np.ndarray` or None, optional 

268 Measured covariance for this exposure pair. 

269 covSqrtWeights : `np.ndarray` or None, optional 

270 Measured sqrt of covariance weights in this exposure pair. 

271 gain : `float`, optional 

272 Estimated gain for this exposure pair. 

273 noise : `float`, optional 

274 Estimated read noise for this exposure pair. 

275 histVar : `float`, optional 

276 Variance estimated from fitting a histogram with a Gaussian model. 

277 histChi2Dof : `float`, optional 

278 Chi-squared per degree of freedom from Gaussian histogram fit. 

279 kspValue : `float`, optional 

280 KS test p-value from the Gaussian histogram fit. 

281 """ 

282 nanMatrix = np.full((self.covMatrixSide, self.covMatrixSide), np.nan) 

283 if covariance is None: 

284 covariance = nanMatrix 

285 if covSqrtWeights is None: 

286 covSqrtWeights = nanMatrix 

287 

288 self.inputExpIdPairs[ampName] = [inputExpIdPair] 

289 self.rawExpTimes[ampName] = np.array([rawExpTime]) 

290 self.rawMeans[ampName] = np.array([rawMean]) 

291 self.rawVars[ampName] = np.array([rawVar]) 

292 self.photoCharges[ampName] = np.array([photoCharge]) 

293 self.expIdMask[ampName] = np.array([expIdMask]) 

294 self.covariances[ampName] = np.array([covariance]) 

295 self.covariancesSqrtWeights[ampName] = np.array([covSqrtWeights]) 

296 self.gain[ampName] = gain 

297 self.noise[ampName] = noise 

298 self.histVars[ampName] = np.array([histVar]) 

299 self.histChi2Dofs[ampName] = np.array([histChi2Dof]) 

300 self.kspValues[ampName] = np.array([kspValue]) 

301 

302 self.covariancesModel[ampName] = np.array([nanMatrix]) 

303 self.covariancesModelNoB[ampName] = np.array([nanMatrix]) 

304 self.aMatrix[ampName] = nanMatrix 

305 self.bMatrix[ampName] = nanMatrix 

306 self.aMatrixNoB[ampName] = nanMatrix 

307 self.noiseMatrix[ampName] = nanMatrix 

308 self.noiseMatrixNoB[ampName] = nanMatrix 

309 

310 def updateMetadata(self, **kwargs): 

311 """Update calibration metadata. 

312 This calls the base class's method after ensuring the required 

313 calibration keywords will be saved. 

314 

315 Parameters 

316 ---------- 

317 setDate : `bool`, optional 

318 Update the CALIBDATE fields in the metadata to the current 

319 time. Defaults to False. 

320 kwargs : 

321 Other keyword parameters to set in the metadata. 

322 """ 

323 super().updateMetadata(PTC_FIT_TYPE=self.ptcFitType, **kwargs) 

324 

325 @classmethod 

326 def fromDict(cls, dictionary): 

327 """Construct a calibration from a dictionary of properties. 

328 Must be implemented by the specific calibration subclasses. 

329 

330 Parameters 

331 ---------- 

332 dictionary : `dict` 

333 Dictionary of properties. 

334 

335 Returns 

336 ------- 

337 calib : `lsst.ip.isr.PhotonTransferCurveDataset` 

338 Constructed calibration. 

339 

340 Raises 

341 ------ 

342 RuntimeError 

343 Raised if the supplied dictionary is for a different 

344 calibration. 

345 """ 

346 calib = cls() 

347 if calib._OBSTYPE != dictionary['metadata']['OBSTYPE']: 

348 raise RuntimeError(f"Incorrect Photon Transfer Curve dataset supplied. " 

349 f"Expected {calib._OBSTYPE}, found {dictionary['metadata']['OBSTYPE']}") 

350 calib.setMetadata(dictionary['metadata']) 

351 calib.ptcFitType = dictionary['ptcFitType'] 

352 calib.covMatrixSide = dictionary['covMatrixSide'] 

353 calib.badAmps = np.array(dictionary['badAmps'], 'str').tolist() 

354 calib.ampNames = [] 

355 

356 # The cov matrices are square 

357 covMatrixSide = calib.covMatrixSide 

358 # Number of final signal levels 

359 covDimensionsProduct = len(np.array(list(dictionary['covariances'].values())[0]).ravel()) 

360 nSignalPoints = int(covDimensionsProduct/(covMatrixSide*covMatrixSide)) 

361 

362 for ampName in dictionary['ampNames']: 

363 calib.ampNames.append(ampName) 

364 calib.inputExpIdPairs[ampName] = dictionary['inputExpIdPairs'][ampName] 

365 calib.expIdMask[ampName] = np.array(dictionary['expIdMask'][ampName]) 

366 calib.rawExpTimes[ampName] = np.array(dictionary['rawExpTimes'][ampName], dtype=np.float64) 

367 calib.rawMeans[ampName] = np.array(dictionary['rawMeans'][ampName], dtype=np.float64) 

368 calib.rawVars[ampName] = np.array(dictionary['rawVars'][ampName], dtype=np.float64) 

369 calib.gain[ampName] = float(dictionary['gain'][ampName]) 

370 calib.gainErr[ampName] = float(dictionary['gainErr'][ampName]) 

371 calib.noise[ampName] = float(dictionary['noise'][ampName]) 

372 calib.noiseErr[ampName] = float(dictionary['noiseErr'][ampName]) 

373 calib.histVars[ampName] = np.array(dictionary['histVars'][ampName], dtype=np.float64) 

374 calib.histChi2Dofs[ampName] = np.array(dictionary['histChi2Dofs'][ampName], dtype=np.float64) 

375 calib.kspValues[ampName] = np.array(dictionary['kspValues'][ampName], dtype=np.float64) 

376 calib.ptcFitPars[ampName] = np.array(dictionary['ptcFitPars'][ampName], dtype=np.float64) 

377 calib.ptcFitParsError[ampName] = np.array(dictionary['ptcFitParsError'][ampName], 

378 dtype=np.float64) 

379 calib.ptcFitChiSq[ampName] = float(dictionary['ptcFitChiSq'][ampName]) 

380 calib.ptcTurnoff[ampName] = float(dictionary['ptcTurnoff'][ampName]) 

381 if nSignalPoints > 0: 

382 # Regular dataset 

383 calib.covariances[ampName] = np.array(dictionary['covariances'][ampName], 

384 dtype=np.float64).reshape( 

385 (nSignalPoints, covMatrixSide, covMatrixSide)) 

386 calib.covariancesModel[ampName] = np.array( 

387 dictionary['covariancesModel'][ampName], 

388 dtype=np.float64).reshape( 

389 (nSignalPoints, covMatrixSide, covMatrixSide)) 

390 calib.covariancesSqrtWeights[ampName] = np.array( 

391 dictionary['covariancesSqrtWeights'][ampName], 

392 dtype=np.float64).reshape( 

393 (nSignalPoints, covMatrixSide, covMatrixSide)) 

394 calib.aMatrix[ampName] = np.array(dictionary['aMatrix'][ampName], 

395 dtype=np.float64).reshape( 

396 (covMatrixSide, covMatrixSide)) 

397 calib.bMatrix[ampName] = np.array(dictionary['bMatrix'][ampName], 

398 dtype=np.float64).reshape( 

399 (covMatrixSide, covMatrixSide)) 

400 calib.covariancesModelNoB[ampName] = np.array( 

401 dictionary['covariancesModelNoB'][ampName], dtype=np.float64).reshape( 

402 (nSignalPoints, covMatrixSide, covMatrixSide)) 

403 calib.aMatrixNoB[ampName] = np.array( 

404 dictionary['aMatrixNoB'][ampName], 

405 dtype=np.float64).reshape((covMatrixSide, covMatrixSide)) 

406 calib.noiseMatrix[ampName] = np.array( 

407 dictionary['noiseMatrix'][ampName], 

408 dtype=np.float64).reshape((covMatrixSide, covMatrixSide)) 

409 calib.noiseMatrixNoB[ampName] = np.array( 

410 dictionary['noiseMatrixNoB'][ampName], 

411 dtype=np.float64).reshape((covMatrixSide, covMatrixSide)) 

412 else: 

413 # Empty dataset 

414 calib.covariances[ampName] = np.array([], dtype=np.float64) 

415 calib.covariancesModel[ampName] = np.array([], dtype=np.float64) 

416 calib.covariancesSqrtWeights[ampName] = np.array([], dtype=np.float64) 

417 calib.aMatrix[ampName] = np.array([], dtype=np.float64) 

418 calib.bMatrix[ampName] = np.array([], dtype=np.float64) 

419 calib.covariancesModelNoB[ampName] = np.array([], dtype=np.float64) 

420 calib.aMatrixNoB[ampName] = np.array([], dtype=np.float64) 

421 calib.noiseMatrix[ampName] = np.array([], dtype=np.float64) 

422 calib.noiseMatrixNoB[ampName] = np.array([], dtype=np.float64) 

423 

424 calib.finalVars[ampName] = np.array(dictionary['finalVars'][ampName], dtype=np.float64) 

425 calib.finalModelVars[ampName] = np.array(dictionary['finalModelVars'][ampName], dtype=np.float64) 

426 calib.finalMeans[ampName] = np.array(dictionary['finalMeans'][ampName], dtype=np.float64) 

427 calib.photoCharges[ampName] = np.array(dictionary['photoCharges'][ampName], dtype=np.float64) 

428 

429 calib.updateMetadata() 

430 return calib 

431 

432 def toDict(self): 

433 """Return a dictionary containing the calibration properties. 

434 The dictionary should be able to be round-tripped through 

435 `fromDict`. 

436 

437 Returns 

438 ------- 

439 dictionary : `dict` 

440 Dictionary of properties. 

441 """ 

442 self.updateMetadata() 

443 

444 outDict = dict() 

445 metadata = self.getMetadata() 

446 outDict['metadata'] = metadata 

447 

448 def _dictOfArraysToDictOfLists(dictOfArrays): 

449 dictOfLists = {} 

450 for key, value in dictOfArrays.items(): 

451 dictOfLists[key] = value.ravel().tolist() 

452 

453 return dictOfLists 

454 

455 outDict['ptcFitType'] = self.ptcFitType 

456 outDict['covMatrixSide'] = self.covMatrixSide 

457 outDict['ampNames'] = self.ampNames 

458 outDict['badAmps'] = self.badAmps 

459 outDict['inputExpIdPairs'] = self.inputExpIdPairs 

460 outDict['expIdMask'] = _dictOfArraysToDictOfLists(self.expIdMask) 

461 outDict['rawExpTimes'] = _dictOfArraysToDictOfLists(self.rawExpTimes) 

462 outDict['rawMeans'] = _dictOfArraysToDictOfLists(self.rawMeans) 

463 outDict['rawVars'] = _dictOfArraysToDictOfLists(self.rawVars) 

464 outDict['gain'] = self.gain 

465 outDict['gainErr'] = self.gainErr 

466 outDict['noise'] = self.noise 

467 outDict['noiseErr'] = self.noiseErr 

468 outDict['histVars'] = self.histVars 

469 outDict['histChi2Dofs'] = self.histChi2Dofs 

470 outDict['kspValues'] = self.kspValues 

471 outDict['ptcFitPars'] = _dictOfArraysToDictOfLists(self.ptcFitPars) 

472 outDict['ptcFitParsError'] = _dictOfArraysToDictOfLists(self.ptcFitParsError) 

473 outDict['ptcFitChiSq'] = self.ptcFitChiSq 

474 outDict['ptcTurnoff'] = self.ptcTurnoff 

475 outDict['covariances'] = _dictOfArraysToDictOfLists(self.covariances) 

476 outDict['covariancesModel'] = _dictOfArraysToDictOfLists(self.covariancesModel) 

477 outDict['covariancesSqrtWeights'] = _dictOfArraysToDictOfLists(self.covariancesSqrtWeights) 

478 outDict['aMatrix'] = _dictOfArraysToDictOfLists(self.aMatrix) 

479 outDict['bMatrix'] = _dictOfArraysToDictOfLists(self.bMatrix) 

480 outDict['noiseMatrix'] = _dictOfArraysToDictOfLists(self.noiseMatrix) 

481 outDict['covariancesModelNoB'] = _dictOfArraysToDictOfLists(self.covariancesModelNoB) 

482 outDict['aMatrixNoB'] = _dictOfArraysToDictOfLists(self.aMatrixNoB) 

483 outDict['noiseMatrixNoB'] = _dictOfArraysToDictOfLists(self.noiseMatrixNoB) 

484 outDict['finalVars'] = _dictOfArraysToDictOfLists(self.finalVars) 

485 outDict['finalModelVars'] = _dictOfArraysToDictOfLists(self.finalModelVars) 

486 outDict['finalMeans'] = _dictOfArraysToDictOfLists(self.finalMeans) 

487 outDict['photoCharges'] = _dictOfArraysToDictOfLists(self.photoCharges) 

488 

489 return outDict 

490 

491 @classmethod 

492 def fromTable(cls, tableList): 

493 """Construct calibration from a list of tables. 

494 This method uses the `fromDict` method to create the 

495 calibration, after constructing an appropriate dictionary from 

496 the input tables. 

497 

498 Parameters 

499 ---------- 

500 tableList : `list` [`lsst.afw.table.Table`] 

501 List of tables to use to construct the datasetPtc. 

502 

503 Returns 

504 ------- 

505 calib : `lsst.ip.isr.PhotonTransferCurveDataset` 

506 The calibration defined in the tables. 

507 """ 

508 ptcTable = tableList[0] 

509 

510 metadata = ptcTable.meta 

511 inDict = dict() 

512 inDict['metadata'] = metadata 

513 inDict['ampNames'] = [] 

514 inDict['ptcFitType'] = [] 

515 inDict['covMatrixSide'] = [] 

516 inDict['inputExpIdPairs'] = dict() 

517 inDict['expIdMask'] = dict() 

518 inDict['rawExpTimes'] = dict() 

519 inDict['rawMeans'] = dict() 

520 inDict['rawVars'] = dict() 

521 inDict['gain'] = dict() 

522 inDict['gainErr'] = dict() 

523 inDict['noise'] = dict() 

524 inDict['noiseErr'] = dict() 

525 inDict['histVars'] = dict() 

526 inDict['histChi2Dofs'] = dict() 

527 inDict['kspValues'] = dict() 

528 inDict['ptcFitPars'] = dict() 

529 inDict['ptcFitParsError'] = dict() 

530 inDict['ptcFitChiSq'] = dict() 

531 inDict['ptcTurnoff'] = dict() 

532 inDict['covariances'] = dict() 

533 inDict['covariancesModel'] = dict() 

534 inDict['covariancesSqrtWeights'] = dict() 

535 inDict['aMatrix'] = dict() 

536 inDict['bMatrix'] = dict() 

537 inDict['noiseMatrix'] = dict() 

538 inDict['covariancesModelNoB'] = dict() 

539 inDict['aMatrixNoB'] = dict() 

540 inDict['noiseMatrixNoB'] = dict() 

541 inDict['finalVars'] = dict() 

542 inDict['finalModelVars'] = dict() 

543 inDict['finalMeans'] = dict() 

544 inDict['badAmps'] = [] 

545 inDict['photoCharges'] = dict() 

546 

547 calibVersion = metadata['PTC_VERSION'] 

548 if calibVersion == 1.0: 

549 cls().log.warning(f"Previous version found for PTC dataset: {calibVersion}. " 

550 f"Setting 'ptcTurnoff' in all amps to last value in 'finalMeans'.") 

551 for record in ptcTable: 

552 ampName = record['AMPLIFIER_NAME'] 

553 

554 inDict['ptcFitType'] = record['PTC_FIT_TYPE'] 

555 inDict['covMatrixSide'] = record['COV_MATRIX_SIDE'] 

556 inDict['ampNames'].append(ampName) 

557 inDict['inputExpIdPairs'][ampName] = record['INPUT_EXP_ID_PAIRS'].tolist() 

558 inDict['expIdMask'][ampName] = record['EXP_ID_MASK'] 

559 inDict['rawExpTimes'][ampName] = record['RAW_EXP_TIMES'] 

560 inDict['rawMeans'][ampName] = record['RAW_MEANS'] 

561 inDict['rawVars'][ampName] = record['RAW_VARS'] 

562 inDict['gain'][ampName] = record['GAIN'] 

563 inDict['gainErr'][ampName] = record['GAIN_ERR'] 

564 inDict['noise'][ampName] = record['NOISE'] 

565 inDict['noiseErr'][ampName] = record['NOISE_ERR'] 

566 inDict['ptcFitPars'][ampName] = record['PTC_FIT_PARS'] 

567 inDict['ptcFitParsError'][ampName] = record['PTC_FIT_PARS_ERROR'] 

568 inDict['ptcFitChiSq'][ampName] = record['PTC_FIT_CHI_SQ'] 

569 inDict['covariances'][ampName] = record['COVARIANCES'] 

570 inDict['covariancesModel'][ampName] = record['COVARIANCES_MODEL'] 

571 inDict['covariancesSqrtWeights'][ampName] = record['COVARIANCES_SQRT_WEIGHTS'] 

572 inDict['aMatrix'][ampName] = record['A_MATRIX'] 

573 inDict['bMatrix'][ampName] = record['B_MATRIX'] 

574 inDict['noiseMatrix'][ampName] = record['NOISE_MATRIX'] 

575 inDict['covariancesModelNoB'][ampName] = record['COVARIANCES_MODEL_NO_B'] 

576 inDict['aMatrixNoB'][ampName] = record['A_MATRIX_NO_B'] 

577 inDict['noiseMatrixNoB'][ampName] = record['NOISE_MATRIX_NO_B'] 

578 inDict['finalVars'][ampName] = record['FINAL_VARS'] 

579 inDict['finalModelVars'][ampName] = record['FINAL_MODEL_VARS'] 

580 inDict['finalMeans'][ampName] = record['FINAL_MEANS'] 

581 inDict['badAmps'] = record['BAD_AMPS'].tolist() 

582 inDict['photoCharges'][ampName] = record['PHOTO_CHARGE'] 

583 if calibVersion == 1.0: 

584 mask = record['FINAL_MEANS'].mask 

585 array = record['FINAL_MEANS'][~mask] 

586 if len(array) > 0: 

587 inDict['ptcTurnoff'][ampName] = record['FINAL_MEANS'][~mask][-1] 

588 else: 

589 inDict['ptcTurnoff'][ampName] = np.nan 

590 else: 

591 inDict['ptcTurnoff'][ampName] = record['PTC_TURNOFF'] 

592 if calibVersion < 1.2: 

593 inDict['histVars'][ampName] = np.array([np.nan]) 

594 inDict['histChi2Dofs'][ampName] = np.array([np.nan]) 

595 inDict['kspValues'][ampName] = np.array([0.0]) 

596 else: 

597 inDict['histVars'][ampName] = record['HIST_VARS'] 

598 inDict['histChi2Dofs'][ampName] = record['HIST_CHI2_DOFS'] 

599 inDict['kspValues'][ampName] = record['KS_PVALUES'] 

600 if calibVersion < 1.3: 

601 nanMatrix = np.full_like(inDict['aMatrix'][ampName], np.nan) 

602 inDict['noiseMatrix'][ampName] = nanMatrix 

603 inDict['noiseMatrixNoB'][ampName] = nanMatrix 

604 else: 

605 inDict['noiseMatrix'][ampName] = record['NOISE_MATRIX'] 

606 inDict['noiseMatrixNoB'][ampName] = record['NOISE_MATRIX_NO_B'] 

607 

608 return cls().fromDict(inDict) 

609 

610 def toTable(self): 

611 """Construct a list of tables containing the information in this 

612 calibration. 

613 

614 The list of tables should create an identical calibration 

615 after being passed to this class's fromTable method. 

616 

617 Returns 

618 ------- 

619 tableList : `list` [`astropy.table.Table`] 

620 List of tables containing the linearity calibration 

621 information. 

622 """ 

623 tableList = [] 

624 self.updateMetadata() 

625 

626 badAmps = np.array(self.badAmps) if len(self.badAmps) else np.array([], dtype="U3") 

627 

628 catalogList = [] 

629 for ampName in self.ampNames: 

630 ampDict = { 

631 'AMPLIFIER_NAME': ampName, 

632 'PTC_FIT_TYPE': self.ptcFitType, 

633 'COV_MATRIX_SIDE': self.covMatrixSide, 

634 'INPUT_EXP_ID_PAIRS': self.inputExpIdPairs[ampName], 

635 'EXP_ID_MASK': self.expIdMask[ampName], 

636 'RAW_EXP_TIMES': self.rawExpTimes[ampName], 

637 'RAW_MEANS': self.rawMeans[ampName], 

638 'RAW_VARS': self.rawVars[ampName], 

639 'GAIN': self.gain[ampName], 

640 'GAIN_ERR': self.gainErr[ampName], 

641 'NOISE': self.noise[ampName], 

642 'NOISE_ERR': self.noiseErr[ampName], 

643 'HIST_VARS': self.histVars[ampName], 

644 'HIST_CHI2_DOFS': self.histChi2Dofs[ampName], 

645 'KS_PVALUES': self.kspValues[ampName], 

646 'PTC_FIT_PARS': np.array(self.ptcFitPars[ampName]), 

647 'PTC_FIT_PARS_ERROR': np.array(self.ptcFitParsError[ampName]), 

648 'PTC_FIT_CHI_SQ': self.ptcFitChiSq[ampName], 

649 'PTC_TURNOFF': self.ptcTurnoff[ampName], 

650 'A_MATRIX': self.aMatrix[ampName].ravel(), 

651 'B_MATRIX': self.bMatrix[ampName].ravel(), 

652 'A_MATRIX_NO_B': self.aMatrixNoB[ampName].ravel(), 

653 'NOISE_MATRIX': self.noiseMatrix[ampName].ravel(), 

654 'NOISE_MATRIX_NO_B': self.noiseMatrixNoB[ampName].ravel(), 

655 'BAD_AMPS': badAmps, 

656 'PHOTO_CHARGE': self.photoCharges[ampName], 

657 'COVARIANCES': self.covariances[ampName].ravel(), 

658 'COVARIANCES_MODEL': self.covariancesModel[ampName].ravel(), 

659 'COVARIANCES_SQRT_WEIGHTS': self.covariancesSqrtWeights[ampName].ravel(), 

660 'COVARIANCES_MODEL_NO_B': self.covariancesModelNoB[ampName].ravel(), 

661 'FINAL_VARS': self.finalVars[ampName], 

662 'FINAL_MODEL_VARS': self.finalModelVars[ampName], 

663 'FINAL_MEANS': self.finalMeans[ampName], 

664 } 

665 catalogList.append(ampDict) 

666 

667 catalog = Table(catalogList) 

668 

669 inMeta = self.getMetadata().toDict() 

670 outMeta = {k: v for k, v in inMeta.items() if v is not None} 

671 outMeta.update({k: "" for k, v in inMeta.items() if v is None}) 

672 catalog.meta = outMeta 

673 tableList.append(catalog) 

674 

675 return(tableList) 

676 

677 def fromDetector(self, detector): 

678 """Read metadata parameters from a detector. 

679 

680 Parameters 

681 ---------- 

682 detector : `lsst.afw.cameraGeom.detector` 

683 Input detector with parameters to use. 

684 

685 Returns 

686 ------- 

687 calib : `lsst.ip.isr.PhotonTransferCurveDataset` 

688 The calibration constructed from the detector. 

689 """ 

690 

691 pass 

692 

693 def getExpIdsUsed(self, ampName): 

694 """Get the exposures used, i.e. not discarded, for a given amp. 

695 If no mask has been created yet, all exposures are returned. 

696 

697 Parameters 

698 ---------- 

699 ampName : `str` 

700 

701 Returns 

702 ------- 

703 expIdsUsed : `list` [`tuple`] 

704 List of pairs of exposure ids used in PTC. 

705 """ 

706 if len(self.expIdMask[ampName]) == 0: 

707 return self.inputExpIdPairs[ampName] 

708 

709 # if the mask exists it had better be the same length as the expIdPairs 

710 assert len(self.expIdMask[ampName]) == len(self.inputExpIdPairs[ampName]) 

711 

712 pairs = self.inputExpIdPairs[ampName] 

713 mask = self.expIdMask[ampName] 

714 # cast to bool required because numpy 

715 try: 

716 expIdsUsed = [(exp1, exp2) for ((exp1, exp2), m) in zip(pairs, mask) if m] 

717 except ValueError: 

718 warnings.warn("The PTC file was written incorrectly; you should rerun the " 

719 "PTC solve task if possible.", RuntimeWarning) 

720 expIdsUsed = [] 

721 for pairList, m in zip(pairs, mask): 

722 if m: 

723 expIdsUsed.append(pairList[0]) 

724 

725 return expIdsUsed 

726 

727 def getGoodAmps(self): 

728 """Get the good amps from this PTC.""" 

729 return [amp for amp in self.ampNames if amp not in self.badAmps] 

730 

731 def getGoodPoints(self, ampName): 

732 """Get the good points used for a given amp in the PTC. 

733 

734 Parameters 

735 ---------- 

736 ampName : `str` 

737 

738 Returns 

739 ------- 

740 goodPoints : `np.ndarray` 

741 Boolean array of good points used in PTC. 

742 """ 

743 return self.expIdMask[ampName]