Coverage for python/lsst/ip/isr/ptcDataset.py: 6%

376 statements  

« prev     ^ index     » next       coverage.py v7.5.0, created at 2024-05-01 04:11 -0700

1# 

2# LSST Data Management System 

3# Copyright 2008-2017 AURA/LSST. 

4# 

5# This product includes software developed by the 

6# LSST Project (http://www.lsst.org/). 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <https://www.lsstcorp.org/LegalNotices/>. 

21# 

22""" 

23Define dataset class for MeasurePhotonTransferCurve task 

24""" 

25 

26__all__ = ['PhotonTransferCurveDataset'] 

27 

28import numpy as np 

29import math 

30from astropy.table import Table 

31 

32from lsst.ip.isr import IsrCalib 

33 

34 

35class PhotonTransferCurveDataset(IsrCalib): 

36 """A simple class to hold the output data from the PTC task. 

37 

38 The dataset is made up of a dictionary for each item, keyed by the 

39 amplifiers' names, which much be supplied at construction time. 

40 New items cannot be added to the class to save accidentally saving to the 

41 wrong property, and the class can be frozen if desired. 

42 inputExpIdPairs records the exposures used to produce the data. 

43 When fitPtc() or fitCovariancesAstier() is run, a mask is built up, which 

44 is by definition always the same length as inputExpIdPairs, rawExpTimes, 

45 rawMeans and rawVars, and is a list of bools, which are incrementally set 

46 to False as points are discarded from the fits. 

47 PTC fit parameters for polynomials are stored in a list in ascending order 

48 of polynomial term, i.e. par[0]*x^0 + par[1]*x + par[2]*x^2 etc 

49 with the length of the list corresponding to the order of the polynomial 

50 plus one. 

51 

52 Parameters 

53 ---------- 

54 ampNames : `list` 

55 List with the names of the amplifiers of the detector at hand. 

56 ptcFitType : `str`, optional 

57 Type of model fitted to the PTC: "POLYNOMIAL", "EXPAPPROXIMATION", 

58 or "FULLCOVARIANCE". 

59 covMatrixSide : `int`, optional 

60 Maximum lag of measured covariances (size of square covariance 

61 matrices). 

62 covMatrixSideFullCovFit : `int`, optional 

63 Maximum covariances lag for FULLCOVARIANCE fit. It should be less or 

64 equal than covMatrixSide. 

65 kwargs : `dict`, optional 

66 Other keyword arguments to pass to the parent init. 

67 

68 Notes 

69 ----- 

70 The stored attributes are: 

71 

72 badAmps : `list` [`str`] 

73 List with bad amplifiers names. 

74 inputExpIdPairs : `dict`, [`str`, `list`] 

75 Dictionary keyed by amp names containing the input exposures IDs. 

76 expIdMask : `dict`, [`str`, `np.ndarray`] 

77 Dictionary keyed by amp names containing the mask produced after 

78 outlier rejection. The mask produced by the "FULLCOVARIANCE" 

79 option may differ from the one produced in the other two PTC 

80 fit types. 

81 rawExpTimes : `dict`, [`str`, `np.ndarray`] 

82 Dictionary keyed by amp names containing the unmasked exposure times. 

83 rawMeans : `dict`, [`str`, `np.ndarray`] 

84 Dictionary keyed by amp names containing the unmasked average of the 

85 means of the exposures in each flat pair. 

86 rawVars : `dict`, [`str`, `np.ndarray`] 

87 Dictionary keyed by amp names containing the variance of the 

88 difference image of the exposures in each flat pair. 

89 rowMeanVariance : `dict`, [`str`, `np.ndarray`] 

90 Dictionary keyed by amp names containing the variance of the 

91 means of the rows of the difference image of the exposures 

92 in each flat pair. 

93 histVars : `dict`, [`str`, `np.ndarray`] 

94 Dictionary keyed by amp names containing the variance of the 

95 difference image of the exposures in each flat pair estimated 

96 by fitting a Gaussian model. 

97 histChi2Dofs : `dict`, [`str`, `np.ndarray`] 

98 Dictionary keyed by amp names containing the chi-squared per degree 

99 of freedom fitting the difference image to a Gaussian model. 

100 kspValues : `dict`, [`str`, `np.ndarray`] 

101 Dictionary keyed by amp names containing the KS test p-value from 

102 fitting the difference image to a Gaussian model. 

103 gain : `dict`, [`str`, `float`] 

104 Dictionary keyed by amp names containing the fitted gains. 

105 gainErr : `dict`, [`str`, `float`] 

106 Dictionary keyed by amp names containing the errors on the 

107 fitted gains. 

108 noiseList : `dict`, [`str`, `np.ndarray`] 

109 Dictionary keyed by amp names containing the mean read noise from 

110 each flat pair (as measured from overscan). 

111 noise : `dict`, [`str`, `float`] 

112 Dictionary keyed by amp names containing the fitted noise. 

113 noiseErr : `dict`, [`str`, `float`] 

114 Dictionary keyed by amp names containing the errors on the fitted 

115 noise. 

116 ptcFitPars : `dict`, [`str`, `np.ndarray`] 

117 Dictionary keyed by amp names containing the fitted parameters of the 

118 PTC model for ptcFitTye in ["POLYNOMIAL", "EXPAPPROXIMATION"]. 

119 ptcFitParsError : `dict`, [`str`, `np.ndarray`] 

120 Dictionary keyed by amp names containing the errors on the fitted 

121 parameters of the PTC model for ptcFitTye in 

122 ["POLYNOMIAL", "EXPAPPROXIMATION"]. 

123 ptcFitChiSq : `dict`, [`str`, `float`] 

124 Dictionary keyed by amp names containing the reduced chi squared 

125 of the fit for ptcFitTye in ["POLYNOMIAL", "EXPAPPROXIMATION"]. 

126 ptcTurnoff : `dict` [`str, `float`] 

127 Flux value (in ADU) where the variance of the PTC curve starts 

128 decreasing consistently. 

129 ptcTurnoffSamplingError : `dict` [`str`, `float`] 

130 ``Sampling`` error on the ptcTurnoff, based on the flux sampling 

131 of the input PTC. 

132 covariances : `dict`, [`str`, `np.ndarray`] 

133 Dictionary keyed by amp names containing a list of measured 

134 covariances per mean flux. 

135 covariancesModel : `dict`, [`str`, `np.ndarray`] 

136 Dictionary keyed by amp names containinging covariances model 

137 (Eq. 20 of Astier+19) per mean flux. 

138 covariancesSqrtWeights : `dict`, [`str`, `np.ndarray`] 

139 Dictionary keyed by amp names containinging sqrt. of covariances 

140 weights. 

141 aMatrix : `dict`, [`str`, `np.ndarray`] 

142 Dictionary keyed by amp names containing the "a" parameters from 

143 the model in Eq. 20 of Astier+19. 

144 bMatrix : `dict`, [`str`, `np.ndarray`] 

145 Dictionary keyed by amp names containing the "b" parameters from 

146 the model in Eq. 20 of Astier+19. 

147 noiseMatrix : `dict`, [`str`, `np.ndarray`] 

148 Dictionary keyed by amp names containing the "noise" parameters from 

149 the model in Eq. 20 of Astier+19. 

150 covariancesModelNoB : `dict`, [`str`, `np.ndarray`] 

151 Dictionary keyed by amp names containing covariances model 

152 (with 'b'=0 in Eq. 20 of Astier+19) 

153 per mean flux. 

154 aMatrixNoB : `dict`, [`str`, `np.ndarray`] 

155 Dictionary keyed by amp names containing the "a" parameters from the 

156 model in Eq. 20 of Astier+19 

157 (and 'b' = 0). 

158 noiseMatrixNoB : `dict`, [`str`, `np.ndarray`] 

159 Dictionary keyed by amp names containing the "noise" parameters from 

160 the model in Eq. 20 of Astier+19, with 'b' = 0. 

161 finalVars : `dict`, [`str`, `np.ndarray`] 

162 Dictionary keyed by amp names containing the masked variance of the 

163 difference image of each flat 

164 pair. If needed, each array will be right-padded with 

165 np.nan to match the length of rawExpTimes. 

166 finalModelVars : `dict`, [`str`, `np.ndarray`] 

167 Dictionary keyed by amp names containing the masked modeled 

168 variance of the difference image of each flat pair. If needed, each 

169 array will be right-padded with np.nan to match the length of 

170 rawExpTimes. 

171 finalMeans : `dict`, [`str`, `np.ndarray`] 

172 Dictionary keyed by amp names containing the masked average of the 

173 means of the exposures in each flat pair. If needed, each array 

174 will be right-padded with np.nan to match the length of 

175 rawExpTimes. 

176 photoCharges : `dict`, [`str`, `np.ndarray`] 

177 Dictionary keyed by amp names containing the integrated photocharge 

178 for linearity calibration. 

179 auxValues : `dict`, [`str`, `np.ndarray`] 

180 Dictionary of per-detector auxiliary header values that can be used 

181 for PTC, linearity computation. 

182 

183 Version 1.1 adds the `ptcTurnoff` attribute. 

184 Version 1.2 adds the `histVars`, `histChi2Dofs`, and `kspValues` 

185 attributes. 

186 Version 1.3 adds the `noiseMatrix` and `noiseMatrixNoB` attributes. 

187 Version 1.4 adds the `auxValues` attribute. 

188 Version 1.5 adds the `covMatrixSideFullCovFit` attribute. 

189 Version 1.6 adds the `rowMeanVariance` attribute. 

190 Version 1.7 adds the `noiseList` attribute. 

191 Version 1.8 adds the `ptcTurnoffSamplingError` attribute. 

192 """ 

193 

194 _OBSTYPE = 'PTC' 

195 _SCHEMA = 'Gen3 Photon Transfer Curve' 

196 _VERSION = 1.8 

197 

198 def __init__(self, ampNames=[], ptcFitType=None, covMatrixSide=1, 

199 covMatrixSideFullCovFit=None, **kwargs): 

200 self.ptcFitType = ptcFitType 

201 self.ampNames = ampNames 

202 self.covMatrixSide = covMatrixSide 

203 if covMatrixSideFullCovFit is None: 

204 self.covMatrixSideFullCovFit = covMatrixSide 

205 else: 

206 self.covMatrixSideFullCovFit = covMatrixSideFullCovFit 

207 

208 self.badAmps = [] 

209 

210 self.inputExpIdPairs = {ampName: [] for ampName in ampNames} 

211 self.expIdMask = {ampName: np.array([], dtype=bool) for ampName in ampNames} 

212 self.rawExpTimes = {ampName: np.array([]) for ampName in ampNames} 

213 self.rawMeans = {ampName: np.array([]) for ampName in ampNames} 

214 self.rawVars = {ampName: np.array([]) for ampName in ampNames} 

215 self.rowMeanVariance = {ampName: np.array([]) for ampName in ampNames} 

216 self.photoCharges = {ampName: np.array([]) for ampName in ampNames} 

217 

218 self.gain = {ampName: np.nan for ampName in ampNames} 

219 self.gainErr = {ampName: np.nan for ampName in ampNames} 

220 self.noiseList = {ampName: np.array([]) for ampName in ampNames} 

221 self.noise = {ampName: np.nan for ampName in ampNames} 

222 self.noiseErr = {ampName: np.nan for ampName in ampNames} 

223 

224 self.histVars = {ampName: np.array([]) for ampName in ampNames} 

225 self.histChi2Dofs = {ampName: np.array([]) for ampName in ampNames} 

226 self.kspValues = {ampName: np.array([]) for ampName in ampNames} 

227 

228 self.ptcFitPars = {ampName: np.array([]) for ampName in ampNames} 

229 self.ptcFitParsError = {ampName: np.array([]) for ampName in ampNames} 

230 self.ptcFitChiSq = {ampName: np.nan for ampName in ampNames} 

231 self.ptcTurnoff = {ampName: np.nan for ampName in ampNames} 

232 self.ptcTurnoffSamplingError = {ampName: np.nan for ampName in ampNames} 

233 

234 self.covariances = {ampName: np.array([]) for ampName in ampNames} 

235 self.covariancesModel = {ampName: np.array([]) for ampName in ampNames} 

236 self.covariancesSqrtWeights = {ampName: np.array([]) for ampName in ampNames} 

237 self.aMatrix = {ampName: np.array([]) for ampName in ampNames} 

238 self.bMatrix = {ampName: np.array([]) for ampName in ampNames} 

239 self.noiseMatrix = {ampName: np.array([]) for ampName in ampNames} 

240 self.covariancesModelNoB = {ampName: np.array([]) for ampName in ampNames} 

241 self.aMatrixNoB = {ampName: np.array([]) for ampName in ampNames} 

242 self.noiseMatrixNoB = {ampName: np.array([]) for ampName in ampNames} 

243 

244 self.finalVars = {ampName: np.array([]) for ampName in ampNames} 

245 self.finalModelVars = {ampName: np.array([]) for ampName in ampNames} 

246 self.finalMeans = {ampName: np.array([]) for ampName in ampNames} 

247 

248 # Try this as a dict of arrays. 

249 self.auxValues = {} 

250 

251 super().__init__(**kwargs) 

252 self.requiredAttributes.update(['badAmps', 'inputExpIdPairs', 'expIdMask', 'rawExpTimes', 

253 'rawMeans', 'rawVars', 'rowMeanVariance', 'gain', 

254 'gainErr', 'noise', 'noiseErr', 'noiseList', 

255 'ptcFitPars', 'ptcFitParsError', 'ptcFitChiSq', 'ptcTurnoff', 

256 'aMatrixNoB', 'covariances', 'covariancesModel', 

257 'covariancesSqrtWeights', 'covariancesModelNoB', 

258 'aMatrix', 'bMatrix', 'noiseMatrix', 'noiseMatrixNoB', 'finalVars', 

259 'finalModelVars', 'finalMeans', 'photoCharges', 'histVars', 

260 'histChi2Dofs', 'kspValues', 'auxValues', 'ptcTurnoffSamplingError']) 

261 

262 self.updateMetadata(setCalibInfo=True, setCalibId=True, **kwargs) 

263 self._validateCovarianceMatrizSizes() 

264 

265 def setAmpValuesPartialDataset( 

266 self, 

267 ampName, 

268 inputExpIdPair=(-1, -1), 

269 rawExpTime=np.nan, 

270 rawMean=np.nan, 

271 rawVar=np.nan, 

272 rowMeanVariance=np.nan, 

273 photoCharge=np.nan, 

274 expIdMask=False, 

275 covariance=None, 

276 covSqrtWeights=None, 

277 gain=np.nan, 

278 noise=np.nan, 

279 histVar=np.nan, 

280 histChi2Dof=np.nan, 

281 kspValue=0.0, 

282 auxValues=None, 

283 ): 

284 """ 

285 Set the amp values for a partial PTC Dataset (from cpExtractPtcTask). 

286 

287 Parameters 

288 ---------- 

289 ampName : `str` 

290 Name of the amp to set the values. 

291 inputExpIdPair : `tuple` [`int`] 

292 Exposure IDs of input pair. 

293 rawExpTime : `float`, optional 

294 Exposure time for this exposure pair. 

295 rawMean : `float`, optional 

296 Average of the means of the exposures in this pair. 

297 rawVar : `float`, optional 

298 Variance of the difference of the exposures in this pair. 

299 rowMeanVariance : `float`, optional 

300 Variance of the means of the rows in the difference image 

301 of the exposures in this pair. 

302 photoCharge : `float`, optional 

303 Integrated photocharge for flat pair for linearity calibration. 

304 expIdMask : `bool`, optional 

305 Flag setting if this exposure pair should be used (True) 

306 or not used (False). 

307 covariance : `np.ndarray` or None, optional 

308 Measured covariance for this exposure pair. 

309 covSqrtWeights : `np.ndarray` or None, optional 

310 Measured sqrt of covariance weights in this exposure pair. 

311 gain : `float`, optional 

312 Estimated gain for this exposure pair. 

313 noise : `float`, optional 

314 Estimated read noise for this exposure pair. 

315 histVar : `float`, optional 

316 Variance estimated from fitting a histogram with a Gaussian model. 

317 histChi2Dof : `float`, optional 

318 Chi-squared per degree of freedom from Gaussian histogram fit. 

319 kspValue : `float`, optional 

320 KS test p-value from the Gaussian histogram fit. 

321 """ 

322 nanMatrix = np.full((self.covMatrixSide, self.covMatrixSide), np.nan) 

323 nanMatrixFit = np.full((self.covMatrixSideFullCovFit, 

324 self.covMatrixSideFullCovFit), np.nan) 

325 if covariance is None: 

326 covariance = nanMatrix 

327 if covSqrtWeights is None: 

328 covSqrtWeights = nanMatrix 

329 

330 self.inputExpIdPairs[ampName] = [inputExpIdPair] 

331 self.rawExpTimes[ampName] = np.array([rawExpTime]) 

332 self.rawMeans[ampName] = np.array([rawMean]) 

333 self.rawVars[ampName] = np.array([rawVar]) 

334 self.rowMeanVariance[ampName] = np.array([rowMeanVariance]) 

335 self.photoCharges[ampName] = np.array([photoCharge]) 

336 self.expIdMask[ampName] = np.array([expIdMask]) 

337 self.covariances[ampName] = np.array([covariance]) 

338 self.covariancesSqrtWeights[ampName] = np.array([covSqrtWeights]) 

339 self.gain[ampName] = gain 

340 self.noise[ampName] = noise 

341 self.histVars[ampName] = np.array([histVar]) 

342 self.histChi2Dofs[ampName] = np.array([histChi2Dof]) 

343 self.kspValues[ampName] = np.array([kspValue]) 

344 

345 # From FULLCOVARIANCE model 

346 self.covariancesModel[ampName] = np.array([nanMatrixFit]) 

347 self.covariancesModelNoB[ampName] = np.array([nanMatrixFit]) 

348 self.aMatrix[ampName] = nanMatrixFit 

349 self.bMatrix[ampName] = nanMatrixFit 

350 self.aMatrixNoB[ampName] = nanMatrixFit 

351 self.noiseMatrix[ampName] = nanMatrixFit 

352 self.noiseMatrixNoB[ampName] = nanMatrixFit 

353 

354 def setAuxValuesPartialDataset(self, auxDict): 

355 """ 

356 Set a dictionary of auxiliary values for a partial dataset. 

357 

358 Parameters 

359 ---------- 

360 auxDict : `dict` [`str`, `float`] 

361 Dictionary of float values. 

362 """ 

363 for key, value in auxDict.items(): 

364 self.auxValues[key] = np.atleast_1d(np.array(value, dtype=np.float64)) 

365 

366 def updateMetadata(self, **kwargs): 

367 """Update calibration metadata. 

368 This calls the base class's method after ensuring the required 

369 calibration keywords will be saved. 

370 

371 Parameters 

372 ---------- 

373 setDate : `bool`, optional 

374 Update the CALIBDATE fields in the metadata to the current 

375 time. Defaults to False. 

376 kwargs : 

377 Other keyword parameters to set in the metadata. 

378 """ 

379 super().updateMetadata(PTC_FIT_TYPE=self.ptcFitType, **kwargs) 

380 

381 @classmethod 

382 def fromDict(cls, dictionary): 

383 """Construct a calibration from a dictionary of properties. 

384 Must be implemented by the specific calibration subclasses. 

385 

386 Parameters 

387 ---------- 

388 dictionary : `dict` 

389 Dictionary of properties. 

390 

391 Returns 

392 ------- 

393 calib : `lsst.ip.isr.PhotonTransferCurveDataset` 

394 Constructed calibration. 

395 

396 Raises 

397 ------ 

398 RuntimeError 

399 Raised if the supplied dictionary is for a different 

400 calibration. 

401 """ 

402 calib = cls() 

403 if calib._OBSTYPE != dictionary['metadata']['OBSTYPE']: 

404 raise RuntimeError(f"Incorrect Photon Transfer Curve dataset supplied. " 

405 f"Expected {calib._OBSTYPE}, found {dictionary['metadata']['OBSTYPE']}") 

406 calib.setMetadata(dictionary['metadata']) 

407 calib.ptcFitType = dictionary['ptcFitType'] 

408 calib.covMatrixSide = dictionary['covMatrixSide'] 

409 calib.covMatrixSideFullCovFit = dictionary['covMatrixSideFullCovFit'] 

410 calib.badAmps = np.array(dictionary['badAmps'], 'str').tolist() 

411 calib.ampNames = [] 

412 

413 # The cov matrices are square 

414 covMatrixSide = calib.covMatrixSide 

415 covMatrixSideFullCovFit = calib.covMatrixSideFullCovFit 

416 # Number of final signal levels 

417 covDimensionsProduct = len(np.array(list(dictionary['covariances'].values())[0]).ravel()) 

418 nSignalPoints = int(covDimensionsProduct/(covMatrixSide*covMatrixSide)) 

419 

420 for ampName in dictionary['ampNames']: 

421 calib.ampNames.append(ampName) 

422 calib.inputExpIdPairs[ampName] = dictionary['inputExpIdPairs'][ampName] 

423 calib.expIdMask[ampName] = np.array(dictionary['expIdMask'][ampName]) 

424 calib.rawExpTimes[ampName] = np.array(dictionary['rawExpTimes'][ampName], dtype=np.float64) 

425 calib.rawMeans[ampName] = np.array(dictionary['rawMeans'][ampName], dtype=np.float64) 

426 calib.rawVars[ampName] = np.array(dictionary['rawVars'][ampName], dtype=np.float64) 

427 calib.rowMeanVariance[ampName] = np.array(dictionary['rowMeanVariance'][ampName], 

428 dtype=np.float64) 

429 calib.gain[ampName] = float(dictionary['gain'][ampName]) 

430 calib.gainErr[ampName] = float(dictionary['gainErr'][ampName]) 

431 calib.noiseList[ampName] = np.array(dictionary['noiseList'][ampName], dtype=np.float64) 

432 calib.noise[ampName] = float(dictionary['noise'][ampName]) 

433 calib.noiseErr[ampName] = float(dictionary['noiseErr'][ampName]) 

434 calib.histVars[ampName] = np.array(dictionary['histVars'][ampName], dtype=np.float64) 

435 calib.histChi2Dofs[ampName] = np.array(dictionary['histChi2Dofs'][ampName], dtype=np.float64) 

436 calib.kspValues[ampName] = np.array(dictionary['kspValues'][ampName], dtype=np.float64) 

437 calib.ptcFitPars[ampName] = np.array(dictionary['ptcFitPars'][ampName], dtype=np.float64) 

438 calib.ptcFitParsError[ampName] = np.array(dictionary['ptcFitParsError'][ampName], 

439 dtype=np.float64) 

440 calib.ptcFitChiSq[ampName] = float(dictionary['ptcFitChiSq'][ampName]) 

441 calib.ptcTurnoff[ampName] = float(dictionary['ptcTurnoff'][ampName]) 

442 calib.ptcTurnoffSamplingError[ampName] = float(dictionary['ptcTurnoffSamplingError'][ampName]) 

443 if nSignalPoints > 0: 

444 # Regular dataset 

445 calib.covariances[ampName] = np.array(dictionary['covariances'][ampName], 

446 dtype=np.float64).reshape( 

447 (nSignalPoints, covMatrixSide, covMatrixSide)) 

448 calib.covariancesModel[ampName] = np.array( 

449 dictionary['covariancesModel'][ampName], 

450 dtype=np.float64).reshape( 

451 (nSignalPoints, covMatrixSideFullCovFit, covMatrixSideFullCovFit)) 

452 calib.covariancesSqrtWeights[ampName] = np.array( 

453 dictionary['covariancesSqrtWeights'][ampName], 

454 dtype=np.float64).reshape( 

455 (nSignalPoints, covMatrixSide, covMatrixSide)) 

456 calib.aMatrix[ampName] = np.array(dictionary['aMatrix'][ampName], 

457 dtype=np.float64).reshape( 

458 (covMatrixSideFullCovFit, covMatrixSideFullCovFit)) 

459 calib.bMatrix[ampName] = np.array(dictionary['bMatrix'][ampName], 

460 dtype=np.float64).reshape( 

461 (covMatrixSideFullCovFit, covMatrixSideFullCovFit)) 

462 calib.covariancesModelNoB[ampName] = np.array( 

463 dictionary['covariancesModelNoB'][ampName], dtype=np.float64).reshape( 

464 (nSignalPoints, covMatrixSideFullCovFit, covMatrixSideFullCovFit)) 

465 calib.aMatrixNoB[ampName] = np.array( 

466 dictionary['aMatrixNoB'][ampName], 

467 dtype=np.float64).reshape((covMatrixSideFullCovFit, covMatrixSideFullCovFit)) 

468 calib.noiseMatrix[ampName] = np.array( 

469 dictionary['noiseMatrix'][ampName], 

470 dtype=np.float64).reshape((covMatrixSideFullCovFit, covMatrixSideFullCovFit)) 

471 calib.noiseMatrixNoB[ampName] = np.array( 

472 dictionary['noiseMatrixNoB'][ampName], 

473 dtype=np.float64).reshape((covMatrixSideFullCovFit, covMatrixSideFullCovFit)) 

474 else: 

475 # Empty dataset 

476 calib.covariances[ampName] = np.array([], dtype=np.float64) 

477 calib.covariancesModel[ampName] = np.array([], dtype=np.float64) 

478 calib.covariancesSqrtWeights[ampName] = np.array([], dtype=np.float64) 

479 calib.aMatrix[ampName] = np.array([], dtype=np.float64) 

480 calib.bMatrix[ampName] = np.array([], dtype=np.float64) 

481 calib.covariancesModelNoB[ampName] = np.array([], dtype=np.float64) 

482 calib.aMatrixNoB[ampName] = np.array([], dtype=np.float64) 

483 calib.noiseMatrix[ampName] = np.array([], dtype=np.float64) 

484 calib.noiseMatrixNoB[ampName] = np.array([], dtype=np.float64) 

485 

486 calib.finalVars[ampName] = np.array(dictionary['finalVars'][ampName], dtype=np.float64) 

487 calib.finalModelVars[ampName] = np.array(dictionary['finalModelVars'][ampName], dtype=np.float64) 

488 calib.finalMeans[ampName] = np.array(dictionary['finalMeans'][ampName], dtype=np.float64) 

489 calib.photoCharges[ampName] = np.array(dictionary['photoCharges'][ampName], dtype=np.float64) 

490 

491 for key, value in dictionary['auxValues'].items(): 

492 calib.auxValues[key] = np.atleast_1d(np.array(value, dtype=np.float64)) 

493 

494 calib.updateMetadata() 

495 return calib 

496 

497 def toDict(self): 

498 """Return a dictionary containing the calibration properties. 

499 The dictionary should be able to be round-tripped through 

500 `fromDict`. 

501 

502 Returns 

503 ------- 

504 dictionary : `dict` 

505 Dictionary of properties. 

506 """ 

507 self.updateMetadata() 

508 

509 outDict = dict() 

510 metadata = self.getMetadata() 

511 outDict['metadata'] = metadata 

512 

513 def _dictOfArraysToDictOfLists(dictOfArrays): 

514 dictOfLists = {} 

515 for key, value in dictOfArrays.items(): 

516 dictOfLists[key] = value.ravel().tolist() 

517 

518 return dictOfLists 

519 

520 outDict['ptcFitType'] = self.ptcFitType 

521 outDict['covMatrixSide'] = self.covMatrixSide 

522 outDict['covMatrixSideFullCovFit'] = self.covMatrixSideFullCovFit 

523 outDict['ampNames'] = self.ampNames 

524 outDict['badAmps'] = self.badAmps 

525 outDict['inputExpIdPairs'] = self.inputExpIdPairs 

526 outDict['expIdMask'] = _dictOfArraysToDictOfLists(self.expIdMask) 

527 outDict['rawExpTimes'] = _dictOfArraysToDictOfLists(self.rawExpTimes) 

528 outDict['rawMeans'] = _dictOfArraysToDictOfLists(self.rawMeans) 

529 outDict['rawVars'] = _dictOfArraysToDictOfLists(self.rawVars) 

530 outDict['rowMeanVariance'] = _dictOfArraysToDictOfLists(self.rowMeanVariance) 

531 outDict['gain'] = self.gain 

532 outDict['gainErr'] = self.gainErr 

533 outDict['noiseList'] = _dictOfArraysToDictOfLists(self.noiseList) 

534 outDict['noise'] = self.noise 

535 outDict['noiseErr'] = self.noiseErr 

536 outDict['histVars'] = self.histVars 

537 outDict['histChi2Dofs'] = self.histChi2Dofs 

538 outDict['kspValues'] = self.kspValues 

539 outDict['ptcFitPars'] = _dictOfArraysToDictOfLists(self.ptcFitPars) 

540 outDict['ptcFitParsError'] = _dictOfArraysToDictOfLists(self.ptcFitParsError) 

541 outDict['ptcFitChiSq'] = self.ptcFitChiSq 

542 outDict['ptcTurnoff'] = self.ptcTurnoff 

543 outDict['ptcTurnoffSamplingError'] = self.ptcTurnoffSamplingError 

544 outDict['covariances'] = _dictOfArraysToDictOfLists(self.covariances) 

545 outDict['covariancesModel'] = _dictOfArraysToDictOfLists(self.covariancesModel) 

546 outDict['covariancesSqrtWeights'] = _dictOfArraysToDictOfLists(self.covariancesSqrtWeights) 

547 outDict['aMatrix'] = _dictOfArraysToDictOfLists(self.aMatrix) 

548 outDict['bMatrix'] = _dictOfArraysToDictOfLists(self.bMatrix) 

549 outDict['noiseMatrix'] = _dictOfArraysToDictOfLists(self.noiseMatrix) 

550 outDict['covariancesModelNoB'] = _dictOfArraysToDictOfLists(self.covariancesModelNoB) 

551 outDict['aMatrixNoB'] = _dictOfArraysToDictOfLists(self.aMatrixNoB) 

552 outDict['noiseMatrixNoB'] = _dictOfArraysToDictOfLists(self.noiseMatrixNoB) 

553 outDict['finalVars'] = _dictOfArraysToDictOfLists(self.finalVars) 

554 outDict['finalModelVars'] = _dictOfArraysToDictOfLists(self.finalModelVars) 

555 outDict['finalMeans'] = _dictOfArraysToDictOfLists(self.finalMeans) 

556 outDict['photoCharges'] = _dictOfArraysToDictOfLists(self.photoCharges) 

557 outDict['auxValues'] = _dictOfArraysToDictOfLists(self.auxValues) 

558 

559 return outDict 

560 

561 @classmethod 

562 def fromTable(cls, tableList): 

563 """Construct calibration from a list of tables. 

564 This method uses the `fromDict` method to create the 

565 calibration, after constructing an appropriate dictionary from 

566 the input tables. 

567 

568 Parameters 

569 ---------- 

570 tableList : `list` [`lsst.afw.table.Table`] 

571 List of tables to use to construct the datasetPtc. 

572 

573 Returns 

574 ------- 

575 calib : `lsst.ip.isr.PhotonTransferCurveDataset` 

576 The calibration defined in the tables. 

577 """ 

578 ptcTable = tableList[0] 

579 

580 metadata = ptcTable.meta 

581 inDict = dict() 

582 inDict['metadata'] = metadata 

583 inDict['ampNames'] = [] 

584 inDict['ptcFitType'] = [] 

585 inDict['covMatrixSide'] = [] 

586 inDict['covMatrixSideFullCovFit'] = [] 

587 inDict['inputExpIdPairs'] = dict() 

588 inDict['expIdMask'] = dict() 

589 inDict['rawExpTimes'] = dict() 

590 inDict['rawMeans'] = dict() 

591 inDict['rawVars'] = dict() 

592 inDict['rowMeanVariance'] = dict() 

593 inDict['gain'] = dict() 

594 inDict['gainErr'] = dict() 

595 inDict['noiseList'] = dict() 

596 inDict['noise'] = dict() 

597 inDict['noiseErr'] = dict() 

598 inDict['histVars'] = dict() 

599 inDict['histChi2Dofs'] = dict() 

600 inDict['kspValues'] = dict() 

601 inDict['ptcFitPars'] = dict() 

602 inDict['ptcFitParsError'] = dict() 

603 inDict['ptcFitChiSq'] = dict() 

604 inDict['ptcTurnoff'] = dict() 

605 inDict['ptcTurnoffSamplingError'] = dict() 

606 inDict['covariances'] = dict() 

607 inDict['covariancesModel'] = dict() 

608 inDict['covariancesSqrtWeights'] = dict() 

609 inDict['aMatrix'] = dict() 

610 inDict['bMatrix'] = dict() 

611 inDict['noiseMatrix'] = dict() 

612 inDict['covariancesModelNoB'] = dict() 

613 inDict['aMatrixNoB'] = dict() 

614 inDict['noiseMatrixNoB'] = dict() 

615 inDict['finalVars'] = dict() 

616 inDict['finalModelVars'] = dict() 

617 inDict['finalMeans'] = dict() 

618 inDict['badAmps'] = [] 

619 inDict['photoCharges'] = dict() 

620 

621 calibVersion = metadata['PTC_VERSION'] 

622 if calibVersion == 1.0: 

623 cls().log.warning(f"Previous version found for PTC dataset: {calibVersion}. " 

624 f"Setting 'ptcTurnoff' in all amps to last value in 'finalMeans'.") 

625 for record in ptcTable: 

626 ampName = record['AMPLIFIER_NAME'] 

627 

628 inDict['ptcFitType'] = record['PTC_FIT_TYPE'] 

629 inDict['covMatrixSide'] = record['COV_MATRIX_SIDE'] 

630 inDict['ampNames'].append(ampName) 

631 inDict['inputExpIdPairs'][ampName] = record['INPUT_EXP_ID_PAIRS'].tolist() 

632 inDict['expIdMask'][ampName] = record['EXP_ID_MASK'] 

633 inDict['rawExpTimes'][ampName] = record['RAW_EXP_TIMES'] 

634 inDict['rawMeans'][ampName] = record['RAW_MEANS'] 

635 inDict['rawVars'][ampName] = record['RAW_VARS'] 

636 inDict['gain'][ampName] = record['GAIN'] 

637 inDict['gainErr'][ampName] = record['GAIN_ERR'] 

638 inDict['noise'][ampName] = record['NOISE'] 

639 inDict['noiseErr'][ampName] = record['NOISE_ERR'] 

640 inDict['ptcFitPars'][ampName] = record['PTC_FIT_PARS'] 

641 inDict['ptcFitParsError'][ampName] = record['PTC_FIT_PARS_ERROR'] 

642 inDict['ptcFitChiSq'][ampName] = record['PTC_FIT_CHI_SQ'] 

643 inDict['covariances'][ampName] = record['COVARIANCES'] 

644 inDict['covariancesModel'][ampName] = record['COVARIANCES_MODEL'] 

645 inDict['covariancesSqrtWeights'][ampName] = record['COVARIANCES_SQRT_WEIGHTS'] 

646 inDict['aMatrix'][ampName] = record['A_MATRIX'] 

647 inDict['bMatrix'][ampName] = record['B_MATRIX'] 

648 inDict['covariancesModelNoB'][ampName] = record['COVARIANCES_MODEL_NO_B'] 

649 inDict['aMatrixNoB'][ampName] = record['A_MATRIX_NO_B'] 

650 inDict['finalVars'][ampName] = record['FINAL_VARS'] 

651 inDict['finalModelVars'][ampName] = record['FINAL_MODEL_VARS'] 

652 inDict['finalMeans'][ampName] = record['FINAL_MEANS'] 

653 inDict['badAmps'] = record['BAD_AMPS'].tolist() 

654 inDict['photoCharges'][ampName] = record['PHOTO_CHARGE'] 

655 if calibVersion == 1.0: 

656 mask = record['FINAL_MEANS'].mask 

657 array = record['FINAL_MEANS'][~mask] 

658 if len(array) > 0: 

659 inDict['ptcTurnoff'][ampName] = record['FINAL_MEANS'][~mask][-1] 

660 else: 

661 inDict['ptcTurnoff'][ampName] = np.nan 

662 else: 

663 inDict['ptcTurnoff'][ampName] = record['PTC_TURNOFF'] 

664 if calibVersion < 1.2: 

665 inDict['histVars'][ampName] = np.array([np.nan]) 

666 inDict['histChi2Dofs'][ampName] = np.array([np.nan]) 

667 inDict['kspValues'][ampName] = np.array([0.0]) 

668 else: 

669 inDict['histVars'][ampName] = record['HIST_VARS'] 

670 inDict['histChi2Dofs'][ampName] = record['HIST_CHI2_DOFS'] 

671 inDict['kspValues'][ampName] = record['KS_PVALUES'] 

672 if calibVersion < 1.3: 

673 nanMatrix = np.full_like(inDict['aMatrix'][ampName], np.nan) 

674 inDict['noiseMatrix'][ampName] = nanMatrix 

675 inDict['noiseMatrixNoB'][ampName] = nanMatrix 

676 else: 

677 inDict['noiseMatrix'][ampName] = record['NOISE_MATRIX'] 

678 inDict['noiseMatrixNoB'][ampName] = record['NOISE_MATRIX_NO_B'] 

679 if calibVersion < 1.5: 

680 # Matched to `COV_MATRIX_SIDE`. Same for all amps. 

681 inDict['covMatrixSideFullCovFit'] = inDict['covMatrixSide'] 

682 else: 

683 inDict['covMatrixSideFullCovFit'] = record['COV_MATRIX_SIDE_FULL_COV_FIT'] 

684 if calibVersion < 1.6: 

685 inDict['rowMeanVariance'][ampName] = np.full((len(inDict['expIdMask'][ampName]),), np.nan) 

686 else: 

687 inDict['rowMeanVariance'][ampName] = record['ROW_MEAN_VARIANCE'] 

688 if calibVersion < 1.7: 

689 inDict['noiseList'][ampName] = np.full_like(inDict['rawMeans'][ampName], np.nan) 

690 else: 

691 inDict['noiseList'][ampName] = record['NOISE_LIST'] 

692 if calibVersion < 1.8: 

693 inDict['ptcTurnoffSamplingError'][ampName] = np.nan 

694 else: 

695 inDict['ptcTurnoffSamplingError'][ampName] = record['PTC_TURNOFF_SAMPLING_ERROR'] 

696 

697 inDict['auxValues'] = {} 

698 record = ptcTable[0] 

699 for col in record.columns.keys(): 

700 if col.startswith('PTCAUX_'): 

701 parts = col.split('PTCAUX_') 

702 inDict['auxValues'][parts[1]] = record[col] 

703 

704 return cls().fromDict(inDict) 

705 

706 def toTable(self): 

707 """Construct a list of tables containing the information in this 

708 calibration. 

709 

710 The list of tables should create an identical calibration 

711 after being passed to this class's fromTable method. 

712 

713 Returns 

714 ------- 

715 tableList : `list` [`astropy.table.Table`] 

716 List of tables containing the linearity calibration 

717 information. 

718 """ 

719 tableList = [] 

720 self.updateMetadata() 

721 

722 badAmps = np.array(self.badAmps) if len(self.badAmps) else np.array([], dtype="U3") 

723 

724 catalogList = [] 

725 for ampName in self.ampNames: 

726 ampDict = { 

727 'AMPLIFIER_NAME': ampName, 

728 'PTC_FIT_TYPE': self.ptcFitType, 

729 'COV_MATRIX_SIDE': self.covMatrixSide, 

730 'COV_MATRIX_SIDE_FULL_COV_FIT': self.covMatrixSideFullCovFit, 

731 'INPUT_EXP_ID_PAIRS': self.inputExpIdPairs[ampName], 

732 'EXP_ID_MASK': self.expIdMask[ampName], 

733 'RAW_EXP_TIMES': self.rawExpTimes[ampName], 

734 'RAW_MEANS': self.rawMeans[ampName], 

735 'RAW_VARS': self.rawVars[ampName], 

736 'ROW_MEAN_VARIANCE': self.rowMeanVariance[ampName], 

737 'GAIN': self.gain[ampName], 

738 'GAIN_ERR': self.gainErr[ampName], 

739 'NOISE_LIST': self.noiseList[ampName], 

740 'NOISE': self.noise[ampName], 

741 'NOISE_ERR': self.noiseErr[ampName], 

742 'HIST_VARS': self.histVars[ampName], 

743 'HIST_CHI2_DOFS': self.histChi2Dofs[ampName], 

744 'KS_PVALUES': self.kspValues[ampName], 

745 'PTC_FIT_PARS': np.array(self.ptcFitPars[ampName]), 

746 'PTC_FIT_PARS_ERROR': np.array(self.ptcFitParsError[ampName]), 

747 'PTC_FIT_CHI_SQ': self.ptcFitChiSq[ampName], 

748 'PTC_TURNOFF': self.ptcTurnoff[ampName], 

749 'PTC_TURNOFF_SAMPLING_ERROR': self.ptcTurnoffSamplingError[ampName], 

750 'A_MATRIX': self.aMatrix[ampName].ravel(), 

751 'B_MATRIX': self.bMatrix[ampName].ravel(), 

752 'A_MATRIX_NO_B': self.aMatrixNoB[ampName].ravel(), 

753 'NOISE_MATRIX': self.noiseMatrix[ampName].ravel(), 

754 'NOISE_MATRIX_NO_B': self.noiseMatrixNoB[ampName].ravel(), 

755 'BAD_AMPS': badAmps, 

756 'PHOTO_CHARGE': self.photoCharges[ampName], 

757 'COVARIANCES': self.covariances[ampName].ravel(), 

758 'COVARIANCES_MODEL': self.covariancesModel[ampName].ravel(), 

759 'COVARIANCES_SQRT_WEIGHTS': self.covariancesSqrtWeights[ampName].ravel(), 

760 'COVARIANCES_MODEL_NO_B': self.covariancesModelNoB[ampName].ravel(), 

761 'FINAL_VARS': self.finalVars[ampName], 

762 'FINAL_MODEL_VARS': self.finalModelVars[ampName], 

763 'FINAL_MEANS': self.finalMeans[ampName], 

764 } 

765 

766 if self.auxValues: 

767 for key, value in self.auxValues.items(): 

768 ampDict[f"PTCAUX_{key}"] = value 

769 

770 catalogList.append(ampDict) 

771 

772 catalog = Table(catalogList) 

773 

774 inMeta = self.getMetadata().toDict() 

775 outMeta = {k: v for k, v in inMeta.items() if v is not None} 

776 outMeta.update({k: "" for k, v in inMeta.items() if v is None}) 

777 catalog.meta = outMeta 

778 tableList.append(catalog) 

779 

780 return tableList 

781 

782 def fromDetector(self, detector): 

783 """Read metadata parameters from a detector. 

784 

785 Parameters 

786 ---------- 

787 detector : `lsst.afw.cameraGeom.detector` 

788 Input detector with parameters to use. 

789 

790 Returns 

791 ------- 

792 calib : `lsst.ip.isr.PhotonTransferCurveDataset` 

793 The calibration constructed from the detector. 

794 """ 

795 

796 pass 

797 

798 def getExpIdsUsed(self, ampName): 

799 """Get the exposures used, i.e. not discarded, for a given amp. 

800 If no mask has been created yet, all exposures are returned. 

801 

802 Parameters 

803 ---------- 

804 ampName : `str` 

805 

806 Returns 

807 ------- 

808 expIdsUsed : `list` [`tuple`] 

809 List of pairs of exposure ids used in PTC. 

810 """ 

811 if len(self.expIdMask[ampName]) == 0: 

812 return self.inputExpIdPairs[ampName] 

813 

814 # if the mask exists it had better be the same length as the expIdPairs 

815 assert len(self.expIdMask[ampName]) == len(self.inputExpIdPairs[ampName]) 

816 

817 pairs = self.inputExpIdPairs[ampName] 

818 mask = self.expIdMask[ampName] 

819 # cast to bool required because numpy 

820 try: 

821 expIdsUsed = [(exp1, exp2) for ((exp1, exp2), m) in zip(pairs, mask) if m] 

822 except ValueError: 

823 self.log.warning("The PTC file was written incorrectly; you should rerun the " 

824 "PTC solve task if possible.") 

825 expIdsUsed = [] 

826 for pairList, m in zip(pairs, mask): 

827 if m: 

828 expIdsUsed.append(pairList[0]) 

829 

830 return expIdsUsed 

831 

832 def getGoodAmps(self): 

833 """Get the good amps from this PTC.""" 

834 return [amp for amp in self.ampNames if amp not in self.badAmps] 

835 

836 def getGoodPoints(self, ampName): 

837 """Get the good points used for a given amp in the PTC. 

838 

839 Parameters 

840 ---------- 

841 ampName : `str` 

842 Amplifier's name. 

843 

844 Returns 

845 ------- 

846 goodPoints : `np.ndarray` 

847 Boolean array of good points used in PTC. 

848 """ 

849 return self.expIdMask[ampName] 

850 

851 def validateGainNoiseTurnoffValues(self, ampName, doWarn=False): 

852 """Ensure the gain, read noise, and PTC turnoff have 

853 sensible values. 

854 

855 Parameters 

856 ---------- 

857 ampName : `str` 

858 Amplifier's name. 

859 """ 

860 

861 gain = self.gain[ampName] 

862 noise = self.noise[ampName] 

863 ptcTurnoff = self.ptcTurnoff[ampName] 

864 

865 # Check if gain is not positive or is np.nan 

866 if not (isinstance(gain, (int, float)) and gain > 0) or math.isnan(gain): 

867 if doWarn: 

868 self.log.warning(f"Invalid gain value {gain}" 

869 " Setting to default: Gain=1") 

870 gain = 1 

871 

872 # Check if noise is not positive or is np.nan 

873 if not (isinstance(noise, (int, float)) and noise > 0) or math.isnan(noise): 

874 if doWarn: 

875 self.log.warning(f"Invalid noise value: {noise}" 

876 " Setting to default: Noise=1") 

877 noise = 1 

878 

879 # Check if ptcTurnoff is not positive or is np.nan 

880 if not (isinstance(ptcTurnoff, (int, float)) and ptcTurnoff > 0) or math.isnan(ptcTurnoff): 

881 if doWarn: 

882 self.log.warning(f"Invalid PTC turnoff value: {ptcTurnoff}" 

883 " Setting to default: PTC Turnoff=2e19") 

884 ptcTurnoff = 2e19 

885 

886 self.gain[ampName] = gain 

887 self.noise[ampName] = noise 

888 self.ptcTurnoff[ampName] = ptcTurnoff 

889 

890 def _validateCovarianceMatrizSizes(self): 

891 """Ensure covMatrixSideFullCovFit <= covMatrixSide.""" 

892 if self.covMatrixSideFullCovFit > self.covMatrixSide: 

893 self.log.warning("covMatrixSideFullCovFit > covMatrixSide " 

894 f"({self.covMatrixSideFullCovFit} > {self.covMatrixSide})." 

895 "Setting the former to the latter.") 

896 self.covMatrixSideFullCovFit = self.covMatrixSide