Coverage for python/lsst/ip/isr/ptcDataset.py: 6%

369 statements  

« prev     ^ index     » next       coverage.py v7.4.4, created at 2024-03-23 04:01 -0700

1# 

2# LSST Data Management System 

3# Copyright 2008-2017 AURA/LSST. 

4# 

5# This product includes software developed by the 

6# LSST Project (http://www.lsst.org/). 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <https://www.lsstcorp.org/LegalNotices/>. 

21# 

22""" 

23Define dataset class for MeasurePhotonTransferCurve task 

24""" 

25 

26__all__ = ['PhotonTransferCurveDataset'] 

27 

28import numpy as np 

29import math 

30from astropy.table import Table 

31 

32from lsst.ip.isr import IsrCalib 

33 

34 

35class PhotonTransferCurveDataset(IsrCalib): 

36 """A simple class to hold the output data from the PTC task. 

37 

38 The dataset is made up of a dictionary for each item, keyed by the 

39 amplifiers' names, which much be supplied at construction time. 

40 New items cannot be added to the class to save accidentally saving to the 

41 wrong property, and the class can be frozen if desired. 

42 inputExpIdPairs records the exposures used to produce the data. 

43 When fitPtc() or fitCovariancesAstier() is run, a mask is built up, which 

44 is by definition always the same length as inputExpIdPairs, rawExpTimes, 

45 rawMeans and rawVars, and is a list of bools, which are incrementally set 

46 to False as points are discarded from the fits. 

47 PTC fit parameters for polynomials are stored in a list in ascending order 

48 of polynomial term, i.e. par[0]*x^0 + par[1]*x + par[2]*x^2 etc 

49 with the length of the list corresponding to the order of the polynomial 

50 plus one. 

51 

52 Parameters 

53 ---------- 

54 ampNames : `list` 

55 List with the names of the amplifiers of the detector at hand. 

56 ptcFitType : `str`, optional 

57 Type of model fitted to the PTC: "POLYNOMIAL", "EXPAPPROXIMATION", 

58 or "FULLCOVARIANCE". 

59 covMatrixSide : `int`, optional 

60 Maximum lag of measured covariances (size of square covariance 

61 matrices). 

62 covMatrixSideFullCovFit : `int`, optional 

63 Maximum covariances lag for FULLCOVARIANCE fit. It should be less or 

64 equal than covMatrixSide. 

65 kwargs : `dict`, optional 

66 Other keyword arguments to pass to the parent init. 

67 

68 Notes 

69 ----- 

70 The stored attributes are: 

71 

72 badAmps : `list` [`str`] 

73 List with bad amplifiers names. 

74 inputExpIdPairs : `dict`, [`str`, `list`] 

75 Dictionary keyed by amp names containing the input exposures IDs. 

76 expIdMask : `dict`, [`str`, `np.ndarray`] 

77 Dictionary keyed by amp names containing the mask produced after 

78 outlier rejection. The mask produced by the "FULLCOVARIANCE" 

79 option may differ from the one produced in the other two PTC 

80 fit types. 

81 rawExpTimes : `dict`, [`str`, `np.ndarray`] 

82 Dictionary keyed by amp names containing the unmasked exposure times. 

83 rawMeans : `dict`, [`str`, `np.ndarray`] 

84 Dictionary keyed by amp names containing the unmasked average of the 

85 means of the exposures in each flat pair. 

86 rawVars : `dict`, [`str`, `np.ndarray`] 

87 Dictionary keyed by amp names containing the variance of the 

88 difference image of the exposures in each flat pair. 

89 rowMeanVariance : `dict`, [`str`, `np.ndarray`] 

90 Dictionary keyed by amp names containing the variance of the 

91 means of the rows of the difference image of the exposures 

92 in each flat pair. 

93 histVars : `dict`, [`str`, `np.ndarray`] 

94 Dictionary keyed by amp names containing the variance of the 

95 difference image of the exposures in each flat pair estimated 

96 by fitting a Gaussian model. 

97 histChi2Dofs : `dict`, [`str`, `np.ndarray`] 

98 Dictionary keyed by amp names containing the chi-squared per degree 

99 of freedom fitting the difference image to a Gaussian model. 

100 kspValues : `dict`, [`str`, `np.ndarray`] 

101 Dictionary keyed by amp names containing the KS test p-value from 

102 fitting the difference image to a Gaussian model. 

103 gain : `dict`, [`str`, `float`] 

104 Dictionary keyed by amp names containing the fitted gains. 

105 gainErr : `dict`, [`str`, `float`] 

106 Dictionary keyed by amp names containing the errors on the 

107 fitted gains. 

108 noiseList : `dict`, [`str`, `np.ndarray`] 

109 Dictionary keyed by amp names containing the mean read noise from 

110 each flat pair (as measured from overscan). 

111 noise : `dict`, [`str`, `float`] 

112 Dictionary keyed by amp names containing the fitted noise. 

113 noiseErr : `dict`, [`str`, `float`] 

114 Dictionary keyed by amp names containing the errors on the fitted 

115 noise. 

116 ptcFitPars : `dict`, [`str`, `np.ndarray`] 

117 Dictionary keyed by amp names containing the fitted parameters of the 

118 PTC model for ptcFitTye in ["POLYNOMIAL", "EXPAPPROXIMATION"]. 

119 ptcFitParsError : `dict`, [`str`, `np.ndarray`] 

120 Dictionary keyed by amp names containing the errors on the fitted 

121 parameters of the PTC model for ptcFitTye in 

122 ["POLYNOMIAL", "EXPAPPROXIMATION"]. 

123 ptcFitChiSq : `dict`, [`str`, `float`] 

124 Dictionary keyed by amp names containing the reduced chi squared 

125 of the fit for ptcFitTye in ["POLYNOMIAL", "EXPAPPROXIMATION"]. 

126 ptcTurnoff : `dict` [`str, `float`] 

127 Flux value (in ADU) where the variance of the PTC curve starts 

128 decreasing consistently. 

129 covariances : `dict`, [`str`, `np.ndarray`] 

130 Dictionary keyed by amp names containing a list of measured 

131 covariances per mean flux. 

132 covariancesModel : `dict`, [`str`, `np.ndarray`] 

133 Dictionary keyed by amp names containinging covariances model 

134 (Eq. 20 of Astier+19) per mean flux. 

135 covariancesSqrtWeights : `dict`, [`str`, `np.ndarray`] 

136 Dictionary keyed by amp names containinging sqrt. of covariances 

137 weights. 

138 aMatrix : `dict`, [`str`, `np.ndarray`] 

139 Dictionary keyed by amp names containing the "a" parameters from 

140 the model in Eq. 20 of Astier+19. 

141 bMatrix : `dict`, [`str`, `np.ndarray`] 

142 Dictionary keyed by amp names containing the "b" parameters from 

143 the model in Eq. 20 of Astier+19. 

144 noiseMatrix : `dict`, [`str`, `np.ndarray`] 

145 Dictionary keyed by amp names containing the "noise" parameters from 

146 the model in Eq. 20 of Astier+19. 

147 covariancesModelNoB : `dict`, [`str`, `np.ndarray`] 

148 Dictionary keyed by amp names containing covariances model 

149 (with 'b'=0 in Eq. 20 of Astier+19) 

150 per mean flux. 

151 aMatrixNoB : `dict`, [`str`, `np.ndarray`] 

152 Dictionary keyed by amp names containing the "a" parameters from the 

153 model in Eq. 20 of Astier+19 

154 (and 'b' = 0). 

155 noiseMatrixNoB : `dict`, [`str`, `np.ndarray`] 

156 Dictionary keyed by amp names containing the "noise" parameters from 

157 the model in Eq. 20 of Astier+19, with 'b' = 0. 

158 finalVars : `dict`, [`str`, `np.ndarray`] 

159 Dictionary keyed by amp names containing the masked variance of the 

160 difference image of each flat 

161 pair. If needed, each array will be right-padded with 

162 np.nan to match the length of rawExpTimes. 

163 finalModelVars : `dict`, [`str`, `np.ndarray`] 

164 Dictionary keyed by amp names containing the masked modeled 

165 variance of the difference image of each flat pair. If needed, each 

166 array will be right-padded with np.nan to match the length of 

167 rawExpTimes. 

168 finalMeans : `dict`, [`str`, `np.ndarray`] 

169 Dictionary keyed by amp names containing the masked average of the 

170 means of the exposures in each flat pair. If needed, each array 

171 will be right-padded with np.nan to match the length of 

172 rawExpTimes. 

173 photoCharges : `dict`, [`str`, `np.ndarray`] 

174 Dictionary keyed by amp names containing the integrated photocharge 

175 for linearity calibration. 

176 auxValues : `dict`, [`str`, `np.ndarray`] 

177 Dictionary of per-detector auxiliary header values that can be used 

178 for PTC, linearity computation. 

179 

180 Version 1.1 adds the `ptcTurnoff` attribute. 

181 Version 1.2 adds the `histVars`, `histChi2Dofs`, and `kspValues` 

182 attributes. 

183 Version 1.3 adds the `noiseMatrix` and `noiseMatrixNoB` attributes. 

184 Version 1.4 adds the `auxValues` attribute. 

185 Version 1.5 adds the `covMatrixSideFullCovFit` attribute. 

186 Version 1.6 adds the `rowMeanVariance` attribute. 

187 Version 1.7 adds the `noiseList` attribute. 

188 """ 

189 

190 _OBSTYPE = 'PTC' 

191 _SCHEMA = 'Gen3 Photon Transfer Curve' 

192 _VERSION = 1.7 

193 

194 def __init__(self, ampNames=[], ptcFitType=None, covMatrixSide=1, 

195 covMatrixSideFullCovFit=None, **kwargs): 

196 self.ptcFitType = ptcFitType 

197 self.ampNames = ampNames 

198 self.covMatrixSide = covMatrixSide 

199 if covMatrixSideFullCovFit is None: 

200 self.covMatrixSideFullCovFit = covMatrixSide 

201 else: 

202 self.covMatrixSideFullCovFit = covMatrixSideFullCovFit 

203 

204 self.badAmps = [] 

205 

206 self.inputExpIdPairs = {ampName: [] for ampName in ampNames} 

207 self.expIdMask = {ampName: np.array([], dtype=bool) for ampName in ampNames} 

208 self.rawExpTimes = {ampName: np.array([]) for ampName in ampNames} 

209 self.rawMeans = {ampName: np.array([]) for ampName in ampNames} 

210 self.rawVars = {ampName: np.array([]) for ampName in ampNames} 

211 self.rowMeanVariance = {ampName: np.array([]) for ampName in ampNames} 

212 self.photoCharges = {ampName: np.array([]) for ampName in ampNames} 

213 

214 self.gain = {ampName: np.nan for ampName in ampNames} 

215 self.gainErr = {ampName: np.nan for ampName in ampNames} 

216 self.noiseList = {ampName: np.array([]) for ampName in ampNames} 

217 self.noise = {ampName: np.nan for ampName in ampNames} 

218 self.noiseErr = {ampName: np.nan for ampName in ampNames} 

219 

220 self.histVars = {ampName: np.array([]) for ampName in ampNames} 

221 self.histChi2Dofs = {ampName: np.array([]) for ampName in ampNames} 

222 self.kspValues = {ampName: np.array([]) for ampName in ampNames} 

223 

224 self.ptcFitPars = {ampName: np.array([]) for ampName in ampNames} 

225 self.ptcFitParsError = {ampName: np.array([]) for ampName in ampNames} 

226 self.ptcFitChiSq = {ampName: np.nan for ampName in ampNames} 

227 self.ptcTurnoff = {ampName: np.nan for ampName in ampNames} 

228 

229 self.covariances = {ampName: np.array([]) for ampName in ampNames} 

230 self.covariancesModel = {ampName: np.array([]) for ampName in ampNames} 

231 self.covariancesSqrtWeights = {ampName: np.array([]) for ampName in ampNames} 

232 self.aMatrix = {ampName: np.array([]) for ampName in ampNames} 

233 self.bMatrix = {ampName: np.array([]) for ampName in ampNames} 

234 self.noiseMatrix = {ampName: np.array([]) for ampName in ampNames} 

235 self.covariancesModelNoB = {ampName: np.array([]) for ampName in ampNames} 

236 self.aMatrixNoB = {ampName: np.array([]) for ampName in ampNames} 

237 self.noiseMatrixNoB = {ampName: np.array([]) for ampName in ampNames} 

238 

239 self.finalVars = {ampName: np.array([]) for ampName in ampNames} 

240 self.finalModelVars = {ampName: np.array([]) for ampName in ampNames} 

241 self.finalMeans = {ampName: np.array([]) for ampName in ampNames} 

242 

243 # Try this as a dict of arrays. 

244 self.auxValues = {} 

245 

246 super().__init__(**kwargs) 

247 self.requiredAttributes.update(['badAmps', 'inputExpIdPairs', 'expIdMask', 'rawExpTimes', 

248 'rawMeans', 'rawVars', 'rowMeanVariance', 'gain', 

249 'gainErr', 'noise', 'noiseErr', 'noiseList', 

250 'ptcFitPars', 'ptcFitParsError', 'ptcFitChiSq', 'ptcTurnoff', 

251 'aMatrixNoB', 'covariances', 'covariancesModel', 

252 'covariancesSqrtWeights', 'covariancesModelNoB', 

253 'aMatrix', 'bMatrix', 'noiseMatrix', 'noiseMatrixNoB', 'finalVars', 

254 'finalModelVars', 'finalMeans', 'photoCharges', 'histVars', 

255 'histChi2Dofs', 'kspValues', 'auxValues']) 

256 

257 self.updateMetadata(setCalibInfo=True, setCalibId=True, **kwargs) 

258 self._validateCovarianceMatrizSizes() 

259 

260 def setAmpValuesPartialDataset( 

261 self, 

262 ampName, 

263 inputExpIdPair=(-1, -1), 

264 rawExpTime=np.nan, 

265 rawMean=np.nan, 

266 rawVar=np.nan, 

267 rowMeanVariance=np.nan, 

268 photoCharge=np.nan, 

269 expIdMask=False, 

270 covariance=None, 

271 covSqrtWeights=None, 

272 gain=np.nan, 

273 noise=np.nan, 

274 histVar=np.nan, 

275 histChi2Dof=np.nan, 

276 kspValue=0.0, 

277 auxValues=None, 

278 ): 

279 """ 

280 Set the amp values for a partial PTC Dataset (from cpExtractPtcTask). 

281 

282 Parameters 

283 ---------- 

284 ampName : `str` 

285 Name of the amp to set the values. 

286 inputExpIdPair : `tuple` [`int`] 

287 Exposure IDs of input pair. 

288 rawExpTime : `float`, optional 

289 Exposure time for this exposure pair. 

290 rawMean : `float`, optional 

291 Average of the means of the exposures in this pair. 

292 rawVar : `float`, optional 

293 Variance of the difference of the exposures in this pair. 

294 rowMeanVariance : `float`, optional 

295 Variance of the means of the rows in the difference image 

296 of the exposures in this pair. 

297 photoCharge : `float`, optional 

298 Integrated photocharge for flat pair for linearity calibration. 

299 expIdMask : `bool`, optional 

300 Flag setting if this exposure pair should be used (True) 

301 or not used (False). 

302 covariance : `np.ndarray` or None, optional 

303 Measured covariance for this exposure pair. 

304 covSqrtWeights : `np.ndarray` or None, optional 

305 Measured sqrt of covariance weights in this exposure pair. 

306 gain : `float`, optional 

307 Estimated gain for this exposure pair. 

308 noise : `float`, optional 

309 Estimated read noise for this exposure pair. 

310 histVar : `float`, optional 

311 Variance estimated from fitting a histogram with a Gaussian model. 

312 histChi2Dof : `float`, optional 

313 Chi-squared per degree of freedom from Gaussian histogram fit. 

314 kspValue : `float`, optional 

315 KS test p-value from the Gaussian histogram fit. 

316 """ 

317 nanMatrix = np.full((self.covMatrixSide, self.covMatrixSide), np.nan) 

318 nanMatrixFit = np.full((self.covMatrixSideFullCovFit, 

319 self.covMatrixSideFullCovFit), np.nan) 

320 if covariance is None: 

321 covariance = nanMatrix 

322 if covSqrtWeights is None: 

323 covSqrtWeights = nanMatrix 

324 

325 self.inputExpIdPairs[ampName] = [inputExpIdPair] 

326 self.rawExpTimes[ampName] = np.array([rawExpTime]) 

327 self.rawMeans[ampName] = np.array([rawMean]) 

328 self.rawVars[ampName] = np.array([rawVar]) 

329 self.rowMeanVariance[ampName] = np.array([rowMeanVariance]) 

330 self.photoCharges[ampName] = np.array([photoCharge]) 

331 self.expIdMask[ampName] = np.array([expIdMask]) 

332 self.covariances[ampName] = np.array([covariance]) 

333 self.covariancesSqrtWeights[ampName] = np.array([covSqrtWeights]) 

334 self.gain[ampName] = gain 

335 self.noise[ampName] = noise 

336 self.histVars[ampName] = np.array([histVar]) 

337 self.histChi2Dofs[ampName] = np.array([histChi2Dof]) 

338 self.kspValues[ampName] = np.array([kspValue]) 

339 

340 # From FULLCOVARIANCE model 

341 self.covariancesModel[ampName] = np.array([nanMatrixFit]) 

342 self.covariancesModelNoB[ampName] = np.array([nanMatrixFit]) 

343 self.aMatrix[ampName] = nanMatrixFit 

344 self.bMatrix[ampName] = nanMatrixFit 

345 self.aMatrixNoB[ampName] = nanMatrixFit 

346 self.noiseMatrix[ampName] = nanMatrixFit 

347 self.noiseMatrixNoB[ampName] = nanMatrixFit 

348 

349 def setAuxValuesPartialDataset(self, auxDict): 

350 """ 

351 Set a dictionary of auxiliary values for a partial dataset. 

352 

353 Parameters 

354 ---------- 

355 auxDict : `dict` [`str`, `float`] 

356 Dictionary of float values. 

357 """ 

358 for key, value in auxDict.items(): 

359 self.auxValues[key] = np.atleast_1d(np.array(value, dtype=np.float64)) 

360 

361 def updateMetadata(self, **kwargs): 

362 """Update calibration metadata. 

363 This calls the base class's method after ensuring the required 

364 calibration keywords will be saved. 

365 

366 Parameters 

367 ---------- 

368 setDate : `bool`, optional 

369 Update the CALIBDATE fields in the metadata to the current 

370 time. Defaults to False. 

371 kwargs : 

372 Other keyword parameters to set in the metadata. 

373 """ 

374 super().updateMetadata(PTC_FIT_TYPE=self.ptcFitType, **kwargs) 

375 

376 @classmethod 

377 def fromDict(cls, dictionary): 

378 """Construct a calibration from a dictionary of properties. 

379 Must be implemented by the specific calibration subclasses. 

380 

381 Parameters 

382 ---------- 

383 dictionary : `dict` 

384 Dictionary of properties. 

385 

386 Returns 

387 ------- 

388 calib : `lsst.ip.isr.PhotonTransferCurveDataset` 

389 Constructed calibration. 

390 

391 Raises 

392 ------ 

393 RuntimeError 

394 Raised if the supplied dictionary is for a different 

395 calibration. 

396 """ 

397 calib = cls() 

398 if calib._OBSTYPE != dictionary['metadata']['OBSTYPE']: 

399 raise RuntimeError(f"Incorrect Photon Transfer Curve dataset supplied. " 

400 f"Expected {calib._OBSTYPE}, found {dictionary['metadata']['OBSTYPE']}") 

401 calib.setMetadata(dictionary['metadata']) 

402 calib.ptcFitType = dictionary['ptcFitType'] 

403 calib.covMatrixSide = dictionary['covMatrixSide'] 

404 calib.covMatrixSideFullCovFit = dictionary['covMatrixSideFullCovFit'] 

405 calib.badAmps = np.array(dictionary['badAmps'], 'str').tolist() 

406 calib.ampNames = [] 

407 

408 # The cov matrices are square 

409 covMatrixSide = calib.covMatrixSide 

410 covMatrixSideFullCovFit = calib.covMatrixSideFullCovFit 

411 # Number of final signal levels 

412 covDimensionsProduct = len(np.array(list(dictionary['covariances'].values())[0]).ravel()) 

413 nSignalPoints = int(covDimensionsProduct/(covMatrixSide*covMatrixSide)) 

414 

415 for ampName in dictionary['ampNames']: 

416 calib.ampNames.append(ampName) 

417 calib.inputExpIdPairs[ampName] = dictionary['inputExpIdPairs'][ampName] 

418 calib.expIdMask[ampName] = np.array(dictionary['expIdMask'][ampName]) 

419 calib.rawExpTimes[ampName] = np.array(dictionary['rawExpTimes'][ampName], dtype=np.float64) 

420 calib.rawMeans[ampName] = np.array(dictionary['rawMeans'][ampName], dtype=np.float64) 

421 calib.rawVars[ampName] = np.array(dictionary['rawVars'][ampName], dtype=np.float64) 

422 calib.rowMeanVariance[ampName] = np.array(dictionary['rowMeanVariance'][ampName], 

423 dtype=np.float64) 

424 calib.gain[ampName] = float(dictionary['gain'][ampName]) 

425 calib.gainErr[ampName] = float(dictionary['gainErr'][ampName]) 

426 calib.noiseList[ampName] = np.array(dictionary['noiseList'][ampName], dtype=np.float64) 

427 calib.noise[ampName] = float(dictionary['noise'][ampName]) 

428 calib.noiseErr[ampName] = float(dictionary['noiseErr'][ampName]) 

429 calib.histVars[ampName] = np.array(dictionary['histVars'][ampName], dtype=np.float64) 

430 calib.histChi2Dofs[ampName] = np.array(dictionary['histChi2Dofs'][ampName], dtype=np.float64) 

431 calib.kspValues[ampName] = np.array(dictionary['kspValues'][ampName], dtype=np.float64) 

432 calib.ptcFitPars[ampName] = np.array(dictionary['ptcFitPars'][ampName], dtype=np.float64) 

433 calib.ptcFitParsError[ampName] = np.array(dictionary['ptcFitParsError'][ampName], 

434 dtype=np.float64) 

435 calib.ptcFitChiSq[ampName] = float(dictionary['ptcFitChiSq'][ampName]) 

436 calib.ptcTurnoff[ampName] = float(dictionary['ptcTurnoff'][ampName]) 

437 if nSignalPoints > 0: 

438 # Regular dataset 

439 calib.covariances[ampName] = np.array(dictionary['covariances'][ampName], 

440 dtype=np.float64).reshape( 

441 (nSignalPoints, covMatrixSide, covMatrixSide)) 

442 calib.covariancesModel[ampName] = np.array( 

443 dictionary['covariancesModel'][ampName], 

444 dtype=np.float64).reshape( 

445 (nSignalPoints, covMatrixSideFullCovFit, covMatrixSideFullCovFit)) 

446 calib.covariancesSqrtWeights[ampName] = np.array( 

447 dictionary['covariancesSqrtWeights'][ampName], 

448 dtype=np.float64).reshape( 

449 (nSignalPoints, covMatrixSide, covMatrixSide)) 

450 calib.aMatrix[ampName] = np.array(dictionary['aMatrix'][ampName], 

451 dtype=np.float64).reshape( 

452 (covMatrixSideFullCovFit, covMatrixSideFullCovFit)) 

453 calib.bMatrix[ampName] = np.array(dictionary['bMatrix'][ampName], 

454 dtype=np.float64).reshape( 

455 (covMatrixSideFullCovFit, covMatrixSideFullCovFit)) 

456 calib.covariancesModelNoB[ampName] = np.array( 

457 dictionary['covariancesModelNoB'][ampName], dtype=np.float64).reshape( 

458 (nSignalPoints, covMatrixSideFullCovFit, covMatrixSideFullCovFit)) 

459 calib.aMatrixNoB[ampName] = np.array( 

460 dictionary['aMatrixNoB'][ampName], 

461 dtype=np.float64).reshape((covMatrixSideFullCovFit, covMatrixSideFullCovFit)) 

462 calib.noiseMatrix[ampName] = np.array( 

463 dictionary['noiseMatrix'][ampName], 

464 dtype=np.float64).reshape((covMatrixSideFullCovFit, covMatrixSideFullCovFit)) 

465 calib.noiseMatrixNoB[ampName] = np.array( 

466 dictionary['noiseMatrixNoB'][ampName], 

467 dtype=np.float64).reshape((covMatrixSideFullCovFit, covMatrixSideFullCovFit)) 

468 else: 

469 # Empty dataset 

470 calib.covariances[ampName] = np.array([], dtype=np.float64) 

471 calib.covariancesModel[ampName] = np.array([], dtype=np.float64) 

472 calib.covariancesSqrtWeights[ampName] = np.array([], dtype=np.float64) 

473 calib.aMatrix[ampName] = np.array([], dtype=np.float64) 

474 calib.bMatrix[ampName] = np.array([], dtype=np.float64) 

475 calib.covariancesModelNoB[ampName] = np.array([], dtype=np.float64) 

476 calib.aMatrixNoB[ampName] = np.array([], dtype=np.float64) 

477 calib.noiseMatrix[ampName] = np.array([], dtype=np.float64) 

478 calib.noiseMatrixNoB[ampName] = np.array([], dtype=np.float64) 

479 

480 calib.finalVars[ampName] = np.array(dictionary['finalVars'][ampName], dtype=np.float64) 

481 calib.finalModelVars[ampName] = np.array(dictionary['finalModelVars'][ampName], dtype=np.float64) 

482 calib.finalMeans[ampName] = np.array(dictionary['finalMeans'][ampName], dtype=np.float64) 

483 calib.photoCharges[ampName] = np.array(dictionary['photoCharges'][ampName], dtype=np.float64) 

484 

485 for key, value in dictionary['auxValues'].items(): 

486 calib.auxValues[key] = np.atleast_1d(np.array(value, dtype=np.float64)) 

487 

488 calib.updateMetadata() 

489 return calib 

490 

491 def toDict(self): 

492 """Return a dictionary containing the calibration properties. 

493 The dictionary should be able to be round-tripped through 

494 `fromDict`. 

495 

496 Returns 

497 ------- 

498 dictionary : `dict` 

499 Dictionary of properties. 

500 """ 

501 self.updateMetadata() 

502 

503 outDict = dict() 

504 metadata = self.getMetadata() 

505 outDict['metadata'] = metadata 

506 

507 def _dictOfArraysToDictOfLists(dictOfArrays): 

508 dictOfLists = {} 

509 for key, value in dictOfArrays.items(): 

510 dictOfLists[key] = value.ravel().tolist() 

511 

512 return dictOfLists 

513 

514 outDict['ptcFitType'] = self.ptcFitType 

515 outDict['covMatrixSide'] = self.covMatrixSide 

516 outDict['covMatrixSideFullCovFit'] = self.covMatrixSideFullCovFit 

517 outDict['ampNames'] = self.ampNames 

518 outDict['badAmps'] = self.badAmps 

519 outDict['inputExpIdPairs'] = self.inputExpIdPairs 

520 outDict['expIdMask'] = _dictOfArraysToDictOfLists(self.expIdMask) 

521 outDict['rawExpTimes'] = _dictOfArraysToDictOfLists(self.rawExpTimes) 

522 outDict['rawMeans'] = _dictOfArraysToDictOfLists(self.rawMeans) 

523 outDict['rawVars'] = _dictOfArraysToDictOfLists(self.rawVars) 

524 outDict['rowMeanVariance'] = _dictOfArraysToDictOfLists(self.rowMeanVariance) 

525 outDict['gain'] = self.gain 

526 outDict['gainErr'] = self.gainErr 

527 outDict['noiseList'] = _dictOfArraysToDictOfLists(self.noiseList) 

528 outDict['noise'] = self.noise 

529 outDict['noiseErr'] = self.noiseErr 

530 outDict['histVars'] = self.histVars 

531 outDict['histChi2Dofs'] = self.histChi2Dofs 

532 outDict['kspValues'] = self.kspValues 

533 outDict['ptcFitPars'] = _dictOfArraysToDictOfLists(self.ptcFitPars) 

534 outDict['ptcFitParsError'] = _dictOfArraysToDictOfLists(self.ptcFitParsError) 

535 outDict['ptcFitChiSq'] = self.ptcFitChiSq 

536 outDict['ptcTurnoff'] = self.ptcTurnoff 

537 outDict['covariances'] = _dictOfArraysToDictOfLists(self.covariances) 

538 outDict['covariancesModel'] = _dictOfArraysToDictOfLists(self.covariancesModel) 

539 outDict['covariancesSqrtWeights'] = _dictOfArraysToDictOfLists(self.covariancesSqrtWeights) 

540 outDict['aMatrix'] = _dictOfArraysToDictOfLists(self.aMatrix) 

541 outDict['bMatrix'] = _dictOfArraysToDictOfLists(self.bMatrix) 

542 outDict['noiseMatrix'] = _dictOfArraysToDictOfLists(self.noiseMatrix) 

543 outDict['covariancesModelNoB'] = _dictOfArraysToDictOfLists(self.covariancesModelNoB) 

544 outDict['aMatrixNoB'] = _dictOfArraysToDictOfLists(self.aMatrixNoB) 

545 outDict['noiseMatrixNoB'] = _dictOfArraysToDictOfLists(self.noiseMatrixNoB) 

546 outDict['finalVars'] = _dictOfArraysToDictOfLists(self.finalVars) 

547 outDict['finalModelVars'] = _dictOfArraysToDictOfLists(self.finalModelVars) 

548 outDict['finalMeans'] = _dictOfArraysToDictOfLists(self.finalMeans) 

549 outDict['photoCharges'] = _dictOfArraysToDictOfLists(self.photoCharges) 

550 outDict['auxValues'] = _dictOfArraysToDictOfLists(self.auxValues) 

551 

552 return outDict 

553 

554 @classmethod 

555 def fromTable(cls, tableList): 

556 """Construct calibration from a list of tables. 

557 This method uses the `fromDict` method to create the 

558 calibration, after constructing an appropriate dictionary from 

559 the input tables. 

560 

561 Parameters 

562 ---------- 

563 tableList : `list` [`lsst.afw.table.Table`] 

564 List of tables to use to construct the datasetPtc. 

565 

566 Returns 

567 ------- 

568 calib : `lsst.ip.isr.PhotonTransferCurveDataset` 

569 The calibration defined in the tables. 

570 """ 

571 ptcTable = tableList[0] 

572 

573 metadata = ptcTable.meta 

574 inDict = dict() 

575 inDict['metadata'] = metadata 

576 inDict['ampNames'] = [] 

577 inDict['ptcFitType'] = [] 

578 inDict['covMatrixSide'] = [] 

579 inDict['covMatrixSideFullCovFit'] = [] 

580 inDict['inputExpIdPairs'] = dict() 

581 inDict['expIdMask'] = dict() 

582 inDict['rawExpTimes'] = dict() 

583 inDict['rawMeans'] = dict() 

584 inDict['rawVars'] = dict() 

585 inDict['rowMeanVariance'] = dict() 

586 inDict['gain'] = dict() 

587 inDict['gainErr'] = dict() 

588 inDict['noiseList'] = dict() 

589 inDict['noise'] = dict() 

590 inDict['noiseErr'] = dict() 

591 inDict['histVars'] = dict() 

592 inDict['histChi2Dofs'] = dict() 

593 inDict['kspValues'] = dict() 

594 inDict['ptcFitPars'] = dict() 

595 inDict['ptcFitParsError'] = dict() 

596 inDict['ptcFitChiSq'] = dict() 

597 inDict['ptcTurnoff'] = dict() 

598 inDict['covariances'] = dict() 

599 inDict['covariancesModel'] = dict() 

600 inDict['covariancesSqrtWeights'] = dict() 

601 inDict['aMatrix'] = dict() 

602 inDict['bMatrix'] = dict() 

603 inDict['noiseMatrix'] = dict() 

604 inDict['covariancesModelNoB'] = dict() 

605 inDict['aMatrixNoB'] = dict() 

606 inDict['noiseMatrixNoB'] = dict() 

607 inDict['finalVars'] = dict() 

608 inDict['finalModelVars'] = dict() 

609 inDict['finalMeans'] = dict() 

610 inDict['badAmps'] = [] 

611 inDict['photoCharges'] = dict() 

612 

613 calibVersion = metadata['PTC_VERSION'] 

614 if calibVersion == 1.0: 

615 cls().log.warning(f"Previous version found for PTC dataset: {calibVersion}. " 

616 f"Setting 'ptcTurnoff' in all amps to last value in 'finalMeans'.") 

617 for record in ptcTable: 

618 ampName = record['AMPLIFIER_NAME'] 

619 

620 inDict['ptcFitType'] = record['PTC_FIT_TYPE'] 

621 inDict['covMatrixSide'] = record['COV_MATRIX_SIDE'] 

622 inDict['ampNames'].append(ampName) 

623 inDict['inputExpIdPairs'][ampName] = record['INPUT_EXP_ID_PAIRS'].tolist() 

624 inDict['expIdMask'][ampName] = record['EXP_ID_MASK'] 

625 inDict['rawExpTimes'][ampName] = record['RAW_EXP_TIMES'] 

626 inDict['rawMeans'][ampName] = record['RAW_MEANS'] 

627 inDict['rawVars'][ampName] = record['RAW_VARS'] 

628 inDict['gain'][ampName] = record['GAIN'] 

629 inDict['gainErr'][ampName] = record['GAIN_ERR'] 

630 inDict['noise'][ampName] = record['NOISE'] 

631 inDict['noiseErr'][ampName] = record['NOISE_ERR'] 

632 inDict['ptcFitPars'][ampName] = record['PTC_FIT_PARS'] 

633 inDict['ptcFitParsError'][ampName] = record['PTC_FIT_PARS_ERROR'] 

634 inDict['ptcFitChiSq'][ampName] = record['PTC_FIT_CHI_SQ'] 

635 inDict['covariances'][ampName] = record['COVARIANCES'] 

636 inDict['covariancesModel'][ampName] = record['COVARIANCES_MODEL'] 

637 inDict['covariancesSqrtWeights'][ampName] = record['COVARIANCES_SQRT_WEIGHTS'] 

638 inDict['aMatrix'][ampName] = record['A_MATRIX'] 

639 inDict['bMatrix'][ampName] = record['B_MATRIX'] 

640 inDict['covariancesModelNoB'][ampName] = record['COVARIANCES_MODEL_NO_B'] 

641 inDict['aMatrixNoB'][ampName] = record['A_MATRIX_NO_B'] 

642 inDict['finalVars'][ampName] = record['FINAL_VARS'] 

643 inDict['finalModelVars'][ampName] = record['FINAL_MODEL_VARS'] 

644 inDict['finalMeans'][ampName] = record['FINAL_MEANS'] 

645 inDict['badAmps'] = record['BAD_AMPS'].tolist() 

646 inDict['photoCharges'][ampName] = record['PHOTO_CHARGE'] 

647 if calibVersion == 1.0: 

648 mask = record['FINAL_MEANS'].mask 

649 array = record['FINAL_MEANS'][~mask] 

650 if len(array) > 0: 

651 inDict['ptcTurnoff'][ampName] = record['FINAL_MEANS'][~mask][-1] 

652 else: 

653 inDict['ptcTurnoff'][ampName] = np.nan 

654 else: 

655 inDict['ptcTurnoff'][ampName] = record['PTC_TURNOFF'] 

656 if calibVersion < 1.2: 

657 inDict['histVars'][ampName] = np.array([np.nan]) 

658 inDict['histChi2Dofs'][ampName] = np.array([np.nan]) 

659 inDict['kspValues'][ampName] = np.array([0.0]) 

660 else: 

661 inDict['histVars'][ampName] = record['HIST_VARS'] 

662 inDict['histChi2Dofs'][ampName] = record['HIST_CHI2_DOFS'] 

663 inDict['kspValues'][ampName] = record['KS_PVALUES'] 

664 if calibVersion < 1.3: 

665 nanMatrix = np.full_like(inDict['aMatrix'][ampName], np.nan) 

666 inDict['noiseMatrix'][ampName] = nanMatrix 

667 inDict['noiseMatrixNoB'][ampName] = nanMatrix 

668 else: 

669 inDict['noiseMatrix'][ampName] = record['NOISE_MATRIX'] 

670 inDict['noiseMatrixNoB'][ampName] = record['NOISE_MATRIX_NO_B'] 

671 if calibVersion < 1.5: 

672 # Matched to `COV_MATRIX_SIDE`. Same for all amps. 

673 inDict['covMatrixSideFullCovFit'] = inDict['covMatrixSide'] 

674 else: 

675 inDict['covMatrixSideFullCovFit'] = record['COV_MATRIX_SIDE_FULL_COV_FIT'] 

676 if calibVersion < 1.6: 

677 inDict['rowMeanVariance'][ampName] = np.full((len(inDict['expIdMask'][ampName]),), np.nan) 

678 else: 

679 inDict['rowMeanVariance'][ampName] = record['ROW_MEAN_VARIANCE'] 

680 if calibVersion < 1.7: 

681 inDict['noiseList'][ampName] = np.full_like(inDict['rawMeans'][ampName], np.nan) 

682 else: 

683 inDict['noiseList'][ampName] = record['NOISE_LIST'] 

684 

685 inDict['auxValues'] = {} 

686 record = ptcTable[0] 

687 for col in record.columns.keys(): 

688 if col.startswith('PTCAUX_'): 

689 parts = col.split('PTCAUX_') 

690 inDict['auxValues'][parts[1]] = record[col] 

691 

692 return cls().fromDict(inDict) 

693 

694 def toTable(self): 

695 """Construct a list of tables containing the information in this 

696 calibration. 

697 

698 The list of tables should create an identical calibration 

699 after being passed to this class's fromTable method. 

700 

701 Returns 

702 ------- 

703 tableList : `list` [`astropy.table.Table`] 

704 List of tables containing the linearity calibration 

705 information. 

706 """ 

707 tableList = [] 

708 self.updateMetadata() 

709 

710 badAmps = np.array(self.badAmps) if len(self.badAmps) else np.array([], dtype="U3") 

711 

712 catalogList = [] 

713 for ampName in self.ampNames: 

714 ampDict = { 

715 'AMPLIFIER_NAME': ampName, 

716 'PTC_FIT_TYPE': self.ptcFitType, 

717 'COV_MATRIX_SIDE': self.covMatrixSide, 

718 'COV_MATRIX_SIDE_FULL_COV_FIT': self.covMatrixSideFullCovFit, 

719 'INPUT_EXP_ID_PAIRS': self.inputExpIdPairs[ampName], 

720 'EXP_ID_MASK': self.expIdMask[ampName], 

721 'RAW_EXP_TIMES': self.rawExpTimes[ampName], 

722 'RAW_MEANS': self.rawMeans[ampName], 

723 'RAW_VARS': self.rawVars[ampName], 

724 'ROW_MEAN_VARIANCE': self.rowMeanVariance[ampName], 

725 'GAIN': self.gain[ampName], 

726 'GAIN_ERR': self.gainErr[ampName], 

727 'NOISE_LIST': self.noiseList[ampName], 

728 'NOISE': self.noise[ampName], 

729 'NOISE_ERR': self.noiseErr[ampName], 

730 'HIST_VARS': self.histVars[ampName], 

731 'HIST_CHI2_DOFS': self.histChi2Dofs[ampName], 

732 'KS_PVALUES': self.kspValues[ampName], 

733 'PTC_FIT_PARS': np.array(self.ptcFitPars[ampName]), 

734 'PTC_FIT_PARS_ERROR': np.array(self.ptcFitParsError[ampName]), 

735 'PTC_FIT_CHI_SQ': self.ptcFitChiSq[ampName], 

736 'PTC_TURNOFF': self.ptcTurnoff[ampName], 

737 'A_MATRIX': self.aMatrix[ampName].ravel(), 

738 'B_MATRIX': self.bMatrix[ampName].ravel(), 

739 'A_MATRIX_NO_B': self.aMatrixNoB[ampName].ravel(), 

740 'NOISE_MATRIX': self.noiseMatrix[ampName].ravel(), 

741 'NOISE_MATRIX_NO_B': self.noiseMatrixNoB[ampName].ravel(), 

742 'BAD_AMPS': badAmps, 

743 'PHOTO_CHARGE': self.photoCharges[ampName], 

744 'COVARIANCES': self.covariances[ampName].ravel(), 

745 'COVARIANCES_MODEL': self.covariancesModel[ampName].ravel(), 

746 'COVARIANCES_SQRT_WEIGHTS': self.covariancesSqrtWeights[ampName].ravel(), 

747 'COVARIANCES_MODEL_NO_B': self.covariancesModelNoB[ampName].ravel(), 

748 'FINAL_VARS': self.finalVars[ampName], 

749 'FINAL_MODEL_VARS': self.finalModelVars[ampName], 

750 'FINAL_MEANS': self.finalMeans[ampName], 

751 } 

752 

753 if self.auxValues: 

754 for key, value in self.auxValues.items(): 

755 ampDict[f"PTCAUX_{key}"] = value 

756 

757 catalogList.append(ampDict) 

758 

759 catalog = Table(catalogList) 

760 

761 inMeta = self.getMetadata().toDict() 

762 outMeta = {k: v for k, v in inMeta.items() if v is not None} 

763 outMeta.update({k: "" for k, v in inMeta.items() if v is None}) 

764 catalog.meta = outMeta 

765 tableList.append(catalog) 

766 

767 return tableList 

768 

769 def fromDetector(self, detector): 

770 """Read metadata parameters from a detector. 

771 

772 Parameters 

773 ---------- 

774 detector : `lsst.afw.cameraGeom.detector` 

775 Input detector with parameters to use. 

776 

777 Returns 

778 ------- 

779 calib : `lsst.ip.isr.PhotonTransferCurveDataset` 

780 The calibration constructed from the detector. 

781 """ 

782 

783 pass 

784 

785 def getExpIdsUsed(self, ampName): 

786 """Get the exposures used, i.e. not discarded, for a given amp. 

787 If no mask has been created yet, all exposures are returned. 

788 

789 Parameters 

790 ---------- 

791 ampName : `str` 

792 

793 Returns 

794 ------- 

795 expIdsUsed : `list` [`tuple`] 

796 List of pairs of exposure ids used in PTC. 

797 """ 

798 if len(self.expIdMask[ampName]) == 0: 

799 return self.inputExpIdPairs[ampName] 

800 

801 # if the mask exists it had better be the same length as the expIdPairs 

802 assert len(self.expIdMask[ampName]) == len(self.inputExpIdPairs[ampName]) 

803 

804 pairs = self.inputExpIdPairs[ampName] 

805 mask = self.expIdMask[ampName] 

806 # cast to bool required because numpy 

807 try: 

808 expIdsUsed = [(exp1, exp2) for ((exp1, exp2), m) in zip(pairs, mask) if m] 

809 except ValueError: 

810 self.log.warning("The PTC file was written incorrectly; you should rerun the " 

811 "PTC solve task if possible.") 

812 expIdsUsed = [] 

813 for pairList, m in zip(pairs, mask): 

814 if m: 

815 expIdsUsed.append(pairList[0]) 

816 

817 return expIdsUsed 

818 

819 def getGoodAmps(self): 

820 """Get the good amps from this PTC.""" 

821 return [amp for amp in self.ampNames if amp not in self.badAmps] 

822 

823 def getGoodPoints(self, ampName): 

824 """Get the good points used for a given amp in the PTC. 

825 

826 Parameters 

827 ---------- 

828 ampName : `str` 

829 Amplifier's name. 

830 

831 Returns 

832 ------- 

833 goodPoints : `np.ndarray` 

834 Boolean array of good points used in PTC. 

835 """ 

836 return self.expIdMask[ampName] 

837 

838 def validateGainNoiseTurnoffValues(self, ampName, doWarn=False): 

839 """Ensure the gain, read noise, and PTC turnoff have 

840 sensible values. 

841 

842 Parameters 

843 ---------- 

844 ampName : `str` 

845 Amplifier's name. 

846 """ 

847 

848 gain = self.gain[ampName] 

849 noise = self.noise[ampName] 

850 ptcTurnoff = self.ptcTurnoff[ampName] 

851 

852 # Check if gain is not positive or is np.nan 

853 if not (isinstance(gain, (int, float)) and gain > 0) or math.isnan(gain): 

854 if doWarn: 

855 self.log.warning(f"Invalid gain value {gain}" 

856 " Setting to default: Gain=1") 

857 gain = 1 

858 

859 # Check if noise is not positive or is np.nan 

860 if not (isinstance(noise, (int, float)) and noise > 0) or math.isnan(noise): 

861 if doWarn: 

862 self.log.warning(f"Invalid noise value: {noise}" 

863 " Setting to default: Noise=1") 

864 noise = 1 

865 

866 # Check if ptcTurnoff is not positive or is np.nan 

867 if not (isinstance(ptcTurnoff, (int, float)) and ptcTurnoff > 0) or math.isnan(ptcTurnoff): 

868 if doWarn: 

869 self.log.warning(f"Invalid PTC turnoff value: {ptcTurnoff}" 

870 " Setting to default: PTC Turnoff=2e19") 

871 ptcTurnoff = 2e19 

872 

873 self.gain[ampName] = gain 

874 self.noise[ampName] = noise 

875 self.ptcTurnoff[ampName] = ptcTurnoff 

876 

877 def _validateCovarianceMatrizSizes(self): 

878 """Ensure covMatrixSideFullCovFit <= covMatrixSide.""" 

879 if self.covMatrixSideFullCovFit > self.covMatrixSide: 

880 self.log.warning("covMatrixSideFullCovFit > covMatrixSide " 

881 f"({self.covMatrixSideFullCovFit} > {self.covMatrixSide})." 

882 "Setting the former to the latter.") 

883 self.covMatrixSideFullCovFit = self.covMatrixSide