Coverage for python/lsst/ip/isr/ptcDataset.py: 6%

354 statements  

« prev     ^ index     » next       coverage.py v7.4.0, created at 2024-01-11 18:34 +0000

1# 

2# LSST Data Management System 

3# Copyright 2008-2017 AURA/LSST. 

4# 

5# This product includes software developed by the 

6# LSST Project (http://www.lsst.org/). 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <https://www.lsstcorp.org/LegalNotices/>. 

21# 

22""" 

23Define dataset class for MeasurePhotonTransferCurve task 

24""" 

25 

26__all__ = ['PhotonTransferCurveDataset'] 

27 

28import numpy as np 

29import math 

30from astropy.table import Table 

31 

32from lsst.ip.isr import IsrCalib 

33 

34 

35class PhotonTransferCurveDataset(IsrCalib): 

36 """A simple class to hold the output data from the PTC task. 

37 

38 The dataset is made up of a dictionary for each item, keyed by the 

39 amplifiers' names, which much be supplied at construction time. 

40 New items cannot be added to the class to save accidentally saving to the 

41 wrong property, and the class can be frozen if desired. 

42 inputExpIdPairs records the exposures used to produce the data. 

43 When fitPtc() or fitCovariancesAstier() is run, a mask is built up, which 

44 is by definition always the same length as inputExpIdPairs, rawExpTimes, 

45 rawMeans and rawVars, and is a list of bools, which are incrementally set 

46 to False as points are discarded from the fits. 

47 PTC fit parameters for polynomials are stored in a list in ascending order 

48 of polynomial term, i.e. par[0]*x^0 + par[1]*x + par[2]*x^2 etc 

49 with the length of the list corresponding to the order of the polynomial 

50 plus one. 

51 

52 Parameters 

53 ---------- 

54 ampNames : `list` 

55 List with the names of the amplifiers of the detector at hand. 

56 ptcFitType : `str`, optional 

57 Type of model fitted to the PTC: "POLYNOMIAL", "EXPAPPROXIMATION", 

58 or "FULLCOVARIANCE". 

59 covMatrixSide : `int`, optional 

60 Maximum lag of measured covariances (size of square covariance 

61 matrices). 

62 covMatrixSideFullCovFit : `int, optional 

63 Maximum covariances lag for FULLCOVARIANCE fit. It should be less or 

64 equal than covMatrixSide. 

65 kwargs : `dict`, optional 

66 Other keyword arguments to pass to the parent init. 

67 

68 Notes 

69 ----- 

70 The stored attributes are: 

71 

72 badAmps : `list` [`str`] 

73 List with bad amplifiers names. 

74 inputExpIdPairs : `dict`, [`str`, `list`] 

75 Dictionary keyed by amp names containing the input exposures IDs. 

76 expIdMask : `dict`, [`str`, `np.ndarray`] 

77 Dictionary keyed by amp names containing the mask produced after 

78 outlier rejection. The mask produced by the "FULLCOVARIANCE" 

79 option may differ from the one produced in the other two PTC 

80 fit types. 

81 rawExpTimes : `dict`, [`str`, `np.ndarray`] 

82 Dictionary keyed by amp names containing the unmasked exposure times. 

83 rawMeans : `dict`, [`str`, `np.ndarray`] 

84 Dictionary keyed by amp names containing the unmasked average of the 

85 means of the exposures in each flat pair. 

86 rawVars : `dict`, [`str`, `np.ndarray`] 

87 Dictionary keyed by amp names containing the variance of the 

88 difference image of the exposures in each flat pair. 

89 histVars : `dict`, [`str`, `np.ndarray`] 

90 Dictionary keyed by amp names containing the variance of the 

91 difference image of the exposures in each flat pair estimated 

92 by fitting a Gaussian model. 

93 histChi2Dofs : `dict`, [`str`, `np.ndarray`] 

94 Dictionary keyed by amp names containing the chi-squared per degree 

95 of freedom fitting the difference image to a Gaussian model. 

96 kspValues : `dict`, [`str`, `np.ndarray`] 

97 Dictionary keyed by amp names containing the KS test p-value from 

98 fitting the difference image to a Gaussian model. 

99 gain : `dict`, [`str`, `float`] 

100 Dictionary keyed by amp names containing the fitted gains. 

101 gainErr : `dict`, [`str`, `float`] 

102 Dictionary keyed by amp names containing the errors on the 

103 fitted gains. 

104 noise : `dict`, [`str`, `float`] 

105 Dictionary keyed by amp names containing the fitted noise. 

106 noiseErr : `dict`, [`str`, `float`] 

107 Dictionary keyed by amp names containing the errors on the fitted 

108 noise. 

109 ptcFitPars : `dict`, [`str`, `np.ndarray`] 

110 Dictionary keyed by amp names containing the fitted parameters of the 

111 PTC model for ptcFitTye in ["POLYNOMIAL", "EXPAPPROXIMATION"]. 

112 ptcFitParsError : `dict`, [`str`, `np.ndarray`] 

113 Dictionary keyed by amp names containing the errors on the fitted 

114 parameters of the PTC model for ptcFitTye in 

115 ["POLYNOMIAL", "EXPAPPROXIMATION"]. 

116 ptcFitChiSq : `dict`, [`str`, `float`] 

117 Dictionary keyed by amp names containing the reduced chi squared 

118 of the fit for ptcFitTye in ["POLYNOMIAL", "EXPAPPROXIMATION"]. 

119 ptcTurnoff : `dict` [`str, `float`] 

120 Flux value (in ADU) where the variance of the PTC curve starts 

121 decreasing consistently. 

122 covariances : `dict`, [`str`, `np.ndarray`] 

123 Dictionary keyed by amp names containing a list of measured 

124 covariances per mean flux. 

125 covariancesModel : `dict`, [`str`, `np.ndarray`] 

126 Dictionary keyed by amp names containinging covariances model 

127 (Eq. 20 of Astier+19) per mean flux. 

128 covariancesSqrtWeights : `dict`, [`str`, `np.ndarray`] 

129 Dictionary keyed by amp names containinging sqrt. of covariances 

130 weights. 

131 aMatrix : `dict`, [`str`, `np.ndarray`] 

132 Dictionary keyed by amp names containing the "a" parameters from 

133 the model in Eq. 20 of Astier+19. 

134 bMatrix : `dict`, [`str`, `np.ndarray`] 

135 Dictionary keyed by amp names containing the "b" parameters from 

136 the model in Eq. 20 of Astier+19. 

137 noiseMatrix : `dict`, [`str`, `np.ndarray`] 

138 Dictionary keyed by amp names containing the "noise" parameters from 

139 the model in Eq. 20 of Astier+19. 

140 covariancesModelNoB : `dict`, [`str`, `np.ndarray`] 

141 Dictionary keyed by amp names containing covariances model 

142 (with 'b'=0 in Eq. 20 of Astier+19) 

143 per mean flux. 

144 aMatrixNoB : `dict`, [`str`, `np.ndarray`] 

145 Dictionary keyed by amp names containing the "a" parameters from the 

146 model in Eq. 20 of Astier+19 

147 (and 'b' = 0). 

148 noiseMatrixNoB : `dict`, [`str`, `np.ndarray`] 

149 Dictionary keyed by amp names containing the "noise" parameters from 

150 the model in Eq. 20 of Astier+19, with 'b' = 0. 

151 finalVars : `dict`, [`str`, `np.ndarray`] 

152 Dictionary keyed by amp names containing the masked variance of the 

153 difference image of each flat 

154 pair. If needed, each array will be right-padded with 

155 np.nan to match the length of rawExpTimes. 

156 finalModelVars : `dict`, [`str`, `np.ndarray`] 

157 Dictionary keyed by amp names containing the masked modeled 

158 variance of the difference image of each flat pair. If needed, each 

159 array will be right-padded with np.nan to match the length of 

160 rawExpTimes. 

161 finalMeans : `dict`, [`str`, `np.ndarray`] 

162 Dictionary keyed by amp names containing the masked average of the 

163 means of the exposures in each flat pair. If needed, each array 

164 will be right-padded with np.nan to match the length of 

165 rawExpTimes. 

166 photoCharges : `dict`, [`str`, `np.ndarray`] 

167 Dictionary keyed by amp names containing the integrated photocharge 

168 for linearity calibration. 

169 auxValues : `dict`, [`str`, `np.ndarray`] 

170 Dictionary of per-detector auxiliary header values that can be used 

171 for PTC, linearity computation. 

172 

173 Version 1.1 adds the `ptcTurnoff` attribute. 

174 Version 1.2 adds the `histVars`, `histChi2Dofs`, and `kspValues` 

175 attributes. 

176 Version 1.3 adds the `noiseMatrix` and `noiseMatrixNoB` attributes. 

177 Version 1.4 adds the `auxValues` attribute. 

178 Version 1.5 adds the `covMatrixSideFullCovFit` attribute. 

179 """ 

180 

181 _OBSTYPE = 'PTC' 

182 _SCHEMA = 'Gen3 Photon Transfer Curve' 

183 _VERSION = 1.5 

184 

185 def __init__(self, ampNames=[], ptcFitType=None, covMatrixSide=1, 

186 covMatrixSideFullCovFit=None, **kwargs): 

187 self.ptcFitType = ptcFitType 

188 self.ampNames = ampNames 

189 self.covMatrixSide = covMatrixSide 

190 if covMatrixSideFullCovFit is None: 

191 self.covMatrixSideFullCovFit = covMatrixSide 

192 else: 

193 self.covMatrixSideFullCovFit = covMatrixSideFullCovFit 

194 

195 self.badAmps = [] 

196 

197 self.inputExpIdPairs = {ampName: [] for ampName in ampNames} 

198 self.expIdMask = {ampName: np.array([], dtype=bool) for ampName in ampNames} 

199 self.rawExpTimes = {ampName: np.array([]) for ampName in ampNames} 

200 self.rawMeans = {ampName: np.array([]) for ampName in ampNames} 

201 self.rawVars = {ampName: np.array([]) for ampName in ampNames} 

202 self.photoCharges = {ampName: np.array([]) for ampName in ampNames} 

203 

204 self.gain = {ampName: np.nan for ampName in ampNames} 

205 self.gainErr = {ampName: np.nan for ampName in ampNames} 

206 self.noise = {ampName: np.nan for ampName in ampNames} 

207 self.noiseErr = {ampName: np.nan for ampName in ampNames} 

208 

209 self.histVars = {ampName: np.array([]) for ampName in ampNames} 

210 self.histChi2Dofs = {ampName: np.array([]) for ampName in ampNames} 

211 self.kspValues = {ampName: np.array([]) for ampName in ampNames} 

212 

213 self.ptcFitPars = {ampName: np.array([]) for ampName in ampNames} 

214 self.ptcFitParsError = {ampName: np.array([]) for ampName in ampNames} 

215 self.ptcFitChiSq = {ampName: np.nan for ampName in ampNames} 

216 self.ptcTurnoff = {ampName: np.nan for ampName in ampNames} 

217 

218 self.covariances = {ampName: np.array([]) for ampName in ampNames} 

219 self.covariancesModel = {ampName: np.array([]) for ampName in ampNames} 

220 self.covariancesSqrtWeights = {ampName: np.array([]) for ampName in ampNames} 

221 self.aMatrix = {ampName: np.array([]) for ampName in ampNames} 

222 self.bMatrix = {ampName: np.array([]) for ampName in ampNames} 

223 self.noiseMatrix = {ampName: np.array([]) for ampName in ampNames} 

224 self.covariancesModelNoB = {ampName: np.array([]) for ampName in ampNames} 

225 self.aMatrixNoB = {ampName: np.array([]) for ampName in ampNames} 

226 self.noiseMatrixNoB = {ampName: np.array([]) for ampName in ampNames} 

227 

228 self.finalVars = {ampName: np.array([]) for ampName in ampNames} 

229 self.finalModelVars = {ampName: np.array([]) for ampName in ampNames} 

230 self.finalMeans = {ampName: np.array([]) for ampName in ampNames} 

231 

232 # Try this as a dict of arrays. 

233 self.auxValues = {} 

234 

235 super().__init__(**kwargs) 

236 self.requiredAttributes.update(['badAmps', 'inputExpIdPairs', 'expIdMask', 'rawExpTimes', 

237 'rawMeans', 'rawVars', 'gain', 'gainErr', 'noise', 'noiseErr', 

238 'ptcFitPars', 'ptcFitParsError', 'ptcFitChiSq', 'ptcTurnoff', 

239 'aMatrixNoB', 'covariances', 'covariancesModel', 

240 'covariancesSqrtWeights', 'covariancesModelNoB', 

241 'aMatrix', 'bMatrix', 'noiseMatrix', 'noiseMatrixNoB', 'finalVars', 

242 'finalModelVars', 'finalMeans', 'photoCharges', 'histVars', 

243 'histChi2Dofs', 'kspValues', 'auxValues']) 

244 

245 self.updateMetadata(setCalibInfo=True, setCalibId=True, **kwargs) 

246 self._validateCovarianceMatrizSizes() 

247 

248 def setAmpValuesPartialDataset( 

249 self, 

250 ampName, 

251 inputExpIdPair=(-1, -1), 

252 rawExpTime=np.nan, 

253 rawMean=np.nan, 

254 rawVar=np.nan, 

255 photoCharge=np.nan, 

256 expIdMask=False, 

257 covariance=None, 

258 covSqrtWeights=None, 

259 gain=np.nan, 

260 noise=np.nan, 

261 histVar=np.nan, 

262 histChi2Dof=np.nan, 

263 kspValue=0.0, 

264 auxValues=None, 

265 ): 

266 """ 

267 Set the amp values for a partial PTC Dataset (from cpExtractPtcTask). 

268 

269 Parameters 

270 ---------- 

271 ampName : `str` 

272 Name of the amp to set the values. 

273 inputExpIdPair : `tuple` [`int`] 

274 Exposure IDs of input pair. 

275 rawExpTime : `float`, optional 

276 Exposure time for this exposure pair. 

277 rawMean : `float`, optional 

278 Average of the means of the exposures in this pair. 

279 rawVar : `float`, optional 

280 Variance of the difference of the exposures in this pair. 

281 photoCharge : `float`, optional 

282 Integrated photocharge for flat pair for linearity calibration. 

283 expIdMask : `bool`, optional 

284 Flag setting if this exposure pair should be used (True) 

285 or not used (False). 

286 covariance : `np.ndarray` or None, optional 

287 Measured covariance for this exposure pair. 

288 covSqrtWeights : `np.ndarray` or None, optional 

289 Measured sqrt of covariance weights in this exposure pair. 

290 gain : `float`, optional 

291 Estimated gain for this exposure pair. 

292 noise : `float`, optional 

293 Estimated read noise for this exposure pair. 

294 histVar : `float`, optional 

295 Variance estimated from fitting a histogram with a Gaussian model. 

296 histChi2Dof : `float`, optional 

297 Chi-squared per degree of freedom from Gaussian histogram fit. 

298 kspValue : `float`, optional 

299 KS test p-value from the Gaussian histogram fit. 

300 """ 

301 nanMatrix = np.full((self.covMatrixSide, self.covMatrixSide), np.nan) 

302 nanMatrixFit = np.full((self.covMatrixSideFullCovFit, 

303 self.covMatrixSideFullCovFit), np.nan) 

304 if covariance is None: 

305 covariance = nanMatrix 

306 if covSqrtWeights is None: 

307 covSqrtWeights = nanMatrix 

308 

309 self.inputExpIdPairs[ampName] = [inputExpIdPair] 

310 self.rawExpTimes[ampName] = np.array([rawExpTime]) 

311 self.rawMeans[ampName] = np.array([rawMean]) 

312 self.rawVars[ampName] = np.array([rawVar]) 

313 self.photoCharges[ampName] = np.array([photoCharge]) 

314 self.expIdMask[ampName] = np.array([expIdMask]) 

315 self.covariances[ampName] = np.array([covariance]) 

316 self.covariancesSqrtWeights[ampName] = np.array([covSqrtWeights]) 

317 self.gain[ampName] = gain 

318 self.noise[ampName] = noise 

319 self.histVars[ampName] = np.array([histVar]) 

320 self.histChi2Dofs[ampName] = np.array([histChi2Dof]) 

321 self.kspValues[ampName] = np.array([kspValue]) 

322 

323 # From FULLCOVARIANCE model 

324 self.covariancesModel[ampName] = np.array([nanMatrixFit]) 

325 self.covariancesModelNoB[ampName] = np.array([nanMatrixFit]) 

326 self.aMatrix[ampName] = nanMatrixFit 

327 self.bMatrix[ampName] = nanMatrixFit 

328 self.aMatrixNoB[ampName] = nanMatrixFit 

329 self.noiseMatrix[ampName] = nanMatrixFit 

330 self.noiseMatrixNoB[ampName] = nanMatrixFit 

331 

332 def setAuxValuesPartialDataset(self, auxDict): 

333 """ 

334 Set a dictionary of auxiliary values for a partial dataset. 

335 

336 Parameters 

337 ---------- 

338 auxDict : `dict` [`str`, `float`] 

339 Dictionary of float values. 

340 """ 

341 for key, value in auxDict.items(): 

342 self.auxValues[key] = np.atleast_1d(np.array(value, dtype=np.float64)) 

343 

344 def updateMetadata(self, **kwargs): 

345 """Update calibration metadata. 

346 This calls the base class's method after ensuring the required 

347 calibration keywords will be saved. 

348 

349 Parameters 

350 ---------- 

351 setDate : `bool`, optional 

352 Update the CALIBDATE fields in the metadata to the current 

353 time. Defaults to False. 

354 kwargs : 

355 Other keyword parameters to set in the metadata. 

356 """ 

357 super().updateMetadata(PTC_FIT_TYPE=self.ptcFitType, **kwargs) 

358 

359 @classmethod 

360 def fromDict(cls, dictionary): 

361 """Construct a calibration from a dictionary of properties. 

362 Must be implemented by the specific calibration subclasses. 

363 

364 Parameters 

365 ---------- 

366 dictionary : `dict` 

367 Dictionary of properties. 

368 

369 Returns 

370 ------- 

371 calib : `lsst.ip.isr.PhotonTransferCurveDataset` 

372 Constructed calibration. 

373 

374 Raises 

375 ------ 

376 RuntimeError 

377 Raised if the supplied dictionary is for a different 

378 calibration. 

379 """ 

380 calib = cls() 

381 if calib._OBSTYPE != dictionary['metadata']['OBSTYPE']: 

382 raise RuntimeError(f"Incorrect Photon Transfer Curve dataset supplied. " 

383 f"Expected {calib._OBSTYPE}, found {dictionary['metadata']['OBSTYPE']}") 

384 calib.setMetadata(dictionary['metadata']) 

385 calib.ptcFitType = dictionary['ptcFitType'] 

386 calib.covMatrixSide = dictionary['covMatrixSide'] 

387 calib.covMatrixSideFullCovFit = dictionary['covMatrixSideFullCovFit'] 

388 calib.badAmps = np.array(dictionary['badAmps'], 'str').tolist() 

389 calib.ampNames = [] 

390 

391 # The cov matrices are square 

392 covMatrixSide = calib.covMatrixSide 

393 covMatrixSideFullCovFit = calib.covMatrixSideFullCovFit 

394 # Number of final signal levels 

395 covDimensionsProduct = len(np.array(list(dictionary['covariances'].values())[0]).ravel()) 

396 nSignalPoints = int(covDimensionsProduct/(covMatrixSide*covMatrixSide)) 

397 

398 for ampName in dictionary['ampNames']: 

399 calib.ampNames.append(ampName) 

400 calib.inputExpIdPairs[ampName] = dictionary['inputExpIdPairs'][ampName] 

401 calib.expIdMask[ampName] = np.array(dictionary['expIdMask'][ampName]) 

402 calib.rawExpTimes[ampName] = np.array(dictionary['rawExpTimes'][ampName], dtype=np.float64) 

403 calib.rawMeans[ampName] = np.array(dictionary['rawMeans'][ampName], dtype=np.float64) 

404 calib.rawVars[ampName] = np.array(dictionary['rawVars'][ampName], dtype=np.float64) 

405 calib.gain[ampName] = float(dictionary['gain'][ampName]) 

406 calib.gainErr[ampName] = float(dictionary['gainErr'][ampName]) 

407 calib.noise[ampName] = float(dictionary['noise'][ampName]) 

408 calib.noiseErr[ampName] = float(dictionary['noiseErr'][ampName]) 

409 calib.histVars[ampName] = np.array(dictionary['histVars'][ampName], dtype=np.float64) 

410 calib.histChi2Dofs[ampName] = np.array(dictionary['histChi2Dofs'][ampName], dtype=np.float64) 

411 calib.kspValues[ampName] = np.array(dictionary['kspValues'][ampName], dtype=np.float64) 

412 calib.ptcFitPars[ampName] = np.array(dictionary['ptcFitPars'][ampName], dtype=np.float64) 

413 calib.ptcFitParsError[ampName] = np.array(dictionary['ptcFitParsError'][ampName], 

414 dtype=np.float64) 

415 calib.ptcFitChiSq[ampName] = float(dictionary['ptcFitChiSq'][ampName]) 

416 calib.ptcTurnoff[ampName] = float(dictionary['ptcTurnoff'][ampName]) 

417 if nSignalPoints > 0: 

418 # Regular dataset 

419 calib.covariances[ampName] = np.array(dictionary['covariances'][ampName], 

420 dtype=np.float64).reshape( 

421 (nSignalPoints, covMatrixSide, covMatrixSide)) 

422 calib.covariancesModel[ampName] = np.array( 

423 dictionary['covariancesModel'][ampName], 

424 dtype=np.float64).reshape( 

425 (nSignalPoints, covMatrixSideFullCovFit, covMatrixSideFullCovFit)) 

426 calib.covariancesSqrtWeights[ampName] = np.array( 

427 dictionary['covariancesSqrtWeights'][ampName], 

428 dtype=np.float64).reshape( 

429 (nSignalPoints, covMatrixSide, covMatrixSide)) 

430 calib.aMatrix[ampName] = np.array(dictionary['aMatrix'][ampName], 

431 dtype=np.float64).reshape( 

432 (covMatrixSideFullCovFit, covMatrixSideFullCovFit)) 

433 calib.bMatrix[ampName] = np.array(dictionary['bMatrix'][ampName], 

434 dtype=np.float64).reshape( 

435 (covMatrixSideFullCovFit, covMatrixSideFullCovFit)) 

436 calib.covariancesModelNoB[ampName] = np.array( 

437 dictionary['covariancesModelNoB'][ampName], dtype=np.float64).reshape( 

438 (nSignalPoints, covMatrixSideFullCovFit, covMatrixSideFullCovFit)) 

439 calib.aMatrixNoB[ampName] = np.array( 

440 dictionary['aMatrixNoB'][ampName], 

441 dtype=np.float64).reshape((covMatrixSideFullCovFit, covMatrixSideFullCovFit)) 

442 calib.noiseMatrix[ampName] = np.array( 

443 dictionary['noiseMatrix'][ampName], 

444 dtype=np.float64).reshape((covMatrixSideFullCovFit, covMatrixSideFullCovFit)) 

445 calib.noiseMatrixNoB[ampName] = np.array( 

446 dictionary['noiseMatrixNoB'][ampName], 

447 dtype=np.float64).reshape((covMatrixSideFullCovFit, covMatrixSideFullCovFit)) 

448 else: 

449 # Empty dataset 

450 calib.covariances[ampName] = np.array([], dtype=np.float64) 

451 calib.covariancesModel[ampName] = np.array([], dtype=np.float64) 

452 calib.covariancesSqrtWeights[ampName] = np.array([], dtype=np.float64) 

453 calib.aMatrix[ampName] = np.array([], dtype=np.float64) 

454 calib.bMatrix[ampName] = np.array([], dtype=np.float64) 

455 calib.covariancesModelNoB[ampName] = np.array([], dtype=np.float64) 

456 calib.aMatrixNoB[ampName] = np.array([], dtype=np.float64) 

457 calib.noiseMatrix[ampName] = np.array([], dtype=np.float64) 

458 calib.noiseMatrixNoB[ampName] = np.array([], dtype=np.float64) 

459 

460 calib.finalVars[ampName] = np.array(dictionary['finalVars'][ampName], dtype=np.float64) 

461 calib.finalModelVars[ampName] = np.array(dictionary['finalModelVars'][ampName], dtype=np.float64) 

462 calib.finalMeans[ampName] = np.array(dictionary['finalMeans'][ampName], dtype=np.float64) 

463 calib.photoCharges[ampName] = np.array(dictionary['photoCharges'][ampName], dtype=np.float64) 

464 

465 for key, value in dictionary['auxValues'].items(): 

466 calib.auxValues[key] = np.atleast_1d(np.array(value, dtype=np.float64)) 

467 

468 calib.updateMetadata() 

469 return calib 

470 

471 def toDict(self): 

472 """Return a dictionary containing the calibration properties. 

473 The dictionary should be able to be round-tripped through 

474 `fromDict`. 

475 

476 Returns 

477 ------- 

478 dictionary : `dict` 

479 Dictionary of properties. 

480 """ 

481 self.updateMetadata() 

482 

483 outDict = dict() 

484 metadata = self.getMetadata() 

485 outDict['metadata'] = metadata 

486 

487 def _dictOfArraysToDictOfLists(dictOfArrays): 

488 dictOfLists = {} 

489 for key, value in dictOfArrays.items(): 

490 dictOfLists[key] = value.ravel().tolist() 

491 

492 return dictOfLists 

493 

494 outDict['ptcFitType'] = self.ptcFitType 

495 outDict['covMatrixSide'] = self.covMatrixSide 

496 outDict['covMatrixSideFullCovFit'] = self.covMatrixSideFullCovFit 

497 outDict['ampNames'] = self.ampNames 

498 outDict['badAmps'] = self.badAmps 

499 outDict['inputExpIdPairs'] = self.inputExpIdPairs 

500 outDict['expIdMask'] = _dictOfArraysToDictOfLists(self.expIdMask) 

501 outDict['rawExpTimes'] = _dictOfArraysToDictOfLists(self.rawExpTimes) 

502 outDict['rawMeans'] = _dictOfArraysToDictOfLists(self.rawMeans) 

503 outDict['rawVars'] = _dictOfArraysToDictOfLists(self.rawVars) 

504 outDict['gain'] = self.gain 

505 outDict['gainErr'] = self.gainErr 

506 outDict['noise'] = self.noise 

507 outDict['noiseErr'] = self.noiseErr 

508 outDict['histVars'] = self.histVars 

509 outDict['histChi2Dofs'] = self.histChi2Dofs 

510 outDict['kspValues'] = self.kspValues 

511 outDict['ptcFitPars'] = _dictOfArraysToDictOfLists(self.ptcFitPars) 

512 outDict['ptcFitParsError'] = _dictOfArraysToDictOfLists(self.ptcFitParsError) 

513 outDict['ptcFitChiSq'] = self.ptcFitChiSq 

514 outDict['ptcTurnoff'] = self.ptcTurnoff 

515 outDict['covariances'] = _dictOfArraysToDictOfLists(self.covariances) 

516 outDict['covariancesModel'] = _dictOfArraysToDictOfLists(self.covariancesModel) 

517 outDict['covariancesSqrtWeights'] = _dictOfArraysToDictOfLists(self.covariancesSqrtWeights) 

518 outDict['aMatrix'] = _dictOfArraysToDictOfLists(self.aMatrix) 

519 outDict['bMatrix'] = _dictOfArraysToDictOfLists(self.bMatrix) 

520 outDict['noiseMatrix'] = _dictOfArraysToDictOfLists(self.noiseMatrix) 

521 outDict['covariancesModelNoB'] = _dictOfArraysToDictOfLists(self.covariancesModelNoB) 

522 outDict['aMatrixNoB'] = _dictOfArraysToDictOfLists(self.aMatrixNoB) 

523 outDict['noiseMatrixNoB'] = _dictOfArraysToDictOfLists(self.noiseMatrixNoB) 

524 outDict['finalVars'] = _dictOfArraysToDictOfLists(self.finalVars) 

525 outDict['finalModelVars'] = _dictOfArraysToDictOfLists(self.finalModelVars) 

526 outDict['finalMeans'] = _dictOfArraysToDictOfLists(self.finalMeans) 

527 outDict['photoCharges'] = _dictOfArraysToDictOfLists(self.photoCharges) 

528 outDict['auxValues'] = _dictOfArraysToDictOfLists(self.auxValues) 

529 

530 return outDict 

531 

532 @classmethod 

533 def fromTable(cls, tableList): 

534 """Construct calibration from a list of tables. 

535 This method uses the `fromDict` method to create the 

536 calibration, after constructing an appropriate dictionary from 

537 the input tables. 

538 

539 Parameters 

540 ---------- 

541 tableList : `list` [`lsst.afw.table.Table`] 

542 List of tables to use to construct the datasetPtc. 

543 

544 Returns 

545 ------- 

546 calib : `lsst.ip.isr.PhotonTransferCurveDataset` 

547 The calibration defined in the tables. 

548 """ 

549 ptcTable = tableList[0] 

550 

551 metadata = ptcTable.meta 

552 inDict = dict() 

553 inDict['metadata'] = metadata 

554 inDict['ampNames'] = [] 

555 inDict['ptcFitType'] = [] 

556 inDict['covMatrixSide'] = [] 

557 inDict['covMatrixSideFullCovFit'] = [] 

558 inDict['inputExpIdPairs'] = dict() 

559 inDict['expIdMask'] = dict() 

560 inDict['rawExpTimes'] = dict() 

561 inDict['rawMeans'] = dict() 

562 inDict['rawVars'] = dict() 

563 inDict['gain'] = dict() 

564 inDict['gainErr'] = dict() 

565 inDict['noise'] = dict() 

566 inDict['noiseErr'] = dict() 

567 inDict['histVars'] = dict() 

568 inDict['histChi2Dofs'] = dict() 

569 inDict['kspValues'] = dict() 

570 inDict['ptcFitPars'] = dict() 

571 inDict['ptcFitParsError'] = dict() 

572 inDict['ptcFitChiSq'] = dict() 

573 inDict['ptcTurnoff'] = dict() 

574 inDict['covariances'] = dict() 

575 inDict['covariancesModel'] = dict() 

576 inDict['covariancesSqrtWeights'] = dict() 

577 inDict['aMatrix'] = dict() 

578 inDict['bMatrix'] = dict() 

579 inDict['noiseMatrix'] = dict() 

580 inDict['covariancesModelNoB'] = dict() 

581 inDict['aMatrixNoB'] = dict() 

582 inDict['noiseMatrixNoB'] = dict() 

583 inDict['finalVars'] = dict() 

584 inDict['finalModelVars'] = dict() 

585 inDict['finalMeans'] = dict() 

586 inDict['badAmps'] = [] 

587 inDict['photoCharges'] = dict() 

588 

589 calibVersion = metadata['PTC_VERSION'] 

590 if calibVersion == 1.0: 

591 cls().log.warning(f"Previous version found for PTC dataset: {calibVersion}. " 

592 f"Setting 'ptcTurnoff' in all amps to last value in 'finalMeans'.") 

593 for record in ptcTable: 

594 ampName = record['AMPLIFIER_NAME'] 

595 

596 inDict['ptcFitType'] = record['PTC_FIT_TYPE'] 

597 inDict['covMatrixSide'] = record['COV_MATRIX_SIDE'] 

598 inDict['ampNames'].append(ampName) 

599 inDict['inputExpIdPairs'][ampName] = record['INPUT_EXP_ID_PAIRS'].tolist() 

600 inDict['expIdMask'][ampName] = record['EXP_ID_MASK'] 

601 inDict['rawExpTimes'][ampName] = record['RAW_EXP_TIMES'] 

602 inDict['rawMeans'][ampName] = record['RAW_MEANS'] 

603 inDict['rawVars'][ampName] = record['RAW_VARS'] 

604 inDict['gain'][ampName] = record['GAIN'] 

605 inDict['gainErr'][ampName] = record['GAIN_ERR'] 

606 inDict['noise'][ampName] = record['NOISE'] 

607 inDict['noiseErr'][ampName] = record['NOISE_ERR'] 

608 inDict['ptcFitPars'][ampName] = record['PTC_FIT_PARS'] 

609 inDict['ptcFitParsError'][ampName] = record['PTC_FIT_PARS_ERROR'] 

610 inDict['ptcFitChiSq'][ampName] = record['PTC_FIT_CHI_SQ'] 

611 inDict['covariances'][ampName] = record['COVARIANCES'] 

612 inDict['covariancesModel'][ampName] = record['COVARIANCES_MODEL'] 

613 inDict['covariancesSqrtWeights'][ampName] = record['COVARIANCES_SQRT_WEIGHTS'] 

614 inDict['aMatrix'][ampName] = record['A_MATRIX'] 

615 inDict['bMatrix'][ampName] = record['B_MATRIX'] 

616 inDict['covariancesModelNoB'][ampName] = record['COVARIANCES_MODEL_NO_B'] 

617 inDict['aMatrixNoB'][ampName] = record['A_MATRIX_NO_B'] 

618 inDict['finalVars'][ampName] = record['FINAL_VARS'] 

619 inDict['finalModelVars'][ampName] = record['FINAL_MODEL_VARS'] 

620 inDict['finalMeans'][ampName] = record['FINAL_MEANS'] 

621 inDict['badAmps'] = record['BAD_AMPS'].tolist() 

622 inDict['photoCharges'][ampName] = record['PHOTO_CHARGE'] 

623 if calibVersion == 1.0: 

624 mask = record['FINAL_MEANS'].mask 

625 array = record['FINAL_MEANS'][~mask] 

626 if len(array) > 0: 

627 inDict['ptcTurnoff'][ampName] = record['FINAL_MEANS'][~mask][-1] 

628 else: 

629 inDict['ptcTurnoff'][ampName] = np.nan 

630 else: 

631 inDict['ptcTurnoff'][ampName] = record['PTC_TURNOFF'] 

632 if calibVersion < 1.2: 

633 inDict['histVars'][ampName] = np.array([np.nan]) 

634 inDict['histChi2Dofs'][ampName] = np.array([np.nan]) 

635 inDict['kspValues'][ampName] = np.array([0.0]) 

636 else: 

637 inDict['histVars'][ampName] = record['HIST_VARS'] 

638 inDict['histChi2Dofs'][ampName] = record['HIST_CHI2_DOFS'] 

639 inDict['kspValues'][ampName] = record['KS_PVALUES'] 

640 if calibVersion < 1.3: 

641 nanMatrix = np.full_like(inDict['aMatrix'][ampName], np.nan) 

642 inDict['noiseMatrix'][ampName] = nanMatrix 

643 inDict['noiseMatrixNoB'][ampName] = nanMatrix 

644 else: 

645 inDict['noiseMatrix'][ampName] = record['NOISE_MATRIX'] 

646 inDict['noiseMatrixNoB'][ampName] = record['NOISE_MATRIX_NO_B'] 

647 if calibVersion < 1.5: 

648 # Matched to `COV_MATRIX_SIDE`. Same for all amps. 

649 inDict['covMatrixSideFullCovFit'] = inDict['covMatrixSide'] 

650 else: 

651 inDict['covMatrixSideFullCovFit'] = record['COV_MATRIX_SIDE_FULL_COV_FIT'] 

652 

653 inDict['auxValues'] = {} 

654 record = ptcTable[0] 

655 for col in record.columns.keys(): 

656 if col.startswith('PTCAUX_'): 

657 parts = col.split('PTCAUX_') 

658 inDict['auxValues'][parts[1]] = record[col] 

659 

660 return cls().fromDict(inDict) 

661 

662 def toTable(self): 

663 """Construct a list of tables containing the information in this 

664 calibration. 

665 

666 The list of tables should create an identical calibration 

667 after being passed to this class's fromTable method. 

668 

669 Returns 

670 ------- 

671 tableList : `list` [`astropy.table.Table`] 

672 List of tables containing the linearity calibration 

673 information. 

674 """ 

675 tableList = [] 

676 self.updateMetadata() 

677 

678 badAmps = np.array(self.badAmps) if len(self.badAmps) else np.array([], dtype="U3") 

679 

680 catalogList = [] 

681 for ampName in self.ampNames: 

682 ampDict = { 

683 'AMPLIFIER_NAME': ampName, 

684 'PTC_FIT_TYPE': self.ptcFitType, 

685 'COV_MATRIX_SIDE': self.covMatrixSide, 

686 'COV_MATRIX_SIDE_FULL_COV_FIT': self.covMatrixSideFullCovFit, 

687 'INPUT_EXP_ID_PAIRS': self.inputExpIdPairs[ampName], 

688 'EXP_ID_MASK': self.expIdMask[ampName], 

689 'RAW_EXP_TIMES': self.rawExpTimes[ampName], 

690 'RAW_MEANS': self.rawMeans[ampName], 

691 'RAW_VARS': self.rawVars[ampName], 

692 'GAIN': self.gain[ampName], 

693 'GAIN_ERR': self.gainErr[ampName], 

694 'NOISE': self.noise[ampName], 

695 'NOISE_ERR': self.noiseErr[ampName], 

696 'HIST_VARS': self.histVars[ampName], 

697 'HIST_CHI2_DOFS': self.histChi2Dofs[ampName], 

698 'KS_PVALUES': self.kspValues[ampName], 

699 'PTC_FIT_PARS': np.array(self.ptcFitPars[ampName]), 

700 'PTC_FIT_PARS_ERROR': np.array(self.ptcFitParsError[ampName]), 

701 'PTC_FIT_CHI_SQ': self.ptcFitChiSq[ampName], 

702 'PTC_TURNOFF': self.ptcTurnoff[ampName], 

703 'A_MATRIX': self.aMatrix[ampName].ravel(), 

704 'B_MATRIX': self.bMatrix[ampName].ravel(), 

705 'A_MATRIX_NO_B': self.aMatrixNoB[ampName].ravel(), 

706 'NOISE_MATRIX': self.noiseMatrix[ampName].ravel(), 

707 'NOISE_MATRIX_NO_B': self.noiseMatrixNoB[ampName].ravel(), 

708 'BAD_AMPS': badAmps, 

709 'PHOTO_CHARGE': self.photoCharges[ampName], 

710 'COVARIANCES': self.covariances[ampName].ravel(), 

711 'COVARIANCES_MODEL': self.covariancesModel[ampName].ravel(), 

712 'COVARIANCES_SQRT_WEIGHTS': self.covariancesSqrtWeights[ampName].ravel(), 

713 'COVARIANCES_MODEL_NO_B': self.covariancesModelNoB[ampName].ravel(), 

714 'FINAL_VARS': self.finalVars[ampName], 

715 'FINAL_MODEL_VARS': self.finalModelVars[ampName], 

716 'FINAL_MEANS': self.finalMeans[ampName], 

717 } 

718 

719 if self.auxValues: 

720 for key, value in self.auxValues.items(): 

721 ampDict[f"PTCAUX_{key}"] = value 

722 

723 catalogList.append(ampDict) 

724 

725 catalog = Table(catalogList) 

726 

727 inMeta = self.getMetadata().toDict() 

728 outMeta = {k: v for k, v in inMeta.items() if v is not None} 

729 outMeta.update({k: "" for k, v in inMeta.items() if v is None}) 

730 catalog.meta = outMeta 

731 tableList.append(catalog) 

732 

733 return tableList 

734 

735 def fromDetector(self, detector): 

736 """Read metadata parameters from a detector. 

737 

738 Parameters 

739 ---------- 

740 detector : `lsst.afw.cameraGeom.detector` 

741 Input detector with parameters to use. 

742 

743 Returns 

744 ------- 

745 calib : `lsst.ip.isr.PhotonTransferCurveDataset` 

746 The calibration constructed from the detector. 

747 """ 

748 

749 pass 

750 

751 def getExpIdsUsed(self, ampName): 

752 """Get the exposures used, i.e. not discarded, for a given amp. 

753 If no mask has been created yet, all exposures are returned. 

754 

755 Parameters 

756 ---------- 

757 ampName : `str` 

758 

759 Returns 

760 ------- 

761 expIdsUsed : `list` [`tuple`] 

762 List of pairs of exposure ids used in PTC. 

763 """ 

764 if len(self.expIdMask[ampName]) == 0: 

765 return self.inputExpIdPairs[ampName] 

766 

767 # if the mask exists it had better be the same length as the expIdPairs 

768 assert len(self.expIdMask[ampName]) == len(self.inputExpIdPairs[ampName]) 

769 

770 pairs = self.inputExpIdPairs[ampName] 

771 mask = self.expIdMask[ampName] 

772 # cast to bool required because numpy 

773 try: 

774 expIdsUsed = [(exp1, exp2) for ((exp1, exp2), m) in zip(pairs, mask) if m] 

775 except ValueError: 

776 self.log.warning("The PTC file was written incorrectly; you should rerun the " 

777 "PTC solve task if possible.") 

778 expIdsUsed = [] 

779 for pairList, m in zip(pairs, mask): 

780 if m: 

781 expIdsUsed.append(pairList[0]) 

782 

783 return expIdsUsed 

784 

785 def getGoodAmps(self): 

786 """Get the good amps from this PTC.""" 

787 return [amp for amp in self.ampNames if amp not in self.badAmps] 

788 

789 def getGoodPoints(self, ampName): 

790 """Get the good points used for a given amp in the PTC. 

791 

792 Parameters 

793 ---------- 

794 ampName : `str` 

795 Amplifier's name. 

796 

797 Returns 

798 ------- 

799 goodPoints : `np.ndarray` 

800 Boolean array of good points used in PTC. 

801 """ 

802 return self.expIdMask[ampName] 

803 

804 def validateGainNoiseTurnoffValues(self, ampName, doWarn=False): 

805 """Ensure the gain, read noise, and PTC turnoff have 

806 sensible values. 

807 

808 Parameters 

809 ---------- 

810 ampName : `str` 

811 Amplifier's name. 

812 """ 

813 

814 gain = self.gain[ampName] 

815 noise = self.noise[ampName] 

816 ptcTurnoff = self.ptcTurnoff[ampName] 

817 

818 # Check if gain is not positive or is np.nan 

819 if not (isinstance(gain, (int, float)) and gain > 0) or math.isnan(gain): 

820 if doWarn: 

821 self.log.warning(f"Invalid gain value {gain}" 

822 " Setting to default: Gain=1") 

823 gain = 1 

824 

825 # Check if noise is not positive or is np.nan 

826 if not (isinstance(noise, (int, float)) and noise > 0) or math.isnan(noise): 

827 if doWarn: 

828 self.log.warning(f"Invalid noise value: {noise}" 

829 " Setting to default: Noise=1") 

830 noise = 1 

831 

832 # Check if ptcTurnoff is not positive or is np.nan 

833 if not (isinstance(ptcTurnoff, (int, float)) and ptcTurnoff > 0) or math.isnan(ptcTurnoff): 

834 if doWarn: 

835 self.log.warning(f"Invalid PTC turnoff value: {ptcTurnoff}" 

836 " Setting to default: PTC Turnoff=2e19") 

837 ptcTurnoff = 2e19 

838 

839 self.gain[ampName] = gain 

840 self.noise[ampName] = noise 

841 self.ptcTurnoff[ampName] = ptcTurnoff 

842 

843 def _validateCovarianceMatrizSizes(self): 

844 """Ensure covMatrixSideFullCovFit <= covMatrixSide.""" 

845 if self.covMatrixSideFullCovFit > self.covMatrixSide: 

846 self.log.warning("covMatrixSideFullCovFit > covMatrixSide " 

847 f"({self.covMatrixSideFullCovFit} > {self.covMatrixSide})." 

848 "Setting the former to the latter.") 

849 self.covMatrixSideFullCovFit = self.covMatrixSide