Coverage for python/lsst/ip/isr/ptcDataset.py: 6%

338 statements  

« prev     ^ index     » next       coverage.py v7.3.2, created at 2023-12-12 13:11 +0000

1# 

2# LSST Data Management System 

3# Copyright 2008-2017 AURA/LSST. 

4# 

5# This product includes software developed by the 

6# LSST Project (http://www.lsst.org/). 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <https://www.lsstcorp.org/LegalNotices/>. 

21# 

22""" 

23Define dataset class for MeasurePhotonTransferCurve task 

24""" 

25 

26__all__ = ['PhotonTransferCurveDataset'] 

27 

28import numpy as np 

29import math 

30from astropy.table import Table 

31 

32from lsst.ip.isr import IsrCalib 

33 

34 

35class PhotonTransferCurveDataset(IsrCalib): 

36 """A simple class to hold the output data from the PTC task. 

37 

38 The dataset is made up of a dictionary for each item, keyed by the 

39 amplifiers' names, which much be supplied at construction time. 

40 New items cannot be added to the class to save accidentally saving to the 

41 wrong property, and the class can be frozen if desired. 

42 inputExpIdPairs records the exposures used to produce the data. 

43 When fitPtc() or fitCovariancesAstier() is run, a mask is built up, which 

44 is by definition always the same length as inputExpIdPairs, rawExpTimes, 

45 rawMeans and rawVars, and is a list of bools, which are incrementally set 

46 to False as points are discarded from the fits. 

47 PTC fit parameters for polynomials are stored in a list in ascending order 

48 of polynomial term, i.e. par[0]*x^0 + par[1]*x + par[2]*x^2 etc 

49 with the length of the list corresponding to the order of the polynomial 

50 plus one. 

51 

52 Parameters 

53 ---------- 

54 ampNames : `list` 

55 List with the names of the amplifiers of the detector at hand. 

56 ptcFitType : `str` 

57 Type of model fitted to the PTC: "POLYNOMIAL", "EXPAPPROXIMATION", 

58 or "FULLCOVARIANCE". 

59 covMatrixSide : `int` 

60 Maximum lag of covariances (size of square covariance matrices). 

61 kwargs : `dict`, optional 

62 Other keyword arguments to pass to the parent init. 

63 

64 Notes 

65 ----- 

66 The stored attributes are: 

67 

68 badAmps : `list` [`str`] 

69 List with bad amplifiers names. 

70 inputExpIdPairs : `dict`, [`str`, `list`] 

71 Dictionary keyed by amp names containing the input exposures IDs. 

72 expIdMask : `dict`, [`str`, `np.ndarray`] 

73 Dictionary keyed by amp names containing the mask produced after 

74 outlier rejection. The mask produced by the "FULLCOVARIANCE" 

75 option may differ from the one produced in the other two PTC 

76 fit types. 

77 rawExpTimes : `dict`, [`str`, `np.ndarray`] 

78 Dictionary keyed by amp names containing the unmasked exposure times. 

79 rawMeans : `dict`, [`str`, `np.ndarray`] 

80 Dictionary keyed by amp names containing the unmasked average of the 

81 means of the exposures in each flat pair. 

82 rawVars : `dict`, [`str`, `np.ndarray`] 

83 Dictionary keyed by amp names containing the variance of the 

84 difference image of the exposures in each flat pair. 

85 histVars : `dict`, [`str`, `np.ndarray`] 

86 Dictionary keyed by amp names containing the variance of the 

87 difference image of the exposures in each flat pair estimated 

88 by fitting a Gaussian model. 

89 histChi2Dofs : `dict`, [`str`, `np.ndarray`] 

90 Dictionary keyed by amp names containing the chi-squared per degree 

91 of freedom fitting the difference image to a Gaussian model. 

92 kspValues : `dict`, [`str`, `np.ndarray`] 

93 Dictionary keyed by amp names containing the KS test p-value from 

94 fitting the difference image to a Gaussian model. 

95 gain : `dict`, [`str`, `float`] 

96 Dictionary keyed by amp names containing the fitted gains. 

97 gainErr : `dict`, [`str`, `float`] 

98 Dictionary keyed by amp names containing the errors on the 

99 fitted gains. 

100 noise : `dict`, [`str`, `float`] 

101 Dictionary keyed by amp names containing the fitted noise. 

102 noiseErr : `dict`, [`str`, `float`] 

103 Dictionary keyed by amp names containing the errors on the fitted 

104 noise. 

105 ptcFitPars : `dict`, [`str`, `np.ndarray`] 

106 Dictionary keyed by amp names containing the fitted parameters of the 

107 PTC model for ptcFitTye in ["POLYNOMIAL", "EXPAPPROXIMATION"]. 

108 ptcFitParsError : `dict`, [`str`, `np.ndarray`] 

109 Dictionary keyed by amp names containing the errors on the fitted 

110 parameters of the PTC model for ptcFitTye in 

111 ["POLYNOMIAL", "EXPAPPROXIMATION"]. 

112 ptcFitChiSq : `dict`, [`str`, `float`] 

113 Dictionary keyed by amp names containing the reduced chi squared 

114 of the fit for ptcFitTye in ["POLYNOMIAL", "EXPAPPROXIMATION"]. 

115 ptcTurnoff : `dict` [`str, `float`] 

116 Flux value (in ADU) where the variance of the PTC curve starts 

117 decreasing consistently. 

118 covariances : `dict`, [`str`, `np.ndarray`] 

119 Dictionary keyed by amp names containing a list of measured 

120 covariances per mean flux. 

121 covariancesModel : `dict`, [`str`, `np.ndarray`] 

122 Dictionary keyed by amp names containinging covariances model 

123 (Eq. 20 of Astier+19) per mean flux. 

124 covariancesSqrtWeights : `dict`, [`str`, `np.ndarray`] 

125 Dictionary keyed by amp names containinging sqrt. of covariances 

126 weights. 

127 aMatrix : `dict`, [`str`, `np.ndarray`] 

128 Dictionary keyed by amp names containing the "a" parameters from 

129 the model in Eq. 20 of Astier+19. 

130 bMatrix : `dict`, [`str`, `np.ndarray`] 

131 Dictionary keyed by amp names containing the "b" parameters from 

132 the model in Eq. 20 of Astier+19. 

133 noiseMatrix : `dict`, [`str`, `np.ndarray`] 

134 Dictionary keyed by amp names containing the "noise" parameters from 

135 the model in Eq. 20 of Astier+19. 

136 covariancesModelNoB : `dict`, [`str`, `np.ndarray`] 

137 Dictionary keyed by amp names containing covariances model 

138 (with 'b'=0 in Eq. 20 of Astier+19) 

139 per mean flux. 

140 aMatrixNoB : `dict`, [`str`, `np.ndarray`] 

141 Dictionary keyed by amp names containing the "a" parameters from the 

142 model in Eq. 20 of Astier+19 

143 (and 'b' = 0). 

144 noiseMatrixNoB : `dict`, [`str`, `np.ndarray`] 

145 Dictionary keyed by amp names containing the "noise" parameters from 

146 the model in Eq. 20 of Astier+19, with 'b' = 0. 

147 finalVars : `dict`, [`str`, `np.ndarray`] 

148 Dictionary keyed by amp names containing the masked variance of the 

149 difference image of each flat 

150 pair. If needed, each array will be right-padded with 

151 np.nan to match the length of rawExpTimes. 

152 finalModelVars : `dict`, [`str`, `np.ndarray`] 

153 Dictionary keyed by amp names containing the masked modeled 

154 variance of the difference image of each flat pair. If needed, each 

155 array will be right-padded with np.nan to match the length of 

156 rawExpTimes. 

157 finalMeans : `dict`, [`str`, `np.ndarray`] 

158 Dictionary keyed by amp names containing the masked average of the 

159 means of the exposures in each flat pair. If needed, each array 

160 will be right-padded with np.nan to match the length of 

161 rawExpTimes. 

162 photoCharges : `dict`, [`str`, `np.ndarray`] 

163 Dictionary keyed by amp names containing the integrated photocharge 

164 for linearity calibration. 

165 auxValues : `dict`, [`str`, `np.ndarray`] 

166 Dictionary of per-detector auxiliary header values that can be used 

167 for PTC, linearity computation. 

168 

169 Version 1.1 adds the `ptcTurnoff` attribute. 

170 Version 1.2 adds the `histVars`, `histChi2Dofs`, and `kspValues` 

171 attributes. 

172 Version 1.3 adds the `noiseMatrix` and `noiseMatrixNoB` attributes. 

173 Version 1.4 adds the `auxValues` attribute. 

174 """ 

175 

176 _OBSTYPE = 'PTC' 

177 _SCHEMA = 'Gen3 Photon Transfer Curve' 

178 _VERSION = 1.4 

179 

180 def __init__(self, ampNames=[], ptcFitType=None, covMatrixSide=1, **kwargs): 

181 self.ptcFitType = ptcFitType 

182 self.ampNames = ampNames 

183 self.covMatrixSide = covMatrixSide 

184 

185 self.badAmps = [] 

186 

187 self.inputExpIdPairs = {ampName: [] for ampName in ampNames} 

188 self.expIdMask = {ampName: np.array([], dtype=bool) for ampName in ampNames} 

189 self.rawExpTimes = {ampName: np.array([]) for ampName in ampNames} 

190 self.rawMeans = {ampName: np.array([]) for ampName in ampNames} 

191 self.rawVars = {ampName: np.array([]) for ampName in ampNames} 

192 self.photoCharges = {ampName: np.array([]) for ampName in ampNames} 

193 

194 self.gain = {ampName: np.nan for ampName in ampNames} 

195 self.gainErr = {ampName: np.nan for ampName in ampNames} 

196 self.noise = {ampName: np.nan for ampName in ampNames} 

197 self.noiseErr = {ampName: np.nan for ampName in ampNames} 

198 

199 self.histVars = {ampName: np.array([]) for ampName in ampNames} 

200 self.histChi2Dofs = {ampName: np.array([]) for ampName in ampNames} 

201 self.kspValues = {ampName: np.array([]) for ampName in ampNames} 

202 

203 self.ptcFitPars = {ampName: np.array([]) for ampName in ampNames} 

204 self.ptcFitParsError = {ampName: np.array([]) for ampName in ampNames} 

205 self.ptcFitChiSq = {ampName: np.nan for ampName in ampNames} 

206 self.ptcTurnoff = {ampName: np.nan for ampName in ampNames} 

207 

208 self.covariances = {ampName: np.array([]) for ampName in ampNames} 

209 self.covariancesModel = {ampName: np.array([]) for ampName in ampNames} 

210 self.covariancesSqrtWeights = {ampName: np.array([]) for ampName in ampNames} 

211 self.aMatrix = {ampName: np.array([]) for ampName in ampNames} 

212 self.bMatrix = {ampName: np.array([]) for ampName in ampNames} 

213 self.noiseMatrix = {ampName: np.array([]) for ampName in ampNames} 

214 self.covariancesModelNoB = {ampName: np.array([]) for ampName in ampNames} 

215 self.aMatrixNoB = {ampName: np.array([]) for ampName in ampNames} 

216 self.noiseMatrixNoB = {ampName: np.array([]) for ampName in ampNames} 

217 

218 self.finalVars = {ampName: np.array([]) for ampName in ampNames} 

219 self.finalModelVars = {ampName: np.array([]) for ampName in ampNames} 

220 self.finalMeans = {ampName: np.array([]) for ampName in ampNames} 

221 

222 # Try this as a dict of arrays. 

223 self.auxValues = {} 

224 

225 super().__init__(**kwargs) 

226 self.requiredAttributes.update(['badAmps', 'inputExpIdPairs', 'expIdMask', 'rawExpTimes', 

227 'rawMeans', 'rawVars', 'gain', 'gainErr', 'noise', 'noiseErr', 

228 'ptcFitPars', 'ptcFitParsError', 'ptcFitChiSq', 'ptcTurnoff', 

229 'aMatrixNoB', 'covariances', 'covariancesModel', 

230 'covariancesSqrtWeights', 'covariancesModelNoB', 

231 'aMatrix', 'bMatrix', 'noiseMatrix', 'noiseMatrixNoB', 'finalVars', 

232 'finalModelVars', 'finalMeans', 'photoCharges', 'histVars', 

233 'histChi2Dofs', 'kspValues', 'auxValues']) 

234 

235 self.updateMetadata(setCalibInfo=True, setCalibId=True, **kwargs) 

236 

237 def setAmpValuesPartialDataset( 

238 self, 

239 ampName, 

240 inputExpIdPair=(-1, -1), 

241 rawExpTime=np.nan, 

242 rawMean=np.nan, 

243 rawVar=np.nan, 

244 photoCharge=np.nan, 

245 expIdMask=False, 

246 covariance=None, 

247 covSqrtWeights=None, 

248 gain=np.nan, 

249 noise=np.nan, 

250 histVar=np.nan, 

251 histChi2Dof=np.nan, 

252 kspValue=0.0, 

253 auxValues=None, 

254 ): 

255 """ 

256 Set the amp values for a partial PTC Dataset (from cpExtractPtcTask). 

257 

258 Parameters 

259 ---------- 

260 ampName : `str` 

261 Name of the amp to set the values. 

262 inputExpIdPair : `tuple` [`int`] 

263 Exposure IDs of input pair. 

264 rawExpTime : `float`, optional 

265 Exposure time for this exposure pair. 

266 rawMean : `float`, optional 

267 Average of the means of the exposures in this pair. 

268 rawVar : `float`, optional 

269 Variance of the difference of the exposures in this pair. 

270 photoCharge : `float`, optional 

271 Integrated photocharge for flat pair for linearity calibration. 

272 expIdMask : `bool`, optional 

273 Flag setting if this exposure pair should be used (True) 

274 or not used (False). 

275 covariance : `np.ndarray` or None, optional 

276 Measured covariance for this exposure pair. 

277 covSqrtWeights : `np.ndarray` or None, optional 

278 Measured sqrt of covariance weights in this exposure pair. 

279 gain : `float`, optional 

280 Estimated gain for this exposure pair. 

281 noise : `float`, optional 

282 Estimated read noise for this exposure pair. 

283 histVar : `float`, optional 

284 Variance estimated from fitting a histogram with a Gaussian model. 

285 histChi2Dof : `float`, optional 

286 Chi-squared per degree of freedom from Gaussian histogram fit. 

287 kspValue : `float`, optional 

288 KS test p-value from the Gaussian histogram fit. 

289 """ 

290 nanMatrix = np.full((self.covMatrixSide, self.covMatrixSide), np.nan) 

291 if covariance is None: 

292 covariance = nanMatrix 

293 if covSqrtWeights is None: 

294 covSqrtWeights = nanMatrix 

295 

296 self.inputExpIdPairs[ampName] = [inputExpIdPair] 

297 self.rawExpTimes[ampName] = np.array([rawExpTime]) 

298 self.rawMeans[ampName] = np.array([rawMean]) 

299 self.rawVars[ampName] = np.array([rawVar]) 

300 self.photoCharges[ampName] = np.array([photoCharge]) 

301 self.expIdMask[ampName] = np.array([expIdMask]) 

302 self.covariances[ampName] = np.array([covariance]) 

303 self.covariancesSqrtWeights[ampName] = np.array([covSqrtWeights]) 

304 self.gain[ampName] = gain 

305 self.noise[ampName] = noise 

306 self.histVars[ampName] = np.array([histVar]) 

307 self.histChi2Dofs[ampName] = np.array([histChi2Dof]) 

308 self.kspValues[ampName] = np.array([kspValue]) 

309 

310 self.covariancesModel[ampName] = np.array([nanMatrix]) 

311 self.covariancesModelNoB[ampName] = np.array([nanMatrix]) 

312 self.aMatrix[ampName] = nanMatrix 

313 self.bMatrix[ampName] = nanMatrix 

314 self.aMatrixNoB[ampName] = nanMatrix 

315 self.noiseMatrix[ampName] = nanMatrix 

316 self.noiseMatrixNoB[ampName] = nanMatrix 

317 

318 def setAuxValuesPartialDataset(self, auxDict): 

319 """ 

320 Set a dictionary of auxiliary values for a partial dataset. 

321 

322 Parameters 

323 ---------- 

324 auxDict : `dict` [`str`, `float`] 

325 Dictionary of float values. 

326 """ 

327 for key, value in auxDict.items(): 

328 self.auxValues[key] = np.atleast_1d(np.array(value, dtype=np.float64)) 

329 

330 def updateMetadata(self, **kwargs): 

331 """Update calibration metadata. 

332 This calls the base class's method after ensuring the required 

333 calibration keywords will be saved. 

334 

335 Parameters 

336 ---------- 

337 setDate : `bool`, optional 

338 Update the CALIBDATE fields in the metadata to the current 

339 time. Defaults to False. 

340 kwargs : 

341 Other keyword parameters to set in the metadata. 

342 """ 

343 super().updateMetadata(PTC_FIT_TYPE=self.ptcFitType, **kwargs) 

344 

345 @classmethod 

346 def fromDict(cls, dictionary): 

347 """Construct a calibration from a dictionary of properties. 

348 Must be implemented by the specific calibration subclasses. 

349 

350 Parameters 

351 ---------- 

352 dictionary : `dict` 

353 Dictionary of properties. 

354 

355 Returns 

356 ------- 

357 calib : `lsst.ip.isr.PhotonTransferCurveDataset` 

358 Constructed calibration. 

359 

360 Raises 

361 ------ 

362 RuntimeError 

363 Raised if the supplied dictionary is for a different 

364 calibration. 

365 """ 

366 calib = cls() 

367 if calib._OBSTYPE != dictionary['metadata']['OBSTYPE']: 

368 raise RuntimeError(f"Incorrect Photon Transfer Curve dataset supplied. " 

369 f"Expected {calib._OBSTYPE}, found {dictionary['metadata']['OBSTYPE']}") 

370 calib.setMetadata(dictionary['metadata']) 

371 calib.ptcFitType = dictionary['ptcFitType'] 

372 calib.covMatrixSide = dictionary['covMatrixSide'] 

373 calib.badAmps = np.array(dictionary['badAmps'], 'str').tolist() 

374 calib.ampNames = [] 

375 

376 # The cov matrices are square 

377 covMatrixSide = calib.covMatrixSide 

378 # Number of final signal levels 

379 covDimensionsProduct = len(np.array(list(dictionary['covariances'].values())[0]).ravel()) 

380 nSignalPoints = int(covDimensionsProduct/(covMatrixSide*covMatrixSide)) 

381 

382 for ampName in dictionary['ampNames']: 

383 calib.ampNames.append(ampName) 

384 calib.inputExpIdPairs[ampName] = dictionary['inputExpIdPairs'][ampName] 

385 calib.expIdMask[ampName] = np.array(dictionary['expIdMask'][ampName]) 

386 calib.rawExpTimes[ampName] = np.array(dictionary['rawExpTimes'][ampName], dtype=np.float64) 

387 calib.rawMeans[ampName] = np.array(dictionary['rawMeans'][ampName], dtype=np.float64) 

388 calib.rawVars[ampName] = np.array(dictionary['rawVars'][ampName], dtype=np.float64) 

389 calib.gain[ampName] = float(dictionary['gain'][ampName]) 

390 calib.gainErr[ampName] = float(dictionary['gainErr'][ampName]) 

391 calib.noise[ampName] = float(dictionary['noise'][ampName]) 

392 calib.noiseErr[ampName] = float(dictionary['noiseErr'][ampName]) 

393 calib.histVars[ampName] = np.array(dictionary['histVars'][ampName], dtype=np.float64) 

394 calib.histChi2Dofs[ampName] = np.array(dictionary['histChi2Dofs'][ampName], dtype=np.float64) 

395 calib.kspValues[ampName] = np.array(dictionary['kspValues'][ampName], dtype=np.float64) 

396 calib.ptcFitPars[ampName] = np.array(dictionary['ptcFitPars'][ampName], dtype=np.float64) 

397 calib.ptcFitParsError[ampName] = np.array(dictionary['ptcFitParsError'][ampName], 

398 dtype=np.float64) 

399 calib.ptcFitChiSq[ampName] = float(dictionary['ptcFitChiSq'][ampName]) 

400 calib.ptcTurnoff[ampName] = float(dictionary['ptcTurnoff'][ampName]) 

401 if nSignalPoints > 0: 

402 # Regular dataset 

403 calib.covariances[ampName] = np.array(dictionary['covariances'][ampName], 

404 dtype=np.float64).reshape( 

405 (nSignalPoints, covMatrixSide, covMatrixSide)) 

406 calib.covariancesModel[ampName] = np.array( 

407 dictionary['covariancesModel'][ampName], 

408 dtype=np.float64).reshape( 

409 (nSignalPoints, covMatrixSide, covMatrixSide)) 

410 calib.covariancesSqrtWeights[ampName] = np.array( 

411 dictionary['covariancesSqrtWeights'][ampName], 

412 dtype=np.float64).reshape( 

413 (nSignalPoints, covMatrixSide, covMatrixSide)) 

414 calib.aMatrix[ampName] = np.array(dictionary['aMatrix'][ampName], 

415 dtype=np.float64).reshape( 

416 (covMatrixSide, covMatrixSide)) 

417 calib.bMatrix[ampName] = np.array(dictionary['bMatrix'][ampName], 

418 dtype=np.float64).reshape( 

419 (covMatrixSide, covMatrixSide)) 

420 calib.covariancesModelNoB[ampName] = np.array( 

421 dictionary['covariancesModelNoB'][ampName], dtype=np.float64).reshape( 

422 (nSignalPoints, covMatrixSide, covMatrixSide)) 

423 calib.aMatrixNoB[ampName] = np.array( 

424 dictionary['aMatrixNoB'][ampName], 

425 dtype=np.float64).reshape((covMatrixSide, covMatrixSide)) 

426 calib.noiseMatrix[ampName] = np.array( 

427 dictionary['noiseMatrix'][ampName], 

428 dtype=np.float64).reshape((covMatrixSide, covMatrixSide)) 

429 calib.noiseMatrixNoB[ampName] = np.array( 

430 dictionary['noiseMatrixNoB'][ampName], 

431 dtype=np.float64).reshape((covMatrixSide, covMatrixSide)) 

432 else: 

433 # Empty dataset 

434 calib.covariances[ampName] = np.array([], dtype=np.float64) 

435 calib.covariancesModel[ampName] = np.array([], dtype=np.float64) 

436 calib.covariancesSqrtWeights[ampName] = np.array([], dtype=np.float64) 

437 calib.aMatrix[ampName] = np.array([], dtype=np.float64) 

438 calib.bMatrix[ampName] = np.array([], dtype=np.float64) 

439 calib.covariancesModelNoB[ampName] = np.array([], dtype=np.float64) 

440 calib.aMatrixNoB[ampName] = np.array([], dtype=np.float64) 

441 calib.noiseMatrix[ampName] = np.array([], dtype=np.float64) 

442 calib.noiseMatrixNoB[ampName] = np.array([], dtype=np.float64) 

443 

444 calib.finalVars[ampName] = np.array(dictionary['finalVars'][ampName], dtype=np.float64) 

445 calib.finalModelVars[ampName] = np.array(dictionary['finalModelVars'][ampName], dtype=np.float64) 

446 calib.finalMeans[ampName] = np.array(dictionary['finalMeans'][ampName], dtype=np.float64) 

447 calib.photoCharges[ampName] = np.array(dictionary['photoCharges'][ampName], dtype=np.float64) 

448 

449 for key, value in dictionary['auxValues'].items(): 

450 calib.auxValues[key] = np.atleast_1d(np.array(value, dtype=np.float64)) 

451 

452 calib.updateMetadata() 

453 return calib 

454 

455 def toDict(self): 

456 """Return a dictionary containing the calibration properties. 

457 The dictionary should be able to be round-tripped through 

458 `fromDict`. 

459 

460 Returns 

461 ------- 

462 dictionary : `dict` 

463 Dictionary of properties. 

464 """ 

465 self.updateMetadata() 

466 

467 outDict = dict() 

468 metadata = self.getMetadata() 

469 outDict['metadata'] = metadata 

470 

471 def _dictOfArraysToDictOfLists(dictOfArrays): 

472 dictOfLists = {} 

473 for key, value in dictOfArrays.items(): 

474 dictOfLists[key] = value.ravel().tolist() 

475 

476 return dictOfLists 

477 

478 outDict['ptcFitType'] = self.ptcFitType 

479 outDict['covMatrixSide'] = self.covMatrixSide 

480 outDict['ampNames'] = self.ampNames 

481 outDict['badAmps'] = self.badAmps 

482 outDict['inputExpIdPairs'] = self.inputExpIdPairs 

483 outDict['expIdMask'] = _dictOfArraysToDictOfLists(self.expIdMask) 

484 outDict['rawExpTimes'] = _dictOfArraysToDictOfLists(self.rawExpTimes) 

485 outDict['rawMeans'] = _dictOfArraysToDictOfLists(self.rawMeans) 

486 outDict['rawVars'] = _dictOfArraysToDictOfLists(self.rawVars) 

487 outDict['gain'] = self.gain 

488 outDict['gainErr'] = self.gainErr 

489 outDict['noise'] = self.noise 

490 outDict['noiseErr'] = self.noiseErr 

491 outDict['histVars'] = self.histVars 

492 outDict['histChi2Dofs'] = self.histChi2Dofs 

493 outDict['kspValues'] = self.kspValues 

494 outDict['ptcFitPars'] = _dictOfArraysToDictOfLists(self.ptcFitPars) 

495 outDict['ptcFitParsError'] = _dictOfArraysToDictOfLists(self.ptcFitParsError) 

496 outDict['ptcFitChiSq'] = self.ptcFitChiSq 

497 outDict['ptcTurnoff'] = self.ptcTurnoff 

498 outDict['covariances'] = _dictOfArraysToDictOfLists(self.covariances) 

499 outDict['covariancesModel'] = _dictOfArraysToDictOfLists(self.covariancesModel) 

500 outDict['covariancesSqrtWeights'] = _dictOfArraysToDictOfLists(self.covariancesSqrtWeights) 

501 outDict['aMatrix'] = _dictOfArraysToDictOfLists(self.aMatrix) 

502 outDict['bMatrix'] = _dictOfArraysToDictOfLists(self.bMatrix) 

503 outDict['noiseMatrix'] = _dictOfArraysToDictOfLists(self.noiseMatrix) 

504 outDict['covariancesModelNoB'] = _dictOfArraysToDictOfLists(self.covariancesModelNoB) 

505 outDict['aMatrixNoB'] = _dictOfArraysToDictOfLists(self.aMatrixNoB) 

506 outDict['noiseMatrixNoB'] = _dictOfArraysToDictOfLists(self.noiseMatrixNoB) 

507 outDict['finalVars'] = _dictOfArraysToDictOfLists(self.finalVars) 

508 outDict['finalModelVars'] = _dictOfArraysToDictOfLists(self.finalModelVars) 

509 outDict['finalMeans'] = _dictOfArraysToDictOfLists(self.finalMeans) 

510 outDict['photoCharges'] = _dictOfArraysToDictOfLists(self.photoCharges) 

511 outDict['auxValues'] = _dictOfArraysToDictOfLists(self.auxValues) 

512 

513 return outDict 

514 

515 @classmethod 

516 def fromTable(cls, tableList): 

517 """Construct calibration from a list of tables. 

518 This method uses the `fromDict` method to create the 

519 calibration, after constructing an appropriate dictionary from 

520 the input tables. 

521 

522 Parameters 

523 ---------- 

524 tableList : `list` [`lsst.afw.table.Table`] 

525 List of tables to use to construct the datasetPtc. 

526 

527 Returns 

528 ------- 

529 calib : `lsst.ip.isr.PhotonTransferCurveDataset` 

530 The calibration defined in the tables. 

531 """ 

532 ptcTable = tableList[0] 

533 

534 metadata = ptcTable.meta 

535 inDict = dict() 

536 inDict['metadata'] = metadata 

537 inDict['ampNames'] = [] 

538 inDict['ptcFitType'] = [] 

539 inDict['covMatrixSide'] = [] 

540 inDict['inputExpIdPairs'] = dict() 

541 inDict['expIdMask'] = dict() 

542 inDict['rawExpTimes'] = dict() 

543 inDict['rawMeans'] = dict() 

544 inDict['rawVars'] = dict() 

545 inDict['gain'] = dict() 

546 inDict['gainErr'] = dict() 

547 inDict['noise'] = dict() 

548 inDict['noiseErr'] = dict() 

549 inDict['histVars'] = dict() 

550 inDict['histChi2Dofs'] = dict() 

551 inDict['kspValues'] = dict() 

552 inDict['ptcFitPars'] = dict() 

553 inDict['ptcFitParsError'] = dict() 

554 inDict['ptcFitChiSq'] = dict() 

555 inDict['ptcTurnoff'] = dict() 

556 inDict['covariances'] = dict() 

557 inDict['covariancesModel'] = dict() 

558 inDict['covariancesSqrtWeights'] = dict() 

559 inDict['aMatrix'] = dict() 

560 inDict['bMatrix'] = dict() 

561 inDict['noiseMatrix'] = dict() 

562 inDict['covariancesModelNoB'] = dict() 

563 inDict['aMatrixNoB'] = dict() 

564 inDict['noiseMatrixNoB'] = dict() 

565 inDict['finalVars'] = dict() 

566 inDict['finalModelVars'] = dict() 

567 inDict['finalMeans'] = dict() 

568 inDict['badAmps'] = [] 

569 inDict['photoCharges'] = dict() 

570 

571 calibVersion = metadata['PTC_VERSION'] 

572 if calibVersion == 1.0: 

573 cls().log.warning(f"Previous version found for PTC dataset: {calibVersion}. " 

574 f"Setting 'ptcTurnoff' in all amps to last value in 'finalMeans'.") 

575 for record in ptcTable: 

576 ampName = record['AMPLIFIER_NAME'] 

577 

578 inDict['ptcFitType'] = record['PTC_FIT_TYPE'] 

579 inDict['covMatrixSide'] = record['COV_MATRIX_SIDE'] 

580 inDict['ampNames'].append(ampName) 

581 inDict['inputExpIdPairs'][ampName] = record['INPUT_EXP_ID_PAIRS'].tolist() 

582 inDict['expIdMask'][ampName] = record['EXP_ID_MASK'] 

583 inDict['rawExpTimes'][ampName] = record['RAW_EXP_TIMES'] 

584 inDict['rawMeans'][ampName] = record['RAW_MEANS'] 

585 inDict['rawVars'][ampName] = record['RAW_VARS'] 

586 inDict['gain'][ampName] = record['GAIN'] 

587 inDict['gainErr'][ampName] = record['GAIN_ERR'] 

588 inDict['noise'][ampName] = record['NOISE'] 

589 inDict['noiseErr'][ampName] = record['NOISE_ERR'] 

590 inDict['ptcFitPars'][ampName] = record['PTC_FIT_PARS'] 

591 inDict['ptcFitParsError'][ampName] = record['PTC_FIT_PARS_ERROR'] 

592 inDict['ptcFitChiSq'][ampName] = record['PTC_FIT_CHI_SQ'] 

593 inDict['covariances'][ampName] = record['COVARIANCES'] 

594 inDict['covariancesModel'][ampName] = record['COVARIANCES_MODEL'] 

595 inDict['covariancesSqrtWeights'][ampName] = record['COVARIANCES_SQRT_WEIGHTS'] 

596 inDict['aMatrix'][ampName] = record['A_MATRIX'] 

597 inDict['bMatrix'][ampName] = record['B_MATRIX'] 

598 inDict['covariancesModelNoB'][ampName] = record['COVARIANCES_MODEL_NO_B'] 

599 inDict['aMatrixNoB'][ampName] = record['A_MATRIX_NO_B'] 

600 inDict['finalVars'][ampName] = record['FINAL_VARS'] 

601 inDict['finalModelVars'][ampName] = record['FINAL_MODEL_VARS'] 

602 inDict['finalMeans'][ampName] = record['FINAL_MEANS'] 

603 inDict['badAmps'] = record['BAD_AMPS'].tolist() 

604 inDict['photoCharges'][ampName] = record['PHOTO_CHARGE'] 

605 if calibVersion == 1.0: 

606 mask = record['FINAL_MEANS'].mask 

607 array = record['FINAL_MEANS'][~mask] 

608 if len(array) > 0: 

609 inDict['ptcTurnoff'][ampName] = record['FINAL_MEANS'][~mask][-1] 

610 else: 

611 inDict['ptcTurnoff'][ampName] = np.nan 

612 else: 

613 inDict['ptcTurnoff'][ampName] = record['PTC_TURNOFF'] 

614 if calibVersion < 1.2: 

615 inDict['histVars'][ampName] = np.array([np.nan]) 

616 inDict['histChi2Dofs'][ampName] = np.array([np.nan]) 

617 inDict['kspValues'][ampName] = np.array([0.0]) 

618 else: 

619 inDict['histVars'][ampName] = record['HIST_VARS'] 

620 inDict['histChi2Dofs'][ampName] = record['HIST_CHI2_DOFS'] 

621 inDict['kspValues'][ampName] = record['KS_PVALUES'] 

622 if calibVersion < 1.3: 

623 nanMatrix = np.full_like(inDict['aMatrix'][ampName], np.nan) 

624 inDict['noiseMatrix'][ampName] = nanMatrix 

625 inDict['noiseMatrixNoB'][ampName] = nanMatrix 

626 else: 

627 inDict['noiseMatrix'][ampName] = record['NOISE_MATRIX'] 

628 inDict['noiseMatrixNoB'][ampName] = record['NOISE_MATRIX_NO_B'] 

629 

630 inDict['auxValues'] = {} 

631 record = ptcTable[0] 

632 for col in record.columns.keys(): 

633 if col.startswith('PTCAUX_'): 

634 parts = col.split('PTCAUX_') 

635 inDict['auxValues'][parts[1]] = record[col] 

636 

637 return cls().fromDict(inDict) 

638 

639 def toTable(self): 

640 """Construct a list of tables containing the information in this 

641 calibration. 

642 

643 The list of tables should create an identical calibration 

644 after being passed to this class's fromTable method. 

645 

646 Returns 

647 ------- 

648 tableList : `list` [`astropy.table.Table`] 

649 List of tables containing the linearity calibration 

650 information. 

651 """ 

652 tableList = [] 

653 self.updateMetadata() 

654 

655 badAmps = np.array(self.badAmps) if len(self.badAmps) else np.array([], dtype="U3") 

656 

657 catalogList = [] 

658 for ampName in self.ampNames: 

659 ampDict = { 

660 'AMPLIFIER_NAME': ampName, 

661 'PTC_FIT_TYPE': self.ptcFitType, 

662 'COV_MATRIX_SIDE': self.covMatrixSide, 

663 'INPUT_EXP_ID_PAIRS': self.inputExpIdPairs[ampName], 

664 'EXP_ID_MASK': self.expIdMask[ampName], 

665 'RAW_EXP_TIMES': self.rawExpTimes[ampName], 

666 'RAW_MEANS': self.rawMeans[ampName], 

667 'RAW_VARS': self.rawVars[ampName], 

668 'GAIN': self.gain[ampName], 

669 'GAIN_ERR': self.gainErr[ampName], 

670 'NOISE': self.noise[ampName], 

671 'NOISE_ERR': self.noiseErr[ampName], 

672 'HIST_VARS': self.histVars[ampName], 

673 'HIST_CHI2_DOFS': self.histChi2Dofs[ampName], 

674 'KS_PVALUES': self.kspValues[ampName], 

675 'PTC_FIT_PARS': np.array(self.ptcFitPars[ampName]), 

676 'PTC_FIT_PARS_ERROR': np.array(self.ptcFitParsError[ampName]), 

677 'PTC_FIT_CHI_SQ': self.ptcFitChiSq[ampName], 

678 'PTC_TURNOFF': self.ptcTurnoff[ampName], 

679 'A_MATRIX': self.aMatrix[ampName].ravel(), 

680 'B_MATRIX': self.bMatrix[ampName].ravel(), 

681 'A_MATRIX_NO_B': self.aMatrixNoB[ampName].ravel(), 

682 'NOISE_MATRIX': self.noiseMatrix[ampName].ravel(), 

683 'NOISE_MATRIX_NO_B': self.noiseMatrixNoB[ampName].ravel(), 

684 'BAD_AMPS': badAmps, 

685 'PHOTO_CHARGE': self.photoCharges[ampName], 

686 'COVARIANCES': self.covariances[ampName].ravel(), 

687 'COVARIANCES_MODEL': self.covariancesModel[ampName].ravel(), 

688 'COVARIANCES_SQRT_WEIGHTS': self.covariancesSqrtWeights[ampName].ravel(), 

689 'COVARIANCES_MODEL_NO_B': self.covariancesModelNoB[ampName].ravel(), 

690 'FINAL_VARS': self.finalVars[ampName], 

691 'FINAL_MODEL_VARS': self.finalModelVars[ampName], 

692 'FINAL_MEANS': self.finalMeans[ampName], 

693 } 

694 

695 if self.auxValues: 

696 for key, value in self.auxValues.items(): 

697 ampDict[f"PTCAUX_{key}"] = value 

698 

699 catalogList.append(ampDict) 

700 

701 catalog = Table(catalogList) 

702 

703 inMeta = self.getMetadata().toDict() 

704 outMeta = {k: v for k, v in inMeta.items() if v is not None} 

705 outMeta.update({k: "" for k, v in inMeta.items() if v is None}) 

706 catalog.meta = outMeta 

707 tableList.append(catalog) 

708 

709 return tableList 

710 

711 def fromDetector(self, detector): 

712 """Read metadata parameters from a detector. 

713 

714 Parameters 

715 ---------- 

716 detector : `lsst.afw.cameraGeom.detector` 

717 Input detector with parameters to use. 

718 

719 Returns 

720 ------- 

721 calib : `lsst.ip.isr.PhotonTransferCurveDataset` 

722 The calibration constructed from the detector. 

723 """ 

724 

725 pass 

726 

727 def getExpIdsUsed(self, ampName): 

728 """Get the exposures used, i.e. not discarded, for a given amp. 

729 If no mask has been created yet, all exposures are returned. 

730 

731 Parameters 

732 ---------- 

733 ampName : `str` 

734 

735 Returns 

736 ------- 

737 expIdsUsed : `list` [`tuple`] 

738 List of pairs of exposure ids used in PTC. 

739 """ 

740 if len(self.expIdMask[ampName]) == 0: 

741 return self.inputExpIdPairs[ampName] 

742 

743 # if the mask exists it had better be the same length as the expIdPairs 

744 assert len(self.expIdMask[ampName]) == len(self.inputExpIdPairs[ampName]) 

745 

746 pairs = self.inputExpIdPairs[ampName] 

747 mask = self.expIdMask[ampName] 

748 # cast to bool required because numpy 

749 try: 

750 expIdsUsed = [(exp1, exp2) for ((exp1, exp2), m) in zip(pairs, mask) if m] 

751 except ValueError: 

752 self.log.warning("The PTC file was written incorrectly; you should rerun the " 

753 "PTC solve task if possible.") 

754 expIdsUsed = [] 

755 for pairList, m in zip(pairs, mask): 

756 if m: 

757 expIdsUsed.append(pairList[0]) 

758 

759 return expIdsUsed 

760 

761 def getGoodAmps(self): 

762 """Get the good amps from this PTC.""" 

763 return [amp for amp in self.ampNames if amp not in self.badAmps] 

764 

765 def getGoodPoints(self, ampName): 

766 """Get the good points used for a given amp in the PTC. 

767 

768 Parameters 

769 ---------- 

770 ampName : `str` 

771 Amplifier's name. 

772 

773 Returns 

774 ------- 

775 goodPoints : `np.ndarray` 

776 Boolean array of good points used in PTC. 

777 """ 

778 return self.expIdMask[ampName] 

779 

780 def validateGainNoiseTurnoffValues(self, ampName, doWarn=False): 

781 """Ensure the gain, read noise, and PTC turnoff have 

782 sensible values. 

783 

784 Parameters 

785 ---------- 

786 ampName : `str` 

787 Amplifier's name. 

788 """ 

789 

790 gain = self.gain[ampName] 

791 noise = self.noise[ampName] 

792 ptcTurnoff = self.ptcTurnoff[ampName] 

793 

794 # Check if gain is not positive or is np.nan 

795 if not (isinstance(gain, (int, float)) and gain > 0) or math.isnan(gain): 

796 if doWarn: 

797 self.log.warning(f"Invalid gain value {gain}" 

798 " Setting to default: Gain=1") 

799 gain = 1 

800 

801 # Check if noise is not positive or is np.nan 

802 if not (isinstance(noise, (int, float)) and noise > 0) or math.isnan(noise): 

803 if doWarn: 

804 self.log.warning(f"Invalid noise value: {noise}" 

805 " Setting to default: Noise=1") 

806 noise = 1 

807 

808 # Check if ptcTurnoff is not positive or is np.nan 

809 if not (isinstance(ptcTurnoff, (int, float)) and ptcTurnoff > 0) or math.isnan(ptcTurnoff): 

810 if doWarn: 

811 self.log.warning(f"Invalid PTC turnoff value: {ptcTurnoff}" 

812 " Setting to default: PTC Turnoff=2e19") 

813 ptcTurnoff = 2e19 

814 

815 self.gain[ampName] = gain 

816 self.noise[ampName] = noise 

817 self.ptcTurnoff[ampName] = ptcTurnoff