Coverage for python/lsst/ip/isr/ptcDataset.py: 6%

285 statements  

« prev     ^ index     » next       coverage.py v7.2.5, created at 2023-05-18 02:35 -0700

1# 

2# LSST Data Management System 

3# Copyright 2008-2017 AURA/LSST. 

4# 

5# This product includes software developed by the 

6# LSST Project (http://www.lsst.org/). 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <https://www.lsstcorp.org/LegalNotices/>. 

21# 

22""" 

23Define dataset class for MeasurePhotonTransferCurve task 

24""" 

25 

26__all__ = ['PhotonTransferCurveDataset'] 

27 

28import numpy as np 

29from astropy.table import Table 

30import warnings 

31 

32from lsst.ip.isr import IsrCalib 

33 

34 

35class PhotonTransferCurveDataset(IsrCalib): 

36 """A simple class to hold the output data from the PTC task. 

37 

38 The dataset is made up of a dictionary for each item, keyed by the 

39 amplifiers' names, which much be supplied at construction time. 

40 New items cannot be added to the class to save accidentally saving to the 

41 wrong property, and the class can be frozen if desired. 

42 inputExpIdPairs records the exposures used to produce the data. 

43 When fitPtc() or fitCovariancesAstier() is run, a mask is built up, which 

44 is by definition always the same length as inputExpIdPairs, rawExpTimes, 

45 rawMeans and rawVars, and is a list of bools, which are incrementally set 

46 to False as points are discarded from the fits. 

47 PTC fit parameters for polynomials are stored in a list in ascending order 

48 of polynomial term, i.e. par[0]*x^0 + par[1]*x + par[2]*x^2 etc 

49 with the length of the list corresponding to the order of the polynomial 

50 plus one. 

51 

52 Parameters 

53 ---------- 

54 ampNames : `list` 

55 List with the names of the amplifiers of the detector at hand. 

56 ptcFitType : `str` 

57 Type of model fitted to the PTC: "POLYNOMIAL", "EXPAPPROXIMATION", 

58 or "FULLCOVARIANCE". 

59 covMatrixSide : `int` 

60 Maximum lag of covariances (size of square covariance matrices). 

61 kwargs : `dict`, optional 

62 Other keyword arguments to pass to the parent init. 

63 

64 Notes 

65 ----- 

66 The stored attributes are: 

67 

68 badAmps : `list` [`str`] 

69 List with bad amplifiers names. 

70 inputExpIdPairs : `dict`, [`str`, `list`] 

71 Dictionary keyed by amp names containing the input exposures IDs. 

72 expIdMask : `dict`, [`str`, `np.ndarray`] 

73 Dictionary keyed by amp names containing the mask produced after 

74 outlier rejection. The mask produced by the "FULLCOVARIANCE" 

75 option may differ from the one produced in the other two PTC 

76 fit types. 

77 rawExpTimes : `dict`, [`str`, `np.ndarray`] 

78 Dictionary keyed by amp names containing the unmasked exposure times. 

79 rawMeans : `dict`, [`str`, `np.ndarray`] 

80 Dictionary keyed by amp names containing the unmasked average of the 

81 means of the exposures in each flat pair. 

82 rawVars : `dict`, [`str`, `np.ndarray`] 

83 Dictionary keyed by amp names containing the variance of the 

84 difference image of the exposures in each flat pair. 

85 histVars : `dict`, [`str`, `np.ndarray`] 

86 Dictionary keyed by amp names containing the variance of the 

87 difference image of the exposures in each flat pair estimated 

88 by fitting a Gaussian model. 

89 histChi2Dofs : `dict`, [`str`, `np.ndarray`] 

90 Dictionary keyed by amp names containing the chi-squared per degree 

91 of freedom fitting the difference image to a Gaussian model. 

92 kspValues : `dict`, [`str`, `np.ndarray`] 

93 Dictionary keyed by amp names containing the KS test p-value from 

94 fitting the difference image to a Gaussian model. 

95 gain : `dict`, [`str`, `float`] 

96 Dictionary keyed by amp names containing the fitted gains. 

97 gainErr : `dict`, [`str`, `float`] 

98 Dictionary keyed by amp names containing the errors on the 

99 fitted gains. 

100 noise : `dict`, [`str`, `float`] 

101 Dictionary keyed by amp names containing the fitted noise. 

102 noiseErr : `dict`, [`str`, `float`] 

103 Dictionary keyed by amp names containing the errors on the fitted 

104 noise. 

105 ptcFitPars : `dict`, [`str`, `np.ndarray`] 

106 Dictionary keyed by amp names containing the fitted parameters of the 

107 PTC model for ptcFitTye in ["POLYNOMIAL", "EXPAPPROXIMATION"]. 

108 ptcFitParsError : `dict`, [`str`, `np.ndarray`] 

109 Dictionary keyed by amp names containing the errors on the fitted 

110 parameters of the PTC model for ptcFitTye in 

111 ["POLYNOMIAL", "EXPAPPROXIMATION"]. 

112 ptcFitChiSq : `dict`, [`str`, `float`] 

113 Dictionary keyed by amp names containing the reduced chi squared 

114 of the fit for ptcFitTye in ["POLYNOMIAL", "EXPAPPROXIMATION"]. 

115 ptcTurnoff : `dict` [`str, `float`] 

116 Flux value (in ADU) where the variance of the PTC curve starts 

117 decreasing consistently. 

118 covariances : `dict`, [`str`, `np.ndarray`] 

119 Dictionary keyed by amp names containing a list of measured 

120 covariances per mean flux. 

121 covariancesModel : `dict`, [`str`, `np.ndarray`] 

122 Dictionary keyed by amp names containinging covariances model 

123 (Eq. 20 of Astier+19) per mean flux. 

124 covariancesSqrtWeights : `dict`, [`str`, `np.ndarray`] 

125 Dictionary keyed by amp names containinging sqrt. of covariances 

126 weights. 

127 aMatrix : `dict`, [`str`, `np.ndarray`] 

128 Dictionary keyed by amp names containing the "a" parameters from 

129 the model in Eq. 20 of Astier+19. 

130 bMatrix : `dict`, [`str`, `np.ndarray`] 

131 Dictionary keyed by amp names containing the "b" parameters from 

132 the model in Eq. 20 of Astier+19. 

133 covariancesModelNoB : `dict`, [`str`, `np.ndarray`] 

134 Dictionary keyed by amp names containing covariances model 

135 (with 'b'=0 in Eq. 20 of Astier+19) 

136 per mean flux. 

137 aMatrixNoB : `dict`, [`str`, `np.ndarray`] 

138 Dictionary keyed by amp names containing the "a" parameters from the 

139 model in Eq. 20 of Astier+19 

140 (and 'b' = 0). 

141 finalVars : `dict`, [`str`, `np.ndarray`] 

142 Dictionary keyed by amp names containing the masked variance of the 

143 difference image of each flat 

144 pair. If needed, each array will be right-padded with 

145 np.nan to match the length of rawExpTimes. 

146 finalModelVars : `dict`, [`str`, `np.ndarray`] 

147 Dictionary keyed by amp names containing the masked modeled 

148 variance of the difference image of each flat pair. If needed, each 

149 array will be right-padded with np.nan to match the length of 

150 rawExpTimes. 

151 finalMeans : `dict`, [`str`, `np.ndarray`] 

152 Dictionary keyed by amp names containing the masked average of the 

153 means of the exposures in each flat pair. If needed, each array 

154 will be right-padded with np.nan to match the length of 

155 rawExpTimes. 

156 photoCharges : `dict`, [`str`, `np.ndarray`] 

157 Dictionary keyed by amp names containing the integrated photocharge 

158 for linearity calibration. 

159 

160 Version 1.1 adds the `ptcTurnoff` attribute. 

161 """ 

162 

163 _OBSTYPE = 'PTC' 

164 _SCHEMA = 'Gen3 Photon Transfer Curve' 

165 _VERSION = 1.2 

166 

167 def __init__(self, ampNames=[], ptcFitType=None, covMatrixSide=1, **kwargs): 

168 self.ptcFitType = ptcFitType 

169 self.ampNames = ampNames 

170 self.covMatrixSide = covMatrixSide 

171 

172 self.badAmps = [] 

173 

174 self.inputExpIdPairs = {ampName: [] for ampName in ampNames} 

175 self.expIdMask = {ampName: np.array([], dtype=bool) for ampName in ampNames} 

176 self.rawExpTimes = {ampName: np.array([]) for ampName in ampNames} 

177 self.rawMeans = {ampName: np.array([]) for ampName in ampNames} 

178 self.rawVars = {ampName: np.array([]) for ampName in ampNames} 

179 self.photoCharges = {ampName: np.array([]) for ampName in ampNames} 

180 

181 self.gain = {ampName: np.nan for ampName in ampNames} 

182 self.gainErr = {ampName: np.nan for ampName in ampNames} 

183 self.noise = {ampName: np.nan for ampName in ampNames} 

184 self.noiseErr = {ampName: np.nan for ampName in ampNames} 

185 

186 self.histVars = {ampName: np.array([]) for ampName in ampNames} 

187 self.histChi2Dofs = {ampName: np.array([]) for ampName in ampNames} 

188 self.kspValues = {ampName: np.array([]) for ampName in ampNames} 

189 

190 self.ptcFitPars = {ampName: np.array([]) for ampName in ampNames} 

191 self.ptcFitParsError = {ampName: np.array([]) for ampName in ampNames} 

192 self.ptcFitChiSq = {ampName: np.nan for ampName in ampNames} 

193 self.ptcTurnoff = {ampName: np.nan for ampName in ampNames} 

194 

195 self.covariances = {ampName: np.array([]) for ampName in ampNames} 

196 self.covariancesModel = {ampName: np.array([]) for ampName in ampNames} 

197 self.covariancesSqrtWeights = {ampName: np.array([]) for ampName in ampNames} 

198 self.aMatrix = {ampName: np.array([]) for ampName in ampNames} 

199 self.bMatrix = {ampName: np.array([]) for ampName in ampNames} 

200 self.covariancesModelNoB = {ampName: np.array([]) for ampName in ampNames} 

201 self.aMatrixNoB = {ampName: np.array([]) for ampName in ampNames} 

202 

203 self.finalVars = {ampName: np.array([]) for ampName in ampNames} 

204 self.finalModelVars = {ampName: np.array([]) for ampName in ampNames} 

205 self.finalMeans = {ampName: np.array([]) for ampName in ampNames} 

206 

207 super().__init__(**kwargs) 

208 self.requiredAttributes.update(['badAmps', 'inputExpIdPairs', 'expIdMask', 'rawExpTimes', 

209 'rawMeans', 'rawVars', 'gain', 'gainErr', 'noise', 'noiseErr', 

210 'ptcFitPars', 'ptcFitParsError', 'ptcFitChiSq', 'ptcTurnoff', 

211 'aMatrixNoB', 'covariances', 'covariancesModel', 

212 'covariancesSqrtWeights', 'covariancesModelNoB', 

213 'aMatrix', 'bMatrix', 'finalVars', 'finalModelVars', 'finalMeans', 

214 'photoCharges', 'histVars', 'histChi2Dofs', 'kspValues']) 

215 

216 self.updateMetadata(setCalibInfo=True, setCalibId=True, **kwargs) 

217 

218 def setAmpValuesPartialDataset( 

219 self, 

220 ampName, 

221 inputExpIdPair=(-1, -1), 

222 rawExpTime=np.nan, 

223 rawMean=np.nan, 

224 rawVar=np.nan, 

225 photoCharge=np.nan, 

226 expIdMask=False, 

227 covariance=None, 

228 covSqrtWeights=None, 

229 gain=np.nan, 

230 noise=np.nan, 

231 histVar=np.nan, 

232 histChi2Dof=np.nan, 

233 kspValue=0.0, 

234 ): 

235 """ 

236 Set the amp values for a partial PTC Dataset (from cpExtractPtcTask). 

237 

238 Parameters 

239 ---------- 

240 ampName : `str` 

241 Name of the amp to set the values. 

242 inputExpIdPair : `tuple` [`int`] 

243 Exposure IDs of input pair. 

244 rawExpTime : `float`, optional 

245 Exposure time for this exposure pair. 

246 rawMean : `float`, optional 

247 Average of the means of the exposures in this pair. 

248 rawVar : `float`, optional 

249 Variance of the difference of the exposures in this pair. 

250 photoCharge : `float`, optional 

251 Integrated photocharge for flat pair for linearity calibration. 

252 expIdMask : `bool`, optional 

253 Flag setting if this exposure pair should be used (True) 

254 or not used (False). 

255 covariance : `np.ndarray` or None, optional 

256 Measured covariance for this exposure pair. 

257 covSqrtWeights : `np.ndarray` or None, optional 

258 Measured sqrt of covariance weights in this exposure pair. 

259 gain : `float`, optional 

260 Estimated gain for this exposure pair. 

261 noise : `float`, optional 

262 Estimated read noise for this exposure pair. 

263 histVar : `float`, optional 

264 Variance estimated from fitting a histogram with a Gaussian model. 

265 histChi2Dof : `float`, optional 

266 Chi-squared per degree of freedom from Gaussian histogram fit. 

267 kspValue : `float`, optional 

268 KS test p-value from the Gaussian histogram fit. 

269 """ 

270 nanMatrix = np.full((self.covMatrixSide, self.covMatrixSide), np.nan) 

271 if covariance is None: 

272 covariance = nanMatrix 

273 if covSqrtWeights is None: 

274 covSqrtWeights = nanMatrix 

275 

276 self.inputExpIdPairs[ampName] = [inputExpIdPair] 

277 self.rawExpTimes[ampName] = np.array([rawExpTime]) 

278 self.rawMeans[ampName] = np.array([rawMean]) 

279 self.rawVars[ampName] = np.array([rawVar]) 

280 self.photoCharges[ampName] = np.array([photoCharge]) 

281 self.expIdMask[ampName] = np.array([expIdMask]) 

282 self.covariances[ampName] = np.array([covariance]) 

283 self.covariancesSqrtWeights[ampName] = np.array([covSqrtWeights]) 

284 self.gain[ampName] = gain 

285 self.noise[ampName] = noise 

286 self.histVars[ampName] = np.array([histVar]) 

287 self.histChi2Dofs[ampName] = np.array([histChi2Dof]) 

288 self.kspValues[ampName] = np.array([kspValue]) 

289 

290 self.covariancesModel[ampName] = np.array([nanMatrix]) 

291 self.covariancesModelNoB[ampName] = np.array([nanMatrix]) 

292 self.aMatrix[ampName] = nanMatrix 

293 self.bMatrix[ampName] = nanMatrix 

294 self.aMatrixNoB[ampName] = nanMatrix 

295 

296 def updateMetadata(self, **kwargs): 

297 """Update calibration metadata. 

298 This calls the base class's method after ensuring the required 

299 calibration keywords will be saved. 

300 

301 Parameters 

302 ---------- 

303 setDate : `bool`, optional 

304 Update the CALIBDATE fields in the metadata to the current 

305 time. Defaults to False. 

306 kwargs : 

307 Other keyword parameters to set in the metadata. 

308 """ 

309 super().updateMetadata(PTC_FIT_TYPE=self.ptcFitType, **kwargs) 

310 

311 @classmethod 

312 def fromDict(cls, dictionary): 

313 """Construct a calibration from a dictionary of properties. 

314 Must be implemented by the specific calibration subclasses. 

315 

316 Parameters 

317 ---------- 

318 dictionary : `dict` 

319 Dictionary of properties. 

320 

321 Returns 

322 ------- 

323 calib : `lsst.ip.isr.PhotonTransferCurveDataset` 

324 Constructed calibration. 

325 

326 Raises 

327 ------ 

328 RuntimeError 

329 Raised if the supplied dictionary is for a different 

330 calibration. 

331 """ 

332 calib = cls() 

333 if calib._OBSTYPE != dictionary['metadata']['OBSTYPE']: 

334 raise RuntimeError(f"Incorrect Photon Transfer Curve dataset supplied. " 

335 f"Expected {calib._OBSTYPE}, found {dictionary['metadata']['OBSTYPE']}") 

336 calib.setMetadata(dictionary['metadata']) 

337 calib.ptcFitType = dictionary['ptcFitType'] 

338 calib.covMatrixSide = dictionary['covMatrixSide'] 

339 calib.badAmps = np.array(dictionary['badAmps'], 'str').tolist() 

340 calib.ampNames = [] 

341 

342 # The cov matrices are square 

343 covMatrixSide = calib.covMatrixSide 

344 # Number of final signal levels 

345 covDimensionsProduct = len(np.array(list(dictionary['covariances'].values())[0]).ravel()) 

346 nSignalPoints = int(covDimensionsProduct/(covMatrixSide*covMatrixSide)) 

347 

348 for ampName in dictionary['ampNames']: 

349 calib.ampNames.append(ampName) 

350 calib.inputExpIdPairs[ampName] = dictionary['inputExpIdPairs'][ampName] 

351 calib.expIdMask[ampName] = np.array(dictionary['expIdMask'][ampName]) 

352 calib.rawExpTimes[ampName] = np.array(dictionary['rawExpTimes'][ampName], dtype=np.float64) 

353 calib.rawMeans[ampName] = np.array(dictionary['rawMeans'][ampName], dtype=np.float64) 

354 calib.rawVars[ampName] = np.array(dictionary['rawVars'][ampName], dtype=np.float64) 

355 calib.gain[ampName] = float(dictionary['gain'][ampName]) 

356 calib.gainErr[ampName] = float(dictionary['gainErr'][ampName]) 

357 calib.noise[ampName] = float(dictionary['noise'][ampName]) 

358 calib.noiseErr[ampName] = float(dictionary['noiseErr'][ampName]) 

359 calib.histVars[ampName] = np.array(dictionary['histVars'][ampName], dtype=np.float64) 

360 calib.histChi2Dofs[ampName] = np.array(dictionary['histChi2Dofs'][ampName], dtype=np.float64) 

361 calib.kspValues[ampName] = np.array(dictionary['kspValues'][ampName], dtype=np.float64) 

362 calib.ptcFitPars[ampName] = np.array(dictionary['ptcFitPars'][ampName], dtype=np.float64) 

363 calib.ptcFitParsError[ampName] = np.array(dictionary['ptcFitParsError'][ampName], 

364 dtype=np.float64) 

365 calib.ptcFitChiSq[ampName] = float(dictionary['ptcFitChiSq'][ampName]) 

366 calib.ptcTurnoff[ampName] = float(dictionary['ptcTurnoff'][ampName]) 

367 if nSignalPoints > 0: 

368 # Regular dataset 

369 calib.covariances[ampName] = np.array(dictionary['covariances'][ampName], 

370 dtype=np.float64).reshape( 

371 (nSignalPoints, covMatrixSide, covMatrixSide)) 

372 calib.covariancesModel[ampName] = np.array( 

373 dictionary['covariancesModel'][ampName], 

374 dtype=np.float64).reshape( 

375 (nSignalPoints, covMatrixSide, covMatrixSide)) 

376 calib.covariancesSqrtWeights[ampName] = np.array( 

377 dictionary['covariancesSqrtWeights'][ampName], 

378 dtype=np.float64).reshape( 

379 (nSignalPoints, covMatrixSide, covMatrixSide)) 

380 calib.aMatrix[ampName] = np.array(dictionary['aMatrix'][ampName], 

381 dtype=np.float64).reshape( 

382 (covMatrixSide, covMatrixSide)) 

383 calib.bMatrix[ampName] = np.array(dictionary['bMatrix'][ampName], 

384 dtype=np.float64).reshape( 

385 (covMatrixSide, covMatrixSide)) 

386 calib.covariancesModelNoB[ampName] = np.array( 

387 dictionary['covariancesModelNoB'][ampName], dtype=np.float64).reshape( 

388 (nSignalPoints, covMatrixSide, covMatrixSide)) 

389 calib.aMatrixNoB[ampName] = np.array( 

390 dictionary['aMatrixNoB'][ampName], 

391 dtype=np.float64).reshape((covMatrixSide, covMatrixSide)) 

392 else: 

393 # Empty dataset 

394 calib.covariances[ampName] = np.array([], dtype=np.float64) 

395 calib.covariancesModel[ampName] = np.array([], dtype=np.float64) 

396 calib.covariancesSqrtWeights[ampName] = np.array([], dtype=np.float64) 

397 calib.aMatrix[ampName] = np.array([], dtype=np.float64) 

398 calib.bMatrix[ampName] = np.array([], dtype=np.float64) 

399 calib.covariancesModelNoB[ampName] = np.array([], dtype=np.float64) 

400 calib.aMatrixNoB[ampName] = np.array([], dtype=np.float64) 

401 

402 calib.finalVars[ampName] = np.array(dictionary['finalVars'][ampName], dtype=np.float64) 

403 calib.finalModelVars[ampName] = np.array(dictionary['finalModelVars'][ampName], dtype=np.float64) 

404 calib.finalMeans[ampName] = np.array(dictionary['finalMeans'][ampName], dtype=np.float64) 

405 calib.photoCharges[ampName] = np.array(dictionary['photoCharges'][ampName], dtype=np.float64) 

406 

407 calib.updateMetadata() 

408 return calib 

409 

410 def toDict(self): 

411 """Return a dictionary containing the calibration properties. 

412 The dictionary should be able to be round-tripped through 

413 `fromDict`. 

414 

415 Returns 

416 ------- 

417 dictionary : `dict` 

418 Dictionary of properties. 

419 """ 

420 self.updateMetadata() 

421 

422 outDict = dict() 

423 metadata = self.getMetadata() 

424 outDict['metadata'] = metadata 

425 

426 def _dictOfArraysToDictOfLists(dictOfArrays): 

427 dictOfLists = {} 

428 for key, value in dictOfArrays.items(): 

429 dictOfLists[key] = value.ravel().tolist() 

430 

431 return dictOfLists 

432 

433 outDict['ptcFitType'] = self.ptcFitType 

434 outDict['covMatrixSide'] = self.covMatrixSide 

435 outDict['ampNames'] = self.ampNames 

436 outDict['badAmps'] = self.badAmps 

437 outDict['inputExpIdPairs'] = self.inputExpIdPairs 

438 outDict['expIdMask'] = _dictOfArraysToDictOfLists(self.expIdMask) 

439 outDict['rawExpTimes'] = _dictOfArraysToDictOfLists(self.rawExpTimes) 

440 outDict['rawMeans'] = _dictOfArraysToDictOfLists(self.rawMeans) 

441 outDict['rawVars'] = _dictOfArraysToDictOfLists(self.rawVars) 

442 outDict['gain'] = self.gain 

443 outDict['gainErr'] = self.gainErr 

444 outDict['noise'] = self.noise 

445 outDict['noiseErr'] = self.noiseErr 

446 outDict['histVars'] = self.histVars 

447 outDict['histChi2Dofs'] = self.histChi2Dofs 

448 outDict['kspValues'] = self.kspValues 

449 outDict['ptcFitPars'] = _dictOfArraysToDictOfLists(self.ptcFitPars) 

450 outDict['ptcFitParsError'] = _dictOfArraysToDictOfLists(self.ptcFitParsError) 

451 outDict['ptcFitChiSq'] = self.ptcFitChiSq 

452 outDict['ptcTurnoff'] = self.ptcTurnoff 

453 outDict['covariances'] = _dictOfArraysToDictOfLists(self.covariances) 

454 outDict['covariancesModel'] = _dictOfArraysToDictOfLists(self.covariancesModel) 

455 outDict['covariancesSqrtWeights'] = _dictOfArraysToDictOfLists(self.covariancesSqrtWeights) 

456 outDict['aMatrix'] = _dictOfArraysToDictOfLists(self.aMatrix) 

457 outDict['bMatrix'] = _dictOfArraysToDictOfLists(self.bMatrix) 

458 outDict['covariancesModelNoB'] = _dictOfArraysToDictOfLists(self.covariancesModelNoB) 

459 outDict['aMatrixNoB'] = _dictOfArraysToDictOfLists(self.aMatrixNoB) 

460 outDict['finalVars'] = _dictOfArraysToDictOfLists(self.finalVars) 

461 outDict['finalModelVars'] = _dictOfArraysToDictOfLists(self.finalModelVars) 

462 outDict['finalMeans'] = _dictOfArraysToDictOfLists(self.finalMeans) 

463 outDict['photoCharges'] = _dictOfArraysToDictOfLists(self.photoCharges) 

464 

465 return outDict 

466 

467 @classmethod 

468 def fromTable(cls, tableList): 

469 """Construct calibration from a list of tables. 

470 This method uses the `fromDict` method to create the 

471 calibration, after constructing an appropriate dictionary from 

472 the input tables. 

473 

474 Parameters 

475 ---------- 

476 tableList : `list` [`lsst.afw.table.Table`] 

477 List of tables to use to construct the datasetPtc. 

478 

479 Returns 

480 ------- 

481 calib : `lsst.ip.isr.PhotonTransferCurveDataset` 

482 The calibration defined in the tables. 

483 """ 

484 ptcTable = tableList[0] 

485 

486 metadata = ptcTable.meta 

487 inDict = dict() 

488 inDict['metadata'] = metadata 

489 inDict['ampNames'] = [] 

490 inDict['ptcFitType'] = [] 

491 inDict['covMatrixSide'] = [] 

492 inDict['inputExpIdPairs'] = dict() 

493 inDict['expIdMask'] = dict() 

494 inDict['rawExpTimes'] = dict() 

495 inDict['rawMeans'] = dict() 

496 inDict['rawVars'] = dict() 

497 inDict['gain'] = dict() 

498 inDict['gainErr'] = dict() 

499 inDict['noise'] = dict() 

500 inDict['noiseErr'] = dict() 

501 inDict['histVars'] = dict() 

502 inDict['histChi2Dofs'] = dict() 

503 inDict['kspValues'] = dict() 

504 inDict['ptcFitPars'] = dict() 

505 inDict['ptcFitParsError'] = dict() 

506 inDict['ptcFitChiSq'] = dict() 

507 inDict['ptcTurnoff'] = dict() 

508 inDict['covariances'] = dict() 

509 inDict['covariancesModel'] = dict() 

510 inDict['covariancesSqrtWeights'] = dict() 

511 inDict['aMatrix'] = dict() 

512 inDict['bMatrix'] = dict() 

513 inDict['covariancesModelNoB'] = dict() 

514 inDict['aMatrixNoB'] = dict() 

515 inDict['finalVars'] = dict() 

516 inDict['finalModelVars'] = dict() 

517 inDict['finalMeans'] = dict() 

518 inDict['badAmps'] = [] 

519 inDict['photoCharges'] = dict() 

520 

521 calibVersion = metadata['PTC_VERSION'] 

522 if calibVersion == 1.0: 

523 cls().log.warning(f"Previous version found for PTC dataset: {calibVersion}. " 

524 f"Setting 'ptcTurnoff' in all amps to last value in 'finalMeans'.") 

525 for record in ptcTable: 

526 ampName = record['AMPLIFIER_NAME'] 

527 

528 inDict['ptcFitType'] = record['PTC_FIT_TYPE'] 

529 inDict['covMatrixSide'] = record['COV_MATRIX_SIDE'] 

530 inDict['ampNames'].append(ampName) 

531 inDict['inputExpIdPairs'][ampName] = record['INPUT_EXP_ID_PAIRS'].tolist() 

532 inDict['expIdMask'][ampName] = record['EXP_ID_MASK'] 

533 inDict['rawExpTimes'][ampName] = record['RAW_EXP_TIMES'] 

534 inDict['rawMeans'][ampName] = record['RAW_MEANS'] 

535 inDict['rawVars'][ampName] = record['RAW_VARS'] 

536 inDict['gain'][ampName] = record['GAIN'] 

537 inDict['gainErr'][ampName] = record['GAIN_ERR'] 

538 inDict['noise'][ampName] = record['NOISE'] 

539 inDict['noiseErr'][ampName] = record['NOISE_ERR'] 

540 inDict['ptcFitPars'][ampName] = record['PTC_FIT_PARS'] 

541 inDict['ptcFitParsError'][ampName] = record['PTC_FIT_PARS_ERROR'] 

542 inDict['ptcFitChiSq'][ampName] = record['PTC_FIT_CHI_SQ'] 

543 inDict['covariances'][ampName] = record['COVARIANCES'] 

544 inDict['covariancesModel'][ampName] = record['COVARIANCES_MODEL'] 

545 inDict['covariancesSqrtWeights'][ampName] = record['COVARIANCES_SQRT_WEIGHTS'] 

546 inDict['aMatrix'][ampName] = record['A_MATRIX'] 

547 inDict['bMatrix'][ampName] = record['B_MATRIX'] 

548 inDict['covariancesModelNoB'][ampName] = record['COVARIANCES_MODEL_NO_B'] 

549 inDict['aMatrixNoB'][ampName] = record['A_MATRIX_NO_B'] 

550 inDict['finalVars'][ampName] = record['FINAL_VARS'] 

551 inDict['finalModelVars'][ampName] = record['FINAL_MODEL_VARS'] 

552 inDict['finalMeans'][ampName] = record['FINAL_MEANS'] 

553 inDict['badAmps'] = record['BAD_AMPS'].tolist() 

554 inDict['photoCharges'][ampName] = record['PHOTO_CHARGE'] 

555 if calibVersion == 1.0: 

556 mask = record['FINAL_MEANS'].mask 

557 array = record['FINAL_MEANS'][~mask] 

558 if len(array) > 0: 

559 inDict['ptcTurnoff'][ampName] = record['FINAL_MEANS'][~mask][-1] 

560 else: 

561 inDict['ptcTurnoff'][ampName] = np.nan 

562 else: 

563 inDict['ptcTurnoff'][ampName] = record['PTC_TURNOFF'] 

564 if calibVersion == 1.1: 

565 inDict['histVars'][ampName] = np.array([np.nan]) 

566 inDict['histChi2Dofs'][ampName] = np.array([np.nan]) 

567 inDict['kspValues'][ampName] = np.array([0.0]) 

568 else: 

569 inDict['histVars'][ampName] = record['HIST_VARS'] 

570 inDict['histChi2Dofs'][ampName] = record['HIST_CHI2_DOFS'] 

571 inDict['kspValues'][ampName] = record['KS_PVALUES'] 

572 

573 return cls().fromDict(inDict) 

574 

575 def toTable(self): 

576 """Construct a list of tables containing the information in this 

577 calibration. 

578 

579 The list of tables should create an identical calibration 

580 after being passed to this class's fromTable method. 

581 

582 Returns 

583 ------- 

584 tableList : `list` [`astropy.table.Table`] 

585 List of tables containing the linearity calibration 

586 information. 

587 """ 

588 tableList = [] 

589 self.updateMetadata() 

590 

591 badAmps = np.array(self.badAmps) if len(self.badAmps) else np.array([], dtype="U3") 

592 

593 catalogList = [] 

594 for ampName in self.ampNames: 

595 ampDict = { 

596 'AMPLIFIER_NAME': ampName, 

597 'PTC_FIT_TYPE': self.ptcFitType, 

598 'COV_MATRIX_SIDE': self.covMatrixSide, 

599 'INPUT_EXP_ID_PAIRS': self.inputExpIdPairs[ampName], 

600 'EXP_ID_MASK': self.expIdMask[ampName], 

601 'RAW_EXP_TIMES': self.rawExpTimes[ampName], 

602 'RAW_MEANS': self.rawMeans[ampName], 

603 'RAW_VARS': self.rawVars[ampName], 

604 'GAIN': self.gain[ampName], 

605 'GAIN_ERR': self.gainErr[ampName], 

606 'NOISE': self.noise[ampName], 

607 'NOISE_ERR': self.noiseErr[ampName], 

608 'HIST_VARS': self.histVars[ampName], 

609 'HIST_CHI2_DOFS': self.histChi2Dofs[ampName], 

610 'KS_PVALUES': self.kspValues[ampName], 

611 'PTC_FIT_PARS': np.array(self.ptcFitPars[ampName]), 

612 'PTC_FIT_PARS_ERROR': np.array(self.ptcFitParsError[ampName]), 

613 'PTC_FIT_CHI_SQ': self.ptcFitChiSq[ampName], 

614 'PTC_TURNOFF': self.ptcTurnoff[ampName], 

615 'A_MATRIX': self.aMatrix[ampName].ravel(), 

616 'B_MATRIX': self.bMatrix[ampName].ravel(), 

617 'A_MATRIX_NO_B': self.aMatrixNoB[ampName].ravel(), 

618 'BAD_AMPS': badAmps, 

619 'PHOTO_CHARGE': self.photoCharges[ampName], 

620 'COVARIANCES': self.covariances[ampName].ravel(), 

621 'COVARIANCES_MODEL': self.covariancesModel[ampName].ravel(), 

622 'COVARIANCES_SQRT_WEIGHTS': self.covariancesSqrtWeights[ampName].ravel(), 

623 'COVARIANCES_MODEL_NO_B': self.covariancesModelNoB[ampName].ravel(), 

624 'FINAL_VARS': self.finalVars[ampName], 

625 'FINAL_MODEL_VARS': self.finalModelVars[ampName], 

626 'FINAL_MEANS': self.finalMeans[ampName], 

627 } 

628 catalogList.append(ampDict) 

629 

630 catalog = Table(catalogList) 

631 

632 inMeta = self.getMetadata().toDict() 

633 outMeta = {k: v for k, v in inMeta.items() if v is not None} 

634 outMeta.update({k: "" for k, v in inMeta.items() if v is None}) 

635 catalog.meta = outMeta 

636 tableList.append(catalog) 

637 

638 return(tableList) 

639 

640 def fromDetector(self, detector): 

641 """Read metadata parameters from a detector. 

642 

643 Parameters 

644 ---------- 

645 detector : `lsst.afw.cameraGeom.detector` 

646 Input detector with parameters to use. 

647 

648 Returns 

649 ------- 

650 calib : `lsst.ip.isr.PhotonTransferCurveDataset` 

651 The calibration constructed from the detector. 

652 """ 

653 

654 pass 

655 

656 def getExpIdsUsed(self, ampName): 

657 """Get the exposures used, i.e. not discarded, for a given amp. 

658 If no mask has been created yet, all exposures are returned. 

659 

660 Parameters 

661 ---------- 

662 ampName : `str` 

663 

664 Returns 

665 ------- 

666 expIdsUsed : `list` [`tuple`] 

667 List of pairs of exposure ids used in PTC. 

668 """ 

669 if len(self.expIdMask[ampName]) == 0: 

670 return self.inputExpIdPairs[ampName] 

671 

672 # if the mask exists it had better be the same length as the expIdPairs 

673 assert len(self.expIdMask[ampName]) == len(self.inputExpIdPairs[ampName]) 

674 

675 pairs = self.inputExpIdPairs[ampName] 

676 mask = self.expIdMask[ampName] 

677 # cast to bool required because numpy 

678 try: 

679 expIdsUsed = [(exp1, exp2) for ((exp1, exp2), m) in zip(pairs, mask) if m] 

680 except ValueError: 

681 warnings.warn("The PTC file was written incorrectly; you should rerun the " 

682 "PTC solve task if possible.", RuntimeWarning) 

683 expIdsUsed = [] 

684 for pairList, m in zip(pairs, mask): 

685 if m: 

686 expIdsUsed.append(pairList[0]) 

687 

688 return expIdsUsed 

689 

690 def getGoodAmps(self): 

691 """Get the good amps from this PTC.""" 

692 return [amp for amp in self.ampNames if amp not in self.badAmps] 

693 

694 def getGoodPoints(self, ampName): 

695 """Get the good points used for a given amp in the PTC. 

696 

697 Parameters 

698 ---------- 

699 ampName : `str` 

700 

701 Returns 

702 ------- 

703 goodPoints : `np.ndarray` 

704 Boolean array of good points used in PTC. 

705 """ 

706 return self.expIdMask[ampName]