lsst.ip.isr gbf28802888+bf8a358e9b
Loading...
Searching...
No Matches
ptcDataset.py
Go to the documentation of this file.
2# LSST Data Management System
3# Copyright 2008-2017 AURA/LSST.
4#
5# This product includes software developed by the
6# LSST Project (http://www.lsst.org/).
7#
8# This program is free software: you can redistribute it and/or modify
9# it under the terms of the GNU General Public License as published by
10# the Free Software Foundation, either version 3 of the License, or
11# (at your option) any later version.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the LSST License Statement and
19# the GNU General Public License along with this program. If not,
20# see <https://www.lsstcorp.org/LegalNotices/>.
21#
22"""
23Define dataset class for MeasurePhotonTransferCurve task
24"""
25
26__all__ = ['PhotonTransferCurveDataset']
27
28import numpy as np
29from astropy.table import Table
30import warnings
31
32from lsst.ip.isr import IsrCalib
33
34
36 """A simple class to hold the output data from the PTC task.
37
38 The dataset is made up of a dictionary for each item, keyed by the
39 amplifiers' names, which much be supplied at construction time.
40 New items cannot be added to the class to save accidentally saving to the
41 wrong property, and the class can be frozen if desired.
42 inputExpIdPairs records the exposures used to produce the data.
43 When fitPtc() or fitCovariancesAstier() is run, a mask is built up, which
44 is by definition always the same length as inputExpIdPairs, rawExpTimes,
45 rawMeans and rawVars, and is a list of bools, which are incrementally set
46 to False as points are discarded from the fits.
47 PTC fit parameters for polynomials are stored in a list in ascending order
48 of polynomial term, i.e. par[0]*x^0 + par[1]*x + par[2]*x^2 etc
49 with the length of the list corresponding to the order of the polynomial
50 plus one.
51
52 Parameters
53 ----------
54 ampNames : `list`
55 List with the names of the amplifiers of the detector at hand.
56 ptcFitType : `str`
57 Type of model fitted to the PTC: "POLYNOMIAL", "EXPAPPROXIMATION",
58 or "FULLCOVARIANCE".
59 covMatrixSide : `int`
60 Maximum lag of covariances (size of square covariance matrices).
61 kwargs : `dict`, optional
62 Other keyword arguments to pass to the parent init.
63
64 Notes
65 -----
66 The stored attributes are:
67
68 badAmps : `list` [`str`]
69 List with bad amplifiers names.
70 inputExpIdPairs : `dict`, [`str`, `list`]
71 Dictionary keyed by amp names containing the input exposures IDs.
72 expIdMask : `dict`, [`str`, `np.ndarray`]
73 Dictionary keyed by amp names containing the mask produced after
74 outlier rejection. The mask produced by the "FULLCOVARIANCE"
75 option may differ from the one produced in the other two PTC
76 fit types.
77 rawExpTimes : `dict`, [`str`, `np.ndarray`]
78 Dictionary keyed by amp names containing the unmasked exposure times.
79 rawMeans : `dict`, [`str`, `np.ndarray`]
80 Dictionary keyed by amp names containing the unmasked average of the
81 means of the exposures in each flat pair.
82 rawVars : `dict`, [`str`, `np.ndarray`]
83 Dictionary keyed by amp names containing the variance of the
84 difference image of the exposures in each flat pair.
85 histVars : `dict`, [`str`, `np.ndarray`]
86 Dictionary keyed by amp names containing the variance of the
87 difference image of the exposures in each flat pair estimated
88 by fitting a Gaussian model.
89 histChi2Dofs : `dict`, [`str`, `np.ndarray`]
90 Dictionary keyed by amp names containing the chi-squared per degree
91 of freedom fitting the difference image to a Gaussian model.
92 kspValues : `dict`, [`str`, `np.ndarray`]
93 Dictionary keyed by amp names containing the KS test p-value from
94 fitting the difference image to a Gaussian model.
95 gain : `dict`, [`str`, `float`]
96 Dictionary keyed by amp names containing the fitted gains.
97 gainErr : `dict`, [`str`, `float`]
98 Dictionary keyed by amp names containing the errors on the
99 fitted gains.
100 noise : `dict`, [`str`, `float`]
101 Dictionary keyed by amp names containing the fitted noise.
102 noiseErr : `dict`, [`str`, `float`]
103 Dictionary keyed by amp names containing the errors on the fitted
104 noise.
105 ptcFitPars : `dict`, [`str`, `np.ndarray`]
106 Dictionary keyed by amp names containing the fitted parameters of the
107 PTC model for ptcFitTye in ["POLYNOMIAL", "EXPAPPROXIMATION"].
108 ptcFitParsError : `dict`, [`str`, `np.ndarray`]
109 Dictionary keyed by amp names containing the errors on the fitted
110 parameters of the PTC model for ptcFitTye in
111 ["POLYNOMIAL", "EXPAPPROXIMATION"].
112 ptcFitChiSq : `dict`, [`str`, `float`]
113 Dictionary keyed by amp names containing the reduced chi squared
114 of the fit for ptcFitTye in ["POLYNOMIAL", "EXPAPPROXIMATION"].
115 ptcTurnoff : `dict` [`str, `float`]
116 Flux value (in ADU) where the variance of the PTC curve starts
117 decreasing consistently.
118 covariances : `dict`, [`str`, `np.ndarray`]
119 Dictionary keyed by amp names containing a list of measured
120 covariances per mean flux.
121 covariancesModel : `dict`, [`str`, `np.ndarray`]
122 Dictionary keyed by amp names containinging covariances model
123 (Eq. 20 of Astier+19) per mean flux.
124 covariancesSqrtWeights : `dict`, [`str`, `np.ndarray`]
125 Dictionary keyed by amp names containinging sqrt. of covariances
126 weights.
127 aMatrix : `dict`, [`str`, `np.ndarray`]
128 Dictionary keyed by amp names containing the "a" parameters from
129 the model in Eq. 20 of Astier+19.
130 bMatrix : `dict`, [`str`, `np.ndarray`]
131 Dictionary keyed by amp names containing the "b" parameters from
132 the model in Eq. 20 of Astier+19.
133 covariancesModelNoB : `dict`, [`str`, `np.ndarray`]
134 Dictionary keyed by amp names containing covariances model
135 (with 'b'=0 in Eq. 20 of Astier+19)
136 per mean flux.
137 aMatrixNoB : `dict`, [`str`, `np.ndarray`]
138 Dictionary keyed by amp names containing the "a" parameters from the
139 model in Eq. 20 of Astier+19
140 (and 'b' = 0).
141 finalVars : `dict`, [`str`, `np.ndarray`]
142 Dictionary keyed by amp names containing the masked variance of the
143 difference image of each flat
144 pair. If needed, each array will be right-padded with
145 np.nan to match the length of rawExpTimes.
146 finalModelVars : `dict`, [`str`, `np.ndarray`]
147 Dictionary keyed by amp names containing the masked modeled
148 variance of the difference image of each flat pair. If needed, each
149 array will be right-padded with np.nan to match the length of
150 rawExpTimes.
151 finalMeans : `dict`, [`str`, `np.ndarray`]
152 Dictionary keyed by amp names containing the masked average of the
153 means of the exposures in each flat pair. If needed, each array
154 will be right-padded with np.nan to match the length of
155 rawExpTimes.
156 photoCharges : `dict`, [`str`, `np.ndarray`]
157 Dictionary keyed by amp names containing the integrated photocharge
158 for linearity calibration.
159
160 Version 1.1 adds the `ptcTurnoff` attribute.
161 """
162
163 _OBSTYPE = 'PTC'
164 _SCHEMA = 'Gen3 Photon Transfer Curve'
165 _VERSION = 1.2
166
167 def __init__(self, ampNames=[], ptcFitType=None, covMatrixSide=1, **kwargs):
168 self.ptcFitType = ptcFitType
169 self.ampNames = ampNames
170 self.covMatrixSide = covMatrixSide
171
172 self.badAmps = []
173
174 self.inputExpIdPairs = {ampName: [] for ampName in ampNames}
175 self.expIdMask = {ampName: np.array([], dtype=bool) for ampName in ampNames}
176 self.rawExpTimes = {ampName: np.array([]) for ampName in ampNames}
177 self.rawMeans = {ampName: np.array([]) for ampName in ampNames}
178 self.rawVars = {ampName: np.array([]) for ampName in ampNames}
179 self.photoCharges = {ampName: np.array([]) for ampName in ampNames}
180
181 self.gain = {ampName: np.nan for ampName in ampNames}
182 self.gainErr = {ampName: np.nan for ampName in ampNames}
183 self.noise = {ampName: np.nan for ampName in ampNames}
184 self.noiseErr = {ampName: np.nan for ampName in ampNames}
185
186 self.histVars = {ampName: np.array([]) for ampName in ampNames}
187 self.histChi2Dofs = {ampName: np.array([]) for ampName in ampNames}
188 self.kspValues = {ampName: np.array([]) for ampName in ampNames}
189
190 self.ptcFitPars = {ampName: np.array([]) for ampName in ampNames}
191 self.ptcFitParsError = {ampName: np.array([]) for ampName in ampNames}
192 self.ptcFitChiSq = {ampName: np.nan for ampName in ampNames}
193 self.ptcTurnoff = {ampName: np.nan for ampName in ampNames}
194
195 self.covariances = {ampName: np.array([]) for ampName in ampNames}
196 self.covariancesModel = {ampName: np.array([]) for ampName in ampNames}
197 self.covariancesSqrtWeights = {ampName: np.array([]) for ampName in ampNames}
198 self.aMatrix = {ampName: np.array([]) for ampName in ampNames}
199 self.bMatrix = {ampName: np.array([]) for ampName in ampNames}
200 self.covariancesModelNoB = {ampName: np.array([]) for ampName in ampNames}
201 self.aMatrixNoB = {ampName: np.array([]) for ampName in ampNames}
202
203 self.finalVars = {ampName: np.array([]) for ampName in ampNames}
204 self.finalModelVars = {ampName: np.array([]) for ampName in ampNames}
205 self.finalMeans = {ampName: np.array([]) for ampName in ampNames}
206
207 super().__init__(**kwargs)
208 self.requiredAttributesrequiredAttributesrequiredAttributes.update(['badAmps', 'inputExpIdPairs', 'expIdMask', 'rawExpTimes',
209 'rawMeans', 'rawVars', 'gain', 'gainErr', 'noise', 'noiseErr',
210 'ptcFitPars', 'ptcFitParsError', 'ptcFitChiSq', 'ptcTurnoff',
211 'aMatrixNoB', 'covariances', 'covariancesModel',
212 'covariancesSqrtWeights', 'covariancesModelNoB',
213 'aMatrix', 'bMatrix', 'finalVars', 'finalModelVars', 'finalMeans',
214 'photoCharges', 'histVars', 'histChi2Dofs', 'kspValues'])
215
216 self.updateMetadataupdateMetadata(setCalibInfo=True, setCalibId=True, **kwargs)
217
219 self,
220 ampName,
221 inputExpIdPair=(-1, -1),
222 rawExpTime=np.nan,
223 rawMean=np.nan,
224 rawVar=np.nan,
225 photoCharge=np.nan,
226 expIdMask=False,
227 covariance=None,
228 covSqrtWeights=None,
229 gain=np.nan,
230 noise=np.nan,
231 histVar=np.nan,
232 histChi2Dof=np.nan,
233 kspValue=0.0,
234 ):
235 """
236 Set the amp values for a partial PTC Dataset (from cpExtractPtcTask).
237
238 Parameters
239 ----------
240 ampName : `str`
241 Name of the amp to set the values.
242 inputExpIdPair : `tuple` [`int`]
243 Exposure IDs of input pair.
244 rawExpTime : `float`, optional
245 Exposure time for this exposure pair.
246 rawMean : `float`, optional
247 Average of the means of the exposures in this pair.
248 rawVar : `float`, optional
249 Variance of the difference of the exposures in this pair.
250 photoCharge : `float`, optional
251 Integrated photocharge for flat pair for linearity calibration.
252 expIdMask : `bool`, optional
253 Flag setting if this exposure pair should be used (True)
254 or not used (False).
255 covariance : `np.ndarray` or None, optional
256 Measured covariance for this exposure pair.
257 covSqrtWeights : `np.ndarray` or None, optional
258 Measured sqrt of covariance weights in this exposure pair.
259 gain : `float`, optional
260 Estimated gain for this exposure pair.
261 noise : `float`, optional
262 Estimated read noise for this exposure pair.
263 histVar : `float`, optional
264 Variance estimated from fitting a histogram with a Gaussian model.
265 histChi2Dof : `float`, optional
266 Chi-squared per degree of freedom from Gaussian histogram fit.
267 kspValue : `float`, optional
268 KS test p-value from the Gaussian histogram fit.
269 """
270 nanMatrix = np.full((self.covMatrixSide, self.covMatrixSide), np.nan)
271 if covariance is None:
272 covariance = nanMatrix
273 if covSqrtWeights is None:
274 covSqrtWeights = nanMatrix
275
276 self.inputExpIdPairs[ampName] = [inputExpIdPair]
277 self.rawExpTimes[ampName] = np.array([rawExpTime])
278 self.rawMeans[ampName] = np.array([rawMean])
279 self.rawVars[ampName] = np.array([rawVar])
280 self.photoCharges[ampName] = np.array([photoCharge])
281 self.expIdMask[ampName] = np.array([expIdMask])
282 self.covariances[ampName] = np.array([covariance])
283 self.covariancesSqrtWeights[ampName] = np.array([covSqrtWeights])
284 self.gain[ampName] = gain
285 self.noise[ampName] = noise
286 self.histVars[ampName] = np.array([histVar])
287 self.histChi2Dofs[ampName] = np.array([histChi2Dof])
288 self.kspValues[ampName] = np.array([kspValue])
289
290 self.covariancesModel[ampName] = np.array([nanMatrix])
291 self.covariancesModelNoB[ampName] = np.array([nanMatrix])
292 self.aMatrix[ampName] = nanMatrix
293 self.bMatrix[ampName] = nanMatrix
294 self.aMatrixNoB[ampName] = nanMatrix
295
296 def updateMetadata(self, **kwargs):
297 """Update calibration metadata.
298 This calls the base class's method after ensuring the required
299 calibration keywords will be saved.
300
301 Parameters
302 ----------
303 setDate : `bool`, optional
304 Update the CALIBDATE fields in the metadata to the current
305 time. Defaults to False.
306 kwargs :
307 Other keyword parameters to set in the metadata.
308 """
309 super().updateMetadata(PTC_FIT_TYPE=self.ptcFitType, **kwargs)
310
311 @classmethod
312 def fromDict(cls, dictionary):
313 """Construct a calibration from a dictionary of properties.
314 Must be implemented by the specific calibration subclasses.
315
316 Parameters
317 ----------
318 dictionary : `dict`
319 Dictionary of properties.
320
321 Returns
322 -------
324 Constructed calibration.
325
326 Raises
327 ------
328 RuntimeError
329 Raised if the supplied dictionary is for a different
330 calibration.
331 """
332 calib = cls()
333 if calib._OBSTYPE != dictionary['metadata']['OBSTYPE']:
334 raise RuntimeError(f"Incorrect Photon Transfer Curve dataset supplied. "
335 f"Expected {calib._OBSTYPE}, found {dictionary['metadata']['OBSTYPE']}")
336 calib.setMetadata(dictionary['metadata'])
337 calib.ptcFitType = dictionary['ptcFitType']
338 calib.covMatrixSide = dictionary['covMatrixSide']
339 calib.badAmps = np.array(dictionary['badAmps'], 'str').tolist()
340 calib.ampNames = []
341
342 # The cov matrices are square
343 covMatrixSide = calib.covMatrixSide
344 # Number of final signal levels
345 covDimensionsProduct = len(np.array(list(dictionary['covariances'].values())[0]).ravel())
346 nSignalPoints = int(covDimensionsProduct/(covMatrixSide*covMatrixSide))
347
348 for ampName in dictionary['ampNames']:
349 calib.ampNames.append(ampName)
350 calib.inputExpIdPairs[ampName] = dictionary['inputExpIdPairs'][ampName]
351 calib.expIdMask[ampName] = np.array(dictionary['expIdMask'][ampName])
352 calib.rawExpTimes[ampName] = np.array(dictionary['rawExpTimes'][ampName], dtype=np.float64)
353 calib.rawMeans[ampName] = np.array(dictionary['rawMeans'][ampName], dtype=np.float64)
354 calib.rawVars[ampName] = np.array(dictionary['rawVars'][ampName], dtype=np.float64)
355 calib.gain[ampName] = float(dictionary['gain'][ampName])
356 calib.gainErr[ampName] = float(dictionary['gainErr'][ampName])
357 calib.noise[ampName] = float(dictionary['noise'][ampName])
358 calib.noiseErr[ampName] = float(dictionary['noiseErr'][ampName])
359 calib.histVars[ampName] = np.array(dictionary['histVars'][ampName], dtype=np.float64)
360 calib.histChi2Dofs[ampName] = np.array(dictionary['histChi2Dofs'][ampName], dtype=np.float64)
361 calib.kspValues[ampName] = np.array(dictionary['kspValues'][ampName], dtype=np.float64)
362 calib.ptcFitPars[ampName] = np.array(dictionary['ptcFitPars'][ampName], dtype=np.float64)
363 calib.ptcFitParsError[ampName] = np.array(dictionary['ptcFitParsError'][ampName],
364 dtype=np.float64)
365 calib.ptcFitChiSq[ampName] = float(dictionary['ptcFitChiSq'][ampName])
366 calib.ptcTurnoff[ampName] = float(dictionary['ptcTurnoff'][ampName])
367 if nSignalPoints > 0:
368 # Regular dataset
369 calib.covariances[ampName] = np.array(dictionary['covariances'][ampName],
370 dtype=np.float64).reshape(
371 (nSignalPoints, covMatrixSide, covMatrixSide))
372 calib.covariancesModel[ampName] = np.array(
373 dictionary['covariancesModel'][ampName],
374 dtype=np.float64).reshape(
375 (nSignalPoints, covMatrixSide, covMatrixSide))
376 calib.covariancesSqrtWeights[ampName] = np.array(
377 dictionary['covariancesSqrtWeights'][ampName],
378 dtype=np.float64).reshape(
379 (nSignalPoints, covMatrixSide, covMatrixSide))
380 calib.aMatrix[ampName] = np.array(dictionary['aMatrix'][ampName],
381 dtype=np.float64).reshape(
382 (covMatrixSide, covMatrixSide))
383 calib.bMatrix[ampName] = np.array(dictionary['bMatrix'][ampName],
384 dtype=np.float64).reshape(
385 (covMatrixSide, covMatrixSide))
386 calib.covariancesModelNoB[ampName] = np.array(
387 dictionary['covariancesModelNoB'][ampName], dtype=np.float64).reshape(
388 (nSignalPoints, covMatrixSide, covMatrixSide))
389 calib.aMatrixNoB[ampName] = np.array(
390 dictionary['aMatrixNoB'][ampName],
391 dtype=np.float64).reshape((covMatrixSide, covMatrixSide))
392 else:
393 # Empty dataset
394 calib.covariances[ampName] = np.array([], dtype=np.float64)
395 calib.covariancesModel[ampName] = np.array([], dtype=np.float64)
396 calib.covariancesSqrtWeights[ampName] = np.array([], dtype=np.float64)
397 calib.aMatrix[ampName] = np.array([], dtype=np.float64)
398 calib.bMatrix[ampName] = np.array([], dtype=np.float64)
399 calib.covariancesModelNoB[ampName] = np.array([], dtype=np.float64)
400 calib.aMatrixNoB[ampName] = np.array([], dtype=np.float64)
401
402 calib.finalVars[ampName] = np.array(dictionary['finalVars'][ampName], dtype=np.float64)
403 calib.finalModelVars[ampName] = np.array(dictionary['finalModelVars'][ampName], dtype=np.float64)
404 calib.finalMeans[ampName] = np.array(dictionary['finalMeans'][ampName], dtype=np.float64)
405 calib.photoCharges[ampName] = np.array(dictionary['photoCharges'][ampName], dtype=np.float64)
406
407 calib.updateMetadata()
408 return calib
409
410 def toDict(self):
411 """Return a dictionary containing the calibration properties.
412 The dictionary should be able to be round-tripped through
413 `fromDict`.
414
415 Returns
416 -------
417 dictionary : `dict`
418 Dictionary of properties.
419 """
421
422 outDict = dict()
423 metadata = self.getMetadata()
424 outDict['metadata'] = metadata
425
426 def _dictOfArraysToDictOfLists(dictOfArrays):
427 dictOfLists = {}
428 for key, value in dictOfArrays.items():
429 dictOfLists[key] = value.ravel().tolist()
430
431 return dictOfLists
432
433 outDict['ptcFitType'] = self.ptcFitType
434 outDict['covMatrixSide'] = self.covMatrixSide
435 outDict['ampNames'] = self.ampNames
436 outDict['badAmps'] = self.badAmps
437 outDict['inputExpIdPairs'] = self.inputExpIdPairs
438 outDict['expIdMask'] = _dictOfArraysToDictOfLists(self.expIdMask)
439 outDict['rawExpTimes'] = _dictOfArraysToDictOfLists(self.rawExpTimes)
440 outDict['rawMeans'] = _dictOfArraysToDictOfLists(self.rawMeans)
441 outDict['rawVars'] = _dictOfArraysToDictOfLists(self.rawVars)
442 outDict['gain'] = self.gain
443 outDict['gainErr'] = self.gainErr
444 outDict['noise'] = self.noise
445 outDict['noiseErr'] = self.noiseErr
446 outDict['histVars'] = self.histVars
447 outDict['histChi2Dofs'] = self.histChi2Dofs
448 outDict['kspValues'] = self.kspValues
449 outDict['ptcFitPars'] = _dictOfArraysToDictOfLists(self.ptcFitPars)
450 outDict['ptcFitParsError'] = _dictOfArraysToDictOfLists(self.ptcFitParsError)
451 outDict['ptcFitChiSq'] = self.ptcFitChiSq
452 outDict['ptcTurnoff'] = self.ptcTurnoff
453 outDict['covariances'] = _dictOfArraysToDictOfLists(self.covariances)
454 outDict['covariancesModel'] = _dictOfArraysToDictOfLists(self.covariancesModel)
455 outDict['covariancesSqrtWeights'] = _dictOfArraysToDictOfLists(self.covariancesSqrtWeights)
456 outDict['aMatrix'] = _dictOfArraysToDictOfLists(self.aMatrix)
457 outDict['bMatrix'] = _dictOfArraysToDictOfLists(self.bMatrix)
458 outDict['covariancesModelNoB'] = _dictOfArraysToDictOfLists(self.covariancesModelNoB)
459 outDict['aMatrixNoB'] = _dictOfArraysToDictOfLists(self.aMatrixNoB)
460 outDict['finalVars'] = _dictOfArraysToDictOfLists(self.finalVars)
461 outDict['finalModelVars'] = _dictOfArraysToDictOfLists(self.finalModelVars)
462 outDict['finalMeans'] = _dictOfArraysToDictOfLists(self.finalMeans)
463 outDict['photoCharges'] = _dictOfArraysToDictOfLists(self.photoCharges)
464
465 return outDict
466
467 @classmethod
468 def fromTable(cls, tableList):
469 """Construct calibration from a list of tables.
470 This method uses the `fromDict` method to create the
471 calibration, after constructing an appropriate dictionary from
472 the input tables.
473
474 Parameters
475 ----------
476 tableList : `list` [`lsst.afw.table.Table`]
477 List of tables to use to construct the datasetPtc.
478
479 Returns
480 -------
482 The calibration defined in the tables.
483 """
484 ptcTable = tableList[0]
485
486 metadata = ptcTable.meta
487 inDict = dict()
488 inDict['metadata'] = metadata
489 inDict['ampNames'] = []
490 inDict['ptcFitType'] = []
491 inDict['covMatrixSide'] = []
492 inDict['inputExpIdPairs'] = dict()
493 inDict['expIdMask'] = dict()
494 inDict['rawExpTimes'] = dict()
495 inDict['rawMeans'] = dict()
496 inDict['rawVars'] = dict()
497 inDict['gain'] = dict()
498 inDict['gainErr'] = dict()
499 inDict['noise'] = dict()
500 inDict['noiseErr'] = dict()
501 inDict['histVars'] = dict()
502 inDict['histChi2Dofs'] = dict()
503 inDict['kspValues'] = dict()
504 inDict['ptcFitPars'] = dict()
505 inDict['ptcFitParsError'] = dict()
506 inDict['ptcFitChiSq'] = dict()
507 inDict['ptcTurnoff'] = dict()
508 inDict['covariances'] = dict()
509 inDict['covariancesModel'] = dict()
510 inDict['covariancesSqrtWeights'] = dict()
511 inDict['aMatrix'] = dict()
512 inDict['bMatrix'] = dict()
513 inDict['covariancesModelNoB'] = dict()
514 inDict['aMatrixNoB'] = dict()
515 inDict['finalVars'] = dict()
516 inDict['finalModelVars'] = dict()
517 inDict['finalMeans'] = dict()
518 inDict['badAmps'] = []
519 inDict['photoCharges'] = dict()
520
521 calibVersion = metadata['PTC_VERSION']
522 if calibVersion == 1.0:
523 cls().log.warning(f"Previous version found for PTC dataset: {calibVersion}. "
524 f"Setting 'ptcTurnoff' in all amps to last value in 'finalMeans'.")
525 for record in ptcTable:
526 ampName = record['AMPLIFIER_NAME']
527
528 inDict['ptcFitType'] = record['PTC_FIT_TYPE']
529 inDict['covMatrixSide'] = record['COV_MATRIX_SIDE']
530 inDict['ampNames'].append(ampName)
531 inDict['inputExpIdPairs'][ampName] = record['INPUT_EXP_ID_PAIRS'].tolist()
532 inDict['expIdMask'][ampName] = record['EXP_ID_MASK']
533 inDict['rawExpTimes'][ampName] = record['RAW_EXP_TIMES']
534 inDict['rawMeans'][ampName] = record['RAW_MEANS']
535 inDict['rawVars'][ampName] = record['RAW_VARS']
536 inDict['gain'][ampName] = record['GAIN']
537 inDict['gainErr'][ampName] = record['GAIN_ERR']
538 inDict['noise'][ampName] = record['NOISE']
539 inDict['noiseErr'][ampName] = record['NOISE_ERR']
540 inDict['ptcFitPars'][ampName] = record['PTC_FIT_PARS']
541 inDict['ptcFitParsError'][ampName] = record['PTC_FIT_PARS_ERROR']
542 inDict['ptcFitChiSq'][ampName] = record['PTC_FIT_CHI_SQ']
543 inDict['covariances'][ampName] = record['COVARIANCES']
544 inDict['covariancesModel'][ampName] = record['COVARIANCES_MODEL']
545 inDict['covariancesSqrtWeights'][ampName] = record['COVARIANCES_SQRT_WEIGHTS']
546 inDict['aMatrix'][ampName] = record['A_MATRIX']
547 inDict['bMatrix'][ampName] = record['B_MATRIX']
548 inDict['covariancesModelNoB'][ampName] = record['COVARIANCES_MODEL_NO_B']
549 inDict['aMatrixNoB'][ampName] = record['A_MATRIX_NO_B']
550 inDict['finalVars'][ampName] = record['FINAL_VARS']
551 inDict['finalModelVars'][ampName] = record['FINAL_MODEL_VARS']
552 inDict['finalMeans'][ampName] = record['FINAL_MEANS']
553 inDict['badAmps'] = record['BAD_AMPS'].tolist()
554 inDict['photoCharges'][ampName] = record['PHOTO_CHARGE']
555 if calibVersion == 1.0:
556 mask = record['FINAL_MEANS'].mask
557 array = record['FINAL_MEANS'][~mask]
558 if len(array) > 0:
559 inDict['ptcTurnoff'][ampName] = record['FINAL_MEANS'][~mask][-1]
560 else:
561 inDict['ptcTurnoff'][ampName] = np.nan
562 else:
563 inDict['ptcTurnoff'][ampName] = record['PTC_TURNOFF']
564 if calibVersion == 1.1:
565 inDict['histVars'][ampName] = np.array([np.nan])
566 inDict['histChi2Dofs'][ampName] = np.array([np.nan])
567 inDict['kspValues'][ampName] = np.array([0.0])
568 else:
569 inDict['histVars'][ampName] = record['HIST_VARS']
570 inDict['histChi2Dofs'][ampName] = record['HIST_CHI2_DOFS']
571 inDict['kspValues'][ampName] = record['KS_PVALUES']
572
573 return cls().fromDict(inDict)
574
575 def toTable(self):
576 """Construct a list of tables containing the information in this
577 calibration.
578
579 The list of tables should create an identical calibration
580 after being passed to this class's fromTable method.
581
582 Returns
583 -------
584 tableList : `list` [`astropy.table.Table`]
585 List of tables containing the linearity calibration
586 information.
587 """
588 tableList = []
590
591 badAmps = np.array(self.badAmps) if len(self.badAmps) else np.array([], dtype="U3")
592
593 catalogList = []
594 for ampName in self.ampNames:
595 ampDict = {
596 'AMPLIFIER_NAME': ampName,
597 'PTC_FIT_TYPE': self.ptcFitType,
598 'COV_MATRIX_SIDE': self.covMatrixSide,
599 'INPUT_EXP_ID_PAIRS': self.inputExpIdPairs[ampName],
600 'EXP_ID_MASK': self.expIdMask[ampName],
601 'RAW_EXP_TIMES': self.rawExpTimes[ampName],
602 'RAW_MEANS': self.rawMeans[ampName],
603 'RAW_VARS': self.rawVars[ampName],
604 'GAIN': self.gain[ampName],
605 'GAIN_ERR': self.gainErr[ampName],
606 'NOISE': self.noise[ampName],
607 'NOISE_ERR': self.noiseErr[ampName],
608 'HIST_VARS': self.histVars[ampName],
609 'HIST_CHI2_DOFS': self.histChi2Dofs[ampName],
610 'KS_PVALUES': self.kspValues[ampName],
611 'PTC_FIT_PARS': np.array(self.ptcFitPars[ampName]),
612 'PTC_FIT_PARS_ERROR': np.array(self.ptcFitParsError[ampName]),
613 'PTC_FIT_CHI_SQ': self.ptcFitChiSq[ampName],
614 'PTC_TURNOFF': self.ptcTurnoff[ampName],
615 'A_MATRIX': self.aMatrix[ampName].ravel(),
616 'B_MATRIX': self.bMatrix[ampName].ravel(),
617 'A_MATRIX_NO_B': self.aMatrixNoB[ampName].ravel(),
618 'BAD_AMPS': badAmps,
619 'PHOTO_CHARGE': self.photoCharges[ampName],
620 'COVARIANCES': self.covariances[ampName].ravel(),
621 'COVARIANCES_MODEL': self.covariancesModel[ampName].ravel(),
622 'COVARIANCES_SQRT_WEIGHTS': self.covariancesSqrtWeights[ampName].ravel(),
623 'COVARIANCES_MODEL_NO_B': self.covariancesModelNoB[ampName].ravel(),
624 'FINAL_VARS': self.finalVars[ampName],
625 'FINAL_MODEL_VARS': self.finalModelVars[ampName],
626 'FINAL_MEANS': self.finalMeans[ampName],
627 }
628 catalogList.append(ampDict)
629
630 catalog = Table(catalogList)
631
632 inMeta = self.getMetadata().toDict()
633 outMeta = {k: v for k, v in inMeta.items() if v is not None}
634 outMeta.update({k: "" for k, v in inMeta.items() if v is None})
635 catalog.meta = outMeta
636 tableList.append(catalog)
637
638 return(tableList)
639
640 def fromDetector(self, detector):
641 """Read metadata parameters from a detector.
642
643 Parameters
644 ----------
645 detector : `lsst.afw.cameraGeom.detector`
646 Input detector with parameters to use.
647
648 Returns
649 -------
651 The calibration constructed from the detector.
652 """
653
654 pass
655
656 def getExpIdsUsed(self, ampName):
657 """Get the exposures used, i.e. not discarded, for a given amp.
658 If no mask has been created yet, all exposures are returned.
659
660 Parameters
661 ----------
662 ampName : `str`
663
664 Returns
665 -------
666 expIdsUsed : `list` [`tuple`]
667 List of pairs of exposure ids used in PTC.
668 """
669 if len(self.expIdMask[ampName]) == 0:
670 return self.inputExpIdPairs[ampName]
671
672 # if the mask exists it had better be the same length as the expIdPairs
673 assert len(self.expIdMask[ampName]) == len(self.inputExpIdPairs[ampName])
674
675 pairs = self.inputExpIdPairs[ampName]
676 mask = self.expIdMask[ampName]
677 # cast to bool required because numpy
678 try:
679 expIdsUsed = [(exp1, exp2) for ((exp1, exp2), m) in zip(pairs, mask) if m]
680 except ValueError:
681 warnings.warn("The PTC file was written incorrectly; you should rerun the "
682 "PTC solve task if possible.", RuntimeWarning)
683 expIdsUsed = []
684 for pairList, m in zip(pairs, mask):
685 if m:
686 expIdsUsed.append(pairList[0])
687
688 return expIdsUsed
689
690 def getGoodAmps(self):
691 """Get the good amps from this PTC."""
692 return [amp for amp in self.ampNames if amp not in self.badAmps]
693
694 def getGoodPoints(self, ampName):
695 """Get the good points used for a given amp in the PTC.
696
697 Parameters
698 ----------
699 ampName : `str`
700
701 Returns
702 -------
703 goodPoints : `np.ndarray`
704 Boolean array of good points used in PTC.
705 """
706 return self.expIdMask[ampName]
def requiredAttributes(self, value)
Definition: calibType.py:158
def updateMetadata(self, camera=None, detector=None, filterName=None, setCalibId=False, setCalibInfo=False, setDate=False, **kwargs)
Definition: calibType.py:197
def __init__(self, ampNames=[], ptcFitType=None, covMatrixSide=1, **kwargs)
Definition: ptcDataset.py:167
def setAmpValuesPartialDataset(self, ampName, inputExpIdPair=(-1, -1), rawExpTime=np.nan, rawMean=np.nan, rawVar=np.nan, photoCharge=np.nan, expIdMask=False, covariance=None, covSqrtWeights=None, gain=np.nan, noise=np.nan, histVar=np.nan, histChi2Dof=np.nan, kspValue=0.0)
Definition: ptcDataset.py:234