Coverage for python/lsst/ip/isr/ptcDataset.py: 6%
363 statements
« prev ^ index » next coverage.py v7.4.1, created at 2024-01-27 10:05 +0000
« prev ^ index » next coverage.py v7.4.1, created at 2024-01-27 10:05 +0000
1#
2# LSST Data Management System
3# Copyright 2008-2017 AURA/LSST.
4#
5# This product includes software developed by the
6# LSST Project (http://www.lsst.org/).
7#
8# This program is free software: you can redistribute it and/or modify
9# it under the terms of the GNU General Public License as published by
10# the Free Software Foundation, either version 3 of the License, or
11# (at your option) any later version.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the LSST License Statement and
19# the GNU General Public License along with this program. If not,
20# see <https://www.lsstcorp.org/LegalNotices/>.
21#
22"""
23Define dataset class for MeasurePhotonTransferCurve task
24"""
26__all__ = ['PhotonTransferCurveDataset']
28import numpy as np
29import math
30from astropy.table import Table
32from lsst.ip.isr import IsrCalib
35class PhotonTransferCurveDataset(IsrCalib):
36 """A simple class to hold the output data from the PTC task.
38 The dataset is made up of a dictionary for each item, keyed by the
39 amplifiers' names, which much be supplied at construction time.
40 New items cannot be added to the class to save accidentally saving to the
41 wrong property, and the class can be frozen if desired.
42 inputExpIdPairs records the exposures used to produce the data.
43 When fitPtc() or fitCovariancesAstier() is run, a mask is built up, which
44 is by definition always the same length as inputExpIdPairs, rawExpTimes,
45 rawMeans and rawVars, and is a list of bools, which are incrementally set
46 to False as points are discarded from the fits.
47 PTC fit parameters for polynomials are stored in a list in ascending order
48 of polynomial term, i.e. par[0]*x^0 + par[1]*x + par[2]*x^2 etc
49 with the length of the list corresponding to the order of the polynomial
50 plus one.
52 Parameters
53 ----------
54 ampNames : `list`
55 List with the names of the amplifiers of the detector at hand.
56 ptcFitType : `str`, optional
57 Type of model fitted to the PTC: "POLYNOMIAL", "EXPAPPROXIMATION",
58 or "FULLCOVARIANCE".
59 covMatrixSide : `int`, optional
60 Maximum lag of measured covariances (size of square covariance
61 matrices).
62 covMatrixSideFullCovFit : `int, optional
63 Maximum covariances lag for FULLCOVARIANCE fit. It should be less or
64 equal than covMatrixSide.
65 kwargs : `dict`, optional
66 Other keyword arguments to pass to the parent init.
68 Notes
69 -----
70 The stored attributes are:
72 badAmps : `list` [`str`]
73 List with bad amplifiers names.
74 inputExpIdPairs : `dict`, [`str`, `list`]
75 Dictionary keyed by amp names containing the input exposures IDs.
76 expIdMask : `dict`, [`str`, `np.ndarray`]
77 Dictionary keyed by amp names containing the mask produced after
78 outlier rejection. The mask produced by the "FULLCOVARIANCE"
79 option may differ from the one produced in the other two PTC
80 fit types.
81 rawExpTimes : `dict`, [`str`, `np.ndarray`]
82 Dictionary keyed by amp names containing the unmasked exposure times.
83 rawMeans : `dict`, [`str`, `np.ndarray`]
84 Dictionary keyed by amp names containing the unmasked average of the
85 means of the exposures in each flat pair.
86 rawVars : `dict`, [`str`, `np.ndarray`]
87 Dictionary keyed by amp names containing the variance of the
88 difference image of the exposures in each flat pair.
89 rowMeanVariance : `dict`, [`str`, `np.ndarray`]
90 Dictionary keyed by amp names containing the variance of the
91 means of the rows of the difference image of the exposures
92 in each flat pair.
93 histVars : `dict`, [`str`, `np.ndarray`]
94 Dictionary keyed by amp names containing the variance of the
95 difference image of the exposures in each flat pair estimated
96 by fitting a Gaussian model.
97 histChi2Dofs : `dict`, [`str`, `np.ndarray`]
98 Dictionary keyed by amp names containing the chi-squared per degree
99 of freedom fitting the difference image to a Gaussian model.
100 kspValues : `dict`, [`str`, `np.ndarray`]
101 Dictionary keyed by amp names containing the KS test p-value from
102 fitting the difference image to a Gaussian model.
103 gain : `dict`, [`str`, `float`]
104 Dictionary keyed by amp names containing the fitted gains.
105 gainErr : `dict`, [`str`, `float`]
106 Dictionary keyed by amp names containing the errors on the
107 fitted gains.
108 noise : `dict`, [`str`, `float`]
109 Dictionary keyed by amp names containing the fitted noise.
110 noiseErr : `dict`, [`str`, `float`]
111 Dictionary keyed by amp names containing the errors on the fitted
112 noise.
113 ptcFitPars : `dict`, [`str`, `np.ndarray`]
114 Dictionary keyed by amp names containing the fitted parameters of the
115 PTC model for ptcFitTye in ["POLYNOMIAL", "EXPAPPROXIMATION"].
116 ptcFitParsError : `dict`, [`str`, `np.ndarray`]
117 Dictionary keyed by amp names containing the errors on the fitted
118 parameters of the PTC model for ptcFitTye in
119 ["POLYNOMIAL", "EXPAPPROXIMATION"].
120 ptcFitChiSq : `dict`, [`str`, `float`]
121 Dictionary keyed by amp names containing the reduced chi squared
122 of the fit for ptcFitTye in ["POLYNOMIAL", "EXPAPPROXIMATION"].
123 ptcTurnoff : `dict` [`str, `float`]
124 Flux value (in ADU) where the variance of the PTC curve starts
125 decreasing consistently.
126 covariances : `dict`, [`str`, `np.ndarray`]
127 Dictionary keyed by amp names containing a list of measured
128 covariances per mean flux.
129 covariancesModel : `dict`, [`str`, `np.ndarray`]
130 Dictionary keyed by amp names containinging covariances model
131 (Eq. 20 of Astier+19) per mean flux.
132 covariancesSqrtWeights : `dict`, [`str`, `np.ndarray`]
133 Dictionary keyed by amp names containinging sqrt. of covariances
134 weights.
135 aMatrix : `dict`, [`str`, `np.ndarray`]
136 Dictionary keyed by amp names containing the "a" parameters from
137 the model in Eq. 20 of Astier+19.
138 bMatrix : `dict`, [`str`, `np.ndarray`]
139 Dictionary keyed by amp names containing the "b" parameters from
140 the model in Eq. 20 of Astier+19.
141 noiseMatrix : `dict`, [`str`, `np.ndarray`]
142 Dictionary keyed by amp names containing the "noise" parameters from
143 the model in Eq. 20 of Astier+19.
144 covariancesModelNoB : `dict`, [`str`, `np.ndarray`]
145 Dictionary keyed by amp names containing covariances model
146 (with 'b'=0 in Eq. 20 of Astier+19)
147 per mean flux.
148 aMatrixNoB : `dict`, [`str`, `np.ndarray`]
149 Dictionary keyed by amp names containing the "a" parameters from the
150 model in Eq. 20 of Astier+19
151 (and 'b' = 0).
152 noiseMatrixNoB : `dict`, [`str`, `np.ndarray`]
153 Dictionary keyed by amp names containing the "noise" parameters from
154 the model in Eq. 20 of Astier+19, with 'b' = 0.
155 finalVars : `dict`, [`str`, `np.ndarray`]
156 Dictionary keyed by amp names containing the masked variance of the
157 difference image of each flat
158 pair. If needed, each array will be right-padded with
159 np.nan to match the length of rawExpTimes.
160 finalModelVars : `dict`, [`str`, `np.ndarray`]
161 Dictionary keyed by amp names containing the masked modeled
162 variance of the difference image of each flat pair. If needed, each
163 array will be right-padded with np.nan to match the length of
164 rawExpTimes.
165 finalMeans : `dict`, [`str`, `np.ndarray`]
166 Dictionary keyed by amp names containing the masked average of the
167 means of the exposures in each flat pair. If needed, each array
168 will be right-padded with np.nan to match the length of
169 rawExpTimes.
170 photoCharges : `dict`, [`str`, `np.ndarray`]
171 Dictionary keyed by amp names containing the integrated photocharge
172 for linearity calibration.
173 auxValues : `dict`, [`str`, `np.ndarray`]
174 Dictionary of per-detector auxiliary header values that can be used
175 for PTC, linearity computation.
177 Version 1.1 adds the `ptcTurnoff` attribute.
178 Version 1.2 adds the `histVars`, `histChi2Dofs`, and `kspValues`
179 attributes.
180 Version 1.3 adds the `noiseMatrix` and `noiseMatrixNoB` attributes.
181 Version 1.4 adds the `auxValues` attribute.
182 Version 1.5 adds the `covMatrixSideFullCovFit` attribute.
183 Version 1.6 adds the `rowMeanVariance` attribute.
184 """
186 _OBSTYPE = 'PTC'
187 _SCHEMA = 'Gen3 Photon Transfer Curve'
188 _VERSION = 1.6
190 def __init__(self, ampNames=[], ptcFitType=None, covMatrixSide=1,
191 covMatrixSideFullCovFit=None, **kwargs):
192 self.ptcFitType = ptcFitType
193 self.ampNames = ampNames
194 self.covMatrixSide = covMatrixSide
195 if covMatrixSideFullCovFit is None:
196 self.covMatrixSideFullCovFit = covMatrixSide
197 else:
198 self.covMatrixSideFullCovFit = covMatrixSideFullCovFit
200 self.badAmps = []
202 self.inputExpIdPairs = {ampName: [] for ampName in ampNames}
203 self.expIdMask = {ampName: np.array([], dtype=bool) for ampName in ampNames}
204 self.rawExpTimes = {ampName: np.array([]) for ampName in ampNames}
205 self.rawMeans = {ampName: np.array([]) for ampName in ampNames}
206 self.rawVars = {ampName: np.array([]) for ampName in ampNames}
207 self.rowMeanVariance = {ampName: np.array([]) for ampName in ampNames}
208 self.photoCharges = {ampName: np.array([]) for ampName in ampNames}
210 self.gain = {ampName: np.nan for ampName in ampNames}
211 self.gainErr = {ampName: np.nan for ampName in ampNames}
212 self.noise = {ampName: np.nan for ampName in ampNames}
213 self.noiseErr = {ampName: np.nan for ampName in ampNames}
215 self.histVars = {ampName: np.array([]) for ampName in ampNames}
216 self.histChi2Dofs = {ampName: np.array([]) for ampName in ampNames}
217 self.kspValues = {ampName: np.array([]) for ampName in ampNames}
219 self.ptcFitPars = {ampName: np.array([]) for ampName in ampNames}
220 self.ptcFitParsError = {ampName: np.array([]) for ampName in ampNames}
221 self.ptcFitChiSq = {ampName: np.nan for ampName in ampNames}
222 self.ptcTurnoff = {ampName: np.nan for ampName in ampNames}
224 self.covariances = {ampName: np.array([]) for ampName in ampNames}
225 self.covariancesModel = {ampName: np.array([]) for ampName in ampNames}
226 self.covariancesSqrtWeights = {ampName: np.array([]) for ampName in ampNames}
227 self.aMatrix = {ampName: np.array([]) for ampName in ampNames}
228 self.bMatrix = {ampName: np.array([]) for ampName in ampNames}
229 self.noiseMatrix = {ampName: np.array([]) for ampName in ampNames}
230 self.covariancesModelNoB = {ampName: np.array([]) for ampName in ampNames}
231 self.aMatrixNoB = {ampName: np.array([]) for ampName in ampNames}
232 self.noiseMatrixNoB = {ampName: np.array([]) for ampName in ampNames}
234 self.finalVars = {ampName: np.array([]) for ampName in ampNames}
235 self.finalModelVars = {ampName: np.array([]) for ampName in ampNames}
236 self.finalMeans = {ampName: np.array([]) for ampName in ampNames}
238 # Try this as a dict of arrays.
239 self.auxValues = {}
241 super().__init__(**kwargs)
242 self.requiredAttributes.update(['badAmps', 'inputExpIdPairs', 'expIdMask', 'rawExpTimes',
243 'rawMeans', 'rawVars', 'rowMeanVariance', 'gain',
244 'gainErr', 'noise', 'noiseErr',
245 'ptcFitPars', 'ptcFitParsError', 'ptcFitChiSq', 'ptcTurnoff',
246 'aMatrixNoB', 'covariances', 'covariancesModel',
247 'covariancesSqrtWeights', 'covariancesModelNoB',
248 'aMatrix', 'bMatrix', 'noiseMatrix', 'noiseMatrixNoB', 'finalVars',
249 'finalModelVars', 'finalMeans', 'photoCharges', 'histVars',
250 'histChi2Dofs', 'kspValues', 'auxValues'])
252 self.updateMetadata(setCalibInfo=True, setCalibId=True, **kwargs)
253 self._validateCovarianceMatrizSizes()
255 def setAmpValuesPartialDataset(
256 self,
257 ampName,
258 inputExpIdPair=(-1, -1),
259 rawExpTime=np.nan,
260 rawMean=np.nan,
261 rawVar=np.nan,
262 rowMeanVariance=np.nan,
263 photoCharge=np.nan,
264 expIdMask=False,
265 covariance=None,
266 covSqrtWeights=None,
267 gain=np.nan,
268 noise=np.nan,
269 histVar=np.nan,
270 histChi2Dof=np.nan,
271 kspValue=0.0,
272 auxValues=None,
273 ):
274 """
275 Set the amp values for a partial PTC Dataset (from cpExtractPtcTask).
277 Parameters
278 ----------
279 ampName : `str`
280 Name of the amp to set the values.
281 inputExpIdPair : `tuple` [`int`]
282 Exposure IDs of input pair.
283 rawExpTime : `float`, optional
284 Exposure time for this exposure pair.
285 rawMean : `float`, optional
286 Average of the means of the exposures in this pair.
287 rawVar : `float`, optional
288 Variance of the difference of the exposures in this pair.
289 rowMeanVariance : `float`, optional
290 Variance of the means of the rows in the difference image
291 of the exposures in this pair.
292 photoCharge : `float`, optional
293 Integrated photocharge for flat pair for linearity calibration.
294 expIdMask : `bool`, optional
295 Flag setting if this exposure pair should be used (True)
296 or not used (False).
297 covariance : `np.ndarray` or None, optional
298 Measured covariance for this exposure pair.
299 covSqrtWeights : `np.ndarray` or None, optional
300 Measured sqrt of covariance weights in this exposure pair.
301 gain : `float`, optional
302 Estimated gain for this exposure pair.
303 noise : `float`, optional
304 Estimated read noise for this exposure pair.
305 histVar : `float`, optional
306 Variance estimated from fitting a histogram with a Gaussian model.
307 histChi2Dof : `float`, optional
308 Chi-squared per degree of freedom from Gaussian histogram fit.
309 kspValue : `float`, optional
310 KS test p-value from the Gaussian histogram fit.
311 """
312 nanMatrix = np.full((self.covMatrixSide, self.covMatrixSide), np.nan)
313 nanMatrixFit = np.full((self.covMatrixSideFullCovFit,
314 self.covMatrixSideFullCovFit), np.nan)
315 if covariance is None:
316 covariance = nanMatrix
317 if covSqrtWeights is None:
318 covSqrtWeights = nanMatrix
320 self.inputExpIdPairs[ampName] = [inputExpIdPair]
321 self.rawExpTimes[ampName] = np.array([rawExpTime])
322 self.rawMeans[ampName] = np.array([rawMean])
323 self.rawVars[ampName] = np.array([rawVar])
324 self.rowMeanVariance[ampName] = np.array([rowMeanVariance])
325 self.photoCharges[ampName] = np.array([photoCharge])
326 self.expIdMask[ampName] = np.array([expIdMask])
327 self.covariances[ampName] = np.array([covariance])
328 self.covariancesSqrtWeights[ampName] = np.array([covSqrtWeights])
329 self.gain[ampName] = gain
330 self.noise[ampName] = noise
331 self.histVars[ampName] = np.array([histVar])
332 self.histChi2Dofs[ampName] = np.array([histChi2Dof])
333 self.kspValues[ampName] = np.array([kspValue])
335 # From FULLCOVARIANCE model
336 self.covariancesModel[ampName] = np.array([nanMatrixFit])
337 self.covariancesModelNoB[ampName] = np.array([nanMatrixFit])
338 self.aMatrix[ampName] = nanMatrixFit
339 self.bMatrix[ampName] = nanMatrixFit
340 self.aMatrixNoB[ampName] = nanMatrixFit
341 self.noiseMatrix[ampName] = nanMatrixFit
342 self.noiseMatrixNoB[ampName] = nanMatrixFit
344 def setAuxValuesPartialDataset(self, auxDict):
345 """
346 Set a dictionary of auxiliary values for a partial dataset.
348 Parameters
349 ----------
350 auxDict : `dict` [`str`, `float`]
351 Dictionary of float values.
352 """
353 for key, value in auxDict.items():
354 self.auxValues[key] = np.atleast_1d(np.array(value, dtype=np.float64))
356 def updateMetadata(self, **kwargs):
357 """Update calibration metadata.
358 This calls the base class's method after ensuring the required
359 calibration keywords will be saved.
361 Parameters
362 ----------
363 setDate : `bool`, optional
364 Update the CALIBDATE fields in the metadata to the current
365 time. Defaults to False.
366 kwargs :
367 Other keyword parameters to set in the metadata.
368 """
369 super().updateMetadata(PTC_FIT_TYPE=self.ptcFitType, **kwargs)
371 @classmethod
372 def fromDict(cls, dictionary):
373 """Construct a calibration from a dictionary of properties.
374 Must be implemented by the specific calibration subclasses.
376 Parameters
377 ----------
378 dictionary : `dict`
379 Dictionary of properties.
381 Returns
382 -------
383 calib : `lsst.ip.isr.PhotonTransferCurveDataset`
384 Constructed calibration.
386 Raises
387 ------
388 RuntimeError
389 Raised if the supplied dictionary is for a different
390 calibration.
391 """
392 calib = cls()
393 if calib._OBSTYPE != dictionary['metadata']['OBSTYPE']:
394 raise RuntimeError(f"Incorrect Photon Transfer Curve dataset supplied. "
395 f"Expected {calib._OBSTYPE}, found {dictionary['metadata']['OBSTYPE']}")
396 calib.setMetadata(dictionary['metadata'])
397 calib.ptcFitType = dictionary['ptcFitType']
398 calib.covMatrixSide = dictionary['covMatrixSide']
399 calib.covMatrixSideFullCovFit = dictionary['covMatrixSideFullCovFit']
400 calib.badAmps = np.array(dictionary['badAmps'], 'str').tolist()
401 calib.ampNames = []
403 # The cov matrices are square
404 covMatrixSide = calib.covMatrixSide
405 covMatrixSideFullCovFit = calib.covMatrixSideFullCovFit
406 # Number of final signal levels
407 covDimensionsProduct = len(np.array(list(dictionary['covariances'].values())[0]).ravel())
408 nSignalPoints = int(covDimensionsProduct/(covMatrixSide*covMatrixSide))
410 for ampName in dictionary['ampNames']:
411 calib.ampNames.append(ampName)
412 calib.inputExpIdPairs[ampName] = dictionary['inputExpIdPairs'][ampName]
413 calib.expIdMask[ampName] = np.array(dictionary['expIdMask'][ampName])
414 calib.rawExpTimes[ampName] = np.array(dictionary['rawExpTimes'][ampName], dtype=np.float64)
415 calib.rawMeans[ampName] = np.array(dictionary['rawMeans'][ampName], dtype=np.float64)
416 calib.rawVars[ampName] = np.array(dictionary['rawVars'][ampName], dtype=np.float64)
417 calib.rowMeanVariance[ampName] = np.array(dictionary['rowMeanVariance'][ampName],
418 dtype=np.float64)
419 calib.gain[ampName] = float(dictionary['gain'][ampName])
420 calib.gainErr[ampName] = float(dictionary['gainErr'][ampName])
421 calib.noise[ampName] = float(dictionary['noise'][ampName])
422 calib.noiseErr[ampName] = float(dictionary['noiseErr'][ampName])
423 calib.histVars[ampName] = np.array(dictionary['histVars'][ampName], dtype=np.float64)
424 calib.histChi2Dofs[ampName] = np.array(dictionary['histChi2Dofs'][ampName], dtype=np.float64)
425 calib.kspValues[ampName] = np.array(dictionary['kspValues'][ampName], dtype=np.float64)
426 calib.ptcFitPars[ampName] = np.array(dictionary['ptcFitPars'][ampName], dtype=np.float64)
427 calib.ptcFitParsError[ampName] = np.array(dictionary['ptcFitParsError'][ampName],
428 dtype=np.float64)
429 calib.ptcFitChiSq[ampName] = float(dictionary['ptcFitChiSq'][ampName])
430 calib.ptcTurnoff[ampName] = float(dictionary['ptcTurnoff'][ampName])
431 if nSignalPoints > 0:
432 # Regular dataset
433 calib.covariances[ampName] = np.array(dictionary['covariances'][ampName],
434 dtype=np.float64).reshape(
435 (nSignalPoints, covMatrixSide, covMatrixSide))
436 calib.covariancesModel[ampName] = np.array(
437 dictionary['covariancesModel'][ampName],
438 dtype=np.float64).reshape(
439 (nSignalPoints, covMatrixSideFullCovFit, covMatrixSideFullCovFit))
440 calib.covariancesSqrtWeights[ampName] = np.array(
441 dictionary['covariancesSqrtWeights'][ampName],
442 dtype=np.float64).reshape(
443 (nSignalPoints, covMatrixSide, covMatrixSide))
444 calib.aMatrix[ampName] = np.array(dictionary['aMatrix'][ampName],
445 dtype=np.float64).reshape(
446 (covMatrixSideFullCovFit, covMatrixSideFullCovFit))
447 calib.bMatrix[ampName] = np.array(dictionary['bMatrix'][ampName],
448 dtype=np.float64).reshape(
449 (covMatrixSideFullCovFit, covMatrixSideFullCovFit))
450 calib.covariancesModelNoB[ampName] = np.array(
451 dictionary['covariancesModelNoB'][ampName], dtype=np.float64).reshape(
452 (nSignalPoints, covMatrixSideFullCovFit, covMatrixSideFullCovFit))
453 calib.aMatrixNoB[ampName] = np.array(
454 dictionary['aMatrixNoB'][ampName],
455 dtype=np.float64).reshape((covMatrixSideFullCovFit, covMatrixSideFullCovFit))
456 calib.noiseMatrix[ampName] = np.array(
457 dictionary['noiseMatrix'][ampName],
458 dtype=np.float64).reshape((covMatrixSideFullCovFit, covMatrixSideFullCovFit))
459 calib.noiseMatrixNoB[ampName] = np.array(
460 dictionary['noiseMatrixNoB'][ampName],
461 dtype=np.float64).reshape((covMatrixSideFullCovFit, covMatrixSideFullCovFit))
462 else:
463 # Empty dataset
464 calib.covariances[ampName] = np.array([], dtype=np.float64)
465 calib.covariancesModel[ampName] = np.array([], dtype=np.float64)
466 calib.covariancesSqrtWeights[ampName] = np.array([], dtype=np.float64)
467 calib.aMatrix[ampName] = np.array([], dtype=np.float64)
468 calib.bMatrix[ampName] = np.array([], dtype=np.float64)
469 calib.covariancesModelNoB[ampName] = np.array([], dtype=np.float64)
470 calib.aMatrixNoB[ampName] = np.array([], dtype=np.float64)
471 calib.noiseMatrix[ampName] = np.array([], dtype=np.float64)
472 calib.noiseMatrixNoB[ampName] = np.array([], dtype=np.float64)
474 calib.finalVars[ampName] = np.array(dictionary['finalVars'][ampName], dtype=np.float64)
475 calib.finalModelVars[ampName] = np.array(dictionary['finalModelVars'][ampName], dtype=np.float64)
476 calib.finalMeans[ampName] = np.array(dictionary['finalMeans'][ampName], dtype=np.float64)
477 calib.photoCharges[ampName] = np.array(dictionary['photoCharges'][ampName], dtype=np.float64)
479 for key, value in dictionary['auxValues'].items():
480 calib.auxValues[key] = np.atleast_1d(np.array(value, dtype=np.float64))
482 calib.updateMetadata()
483 return calib
485 def toDict(self):
486 """Return a dictionary containing the calibration properties.
487 The dictionary should be able to be round-tripped through
488 `fromDict`.
490 Returns
491 -------
492 dictionary : `dict`
493 Dictionary of properties.
494 """
495 self.updateMetadata()
497 outDict = dict()
498 metadata = self.getMetadata()
499 outDict['metadata'] = metadata
501 def _dictOfArraysToDictOfLists(dictOfArrays):
502 dictOfLists = {}
503 for key, value in dictOfArrays.items():
504 dictOfLists[key] = value.ravel().tolist()
506 return dictOfLists
508 outDict['ptcFitType'] = self.ptcFitType
509 outDict['covMatrixSide'] = self.covMatrixSide
510 outDict['covMatrixSideFullCovFit'] = self.covMatrixSideFullCovFit
511 outDict['ampNames'] = self.ampNames
512 outDict['badAmps'] = self.badAmps
513 outDict['inputExpIdPairs'] = self.inputExpIdPairs
514 outDict['expIdMask'] = _dictOfArraysToDictOfLists(self.expIdMask)
515 outDict['rawExpTimes'] = _dictOfArraysToDictOfLists(self.rawExpTimes)
516 outDict['rawMeans'] = _dictOfArraysToDictOfLists(self.rawMeans)
517 outDict['rawVars'] = _dictOfArraysToDictOfLists(self.rawVars)
518 outDict['rowMeanVariance'] = _dictOfArraysToDictOfLists(self.rowMeanVariance)
519 outDict['gain'] = self.gain
520 outDict['gainErr'] = self.gainErr
521 outDict['noise'] = self.noise
522 outDict['noiseErr'] = self.noiseErr
523 outDict['histVars'] = self.histVars
524 outDict['histChi2Dofs'] = self.histChi2Dofs
525 outDict['kspValues'] = self.kspValues
526 outDict['ptcFitPars'] = _dictOfArraysToDictOfLists(self.ptcFitPars)
527 outDict['ptcFitParsError'] = _dictOfArraysToDictOfLists(self.ptcFitParsError)
528 outDict['ptcFitChiSq'] = self.ptcFitChiSq
529 outDict['ptcTurnoff'] = self.ptcTurnoff
530 outDict['covariances'] = _dictOfArraysToDictOfLists(self.covariances)
531 outDict['covariancesModel'] = _dictOfArraysToDictOfLists(self.covariancesModel)
532 outDict['covariancesSqrtWeights'] = _dictOfArraysToDictOfLists(self.covariancesSqrtWeights)
533 outDict['aMatrix'] = _dictOfArraysToDictOfLists(self.aMatrix)
534 outDict['bMatrix'] = _dictOfArraysToDictOfLists(self.bMatrix)
535 outDict['noiseMatrix'] = _dictOfArraysToDictOfLists(self.noiseMatrix)
536 outDict['covariancesModelNoB'] = _dictOfArraysToDictOfLists(self.covariancesModelNoB)
537 outDict['aMatrixNoB'] = _dictOfArraysToDictOfLists(self.aMatrixNoB)
538 outDict['noiseMatrixNoB'] = _dictOfArraysToDictOfLists(self.noiseMatrixNoB)
539 outDict['finalVars'] = _dictOfArraysToDictOfLists(self.finalVars)
540 outDict['finalModelVars'] = _dictOfArraysToDictOfLists(self.finalModelVars)
541 outDict['finalMeans'] = _dictOfArraysToDictOfLists(self.finalMeans)
542 outDict['photoCharges'] = _dictOfArraysToDictOfLists(self.photoCharges)
543 outDict['auxValues'] = _dictOfArraysToDictOfLists(self.auxValues)
545 return outDict
547 @classmethod
548 def fromTable(cls, tableList):
549 """Construct calibration from a list of tables.
550 This method uses the `fromDict` method to create the
551 calibration, after constructing an appropriate dictionary from
552 the input tables.
554 Parameters
555 ----------
556 tableList : `list` [`lsst.afw.table.Table`]
557 List of tables to use to construct the datasetPtc.
559 Returns
560 -------
561 calib : `lsst.ip.isr.PhotonTransferCurveDataset`
562 The calibration defined in the tables.
563 """
564 ptcTable = tableList[0]
566 metadata = ptcTable.meta
567 inDict = dict()
568 inDict['metadata'] = metadata
569 inDict['ampNames'] = []
570 inDict['ptcFitType'] = []
571 inDict['covMatrixSide'] = []
572 inDict['covMatrixSideFullCovFit'] = []
573 inDict['inputExpIdPairs'] = dict()
574 inDict['expIdMask'] = dict()
575 inDict['rawExpTimes'] = dict()
576 inDict['rawMeans'] = dict()
577 inDict['rawVars'] = dict()
578 inDict['rowMeanVariance'] = dict()
579 inDict['gain'] = dict()
580 inDict['gainErr'] = dict()
581 inDict['noise'] = dict()
582 inDict['noiseErr'] = dict()
583 inDict['histVars'] = dict()
584 inDict['histChi2Dofs'] = dict()
585 inDict['kspValues'] = dict()
586 inDict['ptcFitPars'] = dict()
587 inDict['ptcFitParsError'] = dict()
588 inDict['ptcFitChiSq'] = dict()
589 inDict['ptcTurnoff'] = dict()
590 inDict['covariances'] = dict()
591 inDict['covariancesModel'] = dict()
592 inDict['covariancesSqrtWeights'] = dict()
593 inDict['aMatrix'] = dict()
594 inDict['bMatrix'] = dict()
595 inDict['noiseMatrix'] = dict()
596 inDict['covariancesModelNoB'] = dict()
597 inDict['aMatrixNoB'] = dict()
598 inDict['noiseMatrixNoB'] = dict()
599 inDict['finalVars'] = dict()
600 inDict['finalModelVars'] = dict()
601 inDict['finalMeans'] = dict()
602 inDict['badAmps'] = []
603 inDict['photoCharges'] = dict()
605 calibVersion = metadata['PTC_VERSION']
606 if calibVersion == 1.0:
607 cls().log.warning(f"Previous version found for PTC dataset: {calibVersion}. "
608 f"Setting 'ptcTurnoff' in all amps to last value in 'finalMeans'.")
609 for record in ptcTable:
610 ampName = record['AMPLIFIER_NAME']
612 inDict['ptcFitType'] = record['PTC_FIT_TYPE']
613 inDict['covMatrixSide'] = record['COV_MATRIX_SIDE']
614 inDict['ampNames'].append(ampName)
615 inDict['inputExpIdPairs'][ampName] = record['INPUT_EXP_ID_PAIRS'].tolist()
616 inDict['expIdMask'][ampName] = record['EXP_ID_MASK']
617 inDict['rawExpTimes'][ampName] = record['RAW_EXP_TIMES']
618 inDict['rawMeans'][ampName] = record['RAW_MEANS']
619 inDict['rawVars'][ampName] = record['RAW_VARS']
620 inDict['rowMeanVariance'][ampName] = record['ROW_MEAN_VARIANCE']
621 inDict['gain'][ampName] = record['GAIN']
622 inDict['gainErr'][ampName] = record['GAIN_ERR']
623 inDict['noise'][ampName] = record['NOISE']
624 inDict['noiseErr'][ampName] = record['NOISE_ERR']
625 inDict['ptcFitPars'][ampName] = record['PTC_FIT_PARS']
626 inDict['ptcFitParsError'][ampName] = record['PTC_FIT_PARS_ERROR']
627 inDict['ptcFitChiSq'][ampName] = record['PTC_FIT_CHI_SQ']
628 inDict['covariances'][ampName] = record['COVARIANCES']
629 inDict['covariancesModel'][ampName] = record['COVARIANCES_MODEL']
630 inDict['covariancesSqrtWeights'][ampName] = record['COVARIANCES_SQRT_WEIGHTS']
631 inDict['aMatrix'][ampName] = record['A_MATRIX']
632 inDict['bMatrix'][ampName] = record['B_MATRIX']
633 inDict['covariancesModelNoB'][ampName] = record['COVARIANCES_MODEL_NO_B']
634 inDict['aMatrixNoB'][ampName] = record['A_MATRIX_NO_B']
635 inDict['finalVars'][ampName] = record['FINAL_VARS']
636 inDict['finalModelVars'][ampName] = record['FINAL_MODEL_VARS']
637 inDict['finalMeans'][ampName] = record['FINAL_MEANS']
638 inDict['badAmps'] = record['BAD_AMPS'].tolist()
639 inDict['photoCharges'][ampName] = record['PHOTO_CHARGE']
640 if calibVersion == 1.0:
641 mask = record['FINAL_MEANS'].mask
642 array = record['FINAL_MEANS'][~mask]
643 if len(array) > 0:
644 inDict['ptcTurnoff'][ampName] = record['FINAL_MEANS'][~mask][-1]
645 else:
646 inDict['ptcTurnoff'][ampName] = np.nan
647 else:
648 inDict['ptcTurnoff'][ampName] = record['PTC_TURNOFF']
649 if calibVersion < 1.2:
650 inDict['histVars'][ampName] = np.array([np.nan])
651 inDict['histChi2Dofs'][ampName] = np.array([np.nan])
652 inDict['kspValues'][ampName] = np.array([0.0])
653 else:
654 inDict['histVars'][ampName] = record['HIST_VARS']
655 inDict['histChi2Dofs'][ampName] = record['HIST_CHI2_DOFS']
656 inDict['kspValues'][ampName] = record['KS_PVALUES']
657 if calibVersion < 1.3:
658 nanMatrix = np.full_like(inDict['aMatrix'][ampName], np.nan)
659 inDict['noiseMatrix'][ampName] = nanMatrix
660 inDict['noiseMatrixNoB'][ampName] = nanMatrix
661 else:
662 inDict['noiseMatrix'][ampName] = record['NOISE_MATRIX']
663 inDict['noiseMatrixNoB'][ampName] = record['NOISE_MATRIX_NO_B']
664 if calibVersion < 1.5:
665 # Matched to `COV_MATRIX_SIDE`. Same for all amps.
666 inDict['covMatrixSideFullCovFit'] = inDict['covMatrixSide']
667 else:
668 inDict['covMatrixSideFullCovFit'] = record['COV_MATRIX_SIDE_FULL_COV_FIT']
669 if calibVersion < 1.6:
670 inDict['rowMeanVariance'][ampName] = np.array([np.nan])
671 else:
672 inDict['rowMeanVariance'][ampName] = record['ROW_MEAN_VARIANCE']
674 inDict['auxValues'] = {}
675 record = ptcTable[0]
676 for col in record.columns.keys():
677 if col.startswith('PTCAUX_'):
678 parts = col.split('PTCAUX_')
679 inDict['auxValues'][parts[1]] = record[col]
681 return cls().fromDict(inDict)
683 def toTable(self):
684 """Construct a list of tables containing the information in this
685 calibration.
687 The list of tables should create an identical calibration
688 after being passed to this class's fromTable method.
690 Returns
691 -------
692 tableList : `list` [`astropy.table.Table`]
693 List of tables containing the linearity calibration
694 information.
695 """
696 tableList = []
697 self.updateMetadata()
699 badAmps = np.array(self.badAmps) if len(self.badAmps) else np.array([], dtype="U3")
701 catalogList = []
702 for ampName in self.ampNames:
703 ampDict = {
704 'AMPLIFIER_NAME': ampName,
705 'PTC_FIT_TYPE': self.ptcFitType,
706 'COV_MATRIX_SIDE': self.covMatrixSide,
707 'COV_MATRIX_SIDE_FULL_COV_FIT': self.covMatrixSideFullCovFit,
708 'INPUT_EXP_ID_PAIRS': self.inputExpIdPairs[ampName],
709 'EXP_ID_MASK': self.expIdMask[ampName],
710 'RAW_EXP_TIMES': self.rawExpTimes[ampName],
711 'RAW_MEANS': self.rawMeans[ampName],
712 'RAW_VARS': self.rawVars[ampName],
713 'ROW_MEAN_VARIANCE': self.rowMeanVariance[ampName],
714 'GAIN': self.gain[ampName],
715 'GAIN_ERR': self.gainErr[ampName],
716 'NOISE': self.noise[ampName],
717 'NOISE_ERR': self.noiseErr[ampName],
718 'HIST_VARS': self.histVars[ampName],
719 'HIST_CHI2_DOFS': self.histChi2Dofs[ampName],
720 'KS_PVALUES': self.kspValues[ampName],
721 'PTC_FIT_PARS': np.array(self.ptcFitPars[ampName]),
722 'PTC_FIT_PARS_ERROR': np.array(self.ptcFitParsError[ampName]),
723 'PTC_FIT_CHI_SQ': self.ptcFitChiSq[ampName],
724 'PTC_TURNOFF': self.ptcTurnoff[ampName],
725 'A_MATRIX': self.aMatrix[ampName].ravel(),
726 'B_MATRIX': self.bMatrix[ampName].ravel(),
727 'A_MATRIX_NO_B': self.aMatrixNoB[ampName].ravel(),
728 'NOISE_MATRIX': self.noiseMatrix[ampName].ravel(),
729 'NOISE_MATRIX_NO_B': self.noiseMatrixNoB[ampName].ravel(),
730 'BAD_AMPS': badAmps,
731 'PHOTO_CHARGE': self.photoCharges[ampName],
732 'COVARIANCES': self.covariances[ampName].ravel(),
733 'COVARIANCES_MODEL': self.covariancesModel[ampName].ravel(),
734 'COVARIANCES_SQRT_WEIGHTS': self.covariancesSqrtWeights[ampName].ravel(),
735 'COVARIANCES_MODEL_NO_B': self.covariancesModelNoB[ampName].ravel(),
736 'FINAL_VARS': self.finalVars[ampName],
737 'FINAL_MODEL_VARS': self.finalModelVars[ampName],
738 'FINAL_MEANS': self.finalMeans[ampName],
739 }
741 if self.auxValues:
742 for key, value in self.auxValues.items():
743 ampDict[f"PTCAUX_{key}"] = value
745 catalogList.append(ampDict)
747 catalog = Table(catalogList)
749 inMeta = self.getMetadata().toDict()
750 outMeta = {k: v for k, v in inMeta.items() if v is not None}
751 outMeta.update({k: "" for k, v in inMeta.items() if v is None})
752 catalog.meta = outMeta
753 tableList.append(catalog)
755 return tableList
757 def fromDetector(self, detector):
758 """Read metadata parameters from a detector.
760 Parameters
761 ----------
762 detector : `lsst.afw.cameraGeom.detector`
763 Input detector with parameters to use.
765 Returns
766 -------
767 calib : `lsst.ip.isr.PhotonTransferCurveDataset`
768 The calibration constructed from the detector.
769 """
771 pass
773 def getExpIdsUsed(self, ampName):
774 """Get the exposures used, i.e. not discarded, for a given amp.
775 If no mask has been created yet, all exposures are returned.
777 Parameters
778 ----------
779 ampName : `str`
781 Returns
782 -------
783 expIdsUsed : `list` [`tuple`]
784 List of pairs of exposure ids used in PTC.
785 """
786 if len(self.expIdMask[ampName]) == 0:
787 return self.inputExpIdPairs[ampName]
789 # if the mask exists it had better be the same length as the expIdPairs
790 assert len(self.expIdMask[ampName]) == len(self.inputExpIdPairs[ampName])
792 pairs = self.inputExpIdPairs[ampName]
793 mask = self.expIdMask[ampName]
794 # cast to bool required because numpy
795 try:
796 expIdsUsed = [(exp1, exp2) for ((exp1, exp2), m) in zip(pairs, mask) if m]
797 except ValueError:
798 self.log.warning("The PTC file was written incorrectly; you should rerun the "
799 "PTC solve task if possible.")
800 expIdsUsed = []
801 for pairList, m in zip(pairs, mask):
802 if m:
803 expIdsUsed.append(pairList[0])
805 return expIdsUsed
807 def getGoodAmps(self):
808 """Get the good amps from this PTC."""
809 return [amp for amp in self.ampNames if amp not in self.badAmps]
811 def getGoodPoints(self, ampName):
812 """Get the good points used for a given amp in the PTC.
814 Parameters
815 ----------
816 ampName : `str`
817 Amplifier's name.
819 Returns
820 -------
821 goodPoints : `np.ndarray`
822 Boolean array of good points used in PTC.
823 """
824 return self.expIdMask[ampName]
826 def validateGainNoiseTurnoffValues(self, ampName, doWarn=False):
827 """Ensure the gain, read noise, and PTC turnoff have
828 sensible values.
830 Parameters
831 ----------
832 ampName : `str`
833 Amplifier's name.
834 """
836 gain = self.gain[ampName]
837 noise = self.noise[ampName]
838 ptcTurnoff = self.ptcTurnoff[ampName]
840 # Check if gain is not positive or is np.nan
841 if not (isinstance(gain, (int, float)) and gain > 0) or math.isnan(gain):
842 if doWarn:
843 self.log.warning(f"Invalid gain value {gain}"
844 " Setting to default: Gain=1")
845 gain = 1
847 # Check if noise is not positive or is np.nan
848 if not (isinstance(noise, (int, float)) and noise > 0) or math.isnan(noise):
849 if doWarn:
850 self.log.warning(f"Invalid noise value: {noise}"
851 " Setting to default: Noise=1")
852 noise = 1
854 # Check if ptcTurnoff is not positive or is np.nan
855 if not (isinstance(ptcTurnoff, (int, float)) and ptcTurnoff > 0) or math.isnan(ptcTurnoff):
856 if doWarn:
857 self.log.warning(f"Invalid PTC turnoff value: {ptcTurnoff}"
858 " Setting to default: PTC Turnoff=2e19")
859 ptcTurnoff = 2e19
861 self.gain[ampName] = gain
862 self.noise[ampName] = noise
863 self.ptcTurnoff[ampName] = ptcTurnoff
865 def _validateCovarianceMatrizSizes(self):
866 """Ensure covMatrixSideFullCovFit <= covMatrixSide."""
867 if self.covMatrixSideFullCovFit > self.covMatrixSide:
868 self.log.warning("covMatrixSideFullCovFit > covMatrixSide "
869 f"({self.covMatrixSideFullCovFit} > {self.covMatrixSide})."
870 "Setting the former to the latter.")
871 self.covMatrixSideFullCovFit = self.covMatrixSide