Coverage for python/lsst/ip/isr/ptcDataset.py: 6%
354 statements
« prev ^ index » next coverage.py v7.4.0, created at 2024-01-18 12:25 +0000
« prev ^ index » next coverage.py v7.4.0, created at 2024-01-18 12:25 +0000
1#
2# LSST Data Management System
3# Copyright 2008-2017 AURA/LSST.
4#
5# This product includes software developed by the
6# LSST Project (http://www.lsst.org/).
7#
8# This program is free software: you can redistribute it and/or modify
9# it under the terms of the GNU General Public License as published by
10# the Free Software Foundation, either version 3 of the License, or
11# (at your option) any later version.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the LSST License Statement and
19# the GNU General Public License along with this program. If not,
20# see <https://www.lsstcorp.org/LegalNotices/>.
21#
22"""
23Define dataset class for MeasurePhotonTransferCurve task
24"""
26__all__ = ['PhotonTransferCurveDataset']
28import numpy as np
29import math
30from astropy.table import Table
32from lsst.ip.isr import IsrCalib
35class PhotonTransferCurveDataset(IsrCalib):
36 """A simple class to hold the output data from the PTC task.
38 The dataset is made up of a dictionary for each item, keyed by the
39 amplifiers' names, which much be supplied at construction time.
40 New items cannot be added to the class to save accidentally saving to the
41 wrong property, and the class can be frozen if desired.
42 inputExpIdPairs records the exposures used to produce the data.
43 When fitPtc() or fitCovariancesAstier() is run, a mask is built up, which
44 is by definition always the same length as inputExpIdPairs, rawExpTimes,
45 rawMeans and rawVars, and is a list of bools, which are incrementally set
46 to False as points are discarded from the fits.
47 PTC fit parameters for polynomials are stored in a list in ascending order
48 of polynomial term, i.e. par[0]*x^0 + par[1]*x + par[2]*x^2 etc
49 with the length of the list corresponding to the order of the polynomial
50 plus one.
52 Parameters
53 ----------
54 ampNames : `list`
55 List with the names of the amplifiers of the detector at hand.
56 ptcFitType : `str`, optional
57 Type of model fitted to the PTC: "POLYNOMIAL", "EXPAPPROXIMATION",
58 or "FULLCOVARIANCE".
59 covMatrixSide : `int`, optional
60 Maximum lag of measured covariances (size of square covariance
61 matrices).
62 covMatrixSideFullCovFit : `int, optional
63 Maximum covariances lag for FULLCOVARIANCE fit. It should be less or
64 equal than covMatrixSide.
65 kwargs : `dict`, optional
66 Other keyword arguments to pass to the parent init.
68 Notes
69 -----
70 The stored attributes are:
72 badAmps : `list` [`str`]
73 List with bad amplifiers names.
74 inputExpIdPairs : `dict`, [`str`, `list`]
75 Dictionary keyed by amp names containing the input exposures IDs.
76 expIdMask : `dict`, [`str`, `np.ndarray`]
77 Dictionary keyed by amp names containing the mask produced after
78 outlier rejection. The mask produced by the "FULLCOVARIANCE"
79 option may differ from the one produced in the other two PTC
80 fit types.
81 rawExpTimes : `dict`, [`str`, `np.ndarray`]
82 Dictionary keyed by amp names containing the unmasked exposure times.
83 rawMeans : `dict`, [`str`, `np.ndarray`]
84 Dictionary keyed by amp names containing the unmasked average of the
85 means of the exposures in each flat pair.
86 rawVars : `dict`, [`str`, `np.ndarray`]
87 Dictionary keyed by amp names containing the variance of the
88 difference image of the exposures in each flat pair.
89 histVars : `dict`, [`str`, `np.ndarray`]
90 Dictionary keyed by amp names containing the variance of the
91 difference image of the exposures in each flat pair estimated
92 by fitting a Gaussian model.
93 histChi2Dofs : `dict`, [`str`, `np.ndarray`]
94 Dictionary keyed by amp names containing the chi-squared per degree
95 of freedom fitting the difference image to a Gaussian model.
96 kspValues : `dict`, [`str`, `np.ndarray`]
97 Dictionary keyed by amp names containing the KS test p-value from
98 fitting the difference image to a Gaussian model.
99 gain : `dict`, [`str`, `float`]
100 Dictionary keyed by amp names containing the fitted gains.
101 gainErr : `dict`, [`str`, `float`]
102 Dictionary keyed by amp names containing the errors on the
103 fitted gains.
104 noise : `dict`, [`str`, `float`]
105 Dictionary keyed by amp names containing the fitted noise.
106 noiseErr : `dict`, [`str`, `float`]
107 Dictionary keyed by amp names containing the errors on the fitted
108 noise.
109 ptcFitPars : `dict`, [`str`, `np.ndarray`]
110 Dictionary keyed by amp names containing the fitted parameters of the
111 PTC model for ptcFitTye in ["POLYNOMIAL", "EXPAPPROXIMATION"].
112 ptcFitParsError : `dict`, [`str`, `np.ndarray`]
113 Dictionary keyed by amp names containing the errors on the fitted
114 parameters of the PTC model for ptcFitTye in
115 ["POLYNOMIAL", "EXPAPPROXIMATION"].
116 ptcFitChiSq : `dict`, [`str`, `float`]
117 Dictionary keyed by amp names containing the reduced chi squared
118 of the fit for ptcFitTye in ["POLYNOMIAL", "EXPAPPROXIMATION"].
119 ptcTurnoff : `dict` [`str, `float`]
120 Flux value (in ADU) where the variance of the PTC curve starts
121 decreasing consistently.
122 covariances : `dict`, [`str`, `np.ndarray`]
123 Dictionary keyed by amp names containing a list of measured
124 covariances per mean flux.
125 covariancesModel : `dict`, [`str`, `np.ndarray`]
126 Dictionary keyed by amp names containinging covariances model
127 (Eq. 20 of Astier+19) per mean flux.
128 covariancesSqrtWeights : `dict`, [`str`, `np.ndarray`]
129 Dictionary keyed by amp names containinging sqrt. of covariances
130 weights.
131 aMatrix : `dict`, [`str`, `np.ndarray`]
132 Dictionary keyed by amp names containing the "a" parameters from
133 the model in Eq. 20 of Astier+19.
134 bMatrix : `dict`, [`str`, `np.ndarray`]
135 Dictionary keyed by amp names containing the "b" parameters from
136 the model in Eq. 20 of Astier+19.
137 noiseMatrix : `dict`, [`str`, `np.ndarray`]
138 Dictionary keyed by amp names containing the "noise" parameters from
139 the model in Eq. 20 of Astier+19.
140 covariancesModelNoB : `dict`, [`str`, `np.ndarray`]
141 Dictionary keyed by amp names containing covariances model
142 (with 'b'=0 in Eq. 20 of Astier+19)
143 per mean flux.
144 aMatrixNoB : `dict`, [`str`, `np.ndarray`]
145 Dictionary keyed by amp names containing the "a" parameters from the
146 model in Eq. 20 of Astier+19
147 (and 'b' = 0).
148 noiseMatrixNoB : `dict`, [`str`, `np.ndarray`]
149 Dictionary keyed by amp names containing the "noise" parameters from
150 the model in Eq. 20 of Astier+19, with 'b' = 0.
151 finalVars : `dict`, [`str`, `np.ndarray`]
152 Dictionary keyed by amp names containing the masked variance of the
153 difference image of each flat
154 pair. If needed, each array will be right-padded with
155 np.nan to match the length of rawExpTimes.
156 finalModelVars : `dict`, [`str`, `np.ndarray`]
157 Dictionary keyed by amp names containing the masked modeled
158 variance of the difference image of each flat pair. If needed, each
159 array will be right-padded with np.nan to match the length of
160 rawExpTimes.
161 finalMeans : `dict`, [`str`, `np.ndarray`]
162 Dictionary keyed by amp names containing the masked average of the
163 means of the exposures in each flat pair. If needed, each array
164 will be right-padded with np.nan to match the length of
165 rawExpTimes.
166 photoCharges : `dict`, [`str`, `np.ndarray`]
167 Dictionary keyed by amp names containing the integrated photocharge
168 for linearity calibration.
169 auxValues : `dict`, [`str`, `np.ndarray`]
170 Dictionary of per-detector auxiliary header values that can be used
171 for PTC, linearity computation.
173 Version 1.1 adds the `ptcTurnoff` attribute.
174 Version 1.2 adds the `histVars`, `histChi2Dofs`, and `kspValues`
175 attributes.
176 Version 1.3 adds the `noiseMatrix` and `noiseMatrixNoB` attributes.
177 Version 1.4 adds the `auxValues` attribute.
178 Version 1.5 adds the `covMatrixSideFullCovFit` attribute.
179 """
181 _OBSTYPE = 'PTC'
182 _SCHEMA = 'Gen3 Photon Transfer Curve'
183 _VERSION = 1.5
185 def __init__(self, ampNames=[], ptcFitType=None, covMatrixSide=1,
186 covMatrixSideFullCovFit=None, **kwargs):
187 self.ptcFitType = ptcFitType
188 self.ampNames = ampNames
189 self.covMatrixSide = covMatrixSide
190 if covMatrixSideFullCovFit is None:
191 self.covMatrixSideFullCovFit = covMatrixSide
192 else:
193 self.covMatrixSideFullCovFit = covMatrixSideFullCovFit
195 self.badAmps = []
197 self.inputExpIdPairs = {ampName: [] for ampName in ampNames}
198 self.expIdMask = {ampName: np.array([], dtype=bool) for ampName in ampNames}
199 self.rawExpTimes = {ampName: np.array([]) for ampName in ampNames}
200 self.rawMeans = {ampName: np.array([]) for ampName in ampNames}
201 self.rawVars = {ampName: np.array([]) for ampName in ampNames}
202 self.photoCharges = {ampName: np.array([]) for ampName in ampNames}
204 self.gain = {ampName: np.nan for ampName in ampNames}
205 self.gainErr = {ampName: np.nan for ampName in ampNames}
206 self.noise = {ampName: np.nan for ampName in ampNames}
207 self.noiseErr = {ampName: np.nan for ampName in ampNames}
209 self.histVars = {ampName: np.array([]) for ampName in ampNames}
210 self.histChi2Dofs = {ampName: np.array([]) for ampName in ampNames}
211 self.kspValues = {ampName: np.array([]) for ampName in ampNames}
213 self.ptcFitPars = {ampName: np.array([]) for ampName in ampNames}
214 self.ptcFitParsError = {ampName: np.array([]) for ampName in ampNames}
215 self.ptcFitChiSq = {ampName: np.nan for ampName in ampNames}
216 self.ptcTurnoff = {ampName: np.nan for ampName in ampNames}
218 self.covariances = {ampName: np.array([]) for ampName in ampNames}
219 self.covariancesModel = {ampName: np.array([]) for ampName in ampNames}
220 self.covariancesSqrtWeights = {ampName: np.array([]) for ampName in ampNames}
221 self.aMatrix = {ampName: np.array([]) for ampName in ampNames}
222 self.bMatrix = {ampName: np.array([]) for ampName in ampNames}
223 self.noiseMatrix = {ampName: np.array([]) for ampName in ampNames}
224 self.covariancesModelNoB = {ampName: np.array([]) for ampName in ampNames}
225 self.aMatrixNoB = {ampName: np.array([]) for ampName in ampNames}
226 self.noiseMatrixNoB = {ampName: np.array([]) for ampName in ampNames}
228 self.finalVars = {ampName: np.array([]) for ampName in ampNames}
229 self.finalModelVars = {ampName: np.array([]) for ampName in ampNames}
230 self.finalMeans = {ampName: np.array([]) for ampName in ampNames}
232 # Try this as a dict of arrays.
233 self.auxValues = {}
235 super().__init__(**kwargs)
236 self.requiredAttributes.update(['badAmps', 'inputExpIdPairs', 'expIdMask', 'rawExpTimes',
237 'rawMeans', 'rawVars', 'gain', 'gainErr', 'noise', 'noiseErr',
238 'ptcFitPars', 'ptcFitParsError', 'ptcFitChiSq', 'ptcTurnoff',
239 'aMatrixNoB', 'covariances', 'covariancesModel',
240 'covariancesSqrtWeights', 'covariancesModelNoB',
241 'aMatrix', 'bMatrix', 'noiseMatrix', 'noiseMatrixNoB', 'finalVars',
242 'finalModelVars', 'finalMeans', 'photoCharges', 'histVars',
243 'histChi2Dofs', 'kspValues', 'auxValues'])
245 self.updateMetadata(setCalibInfo=True, setCalibId=True, **kwargs)
246 self._validateCovarianceMatrizSizes()
248 def setAmpValuesPartialDataset(
249 self,
250 ampName,
251 inputExpIdPair=(-1, -1),
252 rawExpTime=np.nan,
253 rawMean=np.nan,
254 rawVar=np.nan,
255 photoCharge=np.nan,
256 expIdMask=False,
257 covariance=None,
258 covSqrtWeights=None,
259 gain=np.nan,
260 noise=np.nan,
261 histVar=np.nan,
262 histChi2Dof=np.nan,
263 kspValue=0.0,
264 auxValues=None,
265 ):
266 """
267 Set the amp values for a partial PTC Dataset (from cpExtractPtcTask).
269 Parameters
270 ----------
271 ampName : `str`
272 Name of the amp to set the values.
273 inputExpIdPair : `tuple` [`int`]
274 Exposure IDs of input pair.
275 rawExpTime : `float`, optional
276 Exposure time for this exposure pair.
277 rawMean : `float`, optional
278 Average of the means of the exposures in this pair.
279 rawVar : `float`, optional
280 Variance of the difference of the exposures in this pair.
281 photoCharge : `float`, optional
282 Integrated photocharge for flat pair for linearity calibration.
283 expIdMask : `bool`, optional
284 Flag setting if this exposure pair should be used (True)
285 or not used (False).
286 covariance : `np.ndarray` or None, optional
287 Measured covariance for this exposure pair.
288 covSqrtWeights : `np.ndarray` or None, optional
289 Measured sqrt of covariance weights in this exposure pair.
290 gain : `float`, optional
291 Estimated gain for this exposure pair.
292 noise : `float`, optional
293 Estimated read noise for this exposure pair.
294 histVar : `float`, optional
295 Variance estimated from fitting a histogram with a Gaussian model.
296 histChi2Dof : `float`, optional
297 Chi-squared per degree of freedom from Gaussian histogram fit.
298 kspValue : `float`, optional
299 KS test p-value from the Gaussian histogram fit.
300 """
301 nanMatrix = np.full((self.covMatrixSide, self.covMatrixSide), np.nan)
302 nanMatrixFit = np.full((self.covMatrixSideFullCovFit,
303 self.covMatrixSideFullCovFit), np.nan)
304 if covariance is None:
305 covariance = nanMatrix
306 if covSqrtWeights is None:
307 covSqrtWeights = nanMatrix
309 self.inputExpIdPairs[ampName] = [inputExpIdPair]
310 self.rawExpTimes[ampName] = np.array([rawExpTime])
311 self.rawMeans[ampName] = np.array([rawMean])
312 self.rawVars[ampName] = np.array([rawVar])
313 self.photoCharges[ampName] = np.array([photoCharge])
314 self.expIdMask[ampName] = np.array([expIdMask])
315 self.covariances[ampName] = np.array([covariance])
316 self.covariancesSqrtWeights[ampName] = np.array([covSqrtWeights])
317 self.gain[ampName] = gain
318 self.noise[ampName] = noise
319 self.histVars[ampName] = np.array([histVar])
320 self.histChi2Dofs[ampName] = np.array([histChi2Dof])
321 self.kspValues[ampName] = np.array([kspValue])
323 # From FULLCOVARIANCE model
324 self.covariancesModel[ampName] = np.array([nanMatrixFit])
325 self.covariancesModelNoB[ampName] = np.array([nanMatrixFit])
326 self.aMatrix[ampName] = nanMatrixFit
327 self.bMatrix[ampName] = nanMatrixFit
328 self.aMatrixNoB[ampName] = nanMatrixFit
329 self.noiseMatrix[ampName] = nanMatrixFit
330 self.noiseMatrixNoB[ampName] = nanMatrixFit
332 def setAuxValuesPartialDataset(self, auxDict):
333 """
334 Set a dictionary of auxiliary values for a partial dataset.
336 Parameters
337 ----------
338 auxDict : `dict` [`str`, `float`]
339 Dictionary of float values.
340 """
341 for key, value in auxDict.items():
342 self.auxValues[key] = np.atleast_1d(np.array(value, dtype=np.float64))
344 def updateMetadata(self, **kwargs):
345 """Update calibration metadata.
346 This calls the base class's method after ensuring the required
347 calibration keywords will be saved.
349 Parameters
350 ----------
351 setDate : `bool`, optional
352 Update the CALIBDATE fields in the metadata to the current
353 time. Defaults to False.
354 kwargs :
355 Other keyword parameters to set in the metadata.
356 """
357 super().updateMetadata(PTC_FIT_TYPE=self.ptcFitType, **kwargs)
359 @classmethod
360 def fromDict(cls, dictionary):
361 """Construct a calibration from a dictionary of properties.
362 Must be implemented by the specific calibration subclasses.
364 Parameters
365 ----------
366 dictionary : `dict`
367 Dictionary of properties.
369 Returns
370 -------
371 calib : `lsst.ip.isr.PhotonTransferCurveDataset`
372 Constructed calibration.
374 Raises
375 ------
376 RuntimeError
377 Raised if the supplied dictionary is for a different
378 calibration.
379 """
380 calib = cls()
381 if calib._OBSTYPE != dictionary['metadata']['OBSTYPE']:
382 raise RuntimeError(f"Incorrect Photon Transfer Curve dataset supplied. "
383 f"Expected {calib._OBSTYPE}, found {dictionary['metadata']['OBSTYPE']}")
384 calib.setMetadata(dictionary['metadata'])
385 calib.ptcFitType = dictionary['ptcFitType']
386 calib.covMatrixSide = dictionary['covMatrixSide']
387 calib.covMatrixSideFullCovFit = dictionary['covMatrixSideFullCovFit']
388 calib.badAmps = np.array(dictionary['badAmps'], 'str').tolist()
389 calib.ampNames = []
391 # The cov matrices are square
392 covMatrixSide = calib.covMatrixSide
393 covMatrixSideFullCovFit = calib.covMatrixSideFullCovFit
394 # Number of final signal levels
395 covDimensionsProduct = len(np.array(list(dictionary['covariances'].values())[0]).ravel())
396 nSignalPoints = int(covDimensionsProduct/(covMatrixSide*covMatrixSide))
398 for ampName in dictionary['ampNames']:
399 calib.ampNames.append(ampName)
400 calib.inputExpIdPairs[ampName] = dictionary['inputExpIdPairs'][ampName]
401 calib.expIdMask[ampName] = np.array(dictionary['expIdMask'][ampName])
402 calib.rawExpTimes[ampName] = np.array(dictionary['rawExpTimes'][ampName], dtype=np.float64)
403 calib.rawMeans[ampName] = np.array(dictionary['rawMeans'][ampName], dtype=np.float64)
404 calib.rawVars[ampName] = np.array(dictionary['rawVars'][ampName], dtype=np.float64)
405 calib.gain[ampName] = float(dictionary['gain'][ampName])
406 calib.gainErr[ampName] = float(dictionary['gainErr'][ampName])
407 calib.noise[ampName] = float(dictionary['noise'][ampName])
408 calib.noiseErr[ampName] = float(dictionary['noiseErr'][ampName])
409 calib.histVars[ampName] = np.array(dictionary['histVars'][ampName], dtype=np.float64)
410 calib.histChi2Dofs[ampName] = np.array(dictionary['histChi2Dofs'][ampName], dtype=np.float64)
411 calib.kspValues[ampName] = np.array(dictionary['kspValues'][ampName], dtype=np.float64)
412 calib.ptcFitPars[ampName] = np.array(dictionary['ptcFitPars'][ampName], dtype=np.float64)
413 calib.ptcFitParsError[ampName] = np.array(dictionary['ptcFitParsError'][ampName],
414 dtype=np.float64)
415 calib.ptcFitChiSq[ampName] = float(dictionary['ptcFitChiSq'][ampName])
416 calib.ptcTurnoff[ampName] = float(dictionary['ptcTurnoff'][ampName])
417 if nSignalPoints > 0:
418 # Regular dataset
419 calib.covariances[ampName] = np.array(dictionary['covariances'][ampName],
420 dtype=np.float64).reshape(
421 (nSignalPoints, covMatrixSide, covMatrixSide))
422 calib.covariancesModel[ampName] = np.array(
423 dictionary['covariancesModel'][ampName],
424 dtype=np.float64).reshape(
425 (nSignalPoints, covMatrixSideFullCovFit, covMatrixSideFullCovFit))
426 calib.covariancesSqrtWeights[ampName] = np.array(
427 dictionary['covariancesSqrtWeights'][ampName],
428 dtype=np.float64).reshape(
429 (nSignalPoints, covMatrixSide, covMatrixSide))
430 calib.aMatrix[ampName] = np.array(dictionary['aMatrix'][ampName],
431 dtype=np.float64).reshape(
432 (covMatrixSideFullCovFit, covMatrixSideFullCovFit))
433 calib.bMatrix[ampName] = np.array(dictionary['bMatrix'][ampName],
434 dtype=np.float64).reshape(
435 (covMatrixSideFullCovFit, covMatrixSideFullCovFit))
436 calib.covariancesModelNoB[ampName] = np.array(
437 dictionary['covariancesModelNoB'][ampName], dtype=np.float64).reshape(
438 (nSignalPoints, covMatrixSideFullCovFit, covMatrixSideFullCovFit))
439 calib.aMatrixNoB[ampName] = np.array(
440 dictionary['aMatrixNoB'][ampName],
441 dtype=np.float64).reshape((covMatrixSideFullCovFit, covMatrixSideFullCovFit))
442 calib.noiseMatrix[ampName] = np.array(
443 dictionary['noiseMatrix'][ampName],
444 dtype=np.float64).reshape((covMatrixSideFullCovFit, covMatrixSideFullCovFit))
445 calib.noiseMatrixNoB[ampName] = np.array(
446 dictionary['noiseMatrixNoB'][ampName],
447 dtype=np.float64).reshape((covMatrixSideFullCovFit, covMatrixSideFullCovFit))
448 else:
449 # Empty dataset
450 calib.covariances[ampName] = np.array([], dtype=np.float64)
451 calib.covariancesModel[ampName] = np.array([], dtype=np.float64)
452 calib.covariancesSqrtWeights[ampName] = np.array([], dtype=np.float64)
453 calib.aMatrix[ampName] = np.array([], dtype=np.float64)
454 calib.bMatrix[ampName] = np.array([], dtype=np.float64)
455 calib.covariancesModelNoB[ampName] = np.array([], dtype=np.float64)
456 calib.aMatrixNoB[ampName] = np.array([], dtype=np.float64)
457 calib.noiseMatrix[ampName] = np.array([], dtype=np.float64)
458 calib.noiseMatrixNoB[ampName] = np.array([], dtype=np.float64)
460 calib.finalVars[ampName] = np.array(dictionary['finalVars'][ampName], dtype=np.float64)
461 calib.finalModelVars[ampName] = np.array(dictionary['finalModelVars'][ampName], dtype=np.float64)
462 calib.finalMeans[ampName] = np.array(dictionary['finalMeans'][ampName], dtype=np.float64)
463 calib.photoCharges[ampName] = np.array(dictionary['photoCharges'][ampName], dtype=np.float64)
465 for key, value in dictionary['auxValues'].items():
466 calib.auxValues[key] = np.atleast_1d(np.array(value, dtype=np.float64))
468 calib.updateMetadata()
469 return calib
471 def toDict(self):
472 """Return a dictionary containing the calibration properties.
473 The dictionary should be able to be round-tripped through
474 `fromDict`.
476 Returns
477 -------
478 dictionary : `dict`
479 Dictionary of properties.
480 """
481 self.updateMetadata()
483 outDict = dict()
484 metadata = self.getMetadata()
485 outDict['metadata'] = metadata
487 def _dictOfArraysToDictOfLists(dictOfArrays):
488 dictOfLists = {}
489 for key, value in dictOfArrays.items():
490 dictOfLists[key] = value.ravel().tolist()
492 return dictOfLists
494 outDict['ptcFitType'] = self.ptcFitType
495 outDict['covMatrixSide'] = self.covMatrixSide
496 outDict['covMatrixSideFullCovFit'] = self.covMatrixSideFullCovFit
497 outDict['ampNames'] = self.ampNames
498 outDict['badAmps'] = self.badAmps
499 outDict['inputExpIdPairs'] = self.inputExpIdPairs
500 outDict['expIdMask'] = _dictOfArraysToDictOfLists(self.expIdMask)
501 outDict['rawExpTimes'] = _dictOfArraysToDictOfLists(self.rawExpTimes)
502 outDict['rawMeans'] = _dictOfArraysToDictOfLists(self.rawMeans)
503 outDict['rawVars'] = _dictOfArraysToDictOfLists(self.rawVars)
504 outDict['gain'] = self.gain
505 outDict['gainErr'] = self.gainErr
506 outDict['noise'] = self.noise
507 outDict['noiseErr'] = self.noiseErr
508 outDict['histVars'] = self.histVars
509 outDict['histChi2Dofs'] = self.histChi2Dofs
510 outDict['kspValues'] = self.kspValues
511 outDict['ptcFitPars'] = _dictOfArraysToDictOfLists(self.ptcFitPars)
512 outDict['ptcFitParsError'] = _dictOfArraysToDictOfLists(self.ptcFitParsError)
513 outDict['ptcFitChiSq'] = self.ptcFitChiSq
514 outDict['ptcTurnoff'] = self.ptcTurnoff
515 outDict['covariances'] = _dictOfArraysToDictOfLists(self.covariances)
516 outDict['covariancesModel'] = _dictOfArraysToDictOfLists(self.covariancesModel)
517 outDict['covariancesSqrtWeights'] = _dictOfArraysToDictOfLists(self.covariancesSqrtWeights)
518 outDict['aMatrix'] = _dictOfArraysToDictOfLists(self.aMatrix)
519 outDict['bMatrix'] = _dictOfArraysToDictOfLists(self.bMatrix)
520 outDict['noiseMatrix'] = _dictOfArraysToDictOfLists(self.noiseMatrix)
521 outDict['covariancesModelNoB'] = _dictOfArraysToDictOfLists(self.covariancesModelNoB)
522 outDict['aMatrixNoB'] = _dictOfArraysToDictOfLists(self.aMatrixNoB)
523 outDict['noiseMatrixNoB'] = _dictOfArraysToDictOfLists(self.noiseMatrixNoB)
524 outDict['finalVars'] = _dictOfArraysToDictOfLists(self.finalVars)
525 outDict['finalModelVars'] = _dictOfArraysToDictOfLists(self.finalModelVars)
526 outDict['finalMeans'] = _dictOfArraysToDictOfLists(self.finalMeans)
527 outDict['photoCharges'] = _dictOfArraysToDictOfLists(self.photoCharges)
528 outDict['auxValues'] = _dictOfArraysToDictOfLists(self.auxValues)
530 return outDict
532 @classmethod
533 def fromTable(cls, tableList):
534 """Construct calibration from a list of tables.
535 This method uses the `fromDict` method to create the
536 calibration, after constructing an appropriate dictionary from
537 the input tables.
539 Parameters
540 ----------
541 tableList : `list` [`lsst.afw.table.Table`]
542 List of tables to use to construct the datasetPtc.
544 Returns
545 -------
546 calib : `lsst.ip.isr.PhotonTransferCurveDataset`
547 The calibration defined in the tables.
548 """
549 ptcTable = tableList[0]
551 metadata = ptcTable.meta
552 inDict = dict()
553 inDict['metadata'] = metadata
554 inDict['ampNames'] = []
555 inDict['ptcFitType'] = []
556 inDict['covMatrixSide'] = []
557 inDict['covMatrixSideFullCovFit'] = []
558 inDict['inputExpIdPairs'] = dict()
559 inDict['expIdMask'] = dict()
560 inDict['rawExpTimes'] = dict()
561 inDict['rawMeans'] = dict()
562 inDict['rawVars'] = dict()
563 inDict['gain'] = dict()
564 inDict['gainErr'] = dict()
565 inDict['noise'] = dict()
566 inDict['noiseErr'] = dict()
567 inDict['histVars'] = dict()
568 inDict['histChi2Dofs'] = dict()
569 inDict['kspValues'] = dict()
570 inDict['ptcFitPars'] = dict()
571 inDict['ptcFitParsError'] = dict()
572 inDict['ptcFitChiSq'] = dict()
573 inDict['ptcTurnoff'] = dict()
574 inDict['covariances'] = dict()
575 inDict['covariancesModel'] = dict()
576 inDict['covariancesSqrtWeights'] = dict()
577 inDict['aMatrix'] = dict()
578 inDict['bMatrix'] = dict()
579 inDict['noiseMatrix'] = dict()
580 inDict['covariancesModelNoB'] = dict()
581 inDict['aMatrixNoB'] = dict()
582 inDict['noiseMatrixNoB'] = dict()
583 inDict['finalVars'] = dict()
584 inDict['finalModelVars'] = dict()
585 inDict['finalMeans'] = dict()
586 inDict['badAmps'] = []
587 inDict['photoCharges'] = dict()
589 calibVersion = metadata['PTC_VERSION']
590 if calibVersion == 1.0:
591 cls().log.warning(f"Previous version found for PTC dataset: {calibVersion}. "
592 f"Setting 'ptcTurnoff' in all amps to last value in 'finalMeans'.")
593 for record in ptcTable:
594 ampName = record['AMPLIFIER_NAME']
596 inDict['ptcFitType'] = record['PTC_FIT_TYPE']
597 inDict['covMatrixSide'] = record['COV_MATRIX_SIDE']
598 inDict['ampNames'].append(ampName)
599 inDict['inputExpIdPairs'][ampName] = record['INPUT_EXP_ID_PAIRS'].tolist()
600 inDict['expIdMask'][ampName] = record['EXP_ID_MASK']
601 inDict['rawExpTimes'][ampName] = record['RAW_EXP_TIMES']
602 inDict['rawMeans'][ampName] = record['RAW_MEANS']
603 inDict['rawVars'][ampName] = record['RAW_VARS']
604 inDict['gain'][ampName] = record['GAIN']
605 inDict['gainErr'][ampName] = record['GAIN_ERR']
606 inDict['noise'][ampName] = record['NOISE']
607 inDict['noiseErr'][ampName] = record['NOISE_ERR']
608 inDict['ptcFitPars'][ampName] = record['PTC_FIT_PARS']
609 inDict['ptcFitParsError'][ampName] = record['PTC_FIT_PARS_ERROR']
610 inDict['ptcFitChiSq'][ampName] = record['PTC_FIT_CHI_SQ']
611 inDict['covariances'][ampName] = record['COVARIANCES']
612 inDict['covariancesModel'][ampName] = record['COVARIANCES_MODEL']
613 inDict['covariancesSqrtWeights'][ampName] = record['COVARIANCES_SQRT_WEIGHTS']
614 inDict['aMatrix'][ampName] = record['A_MATRIX']
615 inDict['bMatrix'][ampName] = record['B_MATRIX']
616 inDict['covariancesModelNoB'][ampName] = record['COVARIANCES_MODEL_NO_B']
617 inDict['aMatrixNoB'][ampName] = record['A_MATRIX_NO_B']
618 inDict['finalVars'][ampName] = record['FINAL_VARS']
619 inDict['finalModelVars'][ampName] = record['FINAL_MODEL_VARS']
620 inDict['finalMeans'][ampName] = record['FINAL_MEANS']
621 inDict['badAmps'] = record['BAD_AMPS'].tolist()
622 inDict['photoCharges'][ampName] = record['PHOTO_CHARGE']
623 if calibVersion == 1.0:
624 mask = record['FINAL_MEANS'].mask
625 array = record['FINAL_MEANS'][~mask]
626 if len(array) > 0:
627 inDict['ptcTurnoff'][ampName] = record['FINAL_MEANS'][~mask][-1]
628 else:
629 inDict['ptcTurnoff'][ampName] = np.nan
630 else:
631 inDict['ptcTurnoff'][ampName] = record['PTC_TURNOFF']
632 if calibVersion < 1.2:
633 inDict['histVars'][ampName] = np.array([np.nan])
634 inDict['histChi2Dofs'][ampName] = np.array([np.nan])
635 inDict['kspValues'][ampName] = np.array([0.0])
636 else:
637 inDict['histVars'][ampName] = record['HIST_VARS']
638 inDict['histChi2Dofs'][ampName] = record['HIST_CHI2_DOFS']
639 inDict['kspValues'][ampName] = record['KS_PVALUES']
640 if calibVersion < 1.3:
641 nanMatrix = np.full_like(inDict['aMatrix'][ampName], np.nan)
642 inDict['noiseMatrix'][ampName] = nanMatrix
643 inDict['noiseMatrixNoB'][ampName] = nanMatrix
644 else:
645 inDict['noiseMatrix'][ampName] = record['NOISE_MATRIX']
646 inDict['noiseMatrixNoB'][ampName] = record['NOISE_MATRIX_NO_B']
647 if calibVersion < 1.5:
648 # Matched to `COV_MATRIX_SIDE`. Same for all amps.
649 inDict['covMatrixSideFullCovFit'] = inDict['covMatrixSide']
650 else:
651 inDict['covMatrixSideFullCovFit'] = record['COV_MATRIX_SIDE_FULL_COV_FIT']
653 inDict['auxValues'] = {}
654 record = ptcTable[0]
655 for col in record.columns.keys():
656 if col.startswith('PTCAUX_'):
657 parts = col.split('PTCAUX_')
658 inDict['auxValues'][parts[1]] = record[col]
660 return cls().fromDict(inDict)
662 def toTable(self):
663 """Construct a list of tables containing the information in this
664 calibration.
666 The list of tables should create an identical calibration
667 after being passed to this class's fromTable method.
669 Returns
670 -------
671 tableList : `list` [`astropy.table.Table`]
672 List of tables containing the linearity calibration
673 information.
674 """
675 tableList = []
676 self.updateMetadata()
678 badAmps = np.array(self.badAmps) if len(self.badAmps) else np.array([], dtype="U3")
680 catalogList = []
681 for ampName in self.ampNames:
682 ampDict = {
683 'AMPLIFIER_NAME': ampName,
684 'PTC_FIT_TYPE': self.ptcFitType,
685 'COV_MATRIX_SIDE': self.covMatrixSide,
686 'COV_MATRIX_SIDE_FULL_COV_FIT': self.covMatrixSideFullCovFit,
687 'INPUT_EXP_ID_PAIRS': self.inputExpIdPairs[ampName],
688 'EXP_ID_MASK': self.expIdMask[ampName],
689 'RAW_EXP_TIMES': self.rawExpTimes[ampName],
690 'RAW_MEANS': self.rawMeans[ampName],
691 'RAW_VARS': self.rawVars[ampName],
692 'GAIN': self.gain[ampName],
693 'GAIN_ERR': self.gainErr[ampName],
694 'NOISE': self.noise[ampName],
695 'NOISE_ERR': self.noiseErr[ampName],
696 'HIST_VARS': self.histVars[ampName],
697 'HIST_CHI2_DOFS': self.histChi2Dofs[ampName],
698 'KS_PVALUES': self.kspValues[ampName],
699 'PTC_FIT_PARS': np.array(self.ptcFitPars[ampName]),
700 'PTC_FIT_PARS_ERROR': np.array(self.ptcFitParsError[ampName]),
701 'PTC_FIT_CHI_SQ': self.ptcFitChiSq[ampName],
702 'PTC_TURNOFF': self.ptcTurnoff[ampName],
703 'A_MATRIX': self.aMatrix[ampName].ravel(),
704 'B_MATRIX': self.bMatrix[ampName].ravel(),
705 'A_MATRIX_NO_B': self.aMatrixNoB[ampName].ravel(),
706 'NOISE_MATRIX': self.noiseMatrix[ampName].ravel(),
707 'NOISE_MATRIX_NO_B': self.noiseMatrixNoB[ampName].ravel(),
708 'BAD_AMPS': badAmps,
709 'PHOTO_CHARGE': self.photoCharges[ampName],
710 'COVARIANCES': self.covariances[ampName].ravel(),
711 'COVARIANCES_MODEL': self.covariancesModel[ampName].ravel(),
712 'COVARIANCES_SQRT_WEIGHTS': self.covariancesSqrtWeights[ampName].ravel(),
713 'COVARIANCES_MODEL_NO_B': self.covariancesModelNoB[ampName].ravel(),
714 'FINAL_VARS': self.finalVars[ampName],
715 'FINAL_MODEL_VARS': self.finalModelVars[ampName],
716 'FINAL_MEANS': self.finalMeans[ampName],
717 }
719 if self.auxValues:
720 for key, value in self.auxValues.items():
721 ampDict[f"PTCAUX_{key}"] = value
723 catalogList.append(ampDict)
725 catalog = Table(catalogList)
727 inMeta = self.getMetadata().toDict()
728 outMeta = {k: v for k, v in inMeta.items() if v is not None}
729 outMeta.update({k: "" for k, v in inMeta.items() if v is None})
730 catalog.meta = outMeta
731 tableList.append(catalog)
733 return tableList
735 def fromDetector(self, detector):
736 """Read metadata parameters from a detector.
738 Parameters
739 ----------
740 detector : `lsst.afw.cameraGeom.detector`
741 Input detector with parameters to use.
743 Returns
744 -------
745 calib : `lsst.ip.isr.PhotonTransferCurveDataset`
746 The calibration constructed from the detector.
747 """
749 pass
751 def getExpIdsUsed(self, ampName):
752 """Get the exposures used, i.e. not discarded, for a given amp.
753 If no mask has been created yet, all exposures are returned.
755 Parameters
756 ----------
757 ampName : `str`
759 Returns
760 -------
761 expIdsUsed : `list` [`tuple`]
762 List of pairs of exposure ids used in PTC.
763 """
764 if len(self.expIdMask[ampName]) == 0:
765 return self.inputExpIdPairs[ampName]
767 # if the mask exists it had better be the same length as the expIdPairs
768 assert len(self.expIdMask[ampName]) == len(self.inputExpIdPairs[ampName])
770 pairs = self.inputExpIdPairs[ampName]
771 mask = self.expIdMask[ampName]
772 # cast to bool required because numpy
773 try:
774 expIdsUsed = [(exp1, exp2) for ((exp1, exp2), m) in zip(pairs, mask) if m]
775 except ValueError:
776 self.log.warning("The PTC file was written incorrectly; you should rerun the "
777 "PTC solve task if possible.")
778 expIdsUsed = []
779 for pairList, m in zip(pairs, mask):
780 if m:
781 expIdsUsed.append(pairList[0])
783 return expIdsUsed
785 def getGoodAmps(self):
786 """Get the good amps from this PTC."""
787 return [amp for amp in self.ampNames if amp not in self.badAmps]
789 def getGoodPoints(self, ampName):
790 """Get the good points used for a given amp in the PTC.
792 Parameters
793 ----------
794 ampName : `str`
795 Amplifier's name.
797 Returns
798 -------
799 goodPoints : `np.ndarray`
800 Boolean array of good points used in PTC.
801 """
802 return self.expIdMask[ampName]
804 def validateGainNoiseTurnoffValues(self, ampName, doWarn=False):
805 """Ensure the gain, read noise, and PTC turnoff have
806 sensible values.
808 Parameters
809 ----------
810 ampName : `str`
811 Amplifier's name.
812 """
814 gain = self.gain[ampName]
815 noise = self.noise[ampName]
816 ptcTurnoff = self.ptcTurnoff[ampName]
818 # Check if gain is not positive or is np.nan
819 if not (isinstance(gain, (int, float)) and gain > 0) or math.isnan(gain):
820 if doWarn:
821 self.log.warning(f"Invalid gain value {gain}"
822 " Setting to default: Gain=1")
823 gain = 1
825 # Check if noise is not positive or is np.nan
826 if not (isinstance(noise, (int, float)) and noise > 0) or math.isnan(noise):
827 if doWarn:
828 self.log.warning(f"Invalid noise value: {noise}"
829 " Setting to default: Noise=1")
830 noise = 1
832 # Check if ptcTurnoff is not positive or is np.nan
833 if not (isinstance(ptcTurnoff, (int, float)) and ptcTurnoff > 0) or math.isnan(ptcTurnoff):
834 if doWarn:
835 self.log.warning(f"Invalid PTC turnoff value: {ptcTurnoff}"
836 " Setting to default: PTC Turnoff=2e19")
837 ptcTurnoff = 2e19
839 self.gain[ampName] = gain
840 self.noise[ampName] = noise
841 self.ptcTurnoff[ampName] = ptcTurnoff
843 def _validateCovarianceMatrizSizes(self):
844 """Ensure covMatrixSideFullCovFit <= covMatrixSide."""
845 if self.covMatrixSideFullCovFit > self.covMatrixSide:
846 self.log.warning("covMatrixSideFullCovFit > covMatrixSide "
847 f"({self.covMatrixSideFullCovFit} > {self.covMatrixSide})."
848 "Setting the former to the latter.")
849 self.covMatrixSideFullCovFit = self.covMatrixSide