23Define dataset class for MeasurePhotonTransferCurve task
26__all__ = [
'PhotonTransferCurveDataset']
29from astropy.table
import Table
35 """A simple class to hold the output data from the PTC task.
37 The dataset is made up of a dictionary
for each item, keyed by the
38 amplifiers
' names, which much be supplied at construction time.
39 New items cannot be added to the class to save accidentally saving to the
40 wrong property,
and the
class can be frozen if desired.
41 inputExpIdPairs records the exposures used to produce the data.
42 When fitPtc()
or fitCovariancesAstier()
is run, a mask
is built up, which
43 is by definition always the same length
as inputExpIdPairs, rawExpTimes,
44 rawMeans
and rawVars,
and is a list of bools, which are incrementally set
45 to
False as points are discarded
from the fits.
46 PTC fit parameters
for polynomials are stored
in a list
in ascending order
47 of polynomial term, i.e. par[0]*x^0 + par[1]*x + par[2]*x^2 etc
48 with the length of the list corresponding to the order of the polynomial
54 List
with the names of the amplifiers of the detector at hand.
56 Type of model fitted to the PTC:
"POLYNOMIAL",
"EXPAPPROXIMATION",
59 Maximum lag of covariances (size of square covariance matrices).
60 kwargs : `dict`, optional
61 Other keyword arguments to
pass to the parent init.
65 The stored attributes are:
68 List
with bad amplifiers names.
69 inputExpIdPairs : `dict`, [`str`, `list`]
70 Dictionary keyed by amp names containing the input exposures IDs.
71 expIdMask : `dict`, [`str`, `list`]
72 Dictionary keyed by amp names containing the mask produced after
73 outlier rejection. The mask produced by the
"FULLCOVARIANCE"
74 option may differ
from the one produced
in the other two PTC
76 rawExpTimes : `dict`, [`str`, `list`]
77 Dictionary keyed by amp names containing the unmasked exposure times.
78 rawMeans : `dict`, [`str`, `list`]
79 Dictionary keyed by amp namescontaining the unmasked average of the
80 means of the exposures
in each flat pair.
81 rawVars : `dict`, [`str`, `list`]
82 Dictionary keyed by amp names containing the variance of the
83 difference image of the exposures
in each flat pair.
84 gain : `dict`, [`str`, `list`]
85 Dictionary keyed by amp names containing the fitted gains.
86 gainErr : `dict`, [`str`, `list`]
87 Dictionary keyed by amp names containing the errors on the
89 noise : `dict`, [`str`, `list`]
90 Dictionary keyed by amp names containing the fitted noise.
91 noiseErr : `dict`, [`str`, `list`]
92 Dictionary keyed by amp names containing the errors on the fitted
94 ptcFitPars : `dict`, [`str`, `list`]
95 Dictionary keyed by amp names containing the fitted parameters of the
96 PTC model
for ptcFitTye
in [
"POLYNOMIAL",
"EXPAPPROXIMATION"].
97 ptcFitParsError : `dict`, [`str`, `list`]
98 Dictionary keyed by amp names containing the errors on the fitted
99 parameters of the PTC model
for ptcFitTye
in
100 [
"POLYNOMIAL",
"EXPAPPROXIMATION"].
101 ptcFitChiSq : `dict`, [`str`, `list`]
102 Dictionary keyed by amp names containing the reduced chi squared
103 of the fit
for ptcFitTye
in [
"POLYNOMIAL",
"EXPAPPROXIMATION"].
105 Flux value (
in ADU) where the variance of the PTC curve starts
106 decreasing consistently.
107 covariances : `dict`, [`str`, `list`]
108 Dictionary keyed by amp names containing a list of measured
109 covariances per mean flux.
110 covariancesModel : `dict`, [`str`, `list`]
111 Dictionary keyed by amp names containinging covariances model
112 (Eq. 20 of Astier+19) per mean flux.
113 covariancesSqrtWeights : `dict`, [`str`, `list`]
114 Dictionary keyed by amp names containinging sqrt. of covariances
116 aMatrix : `dict`, [`str`, `list`]
117 Dictionary keyed by amp names containing the
"a" parameters
from
118 the model
in Eq. 20 of Astier+19.
119 bMatrix : `dict`, [`str`, `list`]
120 Dictionary keyed by amp names containing the
"b" parameters
from
121 the model
in Eq. 20 of Astier+19.
122 covariancesModelNoB : `dict`, [`str`, `list`]
123 Dictionary keyed by amp names containing covariances model
124 (
with 'b'=0
in Eq. 20 of Astier+19)
126 aMatrixNoB : `dict`, [`str`, `list`]
127 Dictionary keyed by amp names containing the
"a" parameters
from the
128 model
in Eq. 20 of Astier+19
130 finalVars : `dict`, [`str`, `list`]
131 Dictionary keyed by amp names containing the masked variance of the
132 difference image of each flat
133 pair. If needed, each array will be right-padded
with
134 np.nan to match the length of rawExpTimes.
135 finalModelVars : `dict`, [`str`, `list`]
136 Dictionary keyed by amp names containing the masked modeled
137 variance of the difference image of each flat pair. If needed, each
138 array will be right-padded
with np.nan to match the length of
140 finalMeans : `dict`, [`str`, `list`]
141 Dictionary keyed by amp names containing the masked average of the
142 means of the exposures
in each flat pair. If needed, each array
143 will be right-padded
with np.nan to match the length of
145 photoCharge : `dict`, [`str`, `list`]
146 Dictionary keyed by amp names containing the integrated photocharge
147 for linearity calibration.
149 Version 1.1 adds the `ptcTurnoff` attribute.
153 _SCHEMA =
'Gen3 Photon Transfer Curve'
156 def __init__(self, ampNames=[], ptcFitType=None, covMatrixSide=1, **kwargs):
166 self.
rawMeans = {ampName: []
for ampName
in ampNames}
167 self.
rawVars = {ampName: []
for ampName
in ampNames}
170 self.
gain = {ampName: np.nan
for ampName
in ampNames}
171 self.
gainErr = {ampName: np.nan
for ampName
in ampNames}
172 self.
noise = {ampName: np.nan
for ampName
in ampNames}
173 self.
noiseErr = {ampName: np.nan
for ampName
in ampNames}
183 self.
aMatrix = {ampName: np.nan
for ampName
in ampNames}
184 self.
bMatrix = {ampName: np.nan
for ampName
in ampNames}
194 'rawMeans',
'rawVars',
'gain',
'gainErr',
'noise',
'noiseErr',
195 'ptcFitPars',
'ptcFitParsError',
'ptcFitChiSq',
'ptcTurnoff',
196 'aMatrixNoB',
'covariances',
'covariancesModel',
197 'covariancesSqrtWeights',
'covariancesModelNoB',
198 'aMatrix',
'bMatrix',
'finalVars',
'finalModelVars',
'finalMeans',
203 def setAmpValues(self, ampName, inputExpIdPair=[(np.nan, np.nan)], expIdMask=[np.nan],
204 rawExpTime=[np.nan], rawMean=[np.nan], rawVar=[np.nan], photoCharge=[np.nan],
205 gain=np.nan, gainErr=np.nan, noise=np.nan, noiseErr=np.nan, ptcFitPars=[np.nan],
206 ptcFitParsError=[np.nan], ptcFitChiSq=np.nan, ptcTurnoff=np.nan, covArray=[],
207 covArrayModel=[], covSqrtWeights=[], aMatrix=[], bMatrix=[], covArrayModelNoB=[],
208 aMatrixNoB=[], finalVar=[np.nan], finalModelVar=[np.nan], finalMean=[np.nan]):
209 """Function to initialize an amp of a PhotonTransferCurveDataset.
213 The parameters are all documented in `init`.
216 if len(covArray) == 0:
217 covArray = [nanMatrix]
218 if len(covArrayModel) == 0:
219 covArrayModel = [nanMatrix]
220 if len(covSqrtWeights) == 0:
221 covSqrtWeights = [nanMatrix]
222 if len(covArrayModelNoB) == 0:
223 covArrayModelNoB = [nanMatrix]
224 if len(aMatrix) == 0:
226 if len(bMatrix) == 0:
228 if len(aMatrixNoB) == 0:
229 aMatrixNoB = nanMatrix
237 self.
gain[ampName] = gain
238 self.
gainErr[ampName] = gainErr
239 self.
noise[ampName] = noise
249 self.
aMatrix[ampName] = aMatrix
250 self.
bMatrix[ampName] = bMatrix
260 """Update calibration metadata.
261 This calls the base class's method after ensuring the required
262 calibration keywords will be saved.
266 setDate : `bool`, optional
267 Update the CALIBDATE fields in the metadata to the current
268 time. Defaults to
False.
270 Other keyword parameters to set
in the metadata.
276 """Construct a calibration from a dictionary of properties.
277 Must be implemented by the specific calibration subclasses.
282 Dictionary of properties.
287 Constructed calibration.
292 Raised if the supplied dictionary
is for a different
296 if calib._OBSTYPE != dictionary[
'metadata'][
'OBSTYPE']:
297 raise RuntimeError(f
"Incorrect Photon Transfer Curve dataset supplied. "
298 f
"Expected {calib._OBSTYPE}, found {dictionary['metadata']['OBSTYPE']}")
299 calib.setMetadata(dictionary[
'metadata'])
300 calib.ptcFitType = dictionary[
'ptcFitType']
301 calib.covMatrixSide = dictionary[
'covMatrixSide']
302 calib.badAmps = np.array(dictionary[
'badAmps'],
'str').tolist()
306 covMatrixSide = calib.covMatrixSide
308 covDimensionsProduct = len(np.array(list(dictionary[
'covariances'].values())[0]).ravel())
309 nSignalPoints = int(covDimensionsProduct/(covMatrixSide*covMatrixSide))
311 for ampName
in dictionary[
'ampNames']:
312 covsAmp = np.array(dictionary[
'covariances'][ampName]).reshape((nSignalPoints, covMatrixSide,
321 maskCovsAmp = np.array([~np.isnan(entry).all()
for entry
in covsAmp])
322 maskAmp = ~np.isnan(np.array(dictionary[
'finalMeans'][ampName]))
324 maskCovsAmp = np.array([
True])
325 maskAmp = np.array([
True])
327 calib.ampNames.append(ampName)
328 calib.inputExpIdPairs[ampName] = np.array(dictionary[
'inputExpIdPairs'][ampName]).tolist()
329 calib.expIdMask[ampName] = np.array(dictionary[
'expIdMask'][ampName]).tolist()
330 calib.rawExpTimes[ampName] = np.array(dictionary[
'rawExpTimes'][ampName]).tolist()
331 calib.rawMeans[ampName] = np.array(dictionary[
'rawMeans'][ampName]).tolist()
332 calib.rawVars[ampName] = np.array(dictionary[
'rawVars'][ampName]).tolist()
333 calib.gain[ampName] = np.array(dictionary[
'gain'][ampName]).tolist()
334 calib.gainErr[ampName] = np.array(dictionary[
'gainErr'][ampName]).tolist()
335 calib.noise[ampName] = np.array(dictionary[
'noise'][ampName]).tolist()
336 calib.noiseErr[ampName] = np.array(dictionary[
'noiseErr'][ampName]).tolist()
337 calib.ptcFitPars[ampName] = np.array(dictionary[
'ptcFitPars'][ampName]).tolist()
338 calib.ptcFitParsError[ampName] = np.array(dictionary[
'ptcFitParsError'][ampName]).tolist()
339 calib.ptcFitChiSq[ampName] = np.array(dictionary[
'ptcFitChiSq'][ampName]).tolist()
340 calib.ptcTurnoff[ampName] = np.array(dictionary[
'ptcTurnoff'][ampName]).tolist()
341 calib.covariances[ampName] = covsAmp[maskCovsAmp].tolist()
342 calib.covariancesModel[ampName] = np.array(
343 dictionary[
'covariancesModel'][ampName]).reshape(
344 (nSignalPoints, covMatrixSide, covMatrixSide))[maskCovsAmp].tolist()
345 calib.covariancesSqrtWeights[ampName] = np.array(
346 dictionary[
'covariancesSqrtWeights'][ampName]).reshape(
347 (nSignalPoints, covMatrixSide, covMatrixSide))[maskCovsAmp].tolist()
348 calib.aMatrix[ampName] = np.array(dictionary[
'aMatrix'][ampName]).reshape(
349 (covMatrixSide, covMatrixSide)).tolist()
350 calib.bMatrix[ampName] = np.array(dictionary[
'bMatrix'][ampName]).reshape(
351 (covMatrixSide, covMatrixSide)).tolist()
352 calib.covariancesModelNoB[ampName] = np.array(
353 dictionary[
'covariancesModelNoB'][ampName]).reshape(
354 (nSignalPoints, covMatrixSide, covMatrixSide))[maskCovsAmp].tolist()
355 calib.aMatrixNoB[ampName] = np.array(
356 dictionary[
'aMatrixNoB'][ampName]).reshape((covMatrixSide, covMatrixSide)).tolist()
357 calib.finalVars[ampName] = np.array(dictionary[
'finalVars'][ampName])[maskAmp].tolist()
358 calib.finalModelVars[ampName] = np.array(dictionary[
'finalModelVars'][ampName])[maskAmp].tolist()
359 calib.finalMeans[ampName] = np.array(dictionary[
'finalMeans'][ampName])[maskAmp].tolist()
360 calib.photoCharge[ampName] = np.array(dictionary[
'photoCharge'][ampName]).tolist()
361 calib.updateMetadata()
365 """Return a dictionary containing the calibration properties.
366 The dictionary should be able to be round-tripped through
372 Dictionary of properties.
378 outDict['metadata'] = metadata
383 outDict[
'badAmps'] = self.
badAmps
388 outDict[
'rawVars'] = self.
rawVars
389 outDict[
'gain'] = self.
gain
390 outDict[
'gainErr'] = self.
gainErr
391 outDict[
'noise'] = self.
noise
400 outDict[
'aMatrix'] = self.
aMatrix
401 outDict[
'bMatrix'] = self.
bMatrix
413 """Construct calibration from a list of tables.
414 This method uses the `fromDict` method to create the
415 calibration, after constructing an appropriate dictionary from
420 tableList : `list` [`lsst.afw.table.Table`]
421 List of tables to use to construct the datasetPtc.
426 The calibration defined
in the tables.
428 ptcTable = tableList[0]
430 metadata = ptcTable.meta
432 inDict['metadata'] = metadata
433 inDict[
'ampNames'] = []
434 inDict[
'ptcFitType'] = []
435 inDict[
'covMatrixSide'] = []
436 inDict[
'inputExpIdPairs'] = dict()
437 inDict[
'expIdMask'] = dict()
438 inDict[
'rawExpTimes'] = dict()
439 inDict[
'rawMeans'] = dict()
440 inDict[
'rawVars'] = dict()
441 inDict[
'gain'] = dict()
442 inDict[
'gainErr'] = dict()
443 inDict[
'noise'] = dict()
444 inDict[
'noiseErr'] = dict()
445 inDict[
'ptcFitPars'] = dict()
446 inDict[
'ptcFitParsError'] = dict()
447 inDict[
'ptcFitChiSq'] = dict()
448 inDict[
'ptcTurnoff'] = dict()
449 inDict[
'covariances'] = dict()
450 inDict[
'covariancesModel'] = dict()
451 inDict[
'covariancesSqrtWeights'] = dict()
452 inDict[
'aMatrix'] = dict()
453 inDict[
'bMatrix'] = dict()
454 inDict[
'covariancesModelNoB'] = dict()
455 inDict[
'aMatrixNoB'] = dict()
456 inDict[
'finalVars'] = dict()
457 inDict[
'finalModelVars'] = dict()
458 inDict[
'finalMeans'] = dict()
459 inDict[
'badAmps'] = []
460 inDict[
'photoCharge'] = dict()
462 calibVersion = metadata[
'PTC_VERSION']
463 if calibVersion == 1.0:
464 cls().log.warning(f
"Previous version found for PTC dataset: {calibVersion}. "
465 f
"Setting 'ptcTurnoff' in all amps to last value in 'finalMeans'.")
466 for record
in ptcTable:
467 ampName = record[
'AMPLIFIER_NAME']
469 inDict[
'ptcFitType'] = record[
'PTC_FIT_TYPE']
470 inDict[
'covMatrixSide'] = record[
'COV_MATRIX_SIDE']
471 inDict[
'ampNames'].append(ampName)
472 inDict[
'inputExpIdPairs'][ampName] = record[
'INPUT_EXP_ID_PAIRS']
473 inDict[
'expIdMask'][ampName] = record[
'EXP_ID_MASK']
474 inDict[
'rawExpTimes'][ampName] = record[
'RAW_EXP_TIMES']
475 inDict[
'rawMeans'][ampName] = record[
'RAW_MEANS']
476 inDict[
'rawVars'][ampName] = record[
'RAW_VARS']
477 inDict[
'gain'][ampName] = record[
'GAIN']
478 inDict[
'gainErr'][ampName] = record[
'GAIN_ERR']
479 inDict[
'noise'][ampName] = record[
'NOISE']
480 inDict[
'noiseErr'][ampName] = record[
'NOISE_ERR']
481 inDict[
'ptcFitPars'][ampName] = record[
'PTC_FIT_PARS']
482 inDict[
'ptcFitParsError'][ampName] = record[
'PTC_FIT_PARS_ERROR']
483 inDict[
'ptcFitChiSq'][ampName] = record[
'PTC_FIT_CHI_SQ']
484 inDict[
'covariances'][ampName] = record[
'COVARIANCES']
485 inDict[
'covariancesModel'][ampName] = record[
'COVARIANCES_MODEL']
486 inDict[
'covariancesSqrtWeights'][ampName] = record[
'COVARIANCES_SQRT_WEIGHTS']
487 inDict[
'aMatrix'][ampName] = record[
'A_MATRIX']
488 inDict[
'bMatrix'][ampName] = record[
'B_MATRIX']
489 inDict[
'covariancesModelNoB'][ampName] = record[
'COVARIANCES_MODEL_NO_B']
490 inDict[
'aMatrixNoB'][ampName] = record[
'A_MATRIX_NO_B']
491 inDict[
'finalVars'][ampName] = record[
'FINAL_VARS']
492 inDict[
'finalModelVars'][ampName] = record[
'FINAL_MODEL_VARS']
493 inDict[
'finalMeans'][ampName] = record[
'FINAL_MEANS']
494 inDict[
'badAmps'] = record[
'BAD_AMPS']
495 inDict[
'photoCharge'][ampName] = record[
'PHOTO_CHARGE']
496 if calibVersion == 1.0:
497 mask = record[
'FINAL_MEANS'].mask
498 array = record[
'FINAL_MEANS'][~mask]
500 inDict[
'ptcTurnoff'][ampName] = record[
'FINAL_MEANS'][~mask][-1]
502 inDict[
'ptcTurnoff'][ampName] = np.nan
504 inDict[
'ptcTurnoff'][ampName] = record[
'PTC_TURNOFF']
508 """Construct a list of tables containing the information in this
511 The list of tables should create an identical calibration
512 after being passed to this class's fromTable method.
516 tableList : `list` [`astropy.table.Table`]
517 List of tables containing the linearity calibration
523 for i, ampName
in enumerate(self.
ampNames):
524 nPoints.append(len(list(self.
covariances.values())[i]))
525 nSignalPoints = max(nPoints)
527 for i, ampName
in enumerate(self.
ampNames):
528 nPadPoints[ampName] = nSignalPoints - len(list(self.
covariances.values())[i])
531 catalog = Table([{
'AMPLIFIER_NAME': ampName,
535 if len(self.
expIdMask[ampName])
else np.nan,
537 if len(self.
expIdMask[ampName])
else np.nan,
538 'RAW_EXP_TIMES': np.array(self.
rawExpTimes[ampName]).tolist()
540 'RAW_MEANS': np.array(self.
rawMeans[ampName]).tolist()
541 if len(self.
rawMeans[ampName])
else np.nan,
542 'RAW_VARS': np.array(self.
rawVars[ampName]).tolist()
543 if len(self.
rawVars[ampName])
else np.nan,
544 'GAIN': self.
gain[ampName],
545 'GAIN_ERR': self.
gainErr[ampName],
546 'NOISE': self.
noise[ampName],
547 'NOISE_ERR': self.
noiseErr[ampName],
548 'PTC_FIT_PARS': np.array(self.
ptcFitPars[ampName]).tolist(),
549 'PTC_FIT_PARS_ERROR': np.array(self.
ptcFitParsError[ampName]).tolist(),
552 'COVARIANCES': np.pad(np.array(self.
covariances[ampName]),
553 ((0, nPadPoints[ampName]), (0, 0), (0, 0)),
554 'constant', constant_values=np.nan).reshape(
555 nSignalPoints*covMatrixSide**2).tolist(),
557 ((0, nPadPoints[ampName]), (0, 0), (0, 0)),
558 'constant', constant_values=np.nan).reshape(
559 nSignalPoints*covMatrixSide**2).tolist(),
561 ((0, nPadPoints[ampName]), (0, 0), (0, 0)),
562 'constant', constant_values=0.0).reshape(
563 nSignalPoints*covMatrixSide**2).tolist(),
564 'A_MATRIX': np.array(self.
aMatrix[ampName]).reshape(covMatrixSide**2).tolist(),
565 'B_MATRIX': np.array(self.
bMatrix[ampName]).reshape(covMatrixSide**2).tolist(),
566 'COVARIANCES_MODEL_NO_B':
568 ((0, nPadPoints[ampName]), (0, 0), (0, 0)),
569 'constant', constant_values=np.nan).reshape(
570 nSignalPoints*covMatrixSide**2).tolist(),
571 'A_MATRIX_NO_B': np.array(self.
aMatrixNoB[ampName]).reshape(
572 covMatrixSide**2).tolist(),
573 'FINAL_VARS': np.pad(np.array(self.
finalVars[ampName]), (0, nPadPoints[ampName]),
574 'constant', constant_values=np.nan).tolist(),
575 'FINAL_MODEL_VARS': np.pad(np.array(self.
finalModelVars[ampName]),
576 (0, nPadPoints[ampName]),
577 'constant', constant_values=np.nan).tolist(),
578 'FINAL_MEANS': np.pad(np.array(self.
finalMeans[ampName]),
579 (0, nPadPoints[ampName]),
580 'constant', constant_values=np.nan).tolist(),
581 'BAD_AMPS': np.array(self.
badAmps).tolist()
if len(self.
badAmps)
else np.nan,
582 'PHOTO_CHARGE': np.array(self.
photoCharge[ampName]).tolist(),
585 outMeta = {k: v
for k, v
in inMeta.items()
if v
is not None}
586 outMeta.update({k:
"" for k, v
in inMeta.items()
if v
is None})
587 catalog.meta = outMeta
588 tableList.append(catalog)
593 """Read metadata parameters from a detector.
597 detector : `lsst.afw.cameraGeom.detector`
598 Input detector with parameters to use.
603 The calibration constructed
from the detector.
609 """Get the exposures used, i.e. not discarded, for a given amp.
610 If no mask has been created yet, all exposures are returned.
621 return [(exp1, exp2)
for ((exp1, exp2), m)
in zip(pairs, mask)
if bool(m)
is True]
def requiredAttributes(self, value)
def updateMetadata(self, camera=None, detector=None, filterName=None, setCalibId=False, setCalibInfo=False, setDate=False, **kwargs)
def requiredAttributes(self)
def __init__(self, ampNames=[], ptcFitType=None, covMatrixSide=1, **kwargs)
def getExpIdsUsed(self, ampName)
def fromDict(cls, dictionary)
def fromDetector(self, detector)
def updateMetadata(self, **kwargs)
def fromTable(cls, tableList)
def setAmpValues(self, ampName, inputExpIdPair=[(np.nan, np.nan)], expIdMask=[np.nan], rawExpTime=[np.nan], rawMean=[np.nan], rawVar=[np.nan], photoCharge=[np.nan], gain=np.nan, gainErr=np.nan, noise=np.nan, noiseErr=np.nan, ptcFitPars=[np.nan], ptcFitParsError=[np.nan], ptcFitChiSq=np.nan, ptcTurnoff=np.nan, covArray=[], covArrayModel=[], covSqrtWeights=[], aMatrix=[], bMatrix=[], covArrayModelNoB=[], aMatrixNoB=[], finalVar=[np.nan], finalModelVar=[np.nan], finalMean=[np.nan])