23 Define dataset class for MeasurePhotonTransferCurve task
26 from astropy.table
import Table
30 __all__ = [
'PhotonTransferCurveDataset']
34 """A simple class to hold the output data from the PTC task.
35 The dataset is made up of a dictionary for each item, keyed by the
36 amplifiers' names, which much be supplied at construction time.
37 New items cannot be added to the class to save accidentally saving to the
38 wrong property, and the class can be frozen if desired.
39 inputExpIdPairs records the exposures used to produce the data.
40 When fitPtc() or fitCovariancesAstier() is run, a mask is built up, which
41 is by definition always the same length as inputExpIdPairs, rawExpTimes,
42 rawMeans and rawVars, and is a list of bools, which are incrementally set
43 to False as points are discarded from the fits.
44 PTC fit parameters for polynomials are stored in a list in ascending order
45 of polynomial term, i.e. par[0]*x^0 + par[1]*x + par[2]*x^2 etc
46 with the length of the list corresponding to the order of the polynomial
52 List with the names of the amplifiers of the detector at hand.
55 Type of model fitted to the PTC: "POLYNOMIAL", "EXPAPPROXIMATION",
59 Maximum lag of covariances (size of square covariance matrices).
61 kwargs : `dict`, optional
62 Other keyword arguments to pass to the parent init.
66 The stored attributes are:
68 List with bad amplifiers names.
69 inputExpIdPairs : `dict`, [`str`, `list`]
70 Dictionary keyed by amp names containing the input exposures IDs.
71 expIdMask : `dict`, [`str`, `list`]
72 Dictionary keyed by amp names containing the mask produced after
73 outlier rejection. The mask produced by the "FULLCOVARIANCE"
74 option may differ from the one produced in the other two PTC
76 rawExpTimes : `dict`, [`str`, `list`]
77 Dictionary keyed by amp names containing the unmasked exposure times.
78 rawMeans : `dict`, [`str`, `list`]
79 Dictionary keyed by amp namescontaining the unmasked average of the
80 means of the exposures in each flat pair.
81 rawVars : `dict`, [`str`, `list`]
82 Dictionary keyed by amp names containing the variance of the
83 difference image of the exposures in each flat pair.
84 gain : `dict`, [`str`, `list`]
85 Dictionary keyed by amp names containing the fitted gains.
86 gainErr : `dict`, [`str`, `list`]
87 Dictionary keyed by amp names containing the errors on the
89 noise : `dict`, [`str`, `list`]
90 Dictionary keyed by amp names containing the fitted noise.
91 noiseErr : `dict`, [`str`, `list`]
92 Dictionary keyed by amp names containing the errors on the fitted noise.
93 ptcFitPars : `dict`, [`str`, `list`]
94 Dictionary keyed by amp names containing the fitted parameters of the
95 PTC model for ptcFitTye in ["POLYNOMIAL", "EXPAPPROXIMATION"].
96 ptcFitParsError : `dict`, [`str`, `list`]
97 Dictionary keyed by amp names containing the errors on the fitted
98 parameters of the PTC model for ptcFitTye in
99 ["POLYNOMIAL", "EXPAPPROXIMATION"].
100 ptcFitChiSq : `dict`, [`str`, `list`]
101 Dictionary keyed by amp names containing the reduced chi squared
102 of the fit for ptcFitTye in ["POLYNOMIAL", "EXPAPPROXIMATION"].
103 covariances : `dict`, [`str`, `list`]
104 Dictionary keyed by amp names containing a list of measured
105 covariances per mean flux.
106 covariancesModel : `dict`, [`str`, `list`]
107 Dictionary keyed by amp names containinging covariances model
108 (Eq. 20 of Astier+19) per mean flux.
109 covariancesSqrtWeights : `dict`, [`str`, `list`]
110 Dictionary keyed by amp names containinging sqrt. of covariances
112 aMatrix : `dict`, [`str`, `list`]
113 Dictionary keyed by amp names containing the "a" parameters from
114 the model in Eq. 20 of Astier+19.
115 bMatrix : `dict`, [`str`, `list`]
116 Dictionary keyed by amp names containing the "b" parameters from
117 the model in Eq. 20 of Astier+19.
118 covariancesModelNoB : `dict`, [`str`, `list`]
119 Dictionary keyed by amp names containing covariances model
120 (with 'b'=0 in Eq. 20 of Astier+19)
122 aMatrixNoB : `dict`, [`str`, `list`]
123 Dictionary keyed by amp names containing the "a" parameters from the
124 model in Eq. 20 of Astier+19
126 finalVars : `dict`, [`str`, `list`]
127 Dictionary keyed by amp names containing the masked variance of the
128 difference image of each flat
129 pair. If needed, each array will be right-padded with
130 np.nan to match the length of rawExpTimes.
131 finalModelVars : `dict`, [`str`, `list`]
132 Dictionary keyed by amp names containing the masked modeled
133 variance of the difference image of each flat pair. If needed, each
134 array will be right-padded with np.nan to match the length of
136 finalMeans : `dict`, [`str`, `list`]
137 Dictionary keyed by amp names containing the masked average of the
138 means of the exposures in each flat pair. If needed, each array
139 will be right-padded with np.nan to match the length of
141 photoCharge : `dict`, [`str`, `list`]
142 Dictionary keyed by amp names containing the integrated photocharge
143 for linearity calibration.
147 `lsst.cp.pipe.ptc.PhotonTransferCurveDataset`
148 Output dataset from MeasurePhotonTransferCurveTask.
152 _SCHEMA =
'Gen3 Photon Transfer Curve'
155 def __init__(self, ampNames=[], ptcFitType=None, covMatrixSide=1, **kwargs):
164 self.
expIdMaskexpIdMask = {ampName: []
for ampName
in ampNames}
165 self.
rawExpTimesrawExpTimes = {ampName: []
for ampName
in ampNames}
166 self.
rawMeansrawMeans = {ampName: []
for ampName
in ampNames}
167 self.
rawVarsrawVars = {ampName: []
for ampName
in ampNames}
168 self.
photoChargephotoCharge = {ampName: []
for ampName
in ampNames}
170 self.
gaingain = {ampName: np.nan
for ampName
in ampNames}
171 self.
gainErrgainErr = {ampName: np.nan
for ampName
in ampNames}
172 self.
noisenoise = {ampName: np.nan
for ampName
in ampNames}
173 self.
noiseErrnoiseErr = {ampName: np.nan
for ampName
in ampNames}
175 self.
ptcFitParsptcFitPars = {ampName: []
for ampName
in ampNames}
177 self.
ptcFitChiSqptcFitChiSq = {ampName: np.nan
for ampName
in ampNames}
179 self.
covariancescovariances = {ampName: []
for ampName
in ampNames}
182 self.
aMatrixaMatrix = {ampName: np.nan
for ampName
in ampNames}
183 self.
bMatrixbMatrix = {ampName: np.nan
for ampName
in ampNames}
185 self.
aMatrixNoBaMatrixNoB = {ampName: np.nan
for ampName
in ampNames}
187 self.
finalVarsfinalVars = {ampName: []
for ampName
in ampNames}
189 self.
finalMeansfinalMeans = {ampName: []
for ampName
in ampNames}
193 'rawMeans',
'rawVars',
'gain',
'gainErr',
'noise',
'noiseErr',
194 'ptcFitPars',
'ptcFitParsError',
'ptcFitChiSq',
'aMatrixNoB',
195 'covariances',
'covariancesModel',
'covariancesSqrtWeights',
196 'covariancesModelNoB',
197 'aMatrix',
'bMatrix',
'finalVars',
'finalModelVars',
'finalMeans',
201 """Calibration equivalence
203 if not isinstance(other, self.__class__):
207 attrSelf = getattr(self, attr)
208 attrOther = getattr(other, attr)
209 if isinstance(attrSelf, dict)
and isinstance(attrOther, dict):
210 for ampName
in attrSelf:
211 if not np.allclose(attrSelf[ampName], attrOther[ampName], equal_nan=
True):
214 if attrSelf != attrOther:
219 """Update calibration metadata.
220 This calls the base class's method after ensuring the required
221 calibration keywords will be saved.
224 setDate : `bool`, optional
225 Update the CALIBDATE fields in the metadata to the current
226 time. Defaults to False.
228 Other keyword parameters to set in the metadata.
230 kwargs[
'PTC_FIT_TYPE'] = self.
ptcFitTypeptcFitType
236 """Construct a calibration from a dictionary of properties.
237 Must be implemented by the specific calibration subclasses.
241 Dictionary of properties.
244 calib : `lsst.ip.isr.CalibType`
245 Constructed calibration.
249 Raised if the supplied dictionary is for a different
253 if calib._OBSTYPE != dictionary[
'metadata'][
'OBSTYPE']:
254 raise RuntimeError(f
"Incorrect Photon Transfer Curve dataset supplied. "
255 f
"Expected {calib._OBSTYPE}, found {dictionary['metadata']['OBSTYPE']}")
256 calib.setMetadata(dictionary[
'metadata'])
257 calib.ptcFitType = dictionary[
'ptcFitType']
258 calib.covMatrixSide = dictionary[
'covMatrixSide']
259 calib.badAmps = np.array(dictionary[
'badAmps'],
'str').tolist()
261 covMatrixSide = calib.covMatrixSide
263 covDimensionsProduct = len(np.array(list(dictionary[
'covariances'].values())[0]).ravel())
264 nSignalPoints = int(covDimensionsProduct/(covMatrixSide*covMatrixSide))
266 for ampName
in dictionary[
'ampNames']:
267 calib.ampNames.append(ampName)
268 calib.inputExpIdPairs[ampName] = np.array(dictionary[
'inputExpIdPairs'][ampName]).tolist()
269 calib.expIdMask[ampName] = np.array(dictionary[
'expIdMask'][ampName]).tolist()
270 calib.rawExpTimes[ampName] = np.array(dictionary[
'rawExpTimes'][ampName]).tolist()
271 calib.rawMeans[ampName] = np.array(dictionary[
'rawMeans'][ampName]).tolist()
272 calib.rawVars[ampName] = np.array(dictionary[
'rawVars'][ampName]).tolist()
273 calib.gain[ampName] = np.array(dictionary[
'gain'][ampName]).tolist()
274 calib.gainErr[ampName] = np.array(dictionary[
'gainErr'][ampName]).tolist()
275 calib.noise[ampName] = np.array(dictionary[
'noise'][ampName]).tolist()
276 calib.noiseErr[ampName] = np.array(dictionary[
'noiseErr'][ampName]).tolist()
277 calib.ptcFitPars[ampName] = np.array(dictionary[
'ptcFitPars'][ampName]).tolist()
278 calib.ptcFitParsError[ampName] = np.array(dictionary[
'ptcFitParsError'][ampName]).tolist()
279 calib.ptcFitChiSq[ampName] = np.array(dictionary[
'ptcFitChiSq'][ampName]).tolist()
280 calib.covariances[ampName] = np.array(dictionary[
'covariances'][ampName]).reshape(
281 (nSignalPoints, covMatrixSide, covMatrixSide)).tolist()
282 calib.covariancesModel[ampName] = np.array(
283 dictionary[
'covariancesModel'][ampName]).reshape(
284 (nSignalPoints, covMatrixSide, covMatrixSide)).tolist()
285 calib.covariancesSqrtWeights[ampName] = np.array(
286 dictionary[
'covariancesSqrtWeights'][ampName]).reshape(
287 (nSignalPoints, covMatrixSide, covMatrixSide)).tolist()
288 calib.aMatrix[ampName] = np.array(dictionary[
'aMatrix'][ampName]).reshape(
289 (covMatrixSide, covMatrixSide)).tolist()
290 calib.bMatrix[ampName] = np.array(dictionary[
'bMatrix'][ampName]).reshape(
291 (covMatrixSide, covMatrixSide)).tolist()
292 calib.covariancesModelNoB[ampName] = np.array(
293 dictionary[
'covariancesModelNoB'][ampName]).reshape(
294 (nSignalPoints, covMatrixSide, covMatrixSide)).tolist()
295 calib.aMatrixNoB[ampName] = np.array(
296 dictionary[
'aMatrixNoB'][ampName]).reshape((covMatrixSide, covMatrixSide)).tolist()
297 calib.finalVars[ampName] = np.array(dictionary[
'finalVars'][ampName]).tolist()
298 calib.finalModelVars[ampName] = np.array(dictionary[
'finalModelVars'][ampName]).tolist()
299 calib.finalMeans[ampName] = np.array(dictionary[
'finalMeans'][ampName]).tolist()
300 calib.photoCharge[ampName] = np.array(dictionary[
'photoCharge'][ampName]).tolist()
301 calib.updateMetadata()
305 """Return a dictionary containing the calibration properties.
306 The dictionary should be able to be round-tripped through
311 Dictionary of properties.
317 outDict[
'metadata'] = metadata
319 outDict[
'ptcFitType'] = self.
ptcFitTypeptcFitType
321 outDict[
'ampNames'] = self.
ampNamesampNames
322 outDict[
'badAmps'] = self.
badAmpsbadAmps
324 outDict[
'expIdMask'] = self.
expIdMaskexpIdMask
325 outDict[
'rawExpTimes'] = self.
rawExpTimesrawExpTimes
326 outDict[
'rawMeans'] = self.
rawMeansrawMeans
327 outDict[
'rawVars'] = self.
rawVarsrawVars
328 outDict[
'gain'] = self.
gaingain
329 outDict[
'gainErr'] = self.
gainErrgainErr
330 outDict[
'noise'] = self.
noisenoise
331 outDict[
'noiseErr'] = self.
noiseErrnoiseErr
332 outDict[
'ptcFitPars'] = self.
ptcFitParsptcFitPars
334 outDict[
'ptcFitChiSq'] = self.
ptcFitChiSqptcFitChiSq
335 outDict[
'covariances'] = self.
covariancescovariances
338 outDict[
'aMatrix'] = self.
aMatrixaMatrix
339 outDict[
'bMatrix'] = self.
bMatrixbMatrix
341 outDict[
'aMatrixNoB'] = self.
aMatrixNoBaMatrixNoB
342 outDict[
'finalVars'] = self.
finalVarsfinalVars
344 outDict[
'finalMeans'] = self.
finalMeansfinalMeans
345 outDict[
'photoCharge'] = self.
photoChargephotoCharge
351 """Construct calibration from a list of tables.
352 This method uses the `fromDict` method to create the
353 calibration, after constructing an appropriate dictionary from
357 tableList : `list` [`lsst.afw.table.Table`]
358 List of tables to use to construct the datasetPtc.
361 calib : `lsst.cp.pipe.`
362 The calibration defined in the tables.
364 ptcTable = tableList[0]
366 metadata = ptcTable.meta
368 inDict[
'metadata'] = metadata
369 inDict[
'ampNames'] = []
370 inDict[
'ptcFitType'] = []
371 inDict[
'covMatrixSide'] = []
372 inDict[
'inputExpIdPairs'] = dict()
373 inDict[
'expIdMask'] = dict()
374 inDict[
'rawExpTimes'] = dict()
375 inDict[
'rawMeans'] = dict()
376 inDict[
'rawVars'] = dict()
377 inDict[
'gain'] = dict()
378 inDict[
'gainErr'] = dict()
379 inDict[
'noise'] = dict()
380 inDict[
'noiseErr'] = dict()
381 inDict[
'ptcFitPars'] = dict()
382 inDict[
'ptcFitParsError'] = dict()
383 inDict[
'ptcFitChiSq'] = dict()
384 inDict[
'covariances'] = dict()
385 inDict[
'covariancesModel'] = dict()
386 inDict[
'covariancesSqrtWeights'] = dict()
387 inDict[
'aMatrix'] = dict()
388 inDict[
'bMatrix'] = dict()
389 inDict[
'covariancesModelNoB'] = dict()
390 inDict[
'aMatrixNoB'] = dict()
391 inDict[
'finalVars'] = dict()
392 inDict[
'finalModelVars'] = dict()
393 inDict[
'finalMeans'] = dict()
394 inDict[
'badAmps'] = []
395 inDict[
'photoCharge'] = dict()
397 for record
in ptcTable:
398 ampName = record[
'AMPLIFIER_NAME']
400 inDict[
'ptcFitType'] = record[
'PTC_FIT_TYPE']
401 inDict[
'covMatrixSide'] = record[
'COV_MATRIX_SIDE']
402 inDict[
'ampNames'].append(ampName)
403 inDict[
'inputExpIdPairs'][ampName] = record[
'INPUT_EXP_ID_PAIRS']
404 inDict[
'expIdMask'][ampName] = record[
'EXP_ID_MASK']
405 inDict[
'rawExpTimes'][ampName] = record[
'RAW_EXP_TIMES']
406 inDict[
'rawMeans'][ampName] = record[
'RAW_MEANS']
407 inDict[
'rawVars'][ampName] = record[
'RAW_VARS']
408 inDict[
'gain'][ampName] = record[
'GAIN']
409 inDict[
'gainErr'][ampName] = record[
'GAIN_ERR']
410 inDict[
'noise'][ampName] = record[
'NOISE']
411 inDict[
'noiseErr'][ampName] = record[
'NOISE_ERR']
412 inDict[
'ptcFitPars'][ampName] = record[
'PTC_FIT_PARS']
413 inDict[
'ptcFitParsError'][ampName] = record[
'PTC_FIT_PARS_ERROR']
414 inDict[
'ptcFitChiSq'][ampName] = record[
'PTC_FIT_CHI_SQ']
415 inDict[
'covariances'][ampName] = record[
'COVARIANCES']
416 inDict[
'covariancesModel'][ampName] = record[
'COVARIANCES_MODEL']
417 inDict[
'covariancesSqrtWeights'][ampName] = record[
'COVARIANCES_SQRT_WEIGHTS']
418 inDict[
'aMatrix'][ampName] = record[
'A_MATRIX']
419 inDict[
'bMatrix'][ampName] = record[
'B_MATRIX']
420 inDict[
'covariancesModelNoB'][ampName] = record[
'COVARIANCES_MODEL_NO_B']
421 inDict[
'aMatrixNoB'][ampName] = record[
'A_MATRIX_NO_B']
422 inDict[
'finalVars'][ampName] = record[
'FINAL_VARS']
423 inDict[
'finalModelVars'][ampName] = record[
'FINAL_MODEL_VARS']
424 inDict[
'finalMeans'][ampName] = record[
'FINAL_MEANS']
425 inDict[
'badAmps'] = record[
'BAD_AMPS']
426 inDict[
'photoCharge'][ampName] = record[
'PHOTO_CHARGE']
430 """Construct a list of tables containing the information in this
433 The list of tables should create an identical calibration
434 after being passed to this class's fromTable method.
437 tableList : `list` [`astropy.table.Table`]
438 List of tables containing the linearity calibration
444 for i, ampName
in enumerate(self.
ampNamesampNames):
445 nPoints.append(len(list(self.
covariancescovariances.values())[i]))
446 nSignalPoints = max(nPoints)
448 for i, ampName
in enumerate(self.
ampNamesampNames):
449 nPadPoints[ampName] = nSignalPoints - len(list(self.
covariancescovariances.values())[i])
452 catalog = Table([{
'AMPLIFIER_NAME': ampName,
456 if len(self.
expIdMaskexpIdMask[ampName])
else np.nan,
457 'EXP_ID_MASK': self.
expIdMaskexpIdMask[ampName]
458 if len(self.
expIdMaskexpIdMask[ampName])
else np.nan,
459 'RAW_EXP_TIMES': np.array(self.
rawExpTimesrawExpTimes[ampName]).tolist()
460 if len(self.
rawExpTimesrawExpTimes[ampName])
else np.nan,
461 'RAW_MEANS': np.array(self.
rawMeansrawMeans[ampName]).tolist()
462 if len(self.
rawMeansrawMeans[ampName])
else np.nan,
463 'RAW_VARS': np.array(self.
rawVarsrawVars[ampName]).tolist()
464 if len(self.
rawVarsrawVars[ampName])
else np.nan,
465 'GAIN': self.
gaingain[ampName],
466 'GAIN_ERR': self.
gainErrgainErr[ampName],
467 'NOISE': self.
noisenoise[ampName],
468 'NOISE_ERR': self.
noiseErrnoiseErr[ampName],
469 'PTC_FIT_PARS': np.array(self.
ptcFitParsptcFitPars[ampName]).tolist(),
470 'PTC_FIT_PARS_ERROR': np.array(self.
ptcFitParsErrorptcFitParsError[ampName]).tolist(),
471 'PTC_FIT_CHI_SQ': self.
ptcFitChiSqptcFitChiSq[ampName],
472 'COVARIANCES': np.pad(np.array(self.
covariancescovariances[ampName]),
473 ((0, nPadPoints[ampName]), (0, 0), (0, 0)),
474 'constant', constant_values=np.nan).reshape(
475 nSignalPoints*covMatrixSide**2).tolist(),
476 'COVARIANCES_MODEL': np.pad(np.array(self.
covariancesModelcovariancesModel[ampName]),
477 ((0, nPadPoints[ampName]), (0, 0), (0, 0)),
478 'constant', constant_values=np.nan).reshape(
479 nSignalPoints*covMatrixSide**2).tolist(),
481 ((0, nPadPoints[ampName]), (0, 0), (0, 0)),
482 'constant', constant_values=0.0).reshape(
483 nSignalPoints*covMatrixSide**2).tolist(),
484 'A_MATRIX': np.array(self.
aMatrixaMatrix[ampName]).reshape(covMatrixSide**2).tolist(),
485 'B_MATRIX': np.array(self.
bMatrixbMatrix[ampName]).reshape(covMatrixSide**2).tolist(),
486 'COVARIANCES_MODEL_NO_B':
488 ((0, nPadPoints[ampName]), (0, 0), (0, 0)),
489 'constant', constant_values=np.nan).reshape(
490 nSignalPoints*covMatrixSide**2).tolist(),
491 'A_MATRIX_NO_B': np.array(self.
aMatrixNoBaMatrixNoB[ampName]).reshape(
492 covMatrixSide**2).tolist(),
493 'FINAL_VARS': np.pad(np.array(self.
finalVarsfinalVars[ampName]), (0, nPadPoints[ampName]),
494 'constant', constant_values=np.nan).tolist(),
495 'FINAL_MODEL_VARS': np.pad(np.array(self.
finalModelVarsfinalModelVars[ampName]),
496 (0, nPadPoints[ampName]),
497 'constant', constant_values=np.nan).tolist(),
498 'FINAL_MEANS': np.pad(np.array(self.
finalMeansfinalMeans[ampName]),
499 (0, nPadPoints[ampName]),
500 'constant', constant_values=np.nan).tolist(),
501 'BAD_AMPS': np.array(self.
badAmpsbadAmps).tolist()
if len(self.
badAmpsbadAmps)
else np.nan,
502 'PHOTO_CHARGE': np.array(self.
photoChargephotoCharge[ampName]).tolist(),
503 }
for ampName
in self.
ampNamesampNames])
505 outMeta = {k: v
for k, v
in inMeta.items()
if v
is not None}
506 outMeta.update({k:
"" for k, v
in inMeta.items()
if v
is None})
507 catalog.meta = outMeta
508 tableList.append(catalog)
513 """Get the exposures used, i.e. not discarded, for a given amp.
514 If no mask has been created yet, all exposures are returned.
516 if len(self.
expIdMaskexpIdMask[ampName]) == 0:
525 return [(exp1, exp2)
for ((exp1, exp2), m)
in zip(pairs, mask)
if bool(m)
is True]
528 return [amp
for amp
in self.
ampNamesampNames
if amp
not in self.
badAmpsbadAmps]
def requiredAttributes(self, value)
def updateMetadata(self, camera=None, detector=None, filterName=None, setCalibId=False, setCalibInfo=False, setDate=False, **kwargs)
def requiredAttributes(self)
def __init__(self, ampNames=[], ptcFitType=None, covMatrixSide=1, **kwargs)
def getExpIdsUsed(self, ampName)
def fromDict(cls, dictionary)
def fromTable(cls, tableList)
def updateMetadata(self, setDate=False, **kwargs)