23 Define dataset class for MeasurePhotonTransferCurve task
26 from astropy.table
import Table
30 __all__ = [
'PhotonTransferCurveDataset']
34 """A simple class to hold the output data from the PTC task.
35 The dataset is made up of a dictionary for each item, keyed by the
36 amplifiers' names, which much be supplied at construction time.
37 New items cannot be added to the class to save accidentally saving to the
38 wrong property, and the class can be frozen if desired.
39 inputExpIdPairs records the exposures used to produce the data.
40 When fitPtc() or fitCovariancesAstier() is run, a mask is built up, which
41 is by definition always the same length as inputExpIdPairs, rawExpTimes,
42 rawMeans and rawVars, and is a list of bools, which are incrementally set
43 to False as points are discarded from the fits.
44 PTC fit parameters for polynomials are stored in a list in ascending order
45 of polynomial term, i.e. par[0]*x^0 + par[1]*x + par[2]*x^2 etc
46 with the length of the list corresponding to the order of the polynomial
51 List with the names of the amplifiers of the detector at hand.
53 Type of model fitted to the PTC: "POLYNOMIAL", "EXPAPPROXIMATION",
55 kwargs : `dict`, optional
56 Other keyword arguments to pass to the parent init.
59 The stored attributes are:
61 List with bad amplifiers names.
62 inputExpIdPairs : `dict`, [`str`, `list`]
63 Dictionary keyed by amp names containing the input exposures IDs.
64 expIdMask : `dict`, [`str`, `list`]
65 Dictionary keyed by amp names containing the mask produced after
66 outlier rejection. The mask produced by the "FULLCOVARIANCE"
67 option may differ from the one produced in the other two PTC
69 rawExpTimes : `dict`, [`str`, `list`]
70 Dictionary keyed by amp names containing the unmasked exposure times.
71 rawMeans : `dict`, [`str`, `list`]
72 Dictionary keyed by amp namescontaining the unmasked average of the
73 means of the exposures in each flat pair.
74 rawVars : `dict`, [`str`, `list`]
75 Dictionary keyed by amp names containing the variance of the
76 difference image of the exposures in each flat pair.
77 gain : `dict`, [`str`, `list`]
78 Dictionary keyed by amp names containing the fitted gains.
79 gainErr : `dict`, [`str`, `list`]
80 Dictionary keyed by amp names containing the errors on the
82 noise : `dict`, [`str`, `list`]
83 Dictionary keyed by amp names containing the fitted noise.
84 noiseErr : `dict`, [`str`, `list`]
85 Dictionary keyed by amp names containing the errors on the fitted noise.
86 ptcFitPars : `dict`, [`str`, `list`]
87 Dictionary keyed by amp names containing the fitted parameters of the
88 PTC model for ptcFitTye in ["POLYNOMIAL", "EXPAPPROXIMATION"].
89 ptcFitParsError : `dict`, [`str`, `list`]
90 Dictionary keyed by amp names containing the errors on the fitted
91 parameters of the PTC model for ptcFitTye in
92 ["POLYNOMIAL", "EXPAPPROXIMATION"].
93 ptcFitChiSq : `dict`, [`str`, `list`]
94 Dictionary keyed by amp names containing the reduced chi squared
95 of the fit for ptcFitTye in ["POLYNOMIAL", "EXPAPPROXIMATION"].
96 covariances : `dict`, [`str`, `list`]
97 Dictionary keyed by amp names containing a list of measured
98 covariances per mean flux.
99 covariancesModel : `dict`, [`str`, `list`]
100 Dictionary keyed by amp names containinging covariances model
101 (Eq. 20 of Astier+19) per mean flux.
102 covariancesSqrtWeights : `dict`, [`str`, `list`]
103 Dictionary keyed by amp names containinging sqrt. of covariances
105 aMatrix : `dict`, [`str`, `list`]
106 Dictionary keyed by amp names containing the "a" parameters from
107 the model in Eq. 20 of Astier+19.
108 bMatrix : `dict`, [`str`, `list`]
109 Dictionary keyed by amp names containing the "b" parameters from
110 the model in Eq. 20 of Astier+19.
111 covariancesNoB : `dict`, [`str`, `list`]
112 Dictionary keyed by amp names containing a list of measured
113 covariances per mean flux ('b'=0 in
115 covariancesModelNoB : `dict`, [`str`, `list`]
116 Dictionary keyed by amp names containing covariances model
117 (with 'b'=0 in Eq. 20 of Astier+19)
119 covariancesSqrtWeightsNoB : `dict`, [`str`, `list`]
120 Dictionary keyed by amp names containing sqrt. of covariances weights
121 ('b' = 0 in Eq. 20 of
123 aMatrixNoB : `dict`, [`str`, `list`]
124 Dictionary keyed by amp names containing the "a" parameters from the
125 model in Eq. 20 of Astier+19
127 finalVars : `dict`, [`str`, `list`]
128 Dictionary keyed by amp names containing the masked variance of the
129 difference image of each flat
130 pair. If needed, each array will be right-padded with
131 np.nan to match the length of rawExpTimes.
132 finalModelVars : `dict`, [`str`, `list`]
133 Dictionary keyed by amp names containing the masked modeled
134 variance of the difference image of each flat pair. If needed, each
135 array will be right-padded with np.nan to match the length of
137 finalMeans : `dict`, [`str`, `list`]
138 Dictionary keyed by amp names containing the masked average of the
139 means of the exposures in each flat pair. If needed, each array
140 will be right-padded with np.nan to match the length of
142 photoCharge : `dict`, [`str`, `list`]
143 Dictionary keyed by amp names containing the integrated photocharge
144 for linearity calibration.
148 `lsst.cp.pipe.ptc.PhotonTransferCurveDataset`
149 Output dataset from MeasurePhotonTransferCurveTask.
153 _SCHEMA =
'Gen3 Photon Transfer Curve'
156 def __init__(self, ampNames=[], ptcFitType=None, **kwargs):
165 self.
rawMeans = {ampName: []
for ampName
in ampNames}
166 self.
rawVars = {ampName: []
for ampName
in ampNames}
169 self.
gain = {ampName: -1.
for ampName
in ampNames}
170 self.
gainErr = {ampName: -1.
for ampName
in ampNames}
171 self.
noise = {ampName: -1.
for ampName
in ampNames}
172 self.
noiseErr = {ampName: -1.
for ampName
in ampNames}
181 self.
aMatrix = {ampName: []
for ampName
in ampNames}
182 self.
bMatrix = {ampName: []
for ampName
in ampNames}
193 self.
requiredAttributes.update([
'badAmps',
'inputExpIdPairs',
'expIdMask',
'rawExpTimes',
194 'rawMeans',
'rawVars',
'gain',
'gainErr',
'noise',
'noiseErr',
195 'ptcFitPars',
'ptcFitParsError',
'ptcFitChiSq',
'aMatrixNoB',
196 'covariances',
'covariancesModel',
'covariancesSqrtWeights',
197 'covariancesNoB',
'covariancesModelNoB',
'covariancesSqrtWeightsNoB',
198 'aMatrix',
'bMatrix',
'finalVars',
'finalModelVars',
'finalMeans',
202 """Calibration equivalence
204 if not isinstance(other, self.__class__):
208 attrSelf = getattr(self, attr)
209 attrOther = getattr(other, attr)
210 if isinstance(attrSelf, dict)
and isinstance(attrOther, dict):
211 for ampName
in attrSelf:
212 if not np.allclose(attrSelf[ampName], attrOther[ampName], equal_nan=
True):
215 if attrSelf != attrOther:
220 """Update calibration metadata.
221 This calls the base class's method after ensuring the required
222 calibration keywords will be saved.
225 setDate : `bool`, optional
226 Update the CALIBDATE fields in the metadata to the current
227 time. Defaults to False.
229 Other keyword parameters to set in the metadata.
237 """Construct a calibration from a dictionary of properties.
238 Must be implemented by the specific calibration subclasses.
242 Dictionary of properties.
245 calib : `lsst.ip.isr.CalibType`
246 Constructed calibration.
250 Raised if the supplied dictionary is for a different
254 if calib._OBSTYPE != dictionary[
'metadata'][
'OBSTYPE']:
255 raise RuntimeError(f
"Incorrect Photon Transfer Curve dataset supplied. "
256 f
"Expected {calib._OBSTYPE}, found {dictionary['metadata']['OBSTYPE']}")
257 calib.setMetadata(dictionary[
'metadata'])
258 calib.ptcFitType = dictionary[
'ptcFitType']
259 calib.badAmps = np.array(dictionary[
'badAmps'],
'str').tolist()
261 nSignalPoints = len(list(dictionary[
'rawMeans'].values())[0])
263 covMatrixSide = int(np.sqrt(len(np.array(list(dictionary[
'aMatrix'].values())[0]).ravel())))
264 for ampName
in dictionary[
'ampNames']:
265 calib.ampNames.append(ampName)
266 calib.inputExpIdPairs[ampName] = np.array(dictionary[
'inputExpIdPairs'][ampName]).tolist()
267 calib.expIdMask[ampName] = np.array(dictionary[
'expIdMask'][ampName]).tolist()
268 calib.rawExpTimes[ampName] = np.array(dictionary[
'rawExpTimes'][ampName]).tolist()
269 calib.rawMeans[ampName] = np.array(dictionary[
'rawMeans'][ampName]).tolist()
270 calib.rawVars[ampName] = np.array(dictionary[
'rawVars'][ampName]).tolist()
271 calib.gain[ampName] = np.array(dictionary[
'gain'][ampName]).tolist()
272 calib.gainErr[ampName] = np.array(dictionary[
'gainErr'][ampName]).tolist()
273 calib.noise[ampName] = np.array(dictionary[
'noise'][ampName]).tolist()
274 calib.noiseErr[ampName] = np.array(dictionary[
'noiseErr'][ampName]).tolist()
275 calib.ptcFitPars[ampName] = np.array(dictionary[
'ptcFitPars'][ampName]).tolist()
276 calib.ptcFitParsError[ampName] = np.array(dictionary[
'ptcFitParsError'][ampName]).tolist()
277 calib.ptcFitChiSq[ampName] = np.array(dictionary[
'ptcFitChiSq'][ampName]).tolist()
278 calib.covariances[ampName] = np.array(dictionary[
'covariances'][ampName]).reshape(
279 (nSignalPoints, covMatrixSide, covMatrixSide)).tolist()
280 calib.covariancesModel[ampName] = np.array(
281 dictionary[
'covariancesModel'][ampName]).reshape(
282 (nSignalPoints, covMatrixSide, covMatrixSide)).tolist()
283 calib.covariancesSqrtWeights[ampName] = np.array(
284 dictionary[
'covariancesSqrtWeights'][ampName]).reshape(
285 (nSignalPoints, covMatrixSide, covMatrixSide)).tolist()
286 calib.aMatrix[ampName] = np.array(dictionary[
'aMatrix'][ampName]).reshape(
287 (covMatrixSide, covMatrixSide)).tolist()
288 calib.bMatrix[ampName] = np.array(dictionary[
'bMatrix'][ampName]).reshape(
289 (covMatrixSide, covMatrixSide)).tolist()
290 calib.covariancesNoB[ampName] = np.array(
291 dictionary[
'covariancesNoB'][ampName]).reshape(
292 (nSignalPoints, covMatrixSide, covMatrixSide)).tolist()
293 calib.covariancesModelNoB[ampName] = np.array(
294 dictionary[
'covariancesModelNoB'][ampName]).reshape(
295 (nSignalPoints, covMatrixSide, covMatrixSide)).tolist()
296 calib.covariancesSqrtWeightsNoB[ampName] = np.array(
297 dictionary[
'covariancesSqrtWeightsNoB'][ampName]).reshape(
298 (nSignalPoints, covMatrixSide, covMatrixSide)).tolist()
299 calib.aMatrixNoB[ampName] = np.array(
300 dictionary[
'aMatrixNoB'][ampName]).reshape((covMatrixSide, covMatrixSide)).tolist()
301 calib.finalVars[ampName] = np.array(dictionary[
'finalVars'][ampName]).tolist()
302 calib.finalModelVars[ampName] = np.array(dictionary[
'finalModelVars'][ampName]).tolist()
303 calib.finalMeans[ampName] = np.array(dictionary[
'finalMeans'][ampName]).tolist()
304 calib.photoCharge[ampName] = np.array(dictionary[
'photoCharge'][ampName]).tolist()
305 calib.updateMetadata()
309 """Return a dictionary containing the calibration properties.
310 The dictionary should be able to be round-tripped through
315 Dictionary of properties.
321 outDict[
'metadata'] = metadata
325 outDict[
'badAmps'] = self.
badAmps
330 outDict[
'rawVars'] = self.
rawVars
331 outDict[
'gain'] = self.
gain
332 outDict[
'gainErr'] = self.
gainErr
333 outDict[
'noise'] = self.
noise
341 outDict[
'aMatrix'] = self.
aMatrix
342 outDict[
'bMatrix'] = self.
bMatrix
356 """Construct calibration from a list of tables.
357 This method uses the `fromDict` method to create the
358 calibration, after constructing an appropriate dictionary from
362 tableList : `list` [`lsst.afw.table.Table`]
363 List of tables to use to construct the datasetPtc.
366 calib : `lsst.cp.pipe.`
367 The calibration defined in the tables.
369 ptcTable = tableList[0]
371 metadata = ptcTable.meta
373 inDict[
'metadata'] = metadata
374 inDict[
'ampNames'] = []
375 inDict[
'ptcFitType'] = []
376 inDict[
'inputExpIdPairs'] = dict()
377 inDict[
'expIdMask'] = dict()
378 inDict[
'rawExpTimes'] = dict()
379 inDict[
'rawMeans'] = dict()
380 inDict[
'rawVars'] = dict()
381 inDict[
'gain'] = dict()
382 inDict[
'gainErr'] = dict()
383 inDict[
'noise'] = dict()
384 inDict[
'noiseErr'] = dict()
385 inDict[
'ptcFitPars'] = dict()
386 inDict[
'ptcFitParsError'] = dict()
387 inDict[
'ptcFitChiSq'] = dict()
388 inDict[
'covariances'] = dict()
389 inDict[
'covariancesModel'] = dict()
390 inDict[
'covariancesSqrtWeights'] = dict()
391 inDict[
'aMatrix'] = dict()
392 inDict[
'bMatrix'] = dict()
393 inDict[
'covariancesNoB'] = dict()
394 inDict[
'covariancesModelNoB'] = dict()
395 inDict[
'covariancesSqrtWeightsNoB'] = dict()
396 inDict[
'aMatrixNoB'] = dict()
397 inDict[
'finalVars'] = dict()
398 inDict[
'finalModelVars'] = dict()
399 inDict[
'finalMeans'] = dict()
400 inDict[
'badAmps'] = []
401 inDict[
'photoCharge'] = dict()
403 for record
in ptcTable:
404 ampName = record[
'AMPLIFIER_NAME']
406 inDict[
'ptcFitType'] = record[
'PTC_FIT_TYPE']
407 inDict[
'ampNames'].append(ampName)
408 inDict[
'inputExpIdPairs'][ampName] = record[
'INPUT_EXP_ID_PAIRS']
409 inDict[
'expIdMask'][ampName] = record[
'EXP_ID_MASK']
410 inDict[
'rawExpTimes'][ampName] = record[
'RAW_EXP_TIMES']
411 inDict[
'rawMeans'][ampName] = record[
'RAW_MEANS']
412 inDict[
'rawVars'][ampName] = record[
'RAW_VARS']
413 inDict[
'gain'][ampName] = record[
'GAIN']
414 inDict[
'gainErr'][ampName] = record[
'GAIN_ERR']
415 inDict[
'noise'][ampName] = record[
'NOISE']
416 inDict[
'noiseErr'][ampName] = record[
'NOISE_ERR']
417 inDict[
'ptcFitPars'][ampName] = record[
'PTC_FIT_PARS']
418 inDict[
'ptcFitParsError'][ampName] = record[
'PTC_FIT_PARS_ERROR']
419 inDict[
'ptcFitChiSq'][ampName] = record[
'PTC_FIT_CHI_SQ']
420 inDict[
'covariances'][ampName] = record[
'COVARIANCES']
421 inDict[
'covariancesModel'][ampName] = record[
'COVARIANCES_MODEL']
422 inDict[
'covariancesSqrtWeights'][ampName] = record[
'COVARIANCES_SQRT_WEIGHTS']
423 inDict[
'aMatrix'][ampName] = record[
'A_MATRIX']
424 inDict[
'bMatrix'][ampName] = record[
'B_MATRIX']
425 inDict[
'covariancesNoB'][ampName] = record[
'COVARIANCES_NO_B']
426 inDict[
'covariancesModelNoB'][ampName] = record[
'COVARIANCES_MODEL_NO_B']
427 inDict[
'covariancesSqrtWeightsNoB'][ampName] = record[
'COVARIANCES_SQRT_WEIGHTS_NO_B']
428 inDict[
'aMatrixNoB'][ampName] = record[
'A_MATRIX_NO_B']
429 inDict[
'finalVars'][ampName] = record[
'FINAL_VARS']
430 inDict[
'finalModelVars'][ampName] = record[
'FINAL_MODEL_VARS']
431 inDict[
'finalMeans'][ampName] = record[
'FINAL_MEANS']
432 inDict[
'badAmps'] = record[
'BAD_AMPS']
433 inDict[
'photoCharge'][ampName] = record[
'PHOTO_CHARGE']
437 """Construct a list of tables containing the information in this
440 The list of tables should create an identical calibration
441 after being passed to this class's fromTable method.
444 tableList : `list` [`astropy.table.Table`]
445 List of tables containing the linearity calibration
450 nSignalPoints = len(list(self.
rawExpTimes.values())[0])
451 covMatrixSide = np.array(list(self.
aMatrix.values())[0]).shape[0]
452 catalog = Table([{
'AMPLIFIER_NAME': ampName,
457 'RAW_MEANS': self.
rawMeans[ampName],
458 'RAW_VARS': self.
rawVars[ampName],
459 'GAIN': self.
gain[ampName],
460 'GAIN_ERR': self.
gainErr[ampName],
461 'NOISE': self.
noise[ampName],
462 'NOISE_ERR': self.
noiseErr[ampName],
463 'PTC_FIT_PARS': np.array(self.
ptcFitPars[ampName]).tolist(),
464 'PTC_FIT_PARS_ERROR': np.array(self.
ptcFitParsError[ampName]).tolist(),
466 'COVARIANCES': np.array(self.
covariances[ampName]).reshape(
467 nSignalPoints*covMatrixSide**2).tolist(),
470 nSignalPoints*covMatrixSide**2).tolist(),
471 'COVARIANCES_SQRT_WEIGHTS':
473 nSignalPoints*covMatrixSide**2).tolist(),
474 'A_MATRIX': np.array(self.
aMatrix[ampName]).reshape(covMatrixSide**2).tolist(),
475 'B_MATRIX': np.array(self.
bMatrix[ampName]).reshape(covMatrixSide**2).tolist(),
478 nSignalPoints*covMatrixSide**2).tolist(),
479 'COVARIANCES_MODEL_NO_B':
481 nSignalPoints*covMatrixSide**2).tolist(),
482 'COVARIANCES_SQRT_WEIGHTS_NO_B':
484 nSignalPoints*covMatrixSide**2).tolist(),
486 np.array(self.
aMatrixNoB[ampName]).reshape(covMatrixSide**2).tolist(),
490 'BAD_AMPS': np.array(self.
badAmps).tolist()
if len(self.
badAmps)
else np.nan,
491 'PHOTO_CHARGE': np.array(self.
photoCharge[ampName]).tolist()
495 outMeta = {k: v
for k, v
in inMeta.items()
if v
is not None}
496 outMeta.update({k:
"" for k, v
in inMeta.items()
if v
is None})
497 catalog.meta = outMeta
498 tableList.append(catalog)
503 """Get the exposures used, i.e. not discarded, for a given amp.
504 If no mask has been created yet, all exposures are returned.
515 return [(exp1, exp2)
for ((exp1, exp2), m)
in zip(pairs, mask)
if bool(m)
is True]