23Define dataset class for MeasurePhotonTransferCurve task
26__all__ = [
'PhotonTransferCurveDataset']
29from astropy.table
import Table
36 """A simple class to hold the output data from the PTC task.
38 The dataset is made up of a dictionary
for each item, keyed by the
39 amplifiers
' names, which much be supplied at construction time.
40 New items cannot be added to the class to save accidentally saving to the
41 wrong property,
and the
class can be frozen if desired.
42 inputExpIdPairs records the exposures used to produce the data.
43 When fitPtc()
or fitCovariancesAstier()
is run, a mask
is built up, which
44 is by definition always the same length
as inputExpIdPairs, rawExpTimes,
45 rawMeans
and rawVars,
and is a list of bools, which are incrementally set
46 to
False as points are discarded
from the fits.
47 PTC fit parameters
for polynomials are stored
in a list
in ascending order
48 of polynomial term, i.e. par[0]*x^0 + par[1]*x + par[2]*x^2 etc
49 with the length of the list corresponding to the order of the polynomial
55 List
with the names of the amplifiers of the detector at hand.
57 Type of model fitted to the PTC:
"POLYNOMIAL",
"EXPAPPROXIMATION",
60 Maximum lag of covariances (size of square covariance matrices).
61 kwargs : `dict`, optional
62 Other keyword arguments to
pass to the parent init.
66 The stored attributes are:
68 badAmps : `list` [`str`]
69 List
with bad amplifiers names.
70 inputExpIdPairs : `dict`, [`str`, `list`]
71 Dictionary keyed by amp names containing the input exposures IDs.
72 expIdMask : `dict`, [`str`, `np.ndarray`]
73 Dictionary keyed by amp names containing the mask produced after
74 outlier rejection. The mask produced by the
"FULLCOVARIANCE"
75 option may differ
from the one produced
in the other two PTC
77 rawExpTimes : `dict`, [`str`, `np.ndarray`]
78 Dictionary keyed by amp names containing the unmasked exposure times.
79 rawMeans : `dict`, [`str`, `np.ndarray`]
80 Dictionary keyed by amp namescontaining the unmasked average of the
81 means of the exposures
in each flat pair.
82 rawVars : `dict`, [`str`, `np.ndarray`]
83 Dictionary keyed by amp names containing the variance of the
84 difference image of the exposures
in each flat pair.
85 gain : `dict`, [`str`, `float`]
86 Dictionary keyed by amp names containing the fitted gains.
87 gainErr : `dict`, [`str`, `float`]
88 Dictionary keyed by amp names containing the errors on the
90 noise : `dict`, [`str`, `float`]
91 Dictionary keyed by amp names containing the fitted noise.
92 noiseErr : `dict`, [`str`, `float`]
93 Dictionary keyed by amp names containing the errors on the fitted
95 ptcFitPars : `dict`, [`str`, `np.ndarray`]
96 Dictionary keyed by amp names containing the fitted parameters of the
97 PTC model
for ptcFitTye
in [
"POLYNOMIAL",
"EXPAPPROXIMATION"].
98 ptcFitParsError : `dict`, [`str`, `np.ndarray`]
99 Dictionary keyed by amp names containing the errors on the fitted
100 parameters of the PTC model
for ptcFitTye
in
101 [
"POLYNOMIAL",
"EXPAPPROXIMATION"].
102 ptcFitChiSq : `dict`, [`str`, `float`]
103 Dictionary keyed by amp names containing the reduced chi squared
104 of the fit
for ptcFitTye
in [
"POLYNOMIAL",
"EXPAPPROXIMATION"].
105 ptcTurnoff : `dict` [`str, `float`]
106 Flux value (
in ADU) where the variance of the PTC curve starts
107 decreasing consistently.
108 covariances : `dict`, [`str`, `np.ndarray`]
109 Dictionary keyed by amp names containing a list of measured
110 covariances per mean flux.
111 covariancesModel : `dict`, [`str`, `np.ndarray`]
112 Dictionary keyed by amp names containinging covariances model
113 (Eq. 20 of Astier+19) per mean flux.
114 covariancesSqrtWeights : `dict`, [`str`, `np.ndarray`]
115 Dictionary keyed by amp names containinging sqrt. of covariances
117 aMatrix : `dict`, [`str`, `np.ndarray`]
118 Dictionary keyed by amp names containing the
"a" parameters
from
119 the model
in Eq. 20 of Astier+19.
120 bMatrix : `dict`, [`str`, `np.ndarray`]
121 Dictionary keyed by amp names containing the
"b" parameters
from
122 the model
in Eq. 20 of Astier+19.
123 covariancesModelNoB : `dict`, [`str`, `np.ndarray`]
124 Dictionary keyed by amp names containing covariances model
125 (
with 'b'=0
in Eq. 20 of Astier+19)
127 aMatrixNoB : `dict`, [`str`, `np.ndarray`]
128 Dictionary keyed by amp names containing the
"a" parameters
from the
129 model
in Eq. 20 of Astier+19
131 finalVars : `dict`, [`str`, `np.ndarray`]
132 Dictionary keyed by amp names containing the masked variance of the
133 difference image of each flat
134 pair. If needed, each array will be right-padded
with
135 np.nan to match the length of rawExpTimes.
136 finalModelVars : `dict`, [`str`, `np.ndarray`]
137 Dictionary keyed by amp names containing the masked modeled
138 variance of the difference image of each flat pair. If needed, each
139 array will be right-padded
with np.nan to match the length of
141 finalMeans : `dict`, [`str`, `np.ndarray`]
142 Dictionary keyed by amp names containing the masked average of the
143 means of the exposures
in each flat pair. If needed, each array
144 will be right-padded
with np.nan to match the length of
146 photoCharges : `dict`, [`str`, `np.ndarray`]
147 Dictionary keyed by amp names containing the integrated photocharge
148 for linearity calibration.
150 Version 1.1 adds the `ptcTurnoff` attribute.
154 _SCHEMA =
'Gen3 Photon Transfer Curve'
157 def __init__(self, ampNames=[], ptcFitType=None, covMatrixSide=1, **kwargs):
165 self.
expIdMask = {ampName: np.array([], dtype=bool)
for ampName
in ampNames}
166 self.
rawExpTimes = {ampName: np.array([])
for ampName
in ampNames}
167 self.
rawMeans = {ampName: np.array([])
for ampName
in ampNames}
168 self.
rawVars = {ampName: np.array([])
for ampName
in ampNames}
171 self.
gain = {ampName: np.nan
for ampName
in ampNames}
172 self.
gainErr = {ampName: np.nan
for ampName
in ampNames}
173 self.
noise = {ampName: np.nan
for ampName
in ampNames}
174 self.
noiseErr = {ampName: np.nan
for ampName
in ampNames}
176 self.
ptcFitPars = {ampName: np.array([])
for ampName
in ampNames}
181 self.
covariances = {ampName: np.array([])
for ampName
in ampNames}
184 self.
aMatrix = {ampName: np.array([])
for ampName
in ampNames}
185 self.
bMatrix = {ampName: np.array([])
for ampName
in ampNames}
187 self.
aMatrixNoB = {ampName: np.array([])
for ampName
in ampNames}
189 self.
finalVars = {ampName: np.array([])
for ampName
in ampNames}
191 self.
finalMeans = {ampName: np.array([])
for ampName
in ampNames}
195 'rawMeans',
'rawVars',
'gain',
'gainErr',
'noise',
'noiseErr',
196 'ptcFitPars',
'ptcFitParsError',
'ptcFitChiSq',
'ptcTurnoff',
197 'aMatrixNoB',
'covariances',
'covariancesModel',
198 'covariancesSqrtWeights',
'covariancesModelNoB',
199 'aMatrix',
'bMatrix',
'finalVars',
'finalModelVars',
'finalMeans',
207 inputExpIdPair=(-1, -1),
219 Set the amp values for a partial PTC Dataset (
from cpExtractPtcTask).
224 Name of the amp to set the values.
225 inputExpIdPair : `tuple` [`int`]
226 Exposure IDs of input pair.
227 rawExpTime : `float`, optional
228 Exposure time
for this exposure pair.
229 rawMean : `float`, optional
230 Average of the means of the exposures
in this pair.
231 rawVar : `float`, optional
232 Variance of the difference of the exposures
in this pair.
233 photoCharge : `float`, optional
234 Integrated photocharge
for flat pair
for linearity calibration.
235 expIdMask : `bool`, optional
236 Flag setting
if this exposure pair should be used (
True)
238 covariance : `np.ndarray`
or None, optional
239 Measured covariance
for this exposure pair.
240 covSqrtWeights : `np.ndarray`
or None, optional
241 Measured sqrt of covariance weights
in this exposure pair.
242 gain : `float`, optional
243 Estimated gain
for this exposure pair.
244 noise : `float`, optional
245 Estimated read noise
for this exposure pair.
248 if covariance
is None:
249 covariance = nanMatrix
250 if covSqrtWeights
is None:
251 covSqrtWeights = nanMatrix
255 self.
rawMeans[ampName] = np.array([rawMean])
256 self.
rawVars[ampName] = np.array([rawVar])
258 self.
expIdMask[ampName] = np.array([expIdMask])
261 self.
gain[ampName] = gain
262 self.
noise[ampName] = noise
266 self.
aMatrix[ampName] = nanMatrix
267 self.
bMatrix[ampName] = nanMatrix
271 """Update calibration metadata.
272 This calls the base class's method after ensuring the required
273 calibration keywords will be saved.
277 setDate : `bool`, optional
278 Update the CALIBDATE fields in the metadata to the current
279 time. Defaults to
False.
281 Other keyword parameters to set
in the metadata.
287 """Construct a calibration from a dictionary of properties.
288 Must be implemented by the specific calibration subclasses.
293 Dictionary of properties.
298 Constructed calibration.
303 Raised if the supplied dictionary
is for a different
307 if calib._OBSTYPE != dictionary[
'metadata'][
'OBSTYPE']:
308 raise RuntimeError(f
"Incorrect Photon Transfer Curve dataset supplied. "
309 f
"Expected {calib._OBSTYPE}, found {dictionary['metadata']['OBSTYPE']}")
310 calib.setMetadata(dictionary[
'metadata'])
311 calib.ptcFitType = dictionary[
'ptcFitType']
312 calib.covMatrixSide = dictionary[
'covMatrixSide']
313 calib.badAmps = np.array(dictionary[
'badAmps'],
'str').tolist()
317 covMatrixSide = calib.covMatrixSide
319 covDimensionsProduct = len(np.array(list(dictionary[
'covariances'].values())[0]).ravel())
320 nSignalPoints = int(covDimensionsProduct/(covMatrixSide*covMatrixSide))
322 for ampName
in dictionary[
'ampNames']:
323 calib.ampNames.append(ampName)
324 calib.inputExpIdPairs[ampName] = dictionary[
'inputExpIdPairs'][ampName]
325 calib.expIdMask[ampName] = np.array(dictionary[
'expIdMask'][ampName])
326 calib.rawExpTimes[ampName] = np.array(dictionary[
'rawExpTimes'][ampName])
327 calib.rawMeans[ampName] = np.array(dictionary[
'rawMeans'][ampName])
328 calib.rawVars[ampName] = np.array(dictionary[
'rawVars'][ampName])
329 calib.gain[ampName] = np.array(dictionary[
'gain'][ampName])
330 calib.gainErr[ampName] = np.array(dictionary[
'gainErr'][ampName])
331 calib.noise[ampName] = np.array(dictionary[
'noise'][ampName])
332 calib.noiseErr[ampName] = np.array(dictionary[
'noiseErr'][ampName])
333 calib.ptcFitPars[ampName] = np.array(dictionary[
'ptcFitPars'][ampName])
334 calib.ptcFitParsError[ampName] = np.array(dictionary[
'ptcFitParsError'][ampName])
335 calib.ptcFitChiSq[ampName] = np.array(dictionary[
'ptcFitChiSq'][ampName])
336 calib.ptcTurnoff[ampName] = np.array(dictionary[
'ptcTurnoff'][ampName])
337 if nSignalPoints > 0:
339 calib.covariances[ampName] = np.array(dictionary[
'covariances'][ampName]).reshape(
340 (nSignalPoints, covMatrixSide, covMatrixSide))
341 calib.covariancesModel[ampName] = np.array(
342 dictionary[
'covariancesModel'][ampName]).reshape(
343 (nSignalPoints, covMatrixSide, covMatrixSide))
344 calib.covariancesSqrtWeights[ampName] = np.array(
345 dictionary[
'covariancesSqrtWeights'][ampName]).reshape(
346 (nSignalPoints, covMatrixSide, covMatrixSide))
347 calib.aMatrix[ampName] = np.array(dictionary[
'aMatrix'][ampName]).reshape(
348 (covMatrixSide, covMatrixSide))
349 calib.bMatrix[ampName] = np.array(dictionary[
'bMatrix'][ampName]).reshape(
350 (covMatrixSide, covMatrixSide))
351 calib.covariancesModelNoB[ampName] = np.array(
352 dictionary[
'covariancesModelNoB'][ampName]).reshape(
353 (nSignalPoints, covMatrixSide, covMatrixSide))
354 calib.aMatrixNoB[ampName] = np.array(
355 dictionary[
'aMatrixNoB'][ampName]).reshape((covMatrixSide, covMatrixSide))
358 calib.covariances[ampName] = np.array([], dtype=np.float64)
359 calib.covariancesModel[ampName] = np.array([], dtype=np.float64)
360 calib.covariancesSqrtWeights[ampName] = np.array([], dtype=np.float64)
361 calib.aMatrix[ampName] = np.array([], dtype=np.float64)
362 calib.bMatrix[ampName] = np.array([], dtype=np.float64)
363 calib.covariancesModelNoB[ampName] = np.array([], dtype=np.float64)
364 calib.aMatrixNoB[ampName] = np.array([], dtype=np.float64)
366 calib.finalVars[ampName] = np.array(dictionary[
'finalVars'][ampName])
367 calib.finalModelVars[ampName] = np.array(dictionary[
'finalModelVars'][ampName])
368 calib.finalMeans[ampName] = np.array(dictionary[
'finalMeans'][ampName])
369 calib.photoCharges[ampName] = np.array(dictionary[
'photoCharges'][ampName])
371 calib.updateMetadata()
375 """Return a dictionary containing the calibration properties.
376 The dictionary should be able to be round-tripped through
382 Dictionary of properties.
388 outDict['metadata'] = metadata
390 def _dictOfArraysToDictOfLists(dictOfArrays):
392 for key, value
in dictOfArrays.items():
393 dictOfLists[key] = value.ravel().tolist()
400 outDict[
'badAmps'] = self.
badAmps
402 outDict[
'expIdMask'] = _dictOfArraysToDictOfLists(self.
expIdMask)
403 outDict[
'rawExpTimes'] = _dictOfArraysToDictOfLists(self.
rawExpTimes)
404 outDict[
'rawMeans'] = _dictOfArraysToDictOfLists(self.
rawMeans)
405 outDict[
'rawVars'] = _dictOfArraysToDictOfLists(self.
rawVars)
406 outDict[
'gain'] = self.
gain
407 outDict[
'gainErr'] = self.
gainErr
408 outDict[
'noise'] = self.
noise
410 outDict[
'ptcFitPars'] = _dictOfArraysToDictOfLists(self.
ptcFitPars)
411 outDict[
'ptcFitParsError'] = _dictOfArraysToDictOfLists(self.
ptcFitParsError)
414 outDict[
'covariances'] = _dictOfArraysToDictOfLists(self.
covariances)
415 outDict[
'covariancesModel'] = _dictOfArraysToDictOfLists(self.
covariancesModel)
417 outDict[
'aMatrix'] = _dictOfArraysToDictOfLists(self.
aMatrix)
418 outDict[
'bMatrix'] = _dictOfArraysToDictOfLists(self.
bMatrix)
420 outDict[
'aMatrixNoB'] = _dictOfArraysToDictOfLists(self.
aMatrixNoB)
421 outDict[
'finalVars'] = _dictOfArraysToDictOfLists(self.
finalVars)
422 outDict[
'finalModelVars'] = _dictOfArraysToDictOfLists(self.
finalModelVars)
423 outDict[
'finalMeans'] = _dictOfArraysToDictOfLists(self.
finalMeans)
424 outDict[
'photoCharges'] = _dictOfArraysToDictOfLists(self.
photoCharges)
430 """Construct calibration from a list of tables.
431 This method uses the `fromDict` method to create the
432 calibration, after constructing an appropriate dictionary from
437 tableList : `list` [`lsst.afw.table.Table`]
438 List of tables to use to construct the datasetPtc.
443 The calibration defined
in the tables.
445 ptcTable = tableList[0]
447 metadata = ptcTable.meta
449 inDict['metadata'] = metadata
450 inDict[
'ampNames'] = []
451 inDict[
'ptcFitType'] = []
452 inDict[
'covMatrixSide'] = []
453 inDict[
'inputExpIdPairs'] = dict()
454 inDict[
'expIdMask'] = dict()
455 inDict[
'rawExpTimes'] = dict()
456 inDict[
'rawMeans'] = dict()
457 inDict[
'rawVars'] = dict()
458 inDict[
'gain'] = dict()
459 inDict[
'gainErr'] = dict()
460 inDict[
'noise'] = dict()
461 inDict[
'noiseErr'] = dict()
462 inDict[
'ptcFitPars'] = dict()
463 inDict[
'ptcFitParsError'] = dict()
464 inDict[
'ptcFitChiSq'] = dict()
465 inDict[
'ptcTurnoff'] = dict()
466 inDict[
'covariances'] = dict()
467 inDict[
'covariancesModel'] = dict()
468 inDict[
'covariancesSqrtWeights'] = dict()
469 inDict[
'aMatrix'] = dict()
470 inDict[
'bMatrix'] = dict()
471 inDict[
'covariancesModelNoB'] = dict()
472 inDict[
'aMatrixNoB'] = dict()
473 inDict[
'finalVars'] = dict()
474 inDict[
'finalModelVars'] = dict()
475 inDict[
'finalMeans'] = dict()
476 inDict[
'badAmps'] = []
477 inDict[
'photoCharges'] = dict()
479 calibVersion = metadata[
'PTC_VERSION']
480 if calibVersion == 1.0:
481 cls().log.warning(f
"Previous version found for PTC dataset: {calibVersion}. "
482 f
"Setting 'ptcTurnoff' in all amps to last value in 'finalMeans'.")
483 for record
in ptcTable:
484 ampName = record[
'AMPLIFIER_NAME']
486 inDict[
'ptcFitType'] = record[
'PTC_FIT_TYPE']
487 inDict[
'covMatrixSide'] = record[
'COV_MATRIX_SIDE']
488 inDict[
'ampNames'].append(ampName)
489 inDict[
'inputExpIdPairs'][ampName] = record[
'INPUT_EXP_ID_PAIRS'].tolist()
490 inDict[
'expIdMask'][ampName] = record[
'EXP_ID_MASK']
491 inDict[
'rawExpTimes'][ampName] = record[
'RAW_EXP_TIMES']
492 inDict[
'rawMeans'][ampName] = record[
'RAW_MEANS']
493 inDict[
'rawVars'][ampName] = record[
'RAW_VARS']
494 inDict[
'gain'][ampName] = record[
'GAIN']
495 inDict[
'gainErr'][ampName] = record[
'GAIN_ERR']
496 inDict[
'noise'][ampName] = record[
'NOISE']
497 inDict[
'noiseErr'][ampName] = record[
'NOISE_ERR']
498 inDict[
'ptcFitPars'][ampName] = record[
'PTC_FIT_PARS']
499 inDict[
'ptcFitParsError'][ampName] = record[
'PTC_FIT_PARS_ERROR']
500 inDict[
'ptcFitChiSq'][ampName] = record[
'PTC_FIT_CHI_SQ']
501 inDict[
'covariances'][ampName] = record[
'COVARIANCES']
502 inDict[
'covariancesModel'][ampName] = record[
'COVARIANCES_MODEL']
503 inDict[
'covariancesSqrtWeights'][ampName] = record[
'COVARIANCES_SQRT_WEIGHTS']
504 inDict[
'aMatrix'][ampName] = record[
'A_MATRIX']
505 inDict[
'bMatrix'][ampName] = record[
'B_MATRIX']
506 inDict[
'covariancesModelNoB'][ampName] = record[
'COVARIANCES_MODEL_NO_B']
507 inDict[
'aMatrixNoB'][ampName] = record[
'A_MATRIX_NO_B']
508 inDict[
'finalVars'][ampName] = record[
'FINAL_VARS']
509 inDict[
'finalModelVars'][ampName] = record[
'FINAL_MODEL_VARS']
510 inDict[
'finalMeans'][ampName] = record[
'FINAL_MEANS']
511 inDict[
'badAmps'] = record[
'BAD_AMPS'].tolist()
512 inDict[
'photoCharges'][ampName] = record[
'PHOTO_CHARGE']
513 if calibVersion == 1.0:
514 mask = record[
'FINAL_MEANS'].mask
515 array = record[
'FINAL_MEANS'][~mask]
517 inDict[
'ptcTurnoff'][ampName] = record[
'FINAL_MEANS'][~mask][-1]
519 inDict[
'ptcTurnoff'][ampName] = np.nan
521 inDict[
'ptcTurnoff'][ampName] = record[
'PTC_TURNOFF']
525 """Construct a list of tables containing the information in this
528 The list of tables should create an identical calibration
529 after being passed to this class's fromTable method.
533 tableList : `list` [`astropy.table.Table`]
534 List of tables containing the linearity calibration
540 badAmps = np.array(self.badAmps) if len(self.
badAmps)
else np.array([], dtype=
"U3")
545 'AMPLIFIER_NAME': ampName,
551 'RAW_MEANS': self.
rawMeans[ampName],
552 'RAW_VARS': self.
rawVars[ampName],
553 'GAIN': self.
gain[ampName],
554 'GAIN_ERR': self.
gainErr[ampName],
555 'NOISE': self.
noise[ampName],
556 'NOISE_ERR': self.
noiseErr[ampName],
557 'PTC_FIT_PARS': np.array(self.
ptcFitPars[ampName]),
561 'A_MATRIX': self.
aMatrix[ampName].ravel(),
562 'B_MATRIX': self.
bMatrix[ampName].ravel(),
563 'A_MATRIX_NO_B': self.
aMatrixNoB[ampName].ravel(),
574 catalogList.append(ampDict)
576 catalog = Table(catalogList)
579 outMeta = {k: v
for k, v
in inMeta.items()
if v
is not None}
580 outMeta.update({k:
"" for k, v
in inMeta.items()
if v
is None})
581 catalog.meta = outMeta
582 tableList.append(catalog)
587 """Read metadata parameters from a detector.
591 detector : `lsst.afw.cameraGeom.detector`
592 Input detector with parameters to use.
597 The calibration constructed
from the detector.
603 """Get the exposures used, i.e. not discarded, for a given amp.
604 If no mask has been created yet, all exposures are returned.
612 expIdsUsed : `list` [`tuple`]
613 List of pairs of exposure ids used in PTC.
625 expIdsUsed = [(exp1, exp2)
for ((exp1, exp2), m)
in zip(pairs, mask)
if m]
627 warnings.warn(
"The PTC file was written incorrectly; you should rerun the "
628 "PTC solve task if possible.", RuntimeWarning)
630 for pairList, m
in zip(pairs, mask):
632 expIdsUsed.append(pairList[0])
637 """Get the good amps from this PTC."""
641 """Get the good points used for a given amp in the PTC.
649 goodPoints : `np.ndarray`
650 Boolean array of good points used in PTC.
def requiredAttributes(self, value)
def updateMetadata(self, camera=None, detector=None, filterName=None, setCalibId=False, setCalibInfo=False, setDate=False, **kwargs)
def requiredAttributes(self)
def __init__(self, ampNames=[], ptcFitType=None, covMatrixSide=1, **kwargs)
def getExpIdsUsed(self, ampName)
def fromDict(cls, dictionary)
def fromDetector(self, detector)
def updateMetadata(self, **kwargs)
def fromTable(cls, tableList)
def setAmpValuesPartialDataset(self, ampName, inputExpIdPair=(-1, -1), rawExpTime=np.nan, rawMean=np.nan, rawVar=np.nan, photoCharge=np.nan, expIdMask=False, covariance=None, covSqrtWeights=None, gain=np.nan, noise=np.nan)
def getGoodPoints(self, ampName)