Coverage for python/lsst/cp/pipe/ptc.py : 9%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of cp_pipe.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21#
22import numpy as np
23import matplotlib.pyplot as plt
24from collections import Counter
26import lsst.afw.math as afwMath
27import lsst.pex.config as pexConfig
28import lsst.pipe.base as pipeBase
29from .utils import (fitLeastSq, fitBootstrap, funcPolynomial, funcAstier)
30from scipy.optimize import least_squares
32import datetime
34from .astierCovPtcUtils import (fftSize, CovFft, computeCovDirect, fitData)
35from .linearity import LinearitySolveTask
36from .photodiode import getBOTphotodiodeData
38from lsst.pipe.tasks.getRepositoryData import DataRefListRunner
39from lsst.ip.isr import PhotonTransferCurveDataset
41__all__ = ['MeasurePhotonTransferCurveTask',
42 'MeasurePhotonTransferCurveTaskConfig']
45class MeasurePhotonTransferCurveTaskConfig(pexConfig.Config):
46 """Config class for photon transfer curve measurement task"""
47 ccdKey = pexConfig.Field(
48 dtype=str,
49 doc="The key by which to pull a detector from a dataId, e.g. 'ccd' or 'detector'.",
50 default='ccd',
51 )
52 ptcFitType = pexConfig.ChoiceField(
53 dtype=str,
54 doc="Fit PTC to Eq. 16, Eq. 20 in Astier+19, or to a polynomial.",
55 default="POLYNOMIAL",
56 allowed={
57 "POLYNOMIAL": "n-degree polynomial (use 'polynomialFitDegree' to set 'n').",
58 "EXPAPPROXIMATION": "Approximation in Astier+19 (Eq. 16).",
59 "FULLCOVARIANCE": "Full covariances model in Astier+19 (Eq. 20)"
60 }
61 )
62 sigmaClipFullFitCovariancesAstier = pexConfig.Field(
63 dtype=float,
64 doc="Sigma clip for full model fit for FULLCOVARIANCE ptcFitType ",
65 default=5.0,
66 )
67 maxIterFullFitCovariancesAstier = pexConfig.Field(
68 dtype=int,
69 doc="Maximum number of iterations in full model fit for FULLCOVARIANCE ptcFitType",
70 default=3,
71 )
72 maximumRangeCovariancesAstier = pexConfig.Field(
73 dtype=int,
74 doc="Maximum range of covariances as in Astier+19",
75 default=8,
76 )
77 covAstierRealSpace = pexConfig.Field(
78 dtype=bool,
79 doc="Calculate covariances in real space or via FFT? (see appendix A of Astier+19).",
80 default=False,
81 )
82 polynomialFitDegree = pexConfig.Field(
83 dtype=int,
84 doc="Degree of polynomial to fit the PTC, when 'ptcFitType'=POLYNOMIAL.",
85 default=3,
86 )
87 linearity = pexConfig.ConfigurableField(
88 target=LinearitySolveTask,
89 doc="Task to solve the linearity."
90 )
92 doCreateLinearizer = pexConfig.Field(
93 dtype=bool,
94 doc="Calculate non-linearity and persist linearizer?",
95 default=False,
96 )
98 binSize = pexConfig.Field(
99 dtype=int,
100 doc="Bin the image by this factor in both dimensions.",
101 default=1,
102 )
103 minMeanSignal = pexConfig.DictField(
104 keytype=str,
105 itemtype=float,
106 doc="Minimum values (inclusive) of mean signal (in ADU) above which to consider, per amp."
107 " The same cut is applied to all amps if this dictionary is of the form"
108 " {'ALL_AMPS': value}",
109 default={'ALL_AMPS': 0.0},
110 )
111 maxMeanSignal = pexConfig.DictField(
112 keytype=str,
113 itemtype=float,
114 doc="Maximum values (inclusive) of mean signal (in ADU) below which to consider, per amp."
115 " The same cut is applied to all amps if this dictionary is of the form"
116 " {'ALL_AMPS': value}",
117 default={'ALL_AMPS': 1e6},
118 )
119 initialNonLinearityExclusionThresholdPositive = pexConfig.RangeField(
120 dtype=float,
121 doc="Initially exclude data points with a variance that are more than a factor of this from being"
122 " linear in the positive direction, from the PTC fit. Note that these points will also be"
123 " excluded from the non-linearity fit. This is done before the iterative outlier rejection,"
124 " to allow an accurate determination of the sigmas for said iterative fit.",
125 default=0.12,
126 min=0.0,
127 max=1.0,
128 )
129 initialNonLinearityExclusionThresholdNegative = pexConfig.RangeField(
130 dtype=float,
131 doc="Initially exclude data points with a variance that are more than a factor of this from being"
132 " linear in the negative direction, from the PTC fit. Note that these points will also be"
133 " excluded from the non-linearity fit. This is done before the iterative outlier rejection,"
134 " to allow an accurate determination of the sigmas for said iterative fit.",
135 default=0.25,
136 min=0.0,
137 max=1.0,
138 )
139 sigmaCutPtcOutliers = pexConfig.Field(
140 dtype=float,
141 doc="Sigma cut for outlier rejection in PTC.",
142 default=5.0,
143 )
144 maskNameList = pexConfig.ListField(
145 dtype=str,
146 doc="Mask list to exclude from statistics calculations.",
147 default=['SUSPECT', 'BAD', 'NO_DATA'],
148 )
149 nSigmaClipPtc = pexConfig.Field(
150 dtype=float,
151 doc="Sigma cut for afwMath.StatisticsControl()",
152 default=5.5,
153 )
154 nIterSigmaClipPtc = pexConfig.Field(
155 dtype=int,
156 doc="Number of sigma-clipping iterations for afwMath.StatisticsControl()",
157 default=1,
158 )
159 maxIterationsPtcOutliers = pexConfig.Field(
160 dtype=int,
161 doc="Maximum number of iterations for outlier rejection in PTC.",
162 default=2,
163 )
164 doFitBootstrap = pexConfig.Field(
165 dtype=bool,
166 doc="Use bootstrap for the PTC fit parameters and errors?.",
167 default=False,
168 )
169 doPhotodiode = pexConfig.Field(
170 dtype=bool,
171 doc="Apply a correction based on the photodiode readings if available?",
172 default=True,
173 )
174 photodiodeDataPath = pexConfig.Field(
175 dtype=str,
176 doc="Gen2 only: path to locate the data photodiode data files.",
177 default=""
178 )
179 instrumentName = pexConfig.Field(
180 dtype=str,
181 doc="Instrument name.",
182 default='',
183 )
186class MeasurePhotonTransferCurveTask(pipeBase.CmdLineTask):
187 """A class to calculate, fit, and plot a PTC from a set of flat pairs.
189 The Photon Transfer Curve (var(signal) vs mean(signal)) is a standard tool
190 used in astronomical detectors characterization (e.g., Janesick 2001,
191 Janesick 2007). If ptcFitType is "EXPAPPROXIMATION" or "POLYNOMIAL", this task calculates the
192 PTC from a series of pairs of flat-field images; each pair taken at identical exposure
193 times. The difference image of each pair is formed to eliminate fixed pattern noise,
194 and then the variance of the difference image and the mean of the average image
195 are used to produce the PTC. An n-degree polynomial or the approximation in Equation
196 16 of Astier+19 ("The Shape of the Photon Transfer Curve of CCD sensors",
197 arXiv:1905.08677) can be fitted to the PTC curve. These models include
198 parameters such as the gain (e/DN) and readout noise.
200 Linearizers to correct for signal-chain non-linearity are also calculated.
201 The `Linearizer` class, in general, can support per-amp linearizers, but in this
202 task this is not supported.
204 If ptcFitType is "FULLCOVARIANCE", the covariances of the difference images are calculated via the
205 DFT methods described in Astier+19 and the variances for the PTC are given by the cov[0,0] elements
206 at each signal level. The full model in Equation 20 of Astier+19 is fit to the PTC to get the gain
207 and the noise.
209 Parameters
210 ----------
212 *args: `list`
213 Positional arguments passed to the Task constructor. None used at this
214 time.
215 **kwargs: `dict`
216 Keyword arguments passed on to the Task constructor. None used at this
217 time.
219 """
221 RunnerClass = DataRefListRunner
222 ConfigClass = MeasurePhotonTransferCurveTaskConfig
223 _DefaultName = "measurePhotonTransferCurve"
225 def __init__(self, *args, **kwargs):
226 pipeBase.CmdLineTask.__init__(self, *args, **kwargs)
227 self.makeSubtask("linearity")
228 plt.interactive(False) # stop windows popping up when plotting. When headless, use 'agg' backend too
229 self.config.validate()
230 self.config.freeze()
232 @pipeBase.timeMethod
233 def runDataRef(self, dataRefList):
234 """Run the Photon Transfer Curve (PTC) measurement task.
236 For a dataRef (which is each detector here),
237 and given a list of exposure pairs (postISR) at different exposure times,
238 measure the PTC.
240 Parameters
241 ----------
242 dataRefList : `list` [`lsst.daf.peristence.ButlerDataRef`]
243 Data references for exposures for detectors to process.
244 """
245 if len(dataRefList) < 2:
246 raise RuntimeError("Insufficient inputs to combine.")
248 # setup necessary objects
249 dataRef = dataRefList[0]
251 detNum = dataRef.dataId[self.config.ccdKey]
252 camera = dataRef.get('camera')
253 detector = camera[dataRef.dataId[self.config.ccdKey]]
255 amps = detector.getAmplifiers()
256 ampNames = [amp.getName() for amp in amps]
257 datasetPtc = PhotonTransferCurveDataset(ampNames, self.config.ptcFitType)
259 # Get the pairs of flat indexed by expTime
260 expPairs = self.makePairs(dataRefList)
261 expIds = []
262 for (exp1, exp2) in expPairs.values():
263 id1 = exp1.getInfo().getVisitInfo().getExposureId()
264 id2 = exp2.getInfo().getVisitInfo().getExposureId()
265 expIds.append((id1, id2))
266 self.log.info(f"Measuring PTC using {expIds} exposures for detector {detector.getId()}")
268 # get photodiode data early so that logic can be put in to only use the
269 # data if all files are found, as partial corrections are not possible
270 # or at least require significant logic to deal with
271 if self.config.doPhotodiode:
272 for (expId1, expId2) in expIds:
273 charges = [-1, -1] # necessary to have a not-found value to keep lists in step
274 for i, expId in enumerate([expId1, expId2]):
275 # //1000 is a Gen2 only hack, working around the fact an
276 # exposure's ID is not the same as the expId in the
277 # registry. Currently expId is concatenated with the
278 # zero-padded detector ID. This will all go away in Gen3.
279 dataRef.dataId['expId'] = expId//1000
280 if self.config.photodiodeDataPath:
281 photodiodeData = getBOTphotodiodeData(dataRef, self.config.photodiodeDataPath)
282 else:
283 photodiodeData = getBOTphotodiodeData(dataRef)
284 if photodiodeData: # default path stored in function def to keep task clean
285 charges[i] = photodiodeData.getCharge()
286 else:
287 # full expId (not //1000) here, as that encodes the
288 # the detector number as so is fully qualifying
289 self.log.warn(f"No photodiode data found for {expId}")
291 for ampName in ampNames:
292 datasetPtc.photoCharge[ampName].append((charges[0], charges[1]))
293 else:
294 # Can't be an empty list, as initialized, because astropy.Table won't allow it
295 # when saving as fits
296 for ampName in ampNames:
297 datasetPtc.photoCharge[ampName] = np.repeat(np.nan, len(expIds))
299 for ampName in ampNames:
300 datasetPtc.inputExpIdPairs[ampName] = expIds
302 maxMeanSignalDict = {ampName: 1e6 for ampName in ampNames}
303 minMeanSignalDict = {ampName: 0.0 for ampName in ampNames}
304 for ampName in ampNames:
305 if 'ALL_AMPS' in self.config.maxMeanSignal:
306 maxMeanSignalDict[ampName] = self.config.maxMeanSignal['ALL_AMPS']
307 elif ampName in self.config.maxMeanSignal:
308 maxMeanSignalDict[ampName] = self.config.maxMeanSignal[ampName]
310 if 'ALL_AMPS' in self.config.minMeanSignal:
311 minMeanSignalDict[ampName] = self.config.minMeanSignal['ALL_AMPS']
312 elif ampName in self.config.minMeanSignal:
313 minMeanSignalDict[ampName] = self.config.minMeanSignal[ampName]
315 tupleRecords = []
316 allTags = []
317 for expTime, (exp1, exp2) in expPairs.items():
318 expId1 = exp1.getInfo().getVisitInfo().getExposureId()
319 expId2 = exp2.getInfo().getVisitInfo().getExposureId()
320 tupleRows = []
321 nAmpsNan = 0
322 tags = ['mu', 'i', 'j', 'var', 'cov', 'npix', 'ext', 'expTime', 'ampName']
323 for ampNumber, amp in enumerate(detector):
324 ampName = amp.getName()
325 # covAstier: (i, j, var (cov[0,0]), cov, npix)
326 doRealSpace = self.config.covAstierRealSpace
327 muDiff, varDiff, covAstier = self.measureMeanVarCov(exp1, exp2, region=amp.getBBox(),
328 covAstierRealSpace=doRealSpace)
330 if np.isnan(muDiff) or np.isnan(varDiff) or (covAstier is None):
331 msg = (f"NaN mean or var, or None cov in amp {ampName} in exposure pair {expId1},"
332 f" {expId2} of detector {detNum}.")
333 self.log.warn(msg)
334 nAmpsNan += 1
335 continue
336 if (muDiff <= minMeanSignalDict[ampName]) or (muDiff >= maxMeanSignalDict[ampName]):
337 continue
339 datasetPtc.rawExpTimes[ampName].append(expTime)
340 datasetPtc.rawMeans[ampName].append(muDiff)
341 datasetPtc.rawVars[ampName].append(varDiff)
343 tupleRows += [(muDiff, ) + covRow + (ampNumber, expTime, ampName) for covRow in covAstier]
344 if nAmpsNan == len(ampNames):
345 msg = f"NaN mean in all amps of exposure pair {expId1}, {expId2} of detector {detNum}."
346 self.log.warn(msg)
347 continue
348 allTags += tags
349 tupleRecords += tupleRows
350 covariancesWithTags = np.core.records.fromrecords(tupleRecords, names=allTags)
351 # Sort raw vectors by rawMeans index
352 for ampName in datasetPtc.ampNames:
353 index = np.argsort(datasetPtc.rawMeans[ampName])
354 datasetPtc.rawExpTimes[ampName] = np.array(datasetPtc.rawExpTimes[ampName])[index]
355 datasetPtc.rawMeans[ampName] = np.array(datasetPtc.rawMeans[ampName])[index]
356 datasetPtc.rawVars[ampName] = np.array(datasetPtc.rawVars[ampName])[index]
358 if self.config.ptcFitType in ["FULLCOVARIANCE", ]:
359 # Calculate covariances and fit them, including the PTC, to Astier+19 full model (Eq. 20)
360 datasetPtc = self.fitCovariancesAstier(datasetPtc, covariancesWithTags)
361 elif self.config.ptcFitType in ["EXPAPPROXIMATION", "POLYNOMIAL"]:
362 # Fit the PTC to a polynomial or to Astier+19 exponential approximation (Eq. 16)
363 # Fill up PhotonTransferCurveDataset object.
364 datasetPtc = self.fitPtc(datasetPtc, self.config.ptcFitType)
366 detName = detector.getName()
367 now = datetime.datetime.utcnow()
368 calibDate = now.strftime("%Y-%m-%d")
369 butler = dataRef.getButler()
371 datasetPtc.updateMetadata(setDate=True, camera=camera, detector=detector)
373 # Fit a poynomial to calculate non-linearity and persist linearizer.
374 if self.config.doCreateLinearizer:
375 # Fit (non)linearity of signal vs time curve.
376 # Fill up PhotonTransferCurveDataset object.
377 # Fill up array for LUT linearizer (tableArray).
378 # Produce coefficients for Polynomial and Squared linearizers.
379 # Build linearizer objects.
380 dimensions = {'camera': camera.getName(), 'detector': detector.getId()}
381 linearityResults = self.linearity.run(datasetPtc, camera, dimensions)
382 linearizer = linearityResults.outputLinearizer
384 self.log.info("Writing linearizer:")
386 detName = detector.getName()
387 now = datetime.datetime.utcnow()
388 calibDate = now.strftime("%Y-%m-%d")
390 butler.put(linearizer, datasetType='linearizer',
391 dataId={'detector': detNum, 'detectorName': detName, 'calibDate': calibDate})
393 self.log.info(f"Writing PTC data.")
394 butler.put(datasetPtc, datasetType='photonTransferCurveDataset', dataId={'detector': detNum,
395 'detectorName': detName, 'calibDate': calibDate})
397 return pipeBase.Struct(exitStatus=0)
399 def makePairs(self, dataRefList):
400 """Produce a list of flat pairs indexed by exposure time.
402 Parameters
403 ----------
404 dataRefList : `list` [`lsst.daf.peristence.ButlerDataRef`]
405 Data references for exposures for detectors to process.
407 Return
408 ------
409 flatPairs : `dict` [`float`, `lsst.afw.image.exposure.exposure.ExposureF`]
410 Dictionary that groups flat-field exposures that have the same exposure time (seconds).
412 Notes
413 -----
414 We use the difference of one pair of flat-field images taken at the same exposure time when
415 calculating the PTC to reduce Fixed Pattern Noise. If there are > 2 flat-field images with the
416 same exposure time, the first two are kept and the rest discarded.
417 """
419 # Organize exposures by observation date.
420 expDict = {}
421 for dataRef in dataRefList:
422 try:
423 tempFlat = dataRef.get("postISRCCD")
424 except RuntimeError:
425 self.log.warn("postISR exposure could not be retrieved. Ignoring flat.")
426 continue
427 expDate = tempFlat.getInfo().getVisitInfo().getDate().get()
428 expDict.setdefault(expDate, tempFlat)
429 sortedExps = {k: expDict[k] for k in sorted(expDict)}
431 flatPairs = {}
432 for exp in sortedExps:
433 tempFlat = sortedExps[exp]
434 expTime = tempFlat.getInfo().getVisitInfo().getExposureTime()
435 listAtExpTime = flatPairs.setdefault(expTime, [])
436 if len(listAtExpTime) >= 2:
437 self.log.warn(f"Already found 2 exposures at expTime {expTime}. "
438 f"Ignoring exposure {tempFlat.getInfo().getVisitInfo().getExposureId()}")
439 else:
440 listAtExpTime.append(tempFlat)
442 keysToDrop = []
443 for (key, value) in flatPairs.items():
444 if len(value) < 2:
445 keysToDrop.append(key)
447 if len(keysToDrop):
448 for key in keysToDrop:
449 self.log.warn(f"Only one exposure found at expTime {key}. Dropping exposure "
450 f"{flatPairs[key][0].getInfo().getVisitInfo().getExposureId()}.")
451 flatPairs.pop(key)
452 sortedFlatPairs = {k: flatPairs[k] for k in sorted(flatPairs)}
453 return sortedFlatPairs
455 def fitCovariancesAstier(self, dataset, covariancesWithTagsArray):
456 """Fit measured flat covariances to full model in Astier+19.
458 Parameters
459 ----------
460 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset`
461 The dataset containing information such as the means, variances and exposure times.
463 covariancesWithTagsArray : `numpy.recarray`
464 Tuple with at least (mu, cov, var, i, j, npix), where:
465 mu : 0.5*(m1 + m2), where:
466 mu1: mean value of flat1
467 mu2: mean value of flat2
468 cov: covariance value at lag(i, j)
469 var: variance(covariance value at lag(0, 0))
470 i: lag dimension
471 j: lag dimension
472 npix: number of pixels used for covariance calculation.
474 Returns
475 -------
476 dataset: `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset`
477 This is the same dataset as the input paramter, however, it has been modified
478 to include information such as the fit vectors and the fit parameters. See
479 the class `PhotonTransferCurveDatase`.
480 """
482 covFits, covFitsNoB = fitData(covariancesWithTagsArray,
483 r=self.config.maximumRangeCovariancesAstier,
484 nSigmaFullFit=self.config.sigmaClipFullFitCovariancesAstier,
485 maxIterFullFit=self.config.maxIterFullFitCovariancesAstier)
487 dataset = self.getOutputPtcDataCovAstier(dataset, covFits, covFitsNoB)
489 return dataset
491 def getOutputPtcDataCovAstier(self, dataset, covFits, covFitsNoB):
492 """Get output data for PhotonTransferCurveCovAstierDataset from CovFit objects.
494 Parameters
495 ----------
496 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset`
497 The dataset containing information such as the means, variances and exposure times.
499 covFits: `dict`
500 Dictionary of CovFit objects, with amp names as keys.
502 covFitsNoB : `dict`
503 Dictionary of CovFit objects, with amp names as keys, and 'b=0' in Eq. 20 of Astier+19.
505 Returns
506 -------
507 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset`
508 This is the same dataset as the input paramter, however, it has been modified
509 to include extra information such as the mask 1D array, gains, reoudout noise, measured signal,
510 measured variance, modeled variance, a, and b coefficient matrices (see Astier+19) per amplifier.
511 See the class `PhotonTransferCurveDatase`.
512 """
513 assert(len(covFits) == len(covFitsNoB))
515 for i, amp in enumerate(dataset.ampNames):
516 lenInputTimes = len(dataset.rawExpTimes[amp])
517 # Not used when ptcFitType is 'FULLCOVARIANCE'
518 dataset.ptcFitPars[amp] = np.nan
519 dataset.ptcFitParsError[amp] = np.nan
520 dataset.ptcFitChiSq[amp] = np.nan
521 if (amp in covFits and (covFits[amp].covParams is not None) and
522 (covFitsNoB[amp].covParams is not None)):
523 fit = covFits[amp]
524 fitNoB = covFitsNoB[amp]
525 # Save full covariances, covariances models, and their weights
526 dataset.covariances[amp] = fit.cov
527 dataset.covariancesModel[amp] = fit.evalCovModel()
528 dataset.covariancesSqrtWeights[amp] = fit.sqrtW
529 dataset.aMatrix[amp] = fit.getA()
530 dataset.bMatrix[amp] = fit.getB()
531 dataset.covariancesNoB[amp] = fitNoB.cov
532 dataset.covariancesModelNoB[amp] = fitNoB.evalCovModel()
533 dataset.covariancesSqrtWeightsNoB[amp] = fitNoB.sqrtW
534 dataset.aMatrixNoB[amp] = fitNoB.getA()
536 (meanVecFinal, varVecFinal, varVecModel,
537 wc, varMask) = fit.getFitData(0, 0, divideByMu=False, returnMasked=True)
538 gain = fit.getGain()
539 # adjust mask to original size of rawExpTimes
540 # so far, only the min/max signal cut is in dataset.expIdMask
541 dataset.expIdMask[amp] = varMask
542 dataset.gain[amp] = gain
543 dataset.gainErr[amp] = fit.getGainErr()
544 dataset.noise[amp] = np.sqrt(fit.getRon())
545 dataset.noiseErr[amp] = fit.getRonErr()
547 padLength = lenInputTimes - len(varVecFinal)
548 dataset.finalVars[amp] = np.pad(varVecFinal/(gain**2), (0, padLength), 'constant',
549 constant_values=np.nan)
550 dataset.finalModelVars[amp] = np.pad(varVecModel/(gain**2), (0, padLength), 'constant',
551 constant_values=np.nan)
552 dataset.finalMeans[amp] = np.pad(meanVecFinal/gain, (0, padLength), 'constant',
553 constant_values=np.nan)
554 else:
555 # Bad amp
556 # Entries need to have proper dimensions so read/write with astropy.Table works.
557 matrixSide = self.config.maximumRangeCovariancesAstier
558 nanMatrix = np.full((matrixSide, matrixSide), np.nan)
559 listNanMatrix = np.full((lenInputTimes, matrixSide, matrixSide), np.nan)
561 dataset.covariances[amp] = listNanMatrix
562 dataset.covariancesModel[amp] = listNanMatrix
563 dataset.covariancesSqrtWeights[amp] = listNanMatrix
564 dataset.aMatrix[amp] = nanMatrix
565 dataset.bMatrix[amp] = nanMatrix
566 dataset.covariancesNoB[amp] = listNanMatrix
567 dataset.covariancesModelNoB[amp] = listNanMatrix
568 dataset.covariancesSqrtWeightsNoB[amp] = listNanMatrix
569 dataset.aMatrixNoB[amp] = nanMatrix
571 dataset.expIdMask[amp] = np.repeat(np.nan, lenInputTimes)
572 dataset.gain[amp] = np.nan
573 dataset.gainErr[amp] = np.nan
574 dataset.noise[amp] = np.nan
575 dataset.noiseErr[amp] = np.nan
576 dataset.finalVars[amp] = np.repeat(np.nan, lenInputTimes)
577 dataset.finalModelVars[amp] = np.repeat(np.nan, lenInputTimes)
578 dataset.finalMeans[amp] = np.repeat(np.nan, lenInputTimes)
580 return dataset
582 def measureMeanVarCov(self, exposure1, exposure2, region=None, covAstierRealSpace=False):
583 """Calculate the mean of each of two exposures and the variance and covariance of their difference.
585 The variance is calculated via afwMath, and the covariance via the methods in Astier+19 (appendix A).
586 In theory, var = covariance[0,0]. This should be validated, and in the future, we may decide to just
587 keep one (covariance).
589 Parameters
590 ----------
591 exposure1 : `lsst.afw.image.exposure.exposure.ExposureF`
592 First exposure of flat field pair.
594 exposure2 : `lsst.afw.image.exposure.exposure.ExposureF`
595 Second exposure of flat field pair.
597 region : `lsst.geom.Box2I`, optional
598 Region of each exposure where to perform the calculations (e.g, an amplifier).
600 covAstierRealSpace : `bool`, optional
601 Should the covariannces in Astier+19 be calculated in real space or via FFT?
602 See Appendix A of Astier+19.
604 Returns
605 -------
606 mu : `float` or `NaN`
607 0.5*(mu1 + mu2), where mu1, and mu2 are the clipped means of the regions in
608 both exposures. If either mu1 or m2 are NaN's, the returned value is NaN.
610 varDiff : `float` or `NaN`
611 Half of the clipped variance of the difference of the regions inthe two input
612 exposures. If either mu1 or m2 are NaN's, the returned value is NaN.
614 covDiffAstier : `list` or `NaN`
615 List with tuples of the form (dx, dy, var, cov, npix), where:
616 dx : `int`
617 Lag in x
618 dy : `int`
619 Lag in y
620 var : `float`
621 Variance at (dx, dy).
622 cov : `float`
623 Covariance at (dx, dy).
624 nPix : `int`
625 Number of pixel pairs used to evaluate var and cov.
626 If either mu1 or m2 are NaN's, the returned value is NaN.
627 """
629 if region is not None:
630 im1Area = exposure1.maskedImage[region]
631 im2Area = exposure2.maskedImage[region]
632 else:
633 im1Area = exposure1.maskedImage
634 im2Area = exposure2.maskedImage
636 if self.config.binSize > 1:
637 im1Area = afwMath.binImage(im1Area, self.config.binSize)
638 im2Area = afwMath.binImage(im2Area, self.config.binSize)
640 im1MaskVal = exposure1.getMask().getPlaneBitMask(self.config.maskNameList)
641 im1StatsCtrl = afwMath.StatisticsControl(self.config.nSigmaClipPtc,
642 self.config.nIterSigmaClipPtc,
643 im1MaskVal)
644 im1StatsCtrl.setNanSafe(True)
645 im1StatsCtrl.setAndMask(im1MaskVal)
647 im2MaskVal = exposure2.getMask().getPlaneBitMask(self.config.maskNameList)
648 im2StatsCtrl = afwMath.StatisticsControl(self.config.nSigmaClipPtc,
649 self.config.nIterSigmaClipPtc,
650 im2MaskVal)
651 im2StatsCtrl.setNanSafe(True)
652 im2StatsCtrl.setAndMask(im2MaskVal)
654 # Clipped mean of images; then average of mean.
655 mu1 = afwMath.makeStatistics(im1Area, afwMath.MEANCLIP, im1StatsCtrl).getValue()
656 mu2 = afwMath.makeStatistics(im2Area, afwMath.MEANCLIP, im2StatsCtrl).getValue()
657 if np.isnan(mu1) or np.isnan(mu2):
658 return np.nan, np.nan, None
659 mu = 0.5*(mu1 + mu2)
661 # Take difference of pairs
662 # symmetric formula: diff = (mu2*im1-mu1*im2)/(0.5*(mu1+mu2))
663 temp = im2Area.clone()
664 temp *= mu1
665 diffIm = im1Area.clone()
666 diffIm *= mu2
667 diffIm -= temp
668 diffIm /= mu
670 diffImMaskVal = diffIm.getMask().getPlaneBitMask(self.config.maskNameList)
671 diffImStatsCtrl = afwMath.StatisticsControl(self.config.nSigmaClipPtc,
672 self.config.nIterSigmaClipPtc,
673 diffImMaskVal)
674 diffImStatsCtrl.setNanSafe(True)
675 diffImStatsCtrl.setAndMask(diffImMaskVal)
677 varDiff = 0.5*(afwMath.makeStatistics(diffIm, afwMath.VARIANCECLIP, diffImStatsCtrl).getValue())
679 # Get the mask and identify good pixels as '1', and the rest as '0'.
680 w1 = np.where(im1Area.getMask().getArray() == 0, 1, 0)
681 w2 = np.where(im2Area.getMask().getArray() == 0, 1, 0)
683 w12 = w1*w2
684 wDiff = np.where(diffIm.getMask().getArray() == 0, 1, 0)
685 w = w12*wDiff
687 maxRangeCov = self.config.maximumRangeCovariancesAstier
688 if covAstierRealSpace:
689 covDiffAstier = computeCovDirect(diffIm.getImage().getArray(), w, maxRangeCov)
690 else:
691 shapeDiff = diffIm.getImage().getArray().shape
692 fftShape = (fftSize(shapeDiff[0] + maxRangeCov), fftSize(shapeDiff[1]+maxRangeCov))
693 c = CovFft(diffIm.getImage().getArray(), w, fftShape, maxRangeCov)
694 covDiffAstier = c.reportCovFft(maxRangeCov)
696 return mu, varDiff, covDiffAstier
698 def computeCovDirect(self, diffImage, weightImage, maxRange):
699 """Compute covariances of diffImage in real space.
701 For lags larger than ~25, it is slower than the FFT way.
702 Taken from https://github.com/PierreAstier/bfptc/
704 Parameters
705 ----------
706 diffImage : `numpy.array`
707 Image to compute the covariance of.
709 weightImage : `numpy.array`
710 Weight image of diffImage (1's and 0's for good and bad pixels, respectively).
712 maxRange : `int`
713 Last index of the covariance to be computed.
715 Returns
716 -------
717 outList : `list`
718 List with tuples of the form (dx, dy, var, cov, npix), where:
719 dx : `int`
720 Lag in x
721 dy : `int`
722 Lag in y
723 var : `float`
724 Variance at (dx, dy).
725 cov : `float`
726 Covariance at (dx, dy).
727 nPix : `int`
728 Number of pixel pairs used to evaluate var and cov.
729 """
730 outList = []
731 var = 0
732 # (dy,dx) = (0,0) has to be first
733 for dy in range(maxRange + 1):
734 for dx in range(0, maxRange + 1):
735 if (dx*dy > 0):
736 cov1, nPix1 = self.covDirectValue(diffImage, weightImage, dx, dy)
737 cov2, nPix2 = self.covDirectValue(diffImage, weightImage, dx, -dy)
738 cov = 0.5*(cov1 + cov2)
739 nPix = nPix1 + nPix2
740 else:
741 cov, nPix = self.covDirectValue(diffImage, weightImage, dx, dy)
742 if (dx == 0 and dy == 0):
743 var = cov
744 outList.append((dx, dy, var, cov, nPix))
746 return outList
748 def covDirectValue(self, diffImage, weightImage, dx, dy):
749 """Compute covariances of diffImage in real space at lag (dx, dy).
751 Taken from https://github.com/PierreAstier/bfptc/ (c.f., appendix of Astier+19).
753 Parameters
754 ----------
755 diffImage : `numpy.array`
756 Image to compute the covariance of.
758 weightImage : `numpy.array`
759 Weight image of diffImage (1's and 0's for good and bad pixels, respectively).
761 dx : `int`
762 Lag in x.
764 dy : `int`
765 Lag in y.
767 Returns
768 -------
769 cov : `float`
770 Covariance at (dx, dy)
772 nPix : `int`
773 Number of pixel pairs used to evaluate var and cov.
774 """
775 (nCols, nRows) = diffImage.shape
776 # switching both signs does not change anything:
777 # it just swaps im1 and im2 below
778 if (dx < 0):
779 (dx, dy) = (-dx, -dy)
780 # now, we have dx >0. We have to distinguish two cases
781 # depending on the sign of dy
782 if dy >= 0:
783 im1 = diffImage[dy:, dx:]
784 w1 = weightImage[dy:, dx:]
785 im2 = diffImage[:nCols - dy, :nRows - dx]
786 w2 = weightImage[:nCols - dy, :nRows - dx]
787 else:
788 im1 = diffImage[:nCols + dy, dx:]
789 w1 = weightImage[:nCols + dy, dx:]
790 im2 = diffImage[-dy:, :nRows - dx]
791 w2 = weightImage[-dy:, :nRows - dx]
792 # use the same mask for all 3 calculations
793 wAll = w1*w2
794 # do not use mean() because weightImage=0 pixels would then count
795 nPix = wAll.sum()
796 im1TimesW = im1*wAll
797 s1 = im1TimesW.sum()/nPix
798 s2 = (im2*wAll).sum()/nPix
799 p = (im1TimesW*im2).sum()/nPix
800 cov = p - s1*s2
802 return cov, nPix
804 @staticmethod
805 def _initialParsForPolynomial(order):
806 assert(order >= 2)
807 pars = np.zeros(order, dtype=np.float)
808 pars[0] = 10
809 pars[1] = 1
810 pars[2:] = 0.0001
811 return pars
813 @staticmethod
814 def _boundsForPolynomial(initialPars, lowers=[], uppers=[]):
815 if not len(lowers):
816 lowers = [np.NINF for p in initialPars]
817 if not len(uppers):
818 uppers = [np.inf for p in initialPars]
819 lowers[1] = 0 # no negative gains
820 return (lowers, uppers)
822 @staticmethod
823 def _boundsForAstier(initialPars, lowers=[], uppers=[]):
824 if not len(lowers):
825 lowers = [np.NINF for p in initialPars]
826 if not len(uppers):
827 uppers = [np.inf for p in initialPars]
828 return (lowers, uppers)
830 @staticmethod
831 def _getInitialGoodPoints(means, variances, maxDeviationPositive, maxDeviationNegative):
832 """Return a boolean array to mask bad points.
834 Parameters
835 ----------
836 means : `numpy.array`
837 Input array with mean signal values.
839 variances : `numpy.array`
840 Input array with variances at each mean value.
842 maxDeviationPositive : `float`
843 Maximum deviation from being constant for the variance/mean
844 ratio, in the positive direction.
846 maxDeviationNegative : `float`
847 Maximum deviation from being constant for the variance/mean
848 ratio, in the negative direction.
850 Return
851 ------
852 goodPoints : `numpy.array` [`bool`]
853 Boolean array to select good (`True`) and bad (`False`)
854 points.
856 Notes
857 -----
858 A linear function has a constant ratio, so find the median
859 value of the ratios, and exclude the points that deviate
860 from that by more than a factor of maxDeviationPositive/negative.
861 Asymmetric deviations are supported as we expect the PTC to turn
862 down as the flux increases, but sometimes it anomalously turns
863 upwards just before turning over, which ruins the fits, so it
864 is wise to be stricter about restricting positive outliers than
865 negative ones.
867 Too high and points that are so bad that fit will fail will be included
868 Too low and the non-linear points will be excluded, biasing the NL fit."""
870 assert(len(means) == len(variances))
871 ratios = [b/a for (a, b) in zip(means, variances)]
872 medianRatio = np.nanmedian(ratios)
873 ratioDeviations = [(r/medianRatio)-1 for r in ratios]
875 # so that it doesn't matter if the deviation is expressed as positive or negative
876 maxDeviationPositive = abs(maxDeviationPositive)
877 maxDeviationNegative = -1. * abs(maxDeviationNegative)
879 goodPoints = np.array([True if (r < maxDeviationPositive and r > maxDeviationNegative)
880 else False for r in ratioDeviations])
881 return goodPoints
883 def _makeZeroSafe(self, array, warn=True, substituteValue=1e-9):
884 """"""
885 nBad = Counter(array)[0]
886 if nBad == 0:
887 return array
889 if warn:
890 msg = f"Found {nBad} zeros in array at elements {[x for x in np.where(array==0)[0]]}"
891 self.log.warn(msg)
893 array[array == 0] = substituteValue
894 return array
896 def fitPtc(self, dataset, ptcFitType):
897 """Fit the photon transfer curve to a polynimial or to Astier+19 approximation.
899 Fit the photon transfer curve with either a polynomial of the order
900 specified in the task config, or using the Astier approximation.
902 Sigma clipping is performed iteratively for the fit, as well as an
903 initial clipping of data points that are more than
904 config.initialNonLinearityExclusionThreshold away from lying on a
905 straight line. This other step is necessary because the photon transfer
906 curve turns over catastrophically at very high flux (because saturation
907 drops the variance to ~0) and these far outliers cause the initial fit
908 to fail, meaning the sigma cannot be calculated to perform the
909 sigma-clipping.
911 Parameters
912 ----------
913 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset`
914 The dataset containing the means, variances and exposure times
916 ptcFitType : `str`
917 Fit a 'POLYNOMIAL' (degree: 'polynomialFitDegree') or
918 'EXPAPPROXIMATION' (Eq. 16 of Astier+19) to the PTC
920 Returns
921 -------
922 dataset: `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset`
923 This is the same dataset as the input paramter, however, it has been modified
924 to include information such as the fit vectors and the fit parameters. See
925 the class `PhotonTransferCurveDatase`.
926 """
928 matrixSide = self.config.maximumRangeCovariancesAstier
929 nanMatrix = np.empty((matrixSide, matrixSide))
930 nanMatrix[:] = np.nan
932 for amp in dataset.ampNames:
933 lenInputTimes = len(dataset.rawExpTimes[amp])
934 listNanMatrix = np.empty((lenInputTimes, matrixSide, matrixSide))
935 listNanMatrix[:] = np.nan
937 dataset.covariances[amp] = listNanMatrix
938 dataset.covariancesModel[amp] = listNanMatrix
939 dataset.covariancesSqrtWeights[amp] = listNanMatrix
940 dataset.aMatrix[amp] = nanMatrix
941 dataset.bMatrix[amp] = nanMatrix
942 dataset.covariancesNoB[amp] = listNanMatrix
943 dataset.covariancesModelNoB[amp] = listNanMatrix
944 dataset.covariancesSqrtWeightsNoB[amp] = listNanMatrix
945 dataset.aMatrixNoB[amp] = nanMatrix
947 def errFunc(p, x, y):
948 return ptcFunc(p, x) - y
950 sigmaCutPtcOutliers = self.config.sigmaCutPtcOutliers
951 maxIterationsPtcOutliers = self.config.maxIterationsPtcOutliers
953 for i, ampName in enumerate(dataset.ampNames):
954 timeVecOriginal = np.array(dataset.rawExpTimes[ampName])
955 meanVecOriginal = np.array(dataset.rawMeans[ampName])
956 varVecOriginal = np.array(dataset.rawVars[ampName])
957 varVecOriginal = self._makeZeroSafe(varVecOriginal)
959 goodPoints = self._getInitialGoodPoints(meanVecOriginal, varVecOriginal,
960 self.config.initialNonLinearityExclusionThresholdPositive,
961 self.config.initialNonLinearityExclusionThresholdNegative)
962 if not (goodPoints.any()):
963 msg = (f"\nSERIOUS: All points in goodPoints: {goodPoints} are bad."
964 f"Setting {ampName} to BAD.")
965 self.log.warn(msg)
966 # The first and second parameters of initial fit are discarded (bias and gain)
967 # for the final NL coefficients
968 dataset.badAmps.append(ampName)
969 dataset.expIdMask[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName]))
970 dataset.gain[ampName] = np.nan
971 dataset.gainErr[ampName] = np.nan
972 dataset.noise[ampName] = np.nan
973 dataset.noiseErr[ampName] = np.nan
974 dataset.ptcFitPars[ampName] = (np.repeat(np.nan, self.config.polynomialFitDegree + 1) if
975 ptcFitType in ["POLYNOMIAL", ] else np.repeat(np.nan, 3))
976 dataset.ptcFitParsError[ampName] = (np.repeat(np.nan, self.config.polynomialFitDegree + 1) if
977 ptcFitType in ["POLYNOMIAL", ] else np.repeat(np.nan, 3))
978 dataset.ptcFitChiSq[ampName] = np.nan
979 dataset.finalVars[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName]))
980 dataset.finalModelVars[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName]))
981 dataset.finalMeans[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName]))
982 continue
984 mask = goodPoints
986 if ptcFitType == 'EXPAPPROXIMATION':
987 ptcFunc = funcAstier
988 parsIniPtc = [-1e-9, 1.0, 10.] # a00, gain, noise
989 # lowers and uppers obtained from studies by C. Lage (UC Davis, 11/2020).
990 bounds = self._boundsForAstier(parsIniPtc, lowers=[-1e-4, 0.5, -100],
991 uppers=[1e-4, 2.5, 100])
992 if ptcFitType == 'POLYNOMIAL':
993 ptcFunc = funcPolynomial
994 parsIniPtc = self._initialParsForPolynomial(self.config.polynomialFitDegree + 1)
995 bounds = self._boundsForPolynomial(parsIniPtc)
997 # Before bootstrap fit, do an iterative fit to get rid of outliers
998 count = 1
999 while count <= maxIterationsPtcOutliers:
1000 # Note that application of the mask actually shrinks the array
1001 # to size rather than setting elements to zero (as we want) so
1002 # always update mask itself and re-apply to the original data
1003 meanTempVec = meanVecOriginal[mask]
1004 varTempVec = varVecOriginal[mask]
1005 res = least_squares(errFunc, parsIniPtc, bounds=bounds, args=(meanTempVec, varTempVec))
1006 pars = res.x
1008 # change this to the original from the temp because the masks are ANDed
1009 # meaning once a point is masked it's always masked, and the masks must
1010 # always be the same length for broadcasting
1011 sigResids = (varVecOriginal - ptcFunc(pars, meanVecOriginal))/np.sqrt(varVecOriginal)
1012 newMask = np.array([True if np.abs(r) < sigmaCutPtcOutliers else False for r in sigResids])
1013 mask = mask & newMask
1014 if not (mask.any() and newMask.any()):
1015 msg = (f"\nSERIOUS: All points in either mask: {mask} or newMask: {newMask} are bad. "
1016 f"Setting {ampName} to BAD.")
1017 self.log.warn(msg)
1018 # The first and second parameters of initial fit are discarded (bias and gain)
1019 # for the final NL coefficients
1020 dataset.badAmps.append(ampName)
1021 dataset.expIdMask[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName]))
1022 dataset.gain[ampName] = np.nan
1023 dataset.gainErr[ampName] = np.nan
1024 dataset.noise[ampName] = np.nan
1025 dataset.noiseErr[ampName] = np.nan
1026 dataset.ptcFitPars[ampName] = (np.repeat(np.nan, self.config.polynomialFitDegree + 1)
1027 if ptcFitType in ["POLYNOMIAL", ] else
1028 np.repeat(np.nan, 3))
1029 dataset.ptcFitParsError[ampName] = (np.repeat(np.nan, self.config.polynomialFitDegree + 1)
1030 if ptcFitType in ["POLYNOMIAL", ] else
1031 np.repeat(np.nan, 3))
1032 dataset.ptcFitChiSq[ampName] = np.nan
1033 dataset.finalVars[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName]))
1034 dataset.finalModelVars[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName]))
1035 dataset.finalMeans[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName]))
1036 break
1037 nDroppedTotal = Counter(mask)[False]
1038 self.log.debug(f"Iteration {count}: discarded {nDroppedTotal} points in total for {ampName}")
1039 count += 1
1040 # objects should never shrink
1041 assert (len(mask) == len(timeVecOriginal) == len(meanVecOriginal) == len(varVecOriginal))
1043 if not (mask.any() and newMask.any()):
1044 continue
1045 dataset.expIdMask[ampName] = mask # store the final mask
1046 parsIniPtc = pars
1047 meanVecFinal = meanVecOriginal[mask]
1048 varVecFinal = varVecOriginal[mask]
1050 if Counter(mask)[False] > 0:
1051 self.log.info((f"Number of points discarded in PTC of amplifier {ampName}:" +
1052 f" {Counter(mask)[False]} out of {len(meanVecOriginal)}"))
1054 if (len(meanVecFinal) < len(parsIniPtc)):
1055 msg = (f"\nSERIOUS: Not enough data points ({len(meanVecFinal)}) compared to the number of"
1056 f"parameters of the PTC model({len(parsIniPtc)}). Setting {ampName} to BAD.")
1057 self.log.warn(msg)
1058 # The first and second parameters of initial fit are discarded (bias and gain)
1059 # for the final NL coefficients
1060 dataset.badAmps.append(ampName)
1061 dataset.expIdMask[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName]))
1062 dataset.gain[ampName] = np.nan
1063 dataset.gainErr[ampName] = np.nan
1064 dataset.noise[ampName] = np.nan
1065 dataset.noiseErr[ampName] = np.nan
1066 dataset.ptcFitPars[ampName] = (np.repeat(np.nan, self.config.polynomialFitDegree + 1) if
1067 ptcFitType in ["POLYNOMIAL", ] else np.repeat(np.nan, 3))
1068 dataset.ptcFitParsError[ampName] = (np.repeat(np.nan, self.config.polynomialFitDegree + 1) if
1069 ptcFitType in ["POLYNOMIAL", ] else np.repeat(np.nan, 3))
1070 dataset.ptcFitChiSq[ampName] = np.nan
1071 dataset.finalVars[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName]))
1072 dataset.finalModelVars[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName]))
1073 dataset.finalMeans[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName]))
1074 continue
1076 # Fit the PTC
1077 if self.config.doFitBootstrap:
1078 parsFit, parsFitErr, reducedChiSqPtc = fitBootstrap(parsIniPtc, meanVecFinal,
1079 varVecFinal, ptcFunc,
1080 weightsY=1./np.sqrt(varVecFinal))
1081 else:
1082 parsFit, parsFitErr, reducedChiSqPtc = fitLeastSq(parsIniPtc, meanVecFinal,
1083 varVecFinal, ptcFunc,
1084 weightsY=1./np.sqrt(varVecFinal))
1085 dataset.ptcFitPars[ampName] = parsFit
1086 dataset.ptcFitParsError[ampName] = parsFitErr
1087 dataset.ptcFitChiSq[ampName] = reducedChiSqPtc
1088 # Masked variances (measured and modeled) and means. Need to pad the array so astropy.Table does
1089 # not crash (the mask may vary per amp).
1090 padLength = len(dataset.rawExpTimes[ampName]) - len(varVecFinal)
1091 dataset.finalVars[ampName] = np.pad(varVecFinal, (0, padLength), 'constant',
1092 constant_values=np.nan)
1093 dataset.finalModelVars[ampName] = np.pad(ptcFunc(parsFit, meanVecFinal), (0, padLength),
1094 'constant', constant_values=np.nan)
1095 dataset.finalMeans[ampName] = np.pad(meanVecFinal, (0, padLength), 'constant',
1096 constant_values=np.nan)
1098 if ptcFitType == 'EXPAPPROXIMATION':
1099 ptcGain = parsFit[1]
1100 ptcGainErr = parsFitErr[1]
1101 ptcNoise = np.sqrt(np.fabs(parsFit[2]))
1102 ptcNoiseErr = 0.5*(parsFitErr[2]/np.fabs(parsFit[2]))*np.sqrt(np.fabs(parsFit[2]))
1103 if ptcFitType == 'POLYNOMIAL':
1104 ptcGain = 1./parsFit[1]
1105 ptcGainErr = np.fabs(1./parsFit[1])*(parsFitErr[1]/parsFit[1])
1106 ptcNoise = np.sqrt(np.fabs(parsFit[0]))*ptcGain
1107 ptcNoiseErr = (0.5*(parsFitErr[0]/np.fabs(parsFit[0]))*(np.sqrt(np.fabs(parsFit[0]))))*ptcGain
1108 dataset.gain[ampName] = ptcGain
1109 dataset.gainErr[ampName] = ptcGainErr
1110 dataset.noise[ampName] = ptcNoise
1111 dataset.noiseErr[ampName] = ptcNoiseErr
1112 if not len(dataset.ptcFitType) == 0:
1113 dataset.ptcFitType = ptcFitType
1114 if len(dataset.badAmps) == 0:
1115 dataset.badAmps = np.repeat(np.nan, len(list(dataset.rawExpTimes.values())[0]))
1117 return dataset