Coverage for python/lsst/cp/pipe/ptc.py : 9%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of cp_pipe.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21#
22import numpy as np
23import matplotlib.pyplot as plt
24from collections import Counter
26import lsst.afw.math as afwMath
27import lsst.pex.config as pexConfig
28import lsst.pipe.base as pipeBase
29from .utils import (fitLeastSq, fitBootstrap, funcPolynomial, funcAstier)
30from scipy.optimize import least_squares
32import datetime
34from .astierCovPtcUtils import (fftSize, CovFft, computeCovDirect, fitData)
35from .linearity import LinearitySolveTask
36from .photodiode import getBOTphotodiodeData
38from lsst.pipe.tasks.getRepositoryData import DataRefListRunner
39from lsst.ip.isr import PhotonTransferCurveDataset
41import copy
43__all__ = ['MeasurePhotonTransferCurveTask',
44 'MeasurePhotonTransferCurveTaskConfig']
47class MeasurePhotonTransferCurveTaskConfig(pexConfig.Config):
48 """Config class for photon transfer curve measurement task"""
49 ccdKey = pexConfig.Field(
50 dtype=str,
51 doc="The key by which to pull a detector from a dataId, e.g. 'ccd' or 'detector'.",
52 default='ccd',
53 )
54 ptcFitType = pexConfig.ChoiceField(
55 dtype=str,
56 doc="Fit PTC to Eq. 16, Eq. 20 in Astier+19, or to a polynomial.",
57 default="POLYNOMIAL",
58 allowed={
59 "POLYNOMIAL": "n-degree polynomial (use 'polynomialFitDegree' to set 'n').",
60 "EXPAPPROXIMATION": "Approximation in Astier+19 (Eq. 16).",
61 "FULLCOVARIANCE": "Full covariances model in Astier+19 (Eq. 20)"
62 }
63 )
64 maximumRangeCovariancesAstier = pexConfig.Field(
65 dtype=int,
66 doc="Maximum range of covariances as in Astier+19",
67 default=8,
68 )
69 covAstierRealSpace = pexConfig.Field(
70 dtype=bool,
71 doc="Calculate covariances in real space or via FFT? (see appendix A of Astier+19).",
72 default=False,
73 )
74 polynomialFitDegree = pexConfig.Field(
75 dtype=int,
76 doc="Degree of polynomial to fit the PTC, when 'ptcFitType'=POLYNOMIAL.",
77 default=3,
78 )
79 linearity = pexConfig.ConfigurableField(
80 target=LinearitySolveTask,
81 doc="Task to solve the linearity."
82 )
84 doCreateLinearizer = pexConfig.Field(
85 dtype=bool,
86 doc="Calculate non-linearity and persist linearizer?",
87 default=False,
88 )
90 binSize = pexConfig.Field(
91 dtype=int,
92 doc="Bin the image by this factor in both dimensions.",
93 default=1,
94 )
95 minMeanSignal = pexConfig.DictField(
96 keytype=str,
97 itemtype=float,
98 doc="Minimum values (inclusive) of mean signal (in ADU) above which to consider, per amp."
99 " The same cut is applied to all amps if this dictionary is of the form"
100 " {'ALL_AMPS': value}",
101 default={'ALL_AMPS': 0.0},
102 )
103 maxMeanSignal = pexConfig.DictField(
104 keytype=str,
105 itemtype=float,
106 doc="Maximum values (inclusive) of mean signal (in ADU) below which to consider, per amp."
107 " The same cut is applied to all amps if this dictionary is of the form"
108 " {'ALL_AMPS': value}",
109 default={'ALL_AMPS': 1e6},
110 )
111 initialNonLinearityExclusionThresholdPositive = pexConfig.RangeField(
112 dtype=float,
113 doc="Initially exclude data points with a variance that are more than a factor of this from being"
114 " linear in the positive direction, from the PTC fit. Note that these points will also be"
115 " excluded from the non-linearity fit. This is done before the iterative outlier rejection,"
116 " to allow an accurate determination of the sigmas for said iterative fit.",
117 default=0.12,
118 min=0.0,
119 max=1.0,
120 )
121 initialNonLinearityExclusionThresholdNegative = pexConfig.RangeField(
122 dtype=float,
123 doc="Initially exclude data points with a variance that are more than a factor of this from being"
124 " linear in the negative direction, from the PTC fit. Note that these points will also be"
125 " excluded from the non-linearity fit. This is done before the iterative outlier rejection,"
126 " to allow an accurate determination of the sigmas for said iterative fit.",
127 default=0.25,
128 min=0.0,
129 max=1.0,
130 )
131 sigmaCutPtcOutliers = pexConfig.Field(
132 dtype=float,
133 doc="Sigma cut for outlier rejection in PTC.",
134 default=5.0,
135 )
136 maskNameList = pexConfig.ListField(
137 dtype=str,
138 doc="Mask list to exclude from statistics calculations.",
139 default=['SUSPECT', 'BAD', 'NO_DATA'],
140 )
141 nSigmaClipPtc = pexConfig.Field(
142 dtype=float,
143 doc="Sigma cut for afwMath.StatisticsControl()",
144 default=5.5,
145 )
146 nIterSigmaClipPtc = pexConfig.Field(
147 dtype=int,
148 doc="Number of sigma-clipping iterations for afwMath.StatisticsControl()",
149 default=1,
150 )
151 minNumberGoodPixelsForFft = pexConfig.Field(
152 dtype=int,
153 doc="Minimum number of acceptable good pixels per amp to calculate the covariances via FFT.",
154 default=10000,
155 )
156 maxIterationsPtcOutliers = pexConfig.Field(
157 dtype=int,
158 doc="Maximum number of iterations for outlier rejection in PTC.",
159 default=2,
160 )
161 doFitBootstrap = pexConfig.Field(
162 dtype=bool,
163 doc="Use bootstrap for the PTC fit parameters and errors?.",
164 default=False,
165 )
166 doPhotodiode = pexConfig.Field(
167 dtype=bool,
168 doc="Apply a correction based on the photodiode readings if available?",
169 default=True,
170 )
171 photodiodeDataPath = pexConfig.Field(
172 dtype=str,
173 doc="Gen2 only: path to locate the data photodiode data files.",
174 default=""
175 )
176 instrumentName = pexConfig.Field(
177 dtype=str,
178 doc="Instrument name.",
179 default='',
180 )
183class MeasurePhotonTransferCurveTask(pipeBase.CmdLineTask):
184 """A class to calculate, fit, and plot a PTC from a set of flat pairs.
186 The Photon Transfer Curve (var(signal) vs mean(signal)) is a standard tool
187 used in astronomical detectors characterization (e.g., Janesick 2001,
188 Janesick 2007). If ptcFitType is "EXPAPPROXIMATION" or "POLYNOMIAL", this task calculates the
189 PTC from a series of pairs of flat-field images; each pair taken at identical exposure
190 times. The difference image of each pair is formed to eliminate fixed pattern noise,
191 and then the variance of the difference image and the mean of the average image
192 are used to produce the PTC. An n-degree polynomial or the approximation in Equation
193 16 of Astier+19 ("The Shape of the Photon Transfer Curve of CCD sensors",
194 arXiv:1905.08677) can be fitted to the PTC curve. These models include
195 parameters such as the gain (e/DN) and readout noise.
197 Linearizers to correct for signal-chain non-linearity are also calculated.
198 The `Linearizer` class, in general, can support per-amp linearizers, but in this
199 task this is not supported.
201 If ptcFitType is "FULLCOVARIANCE", the covariances of the difference images are calculated via the
202 DFT methods described in Astier+19 and the variances for the PTC are given by the cov[0,0] elements
203 at each signal level. The full model in Equation 20 of Astier+19 is fit to the PTC to get the gain
204 and the noise.
206 Parameters
207 ----------
209 *args: `list`
210 Positional arguments passed to the Task constructor. None used at this
211 time.
212 **kwargs: `dict`
213 Keyword arguments passed on to the Task constructor. None used at this
214 time.
216 """
218 RunnerClass = DataRefListRunner
219 ConfigClass = MeasurePhotonTransferCurveTaskConfig
220 _DefaultName = "measurePhotonTransferCurve"
222 def __init__(self, *args, **kwargs):
223 pipeBase.CmdLineTask.__init__(self, *args, **kwargs)
224 self.makeSubtask("linearity")
225 plt.interactive(False) # stop windows popping up when plotting. When headless, use 'agg' backend too
226 self.config.validate()
227 self.config.freeze()
229 @pipeBase.timeMethod
230 def runDataRef(self, dataRefList):
231 """Run the Photon Transfer Curve (PTC) measurement task.
233 For a dataRef (which is each detector here),
234 and given a list of exposure pairs (postISR) at different exposure times,
235 measure the PTC.
237 Parameters
238 ----------
239 dataRefList : `list` [`lsst.daf.peristence.ButlerDataRef`]
240 Data references for exposures for detectors to process.
241 """
242 if len(dataRefList) < 2:
243 raise RuntimeError("Insufficient inputs to combine.")
245 # setup necessary objects
246 dataRef = dataRefList[0]
248 detNum = dataRef.dataId[self.config.ccdKey]
249 camera = dataRef.get('camera')
250 detector = camera[dataRef.dataId[self.config.ccdKey]]
252 amps = detector.getAmplifiers()
253 ampNames = [amp.getName() for amp in amps]
254 datasetPtc = PhotonTransferCurveDataset(ampNames, self.config.ptcFitType)
256 # Get the pairs of flat indexed by expTime
257 expPairs = self.makePairs(dataRefList)
258 expIds = []
259 for (exp1, exp2) in expPairs.values():
260 id1 = exp1.getInfo().getVisitInfo().getExposureId()
261 id2 = exp2.getInfo().getVisitInfo().getExposureId()
262 expIds.append((id1, id2))
263 self.log.info(f"Measuring PTC using {expIds} exposures for detector {detector.getId()}")
265 # get photodiode data early so that logic can be put in to only use the
266 # data if all files are found, as partial corrections are not possible
267 # or at least require significant logic to deal with
268 if self.config.doPhotodiode:
269 for (expId1, expId2) in expIds:
270 charges = [-1, -1] # necessary to have a not-found value to keep lists in step
271 for i, expId in enumerate([expId1, expId2]):
272 # //1000 is a Gen2 only hack, working around the fact an
273 # exposure's ID is not the same as the expId in the
274 # registry. Currently expId is concatenated with the
275 # zero-padded detector ID. This will all go away in Gen3.
276 dataRef.dataId['expId'] = expId//1000
277 if self.config.photodiodeDataPath:
278 photodiodeData = getBOTphotodiodeData(dataRef, self.config.photodiodeDataPath)
279 else:
280 photodiodeData = getBOTphotodiodeData(dataRef)
281 if photodiodeData: # default path stored in function def to keep task clean
282 charges[i] = photodiodeData.getCharge()
283 else:
284 # full expId (not //1000) here, as that encodes the
285 # the detector number as so is fully qualifying
286 self.log.warn(f"No photodiode data found for {expId}")
288 for ampName in ampNames:
289 datasetPtc.photoCharge[ampName].append((charges[0], charges[1]))
290 else:
291 # Can't be an empty list, as initialized, because astropy.Table won't allow it
292 # when saving as fits
293 for ampName in ampNames:
294 datasetPtc.photoCharge[ampName] = np.repeat(np.nan, len(expIds))
296 for ampName in ampNames:
297 datasetPtc.inputExpIdPairs[ampName] = expIds
299 maxMeanSignalDict = {ampName: 1e6 for ampName in ampNames}
300 minMeanSignalDict = {ampName: 0.0 for ampName in ampNames}
301 for ampName in ampNames:
302 if 'ALL_AMPS' in self.config.maxMeanSignal:
303 maxMeanSignalDict[ampName] = self.config.maxMeanSignal['ALL_AMPS']
304 elif ampName in self.config.maxMeanSignal:
305 maxMeanSignalDict[ampName] = self.config.maxMeanSignal[ampName]
307 if 'ALL_AMPS' in self.config.minMeanSignal:
308 minMeanSignalDict[ampName] = self.config.minMeanSignal['ALL_AMPS']
309 elif ampName in self.config.minMeanSignal:
310 minMeanSignalDict[ampName] = self.config.minMeanSignal[ampName]
312 tupleRecords = []
313 allTags = []
314 for expTime, (exp1, exp2) in expPairs.items():
315 expId1 = exp1.getInfo().getVisitInfo().getExposureId()
316 expId2 = exp2.getInfo().getVisitInfo().getExposureId()
317 tupleRows = []
318 nAmpsNan = 0
319 tags = ['mu', 'i', 'j', 'var', 'cov', 'npix', 'ext', 'expTime', 'ampName']
320 for ampNumber, amp in enumerate(detector):
321 ampName = amp.getName()
322 # covAstier: (i, j, var (cov[0,0]), cov, npix)
323 doRealSpace = self.config.covAstierRealSpace
324 muDiff, varDiff, covAstier = self.measureMeanVarCov(exp1, exp2, region=amp.getBBox(),
325 covAstierRealSpace=doRealSpace)
327 if np.isnan(muDiff) or np.isnan(varDiff) or (covAstier is None):
328 msg = (f"NaN mean or var, or None cov in amp {ampName} in exposure pair {expId1},"
329 f" {expId2} of detector {detNum}.")
330 self.log.warn(msg)
331 nAmpsNan += 1
332 continue
333 if (muDiff <= minMeanSignalDict[ampName]) or (muDiff >= maxMeanSignalDict[ampName]):
334 continue
336 datasetPtc.rawExpTimes[ampName].append(expTime)
337 datasetPtc.rawMeans[ampName].append(muDiff)
338 datasetPtc.rawVars[ampName].append(varDiff)
340 tupleRows += [(muDiff, ) + covRow + (ampNumber, expTime, ampName) for covRow in covAstier]
341 if nAmpsNan == len(ampNames):
342 msg = f"NaN mean in all amps of exposure pair {expId1}, {expId2} of detector {detNum}."
343 self.log.warn(msg)
344 continue
345 allTags += tags
346 tupleRecords += tupleRows
347 covariancesWithTags = np.core.records.fromrecords(tupleRecords, names=allTags)
349 for ampName in datasetPtc.ampNames:
350 # Sort raw vectors by rawMeans index
351 index = np.argsort(datasetPtc.rawMeans[ampName])
352 datasetPtc.rawExpTimes[ampName] = np.array(datasetPtc.rawExpTimes[ampName])[index]
353 datasetPtc.rawMeans[ampName] = np.array(datasetPtc.rawMeans[ampName])[index]
354 datasetPtc.rawVars[ampName] = np.array(datasetPtc.rawVars[ampName])[index]
356 if self.config.ptcFitType in ["FULLCOVARIANCE", ]:
357 # Calculate covariances and fit them, including the PTC, to Astier+19 full model (Eq. 20)
358 # First, fit get the flat pairs that are masked, according to the regular PTC (C_00 vs mu)
359 # The points at these fluxes will also be masked when calculating the other covariances, C_ij)
360 newDatasetPtc = copy.copy(datasetPtc)
361 newDatasetPtc = self.fitPtc(newDatasetPtc, 'EXPAPPROXIMATION')
362 for ampName in datasetPtc.ampNames:
363 datasetPtc.expIdMask[ampName] = newDatasetPtc.expIdMask[ampName]
365 datasetPtc.fitType = "FULLCOVARIANCE"
366 datasetPtc = self.fitCovariancesAstier(datasetPtc, covariancesWithTags)
367 elif self.config.ptcFitType in ["EXPAPPROXIMATION", "POLYNOMIAL"]:
368 # Fit the PTC to a polynomial or to Astier+19 exponential approximation (Eq. 16)
369 # Fill up PhotonTransferCurveDataset object.
370 datasetPtc = self.fitPtc(datasetPtc, self.config.ptcFitType)
372 detName = detector.getName()
373 now = datetime.datetime.utcnow()
374 calibDate = now.strftime("%Y-%m-%d")
375 butler = dataRef.getButler()
377 datasetPtc.updateMetadata(setDate=True, camera=camera, detector=detector)
379 # Fit a poynomial to calculate non-linearity and persist linearizer.
380 if self.config.doCreateLinearizer:
381 # Fit (non)linearity of signal vs time curve.
382 # Fill up PhotonTransferCurveDataset object.
383 # Fill up array for LUT linearizer (tableArray).
384 # Produce coefficients for Polynomial and Squared linearizers.
385 # Build linearizer objects.
386 dimensions = {'camera': camera.getName(), 'detector': detector.getId()}
387 linearityResults = self.linearity.run(datasetPtc, camera, dimensions)
388 linearizer = linearityResults.outputLinearizer
390 self.log.info("Writing linearizer:")
392 detName = detector.getName()
393 now = datetime.datetime.utcnow()
394 calibDate = now.strftime("%Y-%m-%d")
396 butler.put(linearizer, datasetType='linearizer',
397 dataId={'detector': detNum, 'detectorName': detName, 'calibDate': calibDate})
399 self.log.info(f"Writing PTC data.")
400 butler.put(datasetPtc, datasetType='photonTransferCurveDataset', dataId={'detector': detNum,
401 'detectorName': detName, 'calibDate': calibDate})
403 return pipeBase.Struct(exitStatus=0)
405 def makePairs(self, dataRefList):
406 """Produce a list of flat pairs indexed by exposure time.
408 Parameters
409 ----------
410 dataRefList : `list` [`lsst.daf.peristence.ButlerDataRef`]
411 Data references for exposures for detectors to process.
413 Return
414 ------
415 flatPairs : `dict` [`float`, `lsst.afw.image.exposure.exposure.ExposureF`]
416 Dictionary that groups flat-field exposures that have the same exposure time (seconds).
418 Notes
419 -----
420 We use the difference of one pair of flat-field images taken at the same exposure time when
421 calculating the PTC to reduce Fixed Pattern Noise. If there are > 2 flat-field images with the
422 same exposure time, the first two are kept and the rest discarded.
423 """
425 # Organize exposures by observation date.
426 expDict = {}
427 for dataRef in dataRefList:
428 try:
429 tempFlat = dataRef.get("postISRCCD")
430 except RuntimeError:
431 self.log.warn("postISR exposure could not be retrieved. Ignoring flat.")
432 continue
433 expDate = tempFlat.getInfo().getVisitInfo().getDate().get()
434 expDict.setdefault(expDate, tempFlat)
435 sortedExps = {k: expDict[k] for k in sorted(expDict)}
437 flatPairs = {}
438 for exp in sortedExps:
439 tempFlat = sortedExps[exp]
440 expTime = tempFlat.getInfo().getVisitInfo().getExposureTime()
441 listAtExpTime = flatPairs.setdefault(expTime, [])
442 if len(listAtExpTime) >= 2:
443 self.log.warn(f"Already found 2 exposures at expTime {expTime}. "
444 f"Ignoring exposure {tempFlat.getInfo().getVisitInfo().getExposureId()}")
445 else:
446 listAtExpTime.append(tempFlat)
448 keysToDrop = []
449 for (key, value) in flatPairs.items():
450 if len(value) < 2:
451 keysToDrop.append(key)
453 if len(keysToDrop):
454 for key in keysToDrop:
455 self.log.warn(f"Only one exposure found at expTime {key}. Dropping exposure "
456 f"{flatPairs[key][0].getInfo().getVisitInfo().getExposureId()}.")
457 flatPairs.pop(key)
458 sortedFlatPairs = {k: flatPairs[k] for k in sorted(flatPairs)}
459 return sortedFlatPairs
461 def fitCovariancesAstier(self, dataset, covariancesWithTagsArray):
462 """Fit measured flat covariances to full model in Astier+19.
464 Parameters
465 ----------
466 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset`
467 The dataset containing information such as the means, variances and exposure times.
469 covariancesWithTagsArray : `numpy.recarray`
470 Tuple with at least (mu, cov, var, i, j, npix), where:
471 mu : 0.5*(m1 + m2), where:
472 mu1: mean value of flat1
473 mu2: mean value of flat2
474 cov: covariance value at lag(i, j)
475 var: variance(covariance value at lag(0, 0))
476 i: lag dimension
477 j: lag dimension
478 npix: number of pixels used for covariance calculation.
480 Returns
481 -------
482 dataset: `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset`
483 This is the same dataset as the input paramter, however, it has been modified
484 to include information such as the fit vectors and the fit parameters. See
485 the class `PhotonTransferCurveDatase`.
486 """
488 covFits, covFitsNoB = fitData(covariancesWithTagsArray,
489 r=self.config.maximumRangeCovariancesAstier,
490 expIdMask=dataset.expIdMask)
491 dataset = self.getOutputPtcDataCovAstier(dataset, covFits, covFitsNoB)
492 return dataset
494 def getOutputPtcDataCovAstier(self, dataset, covFits, covFitsNoB):
495 """Get output data for PhotonTransferCurveCovAstierDataset from CovFit objects.
497 Parameters
498 ----------
499 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset`
500 The dataset containing information such as the means, variances and exposure times.
502 covFits: `dict`
503 Dictionary of CovFit objects, with amp names as keys.
505 covFitsNoB : `dict`
506 Dictionary of CovFit objects, with amp names as keys, and 'b=0' in Eq. 20 of Astier+19.
508 Returns
509 -------
510 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset`
511 This is the same dataset as the input paramter, however, it has been modified
512 to include extra information such as the mask 1D array, gains, reoudout noise, measured signal,
513 measured variance, modeled variance, a, and b coefficient matrices (see Astier+19) per amplifier.
514 See the class `PhotonTransferCurveDatase`.
515 """
516 assert(len(covFits) == len(covFitsNoB))
518 for i, amp in enumerate(dataset.ampNames):
519 lenInputTimes = len(dataset.rawExpTimes[amp])
520 # Not used when ptcFitType is 'FULLCOVARIANCE'
521 dataset.ptcFitPars[amp] = np.nan
522 dataset.ptcFitParsError[amp] = np.nan
523 dataset.ptcFitChiSq[amp] = np.nan
524 if (amp in covFits and (covFits[amp].covParams is not None) and
525 (covFitsNoB[amp].covParams is not None)):
526 fit = covFits[amp]
527 fitNoB = covFitsNoB[amp]
528 # Save full covariances, covariances models, and their weights
529 dataset.covariances[amp] = fit.cov
530 dataset.covariancesModel[amp] = fit.evalCovModel()
531 dataset.covariancesSqrtWeights[amp] = fit.sqrtW
532 dataset.aMatrix[amp] = fit.getA()
533 dataset.bMatrix[amp] = fit.getB()
534 dataset.covariancesNoB[amp] = fitNoB.cov
535 dataset.covariancesModelNoB[amp] = fitNoB.evalCovModel()
536 dataset.covariancesSqrtWeightsNoB[amp] = fitNoB.sqrtW
537 dataset.aMatrixNoB[amp] = fitNoB.getA()
539 (meanVecFinal, varVecFinal, varVecModel,
540 wc, varMask) = fit.getFitData(0, 0, divideByMu=False)
541 gain = fit.getGain()
543 dataset.gain[amp] = gain
544 dataset.gainErr[amp] = fit.getGainErr()
545 dataset.noise[amp] = np.sqrt(fit.getRon())
546 dataset.noiseErr[amp] = fit.getRonErr()
548 padLength = lenInputTimes - len(varVecFinal)
549 dataset.finalVars[amp] = np.pad(varVecFinal/(gain**2), (0, padLength), 'constant',
550 constant_values=np.nan)
551 dataset.finalModelVars[amp] = np.pad(varVecModel/(gain**2), (0, padLength), 'constant',
552 constant_values=np.nan)
553 dataset.finalMeans[amp] = np.pad(meanVecFinal/gain, (0, padLength), 'constant',
554 constant_values=np.nan)
555 else:
556 # Bad amp
557 # Entries need to have proper dimensions so read/write with astropy.Table works.
558 matrixSide = self.config.maximumRangeCovariancesAstier
559 nanMatrix = np.full((matrixSide, matrixSide), np.nan)
560 listNanMatrix = np.full((lenInputTimes, matrixSide, matrixSide), np.nan)
562 dataset.covariances[amp] = listNanMatrix
563 dataset.covariancesModel[amp] = listNanMatrix
564 dataset.covariancesSqrtWeights[amp] = listNanMatrix
565 dataset.aMatrix[amp] = nanMatrix
566 dataset.bMatrix[amp] = nanMatrix
567 dataset.covariancesNoB[amp] = listNanMatrix
568 dataset.covariancesModelNoB[amp] = listNanMatrix
569 dataset.covariancesSqrtWeightsNoB[amp] = listNanMatrix
570 dataset.aMatrixNoB[amp] = nanMatrix
572 dataset.expIdMask[amp] = np.repeat(np.nan, lenInputTimes)
573 dataset.gain[amp] = np.nan
574 dataset.gainErr[amp] = np.nan
575 dataset.noise[amp] = np.nan
576 dataset.noiseErr[amp] = np.nan
577 dataset.finalVars[amp] = np.repeat(np.nan, lenInputTimes)
578 dataset.finalModelVars[amp] = np.repeat(np.nan, lenInputTimes)
579 dataset.finalMeans[amp] = np.repeat(np.nan, lenInputTimes)
581 return dataset
583 def measureMeanVarCov(self, exposure1, exposure2, region=None, covAstierRealSpace=False):
584 """Calculate the mean of each of two exposures and the variance and covariance of their difference.
586 The variance is calculated via afwMath, and the covariance via the methods in Astier+19 (appendix A).
587 In theory, var = covariance[0,0]. This should be validated, and in the future, we may decide to just
588 keep one (covariance).
590 Parameters
591 ----------
592 exposure1 : `lsst.afw.image.exposure.exposure.ExposureF`
593 First exposure of flat field pair.
595 exposure2 : `lsst.afw.image.exposure.exposure.ExposureF`
596 Second exposure of flat field pair.
598 region : `lsst.geom.Box2I`, optional
599 Region of each exposure where to perform the calculations (e.g, an amplifier).
601 covAstierRealSpace : `bool`, optional
602 Should the covariannces in Astier+19 be calculated in real space or via FFT?
603 See Appendix A of Astier+19.
605 Returns
606 -------
607 mu : `float` or `NaN`
608 0.5*(mu1 + mu2), where mu1, and mu2 are the clipped means of the regions in
609 both exposures. If either mu1 or m2 are NaN's, the returned value is NaN.
611 varDiff : `float` or `NaN`
612 Half of the clipped variance of the difference of the regions inthe two input
613 exposures. If either mu1 or m2 are NaN's, the returned value is NaN.
615 covDiffAstier : `list` or `None`
616 List with tuples of the form (dx, dy, var, cov, npix), where:
617 dx : `int`
618 Lag in x
619 dy : `int`
620 Lag in y
621 var : `float`
622 Variance at (dx, dy).
623 cov : `float`
624 Covariance at (dx, dy).
625 nPix : `int`
626 Number of pixel pairs used to evaluate var and cov.
627 If either mu1 or m2 are NaN's, the returned value is None.
628 """
630 if region is not None:
631 im1Area = exposure1.maskedImage[region]
632 im2Area = exposure2.maskedImage[region]
633 else:
634 im1Area = exposure1.maskedImage
635 im2Area = exposure2.maskedImage
637 if self.config.binSize > 1:
638 im1Area = afwMath.binImage(im1Area, self.config.binSize)
639 im2Area = afwMath.binImage(im2Area, self.config.binSize)
641 im1MaskVal = exposure1.getMask().getPlaneBitMask(self.config.maskNameList)
642 im1StatsCtrl = afwMath.StatisticsControl(self.config.nSigmaClipPtc,
643 self.config.nIterSigmaClipPtc,
644 im1MaskVal)
645 im1StatsCtrl.setNanSafe(True)
646 im1StatsCtrl.setAndMask(im1MaskVal)
648 im2MaskVal = exposure2.getMask().getPlaneBitMask(self.config.maskNameList)
649 im2StatsCtrl = afwMath.StatisticsControl(self.config.nSigmaClipPtc,
650 self.config.nIterSigmaClipPtc,
651 im2MaskVal)
652 im2StatsCtrl.setNanSafe(True)
653 im2StatsCtrl.setAndMask(im2MaskVal)
655 # Clipped mean of images; then average of mean.
656 mu1 = afwMath.makeStatistics(im1Area, afwMath.MEANCLIP, im1StatsCtrl).getValue()
657 mu2 = afwMath.makeStatistics(im2Area, afwMath.MEANCLIP, im2StatsCtrl).getValue()
658 if np.isnan(mu1) or np.isnan(mu2):
659 self.log.warn(f"Mean of amp in image 1 or 2 is NaN: {mu1}, {mu2}.")
660 return np.nan, np.nan, None
661 mu = 0.5*(mu1 + mu2)
663 # Take difference of pairs
664 # symmetric formula: diff = (mu2*im1-mu1*im2)/(0.5*(mu1+mu2))
665 temp = im2Area.clone()
666 temp *= mu1
667 diffIm = im1Area.clone()
668 diffIm *= mu2
669 diffIm -= temp
670 diffIm /= mu
672 diffImMaskVal = diffIm.getMask().getPlaneBitMask(self.config.maskNameList)
673 diffImStatsCtrl = afwMath.StatisticsControl(self.config.nSigmaClipPtc,
674 self.config.nIterSigmaClipPtc,
675 diffImMaskVal)
676 diffImStatsCtrl.setNanSafe(True)
677 diffImStatsCtrl.setAndMask(diffImMaskVal)
679 varDiff = 0.5*(afwMath.makeStatistics(diffIm, afwMath.VARIANCECLIP, diffImStatsCtrl).getValue())
681 # Get the mask and identify good pixels as '1', and the rest as '0'.
682 w1 = np.where(im1Area.getMask().getArray() == 0, 1, 0)
683 w2 = np.where(im2Area.getMask().getArray() == 0, 1, 0)
685 w12 = w1*w2
686 wDiff = np.where(diffIm.getMask().getArray() == 0, 1, 0)
687 w = w12*wDiff
689 if np.sum(w) < self.config.minNumberGoodPixelsForFft:
690 self.log.warn(f"Number of good points for FFT ({np.sum(w)}) is less than threshold "
691 f"({self.config.minNumberGoodPixelsForFft})")
692 return np.nan, np.nan, None
694 maxRangeCov = self.config.maximumRangeCovariancesAstier
695 if covAstierRealSpace:
696 covDiffAstier = computeCovDirect(diffIm.getImage().getArray(), w, maxRangeCov)
697 else:
698 shapeDiff = diffIm.getImage().getArray().shape
699 fftShape = (fftSize(shapeDiff[0] + maxRangeCov), fftSize(shapeDiff[1]+maxRangeCov))
700 c = CovFft(diffIm.getImage().getArray(), w, fftShape, maxRangeCov)
701 covDiffAstier = c.reportCovFft(maxRangeCov)
703 return mu, varDiff, covDiffAstier
705 def computeCovDirect(self, diffImage, weightImage, maxRange):
706 """Compute covariances of diffImage in real space.
708 For lags larger than ~25, it is slower than the FFT way.
709 Taken from https://github.com/PierreAstier/bfptc/
711 Parameters
712 ----------
713 diffImage : `numpy.array`
714 Image to compute the covariance of.
716 weightImage : `numpy.array`
717 Weight image of diffImage (1's and 0's for good and bad pixels, respectively).
719 maxRange : `int`
720 Last index of the covariance to be computed.
722 Returns
723 -------
724 outList : `list`
725 List with tuples of the form (dx, dy, var, cov, npix), where:
726 dx : `int`
727 Lag in x
728 dy : `int`
729 Lag in y
730 var : `float`
731 Variance at (dx, dy).
732 cov : `float`
733 Covariance at (dx, dy).
734 nPix : `int`
735 Number of pixel pairs used to evaluate var and cov.
736 """
737 outList = []
738 var = 0
739 # (dy,dx) = (0,0) has to be first
740 for dy in range(maxRange + 1):
741 for dx in range(0, maxRange + 1):
742 if (dx*dy > 0):
743 cov1, nPix1 = self.covDirectValue(diffImage, weightImage, dx, dy)
744 cov2, nPix2 = self.covDirectValue(diffImage, weightImage, dx, -dy)
745 cov = 0.5*(cov1 + cov2)
746 nPix = nPix1 + nPix2
747 else:
748 cov, nPix = self.covDirectValue(diffImage, weightImage, dx, dy)
749 if (dx == 0 and dy == 0):
750 var = cov
751 outList.append((dx, dy, var, cov, nPix))
753 return outList
755 def covDirectValue(self, diffImage, weightImage, dx, dy):
756 """Compute covariances of diffImage in real space at lag (dx, dy).
758 Taken from https://github.com/PierreAstier/bfptc/ (c.f., appendix of Astier+19).
760 Parameters
761 ----------
762 diffImage : `numpy.array`
763 Image to compute the covariance of.
765 weightImage : `numpy.array`
766 Weight image of diffImage (1's and 0's for good and bad pixels, respectively).
768 dx : `int`
769 Lag in x.
771 dy : `int`
772 Lag in y.
774 Returns
775 -------
776 cov : `float`
777 Covariance at (dx, dy)
779 nPix : `int`
780 Number of pixel pairs used to evaluate var and cov.
781 """
782 (nCols, nRows) = diffImage.shape
783 # switching both signs does not change anything:
784 # it just swaps im1 and im2 below
785 if (dx < 0):
786 (dx, dy) = (-dx, -dy)
787 # now, we have dx >0. We have to distinguish two cases
788 # depending on the sign of dy
789 if dy >= 0:
790 im1 = diffImage[dy:, dx:]
791 w1 = weightImage[dy:, dx:]
792 im2 = diffImage[:nCols - dy, :nRows - dx]
793 w2 = weightImage[:nCols - dy, :nRows - dx]
794 else:
795 im1 = diffImage[:nCols + dy, dx:]
796 w1 = weightImage[:nCols + dy, dx:]
797 im2 = diffImage[-dy:, :nRows - dx]
798 w2 = weightImage[-dy:, :nRows - dx]
799 # use the same mask for all 3 calculations
800 wAll = w1*w2
801 # do not use mean() because weightImage=0 pixels would then count
802 nPix = wAll.sum()
803 im1TimesW = im1*wAll
804 s1 = im1TimesW.sum()/nPix
805 s2 = (im2*wAll).sum()/nPix
806 p = (im1TimesW*im2).sum()/nPix
807 cov = p - s1*s2
809 return cov, nPix
811 @staticmethod
812 def _initialParsForPolynomial(order):
813 assert(order >= 2)
814 pars = np.zeros(order, dtype=np.float)
815 pars[0] = 10
816 pars[1] = 1
817 pars[2:] = 0.0001
818 return pars
820 @staticmethod
821 def _boundsForPolynomial(initialPars, lowers=[], uppers=[]):
822 if not len(lowers):
823 lowers = [np.NINF for p in initialPars]
824 if not len(uppers):
825 uppers = [np.inf for p in initialPars]
826 lowers[1] = 0 # no negative gains
827 return (lowers, uppers)
829 @staticmethod
830 def _boundsForAstier(initialPars, lowers=[], uppers=[]):
831 if not len(lowers):
832 lowers = [np.NINF for p in initialPars]
833 if not len(uppers):
834 uppers = [np.inf for p in initialPars]
835 return (lowers, uppers)
837 @staticmethod
838 def _getInitialGoodPoints(means, variances, maxDeviationPositive, maxDeviationNegative):
839 """Return a boolean array to mask bad points.
841 Parameters
842 ----------
843 means : `numpy.array`
844 Input array with mean signal values.
846 variances : `numpy.array`
847 Input array with variances at each mean value.
849 maxDeviationPositive : `float`
850 Maximum deviation from being constant for the variance/mean
851 ratio, in the positive direction.
853 maxDeviationNegative : `float`
854 Maximum deviation from being constant for the variance/mean
855 ratio, in the negative direction.
857 Return
858 ------
859 goodPoints : `numpy.array` [`bool`]
860 Boolean array to select good (`True`) and bad (`False`)
861 points.
863 Notes
864 -----
865 A linear function has a constant ratio, so find the median
866 value of the ratios, and exclude the points that deviate
867 from that by more than a factor of maxDeviationPositive/negative.
868 Asymmetric deviations are supported as we expect the PTC to turn
869 down as the flux increases, but sometimes it anomalously turns
870 upwards just before turning over, which ruins the fits, so it
871 is wise to be stricter about restricting positive outliers than
872 negative ones.
874 Too high and points that are so bad that fit will fail will be included
875 Too low and the non-linear points will be excluded, biasing the NL fit.
877 This function also masks points after the variance starts decreasing.
878 """
880 assert(len(means) == len(variances))
881 ratios = [b/a for (a, b) in zip(means, variances)]
882 medianRatio = np.nanmedian(ratios)
883 ratioDeviations = [(r/medianRatio)-1 for r in ratios]
885 # so that it doesn't matter if the deviation is expressed as positive or negative
886 maxDeviationPositive = abs(maxDeviationPositive)
887 maxDeviationNegative = -1. * abs(maxDeviationNegative)
889 goodPoints = np.array([True if (r < maxDeviationPositive and r > maxDeviationNegative)
890 else False for r in ratioDeviations])
892 # Discard points when variance starts decreasing
893 pivot = np.where(np.array(np.diff(variances)) < 0)[0]
894 if len(pivot) == 0:
895 return goodPoints
896 else:
897 pivot = np.min(pivot)
898 goodPoints[pivot:len(goodPoints)] = False
899 return goodPoints
901 def _makeZeroSafe(self, array, warn=True, substituteValue=1e-9):
902 """"""
903 nBad = Counter(array)[0]
904 if nBad == 0:
905 return array
907 if warn:
908 msg = f"Found {nBad} zeros in array at elements {[x for x in np.where(array==0)[0]]}"
909 self.log.warn(msg)
911 array[array == 0] = substituteValue
912 return array
914 def fitPtc(self, dataset, ptcFitType):
915 """Fit the photon transfer curve to a polynimial or to Astier+19 approximation.
917 Fit the photon transfer curve with either a polynomial of the order
918 specified in the task config, or using the Astier approximation.
920 Sigma clipping is performed iteratively for the fit, as well as an
921 initial clipping of data points that are more than
922 config.initialNonLinearityExclusionThreshold away from lying on a
923 straight line. This other step is necessary because the photon transfer
924 curve turns over catastrophically at very high flux (because saturation
925 drops the variance to ~0) and these far outliers cause the initial fit
926 to fail, meaning the sigma cannot be calculated to perform the
927 sigma-clipping.
929 Parameters
930 ----------
931 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset`
932 The dataset containing the means, variances and exposure times
934 ptcFitType : `str`
935 Fit a 'POLYNOMIAL' (degree: 'polynomialFitDegree') or
936 'EXPAPPROXIMATION' (Eq. 16 of Astier+19) to the PTC
938 Returns
939 -------
940 dataset: `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset`
941 This is the same dataset as the input paramter, however, it has been modified
942 to include information such as the fit vectors and the fit parameters. See
943 the class `PhotonTransferCurveDatase`.
944 """
946 matrixSide = self.config.maximumRangeCovariancesAstier
947 nanMatrix = np.empty((matrixSide, matrixSide))
948 nanMatrix[:] = np.nan
950 for amp in dataset.ampNames:
951 lenInputTimes = len(dataset.rawExpTimes[amp])
952 listNanMatrix = np.empty((lenInputTimes, matrixSide, matrixSide))
953 listNanMatrix[:] = np.nan
955 dataset.covariances[amp] = listNanMatrix
956 dataset.covariancesModel[amp] = listNanMatrix
957 dataset.covariancesSqrtWeights[amp] = listNanMatrix
958 dataset.aMatrix[amp] = nanMatrix
959 dataset.bMatrix[amp] = nanMatrix
960 dataset.covariancesNoB[amp] = listNanMatrix
961 dataset.covariancesModelNoB[amp] = listNanMatrix
962 dataset.covariancesSqrtWeightsNoB[amp] = listNanMatrix
963 dataset.aMatrixNoB[amp] = nanMatrix
965 def errFunc(p, x, y):
966 return ptcFunc(p, x) - y
968 sigmaCutPtcOutliers = self.config.sigmaCutPtcOutliers
969 maxIterationsPtcOutliers = self.config.maxIterationsPtcOutliers
971 for i, ampName in enumerate(dataset.ampNames):
972 timeVecOriginal = np.array(dataset.rawExpTimes[ampName])
973 meanVecOriginal = np.array(dataset.rawMeans[ampName])
974 varVecOriginal = np.array(dataset.rawVars[ampName])
975 varVecOriginal = self._makeZeroSafe(varVecOriginal)
977 goodPoints = self._getInitialGoodPoints(meanVecOriginal, varVecOriginal,
978 self.config.initialNonLinearityExclusionThresholdPositive,
979 self.config.initialNonLinearityExclusionThresholdNegative)
980 if not (goodPoints.any()):
981 msg = (f"\nSERIOUS: All points in goodPoints: {goodPoints} are bad."
982 f"Setting {ampName} to BAD.")
983 self.log.warn(msg)
984 # The first and second parameters of initial fit are discarded (bias and gain)
985 # for the final NL coefficients
986 dataset.badAmps.append(ampName)
987 dataset.expIdMask[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName]))
988 dataset.gain[ampName] = np.nan
989 dataset.gainErr[ampName] = np.nan
990 dataset.noise[ampName] = np.nan
991 dataset.noiseErr[ampName] = np.nan
992 dataset.ptcFitPars[ampName] = (np.repeat(np.nan, self.config.polynomialFitDegree + 1) if
993 ptcFitType in ["POLYNOMIAL", ] else np.repeat(np.nan, 3))
994 dataset.ptcFitParsError[ampName] = (np.repeat(np.nan, self.config.polynomialFitDegree + 1) if
995 ptcFitType in ["POLYNOMIAL", ] else np.repeat(np.nan, 3))
996 dataset.ptcFitChiSq[ampName] = np.nan
997 dataset.finalVars[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName]))
998 dataset.finalModelVars[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName]))
999 dataset.finalMeans[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName]))
1000 continue
1002 mask = goodPoints
1004 if ptcFitType == 'EXPAPPROXIMATION':
1005 ptcFunc = funcAstier
1006 parsIniPtc = [-1e-9, 1.0, 10.] # a00, gain, noise
1007 # lowers and uppers obtained from studies by C. Lage (UC Davis, 11/2020).
1008 bounds = self._boundsForAstier(parsIniPtc, lowers=[-1e-4, 0.5, -2000],
1009 uppers=[1e-4, 2.5, 2000])
1010 if ptcFitType == 'POLYNOMIAL':
1011 ptcFunc = funcPolynomial
1012 parsIniPtc = self._initialParsForPolynomial(self.config.polynomialFitDegree + 1)
1013 bounds = self._boundsForPolynomial(parsIniPtc)
1015 # Before bootstrap fit, do an iterative fit to get rid of outliers
1016 count = 1
1017 while count <= maxIterationsPtcOutliers:
1018 # Note that application of the mask actually shrinks the array
1019 # to size rather than setting elements to zero (as we want) so
1020 # always update mask itself and re-apply to the original data
1021 meanTempVec = meanVecOriginal[mask]
1022 varTempVec = varVecOriginal[mask]
1023 res = least_squares(errFunc, parsIniPtc, bounds=bounds, args=(meanTempVec, varTempVec))
1024 pars = res.x
1026 # change this to the original from the temp because the masks are ANDed
1027 # meaning once a point is masked it's always masked, and the masks must
1028 # always be the same length for broadcasting
1029 sigResids = (varVecOriginal - ptcFunc(pars, meanVecOriginal))/np.sqrt(varVecOriginal)
1030 newMask = np.array([True if np.abs(r) < sigmaCutPtcOutliers else False for r in sigResids])
1031 mask = mask & newMask
1032 if not (mask.any() and newMask.any()):
1033 msg = (f"\nSERIOUS: All points in either mask: {mask} or newMask: {newMask} are bad. "
1034 f"Setting {ampName} to BAD.")
1035 self.log.warn(msg)
1036 # The first and second parameters of initial fit are discarded (bias and gain)
1037 # for the final NL coefficients
1038 dataset.badAmps.append(ampName)
1039 dataset.expIdMask[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName]))
1040 dataset.gain[ampName] = np.nan
1041 dataset.gainErr[ampName] = np.nan
1042 dataset.noise[ampName] = np.nan
1043 dataset.noiseErr[ampName] = np.nan
1044 dataset.ptcFitPars[ampName] = (np.repeat(np.nan, self.config.polynomialFitDegree + 1)
1045 if ptcFitType in ["POLYNOMIAL", ] else
1046 np.repeat(np.nan, 3))
1047 dataset.ptcFitParsError[ampName] = (np.repeat(np.nan, self.config.polynomialFitDegree + 1)
1048 if ptcFitType in ["POLYNOMIAL", ] else
1049 np.repeat(np.nan, 3))
1050 dataset.ptcFitChiSq[ampName] = np.nan
1051 dataset.finalVars[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName]))
1052 dataset.finalModelVars[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName]))
1053 dataset.finalMeans[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName]))
1054 break
1055 nDroppedTotal = Counter(mask)[False]
1056 self.log.debug(f"Iteration {count}: discarded {nDroppedTotal} points in total for {ampName}")
1057 count += 1
1058 # objects should never shrink
1059 assert (len(mask) == len(timeVecOriginal) == len(meanVecOriginal) == len(varVecOriginal))
1061 if not (mask.any() and newMask.any()):
1062 continue
1063 dataset.expIdMask[ampName] = mask # store the final mask
1064 parsIniPtc = pars
1065 meanVecFinal = meanVecOriginal[mask]
1066 varVecFinal = varVecOriginal[mask]
1068 if Counter(mask)[False] > 0:
1069 self.log.info((f"Number of points discarded in PTC of amplifier {ampName}:" +
1070 f" {Counter(mask)[False]} out of {len(meanVecOriginal)}"))
1072 if (len(meanVecFinal) < len(parsIniPtc)):
1073 msg = (f"\nSERIOUS: Not enough data points ({len(meanVecFinal)}) compared to the number of"
1074 f"parameters of the PTC model({len(parsIniPtc)}). Setting {ampName} to BAD.")
1075 self.log.warn(msg)
1076 # The first and second parameters of initial fit are discarded (bias and gain)
1077 # for the final NL coefficients
1078 dataset.badAmps.append(ampName)
1079 dataset.expIdMask[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName]))
1080 dataset.gain[ampName] = np.nan
1081 dataset.gainErr[ampName] = np.nan
1082 dataset.noise[ampName] = np.nan
1083 dataset.noiseErr[ampName] = np.nan
1084 dataset.ptcFitPars[ampName] = (np.repeat(np.nan, self.config.polynomialFitDegree + 1) if
1085 ptcFitType in ["POLYNOMIAL", ] else np.repeat(np.nan, 3))
1086 dataset.ptcFitParsError[ampName] = (np.repeat(np.nan, self.config.polynomialFitDegree + 1) if
1087 ptcFitType in ["POLYNOMIAL", ] else np.repeat(np.nan, 3))
1088 dataset.ptcFitChiSq[ampName] = np.nan
1089 dataset.finalVars[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName]))
1090 dataset.finalModelVars[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName]))
1091 dataset.finalMeans[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName]))
1092 continue
1094 # Fit the PTC
1095 if self.config.doFitBootstrap:
1096 parsFit, parsFitErr, reducedChiSqPtc = fitBootstrap(parsIniPtc, meanVecFinal,
1097 varVecFinal, ptcFunc,
1098 weightsY=1./np.sqrt(varVecFinal))
1099 else:
1100 parsFit, parsFitErr, reducedChiSqPtc = fitLeastSq(parsIniPtc, meanVecFinal,
1101 varVecFinal, ptcFunc,
1102 weightsY=1./np.sqrt(varVecFinal))
1103 dataset.ptcFitPars[ampName] = parsFit
1104 dataset.ptcFitParsError[ampName] = parsFitErr
1105 dataset.ptcFitChiSq[ampName] = reducedChiSqPtc
1106 # Masked variances (measured and modeled) and means. Need to pad the array so astropy.Table does
1107 # not crash (the mask may vary per amp).
1108 padLength = len(dataset.rawExpTimes[ampName]) - len(varVecFinal)
1109 dataset.finalVars[ampName] = np.pad(varVecFinal, (0, padLength), 'constant',
1110 constant_values=np.nan)
1111 dataset.finalModelVars[ampName] = np.pad(ptcFunc(parsFit, meanVecFinal), (0, padLength),
1112 'constant', constant_values=np.nan)
1113 dataset.finalMeans[ampName] = np.pad(meanVecFinal, (0, padLength), 'constant',
1114 constant_values=np.nan)
1116 if ptcFitType == 'EXPAPPROXIMATION':
1117 ptcGain = parsFit[1]
1118 ptcGainErr = parsFitErr[1]
1119 ptcNoise = np.sqrt(np.fabs(parsFit[2]))
1120 ptcNoiseErr = 0.5*(parsFitErr[2]/np.fabs(parsFit[2]))*np.sqrt(np.fabs(parsFit[2]))
1121 if ptcFitType == 'POLYNOMIAL':
1122 ptcGain = 1./parsFit[1]
1123 ptcGainErr = np.fabs(1./parsFit[1])*(parsFitErr[1]/parsFit[1])
1124 ptcNoise = np.sqrt(np.fabs(parsFit[0]))*ptcGain
1125 ptcNoiseErr = (0.5*(parsFitErr[0]/np.fabs(parsFit[0]))*(np.sqrt(np.fabs(parsFit[0]))))*ptcGain
1126 dataset.gain[ampName] = ptcGain
1127 dataset.gainErr[ampName] = ptcGainErr
1128 dataset.noise[ampName] = ptcNoise
1129 dataset.noiseErr[ampName] = ptcNoiseErr
1130 if not len(dataset.ptcFitType) == 0:
1131 dataset.ptcFitType = ptcFitType
1132 if len(dataset.badAmps) == 0:
1133 dataset.badAmps = np.repeat(np.nan, len(list(dataset.rawExpTimes.values())[0]))
1135 return dataset