Coverage for python/lsst/cp/pipe/ptc.py : 10%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of cp_pipe.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21#
23__all__ = ['MeasurePhotonTransferCurveTask',
24 'MeasurePhotonTransferCurveTaskConfig',
25 'PhotonTransferCurveDataset']
27import numpy as np
28import matplotlib.pyplot as plt
29import os
30from matplotlib.backends.backend_pdf import PdfPages
31from sqlite3 import OperationalError
32from collections import Counter
34import lsst.afw.math as afwMath
35import lsst.pex.config as pexConfig
36import lsst.pipe.base as pipeBase
37from lsst.ip.isr import IsrTask
38from .utils import (NonexistentDatasetTaskDataIdContainer, PairedVisitListTaskRunner,
39 checkExpLengthEqual, validateIsrConfig)
40from scipy.optimize import leastsq, least_squares
41import numpy.polynomial.polynomial as poly
43from lsst.ip.isr.linearize import Linearizer
44import datetime
47class MeasurePhotonTransferCurveTaskConfig(pexConfig.Config):
48 """Config class for photon transfer curve measurement task"""
49 isr = pexConfig.ConfigurableField(
50 target=IsrTask,
51 doc="""Task to perform instrumental signature removal.""",
52 )
53 isrMandatorySteps = pexConfig.ListField(
54 dtype=str,
55 doc="isr operations that must be performed for valid results. Raises if any of these are False.",
56 default=['doAssembleCcd']
57 )
58 isrForbiddenSteps = pexConfig.ListField(
59 dtype=str,
60 doc="isr operations that must NOT be performed for valid results. Raises if any of these are True",
61 default=['doFlat', 'doFringe', 'doAddDistortionModel', 'doBrighterFatter', 'doUseOpticsTransmission',
62 'doUseFilterTransmission', 'doUseSensorTransmission', 'doUseAtmosphereTransmission']
63 )
64 isrDesirableSteps = pexConfig.ListField(
65 dtype=str,
66 doc="isr operations that it is advisable to perform, but are not mission-critical." +
67 " WARNs are logged for any of these found to be False.",
68 default=['doBias', 'doDark', 'doCrosstalk', 'doDefect']
69 )
70 isrUndesirableSteps = pexConfig.ListField(
71 dtype=str,
72 doc="isr operations that it is *not* advisable to perform in the general case, but are not" +
73 " forbidden as some use-cases might warrant them." +
74 " WARNs are logged for any of these found to be True.",
75 default=['doLinearize']
76 )
77 ccdKey = pexConfig.Field(
78 dtype=str,
79 doc="The key by which to pull a detector from a dataId, e.g. 'ccd' or 'detector'.",
80 default='ccd',
81 )
82 makePlots = pexConfig.Field(
83 dtype=bool,
84 doc="Plot the PTC curves?",
85 default=False,
86 )
87 ptcFitType = pexConfig.ChoiceField(
88 dtype=str,
89 doc="Fit PTC to approximation in Astier+19 (Equation 16) or to a polynomial.",
90 default="POLYNOMIAL",
91 allowed={
92 "POLYNOMIAL": "n-degree polynomial (use 'polynomialFitDegree' to set 'n').",
93 "ASTIERAPPROXIMATION": "Approximation in Astier+19 (Eq. 16)."
94 }
95 )
96 polynomialFitDegree = pexConfig.Field(
97 dtype=int,
98 doc="Degree of polynomial to fit the PTC, when 'ptcFitType'=POLYNOMIAL.",
99 default=2,
100 )
101 polynomialFitDegreeNonLinearity = pexConfig.Field(
102 dtype=int,
103 doc="Degree of polynomial to fit the meanSignal vs exposureTime curve to produce" +
104 " the table for LinearizeLookupTable.",
105 default=3,
106 )
107 binSize = pexConfig.Field(
108 dtype=int,
109 doc="Bin the image by this factor in both dimensions.",
110 default=1,
111 )
112 minMeanSignal = pexConfig.Field(
113 dtype=float,
114 doc="Minimum value (inclusive) of mean signal (in DN) above which to consider.",
115 default=0,
116 )
117 maxMeanSignal = pexConfig.Field(
118 dtype=float,
119 doc="Maximum value (inclusive) of mean signal (in DN) below which to consider.",
120 default=9e6,
121 )
122 initialNonLinearityExclusionThresholdPositive = pexConfig.RangeField(
123 dtype=float,
124 doc="Initially exclude data points with a variance that are more than a factor of this from being"
125 " linear in the positive direction, from the PTC fit. Note that these points will also be"
126 " excluded from the non-linearity fit. This is done before the iterative outlier rejection,"
127 " to allow an accurate determination of the sigmas for said iterative fit.",
128 default=0.12,
129 min=0.0,
130 max=1.0,
131 )
132 initialNonLinearityExclusionThresholdNegative = pexConfig.RangeField(
133 dtype=float,
134 doc="Initially exclude data points with a variance that are more than a factor of this from being"
135 " linear in the negative direction, from the PTC fit. Note that these points will also be"
136 " excluded from the non-linearity fit. This is done before the iterative outlier rejection,"
137 " to allow an accurate determination of the sigmas for said iterative fit.",
138 default=0.25,
139 min=0.0,
140 max=1.0,
141 )
142 sigmaCutPtcOutliers = pexConfig.Field(
143 dtype=float,
144 doc="Sigma cut for outlier rejection in PTC.",
145 default=5.0,
146 )
147 maxIterationsPtcOutliers = pexConfig.Field(
148 dtype=int,
149 doc="Maximum number of iterations for outlier rejection in PTC.",
150 default=2,
151 )
152 doFitBootstrap = pexConfig.Field(
153 dtype=bool,
154 doc="Use bootstrap for the PTC fit parameters and errors?.",
155 default=False,
156 )
157 linResidualTimeIndex = pexConfig.Field(
158 dtype=int,
159 doc="Index position in time array for reference time in linearity residual calculation.",
160 default=2,
161 )
162 maxAduForLookupTableLinearizer = pexConfig.Field(
163 dtype=int,
164 doc="Maximum DN value for the LookupTable linearizer.",
165 default=2**18,
166 )
167 instrumentName = pexConfig.Field(
168 dtype=str,
169 doc="Instrument name.",
170 default='',
171 )
174class PhotonTransferCurveDataset:
175 """A simple class to hold the output data from the PTC task.
177 The dataset is made up of a dictionary for each item, keyed by the
178 amplifiers' names, which much be supplied at construction time.
180 New items cannot be added to the class to save accidentally saving to the
181 wrong property, and the class can be frozen if desired.
183 inputVisitPairs records the visits used to produce the data.
184 When fitPtcAndNonLinearity() is run, a mask is built up, which is by definition
185 always the same length as inputVisitPairs, rawExpTimes, rawMeans
186 and rawVars, and is a list of bools, which are incrementally set to False
187 as points are discarded from the fits.
189 PTC fit parameters for polynomials are stored in a list in ascending order
190 of polynomial term, i.e. par[0]*x^0 + par[1]*x + par[2]*x^2 etc
191 with the length of the list corresponding to the order of the polynomial
192 plus one.
193 """
194 def __init__(self, ampNames):
195 # add items to __dict__ directly because __setattr__ is overridden
197 # instance variables
198 self.__dict__["ampNames"] = ampNames
199 self.__dict__["badAmps"] = []
201 # raw data variables
202 self.__dict__["inputVisitPairs"] = {ampName: [] for ampName in ampNames}
203 self.__dict__["visitMask"] = {ampName: [] for ampName in ampNames}
204 self.__dict__["rawExpTimes"] = {ampName: [] for ampName in ampNames}
205 self.__dict__["rawMeans"] = {ampName: [] for ampName in ampNames}
206 self.__dict__["rawVars"] = {ampName: [] for ampName in ampNames}
208 # fit information
209 self.__dict__["ptcFitType"] = {ampName: "" for ampName in ampNames}
210 self.__dict__["ptcFitPars"] = {ampName: [] for ampName in ampNames}
211 self.__dict__["ptcFitParsError"] = {ampName: [] for ampName in ampNames}
212 self.__dict__["ptcFitReducedChiSquared"] = {ampName: [] for ampName in ampNames}
213 self.__dict__["nonLinearity"] = {ampName: [] for ampName in ampNames}
214 self.__dict__["nonLinearityError"] = {ampName: [] for ampName in ampNames}
215 self.__dict__["nonLinearityResiduals"] = {ampName: [] for ampName in ampNames}
216 self.__dict__["nonLinearityReducedChiSquared"] = {ampName: [] for ampName in ampNames}
218 # final results
219 self.__dict__["gain"] = {ampName: -1. for ampName in ampNames}
220 self.__dict__["gainErr"] = {ampName: -1. for ampName in ampNames}
221 self.__dict__["noise"] = {ampName: -1. for ampName in ampNames}
222 self.__dict__["noiseErr"] = {ampName: -1. for ampName in ampNames}
223 self.__dict__["coefficientsLinearizePolynomial"] = {ampName: [] for ampName in ampNames}
224 self.__dict__["coefficientLinearizeSquared"] = {ampName: [] for ampName in ampNames}
226 def __setattr__(self, attribute, value):
227 """Protect class attributes"""
228 if attribute not in self.__dict__:
229 raise AttributeError(f"{attribute} is not already a member of PhotonTransferCurveDataset, which"
230 " does not support setting of new attributes.")
231 else:
232 self.__dict__[attribute] = value
234 def getVisitsUsed(self, ampName):
235 """Get the visits used, i.e. not discarded, for a given amp.
237 If no mask has been created yet, all visits are returned.
238 """
239 if self.visitMask[ampName] == []:
240 return self.inputVisitPairs[ampName]
242 # if the mask exists it had better be the same length as the visitPairs
243 assert len(self.visitMask[ampName]) == len(self.inputVisitPairs[ampName])
245 pairs = self.inputVisitPairs[ampName]
246 mask = self.visitMask[ampName]
247 # cast to bool required because numpy
248 return [(v1, v2) for ((v1, v2), m) in zip(pairs, mask) if bool(m) is True]
250 def getGoodAmps(self):
251 return [amp for amp in self.ampNames if amp not in self.badAmps]
254class MeasurePhotonTransferCurveTask(pipeBase.CmdLineTask):
255 """A class to calculate, fit, and plot a PTC from a set of flat pairs.
257 The Photon Transfer Curve (var(signal) vs mean(signal)) is a standard tool
258 used in astronomical detectors characterization (e.g., Janesick 2001,
259 Janesick 2007). This task calculates the PTC from a series of pairs of
260 flat-field images; each pair taken at identical exposure times. The
261 difference image of each pair is formed to eliminate fixed pattern noise,
262 and then the variance of the difference image and the mean of the average image
263 are used to produce the PTC. An n-degree polynomial or the approximation in Equation
264 16 of Astier+19 ("The Shape of the Photon Transfer Curve of CCD sensors",
265 arXiv:1905.08677) can be fitted to the PTC curve. These models include
266 parameters such as the gain (e/DN) and readout noise.
268 Linearizers to correct for signal-chain non-linearity are also calculated.
269 The `Linearizer` class, in general, can support per-amp linearizers, but in this
270 task this is not supported.
271 Parameters
272 ----------
274 *args: `list`
275 Positional arguments passed to the Task constructor. None used at this
276 time.
277 **kwargs: `dict`
278 Keyword arguments passed on to the Task constructor. None used at this
279 time.
281 """
283 RunnerClass = PairedVisitListTaskRunner
284 ConfigClass = MeasurePhotonTransferCurveTaskConfig
285 _DefaultName = "measurePhotonTransferCurve"
287 def __init__(self, *args, **kwargs):
288 pipeBase.CmdLineTask.__init__(self, *args, **kwargs)
289 self.makeSubtask("isr")
290 plt.interactive(False) # stop windows popping up when plotting. When headless, use 'agg' backend too
291 validateIsrConfig(self.isr, self.config.isrMandatorySteps,
292 self.config.isrForbiddenSteps, self.config.isrDesirableSteps, checkTrim=False)
293 self.config.validate()
294 self.config.freeze()
296 @classmethod
297 def _makeArgumentParser(cls):
298 """Augment argument parser for the MeasurePhotonTransferCurveTask."""
299 parser = pipeBase.ArgumentParser(name=cls._DefaultName)
300 parser.add_argument("--visit-pairs", dest="visitPairs", nargs="*",
301 help="Visit pairs to use. Each pair must be of the form INT,INT e.g. 123,456")
302 parser.add_id_argument("--id", datasetType="photonTransferCurveDataset",
303 ContainerClass=NonexistentDatasetTaskDataIdContainer,
304 help="The ccds to use, e.g. --id ccd=0..100")
305 return parser
307 @pipeBase.timeMethod
308 def runDataRef(self, dataRef, visitPairs):
309 """Run the Photon Transfer Curve (PTC) measurement task.
311 For a dataRef (which is each detector here),
312 and given a list of visit pairs at different exposure times,
313 measure the PTC.
315 Parameters
316 ----------
317 dataRef : list of lsst.daf.persistence.ButlerDataRef
318 dataRef for the detector for the visits to be fit.
319 visitPairs : `iterable` of `tuple` of `int`
320 Pairs of visit numbers to be processed together
321 """
323 # setup necessary objects
324 detNum = dataRef.dataId[self.config.ccdKey]
325 detector = dataRef.get('camera')[dataRef.dataId[self.config.ccdKey]]
326 # expand some missing fields that we need for lsstCam. This is a work-around
327 # for Gen2 problems that I (RHL) don't feel like solving. The calibs pipelines
328 # (which inherit from CalibTask) use addMissingKeys() to do basically the same thing
329 #
330 # Basically, the butler's trying to look up the fields in `raw_visit` which won't work
331 for name in dataRef.getButler().getKeys('bias'):
332 if name not in dataRef.dataId:
333 try:
334 dataRef.dataId[name] = \
335 dataRef.getButler().queryMetadata('raw', [name], detector=detNum)[0]
336 except OperationalError:
337 pass
339 amps = detector.getAmplifiers()
340 ampNames = [amp.getName() for amp in amps]
341 dataset = PhotonTransferCurveDataset(ampNames)
343 self.log.info('Measuring PTC using %s visits for detector %s' % (visitPairs, detNum))
345 for (v1, v2) in visitPairs:
346 # Perform ISR on each exposure
347 dataRef.dataId['expId'] = v1
348 exp1 = self.isr.runDataRef(dataRef).exposure
349 dataRef.dataId['expId'] = v2
350 exp2 = self.isr.runDataRef(dataRef).exposure
351 del dataRef.dataId['expId']
353 checkExpLengthEqual(exp1, exp2, v1, v2, raiseWithMessage=True)
354 expTime = exp1.getInfo().getVisitInfo().getExposureTime()
356 for amp in detector:
357 mu, varDiff = self.measureMeanVarPair(exp1, exp2, region=amp.getBBox())
358 ampName = amp.getName()
360 dataset.rawExpTimes[ampName].append(expTime)
361 dataset.rawMeans[ampName].append(mu)
362 dataset.rawVars[ampName].append(varDiff)
363 dataset.inputVisitPairs[ampName].append((v1, v2))
365 numberAmps = len(detector.getAmplifiers())
366 numberAduValues = self.config.maxAduForLookupTableLinearizer
367 lookupTableArray = np.zeros((numberAmps, numberAduValues), dtype=np.int)
369 # Fit PTC and (non)linearity of signal vs time curve.
370 # Fill up PhotonTransferCurveDataset object.
371 # Fill up array for LUT linearizer.
372 # Produce coefficients for Polynomial ans Squared linearizers.
373 dataset = self.fitPtcAndNonLinearity(dataset, self.config.ptcFitType,
374 tableArray=lookupTableArray)
376 if self.config.makePlots:
377 self.plot(dataRef, dataset, ptcFitType=self.config.ptcFitType)
379 # Save data, PTC fit, and NL fit dictionaries
380 self.log.info(f"Writing PTC and NL data to {dataRef.getUri(write=True)}")
381 dataRef.put(dataset, datasetType="photonTransferCurveDataset")
383 butler = dataRef.getButler()
384 self.log.info(f"Writing linearizers: \n "
385 "lookup table (linear component of polynomial fit), \n "
386 "polynomial (coefficients for a polynomial correction), \n "
387 "and squared linearizer (quadratic coefficient from polynomial)")
389 detName = detector.getName()
390 now = datetime.datetime.utcnow()
391 calibDate = now.strftime("%Y-%m-%d")
393 for linType, dataType in [("LOOKUPTABLE", 'linearizeLut'),
394 ("LINEARIZEPOLYNOMIAL", 'linearizePolynomial'),
395 ("LINEARIZESQUARED", 'linearizeSquared')]:
397 if linType == "LOOKUPTABLE":
398 tableArray = lookupTableArray
399 else:
400 tableArray = None
402 linearizer = self.buildLinearizerObject(dataset, detector, calibDate, linType,
403 instruName=self.config.instrumentName,
404 tableArray=tableArray,
405 log=self.log)
406 butler.put(linearizer, datasetType=dataType, dataId={'detector': detNum,
407 'detectorName': detName, 'calibDate': calibDate})
409 self.log.info('Finished measuring PTC for in detector %s' % detNum)
411 return pipeBase.Struct(exitStatus=0)
413 def buildLinearizerObject(self, dataset, detector, calibDate, linearizerType, instruName='',
414 tableArray=None, log=None):
415 """Build linearizer object to persist.
417 Parameters
418 ----------
419 dataset : `lsst.cp.pipe.ptc.PhotonTransferCurveDataset`
420 The dataset containing the means, variances, and exposure times
421 detector : `lsst.afw.cameraGeom.Detector`
422 Detector object
423 calibDate : `datetime.datetime`
424 Calibration date
425 linearizerType : `str`
426 'LOOKUPTABLE', 'LINEARIZESQUARED', or 'LINEARIZEPOLYNOMIAL'
427 instruName : `str`, optional
428 Instrument name
429 tableArray : `np.array`, optional
430 Look-up table array with size rows=nAmps and columns=DN values
431 log : `lsst.log.Log`, optional
432 Logger to handle messages
434 Returns
435 -------
436 linearizer : `lsst.ip.isr.Linearizer`
437 Linearizer object
438 """
439 detName = detector.getName()
440 detNum = detector.getId()
441 if linearizerType == "LOOKUPTABLE":
442 if tableArray is not None:
443 linearizer = Linearizer(detector=detector, table=tableArray, log=log)
444 else:
445 raise RuntimeError("tableArray must be provided when creating a LookupTable linearizer")
446 elif linearizerType in ("LINEARIZESQUARED", "LINEARIZEPOLYNOMIAL"):
447 linearizer = Linearizer(log=log)
448 else:
449 raise RuntimeError("Invalid linearizerType {linearizerType} to build a Linearizer object. "
450 "Supported: 'LOOKUPTABLE', 'LINEARIZESQUARED', or 'LINEARIZEPOLYNOMIAL'")
451 for i, amp in enumerate(detector.getAmplifiers()):
452 ampName = amp.getName()
453 if linearizerType == "LOOKUPTABLE":
454 linearizer.linearityCoeffs[ampName] = [i, 0]
455 linearizer.linearityType[ampName] = "LookupTable"
456 elif linearizerType == "LINEARIZESQUARED":
457 linearizer.linearityCoeffs[ampName] = [dataset.coefficientLinearizeSquared[ampName]]
458 linearizer.linearityType[ampName] = "Squared"
459 elif linearizerType == "LINEARIZEPOLYNOMIAL":
460 linearizer.linearityCoeffs[ampName] = dataset.coefficientsLinearizePolynomial[ampName]
461 linearizer.linearityType[ampName] = "Polynomial"
462 linearizer.linearityBBox[ampName] = amp.getBBox()
464 linearizer.validate()
465 calibId = f"detectorName={detName} detector={detNum} calibDate={calibDate} ccd={detNum} filter=NONE"
467 try:
468 raftName = detName.split("_")[0]
469 calibId += f" raftName={raftName}"
470 except Exception:
471 raftname = "NONE"
472 calibId += f" raftName={raftname}"
474 serial = detector.getSerial()
475 linearizer.updateMetadata(instrumentName=instruName, detectorId=f"{detNum}",
476 calibId=calibId, serial=serial, detectorName=f"{detName}")
478 return linearizer
480 def measureMeanVarPair(self, exposure1, exposure2, region=None):
481 """Calculate the mean signal of two exposures and the variance of their difference.
483 Parameters
484 ----------
485 exposure1 : `lsst.afw.image.exposure.exposure.ExposureF`
486 First exposure of flat field pair.
488 exposure2 : `lsst.afw.image.exposure.exposure.ExposureF`
489 Second exposure of flat field pair.
491 region : `lsst.geom.Box2I`
492 Region of each exposure where to perform the calculations (e.g, an amplifier).
494 Return
495 ------
497 mu : `float`
498 0.5*(mu1 + mu2), where mu1, and mu2 are the clipped means of the regions in
499 both exposures.
501 varDiff : `float`
502 Half of the clipped variance of the difference of the regions inthe two input
503 exposures.
504 """
506 if region is not None:
507 im1Area = exposure1.maskedImage[region]
508 im2Area = exposure2.maskedImage[region]
509 else:
510 im1Area = exposure1.maskedImage
511 im2Area = exposure2.maskedImage
513 im1Area = afwMath.binImage(im1Area, self.config.binSize)
514 im2Area = afwMath.binImage(im2Area, self.config.binSize)
516 # Clipped mean of images; then average of mean.
517 mu1 = afwMath.makeStatistics(im1Area, afwMath.MEANCLIP).getValue()
518 mu2 = afwMath.makeStatistics(im2Area, afwMath.MEANCLIP).getValue()
519 mu = 0.5*(mu1 + mu2)
521 # Take difference of pairs
522 # symmetric formula: diff = (mu2*im1-mu1*im2)/(0.5*(mu1+mu2))
523 temp = im2Area.clone()
524 temp *= mu1
525 diffIm = im1Area.clone()
526 diffIm *= mu2
527 diffIm -= temp
528 diffIm /= mu
530 varDiff = 0.5*(afwMath.makeStatistics(diffIm, afwMath.VARIANCECLIP).getValue())
532 return mu, varDiff
534 def _fitLeastSq(self, initialParams, dataX, dataY, function):
535 """Do a fit and estimate the parameter errors using using scipy.optimize.leastq.
537 optimize.leastsq returns the fractional covariance matrix. To estimate the
538 standard deviation of the fit parameters, multiply the entries of this matrix
539 by the unweighted reduced chi squared and take the square root of the diagonal elements.
541 Parameters
542 ----------
543 initialParams : `list` of `float`
544 initial values for fit parameters. For ptcFitType=POLYNOMIAL, its length
545 determines the degree of the polynomial.
547 dataX : `numpy.array` of `float`
548 Data in the abscissa axis.
550 dataY : `numpy.array` of `float`
551 Data in the ordinate axis.
553 function : callable object (function)
554 Function to fit the data with.
556 Return
557 ------
558 pFitSingleLeastSquares : `list` of `float`
559 List with fitted parameters.
561 pErrSingleLeastSquares : `list` of `float`
562 List with errors for fitted parameters.
564 reducedChiSqSingleLeastSquares : `float`
565 Unweighted reduced chi squared
566 """
568 def errFunc(p, x, y):
569 return function(p, x) - y
571 pFit, pCov, infoDict, errMessage, success = leastsq(errFunc, initialParams,
572 args=(dataX, dataY), full_output=1, epsfcn=0.0001)
574 if (len(dataY) > len(initialParams)) and pCov is not None:
575 reducedChiSq = (errFunc(pFit, dataX, dataY)**2).sum()/(len(dataY)-len(initialParams))
576 pCov *= reducedChiSq
577 else:
578 pCov[:, :] = np.inf
580 errorVec = []
581 for i in range(len(pFit)):
582 errorVec.append(np.fabs(pCov[i][i])**0.5)
584 pFitSingleLeastSquares = pFit
585 pErrSingleLeastSquares = np.array(errorVec)
587 return pFitSingleLeastSquares, pErrSingleLeastSquares, reducedChiSq
589 def _fitBootstrap(self, initialParams, dataX, dataY, function, confidenceSigma=1.):
590 """Do a fit using least squares and bootstrap to estimate parameter errors.
592 The bootstrap error bars are calculated by fitting 100 random data sets.
594 Parameters
595 ----------
596 initialParams : `list` of `float`
597 initial values for fit parameters. For ptcFitType=POLYNOMIAL, its length
598 determines the degree of the polynomial.
600 dataX : `numpy.array` of `float`
601 Data in the abscissa axis.
603 dataY : `numpy.array` of `float`
604 Data in the ordinate axis.
606 function : callable object (function)
607 Function to fit the data with.
609 confidenceSigma : `float`
610 Number of sigmas that determine confidence interval for the bootstrap errors.
612 Return
613 ------
614 pFitBootstrap : `list` of `float`
615 List with fitted parameters.
617 pErrBootstrap : `list` of `float`
618 List with errors for fitted parameters.
620 reducedChiSqBootstrap : `float`
621 Reduced chi squared.
622 """
624 def errFunc(p, x, y):
625 return function(p, x) - y
627 # Fit first time
628 pFit, _ = leastsq(errFunc, initialParams, args=(dataX, dataY), full_output=0)
630 # Get the stdev of the residuals
631 residuals = errFunc(pFit, dataX, dataY)
632 sigmaErrTotal = np.std(residuals)
634 # 100 random data sets are generated and fitted
635 pars = []
636 for i in range(100):
637 randomDelta = np.random.normal(0., sigmaErrTotal, len(dataY))
638 randomDataY = dataY + randomDelta
639 randomFit, _ = leastsq(errFunc, initialParams,
640 args=(dataX, randomDataY), full_output=0)
641 pars.append(randomFit)
642 pars = np.array(pars)
643 meanPfit = np.mean(pars, 0)
645 # confidence interval for parameter estimates
646 nSigma = confidenceSigma
647 errPfit = nSigma*np.std(pars, 0)
648 pFitBootstrap = meanPfit
649 pErrBootstrap = errPfit
651 reducedChiSq = (errFunc(pFitBootstrap, dataX, dataY)**2).sum()/(len(dataY)-len(initialParams))
652 return pFitBootstrap, pErrBootstrap, reducedChiSq
654 def funcPolynomial(self, pars, x):
655 """Polynomial function definition"""
656 return poly.polyval(x, [*pars])
658 def funcAstier(self, pars, x):
659 """Single brighter-fatter parameter model for PTC; Equation 16 of Astier+19"""
660 a00, gain, noise = pars
661 return 0.5/(a00*gain*gain)*(np.exp(2*a00*x*gain)-1) + noise/(gain*gain)
663 @staticmethod
664 def _initialParsForPolynomial(order):
665 assert(order >= 2)
666 pars = np.zeros(order, dtype=np.float)
667 pars[0] = 10
668 pars[1] = 1
669 pars[2:] = 0.0001
670 return pars
672 @staticmethod
673 def _boundsForPolynomial(initialPars):
674 lowers = [np.NINF for p in initialPars]
675 uppers = [np.inf for p in initialPars]
676 lowers[1] = 0 # no negative gains
677 return (lowers, uppers)
679 @staticmethod
680 def _boundsForAstier(initialPars):
681 lowers = [np.NINF for p in initialPars]
682 uppers = [np.inf for p in initialPars]
683 return (lowers, uppers)
685 @staticmethod
686 def _getInitialGoodPoints(means, variances, maxDeviationPositive, maxDeviationNegative):
687 """Return a boolean array to mask bad points.
689 A linear function has a constant ratio, so find the median
690 value of the ratios, and exclude the points that deviate
691 from that by more than a factor of maxDeviationPositive/negative.
692 Asymmetric deviations are supported as we expect the PTC to turn
693 down as the flux increases, but sometimes it anomalously turns
694 upwards just before turning over, which ruins the fits, so it
695 is wise to be stricter about restricting positive outliers than
696 negative ones.
698 Too high and points that are so bad that fit will fail will be included
699 Too low and the non-linear points will be excluded, biasing the NL fit."""
700 ratios = [b/a for (a, b) in zip(means, variances)]
701 medianRatio = np.median(ratios)
702 ratioDeviations = [(r/medianRatio)-1 for r in ratios]
704 # so that it doesn't matter if the deviation is expressed as positive or negative
705 maxDeviationPositive = abs(maxDeviationPositive)
706 maxDeviationNegative = -1. * abs(maxDeviationNegative)
708 goodPoints = np.array([True if (r < maxDeviationPositive and r > maxDeviationNegative)
709 else False for r in ratioDeviations])
710 return goodPoints
712 def _makeZeroSafe(self, array, warn=True, substituteValue=1e-9):
713 """"""
714 nBad = Counter(array)[0]
715 if nBad == 0:
716 return array
718 if warn:
719 msg = f"Found {nBad} zeros in array at elements {[x for x in np.where(array==0)[0]]}"
720 self.log.warn(msg)
722 array[array == 0] = substituteValue
723 return array
725 def calculateLinearityResidualAndLinearizers(self, exposureTimeVector, meanSignalVector):
726 """Calculate linearity residual and fit an n-order polynomial to the mean vs time curve
727 to produce corrections (deviation from linear part of polynomial) for a particular amplifier
728 to populate LinearizeLookupTable.
729 Use the coefficients of this fit to calculate the correction coefficients for LinearizePolynomial
730 and LinearizeSquared."
732 Parameters
733 ---------
735 exposureTimeVector: `list` of `float`
736 List of exposure times for each flat pair
738 meanSignalVector: `list` of `float`
739 List of mean signal from diference image of flat pairs
741 Returns
742 -------
743 polynomialLinearizerCoefficients : `list` of `float`
744 Coefficients for LinearizePolynomial, where corrImage = uncorrImage + sum_i c_i uncorrImage^(2 +
745 i).
746 c_(j-2) = -k_j/(k_1^j) with units (DN^(1-j)). The units of k_j are DN/t^j, and they are fit from
747 meanSignalVector = k0 + k1*exposureTimeVector + k2*exposureTimeVector^2 +...
748 + kn*exposureTimeVector^n, with n = "polynomialFitDegreeNonLinearity".
749 k_0 and k_1 and degenerate with bias level and gain, and are not used by the non-linearity
750 correction. Therefore, j = 2...n in the above expression (see `LinearizePolynomial` class in
751 `linearize.py`.)
753 c0 : `float`
754 Coefficient for LinearizeSquared, where corrImage = uncorrImage + c0*uncorrImage^2.
755 c0 = -k2/(k1^2), where k1 and k2 are fit from
756 meanSignalVector = k0 + k1*exposureTimeVector + k2*exposureTimeVector^2 +...
757 + kn*exposureTimeVector^n, with n = "polynomialFitDegreeNonLinearity".
759 linearizerTableRow : `list` of `float`
760 One dimensional array with deviation from linear part of n-order polynomial fit
761 to mean vs time curve. This array will be one row (for the particular amplifier at hand)
762 of the table array for LinearizeLookupTable.
764 linResidual : `list` of `float`
765 Linearity residual from the mean vs time curve, defined as
766 100*(1 - meanSignalReference/expTimeReference/(meanSignal/expTime).
768 parsFit : `list` of `float`
769 Parameters from n-order polynomial fit to meanSignalVector vs exposureTimeVector.
771 parsFitErr : list of `float`
772 Parameters from n-order polynomial fit to meanSignalVector vs exposureTimeVector.
774 reducedChiSquaredNonLinearityFit : `float`
775 Reduced chi squared from polynomial fit to meanSignalVector vs exposureTimeVector.
776 """
778 # Lookup table linearizer
779 parsIniNonLinearity = self._initialParsForPolynomial(self.config.polynomialFitDegreeNonLinearity + 1)
780 if self.config.doFitBootstrap:
781 parsFit, parsFitErr, reducedChiSquaredNonLinearityFit = self._fitBootstrap(parsIniNonLinearity,
782 exposureTimeVector,
783 meanSignalVector,
784 self.funcPolynomial)
785 else:
786 parsFit, parsFitErr, reducedChiSquaredNonLinearityFit = self._fitLeastSq(parsIniNonLinearity,
787 exposureTimeVector,
788 meanSignalVector,
789 self.funcPolynomial)
791 # LinearizeLookupTable:
792 # Use linear part to get time at wich signal is maxAduForLookupTableLinearizer DN
793 tMax = (self.config.maxAduForLookupTableLinearizer - parsFit[0])/parsFit[1]
794 timeRange = np.linspace(0, tMax, self.config.maxAduForLookupTableLinearizer)
795 signalIdeal = (parsFit[0] + parsFit[1]*timeRange).astype(int)
796 signalUncorrected = (self.funcPolynomial(parsFit, timeRange)).astype(int)
797 linearizerTableRow = signalIdeal - signalUncorrected # LinearizerLookupTable has corrections
799 # LinearizePolynomial and LinearizeSquared:
800 # Check that magnitude of higher order (>= 3) coefficents of the polyFit are small,
801 # i.e., less than threshold = 1e-10 (typical quadratic and cubic coefficents are ~1e-6
802 # and ~1e-12).
803 k1 = parsFit[1]
804 polynomialLinearizerCoefficients = []
805 for i, coefficient in enumerate(parsFit):
806 c = -coefficient/(k1**i)
807 polynomialLinearizerCoefficients.append(c)
808 if np.fabs(c) > 1e-10:
809 msg = f"Coefficient {c} in polynomial fit larger than threshold 1e-10."
810 self.log.warn(msg)
811 # Coefficient for LinearizedSquared. Called "c0" in linearize.py
812 c0 = polynomialLinearizerCoefficients[2]
814 # Linearity residual
815 linResidualTimeIndex = self.config.linResidualTimeIndex
816 if exposureTimeVector[linResidualTimeIndex] == 0.0:
817 raise RuntimeError("Reference time for linearity residual can't be 0.0")
818 linResidual = 100*(1 - ((meanSignalVector[linResidualTimeIndex] /
819 exposureTimeVector[linResidualTimeIndex]) /
820 (meanSignalVector/exposureTimeVector)))
822 return (polynomialLinearizerCoefficients, c0, linearizerTableRow, linResidual, parsFit, parsFitErr,
823 reducedChiSquaredNonLinearityFit)
825 def fitPtcAndNonLinearity(self, dataset, ptcFitType, tableArray=None):
826 """Fit the photon transfer curve and calculate linearity and residuals.
828 Fit the photon transfer curve with either a polynomial of the order
829 specified in the task config, or using the Astier approximation.
831 Sigma clipping is performed iteratively for the fit, as well as an
832 initial clipping of data points that are more than
833 config.initialNonLinearityExclusionThreshold away from lying on a
834 straight line. This other step is necessary because the photon transfer
835 curve turns over catastrophically at very high flux (because saturation
836 drops the variance to ~0) and these far outliers cause the initial fit
837 to fail, meaning the sigma cannot be calculated to perform the
838 sigma-clipping.
840 Parameters
841 ----------
842 dataset : `lsst.cp.pipe.ptc.PhotonTransferCurveDataset`
843 The dataset containing the means, variances and exposure times
844 ptcFitType : `str`
845 Fit a 'POLYNOMIAL' (degree: 'polynomialFitDegree') or
846 'ASTIERAPPROXIMATION' to the PTC
847 tableArray : `np.array`
848 Optional. Look-up table array with size rows=nAmps and columns=DN values.
849 It will be modified in-place if supplied.
851 Returns
852 -------
853 dataset: `lsst.cp.pipe.ptc.PhotonTransferCurveDataset`
854 This is the same dataset as the input paramter, however, it has been modified
855 to include information such as the fit vectors and the fit parameters. See
856 the class `PhotonTransferCurveDatase`.
857 """
859 def errFunc(p, x, y):
860 return ptcFunc(p, x) - y
862 sigmaCutPtcOutliers = self.config.sigmaCutPtcOutliers
863 maxIterationsPtcOutliers = self.config.maxIterationsPtcOutliers
865 for i, ampName in enumerate(dataset.ampNames):
866 timeVecOriginal = np.array(dataset.rawExpTimes[ampName])
867 meanVecOriginal = np.array(dataset.rawMeans[ampName])
868 varVecOriginal = np.array(dataset.rawVars[ampName])
869 varVecOriginal = self._makeZeroSafe(varVecOriginal)
871 mask = ((meanVecOriginal >= self.config.minMeanSignal) &
872 (meanVecOriginal <= self.config.maxMeanSignal))
874 goodPoints = self._getInitialGoodPoints(meanVecOriginal, varVecOriginal,
875 self.config.initialNonLinearityExclusionThresholdPositive,
876 self.config.initialNonLinearityExclusionThresholdNegative)
877 mask = mask & goodPoints
879 if ptcFitType == 'ASTIERAPPROXIMATION':
880 ptcFunc = self.funcAstier
881 parsIniPtc = [-1e-9, 1.0, 10.] # a00, gain, noise
882 bounds = self._boundsForAstier(parsIniPtc)
883 if ptcFitType == 'POLYNOMIAL':
884 ptcFunc = self.funcPolynomial
885 parsIniPtc = self._initialParsForPolynomial(self.config.polynomialFitDegree + 1)
886 bounds = self._boundsForPolynomial(parsIniPtc)
888 # Before bootstrap fit, do an iterative fit to get rid of outliers
889 count = 1
890 while count <= maxIterationsPtcOutliers:
891 # Note that application of the mask actually shrinks the array
892 # to size rather than setting elements to zero (as we want) so
893 # always update mask itself and re-apply to the original data
894 meanTempVec = meanVecOriginal[mask]
895 varTempVec = varVecOriginal[mask]
896 res = least_squares(errFunc, parsIniPtc, bounds=bounds, args=(meanTempVec, varTempVec))
897 pars = res.x
899 # change this to the original from the temp because the masks are ANDed
900 # meaning once a point is masked it's always masked, and the masks must
901 # always be the same length for broadcasting
902 sigResids = (varVecOriginal - ptcFunc(pars, meanVecOriginal))/np.sqrt(varVecOriginal)
903 newMask = np.array([True if np.abs(r) < sigmaCutPtcOutliers else False for r in sigResids])
904 mask = mask & newMask
906 nDroppedTotal = Counter(mask)[False]
907 self.log.debug(f"Iteration {count}: discarded {nDroppedTotal} points in total for {ampName}")
908 count += 1
909 # objects should never shrink
910 assert (len(mask) == len(timeVecOriginal) == len(meanVecOriginal) == len(varVecOriginal))
912 dataset.visitMask[ampName] = mask # store the final mask
914 parsIniPtc = pars
915 timeVecFinal = timeVecOriginal[mask]
916 meanVecFinal = meanVecOriginal[mask]
917 varVecFinal = varVecOriginal[mask]
919 if Counter(mask)[False] > 0:
920 self.log.info((f"Number of points discarded in PTC of amplifier {ampName}:" +
921 f" {Counter(mask)[False]} out of {len(meanVecOriginal)}"))
923 if (len(meanVecFinal) < len(parsIniPtc)):
924 msg = (f"\nSERIOUS: Not enough data points ({len(meanVecFinal)}) compared to the number of"
925 f"parameters of the PTC model({len(parsIniPtc)}). Setting {ampName} to BAD.")
926 self.log.warn(msg)
927 dataset.badAmps.append(ampName)
928 dataset.gain[ampName] = np.nan
929 dataset.gainErr[ampName] = np.nan
930 dataset.noise[ampName] = np.nan
931 dataset.noiseErr[ampName] = np.nan
932 dataset.nonLinearity[ampName] = np.nan
933 dataset.nonLinearityError[ampName] = np.nan
934 dataset.nonLinearityResiduals[ampName] = np.nan
935 dataset.coefficientLinearizeSquared[ampName] = np.nan
936 continue
938 # Fit the PTC
939 if self.config.doFitBootstrap:
940 parsFit, parsFitErr, reducedChiSqPtc = self._fitBootstrap(parsIniPtc, meanVecFinal,
941 varVecFinal, ptcFunc)
942 else:
943 parsFit, parsFitErr, reducedChiSqPtc = self._fitLeastSq(parsIniPtc, meanVecFinal,
944 varVecFinal, ptcFunc)
946 dataset.ptcFitPars[ampName] = parsFit
947 dataset.ptcFitParsError[ampName] = parsFitErr
948 dataset.ptcFitReducedChiSquared[ampName] = reducedChiSqPtc
950 if ptcFitType == 'ASTIERAPPROXIMATION':
951 ptcGain = parsFit[1]
952 ptcGainErr = parsFitErr[1]
953 ptcNoise = np.sqrt(np.fabs(parsFit[2]))
954 ptcNoiseErr = 0.5*(parsFitErr[2]/np.fabs(parsFit[2]))*np.sqrt(np.fabs(parsFit[2]))
955 if ptcFitType == 'POLYNOMIAL':
956 ptcGain = 1./parsFit[1]
957 ptcGainErr = np.fabs(1./parsFit[1])*(parsFitErr[1]/parsFit[1])
958 ptcNoise = np.sqrt(np.fabs(parsFit[0]))*ptcGain
959 ptcNoiseErr = (0.5*(parsFitErr[0]/np.fabs(parsFit[0]))*(np.sqrt(np.fabs(parsFit[0]))))*ptcGain
961 dataset.gain[ampName] = ptcGain
962 dataset.gainErr[ampName] = ptcGainErr
963 dataset.noise[ampName] = ptcNoise
964 dataset.noiseErr[ampName] = ptcNoiseErr
965 dataset.ptcFitType[ampName] = ptcFitType
967 # Non-linearity residuals (NL of mean vs time curve): percentage, and fit to a quadratic function
968 # In this case, len(parsIniNonLinearity) = 3 indicates that we want a quadratic fit
970 (coeffsLinPoly, c0, linearizerTableRow, linResidualNonLinearity,
971 parsFitNonLinearity, parsFitErrNonLinearity,
972 reducedChiSqNonLinearity) = self.calculateLinearityResidualAndLinearizers(timeVecFinal,
973 meanVecFinal)
975 # LinearizerLookupTable
976 if tableArray is not None:
977 tableArray[i, :] = linearizerTableRow
979 dataset.nonLinearity[ampName] = parsFitNonLinearity
980 dataset.nonLinearityError[ampName] = parsFitErrNonLinearity
981 dataset.nonLinearityResiduals[ampName] = linResidualNonLinearity
982 dataset.nonLinearityReducedChiSquared[ampName] = reducedChiSqNonLinearity
983 # Slice correction coefficients (starting at 2) for polynomial linearizer. The first
984 # and second are reduntant with the bias and gain, respectively,
985 # and are not used by LinearizerPolynomial.
986 dataset.coefficientsLinearizePolynomial[ampName] = np.array(coeffsLinPoly[2:])
987 dataset.coefficientLinearizeSquared[ampName] = c0
989 return dataset
991 def plot(self, dataRef, dataset, ptcFitType):
992 dirname = dataRef.getUri(datasetType='cpPipePlotRoot', write=True)
993 if not os.path.exists(dirname):
994 os.makedirs(dirname)
996 detNum = dataRef.dataId[self.config.ccdKey]
997 filename = f"PTC_det{detNum}.pdf"
998 filenameFull = os.path.join(dirname, filename)
999 with PdfPages(filenameFull) as pdfPages:
1000 self._plotPtc(dataset, ptcFitType, pdfPages)
1002 def _plotPtc(self, dataset, ptcFitType, pdfPages):
1003 """Plot PTC, linearity, and linearity residual per amplifier"""
1005 reducedChiSqPtc = dataset.ptcFitReducedChiSquared
1006 if ptcFitType == 'ASTIERAPPROXIMATION':
1007 ptcFunc = self.funcAstier
1008 stringTitle = (r"Var = $\frac{1}{2g^2a_{00}}(\exp (2a_{00} \mu g) - 1) + \frac{n_{00}}{g^2}$ "
1009 r" ($chi^2$/dof = %g)" % (reducedChiSqPtc))
1010 if ptcFitType == 'POLYNOMIAL':
1011 ptcFunc = self.funcPolynomial
1012 stringTitle = r"Polynomial (degree: %g)" % (self.config.polynomialFitDegree)
1014 legendFontSize = 7
1015 labelFontSize = 7
1016 titleFontSize = 9
1017 supTitleFontSize = 18
1018 markerSize = 25
1020 # General determination of the size of the plot grid
1021 nAmps = len(dataset.ampNames)
1022 if nAmps == 2:
1023 nRows, nCols = 2, 1
1024 nRows = np.sqrt(nAmps)
1025 mantissa, _ = np.modf(nRows)
1026 if mantissa > 0:
1027 nRows = int(nRows) + 1
1028 nCols = nRows
1029 else:
1030 nRows = int(nRows)
1031 nCols = nRows
1033 f, ax = plt.subplots(nrows=nRows, ncols=nCols, sharex='col', sharey='row', figsize=(13, 10))
1034 f2, ax2 = plt.subplots(nrows=nRows, ncols=nCols, sharex='col', sharey='row', figsize=(13, 10))
1036 for i, (amp, a, a2) in enumerate(zip(dataset.ampNames, ax.flatten(), ax2.flatten())):
1037 meanVecOriginal = np.array(dataset.rawMeans[amp])
1038 varVecOriginal = np.array(dataset.rawVars[amp])
1039 mask = dataset.visitMask[amp]
1040 meanVecFinal = meanVecOriginal[mask]
1041 varVecFinal = varVecOriginal[mask]
1042 meanVecOutliers = meanVecOriginal[np.invert(mask)]
1043 varVecOutliers = varVecOriginal[np.invert(mask)]
1044 pars, parsErr = dataset.ptcFitPars[amp], dataset.ptcFitParsError[amp]
1046 if ptcFitType == 'ASTIERAPPROXIMATION':
1047 ptcA00, ptcA00error = pars[0], parsErr[0]
1048 ptcGain, ptcGainError = pars[1], parsErr[1]
1049 ptcNoise = np.sqrt(np.fabs(pars[2]))
1050 ptcNoiseError = 0.5*(parsErr[2]/np.fabs(pars[2]))*np.sqrt(np.fabs(pars[2]))
1051 stringLegend = (f"a00: {ptcA00:.2e}+/-{ptcA00error:.2e} 1/e"
1052 f"\n Gain: {ptcGain:.4}+/-{ptcGainError:.2e} e/DN"
1053 f"\n Noise: {ptcNoise:.4}+/-{ptcNoiseError:.2e} e \n")
1055 if ptcFitType == 'POLYNOMIAL':
1056 ptcGain, ptcGainError = 1./pars[1], np.fabs(1./pars[1])*(parsErr[1]/pars[1])
1057 ptcNoise = np.sqrt(np.fabs(pars[0]))*ptcGain
1058 ptcNoiseError = (0.5*(parsErr[0]/np.fabs(pars[0]))*(np.sqrt(np.fabs(pars[0]))))*ptcGain
1059 stringLegend = (f"Noise: {ptcNoise:.4}+/-{ptcNoiseError:.2e} e \n"
1060 f"Gain: {ptcGain:.4}+/-{ptcGainError:.2e} e/DN \n")
1062 minMeanVecFinal = np.min(meanVecFinal)
1063 maxMeanVecFinal = np.max(meanVecFinal)
1064 meanVecFit = np.linspace(minMeanVecFinal, maxMeanVecFinal, 100*len(meanVecFinal))
1065 minMeanVecOriginal = np.min(meanVecOriginal)
1066 maxMeanVecOriginal = np.max(meanVecOriginal)
1067 deltaXlim = maxMeanVecOriginal - minMeanVecOriginal
1069 a.plot(meanVecFit, ptcFunc(pars, meanVecFit), color='red')
1070 a.plot(meanVecFinal, pars[0] + pars[1]*meanVecFinal, color='green', linestyle='--')
1071 a.scatter(meanVecFinal, varVecFinal, c='blue', marker='o', s=markerSize)
1072 a.scatter(meanVecOutliers, varVecOutliers, c='magenta', marker='s', s=markerSize)
1073 a.set_xlabel(r'Mean signal ($\mu$, DN)', fontsize=labelFontSize)
1074 a.set_xticks(meanVecOriginal)
1075 a.set_ylabel(r'Variance (DN$^2$)', fontsize=labelFontSize)
1076 a.tick_params(labelsize=11)
1077 a.text(0.03, 0.8, stringLegend, transform=a.transAxes, fontsize=legendFontSize)
1078 a.set_xscale('linear', fontsize=labelFontSize)
1079 a.set_yscale('linear', fontsize=labelFontSize)
1080 a.set_title(amp, fontsize=titleFontSize)
1081 a.set_xlim([minMeanVecOriginal - 0.2*deltaXlim, maxMeanVecOriginal + 0.2*deltaXlim])
1083 # Same, but in log-scale
1084 a2.plot(meanVecFit, ptcFunc(pars, meanVecFit), color='red')
1085 a2.scatter(meanVecFinal, varVecFinal, c='blue', marker='o', s=markerSize)
1086 a2.scatter(meanVecOutliers, varVecOutliers, c='magenta', marker='s', s=markerSize)
1087 a2.set_xlabel(r'Mean Signal ($\mu$, DN)', fontsize=labelFontSize)
1088 a2.set_ylabel(r'Variance (DN$^2$)', fontsize=labelFontSize)
1089 a2.tick_params(labelsize=11)
1090 a2.text(0.03, 0.8, stringLegend, transform=a2.transAxes, fontsize=legendFontSize)
1091 a2.set_xscale('log')
1092 a2.set_yscale('log')
1093 a2.set_title(amp, fontsize=titleFontSize)
1094 a2.set_xlim([minMeanVecOriginal, maxMeanVecOriginal])
1096 f.suptitle(f"PTC \n Fit: " + stringTitle, fontsize=20)
1097 pdfPages.savefig(f)
1098 f2.suptitle(f"PTC (log-log)", fontsize=20)
1099 pdfPages.savefig(f2)
1101 # Plot mean vs time
1102 f, ax = plt.subplots(nrows=4, ncols=4, sharex='col', sharey='row', figsize=(13, 10))
1103 for i, (amp, a) in enumerate(zip(dataset.ampNames, ax.flatten())):
1104 meanVecFinal = np.array(dataset.rawMeans[amp])[dataset.visitMask[amp]]
1105 timeVecFinal = np.array(dataset.rawExpTimes[amp])[dataset.visitMask[amp]]
1107 pars, parsErr = dataset.nonLinearity[amp], dataset.nonLinearityError[amp]
1108 k0, k0Error = pars[0], parsErr[0]
1109 k1, k1Error = pars[1], parsErr[1]
1110 k2, k2Error = pars[2], parsErr[2]
1111 stringLegend = (f"k0: {k0:.4}+/-{k0Error:.2e} DN\n k1: {k1:.4}+/-{k1Error:.2e} DN/t"
1112 f"\n k2: {k2:.2e}+/-{k2Error:.2e} DN/t^2 \n")
1113 a.scatter(timeVecFinal, meanVecFinal)
1114 a.plot(timeVecFinal, self.funcPolynomial(pars, timeVecFinal), color='red')
1115 a.set_xlabel('Time (sec)', fontsize=labelFontSize)
1116 a.set_xticks(timeVecFinal)
1117 a.set_ylabel(r'Mean signal ($\mu$, DN)', fontsize=labelFontSize)
1118 a.tick_params(labelsize=labelFontSize)
1119 a.text(0.03, 0.75, stringLegend, transform=a.transAxes, fontsize=legendFontSize)
1120 a.set_xscale('linear', fontsize=labelFontSize)
1121 a.set_yscale('linear', fontsize=labelFontSize)
1122 a.set_title(amp, fontsize=titleFontSize)
1123 f.suptitle("Linearity \n Fit: Polynomial (degree: %g)"
1124 % (self.config.polynomialFitDegreeNonLinearity),
1125 fontsize=supTitleFontSize)
1126 pdfPages.savefig()
1128 # Plot linearity residual
1129 f, ax = plt.subplots(nrows=4, ncols=4, sharex='col', sharey='row', figsize=(13, 10))
1130 for i, (amp, a) in enumerate(zip(dataset.ampNames, ax.flatten())):
1131 meanVecFinal = np.array(dataset.rawMeans[amp])[dataset.visitMask[amp]]
1132 linRes = np.array(dataset.nonLinearityResiduals[amp])
1134 a.scatter(meanVecFinal, linRes)
1135 a.axhline(y=0, color='k')
1136 a.axvline(x=timeVecFinal[self.config.linResidualTimeIndex], color='g', linestyle='--')
1137 a.set_xlabel(r'Mean signal ($\mu$, DN)', fontsize=labelFontSize)
1138 a.set_xticks(meanVecFinal)
1139 a.set_ylabel('LR (%)', fontsize=labelFontSize)
1140 a.tick_params(labelsize=labelFontSize)
1141 a.set_xscale('linear', fontsize=labelFontSize)
1142 a.set_yscale('linear', fontsize=labelFontSize)
1143 a.set_title(amp, fontsize=titleFontSize)
1145 f.suptitle(r"Linearity Residual: $100(1 - \mu_{\rm{ref}}/t_{\rm{ref}})/(\mu / t))$" + "\n" +
1146 r"$t_{\rm{ref}}$: " + f"{timeVecFinal[2]} s", fontsize=supTitleFontSize)
1147 pdfPages.savefig()
1149 return