Coverage for python/lsst/cp/pipe/ptc.py : 11%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of cp_pipe.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21#
23__all__ = ['MeasurePhotonTransferCurveTask',
24 'MeasurePhotonTransferCurveTaskConfig',
25 'PhotonTransferCurveDataset']
27import numpy as np
28import matplotlib.pyplot as plt
29import os
30from matplotlib.backends.backend_pdf import PdfPages
31from sqlite3 import OperationalError
32from collections import Counter
33from dataclasses import dataclass
35import lsst.afw.math as afwMath
36import lsst.pex.config as pexConfig
37import lsst.pipe.base as pipeBase
38from lsst.ip.isr import IsrTask
39from .utils import (NonexistentDatasetTaskDataIdContainer, PairedVisitListTaskRunner,
40 checkExpLengthEqual, validateIsrConfig)
41from scipy.optimize import leastsq, least_squares
42import numpy.polynomial.polynomial as poly
44from lsst.ip.isr.linearize import Linearizer
45import datetime
48class MeasurePhotonTransferCurveTaskConfig(pexConfig.Config):
49 """Config class for photon transfer curve measurement task"""
50 isr = pexConfig.ConfigurableField(
51 target=IsrTask,
52 doc="""Task to perform instrumental signature removal.""",
53 )
54 isrMandatorySteps = pexConfig.ListField(
55 dtype=str,
56 doc="isr operations that must be performed for valid results. Raises if any of these are False.",
57 default=['doAssembleCcd']
58 )
59 isrForbiddenSteps = pexConfig.ListField(
60 dtype=str,
61 doc="isr operations that must NOT be performed for valid results. Raises if any of these are True",
62 default=['doFlat', 'doFringe', 'doBrighterFatter', 'doUseOpticsTransmission',
63 'doUseFilterTransmission', 'doUseSensorTransmission', 'doUseAtmosphereTransmission']
64 )
65 isrDesirableSteps = pexConfig.ListField(
66 dtype=str,
67 doc="isr operations that it is advisable to perform, but are not mission-critical." +
68 " WARNs are logged for any of these found to be False.",
69 default=['doBias', 'doDark', 'doCrosstalk', 'doDefect']
70 )
71 isrUndesirableSteps = pexConfig.ListField(
72 dtype=str,
73 doc="isr operations that it is *not* advisable to perform in the general case, but are not" +
74 " forbidden as some use-cases might warrant them." +
75 " WARNs are logged for any of these found to be True.",
76 default=['doLinearize']
77 )
78 ccdKey = pexConfig.Field(
79 dtype=str,
80 doc="The key by which to pull a detector from a dataId, e.g. 'ccd' or 'detector'.",
81 default='ccd',
82 )
83 makePlots = pexConfig.Field(
84 dtype=bool,
85 doc="Plot the PTC curves?",
86 default=False,
87 )
88 ptcFitType = pexConfig.ChoiceField(
89 dtype=str,
90 doc="Fit PTC to approximation in Astier+19 (Equation 16) or to a polynomial.",
91 default="POLYNOMIAL",
92 allowed={
93 "POLYNOMIAL": "n-degree polynomial (use 'polynomialFitDegree' to set 'n').",
94 "ASTIERAPPROXIMATION": "Approximation in Astier+19 (Eq. 16)."
95 }
96 )
97 polynomialFitDegree = pexConfig.Field(
98 dtype=int,
99 doc="Degree of polynomial to fit the PTC, when 'ptcFitType'=POLYNOMIAL.",
100 default=2,
101 )
102 polynomialFitDegreeNonLinearity = pexConfig.Field(
103 dtype=int,
104 doc="Degree of polynomial to fit the meanSignal vs exposureTime curve to produce" +
105 " the table for LinearizeLookupTable.",
106 default=3,
107 )
108 binSize = pexConfig.Field(
109 dtype=int,
110 doc="Bin the image by this factor in both dimensions.",
111 default=1,
112 )
113 minMeanSignal = pexConfig.Field(
114 dtype=float,
115 doc="Minimum value (inclusive) of mean signal (in DN) above which to consider.",
116 default=0,
117 )
118 maxMeanSignal = pexConfig.Field(
119 dtype=float,
120 doc="Maximum value (inclusive) of mean signal (in DN) below which to consider.",
121 default=9e6,
122 )
123 initialNonLinearityExclusionThresholdPositive = pexConfig.RangeField(
124 dtype=float,
125 doc="Initially exclude data points with a variance that are more than a factor of this from being"
126 " linear in the positive direction, from the PTC fit. Note that these points will also be"
127 " excluded from the non-linearity fit. This is done before the iterative outlier rejection,"
128 " to allow an accurate determination of the sigmas for said iterative fit.",
129 default=0.12,
130 min=0.0,
131 max=1.0,
132 )
133 initialNonLinearityExclusionThresholdNegative = pexConfig.RangeField(
134 dtype=float,
135 doc="Initially exclude data points with a variance that are more than a factor of this from being"
136 " linear in the negative direction, from the PTC fit. Note that these points will also be"
137 " excluded from the non-linearity fit. This is done before the iterative outlier rejection,"
138 " to allow an accurate determination of the sigmas for said iterative fit.",
139 default=0.25,
140 min=0.0,
141 max=1.0,
142 )
143 sigmaCutPtcOutliers = pexConfig.Field(
144 dtype=float,
145 doc="Sigma cut for outlier rejection in PTC.",
146 default=5.0,
147 )
148 maxIterationsPtcOutliers = pexConfig.Field(
149 dtype=int,
150 doc="Maximum number of iterations for outlier rejection in PTC.",
151 default=2,
152 )
153 doFitBootstrap = pexConfig.Field(
154 dtype=bool,
155 doc="Use bootstrap for the PTC fit parameters and errors?.",
156 default=False,
157 )
158 linResidualTimeIndex = pexConfig.Field(
159 dtype=int,
160 doc="Index position in time array for reference time in linearity residual calculation.",
161 default=2,
162 )
163 maxAduForLookupTableLinearizer = pexConfig.Field(
164 dtype=int,
165 doc="Maximum DN value for the LookupTable linearizer.",
166 default=2**18,
167 )
168 instrumentName = pexConfig.Field(
169 dtype=str,
170 doc="Instrument name.",
171 default='',
172 )
175@dataclass
176class LinearityResidualsAndLinearizersDataset:
177 """A simple class to hold the output from the
178 `calculateLinearityResidualAndLinearizers` function.
179 """
180 # Normalized coefficients for polynomial NL correction
181 polynomialLinearizerCoefficients: list
182 # Normalized coefficient for quadratic polynomial NL correction (c0)
183 quadraticPolynomialLinearizerCoefficient: float
184 # LUT array row for the amplifier at hand
185 linearizerTableRow: list
186 # Linearity residual, Eq. 2.2. of Janesick (2001)
187 linearityResidual: list
188 meanSignalVsTimePolyFitPars: list
189 meanSignalVsTimePolyFitParsErr: list
190 fractionalNonLinearityResidual: list
191 meanSignalVsTimePolyFitReducedChiSq: float
194class PhotonTransferCurveDataset:
195 """A simple class to hold the output data from the PTC task.
197 The dataset is made up of a dictionary for each item, keyed by the
198 amplifiers' names, which much be supplied at construction time.
200 New items cannot be added to the class to save accidentally saving to the
201 wrong property, and the class can be frozen if desired.
203 inputVisitPairs records the visits used to produce the data.
204 When fitPtcAndNonLinearity() is run, a mask is built up, which is by definition
205 always the same length as inputVisitPairs, rawExpTimes, rawMeans
206 and rawVars, and is a list of bools, which are incrementally set to False
207 as points are discarded from the fits.
209 PTC fit parameters for polynomials are stored in a list in ascending order
210 of polynomial term, i.e. par[0]*x^0 + par[1]*x + par[2]*x^2 etc
211 with the length of the list corresponding to the order of the polynomial
212 plus one.
213 """
214 def __init__(self, ampNames):
215 # add items to __dict__ directly because __setattr__ is overridden
217 # instance variables
218 self.__dict__["ampNames"] = ampNames
219 self.__dict__["badAmps"] = []
221 # raw data variables
222 self.__dict__["inputVisitPairs"] = {ampName: [] for ampName in ampNames}
223 self.__dict__["visitMask"] = {ampName: [] for ampName in ampNames}
224 self.__dict__["rawExpTimes"] = {ampName: [] for ampName in ampNames}
225 self.__dict__["rawMeans"] = {ampName: [] for ampName in ampNames}
226 self.__dict__["rawVars"] = {ampName: [] for ampName in ampNames}
228 # fit information
229 self.__dict__["ptcFitType"] = {ampName: "" for ampName in ampNames}
230 self.__dict__["ptcFitPars"] = {ampName: [] for ampName in ampNames}
231 self.__dict__["ptcFitParsError"] = {ampName: [] for ampName in ampNames}
232 self.__dict__["ptcFitReducedChiSquared"] = {ampName: [] for ampName in ampNames}
233 self.__dict__["nonLinearity"] = {ampName: [] for ampName in ampNames}
234 self.__dict__["nonLinearityError"] = {ampName: [] for ampName in ampNames}
235 self.__dict__["nonLinearityResiduals"] = {ampName: [] for ampName in ampNames}
236 self.__dict__["fractionalNonLinearityResiduals"] = {ampName: [] for ampName in ampNames}
237 self.__dict__["nonLinearityReducedChiSquared"] = {ampName: [] for ampName in ampNames}
239 # final results
240 self.__dict__["gain"] = {ampName: -1. for ampName in ampNames}
241 self.__dict__["gainErr"] = {ampName: -1. for ampName in ampNames}
242 self.__dict__["noise"] = {ampName: -1. for ampName in ampNames}
243 self.__dict__["noiseErr"] = {ampName: -1. for ampName in ampNames}
244 self.__dict__["coefficientsLinearizePolynomial"] = {ampName: [] for ampName in ampNames}
245 self.__dict__["coefficientLinearizeSquared"] = {ampName: [] for ampName in ampNames}
247 def __setattr__(self, attribute, value):
248 """Protect class attributes"""
249 if attribute not in self.__dict__:
250 raise AttributeError(f"{attribute} is not already a member of PhotonTransferCurveDataset, which"
251 " does not support setting of new attributes.")
252 else:
253 self.__dict__[attribute] = value
255 def getVisitsUsed(self, ampName):
256 """Get the visits used, i.e. not discarded, for a given amp.
258 If no mask has been created yet, all visits are returned.
259 """
260 if self.visitMask[ampName] == []:
261 return self.inputVisitPairs[ampName]
263 # if the mask exists it had better be the same length as the visitPairs
264 assert len(self.visitMask[ampName]) == len(self.inputVisitPairs[ampName])
266 pairs = self.inputVisitPairs[ampName]
267 mask = self.visitMask[ampName]
268 # cast to bool required because numpy
269 return [(v1, v2) for ((v1, v2), m) in zip(pairs, mask) if bool(m) is True]
271 def getGoodAmps(self):
272 return [amp for amp in self.ampNames if amp not in self.badAmps]
275class MeasurePhotonTransferCurveTask(pipeBase.CmdLineTask):
276 """A class to calculate, fit, and plot a PTC from a set of flat pairs.
278 The Photon Transfer Curve (var(signal) vs mean(signal)) is a standard tool
279 used in astronomical detectors characterization (e.g., Janesick 2001,
280 Janesick 2007). This task calculates the PTC from a series of pairs of
281 flat-field images; each pair taken at identical exposure times. The
282 difference image of each pair is formed to eliminate fixed pattern noise,
283 and then the variance of the difference image and the mean of the average image
284 are used to produce the PTC. An n-degree polynomial or the approximation in Equation
285 16 of Astier+19 ("The Shape of the Photon Transfer Curve of CCD sensors",
286 arXiv:1905.08677) can be fitted to the PTC curve. These models include
287 parameters such as the gain (e/DN) and readout noise.
289 Linearizers to correct for signal-chain non-linearity are also calculated.
290 The `Linearizer` class, in general, can support per-amp linearizers, but in this
291 task this is not supported.
292 Parameters
293 ----------
295 *args: `list`
296 Positional arguments passed to the Task constructor. None used at this
297 time.
298 **kwargs: `dict`
299 Keyword arguments passed on to the Task constructor. None used at this
300 time.
302 """
304 RunnerClass = PairedVisitListTaskRunner
305 ConfigClass = MeasurePhotonTransferCurveTaskConfig
306 _DefaultName = "measurePhotonTransferCurve"
308 def __init__(self, *args, **kwargs):
309 pipeBase.CmdLineTask.__init__(self, *args, **kwargs)
310 self.makeSubtask("isr")
311 plt.interactive(False) # stop windows popping up when plotting. When headless, use 'agg' backend too
312 validateIsrConfig(self.isr, self.config.isrMandatorySteps,
313 self.config.isrForbiddenSteps, self.config.isrDesirableSteps, checkTrim=False)
314 self.config.validate()
315 self.config.freeze()
317 @classmethod
318 def _makeArgumentParser(cls):
319 """Augment argument parser for the MeasurePhotonTransferCurveTask."""
320 parser = pipeBase.ArgumentParser(name=cls._DefaultName)
321 parser.add_argument("--visit-pairs", dest="visitPairs", nargs="*",
322 help="Visit pairs to use. Each pair must be of the form INT,INT e.g. 123,456")
323 parser.add_id_argument("--id", datasetType="photonTransferCurveDataset",
324 ContainerClass=NonexistentDatasetTaskDataIdContainer,
325 help="The ccds to use, e.g. --id ccd=0..100")
326 return parser
328 @pipeBase.timeMethod
329 def runDataRef(self, dataRef, visitPairs):
330 """Run the Photon Transfer Curve (PTC) measurement task.
332 For a dataRef (which is each detector here),
333 and given a list of visit pairs at different exposure times,
334 measure the PTC.
336 Parameters
337 ----------
338 dataRef : list of lsst.daf.persistence.ButlerDataRef
339 dataRef for the detector for the visits to be fit.
340 visitPairs : `iterable` of `tuple` of `int`
341 Pairs of visit numbers to be processed together
342 """
344 # setup necessary objects
345 detNum = dataRef.dataId[self.config.ccdKey]
346 detector = dataRef.get('camera')[dataRef.dataId[self.config.ccdKey]]
347 # expand some missing fields that we need for lsstCam. This is a work-around
348 # for Gen2 problems that I (RHL) don't feel like solving. The calibs pipelines
349 # (which inherit from CalibTask) use addMissingKeys() to do basically the same thing
350 #
351 # Basically, the butler's trying to look up the fields in `raw_visit` which won't work
352 for name in dataRef.getButler().getKeys('bias'):
353 if name not in dataRef.dataId:
354 try:
355 dataRef.dataId[name] = \
356 dataRef.getButler().queryMetadata('raw', [name], detector=detNum)[0]
357 except OperationalError:
358 pass
360 amps = detector.getAmplifiers()
361 ampNames = [amp.getName() for amp in amps]
362 dataset = PhotonTransferCurveDataset(ampNames)
364 self.log.info('Measuring PTC using %s visits for detector %s' % (visitPairs, detNum))
366 for (v1, v2) in visitPairs:
367 # Perform ISR on each exposure
368 dataRef.dataId['expId'] = v1
369 exp1 = self.isr.runDataRef(dataRef).exposure
370 dataRef.dataId['expId'] = v2
371 exp2 = self.isr.runDataRef(dataRef).exposure
372 del dataRef.dataId['expId']
374 checkExpLengthEqual(exp1, exp2, v1, v2, raiseWithMessage=True)
375 expTime = exp1.getInfo().getVisitInfo().getExposureTime()
377 for amp in detector:
378 mu, varDiff = self.measureMeanVarPair(exp1, exp2, region=amp.getBBox())
379 ampName = amp.getName()
381 dataset.rawExpTimes[ampName].append(expTime)
382 dataset.rawMeans[ampName].append(mu)
383 dataset.rawVars[ampName].append(varDiff)
384 dataset.inputVisitPairs[ampName].append((v1, v2))
386 numberAmps = len(detector.getAmplifiers())
387 numberAduValues = self.config.maxAduForLookupTableLinearizer
388 lookupTableArray = np.zeros((numberAmps, numberAduValues), dtype=np.float32)
390 # Fit PTC and (non)linearity of signal vs time curve.
391 # Fill up PhotonTransferCurveDataset object.
392 # Fill up array for LUT linearizer.
393 # Produce coefficients for Polynomial ans Squared linearizers.
394 dataset = self.fitPtcAndNonLinearity(dataset, self.config.ptcFitType,
395 tableArray=lookupTableArray)
397 if self.config.makePlots:
398 self.plot(dataRef, dataset, ptcFitType=self.config.ptcFitType)
400 # Save data, PTC fit, and NL fit dictionaries
401 self.log.info(f"Writing PTC and NL data to {dataRef.getUri(write=True)}")
402 dataRef.put(dataset, datasetType="photonTransferCurveDataset")
404 butler = dataRef.getButler()
405 self.log.info(f"Writing linearizers: \n "
406 "lookup table (linear component of polynomial fit), \n "
407 "polynomial (coefficients for a polynomial correction), \n "
408 "and squared linearizer (quadratic coefficient from polynomial)")
410 detName = detector.getName()
411 now = datetime.datetime.utcnow()
412 calibDate = now.strftime("%Y-%m-%d")
414 for linType, dataType in [("LOOKUPTABLE", 'linearizeLut'),
415 ("LINEARIZEPOLYNOMIAL", 'linearizePolynomial'),
416 ("LINEARIZESQUARED", 'linearizeSquared')]:
418 if linType == "LOOKUPTABLE":
419 tableArray = lookupTableArray
420 else:
421 tableArray = None
423 linearizer = self.buildLinearizerObject(dataset, detector, calibDate, linType,
424 instruName=self.config.instrumentName,
425 tableArray=tableArray,
426 log=self.log)
427 butler.put(linearizer, datasetType=dataType, dataId={'detector': detNum,
428 'detectorName': detName, 'calibDate': calibDate})
430 self.log.info('Finished measuring PTC for in detector %s' % detNum)
432 return pipeBase.Struct(exitStatus=0)
434 def buildLinearizerObject(self, dataset, detector, calibDate, linearizerType, instruName='',
435 tableArray=None, log=None):
436 """Build linearizer object to persist.
438 Parameters
439 ----------
440 dataset : `lsst.cp.pipe.ptc.PhotonTransferCurveDataset`
441 The dataset containing the means, variances, and exposure times
442 detector : `lsst.afw.cameraGeom.Detector`
443 Detector object
444 calibDate : `datetime.datetime`
445 Calibration date
446 linearizerType : `str`
447 'LOOKUPTABLE', 'LINEARIZESQUARED', or 'LINEARIZEPOLYNOMIAL'
448 instruName : `str`, optional
449 Instrument name
450 tableArray : `np.array`, optional
451 Look-up table array with size rows=nAmps and columns=DN values
452 log : `lsst.log.Log`, optional
453 Logger to handle messages
455 Returns
456 -------
457 linearizer : `lsst.ip.isr.Linearizer`
458 Linearizer object
459 """
460 detName = detector.getName()
461 detNum = detector.getId()
462 if linearizerType == "LOOKUPTABLE":
463 if tableArray is not None:
464 linearizer = Linearizer(detector=detector, table=tableArray, log=log)
465 else:
466 raise RuntimeError("tableArray must be provided when creating a LookupTable linearizer")
467 elif linearizerType in ("LINEARIZESQUARED", "LINEARIZEPOLYNOMIAL"):
468 linearizer = Linearizer(log=log)
469 else:
470 raise RuntimeError("Invalid linearizerType {linearizerType} to build a Linearizer object. "
471 "Supported: 'LOOKUPTABLE', 'LINEARIZESQUARED', or 'LINEARIZEPOLYNOMIAL'")
472 for i, amp in enumerate(detector.getAmplifiers()):
473 ampName = amp.getName()
474 if linearizerType == "LOOKUPTABLE":
475 linearizer.linearityCoeffs[ampName] = [i, 0]
476 linearizer.linearityType[ampName] = "LookupTable"
477 elif linearizerType == "LINEARIZESQUARED":
478 linearizer.linearityCoeffs[ampName] = [dataset.coefficientLinearizeSquared[ampName]]
479 linearizer.linearityType[ampName] = "Squared"
480 elif linearizerType == "LINEARIZEPOLYNOMIAL":
481 linearizer.linearityCoeffs[ampName] = dataset.coefficientsLinearizePolynomial[ampName]
482 linearizer.linearityType[ampName] = "Polynomial"
483 linearizer.linearityBBox[ampName] = amp.getBBox()
485 linearizer.validate()
486 calibId = f"detectorName={detName} detector={detNum} calibDate={calibDate} ccd={detNum} filter=NONE"
488 try:
489 raftName = detName.split("_")[0]
490 calibId += f" raftName={raftName}"
491 except Exception:
492 raftname = "NONE"
493 calibId += f" raftName={raftname}"
495 serial = detector.getSerial()
496 linearizer.updateMetadata(instrumentName=instruName, detectorId=f"{detNum}",
497 calibId=calibId, serial=serial, detectorName=f"{detName}")
499 return linearizer
501 def measureMeanVarPair(self, exposure1, exposure2, region=None):
502 """Calculate the mean signal of two exposures and the variance of their difference.
504 Parameters
505 ----------
506 exposure1 : `lsst.afw.image.exposure.exposure.ExposureF`
507 First exposure of flat field pair.
509 exposure2 : `lsst.afw.image.exposure.exposure.ExposureF`
510 Second exposure of flat field pair.
512 region : `lsst.geom.Box2I`
513 Region of each exposure where to perform the calculations (e.g, an amplifier).
515 Return
516 ------
518 mu : `float`
519 0.5*(mu1 + mu2), where mu1, and mu2 are the clipped means of the regions in
520 both exposures.
522 varDiff : `float`
523 Half of the clipped variance of the difference of the regions inthe two input
524 exposures.
525 """
527 if region is not None:
528 im1Area = exposure1.maskedImage[region]
529 im2Area = exposure2.maskedImage[region]
530 else:
531 im1Area = exposure1.maskedImage
532 im2Area = exposure2.maskedImage
534 im1Area = afwMath.binImage(im1Area, self.config.binSize)
535 im2Area = afwMath.binImage(im2Area, self.config.binSize)
537 # Clipped mean of images; then average of mean.
538 mu1 = afwMath.makeStatistics(im1Area, afwMath.MEANCLIP).getValue()
539 mu2 = afwMath.makeStatistics(im2Area, afwMath.MEANCLIP).getValue()
540 mu = 0.5*(mu1 + mu2)
542 # Take difference of pairs
543 # symmetric formula: diff = (mu2*im1-mu1*im2)/(0.5*(mu1+mu2))
544 temp = im2Area.clone()
545 temp *= mu1
546 diffIm = im1Area.clone()
547 diffIm *= mu2
548 diffIm -= temp
549 diffIm /= mu
551 varDiff = 0.5*(afwMath.makeStatistics(diffIm, afwMath.VARIANCECLIP).getValue())
553 return mu, varDiff
555 def _fitLeastSq(self, initialParams, dataX, dataY, function):
556 """Do a fit and estimate the parameter errors using using scipy.optimize.leastq.
558 optimize.leastsq returns the fractional covariance matrix. To estimate the
559 standard deviation of the fit parameters, multiply the entries of this matrix
560 by the unweighted reduced chi squared and take the square root of the diagonal elements.
562 Parameters
563 ----------
564 initialParams : `list` of `float`
565 initial values for fit parameters. For ptcFitType=POLYNOMIAL, its length
566 determines the degree of the polynomial.
568 dataX : `numpy.array` of `float`
569 Data in the abscissa axis.
571 dataY : `numpy.array` of `float`
572 Data in the ordinate axis.
574 function : callable object (function)
575 Function to fit the data with.
577 Return
578 ------
579 pFitSingleLeastSquares : `list` of `float`
580 List with fitted parameters.
582 pErrSingleLeastSquares : `list` of `float`
583 List with errors for fitted parameters.
585 reducedChiSqSingleLeastSquares : `float`
586 Unweighted reduced chi squared
587 """
589 def errFunc(p, x, y):
590 return function(p, x) - y
592 pFit, pCov, infoDict, errMessage, success = leastsq(errFunc, initialParams,
593 args=(dataX, dataY), full_output=1, epsfcn=0.0001)
595 if (len(dataY) > len(initialParams)) and pCov is not None:
596 reducedChiSq = (errFunc(pFit, dataX, dataY)**2).sum()/(len(dataY)-len(initialParams))
597 pCov *= reducedChiSq
598 else:
599 pCov = np.zeros((len(initialParams), len(initialParams)))
600 pCov[:, :] = np.inf
601 reducedChiSq = np.inf
603 errorVec = []
604 for i in range(len(pFit)):
605 errorVec.append(np.fabs(pCov[i][i])**0.5)
607 pFitSingleLeastSquares = pFit
608 pErrSingleLeastSquares = np.array(errorVec)
610 return pFitSingleLeastSquares, pErrSingleLeastSquares, reducedChiSq
612 def _fitBootstrap(self, initialParams, dataX, dataY, function, confidenceSigma=1.):
613 """Do a fit using least squares and bootstrap to estimate parameter errors.
615 The bootstrap error bars are calculated by fitting 100 random data sets.
617 Parameters
618 ----------
619 initialParams : `list` of `float`
620 initial values for fit parameters. For ptcFitType=POLYNOMIAL, its length
621 determines the degree of the polynomial.
623 dataX : `numpy.array` of `float`
624 Data in the abscissa axis.
626 dataY : `numpy.array` of `float`
627 Data in the ordinate axis.
629 function : callable object (function)
630 Function to fit the data with.
632 confidenceSigma : `float`
633 Number of sigmas that determine confidence interval for the bootstrap errors.
635 Return
636 ------
637 pFitBootstrap : `list` of `float`
638 List with fitted parameters.
640 pErrBootstrap : `list` of `float`
641 List with errors for fitted parameters.
643 reducedChiSqBootstrap : `float`
644 Reduced chi squared.
645 """
647 def errFunc(p, x, y):
648 return function(p, x) - y
650 # Fit first time
651 pFit, _ = leastsq(errFunc, initialParams, args=(dataX, dataY), full_output=0)
653 # Get the stdev of the residuals
654 residuals = errFunc(pFit, dataX, dataY)
655 sigmaErrTotal = np.std(residuals)
657 # 100 random data sets are generated and fitted
658 pars = []
659 for i in range(100):
660 randomDelta = np.random.normal(0., sigmaErrTotal, len(dataY))
661 randomDataY = dataY + randomDelta
662 randomFit, _ = leastsq(errFunc, initialParams,
663 args=(dataX, randomDataY), full_output=0)
664 pars.append(randomFit)
665 pars = np.array(pars)
666 meanPfit = np.mean(pars, 0)
668 # confidence interval for parameter estimates
669 nSigma = confidenceSigma
670 errPfit = nSigma*np.std(pars, 0)
671 pFitBootstrap = meanPfit
672 pErrBootstrap = errPfit
674 reducedChiSq = (errFunc(pFitBootstrap, dataX, dataY)**2).sum()/(len(dataY)-len(initialParams))
675 return pFitBootstrap, pErrBootstrap, reducedChiSq
677 def funcPolynomial(self, pars, x):
678 """Polynomial function definition"""
679 return poly.polyval(x, [*pars])
681 def funcAstier(self, pars, x):
682 """Single brighter-fatter parameter model for PTC; Equation 16 of Astier+19"""
683 a00, gain, noise = pars
684 return 0.5/(a00*gain*gain)*(np.exp(2*a00*x*gain)-1) + noise/(gain*gain)
686 @staticmethod
687 def _initialParsForPolynomial(order):
688 assert(order >= 2)
689 pars = np.zeros(order, dtype=np.float)
690 pars[0] = 10
691 pars[1] = 1
692 pars[2:] = 0.0001
693 return pars
695 @staticmethod
696 def _boundsForPolynomial(initialPars):
697 lowers = [np.NINF for p in initialPars]
698 uppers = [np.inf for p in initialPars]
699 lowers[1] = 0 # no negative gains
700 return (lowers, uppers)
702 @staticmethod
703 def _boundsForAstier(initialPars):
704 lowers = [np.NINF for p in initialPars]
705 uppers = [np.inf for p in initialPars]
706 return (lowers, uppers)
708 @staticmethod
709 def _getInitialGoodPoints(means, variances, maxDeviationPositive, maxDeviationNegative):
710 """Return a boolean array to mask bad points.
712 A linear function has a constant ratio, so find the median
713 value of the ratios, and exclude the points that deviate
714 from that by more than a factor of maxDeviationPositive/negative.
715 Asymmetric deviations are supported as we expect the PTC to turn
716 down as the flux increases, but sometimes it anomalously turns
717 upwards just before turning over, which ruins the fits, so it
718 is wise to be stricter about restricting positive outliers than
719 negative ones.
721 Too high and points that are so bad that fit will fail will be included
722 Too low and the non-linear points will be excluded, biasing the NL fit."""
723 ratios = [b/a for (a, b) in zip(means, variances)]
724 medianRatio = np.median(ratios)
725 ratioDeviations = [(r/medianRatio)-1 for r in ratios]
727 # so that it doesn't matter if the deviation is expressed as positive or negative
728 maxDeviationPositive = abs(maxDeviationPositive)
729 maxDeviationNegative = -1. * abs(maxDeviationNegative)
731 goodPoints = np.array([True if (r < maxDeviationPositive and r > maxDeviationNegative)
732 else False for r in ratioDeviations])
733 return goodPoints
735 def _makeZeroSafe(self, array, warn=True, substituteValue=1e-9):
736 """"""
737 nBad = Counter(array)[0]
738 if nBad == 0:
739 return array
741 if warn:
742 msg = f"Found {nBad} zeros in array at elements {[x for x in np.where(array==0)[0]]}"
743 self.log.warn(msg)
745 array[array == 0] = substituteValue
746 return array
748 def calculateLinearityResidualAndLinearizers(self, exposureTimeVector, meanSignalVector):
749 """Calculate linearity residual and fit an n-order polynomial to the mean vs time curve
750 to produce corrections (deviation from linear part of polynomial) for a particular amplifier
751 to populate LinearizeLookupTable.
752 Use the coefficients of this fit to calculate the correction coefficients for LinearizePolynomial
753 and LinearizeSquared."
755 Parameters
756 ---------
758 exposureTimeVector: `list` of `float`
759 List of exposure times for each flat pair
761 meanSignalVector: `list` of `float`
762 List of mean signal from diference image of flat pairs
764 Returns
765 -------
766 dataset : `lsst.cp.pipe.ptc.LinearityResidualsAndLinearizersDataset`
767 The dataset containing the fit parameters, the NL correction coefficients, and the
768 LUT row for the amplifier at hand. Explicitly:
770 dataset.polynomialLinearizerCoefficients : `list` of `float`
771 Coefficients for LinearizePolynomial, where corrImage = uncorrImage + sum_i c_i uncorrImage^(2 +
772 i).
773 c_(j-2) = -k_j/(k_1^j) with units (DN^(1-j)). The units of k_j are DN/t^j, and they are fit from
774 meanSignalVector = k0 + k1*exposureTimeVector + k2*exposureTimeVector^2 +...
775 + kn*exposureTimeVector^n, with n = "polynomialFitDegreeNonLinearity".
776 k_0 and k_1 and degenerate with bias level and gain, and are not used by the non-linearity
777 correction. Therefore, j = 2...n in the above expression (see `LinearizePolynomial` class in
778 `linearize.py`.)
780 dataset.quadraticPolynomialLinearizerCoefficient : `float`
781 Coefficient for LinearizeSquared, where corrImage = uncorrImage + c0*uncorrImage^2.
782 c0 = -k2/(k1^2), where k1 and k2 are fit from
783 meanSignalVector = k0 + k1*exposureTimeVector + k2*exposureTimeVector^2 +...
784 + kn*exposureTimeVector^n, with n = "polynomialFitDegreeNonLinearity".
786 dataset.linearizerTableRow : `list` of `float`
787 One dimensional array with deviation from linear part of n-order polynomial fit
788 to mean vs time curve. This array will be one row (for the particular amplifier at hand)
789 of the table array for LinearizeLookupTable.
791 dataset.linearityResidual : `list` of `float`
792 Linearity residual from the mean vs time curve, defined as
793 100*(1 - meanSignalReference/expTimeReference/(meanSignal/expTime).
795 dataset.meanSignalVsTimePolyFitPars : `list` of `float`
796 Parameters from n-order polynomial fit to meanSignalVector vs exposureTimeVector.
798 dataset.meanSignalVsTimePolyFitParsErr : `list` of `float`
799 Parameters from n-order polynomial fit to meanSignalVector vs exposureTimeVector.
801 dataset.fractionalNonLinearityResidual : `list` of `float`
802 Fractional residuals from the meanSignal vs exposureTime curve with respect to linear part of
803 polynomial fit: 100*(linearPart - meanSignal)/linearPart, where
804 linearPart = k0 + k1*exposureTimeVector.
806 dataset.meanSignalVsTimePolyFitReducedChiSq : `float`
807 Reduced unweighted chi squared from polynomial fit to meanSignalVector vs exposureTimeVector.
808 """
810 # Lookup table linearizer
811 parsIniNonLinearity = self._initialParsForPolynomial(self.config.polynomialFitDegreeNonLinearity + 1)
812 if self.config.doFitBootstrap:
813 parsFit, parsFitErr, reducedChiSquaredNonLinearityFit = self._fitBootstrap(parsIniNonLinearity,
814 exposureTimeVector,
815 meanSignalVector,
816 self.funcPolynomial)
817 else:
818 parsFit, parsFitErr, reducedChiSquaredNonLinearityFit = self._fitLeastSq(parsIniNonLinearity,
819 exposureTimeVector,
820 meanSignalVector,
821 self.funcPolynomial)
823 # LinearizeLookupTable:
824 # Use linear part to get time at wich signal is maxAduForLookupTableLinearizer DN
825 tMax = (self.config.maxAduForLookupTableLinearizer - parsFit[0])/parsFit[1]
826 timeRange = np.linspace(0, tMax, self.config.maxAduForLookupTableLinearizer)
827 signalIdeal = parsFit[0] + parsFit[1]*timeRange
828 signalUncorrected = self.funcPolynomial(parsFit, timeRange)
829 linearizerTableRow = signalIdeal - signalUncorrected # LinearizerLookupTable has corrections
830 # LinearizePolynomial and LinearizeSquared:
831 # Check that magnitude of higher order (>= 3) coefficents of the polyFit are small,
832 # i.e., less than threshold = 1e-10 (typical quadratic and cubic coefficents are ~1e-6
833 # and ~1e-12).
834 k1 = parsFit[1]
835 polynomialLinearizerCoefficients = []
836 for i, coefficient in enumerate(parsFit):
837 c = -coefficient/(k1**i)
838 polynomialLinearizerCoefficients.append(c)
839 if np.fabs(c) > 1e-10:
840 msg = f"Coefficient {c} in polynomial fit larger than threshold 1e-10."
841 self.log.warn(msg)
842 # Coefficient for LinearizedSquared. Called "c0" in linearize.py
843 c0 = polynomialLinearizerCoefficients[2]
845 # Linearity residual
846 linResidualTimeIndex = self.config.linResidualTimeIndex
847 if exposureTimeVector[linResidualTimeIndex] == 0.0:
848 raise RuntimeError("Reference time for linearity residual can't be 0.0")
849 linResidual = 100*(1 - ((meanSignalVector[linResidualTimeIndex] /
850 exposureTimeVector[linResidualTimeIndex]) /
851 (meanSignalVector/exposureTimeVector)))
853 # Fractional non-linearity residual, w.r.t linear part of polynomial fit
854 linearPart = parsFit[0] + k1*exposureTimeVector
855 fracNonLinearityResidual = 100*(linearPart - meanSignalVector)/linearPart
857 dataset = LinearityResidualsAndLinearizersDataset([], None, [], [], [], [], [], None)
858 dataset.polynomialLinearizerCoefficients = polynomialLinearizerCoefficients
859 dataset.quadraticPolynomialLinearizerCoefficient = c0
860 dataset.linearizerTableRow = linearizerTableRow
861 dataset.linearityResidual = linResidual
862 dataset.meanSignalVsTimePolyFitPars = parsFit
863 dataset.meanSignalVsTimePolyFitParsErr = parsFitErr
864 dataset.fractionalNonLinearityResidual = fracNonLinearityResidual
865 dataset.meanSignalVsTimePolyFitReducedChiSq = reducedChiSquaredNonLinearityFit
867 return dataset
869 def fitPtcAndNonLinearity(self, dataset, ptcFitType, tableArray=None):
870 """Fit the photon transfer curve and calculate linearity and residuals.
872 Fit the photon transfer curve with either a polynomial of the order
873 specified in the task config, or using the Astier approximation.
875 Sigma clipping is performed iteratively for the fit, as well as an
876 initial clipping of data points that are more than
877 config.initialNonLinearityExclusionThreshold away from lying on a
878 straight line. This other step is necessary because the photon transfer
879 curve turns over catastrophically at very high flux (because saturation
880 drops the variance to ~0) and these far outliers cause the initial fit
881 to fail, meaning the sigma cannot be calculated to perform the
882 sigma-clipping.
884 Parameters
885 ----------
886 dataset : `lsst.cp.pipe.ptc.PhotonTransferCurveDataset`
887 The dataset containing the means, variances and exposure times
888 ptcFitType : `str`
889 Fit a 'POLYNOMIAL' (degree: 'polynomialFitDegree') or
890 'ASTIERAPPROXIMATION' to the PTC
891 tableArray : `np.array`
892 Optional. Look-up table array with size rows=nAmps and columns=DN values.
893 It will be modified in-place if supplied.
895 Returns
896 -------
897 dataset: `lsst.cp.pipe.ptc.PhotonTransferCurveDataset`
898 This is the same dataset as the input paramter, however, it has been modified
899 to include information such as the fit vectors and the fit parameters. See
900 the class `PhotonTransferCurveDatase`.
901 """
903 def errFunc(p, x, y):
904 return ptcFunc(p, x) - y
906 sigmaCutPtcOutliers = self.config.sigmaCutPtcOutliers
907 maxIterationsPtcOutliers = self.config.maxIterationsPtcOutliers
909 for i, ampName in enumerate(dataset.ampNames):
910 timeVecOriginal = np.array(dataset.rawExpTimes[ampName])
911 meanVecOriginal = np.array(dataset.rawMeans[ampName])
912 varVecOriginal = np.array(dataset.rawVars[ampName])
913 varVecOriginal = self._makeZeroSafe(varVecOriginal)
915 mask = ((meanVecOriginal >= self.config.minMeanSignal) &
916 (meanVecOriginal <= self.config.maxMeanSignal))
918 goodPoints = self._getInitialGoodPoints(meanVecOriginal, varVecOriginal,
919 self.config.initialNonLinearityExclusionThresholdPositive,
920 self.config.initialNonLinearityExclusionThresholdNegative)
921 mask = mask & goodPoints
923 if ptcFitType == 'ASTIERAPPROXIMATION':
924 ptcFunc = self.funcAstier
925 parsIniPtc = [-1e-9, 1.0, 10.] # a00, gain, noise
926 bounds = self._boundsForAstier(parsIniPtc)
927 if ptcFitType == 'POLYNOMIAL':
928 ptcFunc = self.funcPolynomial
929 parsIniPtc = self._initialParsForPolynomial(self.config.polynomialFitDegree + 1)
930 bounds = self._boundsForPolynomial(parsIniPtc)
932 # Before bootstrap fit, do an iterative fit to get rid of outliers
933 count = 1
934 while count <= maxIterationsPtcOutliers:
935 # Note that application of the mask actually shrinks the array
936 # to size rather than setting elements to zero (as we want) so
937 # always update mask itself and re-apply to the original data
938 meanTempVec = meanVecOriginal[mask]
939 varTempVec = varVecOriginal[mask]
940 res = least_squares(errFunc, parsIniPtc, bounds=bounds, args=(meanTempVec, varTempVec))
941 pars = res.x
943 # change this to the original from the temp because the masks are ANDed
944 # meaning once a point is masked it's always masked, and the masks must
945 # always be the same length for broadcasting
946 sigResids = (varVecOriginal - ptcFunc(pars, meanVecOriginal))/np.sqrt(varVecOriginal)
947 newMask = np.array([True if np.abs(r) < sigmaCutPtcOutliers else False for r in sigResids])
948 mask = mask & newMask
950 nDroppedTotal = Counter(mask)[False]
951 self.log.debug(f"Iteration {count}: discarded {nDroppedTotal} points in total for {ampName}")
952 count += 1
953 # objects should never shrink
954 assert (len(mask) == len(timeVecOriginal) == len(meanVecOriginal) == len(varVecOriginal))
956 dataset.visitMask[ampName] = mask # store the final mask
958 parsIniPtc = pars
959 timeVecFinal = timeVecOriginal[mask]
960 meanVecFinal = meanVecOriginal[mask]
961 varVecFinal = varVecOriginal[mask]
963 if Counter(mask)[False] > 0:
964 self.log.info((f"Number of points discarded in PTC of amplifier {ampName}:" +
965 f" {Counter(mask)[False]} out of {len(meanVecOriginal)}"))
967 if (len(meanVecFinal) < len(parsIniPtc)):
968 msg = (f"\nSERIOUS: Not enough data points ({len(meanVecFinal)}) compared to the number of"
969 f"parameters of the PTC model({len(parsIniPtc)}). Setting {ampName} to BAD.")
970 self.log.warn(msg)
971 # The first and second parameters of initial fit are discarded (bias and gain)
972 # for the final NL coefficients
973 lenNonLinPars = self.config.polynomialFitDegreeNonLinearity - 1
974 dataset.badAmps.append(ampName)
975 dataset.gain[ampName] = np.nan
976 dataset.gainErr[ampName] = np.nan
977 dataset.noise[ampName] = np.nan
978 dataset.noiseErr[ampName] = np.nan
979 dataset.nonLinearity[ampName] = np.nan
980 dataset.nonLinearityError[ampName] = np.nan
981 dataset.nonLinearityResiduals[ampName] = np.nan
982 dataset.fractionalNonLinearityResiduals[ampName] = np.nan
983 dataset.coefficientLinearizeSquared[ampName] = np.nan
984 dataset.ptcFitPars[ampName] = np.nan
985 dataset.ptcFitParsError[ampName] = np.nan
986 dataset.ptcFitReducedChiSquared[ampName] = np.nan
987 dataset.coefficientsLinearizePolynomial[ampName] = [np.nan]*lenNonLinPars
988 tableArray[i, :] = [np.nan]*self.config.maxAduForLookupTableLinearizer
989 continue
991 # Fit the PTC
992 if self.config.doFitBootstrap:
993 parsFit, parsFitErr, reducedChiSqPtc = self._fitBootstrap(parsIniPtc, meanVecFinal,
994 varVecFinal, ptcFunc)
995 else:
996 parsFit, parsFitErr, reducedChiSqPtc = self._fitLeastSq(parsIniPtc, meanVecFinal,
997 varVecFinal, ptcFunc)
999 dataset.ptcFitPars[ampName] = parsFit
1000 dataset.ptcFitParsError[ampName] = parsFitErr
1001 dataset.ptcFitReducedChiSquared[ampName] = reducedChiSqPtc
1003 if ptcFitType == 'ASTIERAPPROXIMATION':
1004 ptcGain = parsFit[1]
1005 ptcGainErr = parsFitErr[1]
1006 ptcNoise = np.sqrt(np.fabs(parsFit[2]))
1007 ptcNoiseErr = 0.5*(parsFitErr[2]/np.fabs(parsFit[2]))*np.sqrt(np.fabs(parsFit[2]))
1008 if ptcFitType == 'POLYNOMIAL':
1009 ptcGain = 1./parsFit[1]
1010 ptcGainErr = np.fabs(1./parsFit[1])*(parsFitErr[1]/parsFit[1])
1011 ptcNoise = np.sqrt(np.fabs(parsFit[0]))*ptcGain
1012 ptcNoiseErr = (0.5*(parsFitErr[0]/np.fabs(parsFit[0]))*(np.sqrt(np.fabs(parsFit[0]))))*ptcGain
1014 dataset.gain[ampName] = ptcGain
1015 dataset.gainErr[ampName] = ptcGainErr
1016 dataset.noise[ampName] = ptcNoise
1017 dataset.noiseErr[ampName] = ptcNoiseErr
1018 dataset.ptcFitType[ampName] = ptcFitType
1020 # Non-linearity residuals (NL of mean vs time curve): percentage, and fit to a quadratic function
1021 # In this case, len(parsIniNonLinearity) = 3 indicates that we want a quadratic fit
1023 datasetLinRes = self.calculateLinearityResidualAndLinearizers(timeVecFinal, meanVecFinal)
1025 # LinearizerLookupTable
1026 if tableArray is not None:
1027 tableArray[i, :] = datasetLinRes.linearizerTableRow
1028 dataset.nonLinearity[ampName] = datasetLinRes.meanSignalVsTimePolyFitPars
1029 dataset.nonLinearityError[ampName] = datasetLinRes.meanSignalVsTimePolyFitParsErr
1030 dataset.nonLinearityResiduals[ampName] = datasetLinRes.linearityResidual
1031 dataset.fractionalNonLinearityResiduals[ampName] = datasetLinRes.fractionalNonLinearityResidual
1032 dataset.nonLinearityReducedChiSquared[ampName] = datasetLinRes.meanSignalVsTimePolyFitReducedChiSq
1033 # Slice correction coefficients (starting at 2) for polynomial linearizer. The first
1034 # and second are reduntant with the bias and gain, respectively,
1035 # and are not used by LinearizerPolynomial.
1036 polyLinCoeffs = np.array(datasetLinRes.polynomialLinearizerCoefficients[2:])
1037 dataset.coefficientsLinearizePolynomial[ampName] = polyLinCoeffs
1038 quadPolyLinCoeff = datasetLinRes.quadraticPolynomialLinearizerCoefficient
1039 dataset.coefficientLinearizeSquared[ampName] = quadPolyLinCoeff
1041 return dataset
1043 def plot(self, dataRef, dataset, ptcFitType):
1044 dirname = dataRef.getUri(datasetType='cpPipePlotRoot', write=True)
1045 if not os.path.exists(dirname):
1046 os.makedirs(dirname)
1048 detNum = dataRef.dataId[self.config.ccdKey]
1049 filename = f"PTC_det{detNum}.pdf"
1050 filenameFull = os.path.join(dirname, filename)
1051 with PdfPages(filenameFull) as pdfPages:
1052 self._plotPtc(dataset, ptcFitType, pdfPages)
1054 def _plotPtc(self, dataset, ptcFitType, pdfPages):
1055 """Plot PTC, linearity, and linearity residual per amplifier"""
1057 reducedChiSqPtc = dataset.ptcFitReducedChiSquared
1058 if ptcFitType == 'ASTIERAPPROXIMATION':
1059 ptcFunc = self.funcAstier
1060 stringTitle = (r"Var = $\frac{1}{2g^2a_{00}}(\exp (2a_{00} \mu g) - 1) + \frac{n_{00}}{g^2}$ "
1061 r" ($chi^2$/dof = %g)" % (reducedChiSqPtc))
1062 if ptcFitType == 'POLYNOMIAL':
1063 ptcFunc = self.funcPolynomial
1064 stringTitle = r"Polynomial (degree: %g)" % (self.config.polynomialFitDegree)
1066 legendFontSize = 7
1067 labelFontSize = 7
1068 titleFontSize = 9
1069 supTitleFontSize = 18
1070 markerSize = 25
1072 # General determination of the size of the plot grid
1073 nAmps = len(dataset.ampNames)
1074 if nAmps == 2:
1075 nRows, nCols = 2, 1
1076 nRows = np.sqrt(nAmps)
1077 mantissa, _ = np.modf(nRows)
1078 if mantissa > 0:
1079 nRows = int(nRows) + 1
1080 nCols = nRows
1081 else:
1082 nRows = int(nRows)
1083 nCols = nRows
1085 f, ax = plt.subplots(nrows=nRows, ncols=nCols, sharex='col', sharey='row', figsize=(13, 10))
1086 f2, ax2 = plt.subplots(nrows=nRows, ncols=nCols, sharex='col', sharey='row', figsize=(13, 10))
1088 for i, (amp, a, a2) in enumerate(zip(dataset.ampNames, ax.flatten(), ax2.flatten())):
1089 meanVecOriginal = np.array(dataset.rawMeans[amp])
1090 varVecOriginal = np.array(dataset.rawVars[amp])
1091 mask = dataset.visitMask[amp]
1092 meanVecFinal = meanVecOriginal[mask]
1093 varVecFinal = varVecOriginal[mask]
1094 meanVecOutliers = meanVecOriginal[np.invert(mask)]
1095 varVecOutliers = varVecOriginal[np.invert(mask)]
1096 pars, parsErr = dataset.ptcFitPars[amp], dataset.ptcFitParsError[amp]
1098 if ptcFitType == 'ASTIERAPPROXIMATION':
1099 if len(meanVecFinal):
1100 ptcA00, ptcA00error = pars[0], parsErr[0]
1101 ptcGain, ptcGainError = pars[1], parsErr[1]
1102 ptcNoise = np.sqrt(np.fabs(pars[2]))
1103 ptcNoiseError = 0.5*(parsErr[2]/np.fabs(pars[2]))*np.sqrt(np.fabs(pars[2]))
1104 stringLegend = (f"a00: {ptcA00:.2e}+/-{ptcA00error:.2e} 1/e"
1105 f"\n Gain: {ptcGain:.4}+/-{ptcGainError:.2e} e/DN"
1106 f"\n Noise: {ptcNoise:.4}+/-{ptcNoiseError:.2e} e \n")
1108 if ptcFitType == 'POLYNOMIAL':
1109 if len(meanVecFinal):
1110 ptcGain, ptcGainError = 1./pars[1], np.fabs(1./pars[1])*(parsErr[1]/pars[1])
1111 ptcNoise = np.sqrt(np.fabs(pars[0]))*ptcGain
1112 ptcNoiseError = (0.5*(parsErr[0]/np.fabs(pars[0]))*(np.sqrt(np.fabs(pars[0]))))*ptcGain
1113 stringLegend = (f"Noise: {ptcNoise:.4}+/-{ptcNoiseError:.2e} e \n"
1114 f"Gain: {ptcGain:.4}+/-{ptcGainError:.2e} e/DN \n")
1116 a.set_xlabel(r'Mean signal ($\mu$, DN)', fontsize=labelFontSize)
1117 a.set_ylabel(r'Variance (DN$^2$)', fontsize=labelFontSize)
1118 a.tick_params(labelsize=11)
1119 a.set_xscale('linear', fontsize=labelFontSize)
1120 a.set_yscale('linear', fontsize=labelFontSize)
1122 a2.set_xlabel(r'Mean Signal ($\mu$, DN)', fontsize=labelFontSize)
1123 a2.set_ylabel(r'Variance (DN$^2$)', fontsize=labelFontSize)
1124 a2.tick_params(labelsize=11)
1125 a2.set_xscale('log')
1126 a2.set_yscale('log')
1128 if len(meanVecFinal): # Empty if the whole amp is bad, for example.
1129 minMeanVecFinal = np.min(meanVecFinal)
1130 maxMeanVecFinal = np.max(meanVecFinal)
1131 meanVecFit = np.linspace(minMeanVecFinal, maxMeanVecFinal, 100*len(meanVecFinal))
1132 minMeanVecOriginal = np.min(meanVecOriginal)
1133 maxMeanVecOriginal = np.max(meanVecOriginal)
1134 deltaXlim = maxMeanVecOriginal - minMeanVecOriginal
1136 a.plot(meanVecFit, ptcFunc(pars, meanVecFit), color='red')
1137 a.plot(meanVecFinal, pars[0] + pars[1]*meanVecFinal, color='green', linestyle='--')
1138 a.scatter(meanVecFinal, varVecFinal, c='blue', marker='o', s=markerSize)
1139 a.scatter(meanVecOutliers, varVecOutliers, c='magenta', marker='s', s=markerSize)
1140 a.text(0.03, 0.8, stringLegend, transform=a.transAxes, fontsize=legendFontSize)
1141 a.set_title(amp, fontsize=titleFontSize)
1142 a.set_xlim([minMeanVecOriginal - 0.2*deltaXlim, maxMeanVecOriginal + 0.2*deltaXlim])
1144 # Same, but in log-scale
1145 a2.plot(meanVecFit, ptcFunc(pars, meanVecFit), color='red')
1146 a2.scatter(meanVecFinal, varVecFinal, c='blue', marker='o', s=markerSize)
1147 a2.scatter(meanVecOutliers, varVecOutliers, c='magenta', marker='s', s=markerSize)
1148 a2.text(0.03, 0.8, stringLegend, transform=a2.transAxes, fontsize=legendFontSize)
1149 a2.set_title(amp, fontsize=titleFontSize)
1150 a2.set_xlim([minMeanVecOriginal, maxMeanVecOriginal])
1151 else:
1152 a.set_title(f"{amp} (BAD)", fontsize=titleFontSize)
1153 a2.set_title(f"{amp} (BAD)", fontsize=titleFontSize)
1155 f.suptitle(f"PTC \n Fit: " + stringTitle, fontsize=20)
1156 pdfPages.savefig(f)
1157 f2.suptitle(f"PTC (log-log)", fontsize=20)
1158 pdfPages.savefig(f2)
1160 # Plot mean vs time
1161 f, ax = plt.subplots(nrows=4, ncols=4, sharex='col', sharey='row', figsize=(13, 10))
1162 for i, (amp, a) in enumerate(zip(dataset.ampNames, ax.flatten())):
1163 meanVecFinal = np.array(dataset.rawMeans[amp])[dataset.visitMask[amp]]
1164 timeVecFinal = np.array(dataset.rawExpTimes[amp])[dataset.visitMask[amp]]
1166 a.set_xlabel('Time (sec)', fontsize=labelFontSize)
1167 a.set_ylabel(r'Mean signal ($\mu$, DN)', fontsize=labelFontSize)
1168 a.tick_params(labelsize=labelFontSize)
1169 a.set_xscale('linear', fontsize=labelFontSize)
1170 a.set_yscale('linear', fontsize=labelFontSize)
1172 if len(meanVecFinal):
1173 pars, parsErr = dataset.nonLinearity[amp], dataset.nonLinearityError[amp]
1174 k0, k0Error = pars[0], parsErr[0]
1175 k1, k1Error = pars[1], parsErr[1]
1176 k2, k2Error = pars[2], parsErr[2]
1177 stringLegend = (f"k0: {k0:.4}+/-{k0Error:.2e} DN\n k1: {k1:.4}+/-{k1Error:.2e} DN/t"
1178 f"\n k2: {k2:.2e}+/-{k2Error:.2e} DN/t^2 \n")
1179 a.scatter(timeVecFinal, meanVecFinal)
1180 a.plot(timeVecFinal, self.funcPolynomial(pars, timeVecFinal), color='red')
1181 a.text(0.03, 0.75, stringLegend, transform=a.transAxes, fontsize=legendFontSize)
1182 a.set_title(f"{amp}", fontsize=titleFontSize)
1183 else:
1184 a.set_title(f"{amp} (BAD)", fontsize=titleFontSize)
1186 f.suptitle("Linearity \n Fit: Polynomial (degree: %g)"
1187 % (self.config.polynomialFitDegreeNonLinearity),
1188 fontsize=supTitleFontSize)
1189 pdfPages.savefig(f)
1191 # Plot linearity residual
1192 f, ax = plt.subplots(nrows=4, ncols=4, sharex='col', sharey='row', figsize=(13, 10))
1193 for i, (amp, a) in enumerate(zip(dataset.ampNames, ax.flatten())):
1194 meanVecFinal = np.array(dataset.rawMeans[amp])[dataset.visitMask[amp]]
1195 linRes = np.array(dataset.nonLinearityResiduals[amp])
1197 a.set_xlabel(r'Mean signal ($\mu$, DN)', fontsize=labelFontSize)
1198 a.set_ylabel('LR (%)', fontsize=labelFontSize)
1199 a.set_xscale('linear', fontsize=labelFontSize)
1200 a.set_yscale('linear', fontsize=labelFontSize)
1201 if len(meanVecFinal):
1202 a.axvline(x=timeVecFinal[self.config.linResidualTimeIndex], color='g', linestyle='--')
1203 a.scatter(meanVecFinal, linRes)
1204 a.set_title(f"{amp}", fontsize=titleFontSize)
1205 a.axhline(y=0, color='k')
1206 a.tick_params(labelsize=labelFontSize)
1207 else:
1208 a.set_title(f"{amp} (BAD)", fontsize=titleFontSize)
1210 f.suptitle(r"Linearity Residual: $100\times(1 - \mu_{\rm{ref}}/t_{\rm{ref}})/(\mu / t))$" + "\n" +
1211 r"$t_{\rm{ref}}$: " + f"{timeVecFinal[2]} s", fontsize=supTitleFontSize)
1212 pdfPages.savefig(f)
1214 # Plot fractional non-linearity residual (w.r.t linear part of polynomial fit)
1215 f, ax = plt.subplots(nrows=4, ncols=4, sharex='col', sharey='row', figsize=(13, 10))
1216 for i, (amp, a) in enumerate(zip(dataset.ampNames, ax.flatten())):
1217 meanVecFinal = np.array(dataset.rawMeans[amp])[dataset.visitMask[amp]]
1218 fracLinRes = np.array(dataset.fractionalNonLinearityResiduals[amp])
1220 a.axhline(y=0, color='k')
1221 a.axvline(x=0, color='k', linestyle='-')
1222 a.set_xlabel(r'Mean signal ($\mu$, DN)', fontsize=labelFontSize)
1223 a.set_ylabel('Fractional nonlinearity (%)', fontsize=labelFontSize)
1224 a.tick_params(labelsize=labelFontSize)
1225 a.set_xscale('linear', fontsize=labelFontSize)
1226 a.set_yscale('linear', fontsize=labelFontSize)
1228 if len(meanVecFinal):
1229 a.scatter(meanVecFinal, fracLinRes, c='g')
1230 a.set_title(amp, fontsize=titleFontSize)
1231 else:
1232 a.set_title(f"{amp} (BAD)", fontsize=titleFontSize)
1234 f.suptitle(r"Fractional NL residual" + "\n" +
1235 r"$100\times \frac{(k_0 + k_1*Time-\mu)}{k_0+k_1*Time}$",
1236 fontsize=supTitleFontSize)
1237 pdfPages.savefig(f)
1239 return