Coverage for python/lsst/cp/pipe/ptc.py : 11%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of cp_pipe.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21#
23__all__ = ['MeasurePhotonTransferCurveTask',
24 'MeasurePhotonTransferCurveTaskConfig',
25 'PhotonTransferCurveDataset']
27import numpy as np
28import matplotlib.pyplot as plt
29import os
30from matplotlib.backends.backend_pdf import PdfPages
31from sqlite3 import OperationalError
32from collections import Counter
33from dataclasses import dataclass
35import lsst.afw.math as afwMath
36import lsst.pex.config as pexConfig
37import lsst.pipe.base as pipeBase
38from lsst.ip.isr import IsrTask
39from .utils import (NonexistentDatasetTaskDataIdContainer, PairedVisitListTaskRunner,
40 checkExpLengthEqual, validateIsrConfig)
41from scipy.optimize import leastsq, least_squares
42import numpy.polynomial.polynomial as poly
44from lsst.ip.isr.linearize import Linearizer
45import datetime
48class MeasurePhotonTransferCurveTaskConfig(pexConfig.Config):
49 """Config class for photon transfer curve measurement task"""
50 isr = pexConfig.ConfigurableField(
51 target=IsrTask,
52 doc="""Task to perform instrumental signature removal.""",
53 )
54 isrMandatorySteps = pexConfig.ListField(
55 dtype=str,
56 doc="isr operations that must be performed for valid results. Raises if any of these are False.",
57 default=['doAssembleCcd']
58 )
59 isrForbiddenSteps = pexConfig.ListField(
60 dtype=str,
61 doc="isr operations that must NOT be performed for valid results. Raises if any of these are True",
62 default=['doFlat', 'doFringe', 'doBrighterFatter', 'doUseOpticsTransmission',
63 'doUseFilterTransmission', 'doUseSensorTransmission', 'doUseAtmosphereTransmission']
64 )
65 isrDesirableSteps = pexConfig.ListField(
66 dtype=str,
67 doc="isr operations that it is advisable to perform, but are not mission-critical." +
68 " WARNs are logged for any of these found to be False.",
69 default=['doBias', 'doDark', 'doCrosstalk', 'doDefect']
70 )
71 isrUndesirableSteps = pexConfig.ListField(
72 dtype=str,
73 doc="isr operations that it is *not* advisable to perform in the general case, but are not" +
74 " forbidden as some use-cases might warrant them." +
75 " WARNs are logged for any of these found to be True.",
76 default=['doLinearize']
77 )
78 ccdKey = pexConfig.Field(
79 dtype=str,
80 doc="The key by which to pull a detector from a dataId, e.g. 'ccd' or 'detector'.",
81 default='ccd',
82 )
83 makePlots = pexConfig.Field(
84 dtype=bool,
85 doc="Plot the PTC curves?",
86 default=False,
87 )
88 ptcFitType = pexConfig.ChoiceField(
89 dtype=str,
90 doc="Fit PTC to approximation in Astier+19 (Equation 16) or to a polynomial.",
91 default="POLYNOMIAL",
92 allowed={
93 "POLYNOMIAL": "n-degree polynomial (use 'polynomialFitDegree' to set 'n').",
94 "ASTIERAPPROXIMATION": "Approximation in Astier+19 (Eq. 16)."
95 }
96 )
97 polynomialFitDegree = pexConfig.Field(
98 dtype=int,
99 doc="Degree of polynomial to fit the PTC, when 'ptcFitType'=POLYNOMIAL.",
100 default=2,
101 )
102 polynomialFitDegreeNonLinearity = pexConfig.Field(
103 dtype=int,
104 doc="Degree of polynomial to fit the meanSignal vs exposureTime curve to produce" +
105 " the table for LinearizeLookupTable.",
106 default=3,
107 )
108 binSize = pexConfig.Field(
109 dtype=int,
110 doc="Bin the image by this factor in both dimensions.",
111 default=1,
112 )
113 minMeanSignal = pexConfig.Field(
114 dtype=float,
115 doc="Minimum value (inclusive) of mean signal (in DN) above which to consider.",
116 default=0,
117 )
118 maxMeanSignal = pexConfig.Field(
119 dtype=float,
120 doc="Maximum value (inclusive) of mean signal (in DN) below which to consider.",
121 default=9e6,
122 )
123 initialNonLinearityExclusionThresholdPositive = pexConfig.RangeField(
124 dtype=float,
125 doc="Initially exclude data points with a variance that are more than a factor of this from being"
126 " linear in the positive direction, from the PTC fit. Note that these points will also be"
127 " excluded from the non-linearity fit. This is done before the iterative outlier rejection,"
128 " to allow an accurate determination of the sigmas for said iterative fit.",
129 default=0.12,
130 min=0.0,
131 max=1.0,
132 )
133 initialNonLinearityExclusionThresholdNegative = pexConfig.RangeField(
134 dtype=float,
135 doc="Initially exclude data points with a variance that are more than a factor of this from being"
136 " linear in the negative direction, from the PTC fit. Note that these points will also be"
137 " excluded from the non-linearity fit. This is done before the iterative outlier rejection,"
138 " to allow an accurate determination of the sigmas for said iterative fit.",
139 default=0.25,
140 min=0.0,
141 max=1.0,
142 )
143 sigmaCutPtcOutliers = pexConfig.Field(
144 dtype=float,
145 doc="Sigma cut for outlier rejection in PTC.",
146 default=5.0,
147 )
148 nSigmaClipPtc = pexConfig.Field(
149 dtype=float,
150 doc="Sigma cut for afwMath.StatisticsControl()",
151 default=5.5,
152 )
153 nIterSigmaClipPtc = pexConfig.Field(
154 dtype=int,
155 doc="Number of sigma-clipping iterations for afwMath.StatisticsControl()",
156 default=1,
157 )
158 maxIterationsPtcOutliers = pexConfig.Field(
159 dtype=int,
160 doc="Maximum number of iterations for outlier rejection in PTC.",
161 default=2,
162 )
163 doFitBootstrap = pexConfig.Field(
164 dtype=bool,
165 doc="Use bootstrap for the PTC fit parameters and errors?.",
166 default=False,
167 )
168 linResidualTimeIndex = pexConfig.Field(
169 dtype=int,
170 doc="Index position in time array for reference time in linearity residual calculation.",
171 default=2,
172 )
173 maxAduForLookupTableLinearizer = pexConfig.Field(
174 dtype=int,
175 doc="Maximum DN value for the LookupTable linearizer.",
176 default=2**18,
177 )
178 instrumentName = pexConfig.Field(
179 dtype=str,
180 doc="Instrument name.",
181 default='',
182 )
185@dataclass
186class LinearityResidualsAndLinearizersDataset:
187 """A simple class to hold the output from the
188 `calculateLinearityResidualAndLinearizers` function.
189 """
190 # Normalized coefficients for polynomial NL correction
191 polynomialLinearizerCoefficients: list
192 # Normalized coefficient for quadratic polynomial NL correction (c0)
193 quadraticPolynomialLinearizerCoefficient: float
194 # LUT array row for the amplifier at hand
195 linearizerTableRow: list
196 # Linearity residual, Eq. 2.2. of Janesick (2001)
197 linearityResidual: list
198 meanSignalVsTimePolyFitPars: list
199 meanSignalVsTimePolyFitParsErr: list
200 fractionalNonLinearityResidual: list
201 meanSignalVsTimePolyFitReducedChiSq: float
204class PhotonTransferCurveDataset:
205 """A simple class to hold the output data from the PTC task.
207 The dataset is made up of a dictionary for each item, keyed by the
208 amplifiers' names, which much be supplied at construction time.
210 New items cannot be added to the class to save accidentally saving to the
211 wrong property, and the class can be frozen if desired.
213 inputVisitPairs records the visits used to produce the data.
214 When fitPtcAndNonLinearity() is run, a mask is built up, which is by definition
215 always the same length as inputVisitPairs, rawExpTimes, rawMeans
216 and rawVars, and is a list of bools, which are incrementally set to False
217 as points are discarded from the fits.
219 PTC fit parameters for polynomials are stored in a list in ascending order
220 of polynomial term, i.e. par[0]*x^0 + par[1]*x + par[2]*x^2 etc
221 with the length of the list corresponding to the order of the polynomial
222 plus one.
223 """
224 def __init__(self, ampNames):
225 # add items to __dict__ directly because __setattr__ is overridden
227 # instance variables
228 self.__dict__["ampNames"] = ampNames
229 self.__dict__["badAmps"] = []
231 # raw data variables
232 self.__dict__["inputVisitPairs"] = {ampName: [] for ampName in ampNames}
233 self.__dict__["visitMask"] = {ampName: [] for ampName in ampNames}
234 self.__dict__["rawExpTimes"] = {ampName: [] for ampName in ampNames}
235 self.__dict__["rawMeans"] = {ampName: [] for ampName in ampNames}
236 self.__dict__["rawVars"] = {ampName: [] for ampName in ampNames}
238 # fit information
239 self.__dict__["ptcFitType"] = {ampName: "" for ampName in ampNames}
240 self.__dict__["ptcFitPars"] = {ampName: [] for ampName in ampNames}
241 self.__dict__["ptcFitParsError"] = {ampName: [] for ampName in ampNames}
242 self.__dict__["ptcFitReducedChiSquared"] = {ampName: [] for ampName in ampNames}
243 self.__dict__["nonLinearity"] = {ampName: [] for ampName in ampNames}
244 self.__dict__["nonLinearityError"] = {ampName: [] for ampName in ampNames}
245 self.__dict__["nonLinearityResiduals"] = {ampName: [] for ampName in ampNames}
246 self.__dict__["fractionalNonLinearityResiduals"] = {ampName: [] for ampName in ampNames}
247 self.__dict__["nonLinearityReducedChiSquared"] = {ampName: [] for ampName in ampNames}
249 # final results
250 self.__dict__["gain"] = {ampName: -1. for ampName in ampNames}
251 self.__dict__["gainErr"] = {ampName: -1. for ampName in ampNames}
252 self.__dict__["noise"] = {ampName: -1. for ampName in ampNames}
253 self.__dict__["noiseErr"] = {ampName: -1. for ampName in ampNames}
254 self.__dict__["coefficientsLinearizePolynomial"] = {ampName: [] for ampName in ampNames}
255 self.__dict__["coefficientLinearizeSquared"] = {ampName: [] for ampName in ampNames}
257 def __setattr__(self, attribute, value):
258 """Protect class attributes"""
259 if attribute not in self.__dict__:
260 raise AttributeError(f"{attribute} is not already a member of PhotonTransferCurveDataset, which"
261 " does not support setting of new attributes.")
262 else:
263 self.__dict__[attribute] = value
265 def getVisitsUsed(self, ampName):
266 """Get the visits used, i.e. not discarded, for a given amp.
268 If no mask has been created yet, all visits are returned.
269 """
270 if self.visitMask[ampName] == []:
271 return self.inputVisitPairs[ampName]
273 # if the mask exists it had better be the same length as the visitPairs
274 assert len(self.visitMask[ampName]) == len(self.inputVisitPairs[ampName])
276 pairs = self.inputVisitPairs[ampName]
277 mask = self.visitMask[ampName]
278 # cast to bool required because numpy
279 return [(v1, v2) for ((v1, v2), m) in zip(pairs, mask) if bool(m) is True]
281 def getGoodAmps(self):
282 return [amp for amp in self.ampNames if amp not in self.badAmps]
285class MeasurePhotonTransferCurveTask(pipeBase.CmdLineTask):
286 """A class to calculate, fit, and plot a PTC from a set of flat pairs.
288 The Photon Transfer Curve (var(signal) vs mean(signal)) is a standard tool
289 used in astronomical detectors characterization (e.g., Janesick 2001,
290 Janesick 2007). This task calculates the PTC from a series of pairs of
291 flat-field images; each pair taken at identical exposure times. The
292 difference image of each pair is formed to eliminate fixed pattern noise,
293 and then the variance of the difference image and the mean of the average image
294 are used to produce the PTC. An n-degree polynomial or the approximation in Equation
295 16 of Astier+19 ("The Shape of the Photon Transfer Curve of CCD sensors",
296 arXiv:1905.08677) can be fitted to the PTC curve. These models include
297 parameters such as the gain (e/DN) and readout noise.
299 Linearizers to correct for signal-chain non-linearity are also calculated.
300 The `Linearizer` class, in general, can support per-amp linearizers, but in this
301 task this is not supported.
302 Parameters
303 ----------
305 *args: `list`
306 Positional arguments passed to the Task constructor. None used at this
307 time.
308 **kwargs: `dict`
309 Keyword arguments passed on to the Task constructor. None used at this
310 time.
312 """
314 RunnerClass = PairedVisitListTaskRunner
315 ConfigClass = MeasurePhotonTransferCurveTaskConfig
316 _DefaultName = "measurePhotonTransferCurve"
318 def __init__(self, *args, **kwargs):
319 pipeBase.CmdLineTask.__init__(self, *args, **kwargs)
320 self.makeSubtask("isr")
321 plt.interactive(False) # stop windows popping up when plotting. When headless, use 'agg' backend too
322 validateIsrConfig(self.isr, self.config.isrMandatorySteps,
323 self.config.isrForbiddenSteps, self.config.isrDesirableSteps, checkTrim=False)
324 self.config.validate()
325 self.config.freeze()
327 @classmethod
328 def _makeArgumentParser(cls):
329 """Augment argument parser for the MeasurePhotonTransferCurveTask."""
330 parser = pipeBase.ArgumentParser(name=cls._DefaultName)
331 parser.add_argument("--visit-pairs", dest="visitPairs", nargs="*",
332 help="Visit pairs to use. Each pair must be of the form INT,INT e.g. 123,456")
333 parser.add_id_argument("--id", datasetType="photonTransferCurveDataset",
334 ContainerClass=NonexistentDatasetTaskDataIdContainer,
335 help="The ccds to use, e.g. --id ccd=0..100")
336 return parser
338 @pipeBase.timeMethod
339 def runDataRef(self, dataRef, visitPairs):
340 """Run the Photon Transfer Curve (PTC) measurement task.
342 For a dataRef (which is each detector here),
343 and given a list of visit pairs at different exposure times,
344 measure the PTC.
346 Parameters
347 ----------
348 dataRef : list of lsst.daf.persistence.ButlerDataRef
349 dataRef for the detector for the visits to be fit.
350 visitPairs : `iterable` of `tuple` of `int`
351 Pairs of visit numbers to be processed together
352 """
354 # setup necessary objects
355 detNum = dataRef.dataId[self.config.ccdKey]
356 detector = dataRef.get('camera')[dataRef.dataId[self.config.ccdKey]]
357 # expand some missing fields that we need for lsstCam. This is a work-around
358 # for Gen2 problems that I (RHL) don't feel like solving. The calibs pipelines
359 # (which inherit from CalibTask) use addMissingKeys() to do basically the same thing
360 #
361 # Basically, the butler's trying to look up the fields in `raw_visit` which won't work
362 for name in dataRef.getButler().getKeys('bias'):
363 if name not in dataRef.dataId:
364 try:
365 dataRef.dataId[name] = \
366 dataRef.getButler().queryMetadata('raw', [name], detector=detNum)[0]
367 except OperationalError:
368 pass
370 amps = detector.getAmplifiers()
371 ampNames = [amp.getName() for amp in amps]
372 dataset = PhotonTransferCurveDataset(ampNames)
374 self.log.info('Measuring PTC using %s visits for detector %s' % (visitPairs, detNum))
376 for (v1, v2) in visitPairs:
377 # Perform ISR on each exposure
378 dataRef.dataId['expId'] = v1
379 exp1 = self.isr.runDataRef(dataRef).exposure
380 dataRef.dataId['expId'] = v2
381 exp2 = self.isr.runDataRef(dataRef).exposure
382 del dataRef.dataId['expId']
384 checkExpLengthEqual(exp1, exp2, v1, v2, raiseWithMessage=True)
385 expTime = exp1.getInfo().getVisitInfo().getExposureTime()
387 for amp in detector:
388 mu, varDiff = self.measureMeanVarPair(exp1, exp2, region=amp.getBBox())
389 ampName = amp.getName()
391 dataset.rawExpTimes[ampName].append(expTime)
392 dataset.rawMeans[ampName].append(mu)
393 dataset.rawVars[ampName].append(varDiff)
394 dataset.inputVisitPairs[ampName].append((v1, v2))
396 numberAmps = len(detector.getAmplifiers())
397 numberAduValues = self.config.maxAduForLookupTableLinearizer
398 lookupTableArray = np.zeros((numberAmps, numberAduValues), dtype=np.float32)
400 # Fit PTC and (non)linearity of signal vs time curve.
401 # Fill up PhotonTransferCurveDataset object.
402 # Fill up array for LUT linearizer.
403 # Produce coefficients for Polynomial ans Squared linearizers.
404 dataset = self.fitPtcAndNonLinearity(dataset, self.config.ptcFitType,
405 tableArray=lookupTableArray)
407 if self.config.makePlots:
408 self.plot(dataRef, dataset, ptcFitType=self.config.ptcFitType)
410 # Save data, PTC fit, and NL fit dictionaries
411 self.log.info(f"Writing PTC and NL data to {dataRef.getUri(write=True)}")
412 dataRef.put(dataset, datasetType="photonTransferCurveDataset")
414 butler = dataRef.getButler()
415 self.log.info("Writing linearizers: \n "
416 "lookup table (linear component of polynomial fit), \n "
417 "polynomial (coefficients for a polynomial correction), \n "
418 "and squared linearizer (quadratic coefficient from polynomial)")
420 detName = detector.getName()
421 now = datetime.datetime.utcnow()
422 calibDate = now.strftime("%Y-%m-%d")
424 for linType, dataType in [("LOOKUPTABLE", 'linearizeLut'),
425 ("LINEARIZEPOLYNOMIAL", 'linearizePolynomial'),
426 ("LINEARIZESQUARED", 'linearizeSquared')]:
428 if linType == "LOOKUPTABLE":
429 tableArray = lookupTableArray
430 else:
431 tableArray = None
433 linearizer = self.buildLinearizerObject(dataset, detector, calibDate, linType,
434 instruName=self.config.instrumentName,
435 tableArray=tableArray,
436 log=self.log)
437 butler.put(linearizer, datasetType=dataType, dataId={'detector': detNum,
438 'detectorName': detName, 'calibDate': calibDate})
440 self.log.info('Finished measuring PTC for in detector %s' % detNum)
442 return pipeBase.Struct(exitStatus=0)
444 def buildLinearizerObject(self, dataset, detector, calibDate, linearizerType, instruName='',
445 tableArray=None, log=None):
446 """Build linearizer object to persist.
448 Parameters
449 ----------
450 dataset : `lsst.cp.pipe.ptc.PhotonTransferCurveDataset`
451 The dataset containing the means, variances, and exposure times
452 detector : `lsst.afw.cameraGeom.Detector`
453 Detector object
454 calibDate : `datetime.datetime`
455 Calibration date
456 linearizerType : `str`
457 'LOOKUPTABLE', 'LINEARIZESQUARED', or 'LINEARIZEPOLYNOMIAL'
458 instruName : `str`, optional
459 Instrument name
460 tableArray : `np.array`, optional
461 Look-up table array with size rows=nAmps and columns=DN values
462 log : `lsst.log.Log`, optional
463 Logger to handle messages
465 Returns
466 -------
467 linearizer : `lsst.ip.isr.Linearizer`
468 Linearizer object
469 """
470 detName = detector.getName()
471 detNum = detector.getId()
472 if linearizerType == "LOOKUPTABLE":
473 if tableArray is not None:
474 linearizer = Linearizer(detector=detector, table=tableArray, log=log)
475 else:
476 raise RuntimeError("tableArray must be provided when creating a LookupTable linearizer")
477 elif linearizerType in ("LINEARIZESQUARED", "LINEARIZEPOLYNOMIAL"):
478 linearizer = Linearizer(log=log)
479 else:
480 raise RuntimeError("Invalid linearizerType {linearizerType} to build a Linearizer object. "
481 "Supported: 'LOOKUPTABLE', 'LINEARIZESQUARED', or 'LINEARIZEPOLYNOMIAL'")
482 for i, amp in enumerate(detector.getAmplifiers()):
483 ampName = amp.getName()
484 if linearizerType == "LOOKUPTABLE":
485 linearizer.linearityCoeffs[ampName] = [i, 0]
486 linearizer.linearityType[ampName] = "LookupTable"
487 elif linearizerType == "LINEARIZESQUARED":
488 linearizer.linearityCoeffs[ampName] = [dataset.coefficientLinearizeSquared[ampName]]
489 linearizer.linearityType[ampName] = "Squared"
490 elif linearizerType == "LINEARIZEPOLYNOMIAL":
491 linearizer.linearityCoeffs[ampName] = dataset.coefficientsLinearizePolynomial[ampName]
492 linearizer.linearityType[ampName] = "Polynomial"
493 linearizer.linearityBBox[ampName] = amp.getBBox()
495 linearizer.validate()
496 calibId = f"detectorName={detName} detector={detNum} calibDate={calibDate} ccd={detNum} filter=NONE"
498 try:
499 raftName = detName.split("_")[0]
500 calibId += f" raftName={raftName}"
501 except Exception:
502 raftname = "NONE"
503 calibId += f" raftName={raftname}"
505 serial = detector.getSerial()
506 linearizer.updateMetadata(instrumentName=instruName, detectorId=f"{detNum}",
507 calibId=calibId, serial=serial, detectorName=f"{detName}")
509 return linearizer
511 def measureMeanVarPair(self, exposure1, exposure2, region=None):
512 """Calculate the mean signal of two exposures and the variance of their difference.
514 Parameters
515 ----------
516 exposure1 : `lsst.afw.image.exposure.exposure.ExposureF`
517 First exposure of flat field pair.
519 exposure2 : `lsst.afw.image.exposure.exposure.ExposureF`
520 Second exposure of flat field pair.
522 region : `lsst.geom.Box2I`
523 Region of each exposure where to perform the calculations (e.g, an amplifier).
525 Return
526 ------
528 mu : `float`
529 0.5*(mu1 + mu2), where mu1, and mu2 are the clipped means of the regions in
530 both exposures.
532 varDiff : `float`
533 Half of the clipped variance of the difference of the regions inthe two input
534 exposures.
535 """
537 if region is not None:
538 im1Area = exposure1.maskedImage[region]
539 im2Area = exposure2.maskedImage[region]
540 else:
541 im1Area = exposure1.maskedImage
542 im2Area = exposure2.maskedImage
544 im1Area = afwMath.binImage(im1Area, self.config.binSize)
545 im2Area = afwMath.binImage(im2Area, self.config.binSize)
547 statsCtrl = afwMath.StatisticsControl()
548 statsCtrl.setNumSigmaClip(self.config.nSigmaClipPtc)
549 statsCtrl.setNumIter(self.config.nIterSigmaClipPtc)
550 # Clipped mean of images; then average of mean.
551 mu1 = afwMath.makeStatistics(im1Area, afwMath.MEANCLIP, statsCtrl).getValue()
552 mu2 = afwMath.makeStatistics(im2Area, afwMath.MEANCLIP, statsCtrl).getValue()
553 mu = 0.5*(mu1 + mu2)
555 # Take difference of pairs
556 # symmetric formula: diff = (mu2*im1-mu1*im2)/(0.5*(mu1+mu2))
557 temp = im2Area.clone()
558 temp *= mu1
559 diffIm = im1Area.clone()
560 diffIm *= mu2
561 diffIm -= temp
562 diffIm /= mu
564 varDiff = 0.5*(afwMath.makeStatistics(diffIm, afwMath.VARIANCECLIP, statsCtrl).getValue())
566 return mu, varDiff
568 def _fitLeastSq(self, initialParams, dataX, dataY, function):
569 """Do a fit and estimate the parameter errors using using scipy.optimize.leastq.
571 optimize.leastsq returns the fractional covariance matrix. To estimate the
572 standard deviation of the fit parameters, multiply the entries of this matrix
573 by the unweighted reduced chi squared and take the square root of the diagonal elements.
575 Parameters
576 ----------
577 initialParams : `list` of `float`
578 initial values for fit parameters. For ptcFitType=POLYNOMIAL, its length
579 determines the degree of the polynomial.
581 dataX : `numpy.array` of `float`
582 Data in the abscissa axis.
584 dataY : `numpy.array` of `float`
585 Data in the ordinate axis.
587 function : callable object (function)
588 Function to fit the data with.
590 Return
591 ------
592 pFitSingleLeastSquares : `list` of `float`
593 List with fitted parameters.
595 pErrSingleLeastSquares : `list` of `float`
596 List with errors for fitted parameters.
598 reducedChiSqSingleLeastSquares : `float`
599 Unweighted reduced chi squared
600 """
602 def errFunc(p, x, y):
603 return function(p, x) - y
605 pFit, pCov, infoDict, errMessage, success = leastsq(errFunc, initialParams,
606 args=(dataX, dataY), full_output=1, epsfcn=0.0001)
608 if (len(dataY) > len(initialParams)) and pCov is not None:
609 reducedChiSq = (errFunc(pFit, dataX, dataY)**2).sum()/(len(dataY)-len(initialParams))
610 pCov *= reducedChiSq
611 else:
612 pCov = np.zeros((len(initialParams), len(initialParams)))
613 pCov[:, :] = np.inf
614 reducedChiSq = np.inf
616 errorVec = []
617 for i in range(len(pFit)):
618 errorVec.append(np.fabs(pCov[i][i])**0.5)
620 pFitSingleLeastSquares = pFit
621 pErrSingleLeastSquares = np.array(errorVec)
623 return pFitSingleLeastSquares, pErrSingleLeastSquares, reducedChiSq
625 def _fitBootstrap(self, initialParams, dataX, dataY, function, confidenceSigma=1.):
626 """Do a fit using least squares and bootstrap to estimate parameter errors.
628 The bootstrap error bars are calculated by fitting 100 random data sets.
630 Parameters
631 ----------
632 initialParams : `list` of `float`
633 initial values for fit parameters. For ptcFitType=POLYNOMIAL, its length
634 determines the degree of the polynomial.
636 dataX : `numpy.array` of `float`
637 Data in the abscissa axis.
639 dataY : `numpy.array` of `float`
640 Data in the ordinate axis.
642 function : callable object (function)
643 Function to fit the data with.
645 confidenceSigma : `float`
646 Number of sigmas that determine confidence interval for the bootstrap errors.
648 Return
649 ------
650 pFitBootstrap : `list` of `float`
651 List with fitted parameters.
653 pErrBootstrap : `list` of `float`
654 List with errors for fitted parameters.
656 reducedChiSqBootstrap : `float`
657 Reduced chi squared.
658 """
660 def errFunc(p, x, y):
661 return function(p, x) - y
663 # Fit first time
664 pFit, _ = leastsq(errFunc, initialParams, args=(dataX, dataY), full_output=0)
666 # Get the stdev of the residuals
667 residuals = errFunc(pFit, dataX, dataY)
668 sigmaErrTotal = np.std(residuals)
670 # 100 random data sets are generated and fitted
671 pars = []
672 for i in range(100):
673 randomDelta = np.random.normal(0., sigmaErrTotal, len(dataY))
674 randomDataY = dataY + randomDelta
675 randomFit, _ = leastsq(errFunc, initialParams,
676 args=(dataX, randomDataY), full_output=0)
677 pars.append(randomFit)
678 pars = np.array(pars)
679 meanPfit = np.mean(pars, 0)
681 # confidence interval for parameter estimates
682 nSigma = confidenceSigma
683 errPfit = nSigma*np.std(pars, 0)
684 pFitBootstrap = meanPfit
685 pErrBootstrap = errPfit
687 reducedChiSq = (errFunc(pFitBootstrap, dataX, dataY)**2).sum()/(len(dataY)-len(initialParams))
688 return pFitBootstrap, pErrBootstrap, reducedChiSq
690 def funcPolynomial(self, pars, x):
691 """Polynomial function definition"""
692 return poly.polyval(x, [*pars])
694 def funcAstier(self, pars, x):
695 """Single brighter-fatter parameter model for PTC; Equation 16 of Astier+19"""
696 a00, gain, noise = pars
697 return 0.5/(a00*gain*gain)*(np.exp(2*a00*x*gain)-1) + noise/(gain*gain)
699 @staticmethod
700 def _initialParsForPolynomial(order):
701 assert(order >= 2)
702 pars = np.zeros(order, dtype=np.float)
703 pars[0] = 10
704 pars[1] = 1
705 pars[2:] = 0.0001
706 return pars
708 @staticmethod
709 def _boundsForPolynomial(initialPars):
710 lowers = [np.NINF for p in initialPars]
711 uppers = [np.inf for p in initialPars]
712 lowers[1] = 0 # no negative gains
713 return (lowers, uppers)
715 @staticmethod
716 def _boundsForAstier(initialPars):
717 lowers = [np.NINF for p in initialPars]
718 uppers = [np.inf for p in initialPars]
719 return (lowers, uppers)
721 @staticmethod
722 def _getInitialGoodPoints(means, variances, maxDeviationPositive, maxDeviationNegative):
723 """Return a boolean array to mask bad points.
725 A linear function has a constant ratio, so find the median
726 value of the ratios, and exclude the points that deviate
727 from that by more than a factor of maxDeviationPositive/negative.
728 Asymmetric deviations are supported as we expect the PTC to turn
729 down as the flux increases, but sometimes it anomalously turns
730 upwards just before turning over, which ruins the fits, so it
731 is wise to be stricter about restricting positive outliers than
732 negative ones.
734 Too high and points that are so bad that fit will fail will be included
735 Too low and the non-linear points will be excluded, biasing the NL fit."""
736 ratios = [b/a for (a, b) in zip(means, variances)]
737 medianRatio = np.median(ratios)
738 ratioDeviations = [(r/medianRatio)-1 for r in ratios]
740 # so that it doesn't matter if the deviation is expressed as positive or negative
741 maxDeviationPositive = abs(maxDeviationPositive)
742 maxDeviationNegative = -1. * abs(maxDeviationNegative)
744 goodPoints = np.array([True if (r < maxDeviationPositive and r > maxDeviationNegative)
745 else False for r in ratioDeviations])
746 return goodPoints
748 def _makeZeroSafe(self, array, warn=True, substituteValue=1e-9):
749 """"""
750 nBad = Counter(array)[0]
751 if nBad == 0:
752 return array
754 if warn:
755 msg = f"Found {nBad} zeros in array at elements {[x for x in np.where(array==0)[0]]}"
756 self.log.warn(msg)
758 array[array == 0] = substituteValue
759 return array
761 def calculateLinearityResidualAndLinearizers(self, exposureTimeVector, meanSignalVector):
762 """Calculate linearity residual and fit an n-order polynomial to the mean vs time curve
763 to produce corrections (deviation from linear part of polynomial) for a particular amplifier
764 to populate LinearizeLookupTable.
765 Use the coefficients of this fit to calculate the correction coefficients for LinearizePolynomial
766 and LinearizeSquared."
768 Parameters
769 ---------
771 exposureTimeVector: `list` of `float`
772 List of exposure times for each flat pair
774 meanSignalVector: `list` of `float`
775 List of mean signal from diference image of flat pairs
777 Returns
778 -------
779 dataset : `lsst.cp.pipe.ptc.LinearityResidualsAndLinearizersDataset`
780 The dataset containing the fit parameters, the NL correction coefficients, and the
781 LUT row for the amplifier at hand. Explicitly:
783 dataset.polynomialLinearizerCoefficients : `list` of `float`
784 Coefficients for LinearizePolynomial, where corrImage = uncorrImage + sum_i c_i uncorrImage^(2 +
785 i).
786 c_(j-2) = -k_j/(k_1^j) with units (DN^(1-j)). The units of k_j are DN/t^j, and they are fit from
787 meanSignalVector = k0 + k1*exposureTimeVector + k2*exposureTimeVector^2 +...
788 + kn*exposureTimeVector^n, with n = "polynomialFitDegreeNonLinearity".
789 k_0 and k_1 and degenerate with bias level and gain, and are not used by the non-linearity
790 correction. Therefore, j = 2...n in the above expression (see `LinearizePolynomial` class in
791 `linearize.py`.)
793 dataset.quadraticPolynomialLinearizerCoefficient : `float`
794 Coefficient for LinearizeSquared, where corrImage = uncorrImage + c0*uncorrImage^2.
795 c0 = -k2/(k1^2), where k1 and k2 are fit from
796 meanSignalVector = k0 + k1*exposureTimeVector + k2*exposureTimeVector^2 +...
797 + kn*exposureTimeVector^n, with n = "polynomialFitDegreeNonLinearity".
799 dataset.linearizerTableRow : `list` of `float`
800 One dimensional array with deviation from linear part of n-order polynomial fit
801 to mean vs time curve. This array will be one row (for the particular amplifier at hand)
802 of the table array for LinearizeLookupTable.
804 dataset.linearityResidual : `list` of `float`
805 Linearity residual from the mean vs time curve, defined as
806 100*(1 - meanSignalReference/expTimeReference/(meanSignal/expTime).
808 dataset.meanSignalVsTimePolyFitPars : `list` of `float`
809 Parameters from n-order polynomial fit to meanSignalVector vs exposureTimeVector.
811 dataset.meanSignalVsTimePolyFitParsErr : `list` of `float`
812 Parameters from n-order polynomial fit to meanSignalVector vs exposureTimeVector.
814 dataset.fractionalNonLinearityResidual : `list` of `float`
815 Fractional residuals from the meanSignal vs exposureTime curve with respect to linear part of
816 polynomial fit: 100*(linearPart - meanSignal)/linearPart, where
817 linearPart = k0 + k1*exposureTimeVector.
819 dataset.meanSignalVsTimePolyFitReducedChiSq : `float`
820 Reduced unweighted chi squared from polynomial fit to meanSignalVector vs exposureTimeVector.
821 """
823 # Lookup table linearizer
824 parsIniNonLinearity = self._initialParsForPolynomial(self.config.polynomialFitDegreeNonLinearity + 1)
825 if self.config.doFitBootstrap:
826 parsFit, parsFitErr, reducedChiSquaredNonLinearityFit = self._fitBootstrap(parsIniNonLinearity,
827 exposureTimeVector,
828 meanSignalVector,
829 self.funcPolynomial)
830 else:
831 parsFit, parsFitErr, reducedChiSquaredNonLinearityFit = self._fitLeastSq(parsIniNonLinearity,
832 exposureTimeVector,
833 meanSignalVector,
834 self.funcPolynomial)
836 # LinearizeLookupTable:
837 # Use linear part to get time at wich signal is maxAduForLookupTableLinearizer DN
838 tMax = (self.config.maxAduForLookupTableLinearizer - parsFit[0])/parsFit[1]
839 timeRange = np.linspace(0, tMax, self.config.maxAduForLookupTableLinearizer)
840 signalIdeal = parsFit[0] + parsFit[1]*timeRange
841 signalUncorrected = self.funcPolynomial(parsFit, timeRange)
842 linearizerTableRow = signalIdeal - signalUncorrected # LinearizerLookupTable has corrections
843 # LinearizePolynomial and LinearizeSquared:
844 # Check that magnitude of higher order (>= 3) coefficents of the polyFit are small,
845 # i.e., less than threshold = 1e-10 (typical quadratic and cubic coefficents are ~1e-6
846 # and ~1e-12).
847 k1 = parsFit[1]
848 polynomialLinearizerCoefficients = []
849 for i, coefficient in enumerate(parsFit):
850 c = -coefficient/(k1**i)
851 polynomialLinearizerCoefficients.append(c)
852 if np.fabs(c) > 1e-10:
853 msg = f"Coefficient {c} in polynomial fit larger than threshold 1e-10."
854 self.log.warn(msg)
855 # Coefficient for LinearizedSquared. Called "c0" in linearize.py
856 c0 = polynomialLinearizerCoefficients[2]
858 # Linearity residual
859 linResidualTimeIndex = self.config.linResidualTimeIndex
860 if exposureTimeVector[linResidualTimeIndex] == 0.0:
861 raise RuntimeError("Reference time for linearity residual can't be 0.0")
862 linResidual = 100*(1 - ((meanSignalVector[linResidualTimeIndex] /
863 exposureTimeVector[linResidualTimeIndex]) /
864 (meanSignalVector/exposureTimeVector)))
866 # Fractional non-linearity residual, w.r.t linear part of polynomial fit
867 linearPart = parsFit[0] + k1*exposureTimeVector
868 fracNonLinearityResidual = 100*(linearPart - meanSignalVector)/linearPart
870 dataset = LinearityResidualsAndLinearizersDataset([], None, [], [], [], [], [], None)
871 dataset.polynomialLinearizerCoefficients = polynomialLinearizerCoefficients
872 dataset.quadraticPolynomialLinearizerCoefficient = c0
873 dataset.linearizerTableRow = linearizerTableRow
874 dataset.linearityResidual = linResidual
875 dataset.meanSignalVsTimePolyFitPars = parsFit
876 dataset.meanSignalVsTimePolyFitParsErr = parsFitErr
877 dataset.fractionalNonLinearityResidual = fracNonLinearityResidual
878 dataset.meanSignalVsTimePolyFitReducedChiSq = reducedChiSquaredNonLinearityFit
880 return dataset
882 def fitPtcAndNonLinearity(self, dataset, ptcFitType, tableArray=None):
883 """Fit the photon transfer curve and calculate linearity and residuals.
885 Fit the photon transfer curve with either a polynomial of the order
886 specified in the task config, or using the Astier approximation.
888 Sigma clipping is performed iteratively for the fit, as well as an
889 initial clipping of data points that are more than
890 config.initialNonLinearityExclusionThreshold away from lying on a
891 straight line. This other step is necessary because the photon transfer
892 curve turns over catastrophically at very high flux (because saturation
893 drops the variance to ~0) and these far outliers cause the initial fit
894 to fail, meaning the sigma cannot be calculated to perform the
895 sigma-clipping.
897 Parameters
898 ----------
899 dataset : `lsst.cp.pipe.ptc.PhotonTransferCurveDataset`
900 The dataset containing the means, variances and exposure times
901 ptcFitType : `str`
902 Fit a 'POLYNOMIAL' (degree: 'polynomialFitDegree') or
903 'ASTIERAPPROXIMATION' to the PTC
904 tableArray : `np.array`
905 Optional. Look-up table array with size rows=nAmps and columns=DN values.
906 It will be modified in-place if supplied.
908 Returns
909 -------
910 dataset: `lsst.cp.pipe.ptc.PhotonTransferCurveDataset`
911 This is the same dataset as the input paramter, however, it has been modified
912 to include information such as the fit vectors and the fit parameters. See
913 the class `PhotonTransferCurveDatase`.
914 """
916 def errFunc(p, x, y):
917 return ptcFunc(p, x) - y
919 sigmaCutPtcOutliers = self.config.sigmaCutPtcOutliers
920 maxIterationsPtcOutliers = self.config.maxIterationsPtcOutliers
922 for i, ampName in enumerate(dataset.ampNames):
923 timeVecOriginal = np.array(dataset.rawExpTimes[ampName])
924 meanVecOriginal = np.array(dataset.rawMeans[ampName])
925 varVecOriginal = np.array(dataset.rawVars[ampName])
926 varVecOriginal = self._makeZeroSafe(varVecOriginal)
928 mask = ((meanVecOriginal >= self.config.minMeanSignal) &
929 (meanVecOriginal <= self.config.maxMeanSignal))
931 goodPoints = self._getInitialGoodPoints(meanVecOriginal, varVecOriginal,
932 self.config.initialNonLinearityExclusionThresholdPositive,
933 self.config.initialNonLinearityExclusionThresholdNegative)
934 mask = mask & goodPoints
936 if ptcFitType == 'ASTIERAPPROXIMATION':
937 ptcFunc = self.funcAstier
938 parsIniPtc = [-1e-9, 1.0, 10.] # a00, gain, noise
939 bounds = self._boundsForAstier(parsIniPtc)
940 if ptcFitType == 'POLYNOMIAL':
941 ptcFunc = self.funcPolynomial
942 parsIniPtc = self._initialParsForPolynomial(self.config.polynomialFitDegree + 1)
943 bounds = self._boundsForPolynomial(parsIniPtc)
945 # Before bootstrap fit, do an iterative fit to get rid of outliers
946 count = 1
947 while count <= maxIterationsPtcOutliers:
948 # Note that application of the mask actually shrinks the array
949 # to size rather than setting elements to zero (as we want) so
950 # always update mask itself and re-apply to the original data
951 meanTempVec = meanVecOriginal[mask]
952 varTempVec = varVecOriginal[mask]
953 res = least_squares(errFunc, parsIniPtc, bounds=bounds, args=(meanTempVec, varTempVec))
954 pars = res.x
956 # change this to the original from the temp because the masks are ANDed
957 # meaning once a point is masked it's always masked, and the masks must
958 # always be the same length for broadcasting
959 sigResids = (varVecOriginal - ptcFunc(pars, meanVecOriginal))/np.sqrt(varVecOriginal)
960 newMask = np.array([True if np.abs(r) < sigmaCutPtcOutliers else False for r in sigResids])
961 mask = mask & newMask
963 nDroppedTotal = Counter(mask)[False]
964 self.log.debug(f"Iteration {count}: discarded {nDroppedTotal} points in total for {ampName}")
965 count += 1
966 # objects should never shrink
967 assert (len(mask) == len(timeVecOriginal) == len(meanVecOriginal) == len(varVecOriginal))
969 dataset.visitMask[ampName] = mask # store the final mask
971 parsIniPtc = pars
972 timeVecFinal = timeVecOriginal[mask]
973 meanVecFinal = meanVecOriginal[mask]
974 varVecFinal = varVecOriginal[mask]
976 if Counter(mask)[False] > 0:
977 self.log.info((f"Number of points discarded in PTC of amplifier {ampName}:" +
978 f" {Counter(mask)[False]} out of {len(meanVecOriginal)}"))
980 if (len(meanVecFinal) < len(parsIniPtc)):
981 msg = (f"\nSERIOUS: Not enough data points ({len(meanVecFinal)}) compared to the number of"
982 f"parameters of the PTC model({len(parsIniPtc)}). Setting {ampName} to BAD.")
983 self.log.warn(msg)
984 # The first and second parameters of initial fit are discarded (bias and gain)
985 # for the final NL coefficients
986 lenNonLinPars = self.config.polynomialFitDegreeNonLinearity - 1
987 dataset.badAmps.append(ampName)
988 dataset.gain[ampName] = np.nan
989 dataset.gainErr[ampName] = np.nan
990 dataset.noise[ampName] = np.nan
991 dataset.noiseErr[ampName] = np.nan
992 dataset.nonLinearity[ampName] = np.nan
993 dataset.nonLinearityError[ampName] = np.nan
994 dataset.nonLinearityResiduals[ampName] = np.nan
995 dataset.fractionalNonLinearityResiduals[ampName] = np.nan
996 dataset.coefficientLinearizeSquared[ampName] = np.nan
997 dataset.ptcFitPars[ampName] = np.nan
998 dataset.ptcFitParsError[ampName] = np.nan
999 dataset.ptcFitReducedChiSquared[ampName] = np.nan
1000 dataset.coefficientsLinearizePolynomial[ampName] = [np.nan]*lenNonLinPars
1001 tableArray[i, :] = [np.nan]*self.config.maxAduForLookupTableLinearizer
1002 continue
1004 # Fit the PTC
1005 if self.config.doFitBootstrap:
1006 parsFit, parsFitErr, reducedChiSqPtc = self._fitBootstrap(parsIniPtc, meanVecFinal,
1007 varVecFinal, ptcFunc)
1008 else:
1009 parsFit, parsFitErr, reducedChiSqPtc = self._fitLeastSq(parsIniPtc, meanVecFinal,
1010 varVecFinal, ptcFunc)
1012 dataset.ptcFitPars[ampName] = parsFit
1013 dataset.ptcFitParsError[ampName] = parsFitErr
1014 dataset.ptcFitReducedChiSquared[ampName] = reducedChiSqPtc
1016 if ptcFitType == 'ASTIERAPPROXIMATION':
1017 ptcGain = parsFit[1]
1018 ptcGainErr = parsFitErr[1]
1019 ptcNoise = np.sqrt(np.fabs(parsFit[2]))
1020 ptcNoiseErr = 0.5*(parsFitErr[2]/np.fabs(parsFit[2]))*np.sqrt(np.fabs(parsFit[2]))
1021 if ptcFitType == 'POLYNOMIAL':
1022 ptcGain = 1./parsFit[1]
1023 ptcGainErr = np.fabs(1./parsFit[1])*(parsFitErr[1]/parsFit[1])
1024 ptcNoise = np.sqrt(np.fabs(parsFit[0]))*ptcGain
1025 ptcNoiseErr = (0.5*(parsFitErr[0]/np.fabs(parsFit[0]))*(np.sqrt(np.fabs(parsFit[0]))))*ptcGain
1027 dataset.gain[ampName] = ptcGain
1028 dataset.gainErr[ampName] = ptcGainErr
1029 dataset.noise[ampName] = ptcNoise
1030 dataset.noiseErr[ampName] = ptcNoiseErr
1031 dataset.ptcFitType[ampName] = ptcFitType
1033 # Non-linearity residuals (NL of mean vs time curve): percentage, and fit to a quadratic function
1034 # In this case, len(parsIniNonLinearity) = 3 indicates that we want a quadratic fit
1036 datasetLinRes = self.calculateLinearityResidualAndLinearizers(timeVecFinal, meanVecFinal)
1038 # LinearizerLookupTable
1039 if tableArray is not None:
1040 tableArray[i, :] = datasetLinRes.linearizerTableRow
1041 dataset.nonLinearity[ampName] = datasetLinRes.meanSignalVsTimePolyFitPars
1042 dataset.nonLinearityError[ampName] = datasetLinRes.meanSignalVsTimePolyFitParsErr
1043 dataset.nonLinearityResiduals[ampName] = datasetLinRes.linearityResidual
1044 dataset.fractionalNonLinearityResiduals[ampName] = datasetLinRes.fractionalNonLinearityResidual
1045 dataset.nonLinearityReducedChiSquared[ampName] = datasetLinRes.meanSignalVsTimePolyFitReducedChiSq
1046 # Slice correction coefficients (starting at 2) for polynomial linearizer. The first
1047 # and second are reduntant with the bias and gain, respectively,
1048 # and are not used by LinearizerPolynomial.
1049 polyLinCoeffs = np.array(datasetLinRes.polynomialLinearizerCoefficients[2:])
1050 dataset.coefficientsLinearizePolynomial[ampName] = polyLinCoeffs
1051 quadPolyLinCoeff = datasetLinRes.quadraticPolynomialLinearizerCoefficient
1052 dataset.coefficientLinearizeSquared[ampName] = quadPolyLinCoeff
1054 return dataset
1056 def plot(self, dataRef, dataset, ptcFitType):
1057 dirname = dataRef.getUri(datasetType='cpPipePlotRoot', write=True)
1058 if not os.path.exists(dirname):
1059 os.makedirs(dirname)
1061 detNum = dataRef.dataId[self.config.ccdKey]
1062 filename = f"PTC_det{detNum}.pdf"
1063 filenameFull = os.path.join(dirname, filename)
1064 with PdfPages(filenameFull) as pdfPages:
1065 self._plotPtc(dataset, ptcFitType, pdfPages)
1067 def _plotPtc(self, dataset, ptcFitType, pdfPages):
1068 """Plot PTC, linearity, and linearity residual per amplifier"""
1070 reducedChiSqPtc = dataset.ptcFitReducedChiSquared
1071 if ptcFitType == 'ASTIERAPPROXIMATION':
1072 ptcFunc = self.funcAstier
1073 stringTitle = (r"Var = $\frac{1}{2g^2a_{00}}(\exp (2a_{00} \mu g) - 1) + \frac{n_{00}}{g^2}$ "
1074 r" ($chi^2$/dof = %g)" % (reducedChiSqPtc))
1075 if ptcFitType == 'POLYNOMIAL':
1076 ptcFunc = self.funcPolynomial
1077 stringTitle = r"Polynomial (degree: %g)" % (self.config.polynomialFitDegree)
1079 legendFontSize = 7
1080 labelFontSize = 7
1081 titleFontSize = 9
1082 supTitleFontSize = 18
1083 markerSize = 25
1085 # General determination of the size of the plot grid
1086 nAmps = len(dataset.ampNames)
1087 if nAmps == 2:
1088 nRows, nCols = 2, 1
1089 nRows = np.sqrt(nAmps)
1090 mantissa, _ = np.modf(nRows)
1091 if mantissa > 0:
1092 nRows = int(nRows) + 1
1093 nCols = nRows
1094 else:
1095 nRows = int(nRows)
1096 nCols = nRows
1098 f, ax = plt.subplots(nrows=nRows, ncols=nCols, sharex='col', sharey='row', figsize=(13, 10))
1099 f2, ax2 = plt.subplots(nrows=nRows, ncols=nCols, sharex='col', sharey='row', figsize=(13, 10))
1101 for i, (amp, a, a2) in enumerate(zip(dataset.ampNames, ax.flatten(), ax2.flatten())):
1102 meanVecOriginal = np.array(dataset.rawMeans[amp])
1103 varVecOriginal = np.array(dataset.rawVars[amp])
1104 mask = dataset.visitMask[amp]
1105 meanVecFinal = meanVecOriginal[mask]
1106 varVecFinal = varVecOriginal[mask]
1107 meanVecOutliers = meanVecOriginal[np.invert(mask)]
1108 varVecOutliers = varVecOriginal[np.invert(mask)]
1109 pars, parsErr = dataset.ptcFitPars[amp], dataset.ptcFitParsError[amp]
1111 if ptcFitType == 'ASTIERAPPROXIMATION':
1112 if len(meanVecFinal):
1113 ptcA00, ptcA00error = pars[0], parsErr[0]
1114 ptcGain, ptcGainError = pars[1], parsErr[1]
1115 ptcNoise = np.sqrt(np.fabs(pars[2]))
1116 ptcNoiseError = 0.5*(parsErr[2]/np.fabs(pars[2]))*np.sqrt(np.fabs(pars[2]))
1117 stringLegend = (f"a00: {ptcA00:.2e}+/-{ptcA00error:.2e} 1/e"
1118 f"\n Gain: {ptcGain:.4}+/-{ptcGainError:.2e} e/DN"
1119 f"\n Noise: {ptcNoise:.4}+/-{ptcNoiseError:.2e} e \n")
1121 if ptcFitType == 'POLYNOMIAL':
1122 if len(meanVecFinal):
1123 ptcGain, ptcGainError = 1./pars[1], np.fabs(1./pars[1])*(parsErr[1]/pars[1])
1124 ptcNoise = np.sqrt(np.fabs(pars[0]))*ptcGain
1125 ptcNoiseError = (0.5*(parsErr[0]/np.fabs(pars[0]))*(np.sqrt(np.fabs(pars[0]))))*ptcGain
1126 stringLegend = (f"Noise: {ptcNoise:.4}+/-{ptcNoiseError:.2e} e \n"
1127 f"Gain: {ptcGain:.4}+/-{ptcGainError:.2e} e/DN \n")
1129 a.set_xlabel(r'Mean signal ($\mu$, DN)', fontsize=labelFontSize)
1130 a.set_ylabel(r'Variance (DN$^2$)', fontsize=labelFontSize)
1131 a.tick_params(labelsize=11)
1132 a.set_xscale('linear', fontsize=labelFontSize)
1133 a.set_yscale('linear', fontsize=labelFontSize)
1135 a2.set_xlabel(r'Mean Signal ($\mu$, DN)', fontsize=labelFontSize)
1136 a2.set_ylabel(r'Variance (DN$^2$)', fontsize=labelFontSize)
1137 a2.tick_params(labelsize=11)
1138 a2.set_xscale('log')
1139 a2.set_yscale('log')
1141 if len(meanVecFinal): # Empty if the whole amp is bad, for example.
1142 minMeanVecFinal = np.min(meanVecFinal)
1143 maxMeanVecFinal = np.max(meanVecFinal)
1144 meanVecFit = np.linspace(minMeanVecFinal, maxMeanVecFinal, 100*len(meanVecFinal))
1145 minMeanVecOriginal = np.min(meanVecOriginal)
1146 maxMeanVecOriginal = np.max(meanVecOriginal)
1147 deltaXlim = maxMeanVecOriginal - minMeanVecOriginal
1149 a.plot(meanVecFit, ptcFunc(pars, meanVecFit), color='red')
1150 a.plot(meanVecFinal, pars[0] + pars[1]*meanVecFinal, color='green', linestyle='--')
1151 a.scatter(meanVecFinal, varVecFinal, c='blue', marker='o', s=markerSize)
1152 a.scatter(meanVecOutliers, varVecOutliers, c='magenta', marker='s', s=markerSize)
1153 a.text(0.03, 0.8, stringLegend, transform=a.transAxes, fontsize=legendFontSize)
1154 a.set_title(amp, fontsize=titleFontSize)
1155 a.set_xlim([minMeanVecOriginal - 0.2*deltaXlim, maxMeanVecOriginal + 0.2*deltaXlim])
1157 # Same, but in log-scale
1158 a2.plot(meanVecFit, ptcFunc(pars, meanVecFit), color='red')
1159 a2.scatter(meanVecFinal, varVecFinal, c='blue', marker='o', s=markerSize)
1160 a2.scatter(meanVecOutliers, varVecOutliers, c='magenta', marker='s', s=markerSize)
1161 a2.text(0.03, 0.8, stringLegend, transform=a2.transAxes, fontsize=legendFontSize)
1162 a2.set_title(amp, fontsize=titleFontSize)
1163 a2.set_xlim([minMeanVecOriginal, maxMeanVecOriginal])
1164 else:
1165 a.set_title(f"{amp} (BAD)", fontsize=titleFontSize)
1166 a2.set_title(f"{amp} (BAD)", fontsize=titleFontSize)
1168 f.suptitle("PTC \n Fit: " + stringTitle, fontsize=20)
1169 pdfPages.savefig(f)
1170 f2.suptitle("PTC (log-log)", fontsize=20)
1171 pdfPages.savefig(f2)
1173 # Plot mean vs time
1174 f, ax = plt.subplots(nrows=4, ncols=4, sharex='col', sharey='row', figsize=(13, 10))
1175 for i, (amp, a) in enumerate(zip(dataset.ampNames, ax.flatten())):
1176 meanVecFinal = np.array(dataset.rawMeans[amp])[dataset.visitMask[amp]]
1177 timeVecFinal = np.array(dataset.rawExpTimes[amp])[dataset.visitMask[amp]]
1179 a.set_xlabel('Time (sec)', fontsize=labelFontSize)
1180 a.set_ylabel(r'Mean signal ($\mu$, DN)', fontsize=labelFontSize)
1181 a.tick_params(labelsize=labelFontSize)
1182 a.set_xscale('linear', fontsize=labelFontSize)
1183 a.set_yscale('linear', fontsize=labelFontSize)
1185 if len(meanVecFinal):
1186 pars, parsErr = dataset.nonLinearity[amp], dataset.nonLinearityError[amp]
1187 k0, k0Error = pars[0], parsErr[0]
1188 k1, k1Error = pars[1], parsErr[1]
1189 k2, k2Error = pars[2], parsErr[2]
1190 stringLegend = (f"k0: {k0:.4}+/-{k0Error:.2e} DN\n k1: {k1:.4}+/-{k1Error:.2e} DN/t"
1191 f"\n k2: {k2:.2e}+/-{k2Error:.2e} DN/t^2 \n")
1192 a.scatter(timeVecFinal, meanVecFinal)
1193 a.plot(timeVecFinal, self.funcPolynomial(pars, timeVecFinal), color='red')
1194 a.text(0.03, 0.75, stringLegend, transform=a.transAxes, fontsize=legendFontSize)
1195 a.set_title(f"{amp}", fontsize=titleFontSize)
1196 else:
1197 a.set_title(f"{amp} (BAD)", fontsize=titleFontSize)
1199 f.suptitle("Linearity \n Fit: Polynomial (degree: %g)"
1200 % (self.config.polynomialFitDegreeNonLinearity),
1201 fontsize=supTitleFontSize)
1202 pdfPages.savefig(f)
1204 # Plot linearity residual
1205 f, ax = plt.subplots(nrows=4, ncols=4, sharex='col', sharey='row', figsize=(13, 10))
1206 for i, (amp, a) in enumerate(zip(dataset.ampNames, ax.flatten())):
1207 meanVecFinal = np.array(dataset.rawMeans[amp])[dataset.visitMask[amp]]
1208 linRes = np.array(dataset.nonLinearityResiduals[amp])
1210 a.set_xlabel(r'Mean signal ($\mu$, DN)', fontsize=labelFontSize)
1211 a.set_ylabel('LR (%)', fontsize=labelFontSize)
1212 a.set_xscale('linear', fontsize=labelFontSize)
1213 a.set_yscale('linear', fontsize=labelFontSize)
1214 if len(meanVecFinal):
1215 a.axvline(x=timeVecFinal[self.config.linResidualTimeIndex], color='g', linestyle='--')
1216 a.scatter(meanVecFinal, linRes)
1217 a.set_title(f"{amp}", fontsize=titleFontSize)
1218 a.axhline(y=0, color='k')
1219 a.tick_params(labelsize=labelFontSize)
1220 else:
1221 a.set_title(f"{amp} (BAD)", fontsize=titleFontSize)
1223 f.suptitle(r"Linearity Residual: $100\times(1 - \mu_{\rm{ref}}/t_{\rm{ref}})/(\mu / t))$" + "\n" +
1224 r"$t_{\rm{ref}}$: " + f"{timeVecFinal[2]} s", fontsize=supTitleFontSize)
1225 pdfPages.savefig(f)
1227 # Plot fractional non-linearity residual (w.r.t linear part of polynomial fit)
1228 f, ax = plt.subplots(nrows=4, ncols=4, sharex='col', sharey='row', figsize=(13, 10))
1229 for i, (amp, a) in enumerate(zip(dataset.ampNames, ax.flatten())):
1230 meanVecFinal = np.array(dataset.rawMeans[amp])[dataset.visitMask[amp]]
1231 fracLinRes = np.array(dataset.fractionalNonLinearityResiduals[amp])
1233 a.axhline(y=0, color='k')
1234 a.axvline(x=0, color='k', linestyle='-')
1235 a.set_xlabel(r'Mean signal ($\mu$, DN)', fontsize=labelFontSize)
1236 a.set_ylabel('Fractional nonlinearity (%)', fontsize=labelFontSize)
1237 a.tick_params(labelsize=labelFontSize)
1238 a.set_xscale('linear', fontsize=labelFontSize)
1239 a.set_yscale('linear', fontsize=labelFontSize)
1241 if len(meanVecFinal):
1242 a.scatter(meanVecFinal, fracLinRes, c='g')
1243 a.set_title(amp, fontsize=titleFontSize)
1244 else:
1245 a.set_title(f"{amp} (BAD)", fontsize=titleFontSize)
1247 f.suptitle(r"Fractional NL residual" + "\n" +
1248 r"$100\times \frac{(k_0 + k_1*Time-\mu)}{k_0+k_1*Time}$",
1249 fontsize=supTitleFontSize)
1250 pdfPages.savefig(f)
1252 return