23 __all__ = [
'MeasurePhotonTransferCurveTask',
24 'MeasurePhotonTransferCurveTaskConfig',
25 'PhotonTransferCurveDataset']
28 import matplotlib.pyplot
as plt
30 from matplotlib.backends.backend_pdf
import PdfPages
31 from sqlite3
import OperationalError
32 from collections
import Counter
35 import lsst.pex.config
as pexConfig
38 from .utils
import (NonexistentDatasetTaskDataIdContainer, PairedVisitListTaskRunner,
39 checkExpLengthEqual, validateIsrConfig)
40 from scipy.optimize
import leastsq, least_squares
41 import numpy.polynomial.polynomial
as poly
45 """Config class for photon transfer curve measurement task""" 46 isr = pexConfig.ConfigurableField(
48 doc=
"""Task to perform instrumental signature removal.""",
50 isrMandatorySteps = pexConfig.ListField(
52 doc=
"isr operations that must be performed for valid results. Raises if any of these are False.",
53 default=[
'doAssembleCcd']
55 isrForbiddenSteps = pexConfig.ListField(
57 doc=
"isr operations that must NOT be performed for valid results. Raises if any of these are True",
58 default=[
'doFlat',
'doFringe',
'doAddDistortionModel',
'doBrighterFatter',
'doUseOpticsTransmission',
59 'doUseFilterTransmission',
'doUseSensorTransmission',
'doUseAtmosphereTransmission']
61 isrDesirableSteps = pexConfig.ListField(
63 doc=
"isr operations that it is advisable to perform, but are not mission-critical." +
64 " WARNs are logged for any of these found to be False.",
65 default=[
'doBias',
'doDark',
'doCrosstalk',
'doDefect']
67 isrUndesirableSteps = pexConfig.ListField(
69 doc=
"isr operations that it is *not* advisable to perform in the general case, but are not" +
70 " forbidden as some use-cases might warrant them." +
71 " WARNs are logged for any of these found to be True.",
72 default=[
'doLinearize']
74 ccdKey = pexConfig.Field(
76 doc=
"The key by which to pull a detector from a dataId, e.g. 'ccd' or 'detector'.",
79 makePlots = pexConfig.Field(
81 doc=
"Plot the PTC curves?",
84 ptcFitType = pexConfig.ChoiceField(
86 doc=
"Fit PTC to approximation in Astier+19 (Equation 16) or to a polynomial.",
89 "POLYNOMIAL":
"n-degree polynomial (use 'polynomialFitDegree' to set 'n').",
90 "ASTIERAPPROXIMATION":
"Approximation in Astier+19 (Eq. 16)." 93 polynomialFitDegree = pexConfig.Field(
95 doc=
"Degree of polynomial to fit the PTC, when 'ptcFitType'=POLYNOMIAL.",
98 polynomialFitDegreeNonLinearity = pexConfig.Field(
100 doc=
"Degree of polynomial to fit the meanSignal vs exposureTime curve to produce" +
101 " the table for LinearizerLookupTable.",
104 binSize = pexConfig.Field(
106 doc=
"Bin the image by this factor in both dimensions.",
109 minMeanSignal = pexConfig.Field(
111 doc=
"Minimum value (inclusive) of mean signal (in ADU) above which to consider.",
114 maxMeanSignal = pexConfig.Field(
116 doc=
"Maximum value (inclusive) of mean signal (in ADU) below which to consider.",
119 initialNonLinearityExclusionThresholdPositive = pexConfig.RangeField(
121 doc=
"Initially exclude data points with a variance that are more than a factor of this from being" 122 " linear in the positive direction, from the PTC fit. Note that these points will also be" 123 " excluded from the non-linearity fit. This is done before the iterative outlier rejection," 124 " to allow an accurate determination of the sigmas for said iterative fit.",
129 initialNonLinearityExclusionThresholdNegative = pexConfig.RangeField(
131 doc=
"Initially exclude data points with a variance that are more than a factor of this from being" 132 " linear in the negative direction, from the PTC fit. Note that these points will also be" 133 " excluded from the non-linearity fit. This is done before the iterative outlier rejection," 134 " to allow an accurate determination of the sigmas for said iterative fit.",
139 sigmaCutPtcOutliers = pexConfig.Field(
141 doc=
"Sigma cut for outlier rejection in PTC.",
144 maxIterationsPtcOutliers = pexConfig.Field(
146 doc=
"Maximum number of iterations for outlier rejection in PTC.",
149 doFitBootstrap = pexConfig.Field(
151 doc=
"Use bootstrap for the PTC fit parameters and errors?.",
154 linResidualTimeIndex = pexConfig.Field(
156 doc=
"Index position in time array for reference time in linearity residual calculation.",
159 maxAduForLookupTableLinearizer = pexConfig.Field(
161 doc=
"Maximum ADU value for the LookupTable linearizer.",
167 """A simple class to hold the output data from the PTC task. 169 The dataset is made up of a dictionary for each item, keyed by the 170 amplifiers' names, which much be supplied at construction time. 172 New items cannot be added to the class to save accidentally saving to the 173 wrong property, and the class can be frozen if desired. 175 inputVisitPairs records the visits used to produce the data. 176 When fitPtcAndNonLinearity() is run, a mask is built up, which is by definition 177 always the same length as inputVisitPairs, rawExpTimes, rawMeans 178 and rawVars, and is a list of bools, which are incrementally set to False 179 as points are discarded from the fits. 181 PTC fit parameters for polynomials are stored in a list in ascending order 182 of polynomial term, i.e. par[0]*x^0 + par[1]*x + par[2]*x^2 etc 183 with the length of the list corresponding to the order of the polynomial 190 self.__dict__[
"ampNames"] = ampNames
191 self.__dict__[
"badAmps"] = []
194 self.__dict__[
"inputVisitPairs"] = {ampName: []
for ampName
in ampNames}
195 self.__dict__[
"visitMask"] = {ampName: []
for ampName
in ampNames}
196 self.__dict__[
"rawExpTimes"] = {ampName: []
for ampName
in ampNames}
197 self.__dict__[
"rawMeans"] = {ampName: []
for ampName
in ampNames}
198 self.__dict__[
"rawVars"] = {ampName: []
for ampName
in ampNames}
201 self.__dict__[
"ptcFitType"] = {ampName:
"" for ampName
in ampNames}
202 self.__dict__[
"ptcFitPars"] = {ampName: []
for ampName
in ampNames}
203 self.__dict__[
"ptcFitParsError"] = {ampName: []
for ampName
in ampNames}
204 self.__dict__[
"nonLinearity"] = {ampName: []
for ampName
in ampNames}
205 self.__dict__[
"nonLinearityError"] = {ampName: []
for ampName
in ampNames}
206 self.__dict__[
"nonLinearityResiduals"] = {ampName: []
for ampName
in ampNames}
207 self.__dict__[
"coefficientLinearizeSquared"] = {ampName: []
for ampName
in ampNames}
210 self.__dict__[
"gain"] = {ampName: -1.
for ampName
in ampNames}
211 self.__dict__[
"gainErr"] = {ampName: -1.
for ampName
in ampNames}
212 self.__dict__[
"noise"] = {ampName: -1.
for ampName
in ampNames}
213 self.__dict__[
"noiseErr"] = {ampName: -1.
for ampName
in ampNames}
214 self.__dict__[
"coefficientLinearizeSquared"] = {ampName: []
for ampName
in ampNames}
217 """Protect class attributes""" 218 if attribute
not in self.__dict__:
219 raise AttributeError(f
"{attribute} is not already a member of PhotonTransferCurveDataset, which" 220 " does not support setting of new attributes.")
222 self.__dict__[attribute] = value
225 """Get the visits used, i.e. not discarded, for a given amp. 227 If no mask has been created yet, all visits are returned. 229 if self.visitMask[ampName] == []:
230 return self.inputVisitPairs[ampName]
233 assert len(self.visitMask[ampName]) == len(self.inputVisitPairs[ampName])
235 pairs = self.inputVisitPairs[ampName]
236 mask = self.visitMask[ampName]
238 return [(v1, v2)
for ((v1, v2), m)
in zip(pairs, mask)
if bool(m)
is True]
241 return [amp
for amp
in self.ampNames
if amp
not in self.badAmps]
245 """A class to calculate, fit, and plot a PTC from a set of flat pairs. 247 The Photon Transfer Curve (var(signal) vs mean(signal)) is a standard tool 248 used in astronomical detectors characterization (e.g., Janesick 2001, 249 Janesick 2007). This task calculates the PTC from a series of pairs of 250 flat-field images; each pair taken at identical exposure times. The 251 difference image of each pair is formed to eliminate fixed pattern noise, 252 and then the variance of the difference image and the mean of the average image 253 are used to produce the PTC. An n-degree polynomial or the approximation in Equation 254 16 of Astier+19 ("The Shape of the Photon Transfer Curve of CCD sensors", 255 arXiv:1905.08677) can be fitted to the PTC curve. These models include 256 parameters such as the gain (e/ADU) and readout noise. 262 Positional arguments passed to the Task constructor. None used at this 265 Keyword arguments passed on to the Task constructor. None used at this 270 RunnerClass = PairedVisitListTaskRunner
271 ConfigClass = MeasurePhotonTransferCurveTaskConfig
272 _DefaultName =
"measurePhotonTransferCurve" 275 pipeBase.CmdLineTask.__init__(self, *args, **kwargs)
276 self.makeSubtask(
"isr")
277 plt.interactive(
False)
279 self.config.isrForbiddenSteps, self.config.isrDesirableSteps, checkTrim=
False)
280 self.config.validate()
284 def _makeArgumentParser(cls):
285 """Augment argument parser for the MeasurePhotonTransferCurveTask.""" 287 parser.add_argument(
"--visit-pairs", dest=
"visitPairs", nargs=
"*",
288 help=
"Visit pairs to use. Each pair must be of the form INT,INT e.g. 123,456")
289 parser.add_id_argument(
"--id", datasetType=
"photonTransferCurveDataset",
290 ContainerClass=NonexistentDatasetTaskDataIdContainer,
291 help=
"The ccds to use, e.g. --id ccd=0..100")
296 """Run the Photon Transfer Curve (PTC) measurement task. 298 For a dataRef (which is each detector here), 299 and given a list of visit pairs at different exposure times, 304 dataRef : list of lsst.daf.persistence.ButlerDataRef 305 dataRef for the detector for the visits to be fit. 306 visitPairs : `iterable` of `tuple` of `int` 307 Pairs of visit numbers to be processed together 311 detNum = dataRef.dataId[self.config.ccdKey]
312 detector = dataRef.get(
'camera')[dataRef.dataId[self.config.ccdKey]]
318 for name
in dataRef.getButler().getKeys(
'bias'):
319 if name
not in dataRef.dataId:
321 dataRef.dataId[name] = \
322 dataRef.getButler().queryMetadata(
'raw', [name], detector=detNum)[0]
323 except OperationalError:
326 amps = detector.getAmplifiers()
327 ampNames = [amp.getName()
for amp
in amps]
330 self.log.info(
'Measuring PTC using %s visits for detector %s' % (visitPairs, detNum))
332 for (v1, v2)
in visitPairs:
334 dataRef.dataId[
'expId'] = v1
336 dataRef.dataId[
'expId'] = v2
338 del dataRef.dataId[
'expId']
341 expTime = exp1.getInfo().getVisitInfo().getExposureTime()
345 ampName = amp.getName()
347 dataset.rawExpTimes[ampName].append(expTime)
348 dataset.rawMeans[ampName].append(mu)
349 dataset.rawVars[ampName].append(varDiff)
350 dataset.inputVisitPairs[ampName].append((v1, v2))
352 numberAmps = len(detector.getAmplifiers())
353 numberAduValues = self.config.maxAduForLookupTableLinearizer
354 lookupTableArray = np.zeros((numberAmps, numberAduValues), dtype=np.float32)
361 if self.config.makePlots:
362 self.
plot(dataRef, dataset, ptcFitType=self.config.ptcFitType)
365 self.log.info(f
"Writing PTC and NL data to {dataRef.getUri(write=True)}")
366 dataRef.put(dataset, datasetType=
"photonTransferCurveDataset")
368 self.log.info(
'Finished measuring PTC for in detector %s' % detNum)
370 return pipeBase.Struct(exitStatus=0)
373 """Calculate the mean signal of two exposures and the variance of their difference. 377 exposure1 : `lsst.afw.image.exposure.exposure.ExposureF` 378 First exposure of flat field pair. 380 exposure2 : `lsst.afw.image.exposure.exposure.ExposureF` 381 Second exposure of flat field pair. 383 region : `lsst.geom.Box2I` 384 Region of each exposure where to perform the calculations (e.g, an amplifier). 390 0.5*(mu1 + mu2), where mu1, and mu2 are the clipped means of the regions in 394 Half of the clipped variance of the difference of the regions inthe two input 398 if region
is not None:
399 im1Area = exposure1.maskedImage[region]
400 im2Area = exposure2.maskedImage[region]
402 im1Area = exposure1.maskedImage
403 im2Area = exposure2.maskedImage
405 im1Area = afwMath.binImage(im1Area, self.config.binSize)
406 im2Area = afwMath.binImage(im2Area, self.config.binSize)
409 mu1 = afwMath.makeStatistics(im1Area, afwMath.MEANCLIP).getValue()
410 mu2 = afwMath.makeStatistics(im2Area, afwMath.MEANCLIP).getValue()
415 temp = im2Area.clone()
417 diffIm = im1Area.clone()
422 varDiff = 0.5*(afwMath.makeStatistics(diffIm, afwMath.VARIANCECLIP).getValue())
426 def _fitLeastSq(self, initialParams, dataX, dataY, function):
427 """Do a fit and estimate the parameter errors using using scipy.optimize.leastq. 429 optimize.leastsq returns the fractional covariance matrix. To estimate the 430 standard deviation of the fit parameters, multiply the entries of this matrix 431 by the reduced chi squared and take the square root of the diagonal elements. 435 initialParams : list of np.float 436 initial values for fit parameters. For ptcFitType=POLYNOMIAL, its length 437 determines the degree of the polynomial. 439 dataX : np.array of np.float 440 Data in the abscissa axis. 442 dataY : np.array of np.float 443 Data in the ordinate axis. 445 function : callable object (function) 446 Function to fit the data with. 450 pFitSingleLeastSquares : list of np.float 451 List with fitted parameters. 453 pErrSingleLeastSquares : list of np.float 454 List with errors for fitted parameters. 457 def errFunc(p, x, y):
458 return function(p, x) - y
460 pFit, pCov, infoDict, errMessage, success = leastsq(errFunc, initialParams,
461 args=(dataX, dataY), full_output=1, epsfcn=0.0001)
463 if (len(dataY) > len(initialParams))
and pCov
is not None:
464 reducedChiSq = (errFunc(pFit, dataX, dataY)**2).sum()/(len(dataY)-len(initialParams))
470 for i
in range(len(pFit)):
471 errorVec.append(np.fabs(pCov[i][i])**0.5)
473 pFitSingleLeastSquares = pFit
474 pErrSingleLeastSquares = np.array(errorVec)
476 return pFitSingleLeastSquares, pErrSingleLeastSquares
478 def _fitBootstrap(self, initialParams, dataX, dataY, function, confidenceSigma=1.):
479 """Do a fit using least squares and bootstrap to estimate parameter errors. 481 The bootstrap error bars are calculated by fitting 100 random data sets. 485 initialParams : list of np.float 486 initial values for fit parameters. For ptcFitType=POLYNOMIAL, its length 487 determines the degree of the polynomial. 489 dataX : np.array of np.float 490 Data in the abscissa axis. 492 dataY : np.array of np.float 493 Data in the ordinate axis. 495 function : callable object (function) 496 Function to fit the data with. 498 confidenceSigma : np.float 499 Number of sigmas that determine confidence interval for the bootstrap errors. 503 pFitBootstrap : list of np.float 504 List with fitted parameters. 506 pErrBootstrap : list of np.float 507 List with errors for fitted parameters. 510 def errFunc(p, x, y):
511 return function(p, x) - y
514 pFit, _ = leastsq(errFunc, initialParams, args=(dataX, dataY), full_output=0)
517 residuals = errFunc(pFit, dataX, dataY)
518 sigmaErrTotal = np.std(residuals)
523 randomDelta = np.random.normal(0., sigmaErrTotal, len(dataY))
524 randomDataY = dataY + randomDelta
525 randomFit, _ = leastsq(errFunc, initialParams,
526 args=(dataX, randomDataY), full_output=0)
527 pars.append(randomFit)
528 pars = np.array(pars)
529 meanPfit = np.mean(pars, 0)
532 nSigma = confidenceSigma
533 errPfit = nSigma*np.std(pars, 0)
534 pFitBootstrap = meanPfit
535 pErrBootstrap = errPfit
536 return pFitBootstrap, pErrBootstrap
539 """Polynomial function definition""" 540 return poly.polyval(x, [*pars])
543 """Single brighter-fatter parameter model for PTC; Equation 16 of Astier+19""" 544 a00, gain, noise = pars
545 return 0.5/(a00*gain*gain)*(np.exp(2*a00*x*gain)-1) + noise/(gain*gain)
548 def _initialParsForPolynomial(order):
550 pars = np.zeros(order, dtype=np.float)
557 def _boundsForPolynomial(initialPars):
558 lowers = [np.NINF
for p
in initialPars]
559 uppers = [np.inf
for p
in initialPars]
561 return (lowers, uppers)
564 def _boundsForAstier(initialPars):
565 lowers = [np.NINF
for p
in initialPars]
566 uppers = [np.inf
for p
in initialPars]
567 return (lowers, uppers)
570 def _getInitialGoodPoints(means, variances, maxDeviationPositive, maxDeviationNegative):
571 """Return a boolean array to mask bad points. 573 A linear function has a constant ratio, so find the median 574 value of the ratios, and exclude the points that deviate 575 from that by more than a factor of maxDeviationPositive/negative. 576 Asymmetric deviations are supported as we expect the PTC to turn 577 down as the flux increases, but sometimes it anomalously turns 578 upwards just before turning over, which ruins the fits, so it 579 is wise to be stricter about restricting positive outliers than 582 Too high and points that are so bad that fit will fail will be included 583 Too low and the non-linear points will be excluded, biasing the NL fit.""" 584 ratios = [b/a
for (a, b)
in zip(means, variances)]
585 medianRatio = np.median(ratios)
586 ratioDeviations = [(r/medianRatio)-1
for r
in ratios]
589 maxDeviationPositive = abs(maxDeviationPositive)
590 maxDeviationNegative = -1. * abs(maxDeviationNegative)
592 goodPoints = np.array([
True if (r < maxDeviationPositive
and r > maxDeviationNegative)
593 else False for r
in ratioDeviations])
596 def _makeZeroSafe(self, array, warn=True, substituteValue=1e-9):
598 nBad = Counter(array)[0]
603 msg = f
"Found {nBad} zeros in array at elements {[x for x in np.where(array==0)[0]]}" 606 array[array == 0] = substituteValue
610 """Calculate linearity residual and fit an n-order polynomial to the mean vs time curve 611 to produce corrections (deviation from linear part of polynomial) for a particular amplifier 612 to populate LinearizeLookupTable. Use quadratic and linear parts of this polynomial to approximate 613 c0 for LinearizeSquared." 618 exposureTimeVector: `list` of `np.float` 619 List of exposure times for each flat pair 621 meanSignalVector: `list` of `np.float` 622 List of mean signal from diference image of flat pairs 627 Coefficient for LinearizeSquared, where corrImage = uncorrImage + c0*uncorrImage^2. 628 c0 ~ -k2/(k1^2), where k1 and k2 are fit from 629 meanSignalVector = k0 + k1*exposureTimeVector + k2*exposureTimeVector^2 +... 630 + kn*exposureTimeVector^n, with n = "polynomialFitDegreeNonLinearity". 632 linearizerTableRow: list of `np.float` 633 One dimensional array with deviation from linear part of n-order polynomial fit 634 to mean vs time curve. This array will be one row (for the particular amplifier at hand) 635 of the table array for LinearizeLookupTable. 637 linResidual: list of `np.float` 638 Linearity residual from the mean vs time curve, defined as 639 100*(1 - meanSignalReference/expTimeReference/(meanSignal/expTime). 641 parsFit: list of `np.float` 642 Parameters from n-order polynomial fit to mean vs time curve. 644 parsFitErr: list of `np.float` 645 Parameters from n-order polynomial fit to mean vs time curve. 651 if self.config.doFitBootstrap:
652 parsFit, parsFitErr = self.
_fitBootstrap(parsIniNonLinearity, exposureTimeVector,
655 parsFit, parsFitErr = self.
_fitLeastSq(parsIniNonLinearity, exposureTimeVector, meanSignalVector,
659 tMax = (self.config.maxAduForLookupTableLinearizer - parsFit[0])/parsFit[1]
660 timeRange = np.linspace(0, tMax, self.config.maxAduForLookupTableLinearizer)
661 signalIdeal = parsFit[0] + parsFit[1]*timeRange
663 linearizerTableRow = signalIdeal - signalUncorrected
669 k1, k2 = parsFit[1], parsFit[2]
671 for coefficient
in parsFit[3:]:
672 if np.fabs(coefficient) > 1e-10:
673 msg = f
"Coefficient {coefficient} in polynomial fit larger than threshold 1e-10." 677 linResidualTimeIndex = self.config.linResidualTimeIndex
678 if exposureTimeVector[linResidualTimeIndex] == 0.0:
679 raise RuntimeError(
"Reference time for linearity residual can't be 0.0")
680 linResidual = 100*(1 - ((meanSignalVector[linResidualTimeIndex] /
681 exposureTimeVector[linResidualTimeIndex]) /
682 (meanSignalVector/exposureTimeVector)))
684 return c0, linearizerTableRow, linResidual, parsFit, parsFitErr
687 """Fit the photon transfer curve and calculate linearity and residuals. 689 Fit the photon transfer curve with either a polynomial of the order 690 specified in the task config, or using the Astier approximation. 692 Sigma clipping is performed iteratively for the fit, as well as an 693 initial clipping of data points that are more than 694 config.initialNonLinearityExclusionThreshold away from lying on a 695 straight line. This other step is necessary because the photon transfer 696 curve turns over catastrophically at very high flux (because saturation 697 drops the variance to ~0) and these far outliers cause the initial fit 698 to fail, meaning the sigma cannot be calculated to perform the 703 dataset : `lsst.cp.pipe.ptc.PhotonTransferCurveDataset` 704 The dataset containing the means, variances and exposure times 706 Fit a 'POLYNOMIAL' (degree: 'polynomialFitDegree') or 707 'ASTIERAPPROXIMATION' to the PTC 708 tableArray : `np.array` 709 Look-up table array with size rows=nAmps and columns=ADU values 713 dataset: `lsst.cp.pipe.ptc.PhotonTransferCurveDataset` 714 This is the same dataset as the input paramter, however, it has been modified 715 to include information such as the fit vectors and the fit parameters. See 716 the class `PhotonTransferCurveDatase`. 719 def errFunc(p, x, y):
720 return ptcFunc(p, x) - y
722 sigmaCutPtcOutliers = self.config.sigmaCutPtcOutliers
723 maxIterationsPtcOutliers = self.config.maxIterationsPtcOutliers
725 for i, ampName
in enumerate(dataset.ampNames):
726 timeVecOriginal = np.array(dataset.rawExpTimes[ampName])
727 meanVecOriginal = np.array(dataset.rawMeans[ampName])
728 varVecOriginal = np.array(dataset.rawVars[ampName])
731 mask = ((meanVecOriginal >= self.config.minMeanSignal) &
732 (meanVecOriginal <= self.config.maxMeanSignal))
735 self.config.initialNonLinearityExclusionThresholdPositive,
736 self.config.initialNonLinearityExclusionThresholdNegative)
737 mask = mask & goodPoints
739 if ptcFitType ==
'ASTIERAPPROXIMATION':
741 parsIniPtc = [-1e-9, 1.0, 10.]
743 if ptcFitType ==
'POLYNOMIAL':
750 while count <= maxIterationsPtcOutliers:
754 meanTempVec = meanVecOriginal[mask]
755 varTempVec = varVecOriginal[mask]
756 res = least_squares(errFunc, parsIniPtc, bounds=bounds, args=(meanTempVec, varTempVec))
762 sigResids = (varVecOriginal - ptcFunc(pars, meanVecOriginal))/np.sqrt(varVecOriginal)
763 newMask = np.array([
True if np.abs(r) < sigmaCutPtcOutliers
else False for r
in sigResids])
764 mask = mask & newMask
766 nDroppedTotal = Counter(mask)[
False]
767 self.log.debug(f
"Iteration {count}: discarded {nDroppedTotal} points in total for {ampName}")
770 assert (len(mask) == len(timeVecOriginal) == len(meanVecOriginal) == len(varVecOriginal))
772 dataset.visitMask[ampName] = mask
775 timeVecFinal = timeVecOriginal[mask]
776 meanVecFinal = meanVecOriginal[mask]
777 varVecFinal = varVecOriginal[mask]
779 if Counter(mask)[
False] > 0:
780 self.log.info((f
"Number of points discarded in PTC of amplifier {ampName}:" +
781 f
" {Counter(mask)[False]} out of {len(meanVecOriginal)}"))
783 if (len(meanVecFinal) < len(parsIniPtc)):
784 msg = (f
"\nSERIOUS: Not enough data points ({len(meanVecFinal)}) compared to the number of" 785 f
"parameters of the PTC model({len(parsIniPtc)}). Setting {ampName} to BAD.")
787 dataset.badAmps.append(ampName)
788 dataset.gain[ampName] = np.nan
789 dataset.gainErr[ampName] = np.nan
790 dataset.noise[ampName] = np.nan
791 dataset.noiseErr[ampName] = np.nan
792 dataset.nonLinearity[ampName] = np.nan
793 dataset.nonLinearityError[ampName] = np.nan
794 dataset.nonLinearityResiduals[ampName] = np.nan
798 if self.config.doFitBootstrap:
799 parsFit, parsFitErr = self.
_fitBootstrap(parsIniPtc, meanVecFinal, varVecFinal, ptcFunc)
801 parsFit, parsFitErr = self.
_fitLeastSq(parsIniPtc, meanVecFinal, varVecFinal, ptcFunc)
803 dataset.ptcFitPars[ampName] = parsFit
804 dataset.ptcFitParsError[ampName] = parsFitErr
806 if ptcFitType ==
'ASTIERAPPROXIMATION':
808 ptcGainErr = parsFitErr[1]
809 ptcNoise = np.sqrt(np.fabs(parsFit[2]))
810 ptcNoiseErr = 0.5*(parsFitErr[2]/np.fabs(parsFit[2]))*np.sqrt(np.fabs(parsFit[2]))
811 if ptcFitType ==
'POLYNOMIAL':
812 ptcGain = 1./parsFit[1]
813 ptcGainErr = np.fabs(1./parsFit[1])*(parsFitErr[1]/parsFit[1])
814 ptcNoise = np.sqrt(np.fabs(parsFit[0]))*ptcGain
815 ptcNoiseErr = (0.5*(parsFitErr[0]/np.fabs(parsFit[0]))*(np.sqrt(np.fabs(parsFit[0]))))*ptcGain
817 dataset.gain[ampName] = ptcGain
818 dataset.gainErr[ampName] = ptcGainErr
819 dataset.noise[ampName] = ptcNoise
820 dataset.noiseErr[ampName] = ptcNoiseErr
821 dataset.ptcFitType[ampName] = ptcFitType
826 (c0, linearizerTableRow, linResidualNonLinearity, parsFitNonLinearity,
830 tableArray[i, :] = linearizerTableRow
832 dataset.nonLinearity[ampName] = parsFitNonLinearity
833 dataset.nonLinearityError[ampName] = parsFitErrNonLinearity
834 dataset.nonLinearityResiduals[ampName] = linResidualNonLinearity
835 dataset.coefficientLinearizeSquared[ampName] = c0
839 def plot(self, dataRef, dataset, ptcFitType):
840 dirname = dataRef.getUri(datasetType=
'cpPipePlotRoot', write=
True)
841 if not os.path.exists(dirname):
844 detNum = dataRef.dataId[self.config.ccdKey]
845 filename = f
"PTC_det{detNum}.pdf" 846 filenameFull = os.path.join(dirname, filename)
847 with PdfPages(filenameFull)
as pdfPages:
848 self.
_plotPtc(dataset, ptcFitType, pdfPages)
850 def _plotPtc(self, dataset, ptcFitType, pdfPages):
851 """Plot PTC, linearity, and linearity residual per amplifier""" 853 if ptcFitType ==
'ASTIERAPPROXIMATION':
855 stringTitle =
r"Var = $\frac{1}{2g^2a_{00}}(\exp (2a_{00} \mu g) - 1) + \frac{n_{00}}{g^2}$" 857 if ptcFitType ==
'POLYNOMIAL':
859 stringTitle = f
"Polynomial (degree: {self.config.polynomialFitDegree})" 864 supTitleFontSize = 18
868 nAmps = len(dataset.ampNames)
871 nRows = np.sqrt(nAmps)
872 mantissa, _ = np.modf(nRows)
874 nRows = int(nRows) + 1
880 f, ax = plt.subplots(nrows=nRows, ncols=nCols, sharex=
'col', sharey=
'row', figsize=(13, 10))
881 f2, ax2 = plt.subplots(nrows=nRows, ncols=nCols, sharex=
'col', sharey=
'row', figsize=(13, 10))
883 for i, (amp, a, a2)
in enumerate(zip(dataset.ampNames, ax.flatten(), ax2.flatten())):
884 meanVecOriginal = np.array(dataset.rawMeans[amp])
885 varVecOriginal = np.array(dataset.rawVars[amp])
886 mask = dataset.visitMask[amp]
887 meanVecFinal = meanVecOriginal[mask]
888 varVecFinal = varVecOriginal[mask]
889 meanVecOutliers = meanVecOriginal[np.invert(mask)]
890 varVecOutliers = varVecOriginal[np.invert(mask)]
891 pars, parsErr = dataset.ptcFitPars[amp], dataset.ptcFitParsError[amp]
893 if ptcFitType ==
'ASTIERAPPROXIMATION':
894 ptcA00, ptcA00error = pars[0], parsErr[0]
895 ptcGain, ptcGainError = pars[1], parsErr[1]
896 ptcNoise = np.sqrt(np.fabs(pars[2]))
897 ptcNoiseError = 0.5*(parsErr[2]/np.fabs(pars[2]))*np.sqrt(np.fabs(pars[2]))
898 stringLegend = (f
"a00: {ptcA00:.2e}+/-{ptcA00error:.2e}" 899 f
"\n Gain: {ptcGain:.4}+/-{ptcGainError:.2e}" 900 f
"\n Noise: {ptcNoise:.4}+/-{ptcNoiseError:.2e}")
902 if ptcFitType ==
'POLYNOMIAL':
903 ptcGain, ptcGainError = 1./pars[1], np.fabs(1./pars[1])*(parsErr[1]/pars[1])
904 ptcNoise = np.sqrt(np.fabs(pars[0]))*ptcGain
905 ptcNoiseError = (0.5*(parsErr[0]/np.fabs(pars[0]))*(np.sqrt(np.fabs(pars[0]))))*ptcGain
906 stringLegend = (f
"Noise: {ptcNoise:.4}+/-{ptcNoiseError:.2e} \n" 907 f
"Gain: {ptcGain:.4}+/-{ptcGainError:.2e}")
909 minMeanVecFinal = np.min(meanVecFinal)
910 maxMeanVecFinal = np.max(meanVecFinal)
911 meanVecFit = np.linspace(minMeanVecFinal, maxMeanVecFinal, 100*len(meanVecFinal))
912 minMeanVecOriginal = np.min(meanVecOriginal)
913 maxMeanVecOriginal = np.max(meanVecOriginal)
914 deltaXlim = maxMeanVecOriginal - minMeanVecOriginal
916 a.plot(meanVecFit, ptcFunc(pars, meanVecFit), color=
'red')
917 a.plot(meanVecFinal, pars[0] + pars[1]*meanVecFinal, color=
'green', linestyle=
'--')
918 a.scatter(meanVecFinal, varVecFinal, c=
'blue', marker=
'o', s=markerSize)
919 a.scatter(meanVecOutliers, varVecOutliers, c=
'magenta', marker=
's', s=markerSize)
920 a.set_xlabel(
r'Mean signal ($\mu$, ADU)', fontsize=labelFontSize)
921 a.set_xticks(meanVecOriginal)
922 a.set_ylabel(
r'Variance (ADU$^2$)', fontsize=labelFontSize)
923 a.tick_params(labelsize=11)
924 a.text(0.03, 0.8, stringLegend, transform=a.transAxes, fontsize=legendFontSize)
925 a.set_xscale(
'linear', fontsize=labelFontSize)
926 a.set_yscale(
'linear', fontsize=labelFontSize)
927 a.set_title(amp, fontsize=titleFontSize)
928 a.set_xlim([minMeanVecOriginal - 0.2*deltaXlim, maxMeanVecOriginal + 0.2*deltaXlim])
931 a2.plot(meanVecFit, ptcFunc(pars, meanVecFit), color=
'red')
932 a2.scatter(meanVecFinal, varVecFinal, c=
'blue', marker=
'o', s=markerSize)
933 a2.scatter(meanVecOutliers, varVecOutliers, c=
'magenta', marker=
's', s=markerSize)
934 a2.set_xlabel(
r'Mean Signal ($\mu$, ADU)', fontsize=labelFontSize)
935 a2.set_ylabel(
r'Variance (ADU$^2$)', fontsize=labelFontSize)
936 a2.tick_params(labelsize=11)
937 a2.text(0.03, 0.8, stringLegend, transform=a2.transAxes, fontsize=legendFontSize)
940 a2.set_title(amp, fontsize=titleFontSize)
941 a2.set_xlim([minMeanVecOriginal, maxMeanVecOriginal])
943 f.suptitle(f
"PTC \n Fit: " + stringTitle, fontsize=20)
945 f2.suptitle(f
"PTC (log-log)", fontsize=20)
949 f, ax = plt.subplots(nrows=4, ncols=4, sharex=
'col', sharey=
'row', figsize=(13, 10))
950 for i, (amp, a)
in enumerate(zip(dataset.ampNames, ax.flatten())):
951 meanVecFinal = np.array(dataset.rawMeans[amp])[dataset.visitMask[amp]]
952 timeVecFinal = np.array(dataset.rawExpTimes[amp])[dataset.visitMask[amp]]
954 pars, parsErr = dataset.nonLinearity[amp], dataset.nonLinearityError[amp]
955 c0, c0Error = pars[0], parsErr[0]
956 c1, c1Error = pars[1], parsErr[1]
957 c2, c2Error = pars[2], parsErr[2]
958 stringLegend = f
"c0: {c0:.4}+/-{c0Error:.2e}\n c1: {c1:.4}+/-{c1Error:.2e}" \
959 + f
"\n c2(NL): {c2:.2e}+/-{c2Error:.2e}" 960 a.scatter(timeVecFinal, meanVecFinal)
961 a.plot(timeVecFinal, self.
funcPolynomial(pars, timeVecFinal), color=
'red')
962 a.set_xlabel(
'Time (sec)', fontsize=labelFontSize)
963 a.set_xticks(timeVecFinal)
964 a.set_ylabel(
r'Mean signal ($\mu$, ADU)', fontsize=labelFontSize)
965 a.tick_params(labelsize=labelFontSize)
966 a.text(0.03, 0.75, stringLegend, transform=a.transAxes, fontsize=legendFontSize)
967 a.set_xscale(
'linear', fontsize=labelFontSize)
968 a.set_yscale(
'linear', fontsize=labelFontSize)
969 a.set_title(amp, fontsize=titleFontSize)
971 f.suptitle(
"Linearity \n Fit: " +
r"$\mu = c_0 + c_1 t + c_2 t^2$", fontsize=supTitleFontSize)
975 f, ax = plt.subplots(nrows=4, ncols=4, sharex=
'col', sharey=
'row', figsize=(13, 10))
976 for i, (amp, a)
in enumerate(zip(dataset.ampNames, ax.flatten())):
977 meanVecFinal = np.array(dataset.rawMeans[amp])[dataset.visitMask[amp]]
978 linRes = np.array(dataset.nonLinearityResiduals[amp])
980 a.scatter(meanVecFinal, linRes)
981 a.axhline(y=0, color=
'k')
982 a.axvline(x=timeVecFinal[self.config.linResidualTimeIndex], color=
'g', linestyle=
'--')
983 a.set_xlabel(
r'Mean signal ($\mu$, ADU)', fontsize=labelFontSize)
984 a.set_xticks(meanVecFinal)
985 a.set_ylabel(
'LR (%)', fontsize=labelFontSize)
986 a.tick_params(labelsize=labelFontSize)
987 a.set_xscale(
'linear', fontsize=labelFontSize)
988 a.set_yscale(
'linear', fontsize=labelFontSize)
989 a.set_title(amp, fontsize=titleFontSize)
991 f.suptitle(
r"Linearity Residual: $100(1 - \mu_{\rm{ref}}/t_{\rm{ref}})/(\mu / t))$" +
"\n" +
992 r"$t_{\rm{ref}}$: " + f
"{timeVecFinal[2]} s", fontsize=supTitleFontSize)
def funcAstier(self, pars, x)
def _plotPtc(self, dataset, ptcFitType, pdfPages)
def _boundsForPolynomial(initialPars)
def __init__(self, ampNames)
def runDataRef(self, dataRef, visitPairs)
def calculateLinearityResidualAndLinearizers(self, exposureTimeVector, meanSignalVector)
def _boundsForAstier(initialPars)
def __setattr__(self, attribute, value)
def funcPolynomial(self, pars, x)
def checkExpLengthEqual(exp1, exp2, v1=None, v2=None, raiseWithMessage=False)
def fitPtcAndNonLinearity(self, dataset, tableArray, ptcFitType)
def validateIsrConfig(isrTask, mandatory=None, forbidden=None, desirable=None, undesirable=None, checkTrim=True, logName=None)
def _getInitialGoodPoints(means, variances, maxDeviationPositive, maxDeviationNegative)
def _fitBootstrap(self, initialParams, dataX, dataY, function, confidenceSigma=1.)
def getVisitsUsed(self, ampName)
def __init__(self, args, kwargs)
def measureMeanVarPair(self, exposure1, exposure2, region=None)
def _initialParsForPolynomial(order)
def _makeZeroSafe(self, array, warn=True, substituteValue=1e-9)
def _fitLeastSq(self, initialParams, dataX, dataY, function)
def plot(self, dataRef, dataset, ptcFitType)