23 __all__ = [
'MeasurePhotonTransferCurveTask',
24 'MeasurePhotonTransferCurveTaskConfig',
25 'PhotonTransferCurveDataset']
28 import matplotlib.pyplot
as plt
30 from matplotlib.backends.backend_pdf
import PdfPages
31 from sqlite3
import OperationalError
32 from collections
import Counter
35 import lsst.pex.config
as pexConfig
38 from .utils
import (NonexistentDatasetTaskDataIdContainer, PairedVisitListTaskRunner,
39 checkExpLengthEqual, validateIsrConfig)
40 from scipy.optimize
import leastsq, least_squares
41 import numpy.polynomial.polynomial
as poly
45 """Config class for photon transfer curve measurement task""" 46 isr = pexConfig.ConfigurableField(
48 doc=
"""Task to perform instrumental signature removal.""",
50 isrMandatorySteps = pexConfig.ListField(
52 doc=
"isr operations that must be performed for valid results. Raises if any of these are False.",
53 default=[
'doAssembleCcd']
55 isrForbiddenSteps = pexConfig.ListField(
57 doc=
"isr operations that must NOT be performed for valid results. Raises if any of these are True",
58 default=[
'doFlat',
'doFringe',
'doAddDistortionModel',
'doBrighterFatter',
'doUseOpticsTransmission',
59 'doUseFilterTransmission',
'doUseSensorTransmission',
'doUseAtmosphereTransmission']
61 isrDesirableSteps = pexConfig.ListField(
63 doc=
"isr operations that it is advisable to perform, but are not mission-critical." +
64 " WARNs are logged for any of these found to be False.",
65 default=[
'doBias',
'doDark',
'doCrosstalk',
'doDefect']
67 isrUndesirableSteps = pexConfig.ListField(
69 doc=
"isr operations that it is *not* advisable to perform in the general case, but are not" +
70 " forbidden as some use-cases might warrant them." +
71 " WARNs are logged for any of these found to be True.",
72 default=[
'doLinearize']
74 ccdKey = pexConfig.Field(
76 doc=
"The key by which to pull a detector from a dataId, e.g. 'ccd' or 'detector'.",
79 makePlots = pexConfig.Field(
81 doc=
"Plot the PTC curves?",
84 ptcFitType = pexConfig.ChoiceField(
86 doc=
"Fit PTC to approximation in Astier+19 (Equation 16) or to a polynomial.",
89 "POLYNOMIAL":
"n-degree polynomial (use 'polynomialFitDegree' to set 'n').",
90 "ASTIERAPPROXIMATION":
"Approximation in Astier+19 (Eq. 16)." 93 polynomialFitDegree = pexConfig.Field(
95 doc=
"Degree of polynomial to fit the PTC, when 'ptcFitType'=POLYNOMIAL.",
98 polynomialFitDegreeNonLinearity = pexConfig.Field(
100 doc=
"Degree of polynomial to fit the meanSignal vs exposureTime curve to produce" +
101 " the table for LinearizerLookupTable.",
104 binSize = pexConfig.Field(
106 doc=
"Bin the image by this factor in both dimensions.",
109 minMeanSignal = pexConfig.Field(
111 doc=
"Minimum value (inclusive) of mean signal (in ADU) above which to consider.",
114 maxMeanSignal = pexConfig.Field(
116 doc=
"Maximum value (inclusive) of mean signal (in ADU) below which to consider.",
119 initialNonLinearityExclusionThresholdPositive = pexConfig.RangeField(
121 doc=
"Initially exclude data points with a variance that are more than a factor of this from being" 122 " linear in the positive direction, from the PTC fit. Note that these points will also be" 123 " excluded from the non-linearity fit. This is done before the iterative outlier rejection," 124 " to allow an accurate determination of the sigmas for said iterative fit.",
129 initialNonLinearityExclusionThresholdNegative = pexConfig.RangeField(
131 doc=
"Initially exclude data points with a variance that are more than a factor of this from being" 132 " linear in the negative direction, from the PTC fit. Note that these points will also be" 133 " excluded from the non-linearity fit. This is done before the iterative outlier rejection," 134 " to allow an accurate determination of the sigmas for said iterative fit.",
139 sigmaCutPtcOutliers = pexConfig.Field(
141 doc=
"Sigma cut for outlier rejection in PTC.",
144 maxIterationsPtcOutliers = pexConfig.Field(
146 doc=
"Maximum number of iterations for outlier rejection in PTC.",
149 doFitBootstrap = pexConfig.Field(
151 doc=
"Use bootstrap for the PTC fit parameters and errors?.",
154 linResidualTimeIndex = pexConfig.Field(
156 doc=
"Index position in time array for reference time in linearity residual calculation.",
159 maxAduForLookupTableLinearizer = pexConfig.Field(
161 doc=
"Maximum ADU value for the LookupTable linearizer.",
167 """A simple class to hold the output data from the PTC task. 169 The dataset is made up of a dictionary for each item, keyed by the 170 amplifiers' names, which much be supplied at construction time. 172 New items cannot be added to the class to save accidentally saving to the 173 wrong property, and the class can be frozen if desired. 175 inputVisitPairs records the visits used to produce the data. 176 When fitPtcAndNonLinearity() is run, a mask is built up, which is by definition 177 always the same length as inputVisitPairs, rawExpTimes, rawMeans 178 and rawVars, and is a list of bools, which are incrementally set to False 179 as points are discarded from the fits. 181 PTC fit parameters for polynomials are stored in a list in ascending order 182 of polynomial term, i.e. par[0]*x^0 + par[1]*x + par[2]*x^2 etc 183 with the length of the list corresponding to the order of the polynomial 190 self.__dict__[
"ampNames"] = ampNames
191 self.__dict__[
"badAmps"] = []
194 self.__dict__[
"inputVisitPairs"] = {ampName: []
for ampName
in ampNames}
195 self.__dict__[
"visitMask"] = {ampName: []
for ampName
in ampNames}
196 self.__dict__[
"rawExpTimes"] = {ampName: []
for ampName
in ampNames}
197 self.__dict__[
"rawMeans"] = {ampName: []
for ampName
in ampNames}
198 self.__dict__[
"rawVars"] = {ampName: []
for ampName
in ampNames}
201 self.__dict__[
"ptcFitType"] = {ampName:
"" for ampName
in ampNames}
202 self.__dict__[
"ptcFitPars"] = {ampName: []
for ampName
in ampNames}
203 self.__dict__[
"ptcFitParsError"] = {ampName: []
for ampName
in ampNames}
204 self.__dict__[
"nonLinearity"] = {ampName: []
for ampName
in ampNames}
205 self.__dict__[
"nonLinearityError"] = {ampName: []
for ampName
in ampNames}
206 self.__dict__[
"nonLinearityResiduals"] = {ampName: []
for ampName
in ampNames}
207 self.__dict__[
"coefficientLinearizeSquared"] = {ampName: []
for ampName
in ampNames}
210 self.__dict__[
"gain"] = {ampName: -1.
for ampName
in ampNames}
211 self.__dict__[
"gainErr"] = {ampName: -1.
for ampName
in ampNames}
212 self.__dict__[
"noise"] = {ampName: -1.
for ampName
in ampNames}
213 self.__dict__[
"noiseErr"] = {ampName: -1.
for ampName
in ampNames}
214 self.__dict__[
"coefficientLinearizeSquared"] = {ampName: []
for ampName
in ampNames}
217 """Protect class attributes""" 218 if attribute
not in self.__dict__:
219 raise AttributeError(f
"{attribute} is not already a member of PhotonTransferCurveDataset, which" 220 " does not support setting of new attributes.")
222 self.__dict__[attribute] = value
225 """Get the visits used, i.e. not discarded, for a given amp. 227 If no mask has been created yet, all visits are returned. 229 if self.visitMask[ampName] == []:
230 return self.inputVisitPairs[ampName]
233 assert len(self.visitMask[ampName]) == len(self.inputVisitPairs[ampName])
235 pairs = self.inputVisitPairs[ampName]
236 mask = self.visitMask[ampName]
238 return [(v1, v2)
for ((v1, v2), m)
in zip(pairs, mask)
if bool(m)
is True]
241 return [amp
for amp
in self.ampNames
if amp
not in self.badAmps]
245 """A class to calculate, fit, and plot a PTC from a set of flat pairs. 247 The Photon Transfer Curve (var(signal) vs mean(signal)) is a standard tool 248 used in astronomical detectors characterization (e.g., Janesick 2001, 249 Janesick 2007). This task calculates the PTC from a series of pairs of 250 flat-field images; each pair taken at identical exposure times. The 251 difference image of each pair is formed to eliminate fixed pattern noise, 252 and then the variance of the difference image and the mean of the average image 253 are used to produce the PTC. An n-degree polynomial or the approximation in Equation 254 16 of Astier+19 ("The Shape of the Photon Transfer Curve of CCD sensors", 255 arXiv:1905.08677) can be fitted to the PTC curve. These models include 256 parameters such as the gain (e/ADU) and readout noise. 262 Positional arguments passed to the Task constructor. None used at this 265 Keyword arguments passed on to the Task constructor. None used at this 270 RunnerClass = PairedVisitListTaskRunner
271 ConfigClass = MeasurePhotonTransferCurveTaskConfig
272 _DefaultName =
"measurePhotonTransferCurve" 275 pipeBase.CmdLineTask.__init__(self, *args, **kwargs)
276 self.makeSubtask(
"isr")
277 plt.interactive(
False)
279 self.config.isrForbiddenSteps, self.config.isrDesirableSteps, checkTrim=
False)
280 self.config.validate()
284 def _makeArgumentParser(cls):
285 """Augment argument parser for the MeasurePhotonTransferCurveTask.""" 287 parser.add_argument(
"--visit-pairs", dest=
"visitPairs", nargs=
"*",
288 help=
"Visit pairs to use. Each pair must be of the form INT,INT e.g. 123,456")
289 parser.add_id_argument(
"--id", datasetType=
"photonTransferCurveDataset",
290 ContainerClass=NonexistentDatasetTaskDataIdContainer,
291 help=
"The ccds to use, e.g. --id ccd=0..100")
296 """Run the Photon Transfer Curve (PTC) measurement task. 298 For a dataRef (which is each detector here), 299 and given a list of visit pairs at different exposure times, 304 dataRef : list of lsst.daf.persistence.ButlerDataRef 305 dataRef for the detector for the visits to be fit. 306 visitPairs : `iterable` of `tuple` of `int` 307 Pairs of visit numbers to be processed together 311 detNum = dataRef.dataId[self.config.ccdKey]
312 detector = dataRef.get(
'camera')[dataRef.dataId[self.config.ccdKey]]
318 for name
in dataRef.getButler().getKeys(
'bias'):
319 if name
not in dataRef.dataId:
321 dataRef.dataId[name] = \
322 dataRef.getButler().queryMetadata(
'raw', [name], detector=detNum)[0]
323 except OperationalError:
326 amps = detector.getAmplifiers()
327 ampNames = [amp.getName()
for amp
in amps]
330 self.log.info(
'Measuring PTC using %s visits for detector %s' % (visitPairs, detNum))
332 for (v1, v2)
in visitPairs:
334 dataRef.dataId[
'visit'] = v1
336 dataRef.dataId[
'visit'] = v2
338 del dataRef.dataId[
'visit']
341 expTime = exp1.getInfo().getVisitInfo().getExposureTime()
345 ampName = amp.getName()
347 dataset.rawExpTimes[ampName].append(expTime)
348 dataset.rawMeans[ampName].append(mu)
349 dataset.rawVars[ampName].append(varDiff)
350 dataset.inputVisitPairs[ampName].append((v1, v2))
352 numberAmps = len(detector.getAmplifiers())
353 numberAduValues = self.config.maxAduForLookupTableLinearizer
354 lookupTableArray = np.zeros((numberAmps, numberAduValues), dtype=np.float32)
359 if self.config.makePlots:
360 self.
plot(dataRef, dataset, ptcFitType=self.config.ptcFitType)
363 self.log.info(f
"Writing PTC and NL data to {dataRef.getUri(write=True)}")
364 dataRef.put(dataset, datasetType=
"photonTransferCurveDataset")
366 self.log.info(
'Finished measuring PTC for in detector %s' % detNum)
368 return pipeBase.Struct(exitStatus=0)
371 """Calculate the mean signal of two exposures and the variance of their difference. 375 exposure1 : `lsst.afw.image.exposure.exposure.ExposureF` 376 First exposure of flat field pair. 378 exposure2 : `lsst.afw.image.exposure.exposure.ExposureF` 379 Second exposure of flat field pair. 381 region : `lsst.geom.Box2I` 382 Region of each exposure where to perform the calculations (e.g, an amplifier). 388 0.5*(mu1 + mu2), where mu1, and mu2 are the clipped means of the regions in 392 Half of the clipped variance of the difference of the regions inthe two input 396 if region
is not None:
397 im1Area = exposure1.maskedImage[region]
398 im2Area = exposure2.maskedImage[region]
400 im1Area = exposure1.maskedImage
401 im2Area = exposure2.maskedImage
403 im1Area = afwMath.binImage(im1Area, self.config.binSize)
404 im2Area = afwMath.binImage(im2Area, self.config.binSize)
407 mu1 = afwMath.makeStatistics(im1Area, afwMath.MEANCLIP).getValue()
408 mu2 = afwMath.makeStatistics(im2Area, afwMath.MEANCLIP).getValue()
413 temp = im2Area.clone()
415 diffIm = im1Area.clone()
420 varDiff = 0.5*(afwMath.makeStatistics(diffIm, afwMath.VARIANCECLIP).getValue())
424 def _fitLeastSq(self, initialParams, dataX, dataY, function):
425 """Do a fit and estimate the parameter errors using using scipy.optimize.leastq. 427 optimize.leastsq returns the fractional covariance matrix. To estimate the 428 standard deviation of the fit parameters, multiply the entries of this matrix 429 by the reduced chi squared and take the square root of the diagonal elements. 433 initialParams : list of np.float 434 initial values for fit parameters. For ptcFitType=POLYNOMIAL, its length 435 determines the degree of the polynomial. 437 dataX : np.array of np.float 438 Data in the abscissa axis. 440 dataY : np.array of np.float 441 Data in the ordinate axis. 443 function : callable object (function) 444 Function to fit the data with. 448 pFitSingleLeastSquares : list of np.float 449 List with fitted parameters. 451 pErrSingleLeastSquares : list of np.float 452 List with errors for fitted parameters. 455 def errFunc(p, x, y):
456 return function(p, x) - y
458 pFit, pCov, infoDict, errMessage, success = leastsq(errFunc, initialParams,
459 args=(dataX, dataY), full_output=1, epsfcn=0.0001)
461 if (len(dataY) > len(initialParams))
and pCov
is not None:
462 reducedChiSq = (errFunc(pFit, dataX, dataY)**2).sum()/(len(dataY)-len(initialParams))
468 for i
in range(len(pFit)):
469 errorVec.append(np.fabs(pCov[i][i])**0.5)
471 pFitSingleLeastSquares = pFit
472 pErrSingleLeastSquares = np.array(errorVec)
474 return pFitSingleLeastSquares, pErrSingleLeastSquares
476 def _fitBootstrap(self, initialParams, dataX, dataY, function, confidenceSigma=1.):
477 """Do a fit using least squares and bootstrap to estimate parameter errors. 479 The bootstrap error bars are calculated by fitting 100 random data sets. 483 initialParams : list of np.float 484 initial values for fit parameters. For ptcFitType=POLYNOMIAL, its length 485 determines the degree of the polynomial. 487 dataX : np.array of np.float 488 Data in the abscissa axis. 490 dataY : np.array of np.float 491 Data in the ordinate axis. 493 function : callable object (function) 494 Function to fit the data with. 496 confidenceSigma : np.float 497 Number of sigmas that determine confidence interval for the bootstrap errors. 501 pFitBootstrap : list of np.float 502 List with fitted parameters. 504 pErrBootstrap : list of np.float 505 List with errors for fitted parameters. 508 def errFunc(p, x, y):
509 return function(p, x) - y
512 pFit, _ = leastsq(errFunc, initialParams, args=(dataX, dataY), full_output=0)
515 residuals = errFunc(pFit, dataX, dataY)
516 sigmaErrTotal = np.std(residuals)
521 randomDelta = np.random.normal(0., sigmaErrTotal, len(dataY))
522 randomDataY = dataY + randomDelta
523 randomFit, _ = leastsq(errFunc, initialParams,
524 args=(dataX, randomDataY), full_output=0)
525 pars.append(randomFit)
526 pars = np.array(pars)
527 meanPfit = np.mean(pars, 0)
530 nSigma = confidenceSigma
531 errPfit = nSigma*np.std(pars, 0)
532 pFitBootstrap = meanPfit
533 pErrBootstrap = errPfit
534 return pFitBootstrap, pErrBootstrap
537 """Polynomial function definition""" 538 return poly.polyval(x, [*pars])
541 """Single brighter-fatter parameter model for PTC; Equation 16 of Astier+19""" 542 a00, gain, noise = pars
543 return 0.5/(a00*gain*gain)*(np.exp(2*a00*x*gain)-1) + noise/(gain*gain)
546 def _initialParsForPolynomial(order):
548 pars = np.zeros(order, dtype=np.float)
555 def _boundsForPolynomial(initialPars):
556 lowers = [np.NINF
for p
in initialPars]
557 uppers = [np.inf
for p
in initialPars]
559 return (lowers, uppers)
562 def _boundsForAstier(initialPars):
563 lowers = [np.NINF
for p
in initialPars]
564 uppers = [np.inf
for p
in initialPars]
565 return (lowers, uppers)
568 def _getInitialGoodPoints(means, variances, maxDeviationPositive, maxDeviationNegative):
569 """Return a boolean array to mask bad points. 571 A linear function has a constant ratio, so find the median 572 value of the ratios, and exclude the points that deviate 573 from that by more than a factor of maxDeviationPositive/negative. 574 Asymmetric deviations are supported as we expect the PTC to turn 575 down as the flux increases, but sometimes it anomalously turns 576 upwards just before turning over, which ruins the fits, so it 577 is wise to be stricter about restricting positive outliers than 580 Too high and points that are so bad that fit will fail will be included 581 Too low and the non-linear points will be excluded, biasing the NL fit.""" 582 ratios = [b/a
for (a, b)
in zip(means, variances)]
583 medianRatio = np.median(ratios)
584 ratioDeviations = [(r/medianRatio)-1
for r
in ratios]
587 maxDeviationPositive = abs(maxDeviationPositive)
588 maxDeviationNegative = -1. * abs(maxDeviationNegative)
590 goodPoints = np.array([
True if (r < maxDeviationPositive
and r > maxDeviationNegative)
591 else False for r
in ratioDeviations])
594 def _makeZeroSafe(self, array, warn=True, substituteValue=1e-9):
596 nBad = Counter(array)[0]
601 msg = f
"Found {nBad} zeros in array at elements {[x for x in np.where(array==0)[0]]}" 604 array[array == 0] = substituteValue
608 """Calculate linearity residual and fit an n-order polynomial to the mean vs time curve 609 to produce corrections (deviation from linear part of polynomial) for a particular amplifier 610 to populate LinearizeLookupTable. Use quadratic and linear parts of this polynomial to approximate 611 c0 for LinearizeSquared." 616 exposureTimeVector: `list` of `np.float` 617 List of exposure times for each flat pair 619 meanSignalVector: `list` of `np.float` 620 List of mean signal from diference image of flat pairs 625 Coefficient for LinearizeSquared, where corrImage = uncorrImage + c0*uncorrImage^2. 626 c0 ~ -k2/(k1^2), where k1 and k2 are fit from 627 meanSignalVector = k0 + k1*exposureTimeVector + k2*exposureTimeVector^2 +... 628 + kn*exposureTimeVector^n, with n = "polynomialFitDegreeNonLinearity". 630 linearizerTableRow: list of `np.float` 631 One dimensional array with deviation from linear part of n-order polynomial fit 632 to mean vs time curve. This array will be one row (for the particular amplifier at hand) 633 of the table array for LinearizeLookupTable. 635 linResidual: list of `np.float` 636 Linearity residual from the mean vs time curve, defined as 637 100*(1 - meanSignalReference/expTimeReference/(meanSignal/expTime). 639 parsFit: list of `np.float` 640 Parameters from n-order polynomial fit to mean vs time curve. 642 parsFitErr: list of `np.float` 643 Parameters from n-order polynomial fit to mean vs time curve. 649 if self.config.doFitBootstrap:
650 parsFit, parsFitErr = self.
_fitBootstrap(parsIniNonLinearity, exposureTimeVector,
653 parsFit, parsFitErr = self.
_fitLeastSq(parsIniNonLinearity, exposureTimeVector, meanSignalVector,
657 tMax = (self.config.maxAduForLookupTableLinearizer - parsFit[0])/parsFit[1]
658 timeRange = np.linspace(0, tMax, self.config.maxAduForLookupTableLinearizer)
659 signalIdeal = parsFit[0] + parsFit[1]*timeRange
661 linearizerTableRow = signalIdeal - signalUncorrected
667 k1, k2 = parsFit[1], parsFit[2]
669 for coefficient
in parsFit[3:]:
670 if np.fabs(coefficient) > 1e-10:
671 msg = f
"Coefficient {coefficient} in polynomial fit larger than threshold 1e-10." 675 linResidualTimeIndex = self.config.linResidualTimeIndex
676 if exposureTimeVector[linResidualTimeIndex] == 0.0:
677 raise RuntimeError(
"Reference time for linearity residual can't be 0.0")
678 linResidual = 100*(1 - ((meanSignalVector[linResidualTimeIndex] /
679 exposureTimeVector[linResidualTimeIndex]) /
680 (meanSignalVector/exposureTimeVector)))
682 return c0, linearizerTableRow, linResidual, parsFit, parsFitErr
685 """Fit the photon transfer curve and calculate linearity and residuals. 687 Fit the photon transfer curve with either a polynomial of the order 688 specified in the task config, or using the Astier approximation. 690 Sigma clipping is performed iteratively for the fit, as well as an 691 initial clipping of data points that are more than 692 config.initialNonLinearityExclusionThreshold away from lying on a 693 straight line. This other step is necessary because the photon transfer 694 curve turns over catastrophically at very high flux (because saturation 695 drops the variance to ~0) and these far outliers cause the initial fit 696 to fail, meaning the sigma cannot be calculated to perform the 701 dataset : `lsst.cp.pipe.ptc.PhotonTransferCurveDataset` 702 The dataset containing the means, variances and exposure times 704 Fit a 'POLYNOMIAL' (degree: 'polynomialFitDegree') or 705 'ASTIERAPPROXIMATION' to the PTC 706 tableArray : `np.array` 707 Look-up table array with size rows=nAmps and columns=ADU values 710 def errFunc(p, x, y):
711 return ptcFunc(p, x) - y
713 sigmaCutPtcOutliers = self.config.sigmaCutPtcOutliers
714 maxIterationsPtcOutliers = self.config.maxIterationsPtcOutliers
716 for i, ampName
in enumerate(dataset.ampNames):
717 timeVecOriginal = np.array(dataset.rawExpTimes[ampName])
718 meanVecOriginal = np.array(dataset.rawMeans[ampName])
719 varVecOriginal = np.array(dataset.rawVars[ampName])
722 mask = ((meanVecOriginal >= self.config.minMeanSignal) &
723 (meanVecOriginal <= self.config.maxMeanSignal))
726 self.config.initialNonLinearityExclusionThresholdPositive,
727 self.config.initialNonLinearityExclusionThresholdNegative)
728 mask = mask & goodPoints
730 if ptcFitType ==
'ASTIERAPPROXIMATION':
732 parsIniPtc = [-1e-9, 1.0, 10.]
734 if ptcFitType ==
'POLYNOMIAL':
741 while count <= maxIterationsPtcOutliers:
745 meanTempVec = meanVecOriginal[mask]
746 varTempVec = varVecOriginal[mask]
747 res = least_squares(errFunc, parsIniPtc, bounds=bounds, args=(meanTempVec, varTempVec))
753 sigResids = (varVecOriginal - ptcFunc(pars, meanVecOriginal))/np.sqrt(varVecOriginal)
754 newMask = np.array([
True if np.abs(r) < sigmaCutPtcOutliers
else False for r
in sigResids])
755 mask = mask & newMask
757 nDroppedTotal = Counter(mask)[
False]
758 self.log.debug(f
"Iteration {count}: discarded {nDroppedTotal} points in total for {ampName}")
761 assert (len(mask) == len(timeVecOriginal) == len(meanVecOriginal) == len(varVecOriginal))
763 dataset.visitMask[ampName] = mask
766 timeVecFinal = timeVecOriginal[mask]
767 meanVecFinal = meanVecOriginal[mask]
768 varVecFinal = varVecOriginal[mask]
770 if Counter(mask)[
False] > 0:
771 self.log.info((f
"Number of points discarded in PTC of amplifier {ampName}:" +
772 f
" {Counter(mask)[False]} out of {len(meanVecOriginal)}"))
774 if (len(meanVecFinal) < len(parsIniPtc)):
775 msg = (f
"\nSERIOUS: Not enough data points ({len(meanVecFinal)}) compared to the number of" 776 f
"parameters of the PTC model({len(parsIniPtc)}). Setting {ampName} to BAD.")
778 dataset.badAmps.append(ampName)
779 dataset.gain[ampName] = np.nan
780 dataset.gainErr[ampName] = np.nan
781 dataset.noise[ampName] = np.nan
782 dataset.noiseErr[ampName] = np.nan
783 dataset.nonLinearity[ampName] = np.nan
784 dataset.nonLinearityError[ampName] = np.nan
785 dataset.nonLinearityResiduals[ampName] = np.nan
789 if self.config.doFitBootstrap:
790 parsFit, parsFitErr = self.
_fitBootstrap(parsIniPtc, meanVecFinal, varVecFinal, ptcFunc)
792 parsFit, parsFitErr = self.
_fitLeastSq(parsIniPtc, meanVecFinal, varVecFinal, ptcFunc)
794 dataset.ptcFitPars[ampName] = parsFit
795 dataset.ptcFitParsError[ampName] = parsFitErr
797 if ptcFitType ==
'ASTIERAPPROXIMATION':
799 ptcGainErr = parsFitErr[1]
800 ptcNoise = np.sqrt(np.fabs(parsFit[2]))
801 ptcNoiseErr = 0.5*(parsFitErr[2]/np.fabs(parsFit[2]))*np.sqrt(np.fabs(parsFit[2]))
802 if ptcFitType ==
'POLYNOMIAL':
803 ptcGain = 1./parsFit[1]
804 ptcGainErr = np.fabs(1./parsFit[1])*(parsFitErr[1]/parsFit[1])
805 ptcNoise = np.sqrt(np.fabs(parsFit[0]))*ptcGain
806 ptcNoiseErr = (0.5*(parsFitErr[0]/np.fabs(parsFit[0]))*(np.sqrt(np.fabs(parsFit[0]))))*ptcGain
808 dataset.gain[ampName] = ptcGain
809 dataset.gainErr[ampName] = ptcGainErr
810 dataset.noise[ampName] = ptcNoise
811 dataset.noiseErr[ampName] = ptcNoiseErr
812 dataset.ptcFitType[ampName] = ptcFitType
817 (c0, linearizerTableRow, linResidualNonLinearity, parsFitNonLinearity,
821 tableArray[i, :] = linearizerTableRow
823 dataset.nonLinearity[ampName] = parsFitNonLinearity
824 dataset.nonLinearityError[ampName] = parsFitErrNonLinearity
825 dataset.nonLinearityResiduals[ampName] = linResidualNonLinearity
826 dataset.coefficientLinearizeSquared[ampName] = c0
830 def plot(self, dataRef, dataset, ptcFitType):
831 dirname = dataRef.getUri(datasetType=
'cpPipePlotRoot', write=
True)
832 if not os.path.exists(dirname):
835 detNum = dataRef.dataId[self.config.ccdKey]
836 filename = f
"PTC_det{detNum}.pdf" 837 filenameFull = os.path.join(dirname, filename)
838 with PdfPages(filenameFull)
as pdfPages:
839 self.
_plotPtc(dataset, ptcFitType, pdfPages)
841 def _plotPtc(self, dataset, ptcFitType, pdfPages):
842 """Plot PTC, linearity, and linearity residual per amplifier""" 844 if ptcFitType ==
'ASTIERAPPROXIMATION':
846 stringTitle =
r"Var = $\frac{1}{2g^2a_{00}}(\exp (2a_{00} \mu g) - 1) + \frac{n_{00}}{g^2}$" 848 if ptcFitType ==
'POLYNOMIAL':
850 stringTitle = f
"Polynomial (degree: {self.config.polynomialFitDegree})" 855 supTitleFontSize = 18
859 nAmps = len(dataset.ampNames)
862 nRows = np.sqrt(nAmps)
863 mantissa, _ = np.modf(nRows)
865 nRows = int(nRows) + 1
871 f, ax = plt.subplots(nrows=nRows, ncols=nCols, sharex=
'col', sharey=
'row', figsize=(13, 10))
872 f2, ax2 = plt.subplots(nrows=nRows, ncols=nCols, sharex=
'col', sharey=
'row', figsize=(13, 10))
874 for i, (amp, a, a2)
in enumerate(zip(dataset.ampNames, ax.flatten(), ax2.flatten())):
875 meanVecOriginal = np.array(dataset.rawMeans[amp])
876 varVecOriginal = np.array(dataset.rawVars[amp])
877 mask = dataset.visitMask[amp]
878 meanVecFinal = meanVecOriginal[mask]
879 varVecFinal = varVecOriginal[mask]
880 meanVecOutliers = meanVecOriginal[np.invert(mask)]
881 varVecOutliers = varVecOriginal[np.invert(mask)]
882 pars, parsErr = dataset.ptcFitPars[amp], dataset.ptcFitParsError[amp]
884 if ptcFitType ==
'ASTIERAPPROXIMATION':
885 ptcA00, ptcA00error = pars[0], parsErr[0]
886 ptcGain, ptcGainError = pars[1], parsErr[1]
887 ptcNoise = np.sqrt(np.fabs(pars[2]))
888 ptcNoiseError = 0.5*(parsErr[2]/np.fabs(pars[2]))*np.sqrt(np.fabs(pars[2]))
889 stringLegend = (f
"a00: {ptcA00:.2e}+/-{ptcA00error:.2e}" 890 f
"\n Gain: {ptcGain:.4}+/-{ptcGainError:.2e}" 891 f
"\n Noise: {ptcNoise:.4}+/-{ptcNoiseError:.2e}")
893 if ptcFitType ==
'POLYNOMIAL':
894 ptcGain, ptcGainError = 1./pars[1], np.fabs(1./pars[1])*(parsErr[1]/pars[1])
895 ptcNoise = np.sqrt(np.fabs(pars[0]))*ptcGain
896 ptcNoiseError = (0.5*(parsErr[0]/np.fabs(pars[0]))*(np.sqrt(np.fabs(pars[0]))))*ptcGain
897 stringLegend = (f
"Noise: {ptcNoise:.4}+/-{ptcNoiseError:.2e} \n" 898 f
"Gain: {ptcGain:.4}+/-{ptcGainError:.2e}")
900 minMeanVecFinal = np.min(meanVecFinal)
901 maxMeanVecFinal = np.max(meanVecFinal)
902 meanVecFit = np.linspace(minMeanVecFinal, maxMeanVecFinal, 100*len(meanVecFinal))
903 minMeanVecOriginal = np.min(meanVecOriginal)
904 maxMeanVecOriginal = np.max(meanVecOriginal)
905 deltaXlim = maxMeanVecOriginal - minMeanVecOriginal
907 a.plot(meanVecFit, ptcFunc(pars, meanVecFit), color=
'red')
908 a.plot(meanVecFinal, pars[0] + pars[1]*meanVecFinal, color=
'green', linestyle=
'--')
909 a.scatter(meanVecFinal, varVecFinal, c=
'blue', marker=
'o', s=markerSize)
910 a.scatter(meanVecOutliers, varVecOutliers, c=
'magenta', marker=
's', s=markerSize)
911 a.set_xlabel(
r'Mean signal ($\mu$, ADU)', fontsize=labelFontSize)
912 a.set_xticks(meanVecOriginal)
913 a.set_ylabel(
r'Variance (ADU$^2$)', fontsize=labelFontSize)
914 a.tick_params(labelsize=11)
915 a.text(0.03, 0.8, stringLegend, transform=a.transAxes, fontsize=legendFontSize)
916 a.set_xscale(
'linear', fontsize=labelFontSize)
917 a.set_yscale(
'linear', fontsize=labelFontSize)
918 a.set_title(amp, fontsize=titleFontSize)
919 a.set_xlim([minMeanVecOriginal - 0.2*deltaXlim, maxMeanVecOriginal + 0.2*deltaXlim])
922 a2.plot(meanVecFit, ptcFunc(pars, meanVecFit), color=
'red')
923 a2.scatter(meanVecFinal, varVecFinal, c=
'blue', marker=
'o', s=markerSize)
924 a2.scatter(meanVecOutliers, varVecOutliers, c=
'magenta', marker=
's', s=markerSize)
925 a2.set_xlabel(
r'Mean Signal ($\mu$, ADU)', fontsize=labelFontSize)
926 a2.set_ylabel(
r'Variance (ADU$^2$)', fontsize=labelFontSize)
927 a2.tick_params(labelsize=11)
928 a2.text(0.03, 0.8, stringLegend, transform=a2.transAxes, fontsize=legendFontSize)
931 a2.set_title(amp, fontsize=titleFontSize)
932 a2.set_xlim([minMeanVecOriginal, maxMeanVecOriginal])
934 f.suptitle(f
"PTC \n Fit: " + stringTitle, fontsize=20)
936 f2.suptitle(f
"PTC (log-log)", fontsize=20)
940 f, ax = plt.subplots(nrows=4, ncols=4, sharex=
'col', sharey=
'row', figsize=(13, 10))
941 for i, (amp, a)
in enumerate(zip(dataset.ampNames, ax.flatten())):
942 meanVecFinal = np.array(dataset.rawMeans[amp])[dataset.visitMask[amp]]
943 timeVecFinal = np.array(dataset.rawExpTimes[amp])[dataset.visitMask[amp]]
945 pars, parsErr = dataset.nonLinearity[amp], dataset.nonLinearityError[amp]
946 c0, c0Error = pars[0], parsErr[0]
947 c1, c1Error = pars[1], parsErr[1]
948 c2, c2Error = pars[2], parsErr[2]
949 stringLegend = f
"c0: {c0:.4}+/-{c0Error:.2e}\n c1: {c1:.4}+/-{c1Error:.2e}" \
950 + f
"\n c2(NL): {c2:.2e}+/-{c2Error:.2e}" 951 a.scatter(timeVecFinal, meanVecFinal)
952 a.plot(timeVecFinal, self.
funcPolynomial(pars, timeVecFinal), color=
'red')
953 a.set_xlabel(
'Time (sec)', fontsize=labelFontSize)
954 a.set_xticks(timeVecFinal)
955 a.set_ylabel(
r'Mean signal ($\mu$, ADU)', fontsize=labelFontSize)
956 a.tick_params(labelsize=labelFontSize)
957 a.text(0.03, 0.75, stringLegend, transform=a.transAxes, fontsize=legendFontSize)
958 a.set_xscale(
'linear', fontsize=labelFontSize)
959 a.set_yscale(
'linear', fontsize=labelFontSize)
960 a.set_title(amp, fontsize=titleFontSize)
962 f.suptitle(
"Linearity \n Fit: " +
r"$\mu = c_0 + c_1 t + c_2 t^2$", fontsize=supTitleFontSize)
966 f, ax = plt.subplots(nrows=4, ncols=4, sharex=
'col', sharey=
'row', figsize=(13, 10))
967 for i, (amp, a)
in enumerate(zip(dataset.ampNames, ax.flatten())):
968 meanVecFinal = np.array(dataset.rawMeans[amp])[dataset.visitMask[amp]]
969 linRes = np.array(dataset.nonLinearityResiduals[amp])
971 a.scatter(meanVecFinal, linRes)
972 a.axhline(y=0, color=
'k')
973 a.axvline(x=timeVecFinal[self.config.linResidualTimeIndex], color=
'g', linestyle=
'--')
974 a.set_xlabel(
r'Mean signal ($\mu$, ADU)', fontsize=labelFontSize)
975 a.set_xticks(meanVecFinal)
976 a.set_ylabel(
'LR (%)', fontsize=labelFontSize)
977 a.tick_params(labelsize=labelFontSize)
978 a.set_xscale(
'linear', fontsize=labelFontSize)
979 a.set_yscale(
'linear', fontsize=labelFontSize)
980 a.set_title(amp, fontsize=titleFontSize)
982 f.suptitle(
r"Linearity Residual: $100(1 - \mu_{\rm{ref}}/t_{\rm{ref}})/(\mu / t))$" +
"\n" +
983 r"$t_{\rm{ref}}$: " + f
"{timeVecFinal[2]} s", fontsize=supTitleFontSize)
def funcAstier(self, pars, x)
def _plotPtc(self, dataset, ptcFitType, pdfPages)
def _boundsForPolynomial(initialPars)
def __init__(self, ampNames)
def runDataRef(self, dataRef, visitPairs)
def calculateLinearityResidualAndLinearizers(self, exposureTimeVector, meanSignalVector)
def _boundsForAstier(initialPars)
def __setattr__(self, attribute, value)
def funcPolynomial(self, pars, x)
def checkExpLengthEqual(exp1, exp2, v1=None, v2=None, raiseWithMessage=False)
def fitPtcAndNonLinearity(self, dataset, tableArray, ptcFitType)
def validateIsrConfig(isrTask, mandatory=None, forbidden=None, desirable=None, undesirable=None, checkTrim=True, logName=None)
def _getInitialGoodPoints(means, variances, maxDeviationPositive, maxDeviationNegative)
def _fitBootstrap(self, initialParams, dataX, dataY, function, confidenceSigma=1.)
def getVisitsUsed(self, ampName)
def __init__(self, args, kwargs)
def measureMeanVarPair(self, exposure1, exposure2, region=None)
def _initialParsForPolynomial(order)
def _makeZeroSafe(self, array, warn=True, substituteValue=1e-9)
def _fitLeastSq(self, initialParams, dataX, dataY, function)
def plot(self, dataRef, dataset, ptcFitType)