23 __all__ = [
'MeasurePhotonTransferCurveTask',
24 'MeasurePhotonTransferCurveTaskConfig',
25 'PhotonTransferCurveDataset']
28 import matplotlib.pyplot
as plt
30 from matplotlib.backends.backend_pdf
import PdfPages
31 from sqlite3
import OperationalError
32 from collections
import Counter
35 import lsst.pex.config
as pexConfig
38 from .utils
import (NonexistentDatasetTaskDataIdContainer, PairedVisitListTaskRunner,
39 checkExpLengthEqual, validateIsrConfig)
40 from scipy.optimize
import leastsq, least_squares
41 import numpy.polynomial.polynomial
as poly
48 """Config class for photon transfer curve measurement task""" 49 isr = pexConfig.ConfigurableField(
51 doc=
"""Task to perform instrumental signature removal.""",
53 isrMandatorySteps = pexConfig.ListField(
55 doc=
"isr operations that must be performed for valid results. Raises if any of these are False.",
56 default=[
'doAssembleCcd']
58 isrForbiddenSteps = pexConfig.ListField(
60 doc=
"isr operations that must NOT be performed for valid results. Raises if any of these are True",
61 default=[
'doFlat',
'doFringe',
'doAddDistortionModel',
'doBrighterFatter',
'doUseOpticsTransmission',
62 'doUseFilterTransmission',
'doUseSensorTransmission',
'doUseAtmosphereTransmission']
64 isrDesirableSteps = pexConfig.ListField(
66 doc=
"isr operations that it is advisable to perform, but are not mission-critical." +
67 " WARNs are logged for any of these found to be False.",
68 default=[
'doBias',
'doDark',
'doCrosstalk',
'doDefect']
70 isrUndesirableSteps = pexConfig.ListField(
72 doc=
"isr operations that it is *not* advisable to perform in the general case, but are not" +
73 " forbidden as some use-cases might warrant them." +
74 " WARNs are logged for any of these found to be True.",
75 default=[
'doLinearize']
77 ccdKey = pexConfig.Field(
79 doc=
"The key by which to pull a detector from a dataId, e.g. 'ccd' or 'detector'.",
82 makePlots = pexConfig.Field(
84 doc=
"Plot the PTC curves?",
87 ptcFitType = pexConfig.ChoiceField(
89 doc=
"Fit PTC to approximation in Astier+19 (Equation 16) or to a polynomial.",
92 "POLYNOMIAL":
"n-degree polynomial (use 'polynomialFitDegree' to set 'n').",
93 "ASTIERAPPROXIMATION":
"Approximation in Astier+19 (Eq. 16)." 96 polynomialFitDegree = pexConfig.Field(
98 doc=
"Degree of polynomial to fit the PTC, when 'ptcFitType'=POLYNOMIAL.",
101 polynomialFitDegreeNonLinearity = pexConfig.Field(
103 doc=
"Degree of polynomial to fit the meanSignal vs exposureTime curve to produce" +
104 " the table for LinearizeLookupTable.",
107 binSize = pexConfig.Field(
109 doc=
"Bin the image by this factor in both dimensions.",
112 minMeanSignal = pexConfig.Field(
114 doc=
"Minimum value (inclusive) of mean signal (in DN) above which to consider.",
117 maxMeanSignal = pexConfig.Field(
119 doc=
"Maximum value (inclusive) of mean signal (in DN) below which to consider.",
122 initialNonLinearityExclusionThresholdPositive = pexConfig.RangeField(
124 doc=
"Initially exclude data points with a variance that are more than a factor of this from being" 125 " linear in the positive direction, from the PTC fit. Note that these points will also be" 126 " excluded from the non-linearity fit. This is done before the iterative outlier rejection," 127 " to allow an accurate determination of the sigmas for said iterative fit.",
132 initialNonLinearityExclusionThresholdNegative = pexConfig.RangeField(
134 doc=
"Initially exclude data points with a variance that are more than a factor of this from being" 135 " linear in the negative direction, from the PTC fit. Note that these points will also be" 136 " excluded from the non-linearity fit. This is done before the iterative outlier rejection," 137 " to allow an accurate determination of the sigmas for said iterative fit.",
142 sigmaCutPtcOutliers = pexConfig.Field(
144 doc=
"Sigma cut for outlier rejection in PTC.",
147 maxIterationsPtcOutliers = pexConfig.Field(
149 doc=
"Maximum number of iterations for outlier rejection in PTC.",
152 doFitBootstrap = pexConfig.Field(
154 doc=
"Use bootstrap for the PTC fit parameters and errors?.",
157 linResidualTimeIndex = pexConfig.Field(
159 doc=
"Index position in time array for reference time in linearity residual calculation.",
162 maxAduForLookupTableLinearizer = pexConfig.Field(
164 doc=
"Maximum DN value for the LookupTable linearizer.",
167 instrumentName = pexConfig.Field(
169 doc=
"Instrument name.",
175 """A simple class to hold the output data from the PTC task. 177 The dataset is made up of a dictionary for each item, keyed by the 178 amplifiers' names, which much be supplied at construction time. 180 New items cannot be added to the class to save accidentally saving to the 181 wrong property, and the class can be frozen if desired. 183 inputVisitPairs records the visits used to produce the data. 184 When fitPtcAndNonLinearity() is run, a mask is built up, which is by definition 185 always the same length as inputVisitPairs, rawExpTimes, rawMeans 186 and rawVars, and is a list of bools, which are incrementally set to False 187 as points are discarded from the fits. 189 PTC fit parameters for polynomials are stored in a list in ascending order 190 of polynomial term, i.e. par[0]*x^0 + par[1]*x + par[2]*x^2 etc 191 with the length of the list corresponding to the order of the polynomial 198 self.__dict__[
"ampNames"] = ampNames
199 self.__dict__[
"badAmps"] = []
202 self.__dict__[
"inputVisitPairs"] = {ampName: []
for ampName
in ampNames}
203 self.__dict__[
"visitMask"] = {ampName: []
for ampName
in ampNames}
204 self.__dict__[
"rawExpTimes"] = {ampName: []
for ampName
in ampNames}
205 self.__dict__[
"rawMeans"] = {ampName: []
for ampName
in ampNames}
206 self.__dict__[
"rawVars"] = {ampName: []
for ampName
in ampNames}
209 self.__dict__[
"ptcFitType"] = {ampName:
"" for ampName
in ampNames}
210 self.__dict__[
"ptcFitPars"] = {ampName: []
for ampName
in ampNames}
211 self.__dict__[
"ptcFitParsError"] = {ampName: []
for ampName
in ampNames}
212 self.__dict__[
"ptcFitReducedChiSquared"] = {ampName: []
for ampName
in ampNames}
213 self.__dict__[
"nonLinearity"] = {ampName: []
for ampName
in ampNames}
214 self.__dict__[
"nonLinearityError"] = {ampName: []
for ampName
in ampNames}
215 self.__dict__[
"nonLinearityResiduals"] = {ampName: []
for ampName
in ampNames}
216 self.__dict__[
"nonLinearityReducedChiSquared"] = {ampName: []
for ampName
in ampNames}
219 self.__dict__[
"gain"] = {ampName: -1.
for ampName
in ampNames}
220 self.__dict__[
"gainErr"] = {ampName: -1.
for ampName
in ampNames}
221 self.__dict__[
"noise"] = {ampName: -1.
for ampName
in ampNames}
222 self.__dict__[
"noiseErr"] = {ampName: -1.
for ampName
in ampNames}
223 self.__dict__[
"coefficientsLinearizePolynomial"] = {ampName: []
for ampName
in ampNames}
224 self.__dict__[
"coefficientLinearizeSquared"] = {ampName: []
for ampName
in ampNames}
227 """Protect class attributes""" 228 if attribute
not in self.__dict__:
229 raise AttributeError(f
"{attribute} is not already a member of PhotonTransferCurveDataset, which" 230 " does not support setting of new attributes.")
232 self.__dict__[attribute] = value
235 """Get the visits used, i.e. not discarded, for a given amp. 237 If no mask has been created yet, all visits are returned. 239 if self.visitMask[ampName] == []:
240 return self.inputVisitPairs[ampName]
243 assert len(self.visitMask[ampName]) == len(self.inputVisitPairs[ampName])
245 pairs = self.inputVisitPairs[ampName]
246 mask = self.visitMask[ampName]
248 return [(v1, v2)
for ((v1, v2), m)
in zip(pairs, mask)
if bool(m)
is True]
251 return [amp
for amp
in self.ampNames
if amp
not in self.badAmps]
255 """A class to calculate, fit, and plot a PTC from a set of flat pairs. 257 The Photon Transfer Curve (var(signal) vs mean(signal)) is a standard tool 258 used in astronomical detectors characterization (e.g., Janesick 2001, 259 Janesick 2007). This task calculates the PTC from a series of pairs of 260 flat-field images; each pair taken at identical exposure times. The 261 difference image of each pair is formed to eliminate fixed pattern noise, 262 and then the variance of the difference image and the mean of the average image 263 are used to produce the PTC. An n-degree polynomial or the approximation in Equation 264 16 of Astier+19 ("The Shape of the Photon Transfer Curve of CCD sensors", 265 arXiv:1905.08677) can be fitted to the PTC curve. These models include 266 parameters such as the gain (e/DN) and readout noise. 268 Linearizers to correct for signal-chain non-linearity are also calculated. 269 The `Linearizer` class, in general, can support per-amp linearizers, but in this 270 task this is not supported. 275 Positional arguments passed to the Task constructor. None used at this 278 Keyword arguments passed on to the Task constructor. None used at this 283 RunnerClass = PairedVisitListTaskRunner
284 ConfigClass = MeasurePhotonTransferCurveTaskConfig
285 _DefaultName =
"measurePhotonTransferCurve" 288 pipeBase.CmdLineTask.__init__(self, *args, **kwargs)
289 self.makeSubtask(
"isr")
290 plt.interactive(
False)
292 self.config.isrForbiddenSteps, self.config.isrDesirableSteps, checkTrim=
False)
293 self.config.validate()
297 def _makeArgumentParser(cls):
298 """Augment argument parser for the MeasurePhotonTransferCurveTask.""" 300 parser.add_argument(
"--visit-pairs", dest=
"visitPairs", nargs=
"*",
301 help=
"Visit pairs to use. Each pair must be of the form INT,INT e.g. 123,456")
302 parser.add_id_argument(
"--id", datasetType=
"photonTransferCurveDataset",
303 ContainerClass=NonexistentDatasetTaskDataIdContainer,
304 help=
"The ccds to use, e.g. --id ccd=0..100")
309 """Run the Photon Transfer Curve (PTC) measurement task. 311 For a dataRef (which is each detector here), 312 and given a list of visit pairs at different exposure times, 317 dataRef : list of lsst.daf.persistence.ButlerDataRef 318 dataRef for the detector for the visits to be fit. 319 visitPairs : `iterable` of `tuple` of `int` 320 Pairs of visit numbers to be processed together 324 detNum = dataRef.dataId[self.config.ccdKey]
325 detector = dataRef.get(
'camera')[dataRef.dataId[self.config.ccdKey]]
331 for name
in dataRef.getButler().getKeys(
'bias'):
332 if name
not in dataRef.dataId:
334 dataRef.dataId[name] = \
335 dataRef.getButler().queryMetadata(
'raw', [name], detector=detNum)[0]
336 except OperationalError:
339 amps = detector.getAmplifiers()
340 ampNames = [amp.getName()
for amp
in amps]
343 self.log.info(
'Measuring PTC using %s visits for detector %s' % (visitPairs, detNum))
345 for (v1, v2)
in visitPairs:
347 dataRef.dataId[
'expId'] = v1
349 dataRef.dataId[
'expId'] = v2
351 del dataRef.dataId[
'expId']
354 expTime = exp1.getInfo().getVisitInfo().getExposureTime()
358 ampName = amp.getName()
360 dataset.rawExpTimes[ampName].append(expTime)
361 dataset.rawMeans[ampName].append(mu)
362 dataset.rawVars[ampName].append(varDiff)
363 dataset.inputVisitPairs[ampName].append((v1, v2))
365 numberAmps = len(detector.getAmplifiers())
366 numberAduValues = self.config.maxAduForLookupTableLinearizer
367 lookupTableArray = np.zeros((numberAmps, numberAduValues), dtype=np.int)
374 tableArray=lookupTableArray)
376 if self.config.makePlots:
377 self.
plot(dataRef, dataset, ptcFitType=self.config.ptcFitType)
380 self.log.info(f
"Writing PTC and NL data to {dataRef.getUri(write=True)}")
381 dataRef.put(dataset, datasetType=
"photonTransferCurveDataset")
383 butler = dataRef.getButler()
384 self.log.info(f
"Writing linearizers: \n " 385 "lookup table (linear component of polynomial fit), \n " 386 "polynomial (coefficients for a polynomial correction), \n " 387 "and squared linearizer (quadratic coefficient from polynomial)")
389 detName = detector.getName()
390 now = datetime.datetime.utcnow()
391 calibDate = now.strftime(
"%Y-%m-%d")
393 for linType, dataType
in [(
"LOOKUPTABLE",
'linearizeLut'),
394 (
"LINEARIZEPOLYNOMIAL",
'linearizePolynomial'),
395 (
"LINEARIZESQUARED",
'linearizeSquared')]:
397 if linType ==
"LOOKUPTABLE":
398 tableArray = lookupTableArray
403 instruName=self.config.instrumentName,
404 tableArray=tableArray,
406 butler.put(linearizer, datasetType=dataType, dataId={
'detector': detNum,
407 'detectorName': detName,
'calibDate': calibDate})
409 self.log.info(
'Finished measuring PTC for in detector %s' % detNum)
411 return pipeBase.Struct(exitStatus=0)
414 tableArray=None, log=None):
415 """Build linearizer object to persist. 419 dataset : `lsst.cp.pipe.ptc.PhotonTransferCurveDataset` 420 The dataset containing the means, variances, and exposure times 421 detector : `lsst.afw.cameraGeom.Detector` 423 calibDate : `datetime.datetime` 425 linearizerType : `str` 426 'LOOKUPTABLE', 'LINEARIZESQUARED', or 'LINEARIZEPOLYNOMIAL' 427 instruName : `str`, optional 429 tableArray : `np.array`, optional 430 Look-up table array with size rows=nAmps and columns=DN values 431 log : `lsst.log.Log`, optional 432 Logger to handle messages 436 linearizer : `lsst.ip.isr.Linearizer` 439 detName = detector.getName()
440 detNum = detector.getId()
441 if linearizerType ==
"LOOKUPTABLE":
442 if tableArray
is not None:
443 linearizer =
Linearizer(detector=detector, table=tableArray, log=log)
445 raise RuntimeError(
"tableArray must be provided when creating a LookupTable linearizer")
446 elif linearizerType
in (
"LINEARIZESQUARED",
"LINEARIZEPOLYNOMIAL"):
449 raise RuntimeError(
"Invalid linearizerType {linearizerType} to build a Linearizer object. " 450 "Supported: 'LOOKUPTABLE', 'LINEARIZESQUARED', or 'LINEARIZEPOLYNOMIAL'")
451 for i, amp
in enumerate(detector.getAmplifiers()):
452 ampName = amp.getName()
453 if linearizerType ==
"LOOKUPTABLE":
454 linearizer.linearityCoeffs[ampName] = [i, 0]
455 linearizer.linearityType[ampName] =
"LookupTable" 456 elif linearizerType ==
"LINEARIZESQUARED":
457 linearizer.linearityCoeffs[ampName] = [dataset.coefficientLinearizeSquared[ampName]]
458 linearizer.linearityType[ampName] =
"Squared" 459 elif linearizerType ==
"LINEARIZEPOLYNOMIAL":
460 linearizer.linearityCoeffs[ampName] = dataset.coefficientsLinearizePolynomial[ampName]
461 linearizer.linearityType[ampName] =
"Polynomial" 462 linearizer.linearityBBox[ampName] = amp.getBBox()
464 linearizer.validate()
465 calibId = f
"detectorName={detName} detector={detNum} calibDate={calibDate} ccd={detNum} filter=NONE" 468 raftName = detName.split(
"_")[0]
469 calibId += f
" raftName={raftName}" 472 calibId += f
" raftName={raftname}" 474 serial = detector.getSerial()
475 linearizer.updateMetadata(instrumentName=instruName, detectorId=f
"{detNum}",
476 calibId=calibId, serial=serial, detectorName=f
"{detName}")
481 """Calculate the mean signal of two exposures and the variance of their difference. 485 exposure1 : `lsst.afw.image.exposure.exposure.ExposureF` 486 First exposure of flat field pair. 488 exposure2 : `lsst.afw.image.exposure.exposure.ExposureF` 489 Second exposure of flat field pair. 491 region : `lsst.geom.Box2I` 492 Region of each exposure where to perform the calculations (e.g, an amplifier). 498 0.5*(mu1 + mu2), where mu1, and mu2 are the clipped means of the regions in 502 Half of the clipped variance of the difference of the regions inthe two input 506 if region
is not None:
507 im1Area = exposure1.maskedImage[region]
508 im2Area = exposure2.maskedImage[region]
510 im1Area = exposure1.maskedImage
511 im2Area = exposure2.maskedImage
513 im1Area = afwMath.binImage(im1Area, self.config.binSize)
514 im2Area = afwMath.binImage(im2Area, self.config.binSize)
517 mu1 = afwMath.makeStatistics(im1Area, afwMath.MEANCLIP).getValue()
518 mu2 = afwMath.makeStatistics(im2Area, afwMath.MEANCLIP).getValue()
523 temp = im2Area.clone()
525 diffIm = im1Area.clone()
530 varDiff = 0.5*(afwMath.makeStatistics(diffIm, afwMath.VARIANCECLIP).getValue())
534 def _fitLeastSq(self, initialParams, dataX, dataY, function):
535 """Do a fit and estimate the parameter errors using using scipy.optimize.leastq. 537 optimize.leastsq returns the fractional covariance matrix. To estimate the 538 standard deviation of the fit parameters, multiply the entries of this matrix 539 by the unweighted reduced chi squared and take the square root of the diagonal elements. 543 initialParams : `list` of `float` 544 initial values for fit parameters. For ptcFitType=POLYNOMIAL, its length 545 determines the degree of the polynomial. 547 dataX : `numpy.array` of `float` 548 Data in the abscissa axis. 550 dataY : `numpy.array` of `float` 551 Data in the ordinate axis. 553 function : callable object (function) 554 Function to fit the data with. 558 pFitSingleLeastSquares : `list` of `float` 559 List with fitted parameters. 561 pErrSingleLeastSquares : `list` of `float` 562 List with errors for fitted parameters. 564 reducedChiSqSingleLeastSquares : `float` 565 Unweighted reduced chi squared 568 def errFunc(p, x, y):
569 return function(p, x) - y
571 pFit, pCov, infoDict, errMessage, success = leastsq(errFunc, initialParams,
572 args=(dataX, dataY), full_output=1, epsfcn=0.0001)
574 if (len(dataY) > len(initialParams))
and pCov
is not None:
575 reducedChiSq = (errFunc(pFit, dataX, dataY)**2).sum()/(len(dataY)-len(initialParams))
581 for i
in range(len(pFit)):
582 errorVec.append(np.fabs(pCov[i][i])**0.5)
584 pFitSingleLeastSquares = pFit
585 pErrSingleLeastSquares = np.array(errorVec)
587 return pFitSingleLeastSquares, pErrSingleLeastSquares, reducedChiSq
589 def _fitBootstrap(self, initialParams, dataX, dataY, function, confidenceSigma=1.):
590 """Do a fit using least squares and bootstrap to estimate parameter errors. 592 The bootstrap error bars are calculated by fitting 100 random data sets. 596 initialParams : `list` of `float` 597 initial values for fit parameters. For ptcFitType=POLYNOMIAL, its length 598 determines the degree of the polynomial. 600 dataX : `numpy.array` of `float` 601 Data in the abscissa axis. 603 dataY : `numpy.array` of `float` 604 Data in the ordinate axis. 606 function : callable object (function) 607 Function to fit the data with. 609 confidenceSigma : `float` 610 Number of sigmas that determine confidence interval for the bootstrap errors. 614 pFitBootstrap : `list` of `float` 615 List with fitted parameters. 617 pErrBootstrap : `list` of `float` 618 List with errors for fitted parameters. 620 reducedChiSqBootstrap : `float` 624 def errFunc(p, x, y):
625 return function(p, x) - y
628 pFit, _ = leastsq(errFunc, initialParams, args=(dataX, dataY), full_output=0)
631 residuals = errFunc(pFit, dataX, dataY)
632 sigmaErrTotal = np.std(residuals)
637 randomDelta = np.random.normal(0., sigmaErrTotal, len(dataY))
638 randomDataY = dataY + randomDelta
639 randomFit, _ = leastsq(errFunc, initialParams,
640 args=(dataX, randomDataY), full_output=0)
641 pars.append(randomFit)
642 pars = np.array(pars)
643 meanPfit = np.mean(pars, 0)
646 nSigma = confidenceSigma
647 errPfit = nSigma*np.std(pars, 0)
648 pFitBootstrap = meanPfit
649 pErrBootstrap = errPfit
651 reducedChiSq = (errFunc(pFitBootstrap, dataX, dataY)**2).sum()/(len(dataY)-len(initialParams))
652 return pFitBootstrap, pErrBootstrap, reducedChiSq
655 """Polynomial function definition""" 656 return poly.polyval(x, [*pars])
659 """Single brighter-fatter parameter model for PTC; Equation 16 of Astier+19""" 660 a00, gain, noise = pars
661 return 0.5/(a00*gain*gain)*(np.exp(2*a00*x*gain)-1) + noise/(gain*gain)
664 def _initialParsForPolynomial(order):
666 pars = np.zeros(order, dtype=np.float)
673 def _boundsForPolynomial(initialPars):
674 lowers = [np.NINF
for p
in initialPars]
675 uppers = [np.inf
for p
in initialPars]
677 return (lowers, uppers)
680 def _boundsForAstier(initialPars):
681 lowers = [np.NINF
for p
in initialPars]
682 uppers = [np.inf
for p
in initialPars]
683 return (lowers, uppers)
686 def _getInitialGoodPoints(means, variances, maxDeviationPositive, maxDeviationNegative):
687 """Return a boolean array to mask bad points. 689 A linear function has a constant ratio, so find the median 690 value of the ratios, and exclude the points that deviate 691 from that by more than a factor of maxDeviationPositive/negative. 692 Asymmetric deviations are supported as we expect the PTC to turn 693 down as the flux increases, but sometimes it anomalously turns 694 upwards just before turning over, which ruins the fits, so it 695 is wise to be stricter about restricting positive outliers than 698 Too high and points that are so bad that fit will fail will be included 699 Too low and the non-linear points will be excluded, biasing the NL fit.""" 700 ratios = [b/a
for (a, b)
in zip(means, variances)]
701 medianRatio = np.median(ratios)
702 ratioDeviations = [(r/medianRatio)-1
for r
in ratios]
705 maxDeviationPositive = abs(maxDeviationPositive)
706 maxDeviationNegative = -1. * abs(maxDeviationNegative)
708 goodPoints = np.array([
True if (r < maxDeviationPositive
and r > maxDeviationNegative)
709 else False for r
in ratioDeviations])
712 def _makeZeroSafe(self, array, warn=True, substituteValue=1e-9):
714 nBad = Counter(array)[0]
719 msg = f
"Found {nBad} zeros in array at elements {[x for x in np.where(array==0)[0]]}" 722 array[array == 0] = substituteValue
726 """Calculate linearity residual and fit an n-order polynomial to the mean vs time curve 727 to produce corrections (deviation from linear part of polynomial) for a particular amplifier 728 to populate LinearizeLookupTable. 729 Use the coefficients of this fit to calculate the correction coefficients for LinearizePolynomial 730 and LinearizeSquared." 735 exposureTimeVector: `list` of `float` 736 List of exposure times for each flat pair 738 meanSignalVector: `list` of `float` 739 List of mean signal from diference image of flat pairs 743 polynomialLinearizerCoefficients : `list` of `float` 744 Coefficients for LinearizePolynomial, where corrImage = uncorrImage + sum_i c_i uncorrImage^(2 + 746 c_(j-2) = -k_j/(k_1^j) with units (DN^(1-j)). The units of k_j are DN/t^j, and they are fit from 747 meanSignalVector = k0 + k1*exposureTimeVector + k2*exposureTimeVector^2 +... 748 + kn*exposureTimeVector^n, with n = "polynomialFitDegreeNonLinearity". 749 k_0 and k_1 and degenerate with bias level and gain, and are not used by the non-linearity 750 correction. Therefore, j = 2...n in the above expression (see `LinearizePolynomial` class in 754 Coefficient for LinearizeSquared, where corrImage = uncorrImage + c0*uncorrImage^2. 755 c0 = -k2/(k1^2), where k1 and k2 are fit from 756 meanSignalVector = k0 + k1*exposureTimeVector + k2*exposureTimeVector^2 +... 757 + kn*exposureTimeVector^n, with n = "polynomialFitDegreeNonLinearity". 759 linearizerTableRow : `list` of `float` 760 One dimensional array with deviation from linear part of n-order polynomial fit 761 to mean vs time curve. This array will be one row (for the particular amplifier at hand) 762 of the table array for LinearizeLookupTable. 764 linResidual : `list` of `float` 765 Linearity residual from the mean vs time curve, defined as 766 100*(1 - meanSignalReference/expTimeReference/(meanSignal/expTime). 768 parsFit : `list` of `float` 769 Parameters from n-order polynomial fit to meanSignalVector vs exposureTimeVector. 771 parsFitErr : list of `float` 772 Parameters from n-order polynomial fit to meanSignalVector vs exposureTimeVector. 774 reducedChiSquaredNonLinearityFit : `float` 775 Reduced chi squared from polynomial fit to meanSignalVector vs exposureTimeVector. 780 if self.config.doFitBootstrap:
781 parsFit, parsFitErr, reducedChiSquaredNonLinearityFit = self.
_fitBootstrap(parsIniNonLinearity,
786 parsFit, parsFitErr, reducedChiSquaredNonLinearityFit = self.
_fitLeastSq(parsIniNonLinearity,
793 tMax = (self.config.maxAduForLookupTableLinearizer - parsFit[0])/parsFit[1]
794 timeRange = np.linspace(0, tMax, self.config.maxAduForLookupTableLinearizer)
795 signalIdeal = (parsFit[0] + parsFit[1]*timeRange).astype(int)
796 signalUncorrected = (self.
funcPolynomial(parsFit, timeRange)).astype(int)
797 linearizerTableRow = signalIdeal - signalUncorrected
804 polynomialLinearizerCoefficients = []
805 for i, coefficient
in enumerate(parsFit):
806 c = -coefficient/(k1**i)
807 polynomialLinearizerCoefficients.append(c)
808 if np.fabs(c) > 1e-10:
809 msg = f
"Coefficient {c} in polynomial fit larger than threshold 1e-10." 812 c0 = polynomialLinearizerCoefficients[2]
815 linResidualTimeIndex = self.config.linResidualTimeIndex
816 if exposureTimeVector[linResidualTimeIndex] == 0.0:
817 raise RuntimeError(
"Reference time for linearity residual can't be 0.0")
818 linResidual = 100*(1 - ((meanSignalVector[linResidualTimeIndex] /
819 exposureTimeVector[linResidualTimeIndex]) /
820 (meanSignalVector/exposureTimeVector)))
822 return (polynomialLinearizerCoefficients, c0, linearizerTableRow, linResidual, parsFit, parsFitErr,
823 reducedChiSquaredNonLinearityFit)
826 """Fit the photon transfer curve and calculate linearity and residuals. 828 Fit the photon transfer curve with either a polynomial of the order 829 specified in the task config, or using the Astier approximation. 831 Sigma clipping is performed iteratively for the fit, as well as an 832 initial clipping of data points that are more than 833 config.initialNonLinearityExclusionThreshold away from lying on a 834 straight line. This other step is necessary because the photon transfer 835 curve turns over catastrophically at very high flux (because saturation 836 drops the variance to ~0) and these far outliers cause the initial fit 837 to fail, meaning the sigma cannot be calculated to perform the 842 dataset : `lsst.cp.pipe.ptc.PhotonTransferCurveDataset` 843 The dataset containing the means, variances and exposure times 845 Fit a 'POLYNOMIAL' (degree: 'polynomialFitDegree') or 846 'ASTIERAPPROXIMATION' to the PTC 847 tableArray : `np.array` 848 Optional. Look-up table array with size rows=nAmps and columns=DN values. 849 It will be modified in-place if supplied. 853 dataset: `lsst.cp.pipe.ptc.PhotonTransferCurveDataset` 854 This is the same dataset as the input paramter, however, it has been modified 855 to include information such as the fit vectors and the fit parameters. See 856 the class `PhotonTransferCurveDatase`. 859 def errFunc(p, x, y):
860 return ptcFunc(p, x) - y
862 sigmaCutPtcOutliers = self.config.sigmaCutPtcOutliers
863 maxIterationsPtcOutliers = self.config.maxIterationsPtcOutliers
865 for i, ampName
in enumerate(dataset.ampNames):
866 timeVecOriginal = np.array(dataset.rawExpTimes[ampName])
867 meanVecOriginal = np.array(dataset.rawMeans[ampName])
868 varVecOriginal = np.array(dataset.rawVars[ampName])
871 mask = ((meanVecOriginal >= self.config.minMeanSignal) &
872 (meanVecOriginal <= self.config.maxMeanSignal))
875 self.config.initialNonLinearityExclusionThresholdPositive,
876 self.config.initialNonLinearityExclusionThresholdNegative)
877 mask = mask & goodPoints
879 if ptcFitType ==
'ASTIERAPPROXIMATION':
881 parsIniPtc = [-1e-9, 1.0, 10.]
883 if ptcFitType ==
'POLYNOMIAL':
890 while count <= maxIterationsPtcOutliers:
894 meanTempVec = meanVecOriginal[mask]
895 varTempVec = varVecOriginal[mask]
896 res = least_squares(errFunc, parsIniPtc, bounds=bounds, args=(meanTempVec, varTempVec))
902 sigResids = (varVecOriginal - ptcFunc(pars, meanVecOriginal))/np.sqrt(varVecOriginal)
903 newMask = np.array([
True if np.abs(r) < sigmaCutPtcOutliers
else False for r
in sigResids])
904 mask = mask & newMask
906 nDroppedTotal = Counter(mask)[
False]
907 self.log.debug(f
"Iteration {count}: discarded {nDroppedTotal} points in total for {ampName}")
910 assert (len(mask) == len(timeVecOriginal) == len(meanVecOriginal) == len(varVecOriginal))
912 dataset.visitMask[ampName] = mask
915 timeVecFinal = timeVecOriginal[mask]
916 meanVecFinal = meanVecOriginal[mask]
917 varVecFinal = varVecOriginal[mask]
919 if Counter(mask)[
False] > 0:
920 self.log.info((f
"Number of points discarded in PTC of amplifier {ampName}:" +
921 f
" {Counter(mask)[False]} out of {len(meanVecOriginal)}"))
923 if (len(meanVecFinal) < len(parsIniPtc)):
924 msg = (f
"\nSERIOUS: Not enough data points ({len(meanVecFinal)}) compared to the number of" 925 f
"parameters of the PTC model({len(parsIniPtc)}). Setting {ampName} to BAD.")
927 dataset.badAmps.append(ampName)
928 dataset.gain[ampName] = np.nan
929 dataset.gainErr[ampName] = np.nan
930 dataset.noise[ampName] = np.nan
931 dataset.noiseErr[ampName] = np.nan
932 dataset.nonLinearity[ampName] = np.nan
933 dataset.nonLinearityError[ampName] = np.nan
934 dataset.nonLinearityResiduals[ampName] = np.nan
935 dataset.coefficientLinearizeSquared[ampName] = np.nan
939 if self.config.doFitBootstrap:
940 parsFit, parsFitErr, reducedChiSqPtc = self.
_fitBootstrap(parsIniPtc, meanVecFinal,
941 varVecFinal, ptcFunc)
943 parsFit, parsFitErr, reducedChiSqPtc = self.
_fitLeastSq(parsIniPtc, meanVecFinal,
944 varVecFinal, ptcFunc)
946 dataset.ptcFitPars[ampName] = parsFit
947 dataset.ptcFitParsError[ampName] = parsFitErr
948 dataset.ptcFitReducedChiSquared[ampName] = reducedChiSqPtc
950 if ptcFitType ==
'ASTIERAPPROXIMATION':
952 ptcGainErr = parsFitErr[1]
953 ptcNoise = np.sqrt(np.fabs(parsFit[2]))
954 ptcNoiseErr = 0.5*(parsFitErr[2]/np.fabs(parsFit[2]))*np.sqrt(np.fabs(parsFit[2]))
955 if ptcFitType ==
'POLYNOMIAL':
956 ptcGain = 1./parsFit[1]
957 ptcGainErr = np.fabs(1./parsFit[1])*(parsFitErr[1]/parsFit[1])
958 ptcNoise = np.sqrt(np.fabs(parsFit[0]))*ptcGain
959 ptcNoiseErr = (0.5*(parsFitErr[0]/np.fabs(parsFit[0]))*(np.sqrt(np.fabs(parsFit[0]))))*ptcGain
961 dataset.gain[ampName] = ptcGain
962 dataset.gainErr[ampName] = ptcGainErr
963 dataset.noise[ampName] = ptcNoise
964 dataset.noiseErr[ampName] = ptcNoiseErr
965 dataset.ptcFitType[ampName] = ptcFitType
970 (coeffsLinPoly, c0, linearizerTableRow, linResidualNonLinearity,
971 parsFitNonLinearity, parsFitErrNonLinearity,
976 if tableArray
is not None:
977 tableArray[i, :] = linearizerTableRow
979 dataset.nonLinearity[ampName] = parsFitNonLinearity
980 dataset.nonLinearityError[ampName] = parsFitErrNonLinearity
981 dataset.nonLinearityResiduals[ampName] = linResidualNonLinearity
982 dataset.nonLinearityReducedChiSquared[ampName] = reducedChiSqNonLinearity
986 dataset.coefficientsLinearizePolynomial[ampName] = np.array(coeffsLinPoly[2:])
987 dataset.coefficientLinearizeSquared[ampName] = c0
991 def plot(self, dataRef, dataset, ptcFitType):
992 dirname = dataRef.getUri(datasetType=
'cpPipePlotRoot', write=
True)
993 if not os.path.exists(dirname):
996 detNum = dataRef.dataId[self.config.ccdKey]
997 filename = f
"PTC_det{detNum}.pdf" 998 filenameFull = os.path.join(dirname, filename)
999 with PdfPages(filenameFull)
as pdfPages:
1000 self.
_plotPtc(dataset, ptcFitType, pdfPages)
1002 def _plotPtc(self, dataset, ptcFitType, pdfPages):
1003 """Plot PTC, linearity, and linearity residual per amplifier""" 1005 reducedChiSqPtc = dataset.ptcFitReducedChiSquared
1006 if ptcFitType ==
'ASTIERAPPROXIMATION':
1008 stringTitle = (
r"Var = $\frac{1}{2g^2a_{00}}(\exp (2a_{00} \mu g) - 1) + \frac{n_{00}}{g^2}$ " 1009 r" ($chi^2$/dof = %g)" % (reducedChiSqPtc))
1010 if ptcFitType ==
'POLYNOMIAL':
1012 stringTitle =
r"Polynomial (degree: %g)" % (self.config.polynomialFitDegree)
1017 supTitleFontSize = 18
1021 nAmps = len(dataset.ampNames)
1024 nRows = np.sqrt(nAmps)
1025 mantissa, _ = np.modf(nRows)
1027 nRows = int(nRows) + 1
1033 f, ax = plt.subplots(nrows=nRows, ncols=nCols, sharex=
'col', sharey=
'row', figsize=(13, 10))
1034 f2, ax2 = plt.subplots(nrows=nRows, ncols=nCols, sharex=
'col', sharey=
'row', figsize=(13, 10))
1036 for i, (amp, a, a2)
in enumerate(zip(dataset.ampNames, ax.flatten(), ax2.flatten())):
1037 meanVecOriginal = np.array(dataset.rawMeans[amp])
1038 varVecOriginal = np.array(dataset.rawVars[amp])
1039 mask = dataset.visitMask[amp]
1040 meanVecFinal = meanVecOriginal[mask]
1041 varVecFinal = varVecOriginal[mask]
1042 meanVecOutliers = meanVecOriginal[np.invert(mask)]
1043 varVecOutliers = varVecOriginal[np.invert(mask)]
1044 pars, parsErr = dataset.ptcFitPars[amp], dataset.ptcFitParsError[amp]
1046 if ptcFitType ==
'ASTIERAPPROXIMATION':
1047 ptcA00, ptcA00error = pars[0], parsErr[0]
1048 ptcGain, ptcGainError = pars[1], parsErr[1]
1049 ptcNoise = np.sqrt(np.fabs(pars[2]))
1050 ptcNoiseError = 0.5*(parsErr[2]/np.fabs(pars[2]))*np.sqrt(np.fabs(pars[2]))
1051 stringLegend = (f
"a00: {ptcA00:.2e}+/-{ptcA00error:.2e} 1/e" 1052 f
"\n Gain: {ptcGain:.4}+/-{ptcGainError:.2e} e/DN" 1053 f
"\n Noise: {ptcNoise:.4}+/-{ptcNoiseError:.2e} e \n")
1055 if ptcFitType ==
'POLYNOMIAL':
1056 ptcGain, ptcGainError = 1./pars[1], np.fabs(1./pars[1])*(parsErr[1]/pars[1])
1057 ptcNoise = np.sqrt(np.fabs(pars[0]))*ptcGain
1058 ptcNoiseError = (0.5*(parsErr[0]/np.fabs(pars[0]))*(np.sqrt(np.fabs(pars[0]))))*ptcGain
1059 stringLegend = (f
"Noise: {ptcNoise:.4}+/-{ptcNoiseError:.2e} e \n" 1060 f
"Gain: {ptcGain:.4}+/-{ptcGainError:.2e} e/DN \n")
1062 minMeanVecFinal = np.min(meanVecFinal)
1063 maxMeanVecFinal = np.max(meanVecFinal)
1064 meanVecFit = np.linspace(minMeanVecFinal, maxMeanVecFinal, 100*len(meanVecFinal))
1065 minMeanVecOriginal = np.min(meanVecOriginal)
1066 maxMeanVecOriginal = np.max(meanVecOriginal)
1067 deltaXlim = maxMeanVecOriginal - minMeanVecOriginal
1069 a.plot(meanVecFit, ptcFunc(pars, meanVecFit), color=
'red')
1070 a.plot(meanVecFinal, pars[0] + pars[1]*meanVecFinal, color=
'green', linestyle=
'--')
1071 a.scatter(meanVecFinal, varVecFinal, c=
'blue', marker=
'o', s=markerSize)
1072 a.scatter(meanVecOutliers, varVecOutliers, c=
'magenta', marker=
's', s=markerSize)
1073 a.set_xlabel(
r'Mean signal ($\mu$, DN)', fontsize=labelFontSize)
1074 a.set_xticks(meanVecOriginal)
1075 a.set_ylabel(
r'Variance (DN$^2$)', fontsize=labelFontSize)
1076 a.tick_params(labelsize=11)
1077 a.text(0.03, 0.8, stringLegend, transform=a.transAxes, fontsize=legendFontSize)
1078 a.set_xscale(
'linear', fontsize=labelFontSize)
1079 a.set_yscale(
'linear', fontsize=labelFontSize)
1080 a.set_title(amp, fontsize=titleFontSize)
1081 a.set_xlim([minMeanVecOriginal - 0.2*deltaXlim, maxMeanVecOriginal + 0.2*deltaXlim])
1084 a2.plot(meanVecFit, ptcFunc(pars, meanVecFit), color=
'red')
1085 a2.scatter(meanVecFinal, varVecFinal, c=
'blue', marker=
'o', s=markerSize)
1086 a2.scatter(meanVecOutliers, varVecOutliers, c=
'magenta', marker=
's', s=markerSize)
1087 a2.set_xlabel(
r'Mean Signal ($\mu$, DN)', fontsize=labelFontSize)
1088 a2.set_ylabel(
r'Variance (DN$^2$)', fontsize=labelFontSize)
1089 a2.tick_params(labelsize=11)
1090 a2.text(0.03, 0.8, stringLegend, transform=a2.transAxes, fontsize=legendFontSize)
1091 a2.set_xscale(
'log')
1092 a2.set_yscale(
'log')
1093 a2.set_title(amp, fontsize=titleFontSize)
1094 a2.set_xlim([minMeanVecOriginal, maxMeanVecOriginal])
1096 f.suptitle(f
"PTC \n Fit: " + stringTitle, fontsize=20)
1098 f2.suptitle(f
"PTC (log-log)", fontsize=20)
1099 pdfPages.savefig(f2)
1102 f, ax = plt.subplots(nrows=4, ncols=4, sharex=
'col', sharey=
'row', figsize=(13, 10))
1103 for i, (amp, a)
in enumerate(zip(dataset.ampNames, ax.flatten())):
1104 meanVecFinal = np.array(dataset.rawMeans[amp])[dataset.visitMask[amp]]
1105 timeVecFinal = np.array(dataset.rawExpTimes[amp])[dataset.visitMask[amp]]
1107 pars, parsErr = dataset.nonLinearity[amp], dataset.nonLinearityError[amp]
1108 k0, k0Error = pars[0], parsErr[0]
1109 k1, k1Error = pars[1], parsErr[1]
1110 k2, k2Error = pars[2], parsErr[2]
1111 stringLegend = (f
"k0: {k0:.4}+/-{k0Error:.2e} DN\n k1: {k1:.4}+/-{k1Error:.2e} DN/t" 1112 f
"\n k2: {k2:.2e}+/-{k2Error:.2e} DN/t^2 \n")
1113 a.scatter(timeVecFinal, meanVecFinal)
1114 a.plot(timeVecFinal, self.
funcPolynomial(pars, timeVecFinal), color=
'red')
1115 a.set_xlabel(
'Time (sec)', fontsize=labelFontSize)
1116 a.set_xticks(timeVecFinal)
1117 a.set_ylabel(
r'Mean signal ($\mu$, DN)', fontsize=labelFontSize)
1118 a.tick_params(labelsize=labelFontSize)
1119 a.text(0.03, 0.75, stringLegend, transform=a.transAxes, fontsize=legendFontSize)
1120 a.set_xscale(
'linear', fontsize=labelFontSize)
1121 a.set_yscale(
'linear', fontsize=labelFontSize)
1122 a.set_title(amp, fontsize=titleFontSize)
1123 f.suptitle(
"Linearity \n Fit: Polynomial (degree: %g)" 1124 % (self.config.polynomialFitDegreeNonLinearity),
1125 fontsize=supTitleFontSize)
1129 f, ax = plt.subplots(nrows=4, ncols=4, sharex=
'col', sharey=
'row', figsize=(13, 10))
1130 for i, (amp, a)
in enumerate(zip(dataset.ampNames, ax.flatten())):
1131 meanVecFinal = np.array(dataset.rawMeans[amp])[dataset.visitMask[amp]]
1132 linRes = np.array(dataset.nonLinearityResiduals[amp])
1134 a.scatter(meanVecFinal, linRes)
1135 a.axhline(y=0, color=
'k')
1136 a.axvline(x=timeVecFinal[self.config.linResidualTimeIndex], color=
'g', linestyle=
'--')
1137 a.set_xlabel(
r'Mean signal ($\mu$, DN)', fontsize=labelFontSize)
1138 a.set_xticks(meanVecFinal)
1139 a.set_ylabel(
'LR (%)', fontsize=labelFontSize)
1140 a.tick_params(labelsize=labelFontSize)
1141 a.set_xscale(
'linear', fontsize=labelFontSize)
1142 a.set_yscale(
'linear', fontsize=labelFontSize)
1143 a.set_title(amp, fontsize=titleFontSize)
1145 f.suptitle(
r"Linearity Residual: $100(1 - \mu_{\rm{ref}}/t_{\rm{ref}})/(\mu / t))$" +
"\n" +
1146 r"$t_{\rm{ref}}$: " + f
"{timeVecFinal[2]} s", fontsize=supTitleFontSize)
def funcAstier(self, pars, x)
def _plotPtc(self, dataset, ptcFitType, pdfPages)
def _boundsForPolynomial(initialPars)
def __init__(self, ampNames)
def runDataRef(self, dataRef, visitPairs)
def calculateLinearityResidualAndLinearizers(self, exposureTimeVector, meanSignalVector)
def _boundsForAstier(initialPars)
def fitPtcAndNonLinearity(self, dataset, ptcFitType, tableArray=None)
def __setattr__(self, attribute, value)
def funcPolynomial(self, pars, x)
def checkExpLengthEqual(exp1, exp2, v1=None, v2=None, raiseWithMessage=False)
def buildLinearizerObject(self, dataset, detector, calibDate, linearizerType, instruName='', tableArray=None, log=None)
def validateIsrConfig(isrTask, mandatory=None, forbidden=None, desirable=None, undesirable=None, checkTrim=True, logName=None)
def _getInitialGoodPoints(means, variances, maxDeviationPositive, maxDeviationNegative)
def _fitBootstrap(self, initialParams, dataX, dataY, function, confidenceSigma=1.)
def getVisitsUsed(self, ampName)
def __init__(self, args, kwargs)
def measureMeanVarPair(self, exposure1, exposure2, region=None)
def _initialParsForPolynomial(order)
def _makeZeroSafe(self, array, warn=True, substituteValue=1e-9)
def _fitLeastSq(self, initialParams, dataX, dataY, function)
def plot(self, dataRef, dataset, ptcFitType)