23 __all__ = [
'MeasurePhotonTransferCurveTask',
24 'MeasurePhotonTransferCurveTaskConfig',
25 'PhotonTransferCurveDataset']
28 import matplotlib.pyplot
as plt
30 from matplotlib.backends.backend_pdf
import PdfPages
31 from sqlite3
import OperationalError
32 from collections
import Counter
33 from dataclasses
import dataclass
36 import lsst.pex.config
as pexConfig
39 from .utils
import (NonexistentDatasetTaskDataIdContainer, PairedVisitListTaskRunner,
40 checkExpLengthEqual, validateIsrConfig)
41 from scipy.optimize
import leastsq, least_squares
42 import numpy.polynomial.polynomial
as poly
49 """Config class for photon transfer curve measurement task"""
50 isr = pexConfig.ConfigurableField(
52 doc=
"""Task to perform instrumental signature removal.""",
54 isrMandatorySteps = pexConfig.ListField(
56 doc=
"isr operations that must be performed for valid results. Raises if any of these are False.",
57 default=[
'doAssembleCcd']
59 isrForbiddenSteps = pexConfig.ListField(
61 doc=
"isr operations that must NOT be performed for valid results. Raises if any of these are True",
62 default=[
'doFlat',
'doFringe',
'doBrighterFatter',
'doUseOpticsTransmission',
63 'doUseFilterTransmission',
'doUseSensorTransmission',
'doUseAtmosphereTransmission']
65 isrDesirableSteps = pexConfig.ListField(
67 doc=
"isr operations that it is advisable to perform, but are not mission-critical." +
68 " WARNs are logged for any of these found to be False.",
69 default=[
'doBias',
'doDark',
'doCrosstalk',
'doDefect']
71 isrUndesirableSteps = pexConfig.ListField(
73 doc=
"isr operations that it is *not* advisable to perform in the general case, but are not" +
74 " forbidden as some use-cases might warrant them." +
75 " WARNs are logged for any of these found to be True.",
76 default=[
'doLinearize']
78 ccdKey = pexConfig.Field(
80 doc=
"The key by which to pull a detector from a dataId, e.g. 'ccd' or 'detector'.",
83 makePlots = pexConfig.Field(
85 doc=
"Plot the PTC curves?",
88 ptcFitType = pexConfig.ChoiceField(
90 doc=
"Fit PTC to approximation in Astier+19 (Equation 16) or to a polynomial.",
93 "POLYNOMIAL":
"n-degree polynomial (use 'polynomialFitDegree' to set 'n').",
94 "ASTIERAPPROXIMATION":
"Approximation in Astier+19 (Eq. 16)."
97 polynomialFitDegree = pexConfig.Field(
99 doc=
"Degree of polynomial to fit the PTC, when 'ptcFitType'=POLYNOMIAL.",
102 polynomialFitDegreeNonLinearity = pexConfig.Field(
104 doc=
"Degree of polynomial to fit the meanSignal vs exposureTime curve to produce" +
105 " the table for LinearizeLookupTable.",
108 binSize = pexConfig.Field(
110 doc=
"Bin the image by this factor in both dimensions.",
113 minMeanSignal = pexConfig.Field(
115 doc=
"Minimum value (inclusive) of mean signal (in DN) above which to consider.",
118 maxMeanSignal = pexConfig.Field(
120 doc=
"Maximum value (inclusive) of mean signal (in DN) below which to consider.",
123 initialNonLinearityExclusionThresholdPositive = pexConfig.RangeField(
125 doc=
"Initially exclude data points with a variance that are more than a factor of this from being"
126 " linear in the positive direction, from the PTC fit. Note that these points will also be"
127 " excluded from the non-linearity fit. This is done before the iterative outlier rejection,"
128 " to allow an accurate determination of the sigmas for said iterative fit.",
133 initialNonLinearityExclusionThresholdNegative = pexConfig.RangeField(
135 doc=
"Initially exclude data points with a variance that are more than a factor of this from being"
136 " linear in the negative direction, from the PTC fit. Note that these points will also be"
137 " excluded from the non-linearity fit. This is done before the iterative outlier rejection,"
138 " to allow an accurate determination of the sigmas for said iterative fit.",
143 sigmaCutPtcOutliers = pexConfig.Field(
145 doc=
"Sigma cut for outlier rejection in PTC.",
148 maxIterationsPtcOutliers = pexConfig.Field(
150 doc=
"Maximum number of iterations for outlier rejection in PTC.",
153 doFitBootstrap = pexConfig.Field(
155 doc=
"Use bootstrap for the PTC fit parameters and errors?.",
158 linResidualTimeIndex = pexConfig.Field(
160 doc=
"Index position in time array for reference time in linearity residual calculation.",
163 maxAduForLookupTableLinearizer = pexConfig.Field(
165 doc=
"Maximum DN value for the LookupTable linearizer.",
168 instrumentName = pexConfig.Field(
170 doc=
"Instrument name.",
177 """A simple class to hold the output from the
178 `calculateLinearityResidualAndLinearizers` function.
181 polynomialLinearizerCoefficients: list
183 quadraticPolynomialLinearizerCoefficient: float
185 linearizerTableRow: list
187 linearityResidual: list
188 meanSignalVsTimePolyFitPars: list
189 meanSignalVsTimePolyFitParsErr: list
190 fractionalNonLinearityResidual: list
191 meanSignalVsTimePolyFitReducedChiSq: float
195 """A simple class to hold the output data from the PTC task.
197 The dataset is made up of a dictionary for each item, keyed by the
198 amplifiers' names, which much be supplied at construction time.
200 New items cannot be added to the class to save accidentally saving to the
201 wrong property, and the class can be frozen if desired.
203 inputVisitPairs records the visits used to produce the data.
204 When fitPtcAndNonLinearity() is run, a mask is built up, which is by definition
205 always the same length as inputVisitPairs, rawExpTimes, rawMeans
206 and rawVars, and is a list of bools, which are incrementally set to False
207 as points are discarded from the fits.
209 PTC fit parameters for polynomials are stored in a list in ascending order
210 of polynomial term, i.e. par[0]*x^0 + par[1]*x + par[2]*x^2 etc
211 with the length of the list corresponding to the order of the polynomial
218 self.__dict__[
"ampNames"] = ampNames
219 self.__dict__[
"badAmps"] = []
222 self.__dict__[
"inputVisitPairs"] = {ampName: []
for ampName
in ampNames}
223 self.__dict__[
"visitMask"] = {ampName: []
for ampName
in ampNames}
224 self.__dict__[
"rawExpTimes"] = {ampName: []
for ampName
in ampNames}
225 self.__dict__[
"rawMeans"] = {ampName: []
for ampName
in ampNames}
226 self.__dict__[
"rawVars"] = {ampName: []
for ampName
in ampNames}
229 self.__dict__[
"ptcFitType"] = {ampName:
"" for ampName
in ampNames}
230 self.__dict__[
"ptcFitPars"] = {ampName: []
for ampName
in ampNames}
231 self.__dict__[
"ptcFitParsError"] = {ampName: []
for ampName
in ampNames}
232 self.__dict__[
"ptcFitReducedChiSquared"] = {ampName: []
for ampName
in ampNames}
233 self.__dict__[
"nonLinearity"] = {ampName: []
for ampName
in ampNames}
234 self.__dict__[
"nonLinearityError"] = {ampName: []
for ampName
in ampNames}
235 self.__dict__[
"nonLinearityResiduals"] = {ampName: []
for ampName
in ampNames}
236 self.__dict__[
"fractionalNonLinearityResiduals"] = {ampName: []
for ampName
in ampNames}
237 self.__dict__[
"nonLinearityReducedChiSquared"] = {ampName: []
for ampName
in ampNames}
240 self.__dict__[
"gain"] = {ampName: -1.
for ampName
in ampNames}
241 self.__dict__[
"gainErr"] = {ampName: -1.
for ampName
in ampNames}
242 self.__dict__[
"noise"] = {ampName: -1.
for ampName
in ampNames}
243 self.__dict__[
"noiseErr"] = {ampName: -1.
for ampName
in ampNames}
244 self.__dict__[
"coefficientsLinearizePolynomial"] = {ampName: []
for ampName
in ampNames}
245 self.__dict__[
"coefficientLinearizeSquared"] = {ampName: []
for ampName
in ampNames}
248 """Protect class attributes"""
249 if attribute
not in self.__dict__:
250 raise AttributeError(f
"{attribute} is not already a member of PhotonTransferCurveDataset, which"
251 " does not support setting of new attributes.")
253 self.__dict__[attribute] = value
256 """Get the visits used, i.e. not discarded, for a given amp.
258 If no mask has been created yet, all visits are returned.
260 if self.visitMask[ampName] == []:
261 return self.inputVisitPairs[ampName]
264 assert len(self.visitMask[ampName]) == len(self.inputVisitPairs[ampName])
266 pairs = self.inputVisitPairs[ampName]
267 mask = self.visitMask[ampName]
269 return [(v1, v2)
for ((v1, v2), m)
in zip(pairs, mask)
if bool(m)
is True]
272 return [amp
for amp
in self.ampNames
if amp
not in self.badAmps]
276 """A class to calculate, fit, and plot a PTC from a set of flat pairs.
278 The Photon Transfer Curve (var(signal) vs mean(signal)) is a standard tool
279 used in astronomical detectors characterization (e.g., Janesick 2001,
280 Janesick 2007). This task calculates the PTC from a series of pairs of
281 flat-field images; each pair taken at identical exposure times. The
282 difference image of each pair is formed to eliminate fixed pattern noise,
283 and then the variance of the difference image and the mean of the average image
284 are used to produce the PTC. An n-degree polynomial or the approximation in Equation
285 16 of Astier+19 ("The Shape of the Photon Transfer Curve of CCD sensors",
286 arXiv:1905.08677) can be fitted to the PTC curve. These models include
287 parameters such as the gain (e/DN) and readout noise.
289 Linearizers to correct for signal-chain non-linearity are also calculated.
290 The `Linearizer` class, in general, can support per-amp linearizers, but in this
291 task this is not supported.
296 Positional arguments passed to the Task constructor. None used at this
299 Keyword arguments passed on to the Task constructor. None used at this
304 RunnerClass = PairedVisitListTaskRunner
305 ConfigClass = MeasurePhotonTransferCurveTaskConfig
306 _DefaultName =
"measurePhotonTransferCurve"
309 pipeBase.CmdLineTask.__init__(self, *args, **kwargs)
310 self.makeSubtask(
"isr")
311 plt.interactive(
False)
313 self.config.isrForbiddenSteps, self.config.isrDesirableSteps, checkTrim=
False)
314 self.config.validate()
318 def _makeArgumentParser(cls):
319 """Augment argument parser for the MeasurePhotonTransferCurveTask."""
321 parser.add_argument(
"--visit-pairs", dest=
"visitPairs", nargs=
"*",
322 help=
"Visit pairs to use. Each pair must be of the form INT,INT e.g. 123,456")
323 parser.add_id_argument(
"--id", datasetType=
"photonTransferCurveDataset",
324 ContainerClass=NonexistentDatasetTaskDataIdContainer,
325 help=
"The ccds to use, e.g. --id ccd=0..100")
330 """Run the Photon Transfer Curve (PTC) measurement task.
332 For a dataRef (which is each detector here),
333 and given a list of visit pairs at different exposure times,
338 dataRef : list of lsst.daf.persistence.ButlerDataRef
339 dataRef for the detector for the visits to be fit.
340 visitPairs : `iterable` of `tuple` of `int`
341 Pairs of visit numbers to be processed together
345 detNum = dataRef.dataId[self.config.ccdKey]
346 detector = dataRef.get(
'camera')[dataRef.dataId[self.config.ccdKey]]
352 for name
in dataRef.getButler().getKeys(
'bias'):
353 if name
not in dataRef.dataId:
355 dataRef.dataId[name] = \
356 dataRef.getButler().queryMetadata(
'raw', [name], detector=detNum)[0]
357 except OperationalError:
360 amps = detector.getAmplifiers()
361 ampNames = [amp.getName()
for amp
in amps]
364 self.log.info(
'Measuring PTC using %s visits for detector %s' % (visitPairs, detNum))
366 for (v1, v2)
in visitPairs:
368 dataRef.dataId[
'expId'] = v1
370 dataRef.dataId[
'expId'] = v2
372 del dataRef.dataId[
'expId']
375 expTime = exp1.getInfo().getVisitInfo().getExposureTime()
379 ampName = amp.getName()
381 dataset.rawExpTimes[ampName].append(expTime)
382 dataset.rawMeans[ampName].append(mu)
383 dataset.rawVars[ampName].append(varDiff)
384 dataset.inputVisitPairs[ampName].append((v1, v2))
386 numberAmps = len(detector.getAmplifiers())
387 numberAduValues = self.config.maxAduForLookupTableLinearizer
388 lookupTableArray = np.zeros((numberAmps, numberAduValues), dtype=np.float32)
395 tableArray=lookupTableArray)
397 if self.config.makePlots:
398 self.
plot(dataRef, dataset, ptcFitType=self.config.ptcFitType)
401 self.log.info(f
"Writing PTC and NL data to {dataRef.getUri(write=True)}")
402 dataRef.put(dataset, datasetType=
"photonTransferCurveDataset")
404 butler = dataRef.getButler()
405 self.log.info(f
"Writing linearizers: \n "
406 "lookup table (linear component of polynomial fit), \n "
407 "polynomial (coefficients for a polynomial correction), \n "
408 "and squared linearizer (quadratic coefficient from polynomial)")
410 detName = detector.getName()
411 now = datetime.datetime.utcnow()
412 calibDate = now.strftime(
"%Y-%m-%d")
414 for linType, dataType
in [(
"LOOKUPTABLE",
'linearizeLut'),
415 (
"LINEARIZEPOLYNOMIAL",
'linearizePolynomial'),
416 (
"LINEARIZESQUARED",
'linearizeSquared')]:
418 if linType ==
"LOOKUPTABLE":
419 tableArray = lookupTableArray
424 instruName=self.config.instrumentName,
425 tableArray=tableArray,
427 butler.put(linearizer, datasetType=dataType, dataId={
'detector': detNum,
428 'detectorName': detName,
'calibDate': calibDate})
430 self.log.info(
'Finished measuring PTC for in detector %s' % detNum)
432 return pipeBase.Struct(exitStatus=0)
435 tableArray=None, log=None):
436 """Build linearizer object to persist.
440 dataset : `lsst.cp.pipe.ptc.PhotonTransferCurveDataset`
441 The dataset containing the means, variances, and exposure times
442 detector : `lsst.afw.cameraGeom.Detector`
444 calibDate : `datetime.datetime`
446 linearizerType : `str`
447 'LOOKUPTABLE', 'LINEARIZESQUARED', or 'LINEARIZEPOLYNOMIAL'
448 instruName : `str`, optional
450 tableArray : `np.array`, optional
451 Look-up table array with size rows=nAmps and columns=DN values
452 log : `lsst.log.Log`, optional
453 Logger to handle messages
457 linearizer : `lsst.ip.isr.Linearizer`
460 detName = detector.getName()
461 detNum = detector.getId()
462 if linearizerType ==
"LOOKUPTABLE":
463 if tableArray
is not None:
464 linearizer =
Linearizer(detector=detector, table=tableArray, log=log)
466 raise RuntimeError(
"tableArray must be provided when creating a LookupTable linearizer")
467 elif linearizerType
in (
"LINEARIZESQUARED",
"LINEARIZEPOLYNOMIAL"):
470 raise RuntimeError(
"Invalid linearizerType {linearizerType} to build a Linearizer object. "
471 "Supported: 'LOOKUPTABLE', 'LINEARIZESQUARED', or 'LINEARIZEPOLYNOMIAL'")
472 for i, amp
in enumerate(detector.getAmplifiers()):
473 ampName = amp.getName()
474 if linearizerType ==
"LOOKUPTABLE":
475 linearizer.linearityCoeffs[ampName] = [i, 0]
476 linearizer.linearityType[ampName] =
"LookupTable"
477 elif linearizerType ==
"LINEARIZESQUARED":
478 linearizer.linearityCoeffs[ampName] = [dataset.coefficientLinearizeSquared[ampName]]
479 linearizer.linearityType[ampName] =
"Squared"
480 elif linearizerType ==
"LINEARIZEPOLYNOMIAL":
481 linearizer.linearityCoeffs[ampName] = dataset.coefficientsLinearizePolynomial[ampName]
482 linearizer.linearityType[ampName] =
"Polynomial"
483 linearizer.linearityBBox[ampName] = amp.getBBox()
485 linearizer.validate()
486 calibId = f
"detectorName={detName} detector={detNum} calibDate={calibDate} ccd={detNum} filter=NONE"
489 raftName = detName.split(
"_")[0]
490 calibId += f
" raftName={raftName}"
493 calibId += f
" raftName={raftname}"
495 serial = detector.getSerial()
496 linearizer.updateMetadata(instrumentName=instruName, detectorId=f
"{detNum}",
497 calibId=calibId, serial=serial, detectorName=f
"{detName}")
502 """Calculate the mean signal of two exposures and the variance of their difference.
506 exposure1 : `lsst.afw.image.exposure.exposure.ExposureF`
507 First exposure of flat field pair.
509 exposure2 : `lsst.afw.image.exposure.exposure.ExposureF`
510 Second exposure of flat field pair.
512 region : `lsst.geom.Box2I`
513 Region of each exposure where to perform the calculations (e.g, an amplifier).
519 0.5*(mu1 + mu2), where mu1, and mu2 are the clipped means of the regions in
523 Half of the clipped variance of the difference of the regions inthe two input
527 if region
is not None:
528 im1Area = exposure1.maskedImage[region]
529 im2Area = exposure2.maskedImage[region]
531 im1Area = exposure1.maskedImage
532 im2Area = exposure2.maskedImage
534 im1Area = afwMath.binImage(im1Area, self.config.binSize)
535 im2Area = afwMath.binImage(im2Area, self.config.binSize)
538 mu1 = afwMath.makeStatistics(im1Area, afwMath.MEANCLIP).getValue()
539 mu2 = afwMath.makeStatistics(im2Area, afwMath.MEANCLIP).getValue()
544 temp = im2Area.clone()
546 diffIm = im1Area.clone()
551 varDiff = 0.5*(afwMath.makeStatistics(diffIm, afwMath.VARIANCECLIP).getValue())
555 def _fitLeastSq(self, initialParams, dataX, dataY, function):
556 """Do a fit and estimate the parameter errors using using scipy.optimize.leastq.
558 optimize.leastsq returns the fractional covariance matrix. To estimate the
559 standard deviation of the fit parameters, multiply the entries of this matrix
560 by the unweighted reduced chi squared and take the square root of the diagonal elements.
564 initialParams : `list` of `float`
565 initial values for fit parameters. For ptcFitType=POLYNOMIAL, its length
566 determines the degree of the polynomial.
568 dataX : `numpy.array` of `float`
569 Data in the abscissa axis.
571 dataY : `numpy.array` of `float`
572 Data in the ordinate axis.
574 function : callable object (function)
575 Function to fit the data with.
579 pFitSingleLeastSquares : `list` of `float`
580 List with fitted parameters.
582 pErrSingleLeastSquares : `list` of `float`
583 List with errors for fitted parameters.
585 reducedChiSqSingleLeastSquares : `float`
586 Unweighted reduced chi squared
589 def errFunc(p, x, y):
590 return function(p, x) - y
592 pFit, pCov, infoDict, errMessage, success = leastsq(errFunc, initialParams,
593 args=(dataX, dataY), full_output=1, epsfcn=0.0001)
595 if (len(dataY) > len(initialParams))
and pCov
is not None:
596 reducedChiSq = (errFunc(pFit, dataX, dataY)**2).sum()/(len(dataY)-len(initialParams))
599 pCov = np.zeros((len(initialParams), len(initialParams)))
601 reducedChiSq = np.inf
604 for i
in range(len(pFit)):
605 errorVec.append(np.fabs(pCov[i][i])**0.5)
607 pFitSingleLeastSquares = pFit
608 pErrSingleLeastSquares = np.array(errorVec)
610 return pFitSingleLeastSquares, pErrSingleLeastSquares, reducedChiSq
612 def _fitBootstrap(self, initialParams, dataX, dataY, function, confidenceSigma=1.):
613 """Do a fit using least squares and bootstrap to estimate parameter errors.
615 The bootstrap error bars are calculated by fitting 100 random data sets.
619 initialParams : `list` of `float`
620 initial values for fit parameters. For ptcFitType=POLYNOMIAL, its length
621 determines the degree of the polynomial.
623 dataX : `numpy.array` of `float`
624 Data in the abscissa axis.
626 dataY : `numpy.array` of `float`
627 Data in the ordinate axis.
629 function : callable object (function)
630 Function to fit the data with.
632 confidenceSigma : `float`
633 Number of sigmas that determine confidence interval for the bootstrap errors.
637 pFitBootstrap : `list` of `float`
638 List with fitted parameters.
640 pErrBootstrap : `list` of `float`
641 List with errors for fitted parameters.
643 reducedChiSqBootstrap : `float`
647 def errFunc(p, x, y):
648 return function(p, x) - y
651 pFit, _ = leastsq(errFunc, initialParams, args=(dataX, dataY), full_output=0)
654 residuals = errFunc(pFit, dataX, dataY)
655 sigmaErrTotal = np.std(residuals)
660 randomDelta = np.random.normal(0., sigmaErrTotal, len(dataY))
661 randomDataY = dataY + randomDelta
662 randomFit, _ = leastsq(errFunc, initialParams,
663 args=(dataX, randomDataY), full_output=0)
664 pars.append(randomFit)
665 pars = np.array(pars)
666 meanPfit = np.mean(pars, 0)
669 nSigma = confidenceSigma
670 errPfit = nSigma*np.std(pars, 0)
671 pFitBootstrap = meanPfit
672 pErrBootstrap = errPfit
674 reducedChiSq = (errFunc(pFitBootstrap, dataX, dataY)**2).sum()/(len(dataY)-len(initialParams))
675 return pFitBootstrap, pErrBootstrap, reducedChiSq
678 """Polynomial function definition"""
679 return poly.polyval(x, [*pars])
682 """Single brighter-fatter parameter model for PTC; Equation 16 of Astier+19"""
683 a00, gain, noise = pars
684 return 0.5/(a00*gain*gain)*(np.exp(2*a00*x*gain)-1) + noise/(gain*gain)
687 def _initialParsForPolynomial(order):
689 pars = np.zeros(order, dtype=np.float)
696 def _boundsForPolynomial(initialPars):
697 lowers = [np.NINF
for p
in initialPars]
698 uppers = [np.inf
for p
in initialPars]
700 return (lowers, uppers)
703 def _boundsForAstier(initialPars):
704 lowers = [np.NINF
for p
in initialPars]
705 uppers = [np.inf
for p
in initialPars]
706 return (lowers, uppers)
709 def _getInitialGoodPoints(means, variances, maxDeviationPositive, maxDeviationNegative):
710 """Return a boolean array to mask bad points.
712 A linear function has a constant ratio, so find the median
713 value of the ratios, and exclude the points that deviate
714 from that by more than a factor of maxDeviationPositive/negative.
715 Asymmetric deviations are supported as we expect the PTC to turn
716 down as the flux increases, but sometimes it anomalously turns
717 upwards just before turning over, which ruins the fits, so it
718 is wise to be stricter about restricting positive outliers than
721 Too high and points that are so bad that fit will fail will be included
722 Too low and the non-linear points will be excluded, biasing the NL fit."""
723 ratios = [b/a
for (a, b)
in zip(means, variances)]
724 medianRatio = np.median(ratios)
725 ratioDeviations = [(r/medianRatio)-1
for r
in ratios]
728 maxDeviationPositive = abs(maxDeviationPositive)
729 maxDeviationNegative = -1. * abs(maxDeviationNegative)
731 goodPoints = np.array([
True if (r < maxDeviationPositive
and r > maxDeviationNegative)
732 else False for r
in ratioDeviations])
735 def _makeZeroSafe(self, array, warn=True, substituteValue=1e-9):
737 nBad = Counter(array)[0]
742 msg = f
"Found {nBad} zeros in array at elements {[x for x in np.where(array==0)[0]]}"
745 array[array == 0] = substituteValue
749 """Calculate linearity residual and fit an n-order polynomial to the mean vs time curve
750 to produce corrections (deviation from linear part of polynomial) for a particular amplifier
751 to populate LinearizeLookupTable.
752 Use the coefficients of this fit to calculate the correction coefficients for LinearizePolynomial
753 and LinearizeSquared."
758 exposureTimeVector: `list` of `float`
759 List of exposure times for each flat pair
761 meanSignalVector: `list` of `float`
762 List of mean signal from diference image of flat pairs
766 dataset : `lsst.cp.pipe.ptc.LinearityResidualsAndLinearizersDataset`
767 The dataset containing the fit parameters, the NL correction coefficients, and the
768 LUT row for the amplifier at hand. Explicitly:
770 dataset.polynomialLinearizerCoefficients : `list` of `float`
771 Coefficients for LinearizePolynomial, where corrImage = uncorrImage + sum_i c_i uncorrImage^(2 +
773 c_(j-2) = -k_j/(k_1^j) with units (DN^(1-j)). The units of k_j are DN/t^j, and they are fit from
774 meanSignalVector = k0 + k1*exposureTimeVector + k2*exposureTimeVector^2 +...
775 + kn*exposureTimeVector^n, with n = "polynomialFitDegreeNonLinearity".
776 k_0 and k_1 and degenerate with bias level and gain, and are not used by the non-linearity
777 correction. Therefore, j = 2...n in the above expression (see `LinearizePolynomial` class in
780 dataset.quadraticPolynomialLinearizerCoefficient : `float`
781 Coefficient for LinearizeSquared, where corrImage = uncorrImage + c0*uncorrImage^2.
782 c0 = -k2/(k1^2), where k1 and k2 are fit from
783 meanSignalVector = k0 + k1*exposureTimeVector + k2*exposureTimeVector^2 +...
784 + kn*exposureTimeVector^n, with n = "polynomialFitDegreeNonLinearity".
786 dataset.linearizerTableRow : `list` of `float`
787 One dimensional array with deviation from linear part of n-order polynomial fit
788 to mean vs time curve. This array will be one row (for the particular amplifier at hand)
789 of the table array for LinearizeLookupTable.
791 dataset.linearityResidual : `list` of `float`
792 Linearity residual from the mean vs time curve, defined as
793 100*(1 - meanSignalReference/expTimeReference/(meanSignal/expTime).
795 dataset.meanSignalVsTimePolyFitPars : `list` of `float`
796 Parameters from n-order polynomial fit to meanSignalVector vs exposureTimeVector.
798 dataset.meanSignalVsTimePolyFitParsErr : `list` of `float`
799 Parameters from n-order polynomial fit to meanSignalVector vs exposureTimeVector.
801 dataset.fractionalNonLinearityResidual : `list` of `float`
802 Fractional residuals from the meanSignal vs exposureTime curve with respect to linear part of
803 polynomial fit: 100*(linearPart - meanSignal)/linearPart, where
804 linearPart = k0 + k1*exposureTimeVector.
806 dataset.meanSignalVsTimePolyFitReducedChiSq : `float`
807 Reduced unweighted chi squared from polynomial fit to meanSignalVector vs exposureTimeVector.
812 if self.config.doFitBootstrap:
813 parsFit, parsFitErr, reducedChiSquaredNonLinearityFit = self.
_fitBootstrap(parsIniNonLinearity,
818 parsFit, parsFitErr, reducedChiSquaredNonLinearityFit = self.
_fitLeastSq(parsIniNonLinearity,
825 tMax = (self.config.maxAduForLookupTableLinearizer - parsFit[0])/parsFit[1]
826 timeRange = np.linspace(0, tMax, self.config.maxAduForLookupTableLinearizer)
827 signalIdeal = parsFit[0] + parsFit[1]*timeRange
829 linearizerTableRow = signalIdeal - signalUncorrected
835 polynomialLinearizerCoefficients = []
836 for i, coefficient
in enumerate(parsFit):
837 c = -coefficient/(k1**i)
838 polynomialLinearizerCoefficients.append(c)
839 if np.fabs(c) > 1e-10:
840 msg = f
"Coefficient {c} in polynomial fit larger than threshold 1e-10."
843 c0 = polynomialLinearizerCoefficients[2]
846 linResidualTimeIndex = self.config.linResidualTimeIndex
847 if exposureTimeVector[linResidualTimeIndex] == 0.0:
848 raise RuntimeError(
"Reference time for linearity residual can't be 0.0")
849 linResidual = 100*(1 - ((meanSignalVector[linResidualTimeIndex] /
850 exposureTimeVector[linResidualTimeIndex]) /
851 (meanSignalVector/exposureTimeVector)))
854 linearPart = parsFit[0] + k1*exposureTimeVector
855 fracNonLinearityResidual = 100*(linearPart - meanSignalVector)/linearPart
858 dataset.polynomialLinearizerCoefficients = polynomialLinearizerCoefficients
859 dataset.quadraticPolynomialLinearizerCoefficient = c0
860 dataset.linearizerTableRow = linearizerTableRow
861 dataset.linearityResidual = linResidual
862 dataset.meanSignalVsTimePolyFitPars = parsFit
863 dataset.meanSignalVsTimePolyFitParsErr = parsFitErr
864 dataset.fractionalNonLinearityResidual = fracNonLinearityResidual
865 dataset.meanSignalVsTimePolyFitReducedChiSq = reducedChiSquaredNonLinearityFit
870 """Fit the photon transfer curve and calculate linearity and residuals.
872 Fit the photon transfer curve with either a polynomial of the order
873 specified in the task config, or using the Astier approximation.
875 Sigma clipping is performed iteratively for the fit, as well as an
876 initial clipping of data points that are more than
877 config.initialNonLinearityExclusionThreshold away from lying on a
878 straight line. This other step is necessary because the photon transfer
879 curve turns over catastrophically at very high flux (because saturation
880 drops the variance to ~0) and these far outliers cause the initial fit
881 to fail, meaning the sigma cannot be calculated to perform the
886 dataset : `lsst.cp.pipe.ptc.PhotonTransferCurveDataset`
887 The dataset containing the means, variances and exposure times
889 Fit a 'POLYNOMIAL' (degree: 'polynomialFitDegree') or
890 'ASTIERAPPROXIMATION' to the PTC
891 tableArray : `np.array`
892 Optional. Look-up table array with size rows=nAmps and columns=DN values.
893 It will be modified in-place if supplied.
897 dataset: `lsst.cp.pipe.ptc.PhotonTransferCurveDataset`
898 This is the same dataset as the input paramter, however, it has been modified
899 to include information such as the fit vectors and the fit parameters. See
900 the class `PhotonTransferCurveDatase`.
903 def errFunc(p, x, y):
904 return ptcFunc(p, x) - y
906 sigmaCutPtcOutliers = self.config.sigmaCutPtcOutliers
907 maxIterationsPtcOutliers = self.config.maxIterationsPtcOutliers
909 for i, ampName
in enumerate(dataset.ampNames):
910 timeVecOriginal = np.array(dataset.rawExpTimes[ampName])
911 meanVecOriginal = np.array(dataset.rawMeans[ampName])
912 varVecOriginal = np.array(dataset.rawVars[ampName])
915 mask = ((meanVecOriginal >= self.config.minMeanSignal) &
916 (meanVecOriginal <= self.config.maxMeanSignal))
919 self.config.initialNonLinearityExclusionThresholdPositive,
920 self.config.initialNonLinearityExclusionThresholdNegative)
921 mask = mask & goodPoints
923 if ptcFitType ==
'ASTIERAPPROXIMATION':
925 parsIniPtc = [-1e-9, 1.0, 10.]
927 if ptcFitType ==
'POLYNOMIAL':
934 while count <= maxIterationsPtcOutliers:
938 meanTempVec = meanVecOriginal[mask]
939 varTempVec = varVecOriginal[mask]
940 res = least_squares(errFunc, parsIniPtc, bounds=bounds, args=(meanTempVec, varTempVec))
946 sigResids = (varVecOriginal - ptcFunc(pars, meanVecOriginal))/np.sqrt(varVecOriginal)
947 newMask = np.array([
True if np.abs(r) < sigmaCutPtcOutliers
else False for r
in sigResids])
948 mask = mask & newMask
950 nDroppedTotal = Counter(mask)[
False]
951 self.log.debug(f
"Iteration {count}: discarded {nDroppedTotal} points in total for {ampName}")
954 assert (len(mask) == len(timeVecOriginal) == len(meanVecOriginal) == len(varVecOriginal))
956 dataset.visitMask[ampName] = mask
959 timeVecFinal = timeVecOriginal[mask]
960 meanVecFinal = meanVecOriginal[mask]
961 varVecFinal = varVecOriginal[mask]
963 if Counter(mask)[
False] > 0:
964 self.log.info((f
"Number of points discarded in PTC of amplifier {ampName}:" +
965 f
" {Counter(mask)[False]} out of {len(meanVecOriginal)}"))
967 if (len(meanVecFinal) < len(parsIniPtc)):
968 msg = (f
"\nSERIOUS: Not enough data points ({len(meanVecFinal)}) compared to the number of"
969 f
"parameters of the PTC model({len(parsIniPtc)}). Setting {ampName} to BAD.")
973 lenNonLinPars = self.config.polynomialFitDegreeNonLinearity - 1
974 dataset.badAmps.append(ampName)
975 dataset.gain[ampName] = np.nan
976 dataset.gainErr[ampName] = np.nan
977 dataset.noise[ampName] = np.nan
978 dataset.noiseErr[ampName] = np.nan
979 dataset.nonLinearity[ampName] = np.nan
980 dataset.nonLinearityError[ampName] = np.nan
981 dataset.nonLinearityResiduals[ampName] = np.nan
982 dataset.fractionalNonLinearityResiduals[ampName] = np.nan
983 dataset.coefficientLinearizeSquared[ampName] = np.nan
984 dataset.ptcFitPars[ampName] = np.nan
985 dataset.ptcFitParsError[ampName] = np.nan
986 dataset.ptcFitReducedChiSquared[ampName] = np.nan
987 dataset.coefficientsLinearizePolynomial[ampName] = [np.nan]*lenNonLinPars
988 tableArray[i, :] = [np.nan]*self.config.maxAduForLookupTableLinearizer
992 if self.config.doFitBootstrap:
993 parsFit, parsFitErr, reducedChiSqPtc = self.
_fitBootstrap(parsIniPtc, meanVecFinal,
994 varVecFinal, ptcFunc)
996 parsFit, parsFitErr, reducedChiSqPtc = self.
_fitLeastSq(parsIniPtc, meanVecFinal,
997 varVecFinal, ptcFunc)
999 dataset.ptcFitPars[ampName] = parsFit
1000 dataset.ptcFitParsError[ampName] = parsFitErr
1001 dataset.ptcFitReducedChiSquared[ampName] = reducedChiSqPtc
1003 if ptcFitType ==
'ASTIERAPPROXIMATION':
1004 ptcGain = parsFit[1]
1005 ptcGainErr = parsFitErr[1]
1006 ptcNoise = np.sqrt(np.fabs(parsFit[2]))
1007 ptcNoiseErr = 0.5*(parsFitErr[2]/np.fabs(parsFit[2]))*np.sqrt(np.fabs(parsFit[2]))
1008 if ptcFitType ==
'POLYNOMIAL':
1009 ptcGain = 1./parsFit[1]
1010 ptcGainErr = np.fabs(1./parsFit[1])*(parsFitErr[1]/parsFit[1])
1011 ptcNoise = np.sqrt(np.fabs(parsFit[0]))*ptcGain
1012 ptcNoiseErr = (0.5*(parsFitErr[0]/np.fabs(parsFit[0]))*(np.sqrt(np.fabs(parsFit[0]))))*ptcGain
1014 dataset.gain[ampName] = ptcGain
1015 dataset.gainErr[ampName] = ptcGainErr
1016 dataset.noise[ampName] = ptcNoise
1017 dataset.noiseErr[ampName] = ptcNoiseErr
1018 dataset.ptcFitType[ampName] = ptcFitType
1026 if tableArray
is not None:
1027 tableArray[i, :] = datasetLinRes.linearizerTableRow
1028 dataset.nonLinearity[ampName] = datasetLinRes.meanSignalVsTimePolyFitPars
1029 dataset.nonLinearityError[ampName] = datasetLinRes.meanSignalVsTimePolyFitParsErr
1030 dataset.nonLinearityResiduals[ampName] = datasetLinRes.linearityResidual
1031 dataset.fractionalNonLinearityResiduals[ampName] = datasetLinRes.fractionalNonLinearityResidual
1032 dataset.nonLinearityReducedChiSquared[ampName] = datasetLinRes.meanSignalVsTimePolyFitReducedChiSq
1036 polyLinCoeffs = np.array(datasetLinRes.polynomialLinearizerCoefficients[2:])
1037 dataset.coefficientsLinearizePolynomial[ampName] = polyLinCoeffs
1038 quadPolyLinCoeff = datasetLinRes.quadraticPolynomialLinearizerCoefficient
1039 dataset.coefficientLinearizeSquared[ampName] = quadPolyLinCoeff
1043 def plot(self, dataRef, dataset, ptcFitType):
1044 dirname = dataRef.getUri(datasetType=
'cpPipePlotRoot', write=
True)
1045 if not os.path.exists(dirname):
1046 os.makedirs(dirname)
1048 detNum = dataRef.dataId[self.config.ccdKey]
1049 filename = f
"PTC_det{detNum}.pdf"
1050 filenameFull = os.path.join(dirname, filename)
1051 with PdfPages(filenameFull)
as pdfPages:
1052 self.
_plotPtc(dataset, ptcFitType, pdfPages)
1054 def _plotPtc(self, dataset, ptcFitType, pdfPages):
1055 """Plot PTC, linearity, and linearity residual per amplifier"""
1057 reducedChiSqPtc = dataset.ptcFitReducedChiSquared
1058 if ptcFitType ==
'ASTIERAPPROXIMATION':
1060 stringTitle = (
r"Var = $\frac{1}{2g^2a_{00}}(\exp (2a_{00} \mu g) - 1) + \frac{n_{00}}{g^2}$ "
1061 r" ($chi^2$/dof = %g)" % (reducedChiSqPtc))
1062 if ptcFitType ==
'POLYNOMIAL':
1064 stringTitle =
r"Polynomial (degree: %g)" % (self.config.polynomialFitDegree)
1069 supTitleFontSize = 18
1073 nAmps = len(dataset.ampNames)
1076 nRows = np.sqrt(nAmps)
1077 mantissa, _ = np.modf(nRows)
1079 nRows = int(nRows) + 1
1085 f, ax = plt.subplots(nrows=nRows, ncols=nCols, sharex=
'col', sharey=
'row', figsize=(13, 10))
1086 f2, ax2 = plt.subplots(nrows=nRows, ncols=nCols, sharex=
'col', sharey=
'row', figsize=(13, 10))
1088 for i, (amp, a, a2)
in enumerate(zip(dataset.ampNames, ax.flatten(), ax2.flatten())):
1089 meanVecOriginal = np.array(dataset.rawMeans[amp])
1090 varVecOriginal = np.array(dataset.rawVars[amp])
1091 mask = dataset.visitMask[amp]
1092 meanVecFinal = meanVecOriginal[mask]
1093 varVecFinal = varVecOriginal[mask]
1094 meanVecOutliers = meanVecOriginal[np.invert(mask)]
1095 varVecOutliers = varVecOriginal[np.invert(mask)]
1096 pars, parsErr = dataset.ptcFitPars[amp], dataset.ptcFitParsError[amp]
1098 if ptcFitType ==
'ASTIERAPPROXIMATION':
1099 if len(meanVecFinal):
1100 ptcA00, ptcA00error = pars[0], parsErr[0]
1101 ptcGain, ptcGainError = pars[1], parsErr[1]
1102 ptcNoise = np.sqrt(np.fabs(pars[2]))
1103 ptcNoiseError = 0.5*(parsErr[2]/np.fabs(pars[2]))*np.sqrt(np.fabs(pars[2]))
1104 stringLegend = (f
"a00: {ptcA00:.2e}+/-{ptcA00error:.2e} 1/e"
1105 f
"\n Gain: {ptcGain:.4}+/-{ptcGainError:.2e} e/DN"
1106 f
"\n Noise: {ptcNoise:.4}+/-{ptcNoiseError:.2e} e \n")
1108 if ptcFitType ==
'POLYNOMIAL':
1109 if len(meanVecFinal):
1110 ptcGain, ptcGainError = 1./pars[1], np.fabs(1./pars[1])*(parsErr[1]/pars[1])
1111 ptcNoise = np.sqrt(np.fabs(pars[0]))*ptcGain
1112 ptcNoiseError = (0.5*(parsErr[0]/np.fabs(pars[0]))*(np.sqrt(np.fabs(pars[0]))))*ptcGain
1113 stringLegend = (f
"Noise: {ptcNoise:.4}+/-{ptcNoiseError:.2e} e \n"
1114 f
"Gain: {ptcGain:.4}+/-{ptcGainError:.2e} e/DN \n")
1116 a.set_xlabel(
r'Mean signal ($\mu$, DN)', fontsize=labelFontSize)
1117 a.set_ylabel(
r'Variance (DN$^2$)', fontsize=labelFontSize)
1118 a.tick_params(labelsize=11)
1119 a.set_xscale(
'linear', fontsize=labelFontSize)
1120 a.set_yscale(
'linear', fontsize=labelFontSize)
1122 a2.set_xlabel(
r'Mean Signal ($\mu$, DN)', fontsize=labelFontSize)
1123 a2.set_ylabel(
r'Variance (DN$^2$)', fontsize=labelFontSize)
1124 a2.tick_params(labelsize=11)
1125 a2.set_xscale(
'log')
1126 a2.set_yscale(
'log')
1128 if len(meanVecFinal):
1129 minMeanVecFinal = np.min(meanVecFinal)
1130 maxMeanVecFinal = np.max(meanVecFinal)
1131 meanVecFit = np.linspace(minMeanVecFinal, maxMeanVecFinal, 100*len(meanVecFinal))
1132 minMeanVecOriginal = np.min(meanVecOriginal)
1133 maxMeanVecOriginal = np.max(meanVecOriginal)
1134 deltaXlim = maxMeanVecOriginal - minMeanVecOriginal
1136 a.plot(meanVecFit, ptcFunc(pars, meanVecFit), color=
'red')
1137 a.plot(meanVecFinal, pars[0] + pars[1]*meanVecFinal, color=
'green', linestyle=
'--')
1138 a.scatter(meanVecFinal, varVecFinal, c=
'blue', marker=
'o', s=markerSize)
1139 a.scatter(meanVecOutliers, varVecOutliers, c=
'magenta', marker=
's', s=markerSize)
1140 a.text(0.03, 0.8, stringLegend, transform=a.transAxes, fontsize=legendFontSize)
1141 a.set_title(amp, fontsize=titleFontSize)
1142 a.set_xlim([minMeanVecOriginal - 0.2*deltaXlim, maxMeanVecOriginal + 0.2*deltaXlim])
1145 a2.plot(meanVecFit, ptcFunc(pars, meanVecFit), color=
'red')
1146 a2.scatter(meanVecFinal, varVecFinal, c=
'blue', marker=
'o', s=markerSize)
1147 a2.scatter(meanVecOutliers, varVecOutliers, c=
'magenta', marker=
's', s=markerSize)
1148 a2.text(0.03, 0.8, stringLegend, transform=a2.transAxes, fontsize=legendFontSize)
1149 a2.set_title(amp, fontsize=titleFontSize)
1150 a2.set_xlim([minMeanVecOriginal, maxMeanVecOriginal])
1152 a.set_title(f
"{amp} (BAD)", fontsize=titleFontSize)
1153 a2.set_title(f
"{amp} (BAD)", fontsize=titleFontSize)
1155 f.suptitle(f
"PTC \n Fit: " + stringTitle, fontsize=20)
1157 f2.suptitle(f
"PTC (log-log)", fontsize=20)
1158 pdfPages.savefig(f2)
1161 f, ax = plt.subplots(nrows=4, ncols=4, sharex=
'col', sharey=
'row', figsize=(13, 10))
1162 for i, (amp, a)
in enumerate(zip(dataset.ampNames, ax.flatten())):
1163 meanVecFinal = np.array(dataset.rawMeans[amp])[dataset.visitMask[amp]]
1164 timeVecFinal = np.array(dataset.rawExpTimes[amp])[dataset.visitMask[amp]]
1166 a.set_xlabel(
'Time (sec)', fontsize=labelFontSize)
1167 a.set_ylabel(
r'Mean signal ($\mu$, DN)', fontsize=labelFontSize)
1168 a.tick_params(labelsize=labelFontSize)
1169 a.set_xscale(
'linear', fontsize=labelFontSize)
1170 a.set_yscale(
'linear', fontsize=labelFontSize)
1172 if len(meanVecFinal):
1173 pars, parsErr = dataset.nonLinearity[amp], dataset.nonLinearityError[amp]
1174 k0, k0Error = pars[0], parsErr[0]
1175 k1, k1Error = pars[1], parsErr[1]
1176 k2, k2Error = pars[2], parsErr[2]
1177 stringLegend = (f
"k0: {k0:.4}+/-{k0Error:.2e} DN\n k1: {k1:.4}+/-{k1Error:.2e} DN/t"
1178 f
"\n k2: {k2:.2e}+/-{k2Error:.2e} DN/t^2 \n")
1179 a.scatter(timeVecFinal, meanVecFinal)
1180 a.plot(timeVecFinal, self.
funcPolynomial(pars, timeVecFinal), color=
'red')
1181 a.text(0.03, 0.75, stringLegend, transform=a.transAxes, fontsize=legendFontSize)
1182 a.set_title(f
"{amp}", fontsize=titleFontSize)
1184 a.set_title(f
"{amp} (BAD)", fontsize=titleFontSize)
1186 f.suptitle(
"Linearity \n Fit: Polynomial (degree: %g)"
1187 % (self.config.polynomialFitDegreeNonLinearity),
1188 fontsize=supTitleFontSize)
1192 f, ax = plt.subplots(nrows=4, ncols=4, sharex=
'col', sharey=
'row', figsize=(13, 10))
1193 for i, (amp, a)
in enumerate(zip(dataset.ampNames, ax.flatten())):
1194 meanVecFinal = np.array(dataset.rawMeans[amp])[dataset.visitMask[amp]]
1195 linRes = np.array(dataset.nonLinearityResiduals[amp])
1197 a.set_xlabel(
r'Mean signal ($\mu$, DN)', fontsize=labelFontSize)
1198 a.set_ylabel(
'LR (%)', fontsize=labelFontSize)
1199 a.set_xscale(
'linear', fontsize=labelFontSize)
1200 a.set_yscale(
'linear', fontsize=labelFontSize)
1201 if len(meanVecFinal):
1202 a.axvline(x=timeVecFinal[self.config.linResidualTimeIndex], color=
'g', linestyle=
'--')
1203 a.scatter(meanVecFinal, linRes)
1204 a.set_title(f
"{amp}", fontsize=titleFontSize)
1205 a.axhline(y=0, color=
'k')
1206 a.tick_params(labelsize=labelFontSize)
1208 a.set_title(f
"{amp} (BAD)", fontsize=titleFontSize)
1210 f.suptitle(
r"Linearity Residual: $100\times(1 - \mu_{\rm{ref}}/t_{\rm{ref}})/(\mu / t))$" +
"\n" +
1211 r"$t_{\rm{ref}}$: " + f
"{timeVecFinal[2]} s", fontsize=supTitleFontSize)
1215 f, ax = plt.subplots(nrows=4, ncols=4, sharex=
'col', sharey=
'row', figsize=(13, 10))
1216 for i, (amp, a)
in enumerate(zip(dataset.ampNames, ax.flatten())):
1217 meanVecFinal = np.array(dataset.rawMeans[amp])[dataset.visitMask[amp]]
1218 fracLinRes = np.array(dataset.fractionalNonLinearityResiduals[amp])
1220 a.axhline(y=0, color=
'k')
1221 a.axvline(x=0, color=
'k', linestyle=
'-')
1222 a.set_xlabel(
r'Mean signal ($\mu$, DN)', fontsize=labelFontSize)
1223 a.set_ylabel(
'Fractional nonlinearity (%)', fontsize=labelFontSize)
1224 a.tick_params(labelsize=labelFontSize)
1225 a.set_xscale(
'linear', fontsize=labelFontSize)
1226 a.set_yscale(
'linear', fontsize=labelFontSize)
1228 if len(meanVecFinal):
1229 a.scatter(meanVecFinal, fracLinRes, c=
'g')
1230 a.set_title(amp, fontsize=titleFontSize)
1232 a.set_title(f
"{amp} (BAD)", fontsize=titleFontSize)
1234 f.suptitle(
r"Fractional NL residual" +
"\n" +
1235 r"$100\times \frac{(k_0 + k_1*Time-\mu)}{k_0+k_1*Time}$",
1236 fontsize=supTitleFontSize)