Coverage for python/lsst/cp/pipe/ptc.py : 11%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of cp_pipe.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21#
23__all__ = ['MeasurePhotonTransferCurveTask',
24 'MeasurePhotonTransferCurveTaskConfig',
25 'PhotonTransferCurveDataset']
27import numpy as np
28import matplotlib.pyplot as plt
29import os
30from matplotlib.backends.backend_pdf import PdfPages
31from sqlite3 import OperationalError
32from collections import Counter
34import lsst.afw.math as afwMath
35import lsst.pex.config as pexConfig
36import lsst.pipe.base as pipeBase
37from lsst.ip.isr import IsrTask
38from .utils import (NonexistentDatasetTaskDataIdContainer, PairedVisitListTaskRunner,
39 checkExpLengthEqual, validateIsrConfig)
40from scipy.optimize import leastsq, least_squares
41import numpy.polynomial.polynomial as poly
44class MeasurePhotonTransferCurveTaskConfig(pexConfig.Config):
45 """Config class for photon transfer curve measurement task"""
46 isr = pexConfig.ConfigurableField(
47 target=IsrTask,
48 doc="""Task to perform instrumental signature removal.""",
49 )
50 isrMandatorySteps = pexConfig.ListField(
51 dtype=str,
52 doc="isr operations that must be performed for valid results. Raises if any of these are False.",
53 default=['doAssembleCcd']
54 )
55 isrForbiddenSteps = pexConfig.ListField(
56 dtype=str,
57 doc="isr operations that must NOT be performed for valid results. Raises if any of these are True",
58 default=['doFlat', 'doFringe', 'doAddDistortionModel', 'doBrighterFatter', 'doUseOpticsTransmission',
59 'doUseFilterTransmission', 'doUseSensorTransmission', 'doUseAtmosphereTransmission']
60 )
61 isrDesirableSteps = pexConfig.ListField(
62 dtype=str,
63 doc="isr operations that it is advisable to perform, but are not mission-critical." +
64 " WARNs are logged for any of these found to be False.",
65 default=['doBias', 'doDark', 'doCrosstalk', 'doDefect']
66 )
67 isrUndesirableSteps = pexConfig.ListField(
68 dtype=str,
69 doc="isr operations that it is *not* advisable to perform in the general case, but are not" +
70 " forbidden as some use-cases might warrant them." +
71 " WARNs are logged for any of these found to be True.",
72 default=['doLinearize']
73 )
74 ccdKey = pexConfig.Field(
75 dtype=str,
76 doc="The key by which to pull a detector from a dataId, e.g. 'ccd' or 'detector'.",
77 default='ccd',
78 )
79 makePlots = pexConfig.Field(
80 dtype=bool,
81 doc="Plot the PTC curves?",
82 default=False,
83 )
84 ptcFitType = pexConfig.ChoiceField(
85 dtype=str,
86 doc="Fit PTC to approximation in Astier+19 (Equation 16) or to a polynomial.",
87 default="POLYNOMIAL",
88 allowed={
89 "POLYNOMIAL": "n-degree polynomial (use 'polynomialFitDegree' to set 'n').",
90 "ASTIERAPPROXIMATION": "Approximation in Astier+19 (Eq. 16)."
91 }
92 )
93 polynomialFitDegree = pexConfig.Field(
94 dtype=int,
95 doc="Degree of polynomial to fit the PTC, when 'ptcFitType'=POLYNOMIAL.",
96 default=2,
97 )
98 polynomialFitDegreeNonLinearity = pexConfig.Field(
99 dtype=int,
100 doc="Degree of polynomial to fit the meanSignal vs exposureTime curve to produce" +
101 " the table for LinearizerLookupTable.",
102 default=3,
103 )
104 binSize = pexConfig.Field(
105 dtype=int,
106 doc="Bin the image by this factor in both dimensions.",
107 default=1,
108 )
109 minMeanSignal = pexConfig.Field(
110 dtype=float,
111 doc="Minimum value (inclusive) of mean signal (in ADU) above which to consider.",
112 default=0,
113 )
114 maxMeanSignal = pexConfig.Field(
115 dtype=float,
116 doc="Maximum value (inclusive) of mean signal (in ADU) below which to consider.",
117 default=9e6,
118 )
119 initialNonLinearityExclusionThresholdPositive = pexConfig.RangeField(
120 dtype=float,
121 doc="Initially exclude data points with a variance that are more than a factor of this from being"
122 " linear in the positive direction, from the PTC fit. Note that these points will also be"
123 " excluded from the non-linearity fit. This is done before the iterative outlier rejection,"
124 " to allow an accurate determination of the sigmas for said iterative fit.",
125 default=0.12,
126 min=0.0,
127 max=1.0,
128 )
129 initialNonLinearityExclusionThresholdNegative = pexConfig.RangeField(
130 dtype=float,
131 doc="Initially exclude data points with a variance that are more than a factor of this from being"
132 " linear in the negative direction, from the PTC fit. Note that these points will also be"
133 " excluded from the non-linearity fit. This is done before the iterative outlier rejection,"
134 " to allow an accurate determination of the sigmas for said iterative fit.",
135 default=0.25,
136 min=0.0,
137 max=1.0,
138 )
139 sigmaCutPtcOutliers = pexConfig.Field(
140 dtype=float,
141 doc="Sigma cut for outlier rejection in PTC.",
142 default=5.0,
143 )
144 maxIterationsPtcOutliers = pexConfig.Field(
145 dtype=int,
146 doc="Maximum number of iterations for outlier rejection in PTC.",
147 default=2,
148 )
149 doFitBootstrap = pexConfig.Field(
150 dtype=bool,
151 doc="Use bootstrap for the PTC fit parameters and errors?.",
152 default=False,
153 )
154 linResidualTimeIndex = pexConfig.Field(
155 dtype=int,
156 doc="Index position in time array for reference time in linearity residual calculation.",
157 default=2,
158 )
159 maxAduForLookupTableLinearizer = pexConfig.Field(
160 dtype=int,
161 doc="Maximum ADU value for the LookupTable linearizer.",
162 default=2**18,
163 )
166class PhotonTransferCurveDataset:
167 """A simple class to hold the output data from the PTC task.
169 The dataset is made up of a dictionary for each item, keyed by the
170 amplifiers' names, which much be supplied at construction time.
172 New items cannot be added to the class to save accidentally saving to the
173 wrong property, and the class can be frozen if desired.
175 inputVisitPairs records the visits used to produce the data.
176 When fitPtcAndNonLinearity() is run, a mask is built up, which is by definition
177 always the same length as inputVisitPairs, rawExpTimes, rawMeans
178 and rawVars, and is a list of bools, which are incrementally set to False
179 as points are discarded from the fits.
181 PTC fit parameters for polynomials are stored in a list in ascending order
182 of polynomial term, i.e. par[0]*x^0 + par[1]*x + par[2]*x^2 etc
183 with the length of the list corresponding to the order of the polynomial
184 plus one.
185 """
186 def __init__(self, ampNames):
187 # add items to __dict__ directly because __setattr__ is overridden
189 # instance variables
190 self.__dict__["ampNames"] = ampNames
191 self.__dict__["badAmps"] = []
193 # raw data variables
194 self.__dict__["inputVisitPairs"] = {ampName: [] for ampName in ampNames}
195 self.__dict__["visitMask"] = {ampName: [] for ampName in ampNames}
196 self.__dict__["rawExpTimes"] = {ampName: [] for ampName in ampNames}
197 self.__dict__["rawMeans"] = {ampName: [] for ampName in ampNames}
198 self.__dict__["rawVars"] = {ampName: [] for ampName in ampNames}
200 # fit information
201 self.__dict__["ptcFitType"] = {ampName: "" for ampName in ampNames}
202 self.__dict__["ptcFitPars"] = {ampName: [] for ampName in ampNames}
203 self.__dict__["ptcFitParsError"] = {ampName: [] for ampName in ampNames}
204 self.__dict__["nonLinearity"] = {ampName: [] for ampName in ampNames}
205 self.__dict__["nonLinearityError"] = {ampName: [] for ampName in ampNames}
206 self.__dict__["nonLinearityResiduals"] = {ampName: [] for ampName in ampNames}
208 # final results
209 self.__dict__["gain"] = {ampName: -1. for ampName in ampNames}
210 self.__dict__["gainErr"] = {ampName: -1. for ampName in ampNames}
211 self.__dict__["noise"] = {ampName: -1. for ampName in ampNames}
212 self.__dict__["noiseErr"] = {ampName: -1. for ampName in ampNames}
213 self.__dict__["coefficientLinearizeSquared"] = {ampName: [] for ampName in ampNames}
215 def __setattr__(self, attribute, value):
216 """Protect class attributes"""
217 if attribute not in self.__dict__:
218 raise AttributeError(f"{attribute} is not already a member of PhotonTransferCurveDataset, which"
219 " does not support setting of new attributes.")
220 else:
221 self.__dict__[attribute] = value
223 def getVisitsUsed(self, ampName):
224 """Get the visits used, i.e. not discarded, for a given amp.
226 If no mask has been created yet, all visits are returned.
227 """
228 if self.visitMask[ampName] == []:
229 return self.inputVisitPairs[ampName]
231 # if the mask exists it had better be the same length as the visitPairs
232 assert len(self.visitMask[ampName]) == len(self.inputVisitPairs[ampName])
234 pairs = self.inputVisitPairs[ampName]
235 mask = self.visitMask[ampName]
236 # cast to bool required because numpy
237 return [(v1, v2) for ((v1, v2), m) in zip(pairs, mask) if bool(m) is True]
239 def getGoodAmps(self):
240 return [amp for amp in self.ampNames if amp not in self.badAmps]
243class MeasurePhotonTransferCurveTask(pipeBase.CmdLineTask):
244 """A class to calculate, fit, and plot a PTC from a set of flat pairs.
246 The Photon Transfer Curve (var(signal) vs mean(signal)) is a standard tool
247 used in astronomical detectors characterization (e.g., Janesick 2001,
248 Janesick 2007). This task calculates the PTC from a series of pairs of
249 flat-field images; each pair taken at identical exposure times. The
250 difference image of each pair is formed to eliminate fixed pattern noise,
251 and then the variance of the difference image and the mean of the average image
252 are used to produce the PTC. An n-degree polynomial or the approximation in Equation
253 16 of Astier+19 ("The Shape of the Photon Transfer Curve of CCD sensors",
254 arXiv:1905.08677) can be fitted to the PTC curve. These models include
255 parameters such as the gain (e/ADU) and readout noise.
257 Parameters
258 ----------
260 *args: `list`
261 Positional arguments passed to the Task constructor. None used at this
262 time.
263 **kwargs: `dict`
264 Keyword arguments passed on to the Task constructor. None used at this
265 time.
267 """
269 RunnerClass = PairedVisitListTaskRunner
270 ConfigClass = MeasurePhotonTransferCurveTaskConfig
271 _DefaultName = "measurePhotonTransferCurve"
273 def __init__(self, *args, **kwargs):
274 pipeBase.CmdLineTask.__init__(self, *args, **kwargs)
275 self.makeSubtask("isr")
276 plt.interactive(False) # stop windows popping up when plotting. When headless, use 'agg' backend too
277 validateIsrConfig(self.isr, self.config.isrMandatorySteps,
278 self.config.isrForbiddenSteps, self.config.isrDesirableSteps, checkTrim=False)
279 self.config.validate()
280 self.config.freeze()
282 @classmethod
283 def _makeArgumentParser(cls):
284 """Augment argument parser for the MeasurePhotonTransferCurveTask."""
285 parser = pipeBase.ArgumentParser(name=cls._DefaultName)
286 parser.add_argument("--visit-pairs", dest="visitPairs", nargs="*",
287 help="Visit pairs to use. Each pair must be of the form INT,INT e.g. 123,456")
288 parser.add_id_argument("--id", datasetType="photonTransferCurveDataset",
289 ContainerClass=NonexistentDatasetTaskDataIdContainer,
290 help="The ccds to use, e.g. --id ccd=0..100")
291 return parser
293 @pipeBase.timeMethod
294 def runDataRef(self, dataRef, visitPairs):
295 """Run the Photon Transfer Curve (PTC) measurement task.
297 For a dataRef (which is each detector here),
298 and given a list of visit pairs at different exposure times,
299 measure the PTC.
301 Parameters
302 ----------
303 dataRef : list of lsst.daf.persistence.ButlerDataRef
304 dataRef for the detector for the visits to be fit.
305 visitPairs : `iterable` of `tuple` of `int`
306 Pairs of visit numbers to be processed together
307 """
309 # setup necessary objects
310 detNum = dataRef.dataId[self.config.ccdKey]
311 detector = dataRef.get('camera')[dataRef.dataId[self.config.ccdKey]]
312 # expand some missing fields that we need for lsstCam. This is a work-around
313 # for Gen2 problems that I (RHL) don't feel like solving. The calibs pipelines
314 # (which inherit from CalibTask) use addMissingKeys() to do basically the same thing
315 #
316 # Basically, the butler's trying to look up the fields in `raw_visit` which won't work
317 for name in dataRef.getButler().getKeys('bias'):
318 if name not in dataRef.dataId:
319 try:
320 dataRef.dataId[name] = \
321 dataRef.getButler().queryMetadata('raw', [name], detector=detNum)[0]
322 except OperationalError:
323 pass
325 amps = detector.getAmplifiers()
326 ampNames = [amp.getName() for amp in amps]
327 dataset = PhotonTransferCurveDataset(ampNames)
329 self.log.info('Measuring PTC using %s visits for detector %s' % (visitPairs, detNum))
331 for (v1, v2) in visitPairs:
332 # Perform ISR on each exposure
333 dataRef.dataId['expId'] = v1
334 exp1 = self.isr.runDataRef(dataRef).exposure
335 dataRef.dataId['expId'] = v2
336 exp2 = self.isr.runDataRef(dataRef).exposure
337 del dataRef.dataId['expId']
339 checkExpLengthEqual(exp1, exp2, v1, v2, raiseWithMessage=True)
340 expTime = exp1.getInfo().getVisitInfo().getExposureTime()
342 for amp in detector:
343 mu, varDiff = self.measureMeanVarPair(exp1, exp2, region=amp.getBBox())
344 ampName = amp.getName()
346 dataset.rawExpTimes[ampName].append(expTime)
347 dataset.rawMeans[ampName].append(mu)
348 dataset.rawVars[ampName].append(varDiff)
349 dataset.inputVisitPairs[ampName].append((v1, v2))
351 numberAmps = len(detector.getAmplifiers())
352 numberAduValues = self.config.maxAduForLookupTableLinearizer
353 lookupTableArray = np.zeros((numberAmps, numberAduValues), dtype=np.float32)
355 # Fit PTC and (non)linearity of signal vs time curve.
356 # Fill up PhotonTransferCurveDataset object.
357 # Fill up array for LUT linearizer.
358 dataset = self.fitPtcAndNonLinearity(dataset, tableArray=lookupTableArray,
359 ptcFitType=self.config.ptcFitType)
361 if self.config.makePlots:
362 self.plot(dataRef, dataset, ptcFitType=self.config.ptcFitType)
364 # Save data, PTC fit, and NL fit dictionaries
365 self.log.info(f"Writing PTC and NL data to {dataRef.getUri(write=True)}")
366 dataRef.put(dataset, datasetType="photonTransferCurveDataset")
368 self.log.info('Finished measuring PTC for in detector %s' % detNum)
370 return pipeBase.Struct(exitStatus=0)
372 def measureMeanVarPair(self, exposure1, exposure2, region=None):
373 """Calculate the mean signal of two exposures and the variance of their difference.
375 Parameters
376 ----------
377 exposure1 : `lsst.afw.image.exposure.exposure.ExposureF`
378 First exposure of flat field pair.
380 exposure2 : `lsst.afw.image.exposure.exposure.ExposureF`
381 Second exposure of flat field pair.
383 region : `lsst.geom.Box2I`
384 Region of each exposure where to perform the calculations (e.g, an amplifier).
386 Return
387 ------
389 mu : `np.float`
390 0.5*(mu1 + mu2), where mu1, and mu2 are the clipped means of the regions in
391 both exposures.
393 varDiff : `np.float`
394 Half of the clipped variance of the difference of the regions inthe two input
395 exposures.
396 """
398 if region is not None:
399 im1Area = exposure1.maskedImage[region]
400 im2Area = exposure2.maskedImage[region]
401 else:
402 im1Area = exposure1.maskedImage
403 im2Area = exposure2.maskedImage
405 im1Area = afwMath.binImage(im1Area, self.config.binSize)
406 im2Area = afwMath.binImage(im2Area, self.config.binSize)
408 # Clipped mean of images; then average of mean.
409 mu1 = afwMath.makeStatistics(im1Area, afwMath.MEANCLIP).getValue()
410 mu2 = afwMath.makeStatistics(im2Area, afwMath.MEANCLIP).getValue()
411 mu = 0.5*(mu1 + mu2)
413 # Take difference of pairs
414 # symmetric formula: diff = (mu2*im1-mu1*im2)/(0.5*(mu1+mu2))
415 temp = im2Area.clone()
416 temp *= mu1
417 diffIm = im1Area.clone()
418 diffIm *= mu2
419 diffIm -= temp
420 diffIm /= mu
422 varDiff = 0.5*(afwMath.makeStatistics(diffIm, afwMath.VARIANCECLIP).getValue())
424 return mu, varDiff
426 def _fitLeastSq(self, initialParams, dataX, dataY, function):
427 """Do a fit and estimate the parameter errors using using scipy.optimize.leastq.
429 optimize.leastsq returns the fractional covariance matrix. To estimate the
430 standard deviation of the fit parameters, multiply the entries of this matrix
431 by the reduced chi squared and take the square root of the diagonal elements.
433 Parameters
434 ----------
435 initialParams : list of np.float
436 initial values for fit parameters. For ptcFitType=POLYNOMIAL, its length
437 determines the degree of the polynomial.
439 dataX : np.array of np.float
440 Data in the abscissa axis.
442 dataY : np.array of np.float
443 Data in the ordinate axis.
445 function : callable object (function)
446 Function to fit the data with.
448 Return
449 ------
450 pFitSingleLeastSquares : list of np.float
451 List with fitted parameters.
453 pErrSingleLeastSquares : list of np.float
454 List with errors for fitted parameters.
455 """
457 def errFunc(p, x, y):
458 return function(p, x) - y
460 pFit, pCov, infoDict, errMessage, success = leastsq(errFunc, initialParams,
461 args=(dataX, dataY), full_output=1, epsfcn=0.0001)
463 if (len(dataY) > len(initialParams)) and pCov is not None:
464 reducedChiSq = (errFunc(pFit, dataX, dataY)**2).sum()/(len(dataY)-len(initialParams))
465 pCov *= reducedChiSq
466 else:
467 pCov[:, :] = np.inf
469 errorVec = []
470 for i in range(len(pFit)):
471 errorVec.append(np.fabs(pCov[i][i])**0.5)
473 pFitSingleLeastSquares = pFit
474 pErrSingleLeastSquares = np.array(errorVec)
476 return pFitSingleLeastSquares, pErrSingleLeastSquares
478 def _fitBootstrap(self, initialParams, dataX, dataY, function, confidenceSigma=1.):
479 """Do a fit using least squares and bootstrap to estimate parameter errors.
481 The bootstrap error bars are calculated by fitting 100 random data sets.
483 Parameters
484 ----------
485 initialParams : list of np.float
486 initial values for fit parameters. For ptcFitType=POLYNOMIAL, its length
487 determines the degree of the polynomial.
489 dataX : np.array of np.float
490 Data in the abscissa axis.
492 dataY : np.array of np.float
493 Data in the ordinate axis.
495 function : callable object (function)
496 Function to fit the data with.
498 confidenceSigma : np.float
499 Number of sigmas that determine confidence interval for the bootstrap errors.
501 Return
502 ------
503 pFitBootstrap : list of np.float
504 List with fitted parameters.
506 pErrBootstrap : list of np.float
507 List with errors for fitted parameters.
508 """
510 def errFunc(p, x, y):
511 return function(p, x) - y
513 # Fit first time
514 pFit, _ = leastsq(errFunc, initialParams, args=(dataX, dataY), full_output=0)
516 # Get the stdev of the residuals
517 residuals = errFunc(pFit, dataX, dataY)
518 sigmaErrTotal = np.std(residuals)
520 # 100 random data sets are generated and fitted
521 pars = []
522 for i in range(100):
523 randomDelta = np.random.normal(0., sigmaErrTotal, len(dataY))
524 randomDataY = dataY + randomDelta
525 randomFit, _ = leastsq(errFunc, initialParams,
526 args=(dataX, randomDataY), full_output=0)
527 pars.append(randomFit)
528 pars = np.array(pars)
529 meanPfit = np.mean(pars, 0)
531 # confidence interval for parameter estimates
532 nSigma = confidenceSigma
533 errPfit = nSigma*np.std(pars, 0)
534 pFitBootstrap = meanPfit
535 pErrBootstrap = errPfit
536 return pFitBootstrap, pErrBootstrap
538 def funcPolynomial(self, pars, x):
539 """Polynomial function definition"""
540 return poly.polyval(x, [*pars])
542 def funcAstier(self, pars, x):
543 """Single brighter-fatter parameter model for PTC; Equation 16 of Astier+19"""
544 a00, gain, noise = pars
545 return 0.5/(a00*gain*gain)*(np.exp(2*a00*x*gain)-1) + noise/(gain*gain)
547 @staticmethod
548 def _initialParsForPolynomial(order):
549 assert(order >= 2)
550 pars = np.zeros(order, dtype=np.float)
551 pars[0] = 10
552 pars[1] = 1
553 pars[2:] = 0.0001
554 return pars
556 @staticmethod
557 def _boundsForPolynomial(initialPars):
558 lowers = [np.NINF for p in initialPars]
559 uppers = [np.inf for p in initialPars]
560 lowers[1] = 0 # no negative gains
561 return (lowers, uppers)
563 @staticmethod
564 def _boundsForAstier(initialPars):
565 lowers = [np.NINF for p in initialPars]
566 uppers = [np.inf for p in initialPars]
567 return (lowers, uppers)
569 @staticmethod
570 def _getInitialGoodPoints(means, variances, maxDeviationPositive, maxDeviationNegative):
571 """Return a boolean array to mask bad points.
573 A linear function has a constant ratio, so find the median
574 value of the ratios, and exclude the points that deviate
575 from that by more than a factor of maxDeviationPositive/negative.
576 Asymmetric deviations are supported as we expect the PTC to turn
577 down as the flux increases, but sometimes it anomalously turns
578 upwards just before turning over, which ruins the fits, so it
579 is wise to be stricter about restricting positive outliers than
580 negative ones.
582 Too high and points that are so bad that fit will fail will be included
583 Too low and the non-linear points will be excluded, biasing the NL fit."""
584 ratios = [b/a for (a, b) in zip(means, variances)]
585 medianRatio = np.median(ratios)
586 ratioDeviations = [(r/medianRatio)-1 for r in ratios]
588 # so that it doesn't matter if the deviation is expressed as positive or negative
589 maxDeviationPositive = abs(maxDeviationPositive)
590 maxDeviationNegative = -1. * abs(maxDeviationNegative)
592 goodPoints = np.array([True if (r < maxDeviationPositive and r > maxDeviationNegative)
593 else False for r in ratioDeviations])
594 return goodPoints
596 def _makeZeroSafe(self, array, warn=True, substituteValue=1e-9):
597 """"""
598 nBad = Counter(array)[0]
599 if nBad == 0:
600 return array
602 if warn:
603 msg = f"Found {nBad} zeros in array at elements {[x for x in np.where(array==0)[0]]}"
604 self.log.warn(msg)
606 array[array == 0] = substituteValue
607 return array
609 def calculateLinearityResidualAndLinearizers(self, exposureTimeVector, meanSignalVector):
610 """Calculate linearity residual and fit an n-order polynomial to the mean vs time curve
611 to produce corrections (deviation from linear part of polynomial) for a particular amplifier
612 to populate LinearizeLookupTable. Use quadratic and linear parts of this polynomial to approximate
613 c0 for LinearizeSquared."
615 Parameters
616 ---------
618 exposureTimeVector: `list` of `np.float`
619 List of exposure times for each flat pair
621 meanSignalVector: `list` of `np.float`
622 List of mean signal from diference image of flat pairs
624 Returns
625 -------
626 c0: `np.float`
627 Coefficient for LinearizeSquared, where corrImage = uncorrImage + c0*uncorrImage^2.
628 c0 ~ -k2/(k1^2), where k1 and k2 are fit from
629 meanSignalVector = k0 + k1*exposureTimeVector + k2*exposureTimeVector^2 +...
630 + kn*exposureTimeVector^n, with n = "polynomialFitDegreeNonLinearity".
632 linearizerTableRow: list of `np.float`
633 One dimensional array with deviation from linear part of n-order polynomial fit
634 to mean vs time curve. This array will be one row (for the particular amplifier at hand)
635 of the table array for LinearizeLookupTable.
637 linResidual: list of `np.float`
638 Linearity residual from the mean vs time curve, defined as
639 100*(1 - meanSignalReference/expTimeReference/(meanSignal/expTime).
641 parsFit: list of `np.float`
642 Parameters from n-order polynomial fit to mean vs time curve.
644 parsFitErr: list of `np.float`
645 Parameters from n-order polynomial fit to mean vs time curve.
647 """
649 # Lookup table linearizer
650 parsIniNonLinearity = self._initialParsForPolynomial(self.config.polynomialFitDegreeNonLinearity + 1)
651 if self.config.doFitBootstrap:
652 parsFit, parsFitErr = self._fitBootstrap(parsIniNonLinearity, exposureTimeVector,
653 meanSignalVector, self.funcPolynomial)
654 else:
655 parsFit, parsFitErr = self._fitLeastSq(parsIniNonLinearity, exposureTimeVector, meanSignalVector,
656 self.funcPolynomial)
658 # Use linear part to get time at wich signal is maxAduForLookupTableLinearizer ADU
659 tMax = (self.config.maxAduForLookupTableLinearizer - parsFit[0])/parsFit[1]
660 timeRange = np.linspace(0, tMax, self.config.maxAduForLookupTableLinearizer)
661 signalIdeal = (parsFit[0] + parsFit[1]*timeRange).astype(int)
662 signalUncorrected = (self.funcPolynomial(parsFit, timeRange)).astype(int)
663 linearizerTableRow = signalIdeal - signalUncorrected # LinearizerLookupTable has corrections
665 # Use quadratic and linear part of fit to produce c0 for LinearizeSquared
666 # Check that magnitude of higher order (>= 3) coefficents of the polyFit are small,
667 # i.e., less than threshold = 1e-10 (typical quadratic and cubic coefficents are ~1e-6
668 # and ~1e-12).
669 k1, k2 = parsFit[1], parsFit[2]
670 c0 = -k2/(k1**2) # c0 coefficient for LinearizeSquared
671 for coefficient in parsFit[3:]:
672 if np.fabs(coefficient) > 1e-10:
673 msg = f"Coefficient {coefficient} in polynomial fit larger than threshold 1e-10."
674 self.log.warn(msg)
676 # Linearity residual
677 linResidualTimeIndex = self.config.linResidualTimeIndex
678 if exposureTimeVector[linResidualTimeIndex] == 0.0:
679 raise RuntimeError("Reference time for linearity residual can't be 0.0")
680 linResidual = 100*(1 - ((meanSignalVector[linResidualTimeIndex] /
681 exposureTimeVector[linResidualTimeIndex]) /
682 (meanSignalVector/exposureTimeVector)))
684 return c0, linearizerTableRow, linResidual, parsFit, parsFitErr
686 def fitPtcAndNonLinearity(self, dataset, ptcFitType, tableArray=None):
687 """Fit the photon transfer curve and calculate linearity and residuals.
689 Fit the photon transfer curve with either a polynomial of the order
690 specified in the task config, or using the Astier approximation.
692 Sigma clipping is performed iteratively for the fit, as well as an
693 initial clipping of data points that are more than
694 config.initialNonLinearityExclusionThreshold away from lying on a
695 straight line. This other step is necessary because the photon transfer
696 curve turns over catastrophically at very high flux (because saturation
697 drops the variance to ~0) and these far outliers cause the initial fit
698 to fail, meaning the sigma cannot be calculated to perform the
699 sigma-clipping.
701 Parameters
702 ----------
703 dataset : `lsst.cp.pipe.ptc.PhotonTransferCurveDataset`
704 The dataset containing the means, variances and exposure times
705 ptcFitType : `str`
706 Fit a 'POLYNOMIAL' (degree: 'polynomialFitDegree') or
707 'ASTIERAPPROXIMATION' to the PTC
708 tableArray : `np.array`
709 Optional. Look-up table array with size rows=nAmps and columns=ADU values.
710 It will be modified in-place if supplied.
712 Returns
713 -------
714 dataset: `lsst.cp.pipe.ptc.PhotonTransferCurveDataset`
715 This is the same dataset as the input paramter, however, it has been modified
716 to include information such as the fit vectors and the fit parameters. See
717 the class `PhotonTransferCurveDatase`.
718 """
720 def errFunc(p, x, y):
721 return ptcFunc(p, x) - y
723 sigmaCutPtcOutliers = self.config.sigmaCutPtcOutliers
724 maxIterationsPtcOutliers = self.config.maxIterationsPtcOutliers
726 for i, ampName in enumerate(dataset.ampNames):
727 timeVecOriginal = np.array(dataset.rawExpTimes[ampName])
728 meanVecOriginal = np.array(dataset.rawMeans[ampName])
729 varVecOriginal = np.array(dataset.rawVars[ampName])
730 varVecOriginal = self._makeZeroSafe(varVecOriginal)
732 mask = ((meanVecOriginal >= self.config.minMeanSignal) &
733 (meanVecOriginal <= self.config.maxMeanSignal))
735 goodPoints = self._getInitialGoodPoints(meanVecOriginal, varVecOriginal,
736 self.config.initialNonLinearityExclusionThresholdPositive,
737 self.config.initialNonLinearityExclusionThresholdNegative)
738 mask = mask & goodPoints
740 if ptcFitType == 'ASTIERAPPROXIMATION':
741 ptcFunc = self.funcAstier
742 parsIniPtc = [-1e-9, 1.0, 10.] # a00, gain, noise
743 bounds = self._boundsForAstier(parsIniPtc)
744 if ptcFitType == 'POLYNOMIAL':
745 ptcFunc = self.funcPolynomial
746 parsIniPtc = self._initialParsForPolynomial(self.config.polynomialFitDegree + 1)
747 bounds = self._boundsForPolynomial(parsIniPtc)
749 # Before bootstrap fit, do an iterative fit to get rid of outliers
750 count = 1
751 while count <= maxIterationsPtcOutliers:
752 # Note that application of the mask actually shrinks the array
753 # to size rather than setting elements to zero (as we want) so
754 # always update mask itself and re-apply to the original data
755 meanTempVec = meanVecOriginal[mask]
756 varTempVec = varVecOriginal[mask]
757 res = least_squares(errFunc, parsIniPtc, bounds=bounds, args=(meanTempVec, varTempVec))
758 pars = res.x
760 # change this to the original from the temp because the masks are ANDed
761 # meaning once a point is masked it's always masked, and the masks must
762 # always be the same length for broadcasting
763 sigResids = (varVecOriginal - ptcFunc(pars, meanVecOriginal))/np.sqrt(varVecOriginal)
764 newMask = np.array([True if np.abs(r) < sigmaCutPtcOutliers else False for r in sigResids])
765 mask = mask & newMask
767 nDroppedTotal = Counter(mask)[False]
768 self.log.debug(f"Iteration {count}: discarded {nDroppedTotal} points in total for {ampName}")
769 count += 1
770 # objects should never shrink
771 assert (len(mask) == len(timeVecOriginal) == len(meanVecOriginal) == len(varVecOriginal))
773 dataset.visitMask[ampName] = mask # store the final mask
775 parsIniPtc = pars
776 timeVecFinal = timeVecOriginal[mask]
777 meanVecFinal = meanVecOriginal[mask]
778 varVecFinal = varVecOriginal[mask]
780 if Counter(mask)[False] > 0:
781 self.log.info((f"Number of points discarded in PTC of amplifier {ampName}:" +
782 f" {Counter(mask)[False]} out of {len(meanVecOriginal)}"))
784 if (len(meanVecFinal) < len(parsIniPtc)):
785 msg = (f"\nSERIOUS: Not enough data points ({len(meanVecFinal)}) compared to the number of"
786 f"parameters of the PTC model({len(parsIniPtc)}). Setting {ampName} to BAD.")
787 self.log.warn(msg)
788 dataset.badAmps.append(ampName)
789 dataset.gain[ampName] = np.nan
790 dataset.gainErr[ampName] = np.nan
791 dataset.noise[ampName] = np.nan
792 dataset.noiseErr[ampName] = np.nan
793 dataset.nonLinearity[ampName] = np.nan
794 dataset.nonLinearityError[ampName] = np.nan
795 dataset.nonLinearityResiduals[ampName] = np.nan
796 dataset.coefficientLinearizeSquared[ampName] = np.nan
797 continue
799 # Fit the PTC
800 if self.config.doFitBootstrap:
801 parsFit, parsFitErr = self._fitBootstrap(parsIniPtc, meanVecFinal, varVecFinal, ptcFunc)
802 else:
803 parsFit, parsFitErr = self._fitLeastSq(parsIniPtc, meanVecFinal, varVecFinal, ptcFunc)
805 dataset.ptcFitPars[ampName] = parsFit
806 dataset.ptcFitParsError[ampName] = parsFitErr
808 if ptcFitType == 'ASTIERAPPROXIMATION':
809 ptcGain = parsFit[1]
810 ptcGainErr = parsFitErr[1]
811 ptcNoise = np.sqrt(np.fabs(parsFit[2]))
812 ptcNoiseErr = 0.5*(parsFitErr[2]/np.fabs(parsFit[2]))*np.sqrt(np.fabs(parsFit[2]))
813 if ptcFitType == 'POLYNOMIAL':
814 ptcGain = 1./parsFit[1]
815 ptcGainErr = np.fabs(1./parsFit[1])*(parsFitErr[1]/parsFit[1])
816 ptcNoise = np.sqrt(np.fabs(parsFit[0]))*ptcGain
817 ptcNoiseErr = (0.5*(parsFitErr[0]/np.fabs(parsFit[0]))*(np.sqrt(np.fabs(parsFit[0]))))*ptcGain
819 dataset.gain[ampName] = ptcGain
820 dataset.gainErr[ampName] = ptcGainErr
821 dataset.noise[ampName] = ptcNoise
822 dataset.noiseErr[ampName] = ptcNoiseErr
823 dataset.ptcFitType[ampName] = ptcFitType
825 # Non-linearity residuals (NL of mean vs time curve): percentage, and fit to a quadratic function
826 # In this case, len(parsIniNonLinearity) = 3 indicates that we want a quadratic fit
828 (c0, linearizerTableRow, linResidualNonLinearity, parsFitNonLinearity,
829 parsFitErrNonLinearity) = self.calculateLinearityResidualAndLinearizers(timeVecFinal,
830 meanVecFinal)
831 # LinearizerLookupTable
832 if tableArray is not None:
833 tableArray[i, :] = linearizerTableRow
835 dataset.nonLinearity[ampName] = parsFitNonLinearity
836 dataset.nonLinearityError[ampName] = parsFitErrNonLinearity
837 dataset.nonLinearityResiduals[ampName] = linResidualNonLinearity
838 dataset.coefficientLinearizeSquared[ampName] = c0
840 return dataset
842 def plot(self, dataRef, dataset, ptcFitType):
843 dirname = dataRef.getUri(datasetType='cpPipePlotRoot', write=True)
844 if not os.path.exists(dirname):
845 os.makedirs(dirname)
847 detNum = dataRef.dataId[self.config.ccdKey]
848 filename = f"PTC_det{detNum}.pdf"
849 filenameFull = os.path.join(dirname, filename)
850 with PdfPages(filenameFull) as pdfPages:
851 self._plotPtc(dataset, ptcFitType, pdfPages)
853 def _plotPtc(self, dataset, ptcFitType, pdfPages):
854 """Plot PTC, linearity, and linearity residual per amplifier"""
856 if ptcFitType == 'ASTIERAPPROXIMATION':
857 ptcFunc = self.funcAstier
858 stringTitle = r"Var = $\frac{1}{2g^2a_{00}}(\exp (2a_{00} \mu g) - 1) + \frac{n_{00}}{g^2}$"
860 if ptcFitType == 'POLYNOMIAL':
861 ptcFunc = self.funcPolynomial
862 stringTitle = f"Polynomial (degree: {self.config.polynomialFitDegree})"
864 legendFontSize = 7.5
865 labelFontSize = 8
866 titleFontSize = 10
867 supTitleFontSize = 18
868 markerSize = 25
870 # General determination of the size of the plot grid
871 nAmps = len(dataset.ampNames)
872 if nAmps == 2:
873 nRows, nCols = 2, 1
874 nRows = np.sqrt(nAmps)
875 mantissa, _ = np.modf(nRows)
876 if mantissa > 0:
877 nRows = int(nRows) + 1
878 nCols = nRows
879 else:
880 nRows = int(nRows)
881 nCols = nRows
883 f, ax = plt.subplots(nrows=nRows, ncols=nCols, sharex='col', sharey='row', figsize=(13, 10))
884 f2, ax2 = plt.subplots(nrows=nRows, ncols=nCols, sharex='col', sharey='row', figsize=(13, 10))
886 for i, (amp, a, a2) in enumerate(zip(dataset.ampNames, ax.flatten(), ax2.flatten())):
887 meanVecOriginal = np.array(dataset.rawMeans[amp])
888 varVecOriginal = np.array(dataset.rawVars[amp])
889 mask = dataset.visitMask[amp]
890 meanVecFinal = meanVecOriginal[mask]
891 varVecFinal = varVecOriginal[mask]
892 meanVecOutliers = meanVecOriginal[np.invert(mask)]
893 varVecOutliers = varVecOriginal[np.invert(mask)]
894 pars, parsErr = dataset.ptcFitPars[amp], dataset.ptcFitParsError[amp]
896 if ptcFitType == 'ASTIERAPPROXIMATION':
897 ptcA00, ptcA00error = pars[0], parsErr[0]
898 ptcGain, ptcGainError = pars[1], parsErr[1]
899 ptcNoise = np.sqrt(np.fabs(pars[2]))
900 ptcNoiseError = 0.5*(parsErr[2]/np.fabs(pars[2]))*np.sqrt(np.fabs(pars[2]))
901 stringLegend = (f"a00: {ptcA00:.2e}+/-{ptcA00error:.2e}"
902 f"\n Gain: {ptcGain:.4}+/-{ptcGainError:.2e}"
903 f"\n Noise: {ptcNoise:.4}+/-{ptcNoiseError:.2e}")
905 if ptcFitType == 'POLYNOMIAL':
906 ptcGain, ptcGainError = 1./pars[1], np.fabs(1./pars[1])*(parsErr[1]/pars[1])
907 ptcNoise = np.sqrt(np.fabs(pars[0]))*ptcGain
908 ptcNoiseError = (0.5*(parsErr[0]/np.fabs(pars[0]))*(np.sqrt(np.fabs(pars[0]))))*ptcGain
909 stringLegend = (f"Noise: {ptcNoise:.4}+/-{ptcNoiseError:.2e} \n"
910 f"Gain: {ptcGain:.4}+/-{ptcGainError:.2e}")
912 minMeanVecFinal = np.min(meanVecFinal)
913 maxMeanVecFinal = np.max(meanVecFinal)
914 meanVecFit = np.linspace(minMeanVecFinal, maxMeanVecFinal, 100*len(meanVecFinal))
915 minMeanVecOriginal = np.min(meanVecOriginal)
916 maxMeanVecOriginal = np.max(meanVecOriginal)
917 deltaXlim = maxMeanVecOriginal - minMeanVecOriginal
919 a.plot(meanVecFit, ptcFunc(pars, meanVecFit), color='red')
920 a.plot(meanVecFinal, pars[0] + pars[1]*meanVecFinal, color='green', linestyle='--')
921 a.scatter(meanVecFinal, varVecFinal, c='blue', marker='o', s=markerSize)
922 a.scatter(meanVecOutliers, varVecOutliers, c='magenta', marker='s', s=markerSize)
923 a.set_xlabel(r'Mean signal ($\mu$, ADU)', fontsize=labelFontSize)
924 a.set_xticks(meanVecOriginal)
925 a.set_ylabel(r'Variance (ADU$^2$)', fontsize=labelFontSize)
926 a.tick_params(labelsize=11)
927 a.text(0.03, 0.8, stringLegend, transform=a.transAxes, fontsize=legendFontSize)
928 a.set_xscale('linear', fontsize=labelFontSize)
929 a.set_yscale('linear', fontsize=labelFontSize)
930 a.set_title(amp, fontsize=titleFontSize)
931 a.set_xlim([minMeanVecOriginal - 0.2*deltaXlim, maxMeanVecOriginal + 0.2*deltaXlim])
933 # Same, but in log-scale
934 a2.plot(meanVecFit, ptcFunc(pars, meanVecFit), color='red')
935 a2.scatter(meanVecFinal, varVecFinal, c='blue', marker='o', s=markerSize)
936 a2.scatter(meanVecOutliers, varVecOutliers, c='magenta', marker='s', s=markerSize)
937 a2.set_xlabel(r'Mean Signal ($\mu$, ADU)', fontsize=labelFontSize)
938 a2.set_ylabel(r'Variance (ADU$^2$)', fontsize=labelFontSize)
939 a2.tick_params(labelsize=11)
940 a2.text(0.03, 0.8, stringLegend, transform=a2.transAxes, fontsize=legendFontSize)
941 a2.set_xscale('log')
942 a2.set_yscale('log')
943 a2.set_title(amp, fontsize=titleFontSize)
944 a2.set_xlim([minMeanVecOriginal, maxMeanVecOriginal])
946 f.suptitle(f"PTC \n Fit: " + stringTitle, fontsize=20)
947 pdfPages.savefig(f)
948 f2.suptitle(f"PTC (log-log)", fontsize=20)
949 pdfPages.savefig(f2)
951 # Plot mean vs time
952 f, ax = plt.subplots(nrows=4, ncols=4, sharex='col', sharey='row', figsize=(13, 10))
953 for i, (amp, a) in enumerate(zip(dataset.ampNames, ax.flatten())):
954 meanVecFinal = np.array(dataset.rawMeans[amp])[dataset.visitMask[amp]]
955 timeVecFinal = np.array(dataset.rawExpTimes[amp])[dataset.visitMask[amp]]
957 pars, parsErr = dataset.nonLinearity[amp], dataset.nonLinearityError[amp]
958 c0, c0Error = pars[0], parsErr[0]
959 c1, c1Error = pars[1], parsErr[1]
960 c2, c2Error = pars[2], parsErr[2]
961 stringLegend = f"c0: {c0:.4}+/-{c0Error:.2e}\n c1: {c1:.4}+/-{c1Error:.2e}" \
962 + f"\n c2(NL): {c2:.2e}+/-{c2Error:.2e}"
963 a.scatter(timeVecFinal, meanVecFinal)
964 a.plot(timeVecFinal, self.funcPolynomial(pars, timeVecFinal), color='red')
965 a.set_xlabel('Time (sec)', fontsize=labelFontSize)
966 a.set_xticks(timeVecFinal)
967 a.set_ylabel(r'Mean signal ($\mu$, ADU)', fontsize=labelFontSize)
968 a.tick_params(labelsize=labelFontSize)
969 a.text(0.03, 0.75, stringLegend, transform=a.transAxes, fontsize=legendFontSize)
970 a.set_xscale('linear', fontsize=labelFontSize)
971 a.set_yscale('linear', fontsize=labelFontSize)
972 a.set_title(amp, fontsize=titleFontSize)
974 f.suptitle("Linearity \n Fit: " + r"$\mu = c_0 + c_1 t + c_2 t^2$", fontsize=supTitleFontSize)
975 pdfPages.savefig()
977 # Plot linearity residual
978 f, ax = plt.subplots(nrows=4, ncols=4, sharex='col', sharey='row', figsize=(13, 10))
979 for i, (amp, a) in enumerate(zip(dataset.ampNames, ax.flatten())):
980 meanVecFinal = np.array(dataset.rawMeans[amp])[dataset.visitMask[amp]]
981 linRes = np.array(dataset.nonLinearityResiduals[amp])
983 a.scatter(meanVecFinal, linRes)
984 a.axhline(y=0, color='k')
985 a.axvline(x=timeVecFinal[self.config.linResidualTimeIndex], color='g', linestyle='--')
986 a.set_xlabel(r'Mean signal ($\mu$, ADU)', fontsize=labelFontSize)
987 a.set_xticks(meanVecFinal)
988 a.set_ylabel('LR (%)', fontsize=labelFontSize)
989 a.tick_params(labelsize=labelFontSize)
990 a.set_xscale('linear', fontsize=labelFontSize)
991 a.set_yscale('linear', fontsize=labelFontSize)
992 a.set_title(amp, fontsize=titleFontSize)
994 f.suptitle(r"Linearity Residual: $100(1 - \mu_{\rm{ref}}/t_{\rm{ref}})/(\mu / t))$" + "\n" +
995 r"$t_{\rm{ref}}$: " + f"{timeVecFinal[2]} s", fontsize=supTitleFontSize)
996 pdfPages.savefig()
998 return