lsst.cp.pipe  19.0.0-2-gdbc0a5a+3
ptc.py
Go to the documentation of this file.
1 # This file is part of cp_pipe.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 #
22 
23 __all__ = ['MeasurePhotonTransferCurveTask',
24  'MeasurePhotonTransferCurveTaskConfig',
25  'PhotonTransferCurveDataset']
26 
27 import numpy as np
28 import matplotlib.pyplot as plt
29 import os
30 from matplotlib.backends.backend_pdf import PdfPages
31 from sqlite3 import OperationalError
32 from collections import Counter
33 
34 import lsst.afw.math as afwMath
35 import lsst.pex.config as pexConfig
36 import lsst.pipe.base as pipeBase
37 from lsst.ip.isr import IsrTask
38 from .utils import (NonexistentDatasetTaskDataIdContainer, PairedVisitListTaskRunner,
39  checkExpLengthEqual, validateIsrConfig)
40 from scipy.optimize import leastsq, least_squares
41 import numpy.polynomial.polynomial as poly
42 
43 
44 class MeasurePhotonTransferCurveTaskConfig(pexConfig.Config):
45  """Config class for photon transfer curve measurement task"""
46  isr = pexConfig.ConfigurableField(
47  target=IsrTask,
48  doc="""Task to perform instrumental signature removal.""",
49  )
50  isrMandatorySteps = pexConfig.ListField(
51  dtype=str,
52  doc="isr operations that must be performed for valid results. Raises if any of these are False.",
53  default=['doAssembleCcd']
54  )
55  isrForbiddenSteps = pexConfig.ListField(
56  dtype=str,
57  doc="isr operations that must NOT be performed for valid results. Raises if any of these are True",
58  default=['doFlat', 'doFringe', 'doAddDistortionModel', 'doBrighterFatter', 'doUseOpticsTransmission',
59  'doUseFilterTransmission', 'doUseSensorTransmission', 'doUseAtmosphereTransmission']
60  )
61  isrDesirableSteps = pexConfig.ListField(
62  dtype=str,
63  doc="isr operations that it is advisable to perform, but are not mission-critical." +
64  " WARNs are logged for any of these found to be False.",
65  default=['doBias', 'doDark', 'doCrosstalk', 'doDefect']
66  )
67  isrUndesirableSteps = pexConfig.ListField(
68  dtype=str,
69  doc="isr operations that it is *not* advisable to perform in the general case, but are not" +
70  " forbidden as some use-cases might warrant them." +
71  " WARNs are logged for any of these found to be True.",
72  default=['doLinearize']
73  )
74  ccdKey = pexConfig.Field(
75  dtype=str,
76  doc="The key by which to pull a detector from a dataId, e.g. 'ccd' or 'detector'.",
77  default='ccd',
78  )
79  makePlots = pexConfig.Field(
80  dtype=bool,
81  doc="Plot the PTC curves?",
82  default=False,
83  )
84  ptcFitType = pexConfig.ChoiceField(
85  dtype=str,
86  doc="Fit PTC to approximation in Astier+19 (Equation 16) or to a polynomial.",
87  default="POLYNOMIAL",
88  allowed={
89  "POLYNOMIAL": "n-degree polynomial (use 'polynomialFitDegree' to set 'n').",
90  "ASTIERAPPROXIMATION": "Approximation in Astier+19 (Eq. 16)."
91  }
92  )
93  polynomialFitDegree = pexConfig.Field(
94  dtype=int,
95  doc="Degree of polynomial to fit the PTC, when 'ptcFitType'=POLYNOMIAL.",
96  default=2,
97  )
98  binSize = pexConfig.Field(
99  dtype=int,
100  doc="Bin the image by this factor in both dimensions.",
101  default=1,
102  )
103  minMeanSignal = pexConfig.Field(
104  dtype=float,
105  doc="Minimum value (inclusive) of mean signal (in ADU) above which to consider.",
106  default=0,
107  )
108  maxMeanSignal = pexConfig.Field(
109  dtype=float,
110  doc="Maximum value (inclusive) of mean signal (in ADU) below which to consider.",
111  default=9e6,
112  )
113  initialNonLinearityExclusionThreshold = pexConfig.RangeField(
114  dtype=float,
115  doc="Initially exclude data points with a variance that are more than a factor of this from being"
116  " linear, from the PTC fit. Note that these points will also be excluded from the non-linearity"
117  " fit. This is done before the iterative outlier rejection, to allow an accurate determination"
118  " of the sigmas for said iterative fit.",
119  default=0.25,
120  min=0.0,
121  max=1.0,
122  )
123  sigmaCutPtcOutliers = pexConfig.Field(
124  dtype=float,
125  doc="Sigma cut for outlier rejection in PTC.",
126  default=4.0,
127  )
128  maxIterationsPtcOutliers = pexConfig.Field(
129  dtype=int,
130  doc="Maximum number of iterations for outlier rejection in PTC.",
131  default=2,
132  )
133  doFitBootstrap = pexConfig.Field(
134  dtype=bool,
135  doc="Use bootstrap for the PTC fit parameters and errors?.",
136  default=False,
137  )
138  linResidualTimeIndex = pexConfig.Field(
139  dtype=int,
140  doc="Index position in time array for reference time in linearity residual calculation.",
141  default=2,
142  )
143 
144 
146  """A simple class to hold the output data from the PTC task.
147 
148  The dataset is made up of a dictionary for each item, keyed by the
149  amplifiers' names, which much be supplied at construction time.
150 
151  New items cannot be added to the class to save accidentally saving to the
152  wrong property, and the class can be frozen if desired.
153 
154  inputVisitPairs records the visits used to produce the data.
155  When fitPtcAndNl() is run, a mask is built up, which is by definition
156  always the same length as inputVisitPairs, rawExpTimes, rawMeans
157  and rawVars, and is a list of bools, which are incrementally set to False
158  as points are discarded from the fits.
159  """
160  def __init__(self, ampNames):
161  # add items to __dict__ directly because __setattr__ is overridden
162 
163  # instance variables
164  self.__dict__["ampNames"] = ampNames
165 
166  # raw data variables
167  self.__dict__["inputVisitPairs"] = {ampName: [] for ampName in ampNames}
168  self.__dict__["visitMask"] = {ampName: [] for ampName in ampNames}
169  self.__dict__["rawExpTimes"] = {ampName: [] for ampName in ampNames}
170  self.__dict__["rawMeans"] = {ampName: [] for ampName in ampNames}
171  self.__dict__["rawVars"] = {ampName: [] for ampName in ampNames}
172 
173  # fit information
174  self.__dict__["ptcFitType"] = {ampName: "" for ampName in ampNames}
175  self.__dict__["ptcFitPars"] = {ampName: [] for ampName in ampNames}
176  self.__dict__["ptcFitParsError"] = {ampName: [] for ampName in ampNames}
177  self.__dict__["nonLinearity"] = {ampName: [] for ampName in ampNames}
178  self.__dict__["nonLinearityError"] = {ampName: [] for ampName in ampNames}
179  self.__dict__["nonLinearityResiduals"] = {ampName: [] for ampName in ampNames}
180 
181  # final results
182  self.__dict__["gain"] = {ampName: -1. for ampName in ampNames}
183  self.__dict__["gainErr"] = {ampName: -1. for ampName in ampNames}
184  self.__dict__["noise"] = {ampName: -1. for ampName in ampNames}
185  self.__dict__["noiseErr"] = {ampName: -1. for ampName in ampNames}
186 
187  def __setattr__(self, attribute, value):
188  """Protect class attributes"""
189  if attribute not in self.__dict__:
190  raise AttributeError(f"{attribute} is not already a member of PhotonTransferCurveDataset, which"
191  " does not support setting of new attributes.")
192  else:
193  self.__dict__[attribute] = value
194 
195 
196 class MeasurePhotonTransferCurveTask(pipeBase.CmdLineTask):
197  """A class to calculate, fit, and plot a PTC from a set of flat pairs.
198 
199  The Photon Transfer Curve (var(signal) vs mean(signal)) is a standard tool
200  used in astronomical detectors characterization (e.g., Janesick 2001,
201  Janesick 2007). This task calculates the PTC from a series of pairs of
202  flat-field images; each pair taken at identical exposure times. The
203  difference image of each pair is formed to eliminate fixed pattern noise,
204  and then the variance of the difference image and the mean of the average image
205  are used to produce the PTC. An n-degree polynomial or the approximation in Equation
206  16 of Astier+19 ("The Shape of the Photon Transfer Curve of CCD sensors",
207  arXiv:1905.08677) can be fitted to the PTC curve. These models include
208  parameters such as the gain (e/ADU) and readout noise.
209 
210  Parameters
211  ----------
212 
213  *args: `list`
214  Positional arguments passed to the Task constructor. None used at this
215  time.
216  **kwargs: `dict`
217  Keyword arguments passed on to the Task constructor. None used at this
218  time.
219 
220  """
221 
222  RunnerClass = PairedVisitListTaskRunner
223  ConfigClass = MeasurePhotonTransferCurveTaskConfig
224  _DefaultName = "measurePhotonTransferCurve"
225 
226  def __init__(self, *args, **kwargs):
227  pipeBase.CmdLineTask.__init__(self, *args, **kwargs)
228  self.makeSubtask("isr")
229  plt.interactive(False) # stop windows popping up when plotting. When headless, use 'agg' backend too
230  validateIsrConfig(self.isr, self.config.isrMandatorySteps,
231  self.config.isrForbiddenSteps, self.config.isrDesirableSteps, checkTrim=False)
232  self.config.validate()
233  self.config.freeze()
234 
235  @classmethod
236  def _makeArgumentParser(cls):
237  """Augment argument parser for the MeasurePhotonTransferCurveTask."""
238  parser = pipeBase.ArgumentParser(name=cls._DefaultName)
239  parser.add_argument("--visit-pairs", dest="visitPairs", nargs="*",
240  help="Visit pairs to use. Each pair must be of the form INT,INT e.g. 123,456")
241  parser.add_id_argument("--id", datasetType="photonTransferCurveDataset",
242  ContainerClass=NonexistentDatasetTaskDataIdContainer,
243  help="The ccds to use, e.g. --id ccd=0..100")
244  return parser
245 
246  @pipeBase.timeMethod
247  def runDataRef(self, dataRef, visitPairs):
248  """Run the Photon Transfer Curve (PTC) measurement task.
249 
250  For a dataRef (which is each detector here),
251  and given a list of visit pairs at different exposure times,
252  measure the PTC.
253 
254  Parameters
255  ----------
256  dataRef : list of lsst.daf.persistence.ButlerDataRef
257  dataRef for the detector for the visits to be fit.
258  visitPairs : `iterable` of `tuple` of `int`
259  Pairs of visit numbers to be processed together
260  """
261 
262  # setup necessary objects
263  detNum = dataRef.dataId[self.config.ccdKey]
264  detector = dataRef.get('camera')[dataRef.dataId[self.config.ccdKey]]
265  # expand some missing fields that we need for lsstCam. This is a work-around
266  # for Gen2 problems that I (RHL) don't feel like solving. The calibs pipelines
267  # (which inherit from CalibTask) use addMissingKeys() to do basically the same thing
268  #
269  # Basically, the butler's trying to look up the fields in `raw_visit` which won't work
270  for name in dataRef.getButler().getKeys('bias'):
271  if name not in dataRef.dataId:
272  try:
273  dataRef.dataId[name] = \
274  dataRef.getButler().queryMetadata('raw', [name], detector=detNum)[0]
275  except OperationalError:
276  pass
277 
278  amps = detector.getAmplifiers()
279  ampNames = [amp.getName() for amp in amps]
280  dataset = PhotonTransferCurveDataset(ampNames)
281 
282  self.log.info('Measuring PTC using %s visits for detector %s' % (visitPairs, detNum))
283 
284  for (v1, v2) in visitPairs:
285  # Perform ISR on each exposure
286  dataRef.dataId['visit'] = v1
287  exp1 = self.isr.runDataRef(dataRef).exposure
288  dataRef.dataId['visit'] = v2
289  exp2 = self.isr.runDataRef(dataRef).exposure
290  del dataRef.dataId['visit']
291 
292  checkExpLengthEqual(exp1, exp2, v1, v2, raiseWithMessage=True)
293  expTime = exp1.getInfo().getVisitInfo().getExposureTime()
294 
295  for amp in detector:
296  mu, varDiff = self.measureMeanVarPair(exp1, exp2, region=amp.getBBox())
297  ampName = amp.getName()
298 
299  dataset.rawExpTimes[ampName].append(expTime)
300  dataset.rawMeans[ampName].append(mu)
301  dataset.rawVars[ampName].append(varDiff)
302 
303  # Fit PTC and (non)linearity of signal vs time curve
304  self.fitPtcAndNl(dataset, ptcFitType=self.config.ptcFitType)
305 
306  if self.config.makePlots:
307  self.plot(dataRef, dataset, ptcFitType=self.config.ptcFitType)
308 
309  # Save data, PTC fit, and NL fit dictionaries
310  self.log.info(f"Writing PTC and NL data to {dataRef.getUri(write=True)}")
311  dataRef.put(dataset, datasetType="photonTransferCurveDataset")
312 
313  self.log.info('Finished measuring PTC for in detector %s' % detNum)
314 
315  return pipeBase.Struct(exitStatus=0)
316 
317  def measureMeanVarPair(self, exposure1, exposure2, region=None):
318  """Calculate the mean signal of two exposures and the variance of their difference.
319 
320  Parameters
321  ----------
322  exposure1 : `lsst.afw.image.exposure.exposure.ExposureF`
323  First exposure of flat field pair.
324 
325  exposure2 : `lsst.afw.image.exposure.exposure.ExposureF`
326  Second exposure of flat field pair.
327 
328  region : `lsst.geom.Box2I`
329  Region of each exposure where to perform the calculations (e.g, an amplifier).
330 
331  Return
332  ------
333 
334  mu : `np.float`
335  0.5*(mu1 + mu2), where mu1, and mu2 are the clipped means of the regions in
336  both exposures.
337 
338  varDiff : `np.float`
339  Half of the clipped variance of the difference of the regions inthe two input
340  exposures.
341  """
342 
343  if region is not None:
344  im1Area = exposure1.maskedImage[region]
345  im2Area = exposure2.maskedImage[region]
346  else:
347  im1Area = exposure1.maskedImage
348  im2Area = exposure2.maskedImage
349 
350  im1Area = afwMath.binImage(im1Area, self.config.binSize)
351  im2Area = afwMath.binImage(im2Area, self.config.binSize)
352 
353  # Clipped mean of images; then average of mean.
354  mu1 = afwMath.makeStatistics(im1Area, afwMath.MEANCLIP).getValue()
355  mu2 = afwMath.makeStatistics(im2Area, afwMath.MEANCLIP).getValue()
356  mu = 0.5*(mu1 + mu2)
357 
358  # Take difference of pairs
359  # symmetric formula: diff = (mu2*im1-mu1*im2)/(0.5*(mu1+mu2))
360  temp = im2Area.clone()
361  temp *= mu1
362  diffIm = im1Area.clone()
363  diffIm *= mu2
364  diffIm -= temp
365  diffIm /= mu
366 
367  varDiff = 0.5*(afwMath.makeStatistics(diffIm, afwMath.VARIANCECLIP).getValue())
368 
369  return mu, varDiff
370 
371  def _fitLeastSq(self, initialParams, dataX, dataY, function):
372  """Do a fit and estimate the parameter errors using using scipy.optimize.leastq.
373 
374  optimize.leastsq returns the fractional covariance matrix. To estimate the
375  standard deviation of the fit parameters, multiply the entries of this matrix
376  by the reduced chi squared and take the square root of the diagonal elements.
377 
378  Parameters
379  ----------
380  initialParams : list of np.float
381  initial values for fit parameters. For ptcFitType=POLYNOMIAL, its length
382  determines the degree of the polynomial.
383 
384  dataX : np.array of np.float
385  Data in the abscissa axis.
386 
387  dataY : np.array of np.float
388  Data in the ordinate axis.
389 
390  function : callable object (function)
391  Function to fit the data with.
392 
393  Return
394  ------
395  pFitSingleLeastSquares : list of np.float
396  List with fitted parameters.
397 
398  pErrSingleLeastSquares : list of np.float
399  List with errors for fitted parameters.
400  """
401 
402  def errFunc(p, x, y):
403  return function(p, x) - y
404 
405  pFit, pCov, infoDict, errMessage, success = leastsq(errFunc, initialParams,
406  args=(dataX, dataY), full_output=1, epsfcn=0.0001)
407 
408  if (len(dataY) > len(initialParams)) and pCov is not None:
409  reducedChiSq = (errFunc(pFit, dataX, dataY)**2).sum()/(len(dataY)-len(initialParams))
410  pCov *= reducedChiSq
411  else:
412  pCov[:, :] = np.inf
413 
414  errorVec = []
415  for i in range(len(pFit)):
416  errorVec.append(np.fabs(pCov[i][i])**0.5)
417 
418  pFitSingleLeastSquares = pFit
419  pErrSingleLeastSquares = np.array(errorVec)
420 
421  return pFitSingleLeastSquares, pErrSingleLeastSquares
422 
423  def _fitBootstrap(self, initialParams, dataX, dataY, function, confidenceSigma=1.):
424  """Do a fit using least squares and bootstrap to estimate parameter errors.
425 
426  The bootstrap error bars are calculated by fitting 100 random data sets.
427 
428  Parameters
429  ----------
430  initialParams : list of np.float
431  initial values for fit parameters. For ptcFitType=POLYNOMIAL, its length
432  determines the degree of the polynomial.
433 
434  dataX : np.array of np.float
435  Data in the abscissa axis.
436 
437  dataY : np.array of np.float
438  Data in the ordinate axis.
439 
440  function : callable object (function)
441  Function to fit the data with.
442 
443  confidenceSigma : np.float
444  Number of sigmas that determine confidence interval for the bootstrap errors.
445 
446  Return
447  ------
448  pFitBootstrap : list of np.float
449  List with fitted parameters.
450 
451  pErrBootstrap : list of np.float
452  List with errors for fitted parameters.
453  """
454 
455  def errFunc(p, x, y):
456  return function(p, x) - y
457 
458  # Fit first time
459  pFit, _ = leastsq(errFunc, initialParams, args=(dataX, dataY), full_output=0)
460 
461  # Get the stdev of the residuals
462  residuals = errFunc(pFit, dataX, dataY)
463  sigmaErrTotal = np.std(residuals)
464 
465  # 100 random data sets are generated and fitted
466  pars = []
467  for i in range(100):
468  randomDelta = np.random.normal(0., sigmaErrTotal, len(dataY))
469  randomDataY = dataY + randomDelta
470  randomFit, _ = leastsq(errFunc, initialParams,
471  args=(dataX, randomDataY), full_output=0)
472  pars.append(randomFit)
473  pars = np.array(pars)
474  meanPfit = np.mean(pars, 0)
475 
476  # confidence interval for parameter estimates
477  nSigma = confidenceSigma
478  errPfit = nSigma*np.std(pars, 0)
479  pFitBootstrap = meanPfit
480  pErrBootstrap = errPfit
481  return pFitBootstrap, pErrBootstrap
482 
483  def funcPolynomial(self, pars, x):
484  """Polynomial function definition"""
485  return poly.polyval(x, [*pars])
486 
487  def funcAstier(self, pars, x):
488  """Single brighter-fatter parameter model for PTC; Equation 16 of Astier+19"""
489  a00, gain, noise = pars
490  return 0.5/(a00*gain*gain)*(np.exp(2*a00*x*gain)-1) + noise/(gain*gain)
491 
492  @staticmethod
493  def _initialParsForPolynomial(order):
494  assert(order >= 2)
495  pars = np.zeros(order, dtype=np.float)
496  pars[0] = 10
497  pars[1] = 1
498  pars[2:] = 0.0001
499  return pars
500 
501  @staticmethod
502  def _boundsForPolynomial(initialPars):
503  lowers = [np.NINF for p in initialPars]
504  uppers = [np.inf for p in initialPars]
505  lowers[1] = 0 # no negative gains
506  return (lowers, uppers)
507 
508  @staticmethod
509  def _boundsForAstier(initialPars):
510  lowers = [np.NINF for p in initialPars]
511  uppers = [np.inf for p in initialPars]
512  return (lowers, uppers)
513 
514  @staticmethod
515  def _getInitialGoodPoints(means, variances, maxDeviation):
516  """Return a boolean array to mask bad points.
517 
518  A linear function has a constant ratio, so find the median
519  value of the ratios, and exclude the points that deviate
520  from that by more than a factor of maxDeviation.
521 
522  Too high and points that are so bad that fit will fail will be included
523  Too low and the non-linear points will be excluded, biasing the NL fit."""
524  ratios = [b/a for (a, b) in zip(means, variances)]
525  medianRatio = np.median(ratios)
526  ratioDeviations = [r/medianRatio for r in ratios]
527  goodPoints = [abs(1-r) < maxDeviation for r in ratioDeviations]
528  return np.array(goodPoints)
529 
530  def fitPtcAndNl(self, dataset, ptcFitType):
531  """Fit the photon transfer curve and calculate linearity and residuals.
532 
533  Fit the photon transfer curve with either a polynomial of the order
534  specified in the task config, or using the Astier approximation.
535 
536  Sigma clipping is performed iteratively for the fit, as well as an
537  initial clipping of data points that are more than
538  config.initialNonLinearityExclusionThreshold away from lying on a
539  straight line. This other step is necessary because the photon transfer
540  curve turns over catastrophically at very high flux (because saturation
541  drops the variance to ~0) and these far outliers cause the initial fit
542  to fail, meaning the sigma cannot be calculated to perform the
543  sigma-clipping.
544 
545  Parameters
546  ----------
547  dataset : `lsst.cp.pipe.ptc.PhotonTransferCurveDataset`
548  The dataset containing the means, variances and exposure times
549  ptcFitType : `str`
550  Fit a 'POLYNOMIAL' (degree: 'polynomialFitDegree') or
551  'ASTIERAPPROXIMATION' to the PTC
552  """
553 
554  def errFunc(p, x, y):
555  return ptcFunc(p, x) - y
556 
557  sigmaCutPtcOutliers = self.config.sigmaCutPtcOutliers
558  maxIterationsPtcOutliers = self.config.maxIterationsPtcOutliers
559 
560  for ampName in dataset.ampNames:
561  timeVecOriginal = np.array(dataset.rawExpTimes[ampName])
562  meanVecOriginal = np.array(dataset.rawMeans[ampName])
563  varVecOriginal = np.array(dataset.rawVars[ampName])
564 
565  mask = ((meanVecOriginal >= self.config.minMeanSignal) &
566  (meanVecOriginal <= self.config.maxMeanSignal))
567 
568  goodPoints = self._getInitialGoodPoints(meanVecOriginal, varVecOriginal,
569  self.config.initialNonLinearityExclusionThreshold)
570  mask = mask & goodPoints
571 
572  if ptcFitType == 'ASTIERAPPROXIMATION':
573  ptcFunc = self.funcAstier
574  parsIniPtc = [-1e-9, 1.0, 10.] # a00, gain, noise
575  bounds = self._boundsForAstier(parsIniPtc)
576  if ptcFitType == 'POLYNOMIAL':
577  ptcFunc = self.funcPolynomial
578  parsIniPtc = self._initialParsForPolynomial(self.config.polynomialFitDegree + 1)
579  bounds = self._boundsForPolynomial(parsIniPtc)
580 
581  # Before bootstrap fit, do an iterative fit to get rid of outliers
582  count = 1
583  while count <= maxIterationsPtcOutliers:
584  # Note that application of the mask actually shrinks the array
585  # to size rather than setting elements to zero (as we want) so
586  # always update mask itself and re-apply to the original data
587  meanTempVec = meanVecOriginal[mask]
588  varTempVec = varVecOriginal[mask]
589  res = least_squares(errFunc, parsIniPtc, bounds=bounds, args=(meanTempVec, varTempVec))
590  pars = res.x
591 
592  # change this to the original from the temp because the masks are ANDed
593  # meaning once a point is masked it's always masked, and the masks must
594  # always be the same length for broadcasting
595  sigResids = (varVecOriginal - ptcFunc(pars, meanVecOriginal))/np.sqrt(varVecOriginal)
596  newMask = np.array([True if np.abs(r) < sigmaCutPtcOutliers else False for r in sigResids])
597  mask = mask & newMask
598 
599  nDroppedTotal = Counter(mask)[False]
600  self.log.debug(f"Iteration {count}: discarded {nDroppedTotal} points in total for {ampName}")
601  count += 1
602  # objects should never shrink
603  assert (len(mask) == len(timeVecOriginal) == len(meanVecOriginal) == len(varVecOriginal))
604 
605  dataset.visitMask[ampName] = mask # store the final mask
606 
607  parsIniPtc = pars
608  timeVecFinal = timeVecOriginal[mask]
609  meanVecFinal = meanVecOriginal[mask]
610  varVecFinal = varVecOriginal[mask]
611 
612  if Counter(mask)[False] > 0:
613  self.log.info((f"Number of points discarded in PTC of amplifier {ampName}:" +
614  f" {Counter(mask)[False]} out of {len(meanVecOriginal)}"))
615 
616  if (len(meanVecFinal) < len(parsIniPtc)):
617  raise RuntimeError(f"Not enough data points ({len(meanVecFinal)}) compared to the number of" +
618  f"parameters of the PTC model({len(parsIniPtc)}).")
619  # Fit the PTC
620  if self.config.doFitBootstrap:
621  parsFit, parsFitErr = self._fitBootstrap(parsIniPtc, meanVecFinal, varVecFinal, ptcFunc)
622  else:
623  parsFit, parsFitErr = self._fitLeastSq(parsIniPtc, meanVecFinal, varVecFinal, ptcFunc)
624 
625  dataset.ptcFitPars[ampName] = parsFit
626  dataset.ptcFitParsError[ampName] = parsFitErr
627 
628  if ptcFitType == 'ASTIERAPPROXIMATION':
629  ptcGain = parsFit[1]
630  ptcGainErr = parsFitErr[1]
631  ptcNoise = np.sqrt(np.fabs(parsFit[2]))
632  ptcNoiseErr = 0.5*(parsFitErr[2]/np.fabs(parsFit[2]))*np.sqrt(np.fabs(parsFit[2]))
633  if ptcFitType == 'POLYNOMIAL':
634  ptcGain = 1./parsFit[1]
635  ptcGainErr = np.fabs(1./parsFit[1])*(parsFitErr[1]/parsFit[1])
636  ptcNoise = np.sqrt(np.fabs(parsFit[0]))*ptcGain
637  ptcNoiseErr = (0.5*(parsFitErr[0]/np.fabs(parsFit[0]))*(np.sqrt(np.fabs(parsFit[0]))))*ptcGain
638 
639  dataset.gain[ampName] = ptcGain
640  dataset.gainErr[ampName] = ptcGainErr
641  dataset.noise[ampName] = ptcNoise
642  dataset.noiseErr[ampName] = ptcNoiseErr
643 
644  # Non-linearity residuals (NL of mean vs time curve): percentage, and fit to a quadratic function
645  # In this case, len(parsIniNl) = 3 indicates that we want a quadratic fit
646  parsIniNl = [1., 1., 1.]
647  if self.config.doFitBootstrap:
648  parsFit, parsFitErr = self._fitBootstrap(parsIniNl, timeVecFinal, meanVecFinal,
649  self.funcPolynomial)
650  else:
651  parsFit, parsFitErr = self._fitLeastSq(parsIniNl, timeVecFinal, meanVecFinal,
652  self.funcPolynomial)
653  linResidualTimeIndex = self.config.linResidualTimeIndex
654  if timeVecFinal[linResidualTimeIndex] == 0.0:
655  raise RuntimeError("Reference time for linearity residual can't be 0.0")
656  linResidual = 100*(1 - ((meanVecFinal[linResidualTimeIndex] /
657  timeVecFinal[linResidualTimeIndex]) / (meanVecFinal/timeVecFinal)))
658 
659  dataset.nonLinearity[ampName] = parsFit
660  dataset.nonLinearityError[ampName] = parsFitErr
661  dataset.nonLinearityResiduals[ampName] = linResidual
662 
663  return
664 
665  def plot(self, dataRef, dataset, ptcFitType):
666  dirname = dataRef.getUri(datasetType='cpPipePlotRoot', write=True)
667  if not os.path.exists(dirname):
668  os.makedirs(dirname)
669 
670  detNum = dataRef.dataId[self.config.ccdKey]
671  filename = f"PTC_det{detNum}.pdf"
672  filenameFull = os.path.join(dirname, filename)
673  with PdfPages(filenameFull) as pdfPages:
674  self._plotPtc(dataset, ptcFitType, pdfPages)
675 
676  def _plotPtc(self, dataset, ptcFitType, pdfPages):
677  """Plot PTC, linearity, and linearity residual per amplifier"""
678 
679  if ptcFitType == 'ASTIERAPPROXIMATION':
680  ptcFunc = self.funcAstier
681  stringTitle = r"Var = $\frac{1}{2g^2a_{00}}(\exp (2a_{00} \mu g) - 1) + \frac{n_{00}}{g^2}$"
682 
683  if ptcFitType == 'POLYNOMIAL':
684  ptcFunc = self.funcPolynomial
685  stringTitle = f"Polynomial (degree: {self.config.polynomialFitDegree})"
686 
687  legendFontSize = 7.5
688  labelFontSize = 8
689  titleFontSize = 10
690  supTitleFontSize = 18
691  markerSize = 25
692 
693  # General determination of the size of the plot grid
694  nAmps = len(dataset.ampNames)
695  if nAmps == 2:
696  nRows, nCols = 2, 1
697  nRows = np.sqrt(nAmps)
698  mantissa, _ = np.modf(nRows)
699  if mantissa > 0:
700  nRows = int(nRows) + 1
701  nCols = nRows
702  else:
703  nRows = int(nRows)
704  nCols = nRows
705 
706  f, ax = plt.subplots(nrows=nRows, ncols=nCols, sharex='col', sharey='row', figsize=(13, 10))
707  f2, ax2 = plt.subplots(nrows=nRows, ncols=nCols, sharex='col', sharey='row', figsize=(13, 10))
708 
709  for i, (amp, a, a2) in enumerate(zip(dataset.ampNames, ax.flatten(), ax2.flatten())):
710  meanVecOriginal = np.array(dataset.rawMeans[amp])
711  varVecOriginal = np.array(dataset.rawVars[amp])
712  mask = dataset.visitMask[amp]
713  meanVecFinal = meanVecOriginal[mask]
714  varVecFinal = varVecOriginal[mask]
715  meanVecOutliers = meanVecOriginal[np.invert(mask)]
716  varVecOutliers = varVecOriginal[np.invert(mask)]
717  pars, parsErr = dataset.ptcFitPars[amp], dataset.ptcFitParsError[amp]
718 
719  if ptcFitType == 'ASTIERAPPROXIMATION':
720  ptcA00, ptcA00error = pars[0], parsErr[0]
721  ptcGain, ptcGainError = pars[1], parsErr[1]
722  ptcNoise = np.sqrt(np.fabs(pars[2]))
723  ptcNoiseError = 0.5*(parsErr[2]/np.fabs(pars[2]))*np.sqrt(np.fabs(pars[2]))
724  stringLegend = (f"a00: {ptcA00:.2e}+/-{ptcA00error:.2e}"
725  f"\n Gain: {ptcGain:.4}+/-{ptcGainError:.2e}"
726  f"\n Noise: {ptcNoise:.4}+/-{ptcNoiseError:.2e}")
727 
728  if ptcFitType == 'POLYNOMIAL':
729  ptcGain, ptcGainError = 1./pars[1], np.fabs(1./pars[1])*(parsErr[1]/pars[1])
730  ptcNoise = np.sqrt(np.fabs(pars[0]))*ptcGain
731  ptcNoiseError = (0.5*(parsErr[0]/np.fabs(pars[0]))*(np.sqrt(np.fabs(pars[0]))))*ptcGain
732  stringLegend = (f"Noise: {ptcNoise:.4}+/-{ptcNoiseError:.2e} \n"
733  f"Gain: {ptcGain:.4}+/-{ptcGainError:.2e}")
734 
735  minMeanVecFinal = np.min(meanVecFinal)
736  maxMeanVecFinal = np.max(meanVecFinal)
737  meanVecFit = np.linspace(minMeanVecFinal, maxMeanVecFinal, 100*len(meanVecFinal))
738  minMeanVecOriginal = np.min(meanVecOriginal)
739  maxMeanVecOriginal = np.max(meanVecOriginal)
740  deltaXlim = maxMeanVecOriginal - minMeanVecOriginal
741 
742  a.plot(meanVecFit, ptcFunc(pars, meanVecFit), color='red')
743  a.plot(meanVecFinal, pars[0] + pars[1]*meanVecFinal, color='green', linestyle='--')
744  a.scatter(meanVecFinal, varVecFinal, c='blue', marker='o', s=markerSize)
745  a.scatter(meanVecOutliers, varVecOutliers, c='magenta', marker='s', s=markerSize)
746  a.set_xlabel(r'Mean signal ($\mu$, ADU)', fontsize=labelFontSize)
747  a.set_xticks(meanVecOriginal)
748  a.set_ylabel(r'Variance (ADU$^2$)', fontsize=labelFontSize)
749  a.tick_params(labelsize=11)
750  a.text(0.03, 0.8, stringLegend, transform=a.transAxes, fontsize=legendFontSize)
751  a.set_xscale('linear', fontsize=labelFontSize)
752  a.set_yscale('linear', fontsize=labelFontSize)
753  a.set_title(amp, fontsize=titleFontSize)
754  a.set_xlim([minMeanVecOriginal - 0.2*deltaXlim, maxMeanVecOriginal + 0.2*deltaXlim])
755 
756  # Same, but in log-scale
757  a2.plot(meanVecFit, ptcFunc(pars, meanVecFit), color='red')
758  a2.scatter(meanVecFinal, varVecFinal, c='blue', marker='o', s=markerSize)
759  a2.scatter(meanVecOutliers, varVecOutliers, c='magenta', marker='s', s=markerSize)
760  a2.set_xlabel(r'Mean Signal ($\mu$, ADU)', fontsize=labelFontSize)
761  a2.set_ylabel(r'Variance (ADU$^2$)', fontsize=labelFontSize)
762  a2.tick_params(labelsize=11)
763  a2.text(0.03, 0.8, stringLegend, transform=a2.transAxes, fontsize=legendFontSize)
764  a2.set_xscale('log')
765  a2.set_yscale('log')
766  a2.set_title(amp, fontsize=titleFontSize)
767  a2.set_xlim([minMeanVecOriginal, maxMeanVecOriginal])
768 
769  f.suptitle(f"PTC \n Fit: " + stringTitle, fontsize=20)
770  pdfPages.savefig(f)
771  f2.suptitle(f"PTC (log-log)", fontsize=20)
772  pdfPages.savefig(f2)
773 
774  # Plot mean vs time
775  f, ax = plt.subplots(nrows=4, ncols=4, sharex='col', sharey='row', figsize=(13, 10))
776  for i, (amp, a) in enumerate(zip(dataset.ampNames, ax.flatten())):
777  meanVecFinal = np.array(dataset.rawMeans[amp])[dataset.visitMask[amp]]
778  timeVecFinal = np.array(dataset.rawExpTimes[amp])[dataset.visitMask[amp]]
779 
780  pars, parsErr = dataset.nonLinearity[amp], dataset.nonLinearityError[amp]
781  c0, c0Error = pars[0], parsErr[0]
782  c1, c1Error = pars[1], parsErr[1]
783  c2, c2Error = pars[2], parsErr[2]
784  stringLegend = f"c0: {c0:.4}+/-{c0Error:.2e}\n c1: {c1:.4}+/-{c1Error:.2e}" \
785  + f"\n c2(NL): {c2:.2e}+/-{c2Error:.2e}"
786  a.scatter(timeVecFinal, meanVecFinal)
787  a.plot(timeVecFinal, self.funcPolynomial(pars, timeVecFinal), color='red')
788  a.set_xlabel('Time (sec)', fontsize=labelFontSize)
789  a.set_xticks(timeVecFinal)
790  a.set_ylabel(r'Mean signal ($\mu$, ADU)', fontsize=labelFontSize)
791  a.tick_params(labelsize=labelFontSize)
792  a.text(0.03, 0.75, stringLegend, transform=a.transAxes, fontsize=legendFontSize)
793  a.set_xscale('linear', fontsize=labelFontSize)
794  a.set_yscale('linear', fontsize=labelFontSize)
795  a.set_title(amp, fontsize=titleFontSize)
796 
797  f.suptitle("Linearity \n Fit: " + r"$\mu = c_0 + c_1 t + c_2 t^2$", fontsize=supTitleFontSize)
798  pdfPages.savefig()
799 
800  # Plot linearity residual
801  f, ax = plt.subplots(nrows=4, ncols=4, sharex='col', sharey='row', figsize=(13, 10))
802  for i, (amp, a) in enumerate(zip(dataset.ampNames, ax.flatten())):
803  meanVecFinal = np.array(dataset.rawMeans[amp])[dataset.visitMask[amp]]
804  linRes = np.array(dataset.nonLinearityResiduals[amp])
805 
806  a.scatter(meanVecFinal, linRes)
807  a.axhline(y=0, color='k')
808  a.axvline(x=timeVecFinal[self.config.linResidualTimeIndex], color='g', linestyle='--')
809  a.set_xlabel(r'Mean signal ($\mu$, ADU)', fontsize=labelFontSize)
810  a.set_xticks(meanVecFinal)
811  a.set_ylabel('LR (%)', fontsize=labelFontSize)
812  a.tick_params(labelsize=labelFontSize)
813  a.set_xscale('linear', fontsize=labelFontSize)
814  a.set_yscale('linear', fontsize=labelFontSize)
815  a.set_title(amp, fontsize=titleFontSize)
816 
817  f.suptitle(r"Linearity Residual: $100(1 - \mu_{\rm{ref}}/t_{\rm{ref}})/(\mu / t))$" + "\n" +
818  r"$t_{\rm{ref}}$: " + f"{timeVecFinal[2]} s", fontsize=supTitleFontSize)
819  pdfPages.savefig()
820 
821  return
def _plotPtc(self, dataset, ptcFitType, pdfPages)
Definition: ptc.py:676
def runDataRef(self, dataRef, visitPairs)
Definition: ptc.py:247
def __setattr__(self, attribute, value)
Definition: ptc.py:187
def fitPtcAndNl(self, dataset, ptcFitType)
Definition: ptc.py:530
def checkExpLengthEqual(exp1, exp2, v1=None, v2=None, raiseWithMessage=False)
Definition: utils.py:162
def validateIsrConfig(isrTask, mandatory=None, forbidden=None, desirable=None, undesirable=None, checkTrim=True, logName=None)
Definition: utils.py:200
def _fitBootstrap(self, initialParams, dataX, dataY, function, confidenceSigma=1.)
Definition: ptc.py:423
def __init__(self, args, kwargs)
Definition: ptc.py:226
def measureMeanVarPair(self, exposure1, exposure2, region=None)
Definition: ptc.py:317
def _getInitialGoodPoints(means, variances, maxDeviation)
Definition: ptc.py:515
def _fitLeastSq(self, initialParams, dataX, dataY, function)
Definition: ptc.py:371
def plot(self, dataRef, dataset, ptcFitType)
Definition: ptc.py:665