lsst.cp.pipe  19.0.0-10-ga90c110+4
ptc.py
Go to the documentation of this file.
1 # This file is part of cp_pipe.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 #
22 
23 __all__ = ['MeasurePhotonTransferCurveTask',
24  'MeasurePhotonTransferCurveTaskConfig',
25  'PhotonTransferCurveDataset']
26 
27 import numpy as np
28 import matplotlib.pyplot as plt
29 import os
30 from matplotlib.backends.backend_pdf import PdfPages
31 from sqlite3 import OperationalError
32 from collections import Counter
33 
34 import lsst.afw.math as afwMath
35 import lsst.pex.config as pexConfig
36 import lsst.pipe.base as pipeBase
37 from lsst.ip.isr import IsrTask
38 from .utils import (NonexistentDatasetTaskDataIdContainer, PairedVisitListTaskRunner,
39  checkExpLengthEqual, validateIsrConfig)
40 from scipy.optimize import leastsq, least_squares
41 import numpy.polynomial.polynomial as poly
42 
43 
44 class MeasurePhotonTransferCurveTaskConfig(pexConfig.Config):
45  """Config class for photon transfer curve measurement task"""
46  isr = pexConfig.ConfigurableField(
47  target=IsrTask,
48  doc="""Task to perform instrumental signature removal.""",
49  )
50  isrMandatorySteps = pexConfig.ListField(
51  dtype=str,
52  doc="isr operations that must be performed for valid results. Raises if any of these are False.",
53  default=['doAssembleCcd']
54  )
55  isrForbiddenSteps = pexConfig.ListField(
56  dtype=str,
57  doc="isr operations that must NOT be performed for valid results. Raises if any of these are True",
58  default=['doFlat', 'doFringe', 'doAddDistortionModel', 'doBrighterFatter', 'doUseOpticsTransmission',
59  'doUseFilterTransmission', 'doUseSensorTransmission', 'doUseAtmosphereTransmission']
60  )
61  isrDesirableSteps = pexConfig.ListField(
62  dtype=str,
63  doc="isr operations that it is advisable to perform, but are not mission-critical." +
64  " WARNs are logged for any of these found to be False.",
65  default=['doBias', 'doDark', 'doCrosstalk', 'doDefect']
66  )
67  isrUndesirableSteps = pexConfig.ListField(
68  dtype=str,
69  doc="isr operations that it is *not* advisable to perform in the general case, but are not" +
70  " forbidden as some use-cases might warrant them." +
71  " WARNs are logged for any of these found to be True.",
72  default=['doLinearize']
73  )
74  ccdKey = pexConfig.Field(
75  dtype=str,
76  doc="The key by which to pull a detector from a dataId, e.g. 'ccd' or 'detector'.",
77  default='ccd',
78  )
79  makePlots = pexConfig.Field(
80  dtype=bool,
81  doc="Plot the PTC curves?",
82  default=False,
83  )
84  ptcFitType = pexConfig.ChoiceField(
85  dtype=str,
86  doc="Fit PTC to approximation in Astier+19 (Equation 16) or to a polynomial.",
87  default="POLYNOMIAL",
88  allowed={
89  "POLYNOMIAL": "n-degree polynomial (use 'polynomialFitDegree' to set 'n').",
90  "ASTIERAPPROXIMATION": "Approximation in Astier+19 (Eq. 16)."
91  }
92  )
93  polynomialFitDegree = pexConfig.Field(
94  dtype=int,
95  doc="Degree of polynomial to fit the PTC, when 'ptcFitType'=POLYNOMIAL.",
96  default=2,
97  )
98  polynomialFitDegreeNonLinearity = pexConfig.Field(
99  dtype=int,
100  doc="Degree of polynomial to fit the meanSignal vs exposureTime curve to produce" +
101  " the table for LinearizerLookupTable.",
102  default=3,
103  )
104  binSize = pexConfig.Field(
105  dtype=int,
106  doc="Bin the image by this factor in both dimensions.",
107  default=1,
108  )
109  minMeanSignal = pexConfig.Field(
110  dtype=float,
111  doc="Minimum value (inclusive) of mean signal (in ADU) above which to consider.",
112  default=0,
113  )
114  maxMeanSignal = pexConfig.Field(
115  dtype=float,
116  doc="Maximum value (inclusive) of mean signal (in ADU) below which to consider.",
117  default=9e6,
118  )
119  initialNonLinearityExclusionThresholdPositive = pexConfig.RangeField(
120  dtype=float,
121  doc="Initially exclude data points with a variance that are more than a factor of this from being"
122  " linear in the positive direction, from the PTC fit. Note that these points will also be"
123  " excluded from the non-linearity fit. This is done before the iterative outlier rejection,"
124  " to allow an accurate determination of the sigmas for said iterative fit.",
125  default=0.12,
126  min=0.0,
127  max=1.0,
128  )
129  initialNonLinearityExclusionThresholdNegative = pexConfig.RangeField(
130  dtype=float,
131  doc="Initially exclude data points with a variance that are more than a factor of this from being"
132  " linear in the negative direction, from the PTC fit. Note that these points will also be"
133  " excluded from the non-linearity fit. This is done before the iterative outlier rejection,"
134  " to allow an accurate determination of the sigmas for said iterative fit.",
135  default=0.25,
136  min=0.0,
137  max=1.0,
138  )
139  sigmaCutPtcOutliers = pexConfig.Field(
140  dtype=float,
141  doc="Sigma cut for outlier rejection in PTC.",
142  default=5.0,
143  )
144  maxIterationsPtcOutliers = pexConfig.Field(
145  dtype=int,
146  doc="Maximum number of iterations for outlier rejection in PTC.",
147  default=2,
148  )
149  doFitBootstrap = pexConfig.Field(
150  dtype=bool,
151  doc="Use bootstrap for the PTC fit parameters and errors?.",
152  default=False,
153  )
154  linResidualTimeIndex = pexConfig.Field(
155  dtype=int,
156  doc="Index position in time array for reference time in linearity residual calculation.",
157  default=2,
158  )
159  maxAduForLookupTableLinearizer = pexConfig.Field(
160  dtype=int,
161  doc="Maximum ADU value for the LookupTable linearizer.",
162  default=2**18,
163  )
164 
165 
167  """A simple class to hold the output data from the PTC task.
168 
169  The dataset is made up of a dictionary for each item, keyed by the
170  amplifiers' names, which much be supplied at construction time.
171 
172  New items cannot be added to the class to save accidentally saving to the
173  wrong property, and the class can be frozen if desired.
174 
175  inputVisitPairs records the visits used to produce the data.
176  When fitPtcAndNonLinearity() is run, a mask is built up, which is by definition
177  always the same length as inputVisitPairs, rawExpTimes, rawMeans
178  and rawVars, and is a list of bools, which are incrementally set to False
179  as points are discarded from the fits.
180 
181  PTC fit parameters for polynomials are stored in a list in ascending order
182  of polynomial term, i.e. par[0]*x^0 + par[1]*x + par[2]*x^2 etc
183  with the length of the list corresponding to the order of the polynomial
184  plus one.
185  """
186  def __init__(self, ampNames):
187  # add items to __dict__ directly because __setattr__ is overridden
188 
189  # instance variables
190  self.__dict__["ampNames"] = ampNames
191  self.__dict__["badAmps"] = []
192 
193  # raw data variables
194  self.__dict__["inputVisitPairs"] = {ampName: [] for ampName in ampNames}
195  self.__dict__["visitMask"] = {ampName: [] for ampName in ampNames}
196  self.__dict__["rawExpTimes"] = {ampName: [] for ampName in ampNames}
197  self.__dict__["rawMeans"] = {ampName: [] for ampName in ampNames}
198  self.__dict__["rawVars"] = {ampName: [] for ampName in ampNames}
199 
200  # fit information
201  self.__dict__["ptcFitType"] = {ampName: "" for ampName in ampNames}
202  self.__dict__["ptcFitPars"] = {ampName: [] for ampName in ampNames}
203  self.__dict__["ptcFitParsError"] = {ampName: [] for ampName in ampNames}
204  self.__dict__["nonLinearity"] = {ampName: [] for ampName in ampNames}
205  self.__dict__["nonLinearityError"] = {ampName: [] for ampName in ampNames}
206  self.__dict__["nonLinearityResiduals"] = {ampName: [] for ampName in ampNames}
207 
208  # final results
209  self.__dict__["gain"] = {ampName: -1. for ampName in ampNames}
210  self.__dict__["gainErr"] = {ampName: -1. for ampName in ampNames}
211  self.__dict__["noise"] = {ampName: -1. for ampName in ampNames}
212  self.__dict__["noiseErr"] = {ampName: -1. for ampName in ampNames}
213  self.__dict__["coefficientLinearizeSquared"] = {ampName: [] for ampName in ampNames}
214 
215  def __setattr__(self, attribute, value):
216  """Protect class attributes"""
217  if attribute not in self.__dict__:
218  raise AttributeError(f"{attribute} is not already a member of PhotonTransferCurveDataset, which"
219  " does not support setting of new attributes.")
220  else:
221  self.__dict__[attribute] = value
222 
223  def getVisitsUsed(self, ampName):
224  """Get the visits used, i.e. not discarded, for a given amp.
225 
226  If no mask has been created yet, all visits are returned.
227  """
228  if self.visitMask[ampName] == []:
229  return self.inputVisitPairs[ampName]
230 
231  # if the mask exists it had better be the same length as the visitPairs
232  assert len(self.visitMask[ampName]) == len(self.inputVisitPairs[ampName])
233 
234  pairs = self.inputVisitPairs[ampName]
235  mask = self.visitMask[ampName]
236  # cast to bool required because numpy
237  return [(v1, v2) for ((v1, v2), m) in zip(pairs, mask) if bool(m) is True]
238 
239  def getGoodAmps(self):
240  return [amp for amp in self.ampNames if amp not in self.badAmps]
241 
242 
243 class MeasurePhotonTransferCurveTask(pipeBase.CmdLineTask):
244  """A class to calculate, fit, and plot a PTC from a set of flat pairs.
245 
246  The Photon Transfer Curve (var(signal) vs mean(signal)) is a standard tool
247  used in astronomical detectors characterization (e.g., Janesick 2001,
248  Janesick 2007). This task calculates the PTC from a series of pairs of
249  flat-field images; each pair taken at identical exposure times. The
250  difference image of each pair is formed to eliminate fixed pattern noise,
251  and then the variance of the difference image and the mean of the average image
252  are used to produce the PTC. An n-degree polynomial or the approximation in Equation
253  16 of Astier+19 ("The Shape of the Photon Transfer Curve of CCD sensors",
254  arXiv:1905.08677) can be fitted to the PTC curve. These models include
255  parameters such as the gain (e/ADU) and readout noise.
256 
257  Parameters
258  ----------
259 
260  *args: `list`
261  Positional arguments passed to the Task constructor. None used at this
262  time.
263  **kwargs: `dict`
264  Keyword arguments passed on to the Task constructor. None used at this
265  time.
266 
267  """
268 
269  RunnerClass = PairedVisitListTaskRunner
270  ConfigClass = MeasurePhotonTransferCurveTaskConfig
271  _DefaultName = "measurePhotonTransferCurve"
272 
273  def __init__(self, *args, **kwargs):
274  pipeBase.CmdLineTask.__init__(self, *args, **kwargs)
275  self.makeSubtask("isr")
276  plt.interactive(False) # stop windows popping up when plotting. When headless, use 'agg' backend too
277  validateIsrConfig(self.isr, self.config.isrMandatorySteps,
278  self.config.isrForbiddenSteps, self.config.isrDesirableSteps, checkTrim=False)
279  self.config.validate()
280  self.config.freeze()
281 
282  @classmethod
283  def _makeArgumentParser(cls):
284  """Augment argument parser for the MeasurePhotonTransferCurveTask."""
285  parser = pipeBase.ArgumentParser(name=cls._DefaultName)
286  parser.add_argument("--visit-pairs", dest="visitPairs", nargs="*",
287  help="Visit pairs to use. Each pair must be of the form INT,INT e.g. 123,456")
288  parser.add_id_argument("--id", datasetType="photonTransferCurveDataset",
289  ContainerClass=NonexistentDatasetTaskDataIdContainer,
290  help="The ccds to use, e.g. --id ccd=0..100")
291  return parser
292 
293  @pipeBase.timeMethod
294  def runDataRef(self, dataRef, visitPairs):
295  """Run the Photon Transfer Curve (PTC) measurement task.
296 
297  For a dataRef (which is each detector here),
298  and given a list of visit pairs at different exposure times,
299  measure the PTC.
300 
301  Parameters
302  ----------
303  dataRef : list of lsst.daf.persistence.ButlerDataRef
304  dataRef for the detector for the visits to be fit.
305  visitPairs : `iterable` of `tuple` of `int`
306  Pairs of visit numbers to be processed together
307  """
308 
309  # setup necessary objects
310  detNum = dataRef.dataId[self.config.ccdKey]
311  detector = dataRef.get('camera')[dataRef.dataId[self.config.ccdKey]]
312  # expand some missing fields that we need for lsstCam. This is a work-around
313  # for Gen2 problems that I (RHL) don't feel like solving. The calibs pipelines
314  # (which inherit from CalibTask) use addMissingKeys() to do basically the same thing
315  #
316  # Basically, the butler's trying to look up the fields in `raw_visit` which won't work
317  for name in dataRef.getButler().getKeys('bias'):
318  if name not in dataRef.dataId:
319  try:
320  dataRef.dataId[name] = \
321  dataRef.getButler().queryMetadata('raw', [name], detector=detNum)[0]
322  except OperationalError:
323  pass
324 
325  amps = detector.getAmplifiers()
326  ampNames = [amp.getName() for amp in amps]
327  dataset = PhotonTransferCurveDataset(ampNames)
328 
329  self.log.info('Measuring PTC using %s visits for detector %s' % (visitPairs, detNum))
330 
331  for (v1, v2) in visitPairs:
332  # Perform ISR on each exposure
333  dataRef.dataId['expId'] = v1
334  exp1 = self.isr.runDataRef(dataRef).exposure
335  dataRef.dataId['expId'] = v2
336  exp2 = self.isr.runDataRef(dataRef).exposure
337  del dataRef.dataId['expId']
338 
339  checkExpLengthEqual(exp1, exp2, v1, v2, raiseWithMessage=True)
340  expTime = exp1.getInfo().getVisitInfo().getExposureTime()
341 
342  for amp in detector:
343  mu, varDiff = self.measureMeanVarPair(exp1, exp2, region=amp.getBBox())
344  ampName = amp.getName()
345 
346  dataset.rawExpTimes[ampName].append(expTime)
347  dataset.rawMeans[ampName].append(mu)
348  dataset.rawVars[ampName].append(varDiff)
349  dataset.inputVisitPairs[ampName].append((v1, v2))
350 
351  numberAmps = len(detector.getAmplifiers())
352  numberAduValues = self.config.maxAduForLookupTableLinearizer
353  lookupTableArray = np.zeros((numberAmps, numberAduValues), dtype=np.float32)
354 
355  # Fit PTC and (non)linearity of signal vs time curve.
356  # Fill up PhotonTransferCurveDataset object.
357  # Fill up array for LUT linearizer.
358  dataset = self.fitPtcAndNonLinearity(dataset, tableArray=lookupTableArray,
359  ptcFitType=self.config.ptcFitType)
360 
361  if self.config.makePlots:
362  self.plot(dataRef, dataset, ptcFitType=self.config.ptcFitType)
363 
364  # Save data, PTC fit, and NL fit dictionaries
365  self.log.info(f"Writing PTC and NL data to {dataRef.getUri(write=True)}")
366  dataRef.put(dataset, datasetType="photonTransferCurveDataset")
367 
368  self.log.info('Finished measuring PTC for in detector %s' % detNum)
369 
370  return pipeBase.Struct(exitStatus=0)
371 
372  def measureMeanVarPair(self, exposure1, exposure2, region=None):
373  """Calculate the mean signal of two exposures and the variance of their difference.
374 
375  Parameters
376  ----------
377  exposure1 : `lsst.afw.image.exposure.exposure.ExposureF`
378  First exposure of flat field pair.
379 
380  exposure2 : `lsst.afw.image.exposure.exposure.ExposureF`
381  Second exposure of flat field pair.
382 
383  region : `lsst.geom.Box2I`
384  Region of each exposure where to perform the calculations (e.g, an amplifier).
385 
386  Return
387  ------
388 
389  mu : `np.float`
390  0.5*(mu1 + mu2), where mu1, and mu2 are the clipped means of the regions in
391  both exposures.
392 
393  varDiff : `np.float`
394  Half of the clipped variance of the difference of the regions inthe two input
395  exposures.
396  """
397 
398  if region is not None:
399  im1Area = exposure1.maskedImage[region]
400  im2Area = exposure2.maskedImage[region]
401  else:
402  im1Area = exposure1.maskedImage
403  im2Area = exposure2.maskedImage
404 
405  im1Area = afwMath.binImage(im1Area, self.config.binSize)
406  im2Area = afwMath.binImage(im2Area, self.config.binSize)
407 
408  # Clipped mean of images; then average of mean.
409  mu1 = afwMath.makeStatistics(im1Area, afwMath.MEANCLIP).getValue()
410  mu2 = afwMath.makeStatistics(im2Area, afwMath.MEANCLIP).getValue()
411  mu = 0.5*(mu1 + mu2)
412 
413  # Take difference of pairs
414  # symmetric formula: diff = (mu2*im1-mu1*im2)/(0.5*(mu1+mu2))
415  temp = im2Area.clone()
416  temp *= mu1
417  diffIm = im1Area.clone()
418  diffIm *= mu2
419  diffIm -= temp
420  diffIm /= mu
421 
422  varDiff = 0.5*(afwMath.makeStatistics(diffIm, afwMath.VARIANCECLIP).getValue())
423 
424  return mu, varDiff
425 
426  def _fitLeastSq(self, initialParams, dataX, dataY, function):
427  """Do a fit and estimate the parameter errors using using scipy.optimize.leastq.
428 
429  optimize.leastsq returns the fractional covariance matrix. To estimate the
430  standard deviation of the fit parameters, multiply the entries of this matrix
431  by the reduced chi squared and take the square root of the diagonal elements.
432 
433  Parameters
434  ----------
435  initialParams : list of np.float
436  initial values for fit parameters. For ptcFitType=POLYNOMIAL, its length
437  determines the degree of the polynomial.
438 
439  dataX : np.array of np.float
440  Data in the abscissa axis.
441 
442  dataY : np.array of np.float
443  Data in the ordinate axis.
444 
445  function : callable object (function)
446  Function to fit the data with.
447 
448  Return
449  ------
450  pFitSingleLeastSquares : list of np.float
451  List with fitted parameters.
452 
453  pErrSingleLeastSquares : list of np.float
454  List with errors for fitted parameters.
455  """
456 
457  def errFunc(p, x, y):
458  return function(p, x) - y
459 
460  pFit, pCov, infoDict, errMessage, success = leastsq(errFunc, initialParams,
461  args=(dataX, dataY), full_output=1, epsfcn=0.0001)
462 
463  if (len(dataY) > len(initialParams)) and pCov is not None:
464  reducedChiSq = (errFunc(pFit, dataX, dataY)**2).sum()/(len(dataY)-len(initialParams))
465  pCov *= reducedChiSq
466  else:
467  pCov[:, :] = np.inf
468 
469  errorVec = []
470  for i in range(len(pFit)):
471  errorVec.append(np.fabs(pCov[i][i])**0.5)
472 
473  pFitSingleLeastSquares = pFit
474  pErrSingleLeastSquares = np.array(errorVec)
475 
476  return pFitSingleLeastSquares, pErrSingleLeastSquares
477 
478  def _fitBootstrap(self, initialParams, dataX, dataY, function, confidenceSigma=1.):
479  """Do a fit using least squares and bootstrap to estimate parameter errors.
480 
481  The bootstrap error bars are calculated by fitting 100 random data sets.
482 
483  Parameters
484  ----------
485  initialParams : list of np.float
486  initial values for fit parameters. For ptcFitType=POLYNOMIAL, its length
487  determines the degree of the polynomial.
488 
489  dataX : np.array of np.float
490  Data in the abscissa axis.
491 
492  dataY : np.array of np.float
493  Data in the ordinate axis.
494 
495  function : callable object (function)
496  Function to fit the data with.
497 
498  confidenceSigma : np.float
499  Number of sigmas that determine confidence interval for the bootstrap errors.
500 
501  Return
502  ------
503  pFitBootstrap : list of np.float
504  List with fitted parameters.
505 
506  pErrBootstrap : list of np.float
507  List with errors for fitted parameters.
508  """
509 
510  def errFunc(p, x, y):
511  return function(p, x) - y
512 
513  # Fit first time
514  pFit, _ = leastsq(errFunc, initialParams, args=(dataX, dataY), full_output=0)
515 
516  # Get the stdev of the residuals
517  residuals = errFunc(pFit, dataX, dataY)
518  sigmaErrTotal = np.std(residuals)
519 
520  # 100 random data sets are generated and fitted
521  pars = []
522  for i in range(100):
523  randomDelta = np.random.normal(0., sigmaErrTotal, len(dataY))
524  randomDataY = dataY + randomDelta
525  randomFit, _ = leastsq(errFunc, initialParams,
526  args=(dataX, randomDataY), full_output=0)
527  pars.append(randomFit)
528  pars = np.array(pars)
529  meanPfit = np.mean(pars, 0)
530 
531  # confidence interval for parameter estimates
532  nSigma = confidenceSigma
533  errPfit = nSigma*np.std(pars, 0)
534  pFitBootstrap = meanPfit
535  pErrBootstrap = errPfit
536  return pFitBootstrap, pErrBootstrap
537 
538  def funcPolynomial(self, pars, x):
539  """Polynomial function definition"""
540  return poly.polyval(x, [*pars])
541 
542  def funcAstier(self, pars, x):
543  """Single brighter-fatter parameter model for PTC; Equation 16 of Astier+19"""
544  a00, gain, noise = pars
545  return 0.5/(a00*gain*gain)*(np.exp(2*a00*x*gain)-1) + noise/(gain*gain)
546 
547  @staticmethod
548  def _initialParsForPolynomial(order):
549  assert(order >= 2)
550  pars = np.zeros(order, dtype=np.float)
551  pars[0] = 10
552  pars[1] = 1
553  pars[2:] = 0.0001
554  return pars
555 
556  @staticmethod
557  def _boundsForPolynomial(initialPars):
558  lowers = [np.NINF for p in initialPars]
559  uppers = [np.inf for p in initialPars]
560  lowers[1] = 0 # no negative gains
561  return (lowers, uppers)
562 
563  @staticmethod
564  def _boundsForAstier(initialPars):
565  lowers = [np.NINF for p in initialPars]
566  uppers = [np.inf for p in initialPars]
567  return (lowers, uppers)
568 
569  @staticmethod
570  def _getInitialGoodPoints(means, variances, maxDeviationPositive, maxDeviationNegative):
571  """Return a boolean array to mask bad points.
572 
573  A linear function has a constant ratio, so find the median
574  value of the ratios, and exclude the points that deviate
575  from that by more than a factor of maxDeviationPositive/negative.
576  Asymmetric deviations are supported as we expect the PTC to turn
577  down as the flux increases, but sometimes it anomalously turns
578  upwards just before turning over, which ruins the fits, so it
579  is wise to be stricter about restricting positive outliers than
580  negative ones.
581 
582  Too high and points that are so bad that fit will fail will be included
583  Too low and the non-linear points will be excluded, biasing the NL fit."""
584  ratios = [b/a for (a, b) in zip(means, variances)]
585  medianRatio = np.median(ratios)
586  ratioDeviations = [(r/medianRatio)-1 for r in ratios]
587 
588  # so that it doesn't matter if the deviation is expressed as positive or negative
589  maxDeviationPositive = abs(maxDeviationPositive)
590  maxDeviationNegative = -1. * abs(maxDeviationNegative)
591 
592  goodPoints = np.array([True if (r < maxDeviationPositive and r > maxDeviationNegative)
593  else False for r in ratioDeviations])
594  return goodPoints
595 
596  def _makeZeroSafe(self, array, warn=True, substituteValue=1e-9):
597  """"""
598  nBad = Counter(array)[0]
599  if nBad == 0:
600  return array
601 
602  if warn:
603  msg = f"Found {nBad} zeros in array at elements {[x for x in np.where(array==0)[0]]}"
604  self.log.warn(msg)
605 
606  array[array == 0] = substituteValue
607  return array
608 
609  def calculateLinearityResidualAndLinearizers(self, exposureTimeVector, meanSignalVector):
610  """Calculate linearity residual and fit an n-order polynomial to the mean vs time curve
611  to produce corrections (deviation from linear part of polynomial) for a particular amplifier
612  to populate LinearizeLookupTable. Use quadratic and linear parts of this polynomial to approximate
613  c0 for LinearizeSquared."
614 
615  Parameters
616  ---------
617 
618  exposureTimeVector: `list` of `np.float`
619  List of exposure times for each flat pair
620 
621  meanSignalVector: `list` of `np.float`
622  List of mean signal from diference image of flat pairs
623 
624  Returns
625  -------
626  c0: `np.float`
627  Coefficient for LinearizeSquared, where corrImage = uncorrImage + c0*uncorrImage^2.
628  c0 ~ -k2/(k1^2), where k1 and k2 are fit from
629  meanSignalVector = k0 + k1*exposureTimeVector + k2*exposureTimeVector^2 +...
630  + kn*exposureTimeVector^n, with n = "polynomialFitDegreeNonLinearity".
631 
632  linearizerTableRow: list of `np.float`
633  One dimensional array with deviation from linear part of n-order polynomial fit
634  to mean vs time curve. This array will be one row (for the particular amplifier at hand)
635  of the table array for LinearizeLookupTable.
636 
637  linResidual: list of `np.float`
638  Linearity residual from the mean vs time curve, defined as
639  100*(1 - meanSignalReference/expTimeReference/(meanSignal/expTime).
640 
641  parsFit: list of `np.float`
642  Parameters from n-order polynomial fit to mean vs time curve.
643 
644  parsFitErr: list of `np.float`
645  Parameters from n-order polynomial fit to mean vs time curve.
646 
647  """
648 
649  # Lookup table linearizer
650  parsIniNonLinearity = self._initialParsForPolynomial(self.config.polynomialFitDegreeNonLinearity + 1)
651  if self.config.doFitBootstrap:
652  parsFit, parsFitErr = self._fitBootstrap(parsIniNonLinearity, exposureTimeVector,
653  meanSignalVector, self.funcPolynomial)
654  else:
655  parsFit, parsFitErr = self._fitLeastSq(parsIniNonLinearity, exposureTimeVector, meanSignalVector,
656  self.funcPolynomial)
657 
658  # Use linear part to get time at wich signal is maxAduForLookupTableLinearizer ADU
659  tMax = (self.config.maxAduForLookupTableLinearizer - parsFit[0])/parsFit[1]
660  timeRange = np.linspace(0, tMax, self.config.maxAduForLookupTableLinearizer)
661  signalIdeal = (parsFit[0] + parsFit[1]*timeRange).astype(int)
662  signalUncorrected = (self.funcPolynomial(parsFit, timeRange)).astype(int)
663  linearizerTableRow = signalIdeal - signalUncorrected # LinearizerLookupTable has corrections
664 
665  # Use quadratic and linear part of fit to produce c0 for LinearizeSquared
666  # Check that magnitude of higher order (>= 3) coefficents of the polyFit are small,
667  # i.e., less than threshold = 1e-10 (typical quadratic and cubic coefficents are ~1e-6
668  # and ~1e-12).
669  k1, k2 = parsFit[1], parsFit[2]
670  c0 = -k2/(k1**2) # c0 coefficient for LinearizeSquared
671  for coefficient in parsFit[3:]:
672  if np.fabs(coefficient) > 1e-10:
673  msg = f"Coefficient {coefficient} in polynomial fit larger than threshold 1e-10."
674  self.log.warn(msg)
675 
676  # Linearity residual
677  linResidualTimeIndex = self.config.linResidualTimeIndex
678  if exposureTimeVector[linResidualTimeIndex] == 0.0:
679  raise RuntimeError("Reference time for linearity residual can't be 0.0")
680  linResidual = 100*(1 - ((meanSignalVector[linResidualTimeIndex] /
681  exposureTimeVector[linResidualTimeIndex]) /
682  (meanSignalVector/exposureTimeVector)))
683 
684  return c0, linearizerTableRow, linResidual, parsFit, parsFitErr
685 
686  def fitPtcAndNonLinearity(self, dataset, ptcFitType, tableArray=None):
687  """Fit the photon transfer curve and calculate linearity and residuals.
688 
689  Fit the photon transfer curve with either a polynomial of the order
690  specified in the task config, or using the Astier approximation.
691 
692  Sigma clipping is performed iteratively for the fit, as well as an
693  initial clipping of data points that are more than
694  config.initialNonLinearityExclusionThreshold away from lying on a
695  straight line. This other step is necessary because the photon transfer
696  curve turns over catastrophically at very high flux (because saturation
697  drops the variance to ~0) and these far outliers cause the initial fit
698  to fail, meaning the sigma cannot be calculated to perform the
699  sigma-clipping.
700 
701  Parameters
702  ----------
703  dataset : `lsst.cp.pipe.ptc.PhotonTransferCurveDataset`
704  The dataset containing the means, variances and exposure times
705  ptcFitType : `str`
706  Fit a 'POLYNOMIAL' (degree: 'polynomialFitDegree') or
707  'ASTIERAPPROXIMATION' to the PTC
708  tableArray : `np.array`
709  Optional. Look-up table array with size rows=nAmps and columns=ADU values.
710  It will be modified in-place if supplied.
711 
712  Returns
713  -------
714  dataset: `lsst.cp.pipe.ptc.PhotonTransferCurveDataset`
715  This is the same dataset as the input paramter, however, it has been modified
716  to include information such as the fit vectors and the fit parameters. See
717  the class `PhotonTransferCurveDatase`.
718  """
719 
720  def errFunc(p, x, y):
721  return ptcFunc(p, x) - y
722 
723  sigmaCutPtcOutliers = self.config.sigmaCutPtcOutliers
724  maxIterationsPtcOutliers = self.config.maxIterationsPtcOutliers
725 
726  for i, ampName in enumerate(dataset.ampNames):
727  timeVecOriginal = np.array(dataset.rawExpTimes[ampName])
728  meanVecOriginal = np.array(dataset.rawMeans[ampName])
729  varVecOriginal = np.array(dataset.rawVars[ampName])
730  varVecOriginal = self._makeZeroSafe(varVecOriginal)
731 
732  mask = ((meanVecOriginal >= self.config.minMeanSignal) &
733  (meanVecOriginal <= self.config.maxMeanSignal))
734 
735  goodPoints = self._getInitialGoodPoints(meanVecOriginal, varVecOriginal,
736  self.config.initialNonLinearityExclusionThresholdPositive,
737  self.config.initialNonLinearityExclusionThresholdNegative)
738  mask = mask & goodPoints
739 
740  if ptcFitType == 'ASTIERAPPROXIMATION':
741  ptcFunc = self.funcAstier
742  parsIniPtc = [-1e-9, 1.0, 10.] # a00, gain, noise
743  bounds = self._boundsForAstier(parsIniPtc)
744  if ptcFitType == 'POLYNOMIAL':
745  ptcFunc = self.funcPolynomial
746  parsIniPtc = self._initialParsForPolynomial(self.config.polynomialFitDegree + 1)
747  bounds = self._boundsForPolynomial(parsIniPtc)
748 
749  # Before bootstrap fit, do an iterative fit to get rid of outliers
750  count = 1
751  while count <= maxIterationsPtcOutliers:
752  # Note that application of the mask actually shrinks the array
753  # to size rather than setting elements to zero (as we want) so
754  # always update mask itself and re-apply to the original data
755  meanTempVec = meanVecOriginal[mask]
756  varTempVec = varVecOriginal[mask]
757  res = least_squares(errFunc, parsIniPtc, bounds=bounds, args=(meanTempVec, varTempVec))
758  pars = res.x
759 
760  # change this to the original from the temp because the masks are ANDed
761  # meaning once a point is masked it's always masked, and the masks must
762  # always be the same length for broadcasting
763  sigResids = (varVecOriginal - ptcFunc(pars, meanVecOriginal))/np.sqrt(varVecOriginal)
764  newMask = np.array([True if np.abs(r) < sigmaCutPtcOutliers else False for r in sigResids])
765  mask = mask & newMask
766 
767  nDroppedTotal = Counter(mask)[False]
768  self.log.debug(f"Iteration {count}: discarded {nDroppedTotal} points in total for {ampName}")
769  count += 1
770  # objects should never shrink
771  assert (len(mask) == len(timeVecOriginal) == len(meanVecOriginal) == len(varVecOriginal))
772 
773  dataset.visitMask[ampName] = mask # store the final mask
774 
775  parsIniPtc = pars
776  timeVecFinal = timeVecOriginal[mask]
777  meanVecFinal = meanVecOriginal[mask]
778  varVecFinal = varVecOriginal[mask]
779 
780  if Counter(mask)[False] > 0:
781  self.log.info((f"Number of points discarded in PTC of amplifier {ampName}:" +
782  f" {Counter(mask)[False]} out of {len(meanVecOriginal)}"))
783 
784  if (len(meanVecFinal) < len(parsIniPtc)):
785  msg = (f"\nSERIOUS: Not enough data points ({len(meanVecFinal)}) compared to the number of"
786  f"parameters of the PTC model({len(parsIniPtc)}). Setting {ampName} to BAD.")
787  self.log.warn(msg)
788  dataset.badAmps.append(ampName)
789  dataset.gain[ampName] = np.nan
790  dataset.gainErr[ampName] = np.nan
791  dataset.noise[ampName] = np.nan
792  dataset.noiseErr[ampName] = np.nan
793  dataset.nonLinearity[ampName] = np.nan
794  dataset.nonLinearityError[ampName] = np.nan
795  dataset.nonLinearityResiduals[ampName] = np.nan
796  dataset.coefficientLinearizeSquared[ampName] = np.nan
797  continue
798 
799  # Fit the PTC
800  if self.config.doFitBootstrap:
801  parsFit, parsFitErr = self._fitBootstrap(parsIniPtc, meanVecFinal, varVecFinal, ptcFunc)
802  else:
803  parsFit, parsFitErr = self._fitLeastSq(parsIniPtc, meanVecFinal, varVecFinal, ptcFunc)
804 
805  dataset.ptcFitPars[ampName] = parsFit
806  dataset.ptcFitParsError[ampName] = parsFitErr
807 
808  if ptcFitType == 'ASTIERAPPROXIMATION':
809  ptcGain = parsFit[1]
810  ptcGainErr = parsFitErr[1]
811  ptcNoise = np.sqrt(np.fabs(parsFit[2]))
812  ptcNoiseErr = 0.5*(parsFitErr[2]/np.fabs(parsFit[2]))*np.sqrt(np.fabs(parsFit[2]))
813  if ptcFitType == 'POLYNOMIAL':
814  ptcGain = 1./parsFit[1]
815  ptcGainErr = np.fabs(1./parsFit[1])*(parsFitErr[1]/parsFit[1])
816  ptcNoise = np.sqrt(np.fabs(parsFit[0]))*ptcGain
817  ptcNoiseErr = (0.5*(parsFitErr[0]/np.fabs(parsFit[0]))*(np.sqrt(np.fabs(parsFit[0]))))*ptcGain
818 
819  dataset.gain[ampName] = ptcGain
820  dataset.gainErr[ampName] = ptcGainErr
821  dataset.noise[ampName] = ptcNoise
822  dataset.noiseErr[ampName] = ptcNoiseErr
823  dataset.ptcFitType[ampName] = ptcFitType
824 
825  # Non-linearity residuals (NL of mean vs time curve): percentage, and fit to a quadratic function
826  # In this case, len(parsIniNonLinearity) = 3 indicates that we want a quadratic fit
827 
828  (c0, linearizerTableRow, linResidualNonLinearity, parsFitNonLinearity,
829  parsFitErrNonLinearity) = self.calculateLinearityResidualAndLinearizers(timeVecFinal,
830  meanVecFinal)
831  # LinearizerLookupTable
832  if tableArray is not None:
833  tableArray[i, :] = linearizerTableRow
834 
835  dataset.nonLinearity[ampName] = parsFitNonLinearity
836  dataset.nonLinearityError[ampName] = parsFitErrNonLinearity
837  dataset.nonLinearityResiduals[ampName] = linResidualNonLinearity
838  dataset.coefficientLinearizeSquared[ampName] = c0
839 
840  return dataset
841 
842  def plot(self, dataRef, dataset, ptcFitType):
843  dirname = dataRef.getUri(datasetType='cpPipePlotRoot', write=True)
844  if not os.path.exists(dirname):
845  os.makedirs(dirname)
846 
847  detNum = dataRef.dataId[self.config.ccdKey]
848  filename = f"PTC_det{detNum}.pdf"
849  filenameFull = os.path.join(dirname, filename)
850  with PdfPages(filenameFull) as pdfPages:
851  self._plotPtc(dataset, ptcFitType, pdfPages)
852 
853  def _plotPtc(self, dataset, ptcFitType, pdfPages):
854  """Plot PTC, linearity, and linearity residual per amplifier"""
855 
856  if ptcFitType == 'ASTIERAPPROXIMATION':
857  ptcFunc = self.funcAstier
858  stringTitle = r"Var = $\frac{1}{2g^2a_{00}}(\exp (2a_{00} \mu g) - 1) + \frac{n_{00}}{g^2}$"
859 
860  if ptcFitType == 'POLYNOMIAL':
861  ptcFunc = self.funcPolynomial
862  stringTitle = f"Polynomial (degree: {self.config.polynomialFitDegree})"
863 
864  legendFontSize = 7.5
865  labelFontSize = 8
866  titleFontSize = 10
867  supTitleFontSize = 18
868  markerSize = 25
869 
870  # General determination of the size of the plot grid
871  nAmps = len(dataset.ampNames)
872  if nAmps == 2:
873  nRows, nCols = 2, 1
874  nRows = np.sqrt(nAmps)
875  mantissa, _ = np.modf(nRows)
876  if mantissa > 0:
877  nRows = int(nRows) + 1
878  nCols = nRows
879  else:
880  nRows = int(nRows)
881  nCols = nRows
882 
883  f, ax = plt.subplots(nrows=nRows, ncols=nCols, sharex='col', sharey='row', figsize=(13, 10))
884  f2, ax2 = plt.subplots(nrows=nRows, ncols=nCols, sharex='col', sharey='row', figsize=(13, 10))
885 
886  for i, (amp, a, a2) in enumerate(zip(dataset.ampNames, ax.flatten(), ax2.flatten())):
887  meanVecOriginal = np.array(dataset.rawMeans[amp])
888  varVecOriginal = np.array(dataset.rawVars[amp])
889  mask = dataset.visitMask[amp]
890  meanVecFinal = meanVecOriginal[mask]
891  varVecFinal = varVecOriginal[mask]
892  meanVecOutliers = meanVecOriginal[np.invert(mask)]
893  varVecOutliers = varVecOriginal[np.invert(mask)]
894  pars, parsErr = dataset.ptcFitPars[amp], dataset.ptcFitParsError[amp]
895 
896  if ptcFitType == 'ASTIERAPPROXIMATION':
897  ptcA00, ptcA00error = pars[0], parsErr[0]
898  ptcGain, ptcGainError = pars[1], parsErr[1]
899  ptcNoise = np.sqrt(np.fabs(pars[2]))
900  ptcNoiseError = 0.5*(parsErr[2]/np.fabs(pars[2]))*np.sqrt(np.fabs(pars[2]))
901  stringLegend = (f"a00: {ptcA00:.2e}+/-{ptcA00error:.2e}"
902  f"\n Gain: {ptcGain:.4}+/-{ptcGainError:.2e}"
903  f"\n Noise: {ptcNoise:.4}+/-{ptcNoiseError:.2e}")
904 
905  if ptcFitType == 'POLYNOMIAL':
906  ptcGain, ptcGainError = 1./pars[1], np.fabs(1./pars[1])*(parsErr[1]/pars[1])
907  ptcNoise = np.sqrt(np.fabs(pars[0]))*ptcGain
908  ptcNoiseError = (0.5*(parsErr[0]/np.fabs(pars[0]))*(np.sqrt(np.fabs(pars[0]))))*ptcGain
909  stringLegend = (f"Noise: {ptcNoise:.4}+/-{ptcNoiseError:.2e} \n"
910  f"Gain: {ptcGain:.4}+/-{ptcGainError:.2e}")
911 
912  minMeanVecFinal = np.min(meanVecFinal)
913  maxMeanVecFinal = np.max(meanVecFinal)
914  meanVecFit = np.linspace(minMeanVecFinal, maxMeanVecFinal, 100*len(meanVecFinal))
915  minMeanVecOriginal = np.min(meanVecOriginal)
916  maxMeanVecOriginal = np.max(meanVecOriginal)
917  deltaXlim = maxMeanVecOriginal - minMeanVecOriginal
918 
919  a.plot(meanVecFit, ptcFunc(pars, meanVecFit), color='red')
920  a.plot(meanVecFinal, pars[0] + pars[1]*meanVecFinal, color='green', linestyle='--')
921  a.scatter(meanVecFinal, varVecFinal, c='blue', marker='o', s=markerSize)
922  a.scatter(meanVecOutliers, varVecOutliers, c='magenta', marker='s', s=markerSize)
923  a.set_xlabel(r'Mean signal ($\mu$, ADU)', fontsize=labelFontSize)
924  a.set_xticks(meanVecOriginal)
925  a.set_ylabel(r'Variance (ADU$^2$)', fontsize=labelFontSize)
926  a.tick_params(labelsize=11)
927  a.text(0.03, 0.8, stringLegend, transform=a.transAxes, fontsize=legendFontSize)
928  a.set_xscale('linear', fontsize=labelFontSize)
929  a.set_yscale('linear', fontsize=labelFontSize)
930  a.set_title(amp, fontsize=titleFontSize)
931  a.set_xlim([minMeanVecOriginal - 0.2*deltaXlim, maxMeanVecOriginal + 0.2*deltaXlim])
932 
933  # Same, but in log-scale
934  a2.plot(meanVecFit, ptcFunc(pars, meanVecFit), color='red')
935  a2.scatter(meanVecFinal, varVecFinal, c='blue', marker='o', s=markerSize)
936  a2.scatter(meanVecOutliers, varVecOutliers, c='magenta', marker='s', s=markerSize)
937  a2.set_xlabel(r'Mean Signal ($\mu$, ADU)', fontsize=labelFontSize)
938  a2.set_ylabel(r'Variance (ADU$^2$)', fontsize=labelFontSize)
939  a2.tick_params(labelsize=11)
940  a2.text(0.03, 0.8, stringLegend, transform=a2.transAxes, fontsize=legendFontSize)
941  a2.set_xscale('log')
942  a2.set_yscale('log')
943  a2.set_title(amp, fontsize=titleFontSize)
944  a2.set_xlim([minMeanVecOriginal, maxMeanVecOriginal])
945 
946  f.suptitle(f"PTC \n Fit: " + stringTitle, fontsize=20)
947  pdfPages.savefig(f)
948  f2.suptitle(f"PTC (log-log)", fontsize=20)
949  pdfPages.savefig(f2)
950 
951  # Plot mean vs time
952  f, ax = plt.subplots(nrows=4, ncols=4, sharex='col', sharey='row', figsize=(13, 10))
953  for i, (amp, a) in enumerate(zip(dataset.ampNames, ax.flatten())):
954  meanVecFinal = np.array(dataset.rawMeans[amp])[dataset.visitMask[amp]]
955  timeVecFinal = np.array(dataset.rawExpTimes[amp])[dataset.visitMask[amp]]
956 
957  pars, parsErr = dataset.nonLinearity[amp], dataset.nonLinearityError[amp]
958  c0, c0Error = pars[0], parsErr[0]
959  c1, c1Error = pars[1], parsErr[1]
960  c2, c2Error = pars[2], parsErr[2]
961  stringLegend = f"c0: {c0:.4}+/-{c0Error:.2e}\n c1: {c1:.4}+/-{c1Error:.2e}" \
962  + f"\n c2(NL): {c2:.2e}+/-{c2Error:.2e}"
963  a.scatter(timeVecFinal, meanVecFinal)
964  a.plot(timeVecFinal, self.funcPolynomial(pars, timeVecFinal), color='red')
965  a.set_xlabel('Time (sec)', fontsize=labelFontSize)
966  a.set_xticks(timeVecFinal)
967  a.set_ylabel(r'Mean signal ($\mu$, ADU)', fontsize=labelFontSize)
968  a.tick_params(labelsize=labelFontSize)
969  a.text(0.03, 0.75, stringLegend, transform=a.transAxes, fontsize=legendFontSize)
970  a.set_xscale('linear', fontsize=labelFontSize)
971  a.set_yscale('linear', fontsize=labelFontSize)
972  a.set_title(amp, fontsize=titleFontSize)
973 
974  f.suptitle("Linearity \n Fit: " + r"$\mu = c_0 + c_1 t + c_2 t^2$", fontsize=supTitleFontSize)
975  pdfPages.savefig()
976 
977  # Plot linearity residual
978  f, ax = plt.subplots(nrows=4, ncols=4, sharex='col', sharey='row', figsize=(13, 10))
979  for i, (amp, a) in enumerate(zip(dataset.ampNames, ax.flatten())):
980  meanVecFinal = np.array(dataset.rawMeans[amp])[dataset.visitMask[amp]]
981  linRes = np.array(dataset.nonLinearityResiduals[amp])
982 
983  a.scatter(meanVecFinal, linRes)
984  a.axhline(y=0, color='k')
985  a.axvline(x=timeVecFinal[self.config.linResidualTimeIndex], color='g', linestyle='--')
986  a.set_xlabel(r'Mean signal ($\mu$, ADU)', fontsize=labelFontSize)
987  a.set_xticks(meanVecFinal)
988  a.set_ylabel('LR (%)', fontsize=labelFontSize)
989  a.tick_params(labelsize=labelFontSize)
990  a.set_xscale('linear', fontsize=labelFontSize)
991  a.set_yscale('linear', fontsize=labelFontSize)
992  a.set_title(amp, fontsize=titleFontSize)
993 
994  f.suptitle(r"Linearity Residual: $100(1 - \mu_{\rm{ref}}/t_{\rm{ref}})/(\mu / t))$" + "\n" +
995  r"$t_{\rm{ref}}$: " + f"{timeVecFinal[2]} s", fontsize=supTitleFontSize)
996  pdfPages.savefig()
997 
998  return
def _plotPtc(self, dataset, ptcFitType, pdfPages)
Definition: ptc.py:853
def runDataRef(self, dataRef, visitPairs)
Definition: ptc.py:294
def calculateLinearityResidualAndLinearizers(self, exposureTimeVector, meanSignalVector)
Definition: ptc.py:609
def fitPtcAndNonLinearity(self, dataset, ptcFitType, tableArray=None)
Definition: ptc.py:686
def __setattr__(self, attribute, value)
Definition: ptc.py:215
def checkExpLengthEqual(exp1, exp2, v1=None, v2=None, raiseWithMessage=False)
Definition: utils.py:162
def validateIsrConfig(isrTask, mandatory=None, forbidden=None, desirable=None, undesirable=None, checkTrim=True, logName=None)
Definition: utils.py:200
def _getInitialGoodPoints(means, variances, maxDeviationPositive, maxDeviationNegative)
Definition: ptc.py:570
def _fitBootstrap(self, initialParams, dataX, dataY, function, confidenceSigma=1.)
Definition: ptc.py:478
def __init__(self, args, kwargs)
Definition: ptc.py:273
def measureMeanVarPair(self, exposure1, exposure2, region=None)
Definition: ptc.py:372
def _makeZeroSafe(self, array, warn=True, substituteValue=1e-9)
Definition: ptc.py:596
def _fitLeastSq(self, initialParams, dataX, dataY, function)
Definition: ptc.py:426
def plot(self, dataRef, dataset, ptcFitType)
Definition: ptc.py:842