lsst.cp.pipe  19.0.0-5-gf4bae50+9
ptc.py
Go to the documentation of this file.
1 # This file is part of cp_pipe.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 #
22 
23 __all__ = ['MeasurePhotonTransferCurveTask',
24  'MeasurePhotonTransferCurveTaskConfig',
25  'PhotonTransferCurveDataset']
26 
27 import numpy as np
28 import matplotlib.pyplot as plt
29 import os
30 from matplotlib.backends.backend_pdf import PdfPages
31 from sqlite3 import OperationalError
32 from collections import Counter
33 
34 import lsst.afw.math as afwMath
35 import lsst.pex.config as pexConfig
36 import lsst.pipe.base as pipeBase
37 from lsst.ip.isr import IsrTask
38 from .utils import (NonexistentDatasetTaskDataIdContainer, PairedVisitListTaskRunner,
39  checkExpLengthEqual, validateIsrConfig)
40 from scipy.optimize import leastsq, least_squares
41 import numpy.polynomial.polynomial as poly
42 
43 
44 class MeasurePhotonTransferCurveTaskConfig(pexConfig.Config):
45  """Config class for photon transfer curve measurement task"""
46  isr = pexConfig.ConfigurableField(
47  target=IsrTask,
48  doc="""Task to perform instrumental signature removal.""",
49  )
50  isrMandatorySteps = pexConfig.ListField(
51  dtype=str,
52  doc="isr operations that must be performed for valid results. Raises if any of these are False.",
53  default=['doAssembleCcd']
54  )
55  isrForbiddenSteps = pexConfig.ListField(
56  dtype=str,
57  doc="isr operations that must NOT be performed for valid results. Raises if any of these are True",
58  default=['doFlat', 'doFringe', 'doAddDistortionModel', 'doBrighterFatter', 'doUseOpticsTransmission',
59  'doUseFilterTransmission', 'doUseSensorTransmission', 'doUseAtmosphereTransmission']
60  )
61  isrDesirableSteps = pexConfig.ListField(
62  dtype=str,
63  doc="isr operations that it is advisable to perform, but are not mission-critical." +
64  " WARNs are logged for any of these found to be False.",
65  default=['doBias', 'doDark', 'doCrosstalk', 'doDefect']
66  )
67  isrUndesirableSteps = pexConfig.ListField(
68  dtype=str,
69  doc="isr operations that it is *not* advisable to perform in the general case, but are not" +
70  " forbidden as some use-cases might warrant them." +
71  " WARNs are logged for any of these found to be True.",
72  default=['doLinearize']
73  )
74  ccdKey = pexConfig.Field(
75  dtype=str,
76  doc="The key by which to pull a detector from a dataId, e.g. 'ccd' or 'detector'.",
77  default='ccd',
78  )
79  makePlots = pexConfig.Field(
80  dtype=bool,
81  doc="Plot the PTC curves?",
82  default=False,
83  )
84  ptcFitType = pexConfig.ChoiceField(
85  dtype=str,
86  doc="Fit PTC to approximation in Astier+19 (Equation 16) or to a polynomial.",
87  default="POLYNOMIAL",
88  allowed={
89  "POLYNOMIAL": "n-degree polynomial (use 'polynomialFitDegree' to set 'n').",
90  "ASTIERAPPROXIMATION": "Approximation in Astier+19 (Eq. 16)."
91  }
92  )
93  polynomialFitDegree = pexConfig.Field(
94  dtype=int,
95  doc="Degree of polynomial to fit the PTC, when 'ptcFitType'=POLYNOMIAL.",
96  default=2,
97  )
98  polynomialFitDegreeNonLinearity = pexConfig.Field(
99  dtype=int,
100  doc="Degree of polynomial to fit the meanSignal vs exposureTime curve to produce" +
101  " the table for LinearizerLookupTable.",
102  default=3,
103  )
104  binSize = pexConfig.Field(
105  dtype=int,
106  doc="Bin the image by this factor in both dimensions.",
107  default=1,
108  )
109  minMeanSignal = pexConfig.Field(
110  dtype=float,
111  doc="Minimum value (inclusive) of mean signal (in ADU) above which to consider.",
112  default=0,
113  )
114  maxMeanSignal = pexConfig.Field(
115  dtype=float,
116  doc="Maximum value (inclusive) of mean signal (in ADU) below which to consider.",
117  default=9e6,
118  )
119  initialNonLinearityExclusionThresholdPositive = pexConfig.RangeField(
120  dtype=float,
121  doc="Initially exclude data points with a variance that are more than a factor of this from being"
122  " linear in the positive direction, from the PTC fit. Note that these points will also be"
123  " excluded from the non-linearity fit. This is done before the iterative outlier rejection,"
124  " to allow an accurate determination of the sigmas for said iterative fit.",
125  default=0.12,
126  min=0.0,
127  max=1.0,
128  )
129  initialNonLinearityExclusionThresholdNegative = pexConfig.RangeField(
130  dtype=float,
131  doc="Initially exclude data points with a variance that are more than a factor of this from being"
132  " linear in the negative direction, from the PTC fit. Note that these points will also be"
133  " excluded from the non-linearity fit. This is done before the iterative outlier rejection,"
134  " to allow an accurate determination of the sigmas for said iterative fit.",
135  default=0.25,
136  min=0.0,
137  max=1.0,
138  )
139  sigmaCutPtcOutliers = pexConfig.Field(
140  dtype=float,
141  doc="Sigma cut for outlier rejection in PTC.",
142  default=5.0,
143  )
144  maxIterationsPtcOutliers = pexConfig.Field(
145  dtype=int,
146  doc="Maximum number of iterations for outlier rejection in PTC.",
147  default=2,
148  )
149  doFitBootstrap = pexConfig.Field(
150  dtype=bool,
151  doc="Use bootstrap for the PTC fit parameters and errors?.",
152  default=False,
153  )
154  linResidualTimeIndex = pexConfig.Field(
155  dtype=int,
156  doc="Index position in time array for reference time in linearity residual calculation.",
157  default=2,
158  )
159  maxAduForLookupTableLinearizer = pexConfig.Field(
160  dtype=int,
161  doc="Maximum ADU value for the LookupTable linearizer.",
162  default=2**18,
163  )
164 
165 
167  """A simple class to hold the output data from the PTC task.
168 
169  The dataset is made up of a dictionary for each item, keyed by the
170  amplifiers' names, which much be supplied at construction time.
171 
172  New items cannot be added to the class to save accidentally saving to the
173  wrong property, and the class can be frozen if desired.
174 
175  inputVisitPairs records the visits used to produce the data.
176  When fitPtcAndNonLinearity() is run, a mask is built up, which is by definition
177  always the same length as inputVisitPairs, rawExpTimes, rawMeans
178  and rawVars, and is a list of bools, which are incrementally set to False
179  as points are discarded from the fits.
180 
181  PTC fit parameters for polynomials are stored in a list in ascending order
182  of polynomial term, i.e. par[0]*x^0 + par[1]*x + par[2]*x^2 etc
183  with the length of the list corresponding to the order of the polynomial
184  plus one.
185  """
186  def __init__(self, ampNames):
187  # add items to __dict__ directly because __setattr__ is overridden
188 
189  # instance variables
190  self.__dict__["ampNames"] = ampNames
191  self.__dict__["badAmps"] = []
192 
193  # raw data variables
194  self.__dict__["inputVisitPairs"] = {ampName: [] for ampName in ampNames}
195  self.__dict__["visitMask"] = {ampName: [] for ampName in ampNames}
196  self.__dict__["rawExpTimes"] = {ampName: [] for ampName in ampNames}
197  self.__dict__["rawMeans"] = {ampName: [] for ampName in ampNames}
198  self.__dict__["rawVars"] = {ampName: [] for ampName in ampNames}
199 
200  # fit information
201  self.__dict__["ptcFitType"] = {ampName: "" for ampName in ampNames}
202  self.__dict__["ptcFitPars"] = {ampName: [] for ampName in ampNames}
203  self.__dict__["ptcFitParsError"] = {ampName: [] for ampName in ampNames}
204  self.__dict__["nonLinearity"] = {ampName: [] for ampName in ampNames}
205  self.__dict__["nonLinearityError"] = {ampName: [] for ampName in ampNames}
206  self.__dict__["nonLinearityResiduals"] = {ampName: [] for ampName in ampNames}
207  self.__dict__["coefficientLinearizeSquared"] = {ampName: [] for ampName in ampNames}
208 
209  # final results
210  self.__dict__["gain"] = {ampName: -1. for ampName in ampNames}
211  self.__dict__["gainErr"] = {ampName: -1. for ampName in ampNames}
212  self.__dict__["noise"] = {ampName: -1. for ampName in ampNames}
213  self.__dict__["noiseErr"] = {ampName: -1. for ampName in ampNames}
214  self.__dict__["coefficientLinearizeSquared"] = {ampName: [] for ampName in ampNames}
215 
216  def __setattr__(self, attribute, value):
217  """Protect class attributes"""
218  if attribute not in self.__dict__:
219  raise AttributeError(f"{attribute} is not already a member of PhotonTransferCurveDataset, which"
220  " does not support setting of new attributes.")
221  else:
222  self.__dict__[attribute] = value
223 
224  def getVisitsUsed(self, ampName):
225  """Get the visits used, i.e. not discarded, for a given amp.
226 
227  If no mask has been created yet, all visits are returned.
228  """
229  if self.visitMask[ampName] == []:
230  return self.inputVisitPairs[ampName]
231 
232  # if the mask exists it had better be the same length as the visitPairs
233  assert len(self.visitMask[ampName]) == len(self.inputVisitPairs[ampName])
234 
235  pairs = self.inputVisitPairs[ampName]
236  mask = self.visitMask[ampName]
237  # cast to bool required because numpy
238  return [(v1, v2) for ((v1, v2), m) in zip(pairs, mask) if bool(m) is True]
239 
240  def getGoodAmps(self):
241  return [amp for amp in self.ampNames if amp not in self.badAmps]
242 
243 
244 class MeasurePhotonTransferCurveTask(pipeBase.CmdLineTask):
245  """A class to calculate, fit, and plot a PTC from a set of flat pairs.
246 
247  The Photon Transfer Curve (var(signal) vs mean(signal)) is a standard tool
248  used in astronomical detectors characterization (e.g., Janesick 2001,
249  Janesick 2007). This task calculates the PTC from a series of pairs of
250  flat-field images; each pair taken at identical exposure times. The
251  difference image of each pair is formed to eliminate fixed pattern noise,
252  and then the variance of the difference image and the mean of the average image
253  are used to produce the PTC. An n-degree polynomial or the approximation in Equation
254  16 of Astier+19 ("The Shape of the Photon Transfer Curve of CCD sensors",
255  arXiv:1905.08677) can be fitted to the PTC curve. These models include
256  parameters such as the gain (e/ADU) and readout noise.
257 
258  Parameters
259  ----------
260 
261  *args: `list`
262  Positional arguments passed to the Task constructor. None used at this
263  time.
264  **kwargs: `dict`
265  Keyword arguments passed on to the Task constructor. None used at this
266  time.
267 
268  """
269 
270  RunnerClass = PairedVisitListTaskRunner
271  ConfigClass = MeasurePhotonTransferCurveTaskConfig
272  _DefaultName = "measurePhotonTransferCurve"
273 
274  def __init__(self, *args, **kwargs):
275  pipeBase.CmdLineTask.__init__(self, *args, **kwargs)
276  self.makeSubtask("isr")
277  plt.interactive(False) # stop windows popping up when plotting. When headless, use 'agg' backend too
278  validateIsrConfig(self.isr, self.config.isrMandatorySteps,
279  self.config.isrForbiddenSteps, self.config.isrDesirableSteps, checkTrim=False)
280  self.config.validate()
281  self.config.freeze()
282 
283  @classmethod
284  def _makeArgumentParser(cls):
285  """Augment argument parser for the MeasurePhotonTransferCurveTask."""
286  parser = pipeBase.ArgumentParser(name=cls._DefaultName)
287  parser.add_argument("--visit-pairs", dest="visitPairs", nargs="*",
288  help="Visit pairs to use. Each pair must be of the form INT,INT e.g. 123,456")
289  parser.add_id_argument("--id", datasetType="photonTransferCurveDataset",
290  ContainerClass=NonexistentDatasetTaskDataIdContainer,
291  help="The ccds to use, e.g. --id ccd=0..100")
292  return parser
293 
294  @pipeBase.timeMethod
295  def runDataRef(self, dataRef, visitPairs):
296  """Run the Photon Transfer Curve (PTC) measurement task.
297 
298  For a dataRef (which is each detector here),
299  and given a list of visit pairs at different exposure times,
300  measure the PTC.
301 
302  Parameters
303  ----------
304  dataRef : list of lsst.daf.persistence.ButlerDataRef
305  dataRef for the detector for the visits to be fit.
306  visitPairs : `iterable` of `tuple` of `int`
307  Pairs of visit numbers to be processed together
308  """
309 
310  # setup necessary objects
311  detNum = dataRef.dataId[self.config.ccdKey]
312  detector = dataRef.get('camera')[dataRef.dataId[self.config.ccdKey]]
313  # expand some missing fields that we need for lsstCam. This is a work-around
314  # for Gen2 problems that I (RHL) don't feel like solving. The calibs pipelines
315  # (which inherit from CalibTask) use addMissingKeys() to do basically the same thing
316  #
317  # Basically, the butler's trying to look up the fields in `raw_visit` which won't work
318  for name in dataRef.getButler().getKeys('bias'):
319  if name not in dataRef.dataId:
320  try:
321  dataRef.dataId[name] = \
322  dataRef.getButler().queryMetadata('raw', [name], detector=detNum)[0]
323  except OperationalError:
324  pass
325 
326  amps = detector.getAmplifiers()
327  ampNames = [amp.getName() for amp in amps]
328  dataset = PhotonTransferCurveDataset(ampNames)
329 
330  self.log.info('Measuring PTC using %s visits for detector %s' % (visitPairs, detNum))
331 
332  for (v1, v2) in visitPairs:
333  # Perform ISR on each exposure
334  dataRef.dataId['visit'] = v1
335  exp1 = self.isr.runDataRef(dataRef).exposure
336  dataRef.dataId['visit'] = v2
337  exp2 = self.isr.runDataRef(dataRef).exposure
338  del dataRef.dataId['visit']
339 
340  checkExpLengthEqual(exp1, exp2, v1, v2, raiseWithMessage=True)
341  expTime = exp1.getInfo().getVisitInfo().getExposureTime()
342 
343  for amp in detector:
344  mu, varDiff = self.measureMeanVarPair(exp1, exp2, region=amp.getBBox())
345  ampName = amp.getName()
346 
347  dataset.rawExpTimes[ampName].append(expTime)
348  dataset.rawMeans[ampName].append(mu)
349  dataset.rawVars[ampName].append(varDiff)
350  dataset.inputVisitPairs[ampName].append((v1, v2))
351 
352  numberAmps = len(detector.getAmplifiers())
353  numberAduValues = self.config.maxAduForLookupTableLinearizer
354  lookupTableArray = np.zeros((numberAmps, numberAduValues), dtype=np.float32)
355 
356  # Fit PTC and (non)linearity of signal vs time curve, produce linearizer
357  self.fitPtcAndNonLinearity(dataset, lookupTableArray, ptcFitType=self.config.ptcFitType)
358 
359  if self.config.makePlots:
360  self.plot(dataRef, dataset, ptcFitType=self.config.ptcFitType)
361 
362  # Save data, PTC fit, and NL fit dictionaries
363  self.log.info(f"Writing PTC and NL data to {dataRef.getUri(write=True)}")
364  dataRef.put(dataset, datasetType="photonTransferCurveDataset")
365 
366  self.log.info('Finished measuring PTC for in detector %s' % detNum)
367 
368  return pipeBase.Struct(exitStatus=0)
369 
370  def measureMeanVarPair(self, exposure1, exposure2, region=None):
371  """Calculate the mean signal of two exposures and the variance of their difference.
372 
373  Parameters
374  ----------
375  exposure1 : `lsst.afw.image.exposure.exposure.ExposureF`
376  First exposure of flat field pair.
377 
378  exposure2 : `lsst.afw.image.exposure.exposure.ExposureF`
379  Second exposure of flat field pair.
380 
381  region : `lsst.geom.Box2I`
382  Region of each exposure where to perform the calculations (e.g, an amplifier).
383 
384  Return
385  ------
386 
387  mu : `np.float`
388  0.5*(mu1 + mu2), where mu1, and mu2 are the clipped means of the regions in
389  both exposures.
390 
391  varDiff : `np.float`
392  Half of the clipped variance of the difference of the regions inthe two input
393  exposures.
394  """
395 
396  if region is not None:
397  im1Area = exposure1.maskedImage[region]
398  im2Area = exposure2.maskedImage[region]
399  else:
400  im1Area = exposure1.maskedImage
401  im2Area = exposure2.maskedImage
402 
403  im1Area = afwMath.binImage(im1Area, self.config.binSize)
404  im2Area = afwMath.binImage(im2Area, self.config.binSize)
405 
406  # Clipped mean of images; then average of mean.
407  mu1 = afwMath.makeStatistics(im1Area, afwMath.MEANCLIP).getValue()
408  mu2 = afwMath.makeStatistics(im2Area, afwMath.MEANCLIP).getValue()
409  mu = 0.5*(mu1 + mu2)
410 
411  # Take difference of pairs
412  # symmetric formula: diff = (mu2*im1-mu1*im2)/(0.5*(mu1+mu2))
413  temp = im2Area.clone()
414  temp *= mu1
415  diffIm = im1Area.clone()
416  diffIm *= mu2
417  diffIm -= temp
418  diffIm /= mu
419 
420  varDiff = 0.5*(afwMath.makeStatistics(diffIm, afwMath.VARIANCECLIP).getValue())
421 
422  return mu, varDiff
423 
424  def _fitLeastSq(self, initialParams, dataX, dataY, function):
425  """Do a fit and estimate the parameter errors using using scipy.optimize.leastq.
426 
427  optimize.leastsq returns the fractional covariance matrix. To estimate the
428  standard deviation of the fit parameters, multiply the entries of this matrix
429  by the reduced chi squared and take the square root of the diagonal elements.
430 
431  Parameters
432  ----------
433  initialParams : list of np.float
434  initial values for fit parameters. For ptcFitType=POLYNOMIAL, its length
435  determines the degree of the polynomial.
436 
437  dataX : np.array of np.float
438  Data in the abscissa axis.
439 
440  dataY : np.array of np.float
441  Data in the ordinate axis.
442 
443  function : callable object (function)
444  Function to fit the data with.
445 
446  Return
447  ------
448  pFitSingleLeastSquares : list of np.float
449  List with fitted parameters.
450 
451  pErrSingleLeastSquares : list of np.float
452  List with errors for fitted parameters.
453  """
454 
455  def errFunc(p, x, y):
456  return function(p, x) - y
457 
458  pFit, pCov, infoDict, errMessage, success = leastsq(errFunc, initialParams,
459  args=(dataX, dataY), full_output=1, epsfcn=0.0001)
460 
461  if (len(dataY) > len(initialParams)) and pCov is not None:
462  reducedChiSq = (errFunc(pFit, dataX, dataY)**2).sum()/(len(dataY)-len(initialParams))
463  pCov *= reducedChiSq
464  else:
465  pCov[:, :] = np.inf
466 
467  errorVec = []
468  for i in range(len(pFit)):
469  errorVec.append(np.fabs(pCov[i][i])**0.5)
470 
471  pFitSingleLeastSquares = pFit
472  pErrSingleLeastSquares = np.array(errorVec)
473 
474  return pFitSingleLeastSquares, pErrSingleLeastSquares
475 
476  def _fitBootstrap(self, initialParams, dataX, dataY, function, confidenceSigma=1.):
477  """Do a fit using least squares and bootstrap to estimate parameter errors.
478 
479  The bootstrap error bars are calculated by fitting 100 random data sets.
480 
481  Parameters
482  ----------
483  initialParams : list of np.float
484  initial values for fit parameters. For ptcFitType=POLYNOMIAL, its length
485  determines the degree of the polynomial.
486 
487  dataX : np.array of np.float
488  Data in the abscissa axis.
489 
490  dataY : np.array of np.float
491  Data in the ordinate axis.
492 
493  function : callable object (function)
494  Function to fit the data with.
495 
496  confidenceSigma : np.float
497  Number of sigmas that determine confidence interval for the bootstrap errors.
498 
499  Return
500  ------
501  pFitBootstrap : list of np.float
502  List with fitted parameters.
503 
504  pErrBootstrap : list of np.float
505  List with errors for fitted parameters.
506  """
507 
508  def errFunc(p, x, y):
509  return function(p, x) - y
510 
511  # Fit first time
512  pFit, _ = leastsq(errFunc, initialParams, args=(dataX, dataY), full_output=0)
513 
514  # Get the stdev of the residuals
515  residuals = errFunc(pFit, dataX, dataY)
516  sigmaErrTotal = np.std(residuals)
517 
518  # 100 random data sets are generated and fitted
519  pars = []
520  for i in range(100):
521  randomDelta = np.random.normal(0., sigmaErrTotal, len(dataY))
522  randomDataY = dataY + randomDelta
523  randomFit, _ = leastsq(errFunc, initialParams,
524  args=(dataX, randomDataY), full_output=0)
525  pars.append(randomFit)
526  pars = np.array(pars)
527  meanPfit = np.mean(pars, 0)
528 
529  # confidence interval for parameter estimates
530  nSigma = confidenceSigma
531  errPfit = nSigma*np.std(pars, 0)
532  pFitBootstrap = meanPfit
533  pErrBootstrap = errPfit
534  return pFitBootstrap, pErrBootstrap
535 
536  def funcPolynomial(self, pars, x):
537  """Polynomial function definition"""
538  return poly.polyval(x, [*pars])
539 
540  def funcAstier(self, pars, x):
541  """Single brighter-fatter parameter model for PTC; Equation 16 of Astier+19"""
542  a00, gain, noise = pars
543  return 0.5/(a00*gain*gain)*(np.exp(2*a00*x*gain)-1) + noise/(gain*gain)
544 
545  @staticmethod
546  def _initialParsForPolynomial(order):
547  assert(order >= 2)
548  pars = np.zeros(order, dtype=np.float)
549  pars[0] = 10
550  pars[1] = 1
551  pars[2:] = 0.0001
552  return pars
553 
554  @staticmethod
555  def _boundsForPolynomial(initialPars):
556  lowers = [np.NINF for p in initialPars]
557  uppers = [np.inf for p in initialPars]
558  lowers[1] = 0 # no negative gains
559  return (lowers, uppers)
560 
561  @staticmethod
562  def _boundsForAstier(initialPars):
563  lowers = [np.NINF for p in initialPars]
564  uppers = [np.inf for p in initialPars]
565  return (lowers, uppers)
566 
567  @staticmethod
568  def _getInitialGoodPoints(means, variances, maxDeviationPositive, maxDeviationNegative):
569  """Return a boolean array to mask bad points.
570 
571  A linear function has a constant ratio, so find the median
572  value of the ratios, and exclude the points that deviate
573  from that by more than a factor of maxDeviationPositive/negative.
574  Asymmetric deviations are supported as we expect the PTC to turn
575  down as the flux increases, but sometimes it anomalously turns
576  upwards just before turning over, which ruins the fits, so it
577  is wise to be stricter about restricting positive outliers than
578  negative ones.
579 
580  Too high and points that are so bad that fit will fail will be included
581  Too low and the non-linear points will be excluded, biasing the NL fit."""
582  ratios = [b/a for (a, b) in zip(means, variances)]
583  medianRatio = np.median(ratios)
584  ratioDeviations = [(r/medianRatio)-1 for r in ratios]
585 
586  # so that it doesn't matter if the deviation is expressed as positive or negative
587  maxDeviationPositive = abs(maxDeviationPositive)
588  maxDeviationNegative = -1. * abs(maxDeviationNegative)
589 
590  goodPoints = np.array([True if (r < maxDeviationPositive and r > maxDeviationNegative)
591  else False for r in ratioDeviations])
592  return goodPoints
593 
594  def _makeZeroSafe(self, array, warn=True, substituteValue=1e-9):
595  """"""
596  nBad = Counter(array)[0]
597  if nBad == 0:
598  return array
599 
600  if warn:
601  msg = f"Found {nBad} zeros in array at elements {[x for x in np.where(array==0)[0]]}"
602  self.log.warn(msg)
603 
604  array[array == 0] = substituteValue
605  return array
606 
607  def calculateLinearityResidualAndLinearizers(self, exposureTimeVector, meanSignalVector):
608  """Calculate linearity residual and fit an n-order polynomial to the mean vs time curve
609  to produce corrections (deviation from linear part of polynomial) for a particular amplifier
610  to populate LinearizeLookupTable. Use quadratic and linear parts of this polynomial to approximate
611  c0 for LinearizeSquared."
612 
613  Parameters
614  ---------
615 
616  exposureTimeVector: `list` of `np.float`
617  List of exposure times for each flat pair
618 
619  meanSignalVector: `list` of `np.float`
620  List of mean signal from diference image of flat pairs
621 
622  Returns
623  -------
624  c0: `np.float`
625  Coefficient for LinearizeSquared, where corrImage = uncorrImage + c0*uncorrImage^2.
626  c0 ~ -k2/(k1^2), where k1 and k2 are fit from
627  meanSignalVector = k0 + k1*exposureTimeVector + k2*exposureTimeVector^2 +...
628  + kn*exposureTimeVector^n, with n = "polynomialFitDegreeNonLinearity".
629 
630  linearizerTableRow: list of `np.float`
631  One dimensional array with deviation from linear part of n-order polynomial fit
632  to mean vs time curve. This array will be one row (for the particular amplifier at hand)
633  of the table array for LinearizeLookupTable.
634 
635  linResidual: list of `np.float`
636  Linearity residual from the mean vs time curve, defined as
637  100*(1 - meanSignalReference/expTimeReference/(meanSignal/expTime).
638 
639  parsFit: list of `np.float`
640  Parameters from n-order polynomial fit to mean vs time curve.
641 
642  parsFitErr: list of `np.float`
643  Parameters from n-order polynomial fit to mean vs time curve.
644 
645  """
646 
647  # Lookup table linearizer
648  parsIniNonLinearity = self._initialParsForPolynomial(self.config.polynomialFitDegreeNonLinearity + 1)
649  if self.config.doFitBootstrap:
650  parsFit, parsFitErr = self._fitBootstrap(parsIniNonLinearity, exposureTimeVector,
651  meanSignalVector, self.funcPolynomial)
652  else:
653  parsFit, parsFitErr = self._fitLeastSq(parsIniNonLinearity, exposureTimeVector, meanSignalVector,
654  self.funcPolynomial)
655 
656  # Use linear part to get time at wich signal is maxAduForLookupTableLinearizer ADU
657  tMax = (self.config.maxAduForLookupTableLinearizer - parsFit[0])/parsFit[1]
658  timeRange = np.linspace(0, tMax, self.config.maxAduForLookupTableLinearizer)
659  signalIdeal = parsFit[0] + parsFit[1]*timeRange
660  signalUncorrected = self.funcPolynomial(parsFit, timeRange)
661  linearizerTableRow = signalIdeal - signalUncorrected # LinearizerLookupTable has corrections
662 
663  # Use quadratic and linear part of fit to produce c0 for LinearizeSquared
664  # Check that magnitude of higher order (>= 3) coefficents of the polyFit are small,
665  # i.e., less than threshold = 1e-10 (typical quadratic and cubic coefficents are ~1e-6
666  # and ~1e-12).
667  k1, k2 = parsFit[1], parsFit[2]
668  c0 = -k2/(k1**2) # c0 coefficient for LinearizeSquared
669  for coefficient in parsFit[3:]:
670  if np.fabs(coefficient) > 1e-10:
671  msg = f"Coefficient {coefficient} in polynomial fit larger than threshold 1e-10."
672  self.log.warn(msg)
673 
674  # Linearity residual
675  linResidualTimeIndex = self.config.linResidualTimeIndex
676  if exposureTimeVector[linResidualTimeIndex] == 0.0:
677  raise RuntimeError("Reference time for linearity residual can't be 0.0")
678  linResidual = 100*(1 - ((meanSignalVector[linResidualTimeIndex] /
679  exposureTimeVector[linResidualTimeIndex]) /
680  (meanSignalVector/exposureTimeVector)))
681 
682  return c0, linearizerTableRow, linResidual, parsFit, parsFitErr
683 
684  def fitPtcAndNonLinearity(self, dataset, tableArray, ptcFitType):
685  """Fit the photon transfer curve and calculate linearity and residuals.
686 
687  Fit the photon transfer curve with either a polynomial of the order
688  specified in the task config, or using the Astier approximation.
689 
690  Sigma clipping is performed iteratively for the fit, as well as an
691  initial clipping of data points that are more than
692  config.initialNonLinearityExclusionThreshold away from lying on a
693  straight line. This other step is necessary because the photon transfer
694  curve turns over catastrophically at very high flux (because saturation
695  drops the variance to ~0) and these far outliers cause the initial fit
696  to fail, meaning the sigma cannot be calculated to perform the
697  sigma-clipping.
698 
699  Parameters
700  ----------
701  dataset : `lsst.cp.pipe.ptc.PhotonTransferCurveDataset`
702  The dataset containing the means, variances and exposure times
703  ptcFitType : `str`
704  Fit a 'POLYNOMIAL' (degree: 'polynomialFitDegree') or
705  'ASTIERAPPROXIMATION' to the PTC
706  tableArray : `np.array`
707  Look-up table array with size rows=nAmps and columns=ADU values
708  """
709 
710  def errFunc(p, x, y):
711  return ptcFunc(p, x) - y
712 
713  sigmaCutPtcOutliers = self.config.sigmaCutPtcOutliers
714  maxIterationsPtcOutliers = self.config.maxIterationsPtcOutliers
715 
716  for i, ampName in enumerate(dataset.ampNames):
717  timeVecOriginal = np.array(dataset.rawExpTimes[ampName])
718  meanVecOriginal = np.array(dataset.rawMeans[ampName])
719  varVecOriginal = np.array(dataset.rawVars[ampName])
720  varVecOriginal = self._makeZeroSafe(varVecOriginal)
721 
722  mask = ((meanVecOriginal >= self.config.minMeanSignal) &
723  (meanVecOriginal <= self.config.maxMeanSignal))
724 
725  goodPoints = self._getInitialGoodPoints(meanVecOriginal, varVecOriginal,
726  self.config.initialNonLinearityExclusionThresholdPositive,
727  self.config.initialNonLinearityExclusionThresholdNegative)
728  mask = mask & goodPoints
729 
730  if ptcFitType == 'ASTIERAPPROXIMATION':
731  ptcFunc = self.funcAstier
732  parsIniPtc = [-1e-9, 1.0, 10.] # a00, gain, noise
733  bounds = self._boundsForAstier(parsIniPtc)
734  if ptcFitType == 'POLYNOMIAL':
735  ptcFunc = self.funcPolynomial
736  parsIniPtc = self._initialParsForPolynomial(self.config.polynomialFitDegree + 1)
737  bounds = self._boundsForPolynomial(parsIniPtc)
738 
739  # Before bootstrap fit, do an iterative fit to get rid of outliers
740  count = 1
741  while count <= maxIterationsPtcOutliers:
742  # Note that application of the mask actually shrinks the array
743  # to size rather than setting elements to zero (as we want) so
744  # always update mask itself and re-apply to the original data
745  meanTempVec = meanVecOriginal[mask]
746  varTempVec = varVecOriginal[mask]
747  res = least_squares(errFunc, parsIniPtc, bounds=bounds, args=(meanTempVec, varTempVec))
748  pars = res.x
749 
750  # change this to the original from the temp because the masks are ANDed
751  # meaning once a point is masked it's always masked, and the masks must
752  # always be the same length for broadcasting
753  sigResids = (varVecOriginal - ptcFunc(pars, meanVecOriginal))/np.sqrt(varVecOriginal)
754  newMask = np.array([True if np.abs(r) < sigmaCutPtcOutliers else False for r in sigResids])
755  mask = mask & newMask
756 
757  nDroppedTotal = Counter(mask)[False]
758  self.log.debug(f"Iteration {count}: discarded {nDroppedTotal} points in total for {ampName}")
759  count += 1
760  # objects should never shrink
761  assert (len(mask) == len(timeVecOriginal) == len(meanVecOriginal) == len(varVecOriginal))
762 
763  dataset.visitMask[ampName] = mask # store the final mask
764 
765  parsIniPtc = pars
766  timeVecFinal = timeVecOriginal[mask]
767  meanVecFinal = meanVecOriginal[mask]
768  varVecFinal = varVecOriginal[mask]
769 
770  if Counter(mask)[False] > 0:
771  self.log.info((f"Number of points discarded in PTC of amplifier {ampName}:" +
772  f" {Counter(mask)[False]} out of {len(meanVecOriginal)}"))
773 
774  if (len(meanVecFinal) < len(parsIniPtc)):
775  msg = (f"\nSERIOUS: Not enough data points ({len(meanVecFinal)}) compared to the number of"
776  f"parameters of the PTC model({len(parsIniPtc)}). Setting {ampName} to BAD.")
777  self.log.warn(msg)
778  dataset.badAmps.append(ampName)
779  dataset.gain[ampName] = np.nan
780  dataset.gainErr[ampName] = np.nan
781  dataset.noise[ampName] = np.nan
782  dataset.noiseErr[ampName] = np.nan
783  dataset.nonLinearity[ampName] = np.nan
784  dataset.nonLinearityError[ampName] = np.nan
785  dataset.nonLinearityResiduals[ampName] = np.nan
786  continue
787 
788  # Fit the PTC
789  if self.config.doFitBootstrap:
790  parsFit, parsFitErr = self._fitBootstrap(parsIniPtc, meanVecFinal, varVecFinal, ptcFunc)
791  else:
792  parsFit, parsFitErr = self._fitLeastSq(parsIniPtc, meanVecFinal, varVecFinal, ptcFunc)
793 
794  dataset.ptcFitPars[ampName] = parsFit
795  dataset.ptcFitParsError[ampName] = parsFitErr
796 
797  if ptcFitType == 'ASTIERAPPROXIMATION':
798  ptcGain = parsFit[1]
799  ptcGainErr = parsFitErr[1]
800  ptcNoise = np.sqrt(np.fabs(parsFit[2]))
801  ptcNoiseErr = 0.5*(parsFitErr[2]/np.fabs(parsFit[2]))*np.sqrt(np.fabs(parsFit[2]))
802  if ptcFitType == 'POLYNOMIAL':
803  ptcGain = 1./parsFit[1]
804  ptcGainErr = np.fabs(1./parsFit[1])*(parsFitErr[1]/parsFit[1])
805  ptcNoise = np.sqrt(np.fabs(parsFit[0]))*ptcGain
806  ptcNoiseErr = (0.5*(parsFitErr[0]/np.fabs(parsFit[0]))*(np.sqrt(np.fabs(parsFit[0]))))*ptcGain
807 
808  dataset.gain[ampName] = ptcGain
809  dataset.gainErr[ampName] = ptcGainErr
810  dataset.noise[ampName] = ptcNoise
811  dataset.noiseErr[ampName] = ptcNoiseErr
812  dataset.ptcFitType[ampName] = ptcFitType
813 
814  # Non-linearity residuals (NL of mean vs time curve): percentage, and fit to a quadratic function
815  # In this case, len(parsIniNonLinearity) = 3 indicates that we want a quadratic fit
816 
817  (c0, linearizerTableRow, linResidualNonLinearity, parsFitNonLinearity,
818  parsFitErrNonLinearity) = self.calculateLinearityResidualAndLinearizers(timeVecFinal,
819  meanVecFinal)
820  # LinearizerLookupTable
821  tableArray[i, :] = linearizerTableRow
822 
823  dataset.nonLinearity[ampName] = parsFitNonLinearity
824  dataset.nonLinearityError[ampName] = parsFitErrNonLinearity
825  dataset.nonLinearityResiduals[ampName] = linResidualNonLinearity
826  dataset.coefficientLinearizeSquared[ampName] = c0
827 
828  return
829 
830  def plot(self, dataRef, dataset, ptcFitType):
831  dirname = dataRef.getUri(datasetType='cpPipePlotRoot', write=True)
832  if not os.path.exists(dirname):
833  os.makedirs(dirname)
834 
835  detNum = dataRef.dataId[self.config.ccdKey]
836  filename = f"PTC_det{detNum}.pdf"
837  filenameFull = os.path.join(dirname, filename)
838  with PdfPages(filenameFull) as pdfPages:
839  self._plotPtc(dataset, ptcFitType, pdfPages)
840 
841  def _plotPtc(self, dataset, ptcFitType, pdfPages):
842  """Plot PTC, linearity, and linearity residual per amplifier"""
843 
844  if ptcFitType == 'ASTIERAPPROXIMATION':
845  ptcFunc = self.funcAstier
846  stringTitle = r"Var = $\frac{1}{2g^2a_{00}}(\exp (2a_{00} \mu g) - 1) + \frac{n_{00}}{g^2}$"
847 
848  if ptcFitType == 'POLYNOMIAL':
849  ptcFunc = self.funcPolynomial
850  stringTitle = f"Polynomial (degree: {self.config.polynomialFitDegree})"
851 
852  legendFontSize = 7.5
853  labelFontSize = 8
854  titleFontSize = 10
855  supTitleFontSize = 18
856  markerSize = 25
857 
858  # General determination of the size of the plot grid
859  nAmps = len(dataset.ampNames)
860  if nAmps == 2:
861  nRows, nCols = 2, 1
862  nRows = np.sqrt(nAmps)
863  mantissa, _ = np.modf(nRows)
864  if mantissa > 0:
865  nRows = int(nRows) + 1
866  nCols = nRows
867  else:
868  nRows = int(nRows)
869  nCols = nRows
870 
871  f, ax = plt.subplots(nrows=nRows, ncols=nCols, sharex='col', sharey='row', figsize=(13, 10))
872  f2, ax2 = plt.subplots(nrows=nRows, ncols=nCols, sharex='col', sharey='row', figsize=(13, 10))
873 
874  for i, (amp, a, a2) in enumerate(zip(dataset.ampNames, ax.flatten(), ax2.flatten())):
875  meanVecOriginal = np.array(dataset.rawMeans[amp])
876  varVecOriginal = np.array(dataset.rawVars[amp])
877  mask = dataset.visitMask[amp]
878  meanVecFinal = meanVecOriginal[mask]
879  varVecFinal = varVecOriginal[mask]
880  meanVecOutliers = meanVecOriginal[np.invert(mask)]
881  varVecOutliers = varVecOriginal[np.invert(mask)]
882  pars, parsErr = dataset.ptcFitPars[amp], dataset.ptcFitParsError[amp]
883 
884  if ptcFitType == 'ASTIERAPPROXIMATION':
885  ptcA00, ptcA00error = pars[0], parsErr[0]
886  ptcGain, ptcGainError = pars[1], parsErr[1]
887  ptcNoise = np.sqrt(np.fabs(pars[2]))
888  ptcNoiseError = 0.5*(parsErr[2]/np.fabs(pars[2]))*np.sqrt(np.fabs(pars[2]))
889  stringLegend = (f"a00: {ptcA00:.2e}+/-{ptcA00error:.2e}"
890  f"\n Gain: {ptcGain:.4}+/-{ptcGainError:.2e}"
891  f"\n Noise: {ptcNoise:.4}+/-{ptcNoiseError:.2e}")
892 
893  if ptcFitType == 'POLYNOMIAL':
894  ptcGain, ptcGainError = 1./pars[1], np.fabs(1./pars[1])*(parsErr[1]/pars[1])
895  ptcNoise = np.sqrt(np.fabs(pars[0]))*ptcGain
896  ptcNoiseError = (0.5*(parsErr[0]/np.fabs(pars[0]))*(np.sqrt(np.fabs(pars[0]))))*ptcGain
897  stringLegend = (f"Noise: {ptcNoise:.4}+/-{ptcNoiseError:.2e} \n"
898  f"Gain: {ptcGain:.4}+/-{ptcGainError:.2e}")
899 
900  minMeanVecFinal = np.min(meanVecFinal)
901  maxMeanVecFinal = np.max(meanVecFinal)
902  meanVecFit = np.linspace(minMeanVecFinal, maxMeanVecFinal, 100*len(meanVecFinal))
903  minMeanVecOriginal = np.min(meanVecOriginal)
904  maxMeanVecOriginal = np.max(meanVecOriginal)
905  deltaXlim = maxMeanVecOriginal - minMeanVecOriginal
906 
907  a.plot(meanVecFit, ptcFunc(pars, meanVecFit), color='red')
908  a.plot(meanVecFinal, pars[0] + pars[1]*meanVecFinal, color='green', linestyle='--')
909  a.scatter(meanVecFinal, varVecFinal, c='blue', marker='o', s=markerSize)
910  a.scatter(meanVecOutliers, varVecOutliers, c='magenta', marker='s', s=markerSize)
911  a.set_xlabel(r'Mean signal ($\mu$, ADU)', fontsize=labelFontSize)
912  a.set_xticks(meanVecOriginal)
913  a.set_ylabel(r'Variance (ADU$^2$)', fontsize=labelFontSize)
914  a.tick_params(labelsize=11)
915  a.text(0.03, 0.8, stringLegend, transform=a.transAxes, fontsize=legendFontSize)
916  a.set_xscale('linear', fontsize=labelFontSize)
917  a.set_yscale('linear', fontsize=labelFontSize)
918  a.set_title(amp, fontsize=titleFontSize)
919  a.set_xlim([minMeanVecOriginal - 0.2*deltaXlim, maxMeanVecOriginal + 0.2*deltaXlim])
920 
921  # Same, but in log-scale
922  a2.plot(meanVecFit, ptcFunc(pars, meanVecFit), color='red')
923  a2.scatter(meanVecFinal, varVecFinal, c='blue', marker='o', s=markerSize)
924  a2.scatter(meanVecOutliers, varVecOutliers, c='magenta', marker='s', s=markerSize)
925  a2.set_xlabel(r'Mean Signal ($\mu$, ADU)', fontsize=labelFontSize)
926  a2.set_ylabel(r'Variance (ADU$^2$)', fontsize=labelFontSize)
927  a2.tick_params(labelsize=11)
928  a2.text(0.03, 0.8, stringLegend, transform=a2.transAxes, fontsize=legendFontSize)
929  a2.set_xscale('log')
930  a2.set_yscale('log')
931  a2.set_title(amp, fontsize=titleFontSize)
932  a2.set_xlim([minMeanVecOriginal, maxMeanVecOriginal])
933 
934  f.suptitle(f"PTC \n Fit: " + stringTitle, fontsize=20)
935  pdfPages.savefig(f)
936  f2.suptitle(f"PTC (log-log)", fontsize=20)
937  pdfPages.savefig(f2)
938 
939  # Plot mean vs time
940  f, ax = plt.subplots(nrows=4, ncols=4, sharex='col', sharey='row', figsize=(13, 10))
941  for i, (amp, a) in enumerate(zip(dataset.ampNames, ax.flatten())):
942  meanVecFinal = np.array(dataset.rawMeans[amp])[dataset.visitMask[amp]]
943  timeVecFinal = np.array(dataset.rawExpTimes[amp])[dataset.visitMask[amp]]
944 
945  pars, parsErr = dataset.nonLinearity[amp], dataset.nonLinearityError[amp]
946  c0, c0Error = pars[0], parsErr[0]
947  c1, c1Error = pars[1], parsErr[1]
948  c2, c2Error = pars[2], parsErr[2]
949  stringLegend = f"c0: {c0:.4}+/-{c0Error:.2e}\n c1: {c1:.4}+/-{c1Error:.2e}" \
950  + f"\n c2(NL): {c2:.2e}+/-{c2Error:.2e}"
951  a.scatter(timeVecFinal, meanVecFinal)
952  a.plot(timeVecFinal, self.funcPolynomial(pars, timeVecFinal), color='red')
953  a.set_xlabel('Time (sec)', fontsize=labelFontSize)
954  a.set_xticks(timeVecFinal)
955  a.set_ylabel(r'Mean signal ($\mu$, ADU)', fontsize=labelFontSize)
956  a.tick_params(labelsize=labelFontSize)
957  a.text(0.03, 0.75, stringLegend, transform=a.transAxes, fontsize=legendFontSize)
958  a.set_xscale('linear', fontsize=labelFontSize)
959  a.set_yscale('linear', fontsize=labelFontSize)
960  a.set_title(amp, fontsize=titleFontSize)
961 
962  f.suptitle("Linearity \n Fit: " + r"$\mu = c_0 + c_1 t + c_2 t^2$", fontsize=supTitleFontSize)
963  pdfPages.savefig()
964 
965  # Plot linearity residual
966  f, ax = plt.subplots(nrows=4, ncols=4, sharex='col', sharey='row', figsize=(13, 10))
967  for i, (amp, a) in enumerate(zip(dataset.ampNames, ax.flatten())):
968  meanVecFinal = np.array(dataset.rawMeans[amp])[dataset.visitMask[amp]]
969  linRes = np.array(dataset.nonLinearityResiduals[amp])
970 
971  a.scatter(meanVecFinal, linRes)
972  a.axhline(y=0, color='k')
973  a.axvline(x=timeVecFinal[self.config.linResidualTimeIndex], color='g', linestyle='--')
974  a.set_xlabel(r'Mean signal ($\mu$, ADU)', fontsize=labelFontSize)
975  a.set_xticks(meanVecFinal)
976  a.set_ylabel('LR (%)', fontsize=labelFontSize)
977  a.tick_params(labelsize=labelFontSize)
978  a.set_xscale('linear', fontsize=labelFontSize)
979  a.set_yscale('linear', fontsize=labelFontSize)
980  a.set_title(amp, fontsize=titleFontSize)
981 
982  f.suptitle(r"Linearity Residual: $100(1 - \mu_{\rm{ref}}/t_{\rm{ref}})/(\mu / t))$" + "\n" +
983  r"$t_{\rm{ref}}$: " + f"{timeVecFinal[2]} s", fontsize=supTitleFontSize)
984  pdfPages.savefig()
985 
986  return
def _plotPtc(self, dataset, ptcFitType, pdfPages)
Definition: ptc.py:841
def runDataRef(self, dataRef, visitPairs)
Definition: ptc.py:295
def calculateLinearityResidualAndLinearizers(self, exposureTimeVector, meanSignalVector)
Definition: ptc.py:607
def __setattr__(self, attribute, value)
Definition: ptc.py:216
def checkExpLengthEqual(exp1, exp2, v1=None, v2=None, raiseWithMessage=False)
Definition: utils.py:162
def fitPtcAndNonLinearity(self, dataset, tableArray, ptcFitType)
Definition: ptc.py:684
def validateIsrConfig(isrTask, mandatory=None, forbidden=None, desirable=None, undesirable=None, checkTrim=True, logName=None)
Definition: utils.py:200
def _getInitialGoodPoints(means, variances, maxDeviationPositive, maxDeviationNegative)
Definition: ptc.py:568
def _fitBootstrap(self, initialParams, dataX, dataY, function, confidenceSigma=1.)
Definition: ptc.py:476
def __init__(self, args, kwargs)
Definition: ptc.py:274
def measureMeanVarPair(self, exposure1, exposure2, region=None)
Definition: ptc.py:370
def _makeZeroSafe(self, array, warn=True, substituteValue=1e-9)
Definition: ptc.py:594
def _fitLeastSq(self, initialParams, dataX, dataY, function)
Definition: ptc.py:424
def plot(self, dataRef, dataset, ptcFitType)
Definition: ptc.py:830