lsst.cp.pipe  19.0.0-9-gfb54480+2
ptc.py
Go to the documentation of this file.
1 # This file is part of cp_pipe.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 #
22 
23 __all__ = ['MeasurePhotonTransferCurveTask',
24  'MeasurePhotonTransferCurveTaskConfig',
25  'PhotonTransferCurveDataset']
26 
27 import numpy as np
28 import matplotlib.pyplot as plt
29 import os
30 from matplotlib.backends.backend_pdf import PdfPages
31 from sqlite3 import OperationalError
32 from collections import Counter
33 
34 import lsst.afw.math as afwMath
35 import lsst.pex.config as pexConfig
36 import lsst.pipe.base as pipeBase
37 from lsst.ip.isr import IsrTask
38 from .utils import (NonexistentDatasetTaskDataIdContainer, PairedVisitListTaskRunner,
39  checkExpLengthEqual, validateIsrConfig)
40 from scipy.optimize import leastsq, least_squares
41 import numpy.polynomial.polynomial as poly
42 
43 
44 class MeasurePhotonTransferCurveTaskConfig(pexConfig.Config):
45  """Config class for photon transfer curve measurement task"""
46  isr = pexConfig.ConfigurableField(
47  target=IsrTask,
48  doc="""Task to perform instrumental signature removal.""",
49  )
50  isrMandatorySteps = pexConfig.ListField(
51  dtype=str,
52  doc="isr operations that must be performed for valid results. Raises if any of these are False.",
53  default=['doAssembleCcd']
54  )
55  isrForbiddenSteps = pexConfig.ListField(
56  dtype=str,
57  doc="isr operations that must NOT be performed for valid results. Raises if any of these are True",
58  default=['doFlat', 'doFringe', 'doAddDistortionModel', 'doBrighterFatter', 'doUseOpticsTransmission',
59  'doUseFilterTransmission', 'doUseSensorTransmission', 'doUseAtmosphereTransmission']
60  )
61  isrDesirableSteps = pexConfig.ListField(
62  dtype=str,
63  doc="isr operations that it is advisable to perform, but are not mission-critical." +
64  " WARNs are logged for any of these found to be False.",
65  default=['doBias', 'doDark', 'doCrosstalk', 'doDefect']
66  )
67  isrUndesirableSteps = pexConfig.ListField(
68  dtype=str,
69  doc="isr operations that it is *not* advisable to perform in the general case, but are not" +
70  " forbidden as some use-cases might warrant them." +
71  " WARNs are logged for any of these found to be True.",
72  default=['doLinearize']
73  )
74  ccdKey = pexConfig.Field(
75  dtype=str,
76  doc="The key by which to pull a detector from a dataId, e.g. 'ccd' or 'detector'.",
77  default='ccd',
78  )
79  makePlots = pexConfig.Field(
80  dtype=bool,
81  doc="Plot the PTC curves?",
82  default=False,
83  )
84  ptcFitType = pexConfig.ChoiceField(
85  dtype=str,
86  doc="Fit PTC to approximation in Astier+19 (Equation 16) or to a polynomial.",
87  default="POLYNOMIAL",
88  allowed={
89  "POLYNOMIAL": "n-degree polynomial (use 'polynomialFitDegree' to set 'n').",
90  "ASTIERAPPROXIMATION": "Approximation in Astier+19 (Eq. 16)."
91  }
92  )
93  polynomialFitDegree = pexConfig.Field(
94  dtype=int,
95  doc="Degree of polynomial to fit the PTC, when 'ptcFitType'=POLYNOMIAL.",
96  default=2,
97  )
98  polynomialFitDegreeNonLinearity = pexConfig.Field(
99  dtype=int,
100  doc="Degree of polynomial to fit the meanSignal vs exposureTime curve to produce" +
101  " the table for LinearizerLookupTable.",
102  default=3,
103  )
104  binSize = pexConfig.Field(
105  dtype=int,
106  doc="Bin the image by this factor in both dimensions.",
107  default=1,
108  )
109  minMeanSignal = pexConfig.Field(
110  dtype=float,
111  doc="Minimum value (inclusive) of mean signal (in ADU) above which to consider.",
112  default=0,
113  )
114  maxMeanSignal = pexConfig.Field(
115  dtype=float,
116  doc="Maximum value (inclusive) of mean signal (in ADU) below which to consider.",
117  default=9e6,
118  )
119  initialNonLinearityExclusionThresholdPositive = pexConfig.RangeField(
120  dtype=float,
121  doc="Initially exclude data points with a variance that are more than a factor of this from being"
122  " linear in the positive direction, from the PTC fit. Note that these points will also be"
123  " excluded from the non-linearity fit. This is done before the iterative outlier rejection,"
124  " to allow an accurate determination of the sigmas for said iterative fit.",
125  default=0.12,
126  min=0.0,
127  max=1.0,
128  )
129  initialNonLinearityExclusionThresholdNegative = pexConfig.RangeField(
130  dtype=float,
131  doc="Initially exclude data points with a variance that are more than a factor of this from being"
132  " linear in the negative direction, from the PTC fit. Note that these points will also be"
133  " excluded from the non-linearity fit. This is done before the iterative outlier rejection,"
134  " to allow an accurate determination of the sigmas for said iterative fit.",
135  default=0.25,
136  min=0.0,
137  max=1.0,
138  )
139  sigmaCutPtcOutliers = pexConfig.Field(
140  dtype=float,
141  doc="Sigma cut for outlier rejection in PTC.",
142  default=5.0,
143  )
144  maxIterationsPtcOutliers = pexConfig.Field(
145  dtype=int,
146  doc="Maximum number of iterations for outlier rejection in PTC.",
147  default=2,
148  )
149  doFitBootstrap = pexConfig.Field(
150  dtype=bool,
151  doc="Use bootstrap for the PTC fit parameters and errors?.",
152  default=False,
153  )
154  linResidualTimeIndex = pexConfig.Field(
155  dtype=int,
156  doc="Index position in time array for reference time in linearity residual calculation.",
157  default=2,
158  )
159  maxAduForLookupTableLinearizer = pexConfig.Field(
160  dtype=int,
161  doc="Maximum ADU value for the LookupTable linearizer.",
162  default=2**18,
163  )
164 
165 
167  """A simple class to hold the output data from the PTC task.
168 
169  The dataset is made up of a dictionary for each item, keyed by the
170  amplifiers' names, which much be supplied at construction time.
171 
172  New items cannot be added to the class to save accidentally saving to the
173  wrong property, and the class can be frozen if desired.
174 
175  inputVisitPairs records the visits used to produce the data.
176  When fitPtcAndNonLinearity() is run, a mask is built up, which is by definition
177  always the same length as inputVisitPairs, rawExpTimes, rawMeans
178  and rawVars, and is a list of bools, which are incrementally set to False
179  as points are discarded from the fits.
180 
181  PTC fit parameters for polynomials are stored in a list in ascending order
182  of polynomial term, i.e. par[0]*x^0 + par[1]*x + par[2]*x^2 etc
183  with the length of the list corresponding to the order of the polynomial
184  plus one.
185  """
186  def __init__(self, ampNames):
187  # add items to __dict__ directly because __setattr__ is overridden
188 
189  # instance variables
190  self.__dict__["ampNames"] = ampNames
191  self.__dict__["badAmps"] = []
192 
193  # raw data variables
194  self.__dict__["inputVisitPairs"] = {ampName: [] for ampName in ampNames}
195  self.__dict__["visitMask"] = {ampName: [] for ampName in ampNames}
196  self.__dict__["rawExpTimes"] = {ampName: [] for ampName in ampNames}
197  self.__dict__["rawMeans"] = {ampName: [] for ampName in ampNames}
198  self.__dict__["rawVars"] = {ampName: [] for ampName in ampNames}
199 
200  # fit information
201  self.__dict__["ptcFitType"] = {ampName: "" for ampName in ampNames}
202  self.__dict__["ptcFitPars"] = {ampName: [] for ampName in ampNames}
203  self.__dict__["ptcFitParsError"] = {ampName: [] for ampName in ampNames}
204  self.__dict__["nonLinearity"] = {ampName: [] for ampName in ampNames}
205  self.__dict__["nonLinearityError"] = {ampName: [] for ampName in ampNames}
206  self.__dict__["nonLinearityResiduals"] = {ampName: [] for ampName in ampNames}
207  self.__dict__["coefficientLinearizeSquared"] = {ampName: [] for ampName in ampNames}
208 
209  # final results
210  self.__dict__["gain"] = {ampName: -1. for ampName in ampNames}
211  self.__dict__["gainErr"] = {ampName: -1. for ampName in ampNames}
212  self.__dict__["noise"] = {ampName: -1. for ampName in ampNames}
213  self.__dict__["noiseErr"] = {ampName: -1. for ampName in ampNames}
214  self.__dict__["coefficientLinearizeSquared"] = {ampName: [] for ampName in ampNames}
215 
216  def __setattr__(self, attribute, value):
217  """Protect class attributes"""
218  if attribute not in self.__dict__:
219  raise AttributeError(f"{attribute} is not already a member of PhotonTransferCurveDataset, which"
220  " does not support setting of new attributes.")
221  else:
222  self.__dict__[attribute] = value
223 
224  def getVisitsUsed(self, ampName):
225  """Get the visits used, i.e. not discarded, for a given amp.
226 
227  If no mask has been created yet, all visits are returned.
228  """
229  if self.visitMask[ampName] == []:
230  return self.inputVisitPairs[ampName]
231 
232  # if the mask exists it had better be the same length as the visitPairs
233  assert len(self.visitMask[ampName]) == len(self.inputVisitPairs[ampName])
234 
235  pairs = self.inputVisitPairs[ampName]
236  mask = self.visitMask[ampName]
237  # cast to bool required because numpy
238  return [(v1, v2) for ((v1, v2), m) in zip(pairs, mask) if bool(m) is True]
239 
240  def getGoodAmps(self):
241  return [amp for amp in self.ampNames if amp not in self.badAmps]
242 
243 
244 class MeasurePhotonTransferCurveTask(pipeBase.CmdLineTask):
245  """A class to calculate, fit, and plot a PTC from a set of flat pairs.
246 
247  The Photon Transfer Curve (var(signal) vs mean(signal)) is a standard tool
248  used in astronomical detectors characterization (e.g., Janesick 2001,
249  Janesick 2007). This task calculates the PTC from a series of pairs of
250  flat-field images; each pair taken at identical exposure times. The
251  difference image of each pair is formed to eliminate fixed pattern noise,
252  and then the variance of the difference image and the mean of the average image
253  are used to produce the PTC. An n-degree polynomial or the approximation in Equation
254  16 of Astier+19 ("The Shape of the Photon Transfer Curve of CCD sensors",
255  arXiv:1905.08677) can be fitted to the PTC curve. These models include
256  parameters such as the gain (e/ADU) and readout noise.
257 
258  Parameters
259  ----------
260 
261  *args: `list`
262  Positional arguments passed to the Task constructor. None used at this
263  time.
264  **kwargs: `dict`
265  Keyword arguments passed on to the Task constructor. None used at this
266  time.
267 
268  """
269 
270  RunnerClass = PairedVisitListTaskRunner
271  ConfigClass = MeasurePhotonTransferCurveTaskConfig
272  _DefaultName = "measurePhotonTransferCurve"
273 
274  def __init__(self, *args, **kwargs):
275  pipeBase.CmdLineTask.__init__(self, *args, **kwargs)
276  self.makeSubtask("isr")
277  plt.interactive(False) # stop windows popping up when plotting. When headless, use 'agg' backend too
278  validateIsrConfig(self.isr, self.config.isrMandatorySteps,
279  self.config.isrForbiddenSteps, self.config.isrDesirableSteps, checkTrim=False)
280  self.config.validate()
281  self.config.freeze()
282 
283  @classmethod
284  def _makeArgumentParser(cls):
285  """Augment argument parser for the MeasurePhotonTransferCurveTask."""
286  parser = pipeBase.ArgumentParser(name=cls._DefaultName)
287  parser.add_argument("--visit-pairs", dest="visitPairs", nargs="*",
288  help="Visit pairs to use. Each pair must be of the form INT,INT e.g. 123,456")
289  parser.add_id_argument("--id", datasetType="photonTransferCurveDataset",
290  ContainerClass=NonexistentDatasetTaskDataIdContainer,
291  help="The ccds to use, e.g. --id ccd=0..100")
292  return parser
293 
294  @pipeBase.timeMethod
295  def runDataRef(self, dataRef, visitPairs):
296  """Run the Photon Transfer Curve (PTC) measurement task.
297 
298  For a dataRef (which is each detector here),
299  and given a list of visit pairs at different exposure times,
300  measure the PTC.
301 
302  Parameters
303  ----------
304  dataRef : list of lsst.daf.persistence.ButlerDataRef
305  dataRef for the detector for the visits to be fit.
306  visitPairs : `iterable` of `tuple` of `int`
307  Pairs of visit numbers to be processed together
308  """
309 
310  # setup necessary objects
311  detNum = dataRef.dataId[self.config.ccdKey]
312  detector = dataRef.get('camera')[dataRef.dataId[self.config.ccdKey]]
313  # expand some missing fields that we need for lsstCam. This is a work-around
314  # for Gen2 problems that I (RHL) don't feel like solving. The calibs pipelines
315  # (which inherit from CalibTask) use addMissingKeys() to do basically the same thing
316  #
317  # Basically, the butler's trying to look up the fields in `raw_visit` which won't work
318  for name in dataRef.getButler().getKeys('bias'):
319  if name not in dataRef.dataId:
320  try:
321  dataRef.dataId[name] = \
322  dataRef.getButler().queryMetadata('raw', [name], detector=detNum)[0]
323  except OperationalError:
324  pass
325 
326  amps = detector.getAmplifiers()
327  ampNames = [amp.getName() for amp in amps]
328  dataset = PhotonTransferCurveDataset(ampNames)
329 
330  self.log.info('Measuring PTC using %s visits for detector %s' % (visitPairs, detNum))
331 
332  for (v1, v2) in visitPairs:
333  # Perform ISR on each exposure
334  dataRef.dataId['expId'] = v1
335  exp1 = self.isr.runDataRef(dataRef).exposure
336  dataRef.dataId['expId'] = v2
337  exp2 = self.isr.runDataRef(dataRef).exposure
338  del dataRef.dataId['expId']
339 
340  checkExpLengthEqual(exp1, exp2, v1, v2, raiseWithMessage=True)
341  expTime = exp1.getInfo().getVisitInfo().getExposureTime()
342 
343  for amp in detector:
344  mu, varDiff = self.measureMeanVarPair(exp1, exp2, region=amp.getBBox())
345  ampName = amp.getName()
346 
347  dataset.rawExpTimes[ampName].append(expTime)
348  dataset.rawMeans[ampName].append(mu)
349  dataset.rawVars[ampName].append(varDiff)
350  dataset.inputVisitPairs[ampName].append((v1, v2))
351 
352  numberAmps = len(detector.getAmplifiers())
353  numberAduValues = self.config.maxAduForLookupTableLinearizer
354  lookupTableArray = np.zeros((numberAmps, numberAduValues), dtype=np.float32)
355 
356  # Fit PTC and (non)linearity of signal vs time curve.
357  # Fill up PhotonTransferCurveDataset object.
358  # Fill up array for LUT linearizer.
359  dataset = self.fitPtcAndNonLinearity(dataset, lookupTableArray, ptcFitType=self.config.ptcFitType)
360 
361  if self.config.makePlots:
362  self.plot(dataRef, dataset, ptcFitType=self.config.ptcFitType)
363 
364  # Save data, PTC fit, and NL fit dictionaries
365  self.log.info(f"Writing PTC and NL data to {dataRef.getUri(write=True)}")
366  dataRef.put(dataset, datasetType="photonTransferCurveDataset")
367 
368  self.log.info('Finished measuring PTC for in detector %s' % detNum)
369 
370  return pipeBase.Struct(exitStatus=0)
371 
372  def measureMeanVarPair(self, exposure1, exposure2, region=None):
373  """Calculate the mean signal of two exposures and the variance of their difference.
374 
375  Parameters
376  ----------
377  exposure1 : `lsst.afw.image.exposure.exposure.ExposureF`
378  First exposure of flat field pair.
379 
380  exposure2 : `lsst.afw.image.exposure.exposure.ExposureF`
381  Second exposure of flat field pair.
382 
383  region : `lsst.geom.Box2I`
384  Region of each exposure where to perform the calculations (e.g, an amplifier).
385 
386  Return
387  ------
388 
389  mu : `np.float`
390  0.5*(mu1 + mu2), where mu1, and mu2 are the clipped means of the regions in
391  both exposures.
392 
393  varDiff : `np.float`
394  Half of the clipped variance of the difference of the regions inthe two input
395  exposures.
396  """
397 
398  if region is not None:
399  im1Area = exposure1.maskedImage[region]
400  im2Area = exposure2.maskedImage[region]
401  else:
402  im1Area = exposure1.maskedImage
403  im2Area = exposure2.maskedImage
404 
405  im1Area = afwMath.binImage(im1Area, self.config.binSize)
406  im2Area = afwMath.binImage(im2Area, self.config.binSize)
407 
408  # Clipped mean of images; then average of mean.
409  mu1 = afwMath.makeStatistics(im1Area, afwMath.MEANCLIP).getValue()
410  mu2 = afwMath.makeStatistics(im2Area, afwMath.MEANCLIP).getValue()
411  mu = 0.5*(mu1 + mu2)
412 
413  # Take difference of pairs
414  # symmetric formula: diff = (mu2*im1-mu1*im2)/(0.5*(mu1+mu2))
415  temp = im2Area.clone()
416  temp *= mu1
417  diffIm = im1Area.clone()
418  diffIm *= mu2
419  diffIm -= temp
420  diffIm /= mu
421 
422  varDiff = 0.5*(afwMath.makeStatistics(diffIm, afwMath.VARIANCECLIP).getValue())
423 
424  return mu, varDiff
425 
426  def _fitLeastSq(self, initialParams, dataX, dataY, function):
427  """Do a fit and estimate the parameter errors using using scipy.optimize.leastq.
428 
429  optimize.leastsq returns the fractional covariance matrix. To estimate the
430  standard deviation of the fit parameters, multiply the entries of this matrix
431  by the reduced chi squared and take the square root of the diagonal elements.
432 
433  Parameters
434  ----------
435  initialParams : list of np.float
436  initial values for fit parameters. For ptcFitType=POLYNOMIAL, its length
437  determines the degree of the polynomial.
438 
439  dataX : np.array of np.float
440  Data in the abscissa axis.
441 
442  dataY : np.array of np.float
443  Data in the ordinate axis.
444 
445  function : callable object (function)
446  Function to fit the data with.
447 
448  Return
449  ------
450  pFitSingleLeastSquares : list of np.float
451  List with fitted parameters.
452 
453  pErrSingleLeastSquares : list of np.float
454  List with errors for fitted parameters.
455  """
456 
457  def errFunc(p, x, y):
458  return function(p, x) - y
459 
460  pFit, pCov, infoDict, errMessage, success = leastsq(errFunc, initialParams,
461  args=(dataX, dataY), full_output=1, epsfcn=0.0001)
462 
463  if (len(dataY) > len(initialParams)) and pCov is not None:
464  reducedChiSq = (errFunc(pFit, dataX, dataY)**2).sum()/(len(dataY)-len(initialParams))
465  pCov *= reducedChiSq
466  else:
467  pCov[:, :] = np.inf
468 
469  errorVec = []
470  for i in range(len(pFit)):
471  errorVec.append(np.fabs(pCov[i][i])**0.5)
472 
473  pFitSingleLeastSquares = pFit
474  pErrSingleLeastSquares = np.array(errorVec)
475 
476  return pFitSingleLeastSquares, pErrSingleLeastSquares
477 
478  def _fitBootstrap(self, initialParams, dataX, dataY, function, confidenceSigma=1.):
479  """Do a fit using least squares and bootstrap to estimate parameter errors.
480 
481  The bootstrap error bars are calculated by fitting 100 random data sets.
482 
483  Parameters
484  ----------
485  initialParams : list of np.float
486  initial values for fit parameters. For ptcFitType=POLYNOMIAL, its length
487  determines the degree of the polynomial.
488 
489  dataX : np.array of np.float
490  Data in the abscissa axis.
491 
492  dataY : np.array of np.float
493  Data in the ordinate axis.
494 
495  function : callable object (function)
496  Function to fit the data with.
497 
498  confidenceSigma : np.float
499  Number of sigmas that determine confidence interval for the bootstrap errors.
500 
501  Return
502  ------
503  pFitBootstrap : list of np.float
504  List with fitted parameters.
505 
506  pErrBootstrap : list of np.float
507  List with errors for fitted parameters.
508  """
509 
510  def errFunc(p, x, y):
511  return function(p, x) - y
512 
513  # Fit first time
514  pFit, _ = leastsq(errFunc, initialParams, args=(dataX, dataY), full_output=0)
515 
516  # Get the stdev of the residuals
517  residuals = errFunc(pFit, dataX, dataY)
518  sigmaErrTotal = np.std(residuals)
519 
520  # 100 random data sets are generated and fitted
521  pars = []
522  for i in range(100):
523  randomDelta = np.random.normal(0., sigmaErrTotal, len(dataY))
524  randomDataY = dataY + randomDelta
525  randomFit, _ = leastsq(errFunc, initialParams,
526  args=(dataX, randomDataY), full_output=0)
527  pars.append(randomFit)
528  pars = np.array(pars)
529  meanPfit = np.mean(pars, 0)
530 
531  # confidence interval for parameter estimates
532  nSigma = confidenceSigma
533  errPfit = nSigma*np.std(pars, 0)
534  pFitBootstrap = meanPfit
535  pErrBootstrap = errPfit
536  return pFitBootstrap, pErrBootstrap
537 
538  def funcPolynomial(self, pars, x):
539  """Polynomial function definition"""
540  return poly.polyval(x, [*pars])
541 
542  def funcAstier(self, pars, x):
543  """Single brighter-fatter parameter model for PTC; Equation 16 of Astier+19"""
544  a00, gain, noise = pars
545  return 0.5/(a00*gain*gain)*(np.exp(2*a00*x*gain)-1) + noise/(gain*gain)
546 
547  @staticmethod
548  def _initialParsForPolynomial(order):
549  assert(order >= 2)
550  pars = np.zeros(order, dtype=np.float)
551  pars[0] = 10
552  pars[1] = 1
553  pars[2:] = 0.0001
554  return pars
555 
556  @staticmethod
557  def _boundsForPolynomial(initialPars):
558  lowers = [np.NINF for p in initialPars]
559  uppers = [np.inf for p in initialPars]
560  lowers[1] = 0 # no negative gains
561  return (lowers, uppers)
562 
563  @staticmethod
564  def _boundsForAstier(initialPars):
565  lowers = [np.NINF for p in initialPars]
566  uppers = [np.inf for p in initialPars]
567  return (lowers, uppers)
568 
569  @staticmethod
570  def _getInitialGoodPoints(means, variances, maxDeviationPositive, maxDeviationNegative):
571  """Return a boolean array to mask bad points.
572 
573  A linear function has a constant ratio, so find the median
574  value of the ratios, and exclude the points that deviate
575  from that by more than a factor of maxDeviationPositive/negative.
576  Asymmetric deviations are supported as we expect the PTC to turn
577  down as the flux increases, but sometimes it anomalously turns
578  upwards just before turning over, which ruins the fits, so it
579  is wise to be stricter about restricting positive outliers than
580  negative ones.
581 
582  Too high and points that are so bad that fit will fail will be included
583  Too low and the non-linear points will be excluded, biasing the NL fit."""
584  ratios = [b/a for (a, b) in zip(means, variances)]
585  medianRatio = np.median(ratios)
586  ratioDeviations = [(r/medianRatio)-1 for r in ratios]
587 
588  # so that it doesn't matter if the deviation is expressed as positive or negative
589  maxDeviationPositive = abs(maxDeviationPositive)
590  maxDeviationNegative = -1. * abs(maxDeviationNegative)
591 
592  goodPoints = np.array([True if (r < maxDeviationPositive and r > maxDeviationNegative)
593  else False for r in ratioDeviations])
594  return goodPoints
595 
596  def _makeZeroSafe(self, array, warn=True, substituteValue=1e-9):
597  """"""
598  nBad = Counter(array)[0]
599  if nBad == 0:
600  return array
601 
602  if warn:
603  msg = f"Found {nBad} zeros in array at elements {[x for x in np.where(array==0)[0]]}"
604  self.log.warn(msg)
605 
606  array[array == 0] = substituteValue
607  return array
608 
609  def calculateLinearityResidualAndLinearizers(self, exposureTimeVector, meanSignalVector):
610  """Calculate linearity residual and fit an n-order polynomial to the mean vs time curve
611  to produce corrections (deviation from linear part of polynomial) for a particular amplifier
612  to populate LinearizeLookupTable. Use quadratic and linear parts of this polynomial to approximate
613  c0 for LinearizeSquared."
614 
615  Parameters
616  ---------
617 
618  exposureTimeVector: `list` of `np.float`
619  List of exposure times for each flat pair
620 
621  meanSignalVector: `list` of `np.float`
622  List of mean signal from diference image of flat pairs
623 
624  Returns
625  -------
626  c0: `np.float`
627  Coefficient for LinearizeSquared, where corrImage = uncorrImage + c0*uncorrImage^2.
628  c0 ~ -k2/(k1^2), where k1 and k2 are fit from
629  meanSignalVector = k0 + k1*exposureTimeVector + k2*exposureTimeVector^2 +...
630  + kn*exposureTimeVector^n, with n = "polynomialFitDegreeNonLinearity".
631 
632  linearizerTableRow: list of `np.float`
633  One dimensional array with deviation from linear part of n-order polynomial fit
634  to mean vs time curve. This array will be one row (for the particular amplifier at hand)
635  of the table array for LinearizeLookupTable.
636 
637  linResidual: list of `np.float`
638  Linearity residual from the mean vs time curve, defined as
639  100*(1 - meanSignalReference/expTimeReference/(meanSignal/expTime).
640 
641  parsFit: list of `np.float`
642  Parameters from n-order polynomial fit to mean vs time curve.
643 
644  parsFitErr: list of `np.float`
645  Parameters from n-order polynomial fit to mean vs time curve.
646 
647  """
648 
649  # Lookup table linearizer
650  parsIniNonLinearity = self._initialParsForPolynomial(self.config.polynomialFitDegreeNonLinearity + 1)
651  if self.config.doFitBootstrap:
652  parsFit, parsFitErr = self._fitBootstrap(parsIniNonLinearity, exposureTimeVector,
653  meanSignalVector, self.funcPolynomial)
654  else:
655  parsFit, parsFitErr = self._fitLeastSq(parsIniNonLinearity, exposureTimeVector, meanSignalVector,
656  self.funcPolynomial)
657 
658  # Use linear part to get time at wich signal is maxAduForLookupTableLinearizer ADU
659  tMax = (self.config.maxAduForLookupTableLinearizer - parsFit[0])/parsFit[1]
660  timeRange = np.linspace(0, tMax, self.config.maxAduForLookupTableLinearizer)
661  signalIdeal = parsFit[0] + parsFit[1]*timeRange
662  signalUncorrected = self.funcPolynomial(parsFit, timeRange)
663  linearizerTableRow = signalIdeal - signalUncorrected # LinearizerLookupTable has corrections
664 
665  # Use quadratic and linear part of fit to produce c0 for LinearizeSquared
666  # Check that magnitude of higher order (>= 3) coefficents of the polyFit are small,
667  # i.e., less than threshold = 1e-10 (typical quadratic and cubic coefficents are ~1e-6
668  # and ~1e-12).
669  k1, k2 = parsFit[1], parsFit[2]
670  c0 = -k2/(k1**2) # c0 coefficient for LinearizeSquared
671  for coefficient in parsFit[3:]:
672  if np.fabs(coefficient) > 1e-10:
673  msg = f"Coefficient {coefficient} in polynomial fit larger than threshold 1e-10."
674  self.log.warn(msg)
675 
676  # Linearity residual
677  linResidualTimeIndex = self.config.linResidualTimeIndex
678  if exposureTimeVector[linResidualTimeIndex] == 0.0:
679  raise RuntimeError("Reference time for linearity residual can't be 0.0")
680  linResidual = 100*(1 - ((meanSignalVector[linResidualTimeIndex] /
681  exposureTimeVector[linResidualTimeIndex]) /
682  (meanSignalVector/exposureTimeVector)))
683 
684  return c0, linearizerTableRow, linResidual, parsFit, parsFitErr
685 
686  def fitPtcAndNonLinearity(self, dataset, ptcFitType, tableArray=None):
687  """Fit the photon transfer curve and calculate linearity and residuals.
688 
689  Fit the photon transfer curve with either a polynomial of the order
690  specified in the task config, or using the Astier approximation.
691 
692  Sigma clipping is performed iteratively for the fit, as well as an
693  initial clipping of data points that are more than
694  config.initialNonLinearityExclusionThreshold away from lying on a
695  straight line. This other step is necessary because the photon transfer
696  curve turns over catastrophically at very high flux (because saturation
697  drops the variance to ~0) and these far outliers cause the initial fit
698  to fail, meaning the sigma cannot be calculated to perform the
699  sigma-clipping.
700 
701  Parameters
702  ----------
703  dataset : `lsst.cp.pipe.ptc.PhotonTransferCurveDataset`
704  The dataset containing the means, variances and exposure times
705  ptcFitType : `str`
706  Fit a 'POLYNOMIAL' (degree: 'polynomialFitDegree') or
707  'ASTIERAPPROXIMATION' to the PTC
708  tableArray : `np.array`
709  Optional. Look-up table array with size rows=nAmps and columns=ADU values.
710  It will be modified in-place if supplied.
711 
712  Returns
713  -------
714  dataset: `lsst.cp.pipe.ptc.PhotonTransferCurveDataset`
715  This is the same dataset as the input paramter, however, it has been modified
716  to include information such as the fit vectors and the fit parameters. See
717  the class `PhotonTransferCurveDatase`.
718  """
719 
720  def errFunc(p, x, y):
721  return ptcFunc(p, x) - y
722 
723  sigmaCutPtcOutliers = self.config.sigmaCutPtcOutliers
724  maxIterationsPtcOutliers = self.config.maxIterationsPtcOutliers
725 
726  for i, ampName in enumerate(dataset.ampNames):
727  timeVecOriginal = np.array(dataset.rawExpTimes[ampName])
728  meanVecOriginal = np.array(dataset.rawMeans[ampName])
729  varVecOriginal = np.array(dataset.rawVars[ampName])
730  varVecOriginal = self._makeZeroSafe(varVecOriginal)
731 
732  mask = ((meanVecOriginal >= self.config.minMeanSignal) &
733  (meanVecOriginal <= self.config.maxMeanSignal))
734 
735  goodPoints = self._getInitialGoodPoints(meanVecOriginal, varVecOriginal,
736  self.config.initialNonLinearityExclusionThresholdPositive,
737  self.config.initialNonLinearityExclusionThresholdNegative)
738  mask = mask & goodPoints
739 
740  if ptcFitType == 'ASTIERAPPROXIMATION':
741  ptcFunc = self.funcAstier
742  parsIniPtc = [-1e-9, 1.0, 10.] # a00, gain, noise
743  bounds = self._boundsForAstier(parsIniPtc)
744  if ptcFitType == 'POLYNOMIAL':
745  ptcFunc = self.funcPolynomial
746  parsIniPtc = self._initialParsForPolynomial(self.config.polynomialFitDegree + 1)
747  bounds = self._boundsForPolynomial(parsIniPtc)
748 
749  # Before bootstrap fit, do an iterative fit to get rid of outliers
750  count = 1
751  while count <= maxIterationsPtcOutliers:
752  # Note that application of the mask actually shrinks the array
753  # to size rather than setting elements to zero (as we want) so
754  # always update mask itself and re-apply to the original data
755  meanTempVec = meanVecOriginal[mask]
756  varTempVec = varVecOriginal[mask]
757  res = least_squares(errFunc, parsIniPtc, bounds=bounds, args=(meanTempVec, varTempVec))
758  pars = res.x
759 
760  # change this to the original from the temp because the masks are ANDed
761  # meaning once a point is masked it's always masked, and the masks must
762  # always be the same length for broadcasting
763  sigResids = (varVecOriginal - ptcFunc(pars, meanVecOriginal))/np.sqrt(varVecOriginal)
764  newMask = np.array([True if np.abs(r) < sigmaCutPtcOutliers else False for r in sigResids])
765  mask = mask & newMask
766 
767  nDroppedTotal = Counter(mask)[False]
768  self.log.debug(f"Iteration {count}: discarded {nDroppedTotal} points in total for {ampName}")
769  count += 1
770  # objects should never shrink
771  assert (len(mask) == len(timeVecOriginal) == len(meanVecOriginal) == len(varVecOriginal))
772 
773  dataset.visitMask[ampName] = mask # store the final mask
774 
775  parsIniPtc = pars
776  timeVecFinal = timeVecOriginal[mask]
777  meanVecFinal = meanVecOriginal[mask]
778  varVecFinal = varVecOriginal[mask]
779 
780  if Counter(mask)[False] > 0:
781  self.log.info((f"Number of points discarded in PTC of amplifier {ampName}:" +
782  f" {Counter(mask)[False]} out of {len(meanVecOriginal)}"))
783 
784  if (len(meanVecFinal) < len(parsIniPtc)):
785  msg = (f"\nSERIOUS: Not enough data points ({len(meanVecFinal)}) compared to the number of"
786  f"parameters of the PTC model({len(parsIniPtc)}). Setting {ampName} to BAD.")
787  self.log.warn(msg)
788  dataset.badAmps.append(ampName)
789  dataset.gain[ampName] = np.nan
790  dataset.gainErr[ampName] = np.nan
791  dataset.noise[ampName] = np.nan
792  dataset.noiseErr[ampName] = np.nan
793  dataset.nonLinearity[ampName] = np.nan
794  dataset.nonLinearityError[ampName] = np.nan
795  dataset.nonLinearityResiduals[ampName] = np.nan
796  continue
797 
798  # Fit the PTC
799  if self.config.doFitBootstrap:
800  parsFit, parsFitErr = self._fitBootstrap(parsIniPtc, meanVecFinal, varVecFinal, ptcFunc)
801  else:
802  parsFit, parsFitErr = self._fitLeastSq(parsIniPtc, meanVecFinal, varVecFinal, ptcFunc)
803 
804  dataset.ptcFitPars[ampName] = parsFit
805  dataset.ptcFitParsError[ampName] = parsFitErr
806 
807  if ptcFitType == 'ASTIERAPPROXIMATION':
808  ptcGain = parsFit[1]
809  ptcGainErr = parsFitErr[1]
810  ptcNoise = np.sqrt(np.fabs(parsFit[2]))
811  ptcNoiseErr = 0.5*(parsFitErr[2]/np.fabs(parsFit[2]))*np.sqrt(np.fabs(parsFit[2]))
812  if ptcFitType == 'POLYNOMIAL':
813  ptcGain = 1./parsFit[1]
814  ptcGainErr = np.fabs(1./parsFit[1])*(parsFitErr[1]/parsFit[1])
815  ptcNoise = np.sqrt(np.fabs(parsFit[0]))*ptcGain
816  ptcNoiseErr = (0.5*(parsFitErr[0]/np.fabs(parsFit[0]))*(np.sqrt(np.fabs(parsFit[0]))))*ptcGain
817 
818  dataset.gain[ampName] = ptcGain
819  dataset.gainErr[ampName] = ptcGainErr
820  dataset.noise[ampName] = ptcNoise
821  dataset.noiseErr[ampName] = ptcNoiseErr
822  dataset.ptcFitType[ampName] = ptcFitType
823 
824  # Non-linearity residuals (NL of mean vs time curve): percentage, and fit to a quadratic function
825  # In this case, len(parsIniNonLinearity) = 3 indicates that we want a quadratic fit
826 
827  (c0, linearizerTableRow, linResidualNonLinearity, parsFitNonLinearity,
828  parsFitErrNonLinearity) = self.calculateLinearityResidualAndLinearizers(timeVecFinal,
829  meanVecFinal)
830  # LinearizerLookupTable
831  if tableArray is not None:
832  tableArray[i, :] = linearizerTableRow
833 
834  dataset.nonLinearity[ampName] = parsFitNonLinearity
835  dataset.nonLinearityError[ampName] = parsFitErrNonLinearity
836  dataset.nonLinearityResiduals[ampName] = linResidualNonLinearity
837  dataset.coefficientLinearizeSquared[ampName] = c0
838 
839  return dataset
840 
841  def plot(self, dataRef, dataset, ptcFitType):
842  dirname = dataRef.getUri(datasetType='cpPipePlotRoot', write=True)
843  if not os.path.exists(dirname):
844  os.makedirs(dirname)
845 
846  detNum = dataRef.dataId[self.config.ccdKey]
847  filename = f"PTC_det{detNum}.pdf"
848  filenameFull = os.path.join(dirname, filename)
849  with PdfPages(filenameFull) as pdfPages:
850  self._plotPtc(dataset, ptcFitType, pdfPages)
851 
852  def _plotPtc(self, dataset, ptcFitType, pdfPages):
853  """Plot PTC, linearity, and linearity residual per amplifier"""
854 
855  if ptcFitType == 'ASTIERAPPROXIMATION':
856  ptcFunc = self.funcAstier
857  stringTitle = r"Var = $\frac{1}{2g^2a_{00}}(\exp (2a_{00} \mu g) - 1) + \frac{n_{00}}{g^2}$"
858 
859  if ptcFitType == 'POLYNOMIAL':
860  ptcFunc = self.funcPolynomial
861  stringTitle = f"Polynomial (degree: {self.config.polynomialFitDegree})"
862 
863  legendFontSize = 7.5
864  labelFontSize = 8
865  titleFontSize = 10
866  supTitleFontSize = 18
867  markerSize = 25
868 
869  # General determination of the size of the plot grid
870  nAmps = len(dataset.ampNames)
871  if nAmps == 2:
872  nRows, nCols = 2, 1
873  nRows = np.sqrt(nAmps)
874  mantissa, _ = np.modf(nRows)
875  if mantissa > 0:
876  nRows = int(nRows) + 1
877  nCols = nRows
878  else:
879  nRows = int(nRows)
880  nCols = nRows
881 
882  f, ax = plt.subplots(nrows=nRows, ncols=nCols, sharex='col', sharey='row', figsize=(13, 10))
883  f2, ax2 = plt.subplots(nrows=nRows, ncols=nCols, sharex='col', sharey='row', figsize=(13, 10))
884 
885  for i, (amp, a, a2) in enumerate(zip(dataset.ampNames, ax.flatten(), ax2.flatten())):
886  meanVecOriginal = np.array(dataset.rawMeans[amp])
887  varVecOriginal = np.array(dataset.rawVars[amp])
888  mask = dataset.visitMask[amp]
889  meanVecFinal = meanVecOriginal[mask]
890  varVecFinal = varVecOriginal[mask]
891  meanVecOutliers = meanVecOriginal[np.invert(mask)]
892  varVecOutliers = varVecOriginal[np.invert(mask)]
893  pars, parsErr = dataset.ptcFitPars[amp], dataset.ptcFitParsError[amp]
894 
895  if ptcFitType == 'ASTIERAPPROXIMATION':
896  ptcA00, ptcA00error = pars[0], parsErr[0]
897  ptcGain, ptcGainError = pars[1], parsErr[1]
898  ptcNoise = np.sqrt(np.fabs(pars[2]))
899  ptcNoiseError = 0.5*(parsErr[2]/np.fabs(pars[2]))*np.sqrt(np.fabs(pars[2]))
900  stringLegend = (f"a00: {ptcA00:.2e}+/-{ptcA00error:.2e}"
901  f"\n Gain: {ptcGain:.4}+/-{ptcGainError:.2e}"
902  f"\n Noise: {ptcNoise:.4}+/-{ptcNoiseError:.2e}")
903 
904  if ptcFitType == 'POLYNOMIAL':
905  ptcGain, ptcGainError = 1./pars[1], np.fabs(1./pars[1])*(parsErr[1]/pars[1])
906  ptcNoise = np.sqrt(np.fabs(pars[0]))*ptcGain
907  ptcNoiseError = (0.5*(parsErr[0]/np.fabs(pars[0]))*(np.sqrt(np.fabs(pars[0]))))*ptcGain
908  stringLegend = (f"Noise: {ptcNoise:.4}+/-{ptcNoiseError:.2e} \n"
909  f"Gain: {ptcGain:.4}+/-{ptcGainError:.2e}")
910 
911  minMeanVecFinal = np.min(meanVecFinal)
912  maxMeanVecFinal = np.max(meanVecFinal)
913  meanVecFit = np.linspace(minMeanVecFinal, maxMeanVecFinal, 100*len(meanVecFinal))
914  minMeanVecOriginal = np.min(meanVecOriginal)
915  maxMeanVecOriginal = np.max(meanVecOriginal)
916  deltaXlim = maxMeanVecOriginal - minMeanVecOriginal
917 
918  a.plot(meanVecFit, ptcFunc(pars, meanVecFit), color='red')
919  a.plot(meanVecFinal, pars[0] + pars[1]*meanVecFinal, color='green', linestyle='--')
920  a.scatter(meanVecFinal, varVecFinal, c='blue', marker='o', s=markerSize)
921  a.scatter(meanVecOutliers, varVecOutliers, c='magenta', marker='s', s=markerSize)
922  a.set_xlabel(r'Mean signal ($\mu$, ADU)', fontsize=labelFontSize)
923  a.set_xticks(meanVecOriginal)
924  a.set_ylabel(r'Variance (ADU$^2$)', fontsize=labelFontSize)
925  a.tick_params(labelsize=11)
926  a.text(0.03, 0.8, stringLegend, transform=a.transAxes, fontsize=legendFontSize)
927  a.set_xscale('linear', fontsize=labelFontSize)
928  a.set_yscale('linear', fontsize=labelFontSize)
929  a.set_title(amp, fontsize=titleFontSize)
930  a.set_xlim([minMeanVecOriginal - 0.2*deltaXlim, maxMeanVecOriginal + 0.2*deltaXlim])
931 
932  # Same, but in log-scale
933  a2.plot(meanVecFit, ptcFunc(pars, meanVecFit), color='red')
934  a2.scatter(meanVecFinal, varVecFinal, c='blue', marker='o', s=markerSize)
935  a2.scatter(meanVecOutliers, varVecOutliers, c='magenta', marker='s', s=markerSize)
936  a2.set_xlabel(r'Mean Signal ($\mu$, ADU)', fontsize=labelFontSize)
937  a2.set_ylabel(r'Variance (ADU$^2$)', fontsize=labelFontSize)
938  a2.tick_params(labelsize=11)
939  a2.text(0.03, 0.8, stringLegend, transform=a2.transAxes, fontsize=legendFontSize)
940  a2.set_xscale('log')
941  a2.set_yscale('log')
942  a2.set_title(amp, fontsize=titleFontSize)
943  a2.set_xlim([minMeanVecOriginal, maxMeanVecOriginal])
944 
945  f.suptitle(f"PTC \n Fit: " + stringTitle, fontsize=20)
946  pdfPages.savefig(f)
947  f2.suptitle(f"PTC (log-log)", fontsize=20)
948  pdfPages.savefig(f2)
949 
950  # Plot mean vs time
951  f, ax = plt.subplots(nrows=4, ncols=4, sharex='col', sharey='row', figsize=(13, 10))
952  for i, (amp, a) in enumerate(zip(dataset.ampNames, ax.flatten())):
953  meanVecFinal = np.array(dataset.rawMeans[amp])[dataset.visitMask[amp]]
954  timeVecFinal = np.array(dataset.rawExpTimes[amp])[dataset.visitMask[amp]]
955 
956  pars, parsErr = dataset.nonLinearity[amp], dataset.nonLinearityError[amp]
957  c0, c0Error = pars[0], parsErr[0]
958  c1, c1Error = pars[1], parsErr[1]
959  c2, c2Error = pars[2], parsErr[2]
960  stringLegend = f"c0: {c0:.4}+/-{c0Error:.2e}\n c1: {c1:.4}+/-{c1Error:.2e}" \
961  + f"\n c2(NL): {c2:.2e}+/-{c2Error:.2e}"
962  a.scatter(timeVecFinal, meanVecFinal)
963  a.plot(timeVecFinal, self.funcPolynomial(pars, timeVecFinal), color='red')
964  a.set_xlabel('Time (sec)', fontsize=labelFontSize)
965  a.set_xticks(timeVecFinal)
966  a.set_ylabel(r'Mean signal ($\mu$, ADU)', fontsize=labelFontSize)
967  a.tick_params(labelsize=labelFontSize)
968  a.text(0.03, 0.75, stringLegend, transform=a.transAxes, fontsize=legendFontSize)
969  a.set_xscale('linear', fontsize=labelFontSize)
970  a.set_yscale('linear', fontsize=labelFontSize)
971  a.set_title(amp, fontsize=titleFontSize)
972 
973  f.suptitle("Linearity \n Fit: " + r"$\mu = c_0 + c_1 t + c_2 t^2$", fontsize=supTitleFontSize)
974  pdfPages.savefig()
975 
976  # Plot linearity residual
977  f, ax = plt.subplots(nrows=4, ncols=4, sharex='col', sharey='row', figsize=(13, 10))
978  for i, (amp, a) in enumerate(zip(dataset.ampNames, ax.flatten())):
979  meanVecFinal = np.array(dataset.rawMeans[amp])[dataset.visitMask[amp]]
980  linRes = np.array(dataset.nonLinearityResiduals[amp])
981 
982  a.scatter(meanVecFinal, linRes)
983  a.axhline(y=0, color='k')
984  a.axvline(x=timeVecFinal[self.config.linResidualTimeIndex], color='g', linestyle='--')
985  a.set_xlabel(r'Mean signal ($\mu$, ADU)', fontsize=labelFontSize)
986  a.set_xticks(meanVecFinal)
987  a.set_ylabel('LR (%)', fontsize=labelFontSize)
988  a.tick_params(labelsize=labelFontSize)
989  a.set_xscale('linear', fontsize=labelFontSize)
990  a.set_yscale('linear', fontsize=labelFontSize)
991  a.set_title(amp, fontsize=titleFontSize)
992 
993  f.suptitle(r"Linearity Residual: $100(1 - \mu_{\rm{ref}}/t_{\rm{ref}})/(\mu / t))$" + "\n" +
994  r"$t_{\rm{ref}}$: " + f"{timeVecFinal[2]} s", fontsize=supTitleFontSize)
995  pdfPages.savefig()
996 
997  return
def _plotPtc(self, dataset, ptcFitType, pdfPages)
Definition: ptc.py:852
def runDataRef(self, dataRef, visitPairs)
Definition: ptc.py:295
def calculateLinearityResidualAndLinearizers(self, exposureTimeVector, meanSignalVector)
Definition: ptc.py:609
def fitPtcAndNonLinearity(self, dataset, ptcFitType, tableArray=None)
Definition: ptc.py:686
def __setattr__(self, attribute, value)
Definition: ptc.py:216
def checkExpLengthEqual(exp1, exp2, v1=None, v2=None, raiseWithMessage=False)
Definition: utils.py:162
def validateIsrConfig(isrTask, mandatory=None, forbidden=None, desirable=None, undesirable=None, checkTrim=True, logName=None)
Definition: utils.py:200
def _getInitialGoodPoints(means, variances, maxDeviationPositive, maxDeviationNegative)
Definition: ptc.py:570
def _fitBootstrap(self, initialParams, dataX, dataY, function, confidenceSigma=1.)
Definition: ptc.py:478
def __init__(self, args, kwargs)
Definition: ptc.py:274
def measureMeanVarPair(self, exposure1, exposure2, region=None)
Definition: ptc.py:372
def _makeZeroSafe(self, array, warn=True, substituteValue=1e-9)
Definition: ptc.py:596
def _fitLeastSq(self, initialParams, dataX, dataY, function)
Definition: ptc.py:426
def plot(self, dataRef, dataset, ptcFitType)
Definition: ptc.py:841