lsst.cp.pipe  19.0.0-8-g079e426+8
ptc.py
Go to the documentation of this file.
1 # This file is part of cp_pipe.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 #
22 
23 __all__ = ['MeasurePhotonTransferCurveTask',
24  'MeasurePhotonTransferCurveTaskConfig',
25  'PhotonTransferCurveDataset']
26 
27 import numpy as np
28 import matplotlib.pyplot as plt
29 import os
30 from matplotlib.backends.backend_pdf import PdfPages
31 from sqlite3 import OperationalError
32 from collections import Counter
33 
34 import lsst.afw.math as afwMath
35 import lsst.pex.config as pexConfig
36 import lsst.pipe.base as pipeBase
37 from lsst.ip.isr import IsrTask
38 from .utils import (NonexistentDatasetTaskDataIdContainer, PairedVisitListTaskRunner,
39  checkExpLengthEqual, validateIsrConfig)
40 from scipy.optimize import leastsq, least_squares
41 import numpy.polynomial.polynomial as poly
42 
43 
44 class MeasurePhotonTransferCurveTaskConfig(pexConfig.Config):
45  """Config class for photon transfer curve measurement task"""
46  isr = pexConfig.ConfigurableField(
47  target=IsrTask,
48  doc="""Task to perform instrumental signature removal.""",
49  )
50  isrMandatorySteps = pexConfig.ListField(
51  dtype=str,
52  doc="isr operations that must be performed for valid results. Raises if any of these are False.",
53  default=['doAssembleCcd']
54  )
55  isrForbiddenSteps = pexConfig.ListField(
56  dtype=str,
57  doc="isr operations that must NOT be performed for valid results. Raises if any of these are True",
58  default=['doFlat', 'doFringe', 'doAddDistortionModel', 'doBrighterFatter', 'doUseOpticsTransmission',
59  'doUseFilterTransmission', 'doUseSensorTransmission', 'doUseAtmosphereTransmission']
60  )
61  isrDesirableSteps = pexConfig.ListField(
62  dtype=str,
63  doc="isr operations that it is advisable to perform, but are not mission-critical." +
64  " WARNs are logged for any of these found to be False.",
65  default=['doBias', 'doDark', 'doCrosstalk', 'doDefect']
66  )
67  isrUndesirableSteps = pexConfig.ListField(
68  dtype=str,
69  doc="isr operations that it is *not* advisable to perform in the general case, but are not" +
70  " forbidden as some use-cases might warrant them." +
71  " WARNs are logged for any of these found to be True.",
72  default=['doLinearize']
73  )
74  ccdKey = pexConfig.Field(
75  dtype=str,
76  doc="The key by which to pull a detector from a dataId, e.g. 'ccd' or 'detector'.",
77  default='ccd',
78  )
79  makePlots = pexConfig.Field(
80  dtype=bool,
81  doc="Plot the PTC curves?",
82  default=False,
83  )
84  ptcFitType = pexConfig.ChoiceField(
85  dtype=str,
86  doc="Fit PTC to approximation in Astier+19 (Equation 16) or to a polynomial.",
87  default="POLYNOMIAL",
88  allowed={
89  "POLYNOMIAL": "n-degree polynomial (use 'polynomialFitDegree' to set 'n').",
90  "ASTIERAPPROXIMATION": "Approximation in Astier+19 (Eq. 16)."
91  }
92  )
93  polynomialFitDegree = pexConfig.Field(
94  dtype=int,
95  doc="Degree of polynomial to fit the PTC, when 'ptcFitType'=POLYNOMIAL.",
96  default=2,
97  )
98  polynomialFitDegreeNonLinearity = pexConfig.Field(
99  dtype=int,
100  doc="Degree of polynomial to fit the meanSignal vs exposureTime curve to produce" +
101  " the table for LinearizerLookupTable.",
102  default=3,
103  )
104  binSize = pexConfig.Field(
105  dtype=int,
106  doc="Bin the image by this factor in both dimensions.",
107  default=1,
108  )
109  minMeanSignal = pexConfig.Field(
110  dtype=float,
111  doc="Minimum value (inclusive) of mean signal (in ADU) above which to consider.",
112  default=0,
113  )
114  maxMeanSignal = pexConfig.Field(
115  dtype=float,
116  doc="Maximum value (inclusive) of mean signal (in ADU) below which to consider.",
117  default=9e6,
118  )
119  initialNonLinearityExclusionThresholdPositive = pexConfig.RangeField(
120  dtype=float,
121  doc="Initially exclude data points with a variance that are more than a factor of this from being"
122  " linear in the positive direction, from the PTC fit. Note that these points will also be"
123  " excluded from the non-linearity fit. This is done before the iterative outlier rejection,"
124  " to allow an accurate determination of the sigmas for said iterative fit.",
125  default=0.12,
126  min=0.0,
127  max=1.0,
128  )
129  initialNonLinearityExclusionThresholdNegative = pexConfig.RangeField(
130  dtype=float,
131  doc="Initially exclude data points with a variance that are more than a factor of this from being"
132  " linear in the negative direction, from the PTC fit. Note that these points will also be"
133  " excluded from the non-linearity fit. This is done before the iterative outlier rejection,"
134  " to allow an accurate determination of the sigmas for said iterative fit.",
135  default=0.25,
136  min=0.0,
137  max=1.0,
138  )
139  sigmaCutPtcOutliers = pexConfig.Field(
140  dtype=float,
141  doc="Sigma cut for outlier rejection in PTC.",
142  default=5.0,
143  )
144  maxIterationsPtcOutliers = pexConfig.Field(
145  dtype=int,
146  doc="Maximum number of iterations for outlier rejection in PTC.",
147  default=2,
148  )
149  doFitBootstrap = pexConfig.Field(
150  dtype=bool,
151  doc="Use bootstrap for the PTC fit parameters and errors?.",
152  default=False,
153  )
154  linResidualTimeIndex = pexConfig.Field(
155  dtype=int,
156  doc="Index position in time array for reference time in linearity residual calculation.",
157  default=2,
158  )
159  maxAduForLookupTableLinearizer = pexConfig.Field(
160  dtype=int,
161  doc="Maximum ADU value for the LookupTable linearizer.",
162  default=2**18,
163  )
164 
165 
167  """A simple class to hold the output data from the PTC task.
168 
169  The dataset is made up of a dictionary for each item, keyed by the
170  amplifiers' names, which much be supplied at construction time.
171 
172  New items cannot be added to the class to save accidentally saving to the
173  wrong property, and the class can be frozen if desired.
174 
175  inputVisitPairs records the visits used to produce the data.
176  When fitPtcAndNonLinearity() is run, a mask is built up, which is by definition
177  always the same length as inputVisitPairs, rawExpTimes, rawMeans
178  and rawVars, and is a list of bools, which are incrementally set to False
179  as points are discarded from the fits.
180 
181  PTC fit parameters for polynomials are stored in a list in ascending order
182  of polynomial term, i.e. par[0]*x^0 + par[1]*x + par[2]*x^2 etc
183  with the length of the list corresponding to the order of the polynomial
184  plus one.
185  """
186  def __init__(self, ampNames):
187  # add items to __dict__ directly because __setattr__ is overridden
188 
189  # instance variables
190  self.__dict__["ampNames"] = ampNames
191  self.__dict__["badAmps"] = []
192 
193  # raw data variables
194  self.__dict__["inputVisitPairs"] = {ampName: [] for ampName in ampNames}
195  self.__dict__["visitMask"] = {ampName: [] for ampName in ampNames}
196  self.__dict__["rawExpTimes"] = {ampName: [] for ampName in ampNames}
197  self.__dict__["rawMeans"] = {ampName: [] for ampName in ampNames}
198  self.__dict__["rawVars"] = {ampName: [] for ampName in ampNames}
199 
200  # fit information
201  self.__dict__["ptcFitType"] = {ampName: "" for ampName in ampNames}
202  self.__dict__["ptcFitPars"] = {ampName: [] for ampName in ampNames}
203  self.__dict__["ptcFitParsError"] = {ampName: [] for ampName in ampNames}
204  self.__dict__["nonLinearity"] = {ampName: [] for ampName in ampNames}
205  self.__dict__["nonLinearityError"] = {ampName: [] for ampName in ampNames}
206  self.__dict__["nonLinearityResiduals"] = {ampName: [] for ampName in ampNames}
207  self.__dict__["coefficientLinearizeSquared"] = {ampName: [] for ampName in ampNames}
208 
209  # final results
210  self.__dict__["gain"] = {ampName: -1. for ampName in ampNames}
211  self.__dict__["gainErr"] = {ampName: -1. for ampName in ampNames}
212  self.__dict__["noise"] = {ampName: -1. for ampName in ampNames}
213  self.__dict__["noiseErr"] = {ampName: -1. for ampName in ampNames}
214  self.__dict__["coefficientLinearizeSquared"] = {ampName: [] for ampName in ampNames}
215 
216  def __setattr__(self, attribute, value):
217  """Protect class attributes"""
218  if attribute not in self.__dict__:
219  raise AttributeError(f"{attribute} is not already a member of PhotonTransferCurveDataset, which"
220  " does not support setting of new attributes.")
221  else:
222  self.__dict__[attribute] = value
223 
224  def getVisitsUsed(self, ampName):
225  """Get the visits used, i.e. not discarded, for a given amp.
226 
227  If no mask has been created yet, all visits are returned.
228  """
229  if self.visitMask[ampName] == []:
230  return self.inputVisitPairs[ampName]
231 
232  # if the mask exists it had better be the same length as the visitPairs
233  assert len(self.visitMask[ampName]) == len(self.inputVisitPairs[ampName])
234 
235  pairs = self.inputVisitPairs[ampName]
236  mask = self.visitMask[ampName]
237  # cast to bool required because numpy
238  return [(v1, v2) for ((v1, v2), m) in zip(pairs, mask) if bool(m) is True]
239 
240  def getGoodAmps(self):
241  return [amp for amp in self.ampNames if amp not in self.badAmps]
242 
243 
244 class MeasurePhotonTransferCurveTask(pipeBase.CmdLineTask):
245  """A class to calculate, fit, and plot a PTC from a set of flat pairs.
246 
247  The Photon Transfer Curve (var(signal) vs mean(signal)) is a standard tool
248  used in astronomical detectors characterization (e.g., Janesick 2001,
249  Janesick 2007). This task calculates the PTC from a series of pairs of
250  flat-field images; each pair taken at identical exposure times. The
251  difference image of each pair is formed to eliminate fixed pattern noise,
252  and then the variance of the difference image and the mean of the average image
253  are used to produce the PTC. An n-degree polynomial or the approximation in Equation
254  16 of Astier+19 ("The Shape of the Photon Transfer Curve of CCD sensors",
255  arXiv:1905.08677) can be fitted to the PTC curve. These models include
256  parameters such as the gain (e/ADU) and readout noise.
257 
258  Parameters
259  ----------
260 
261  *args: `list`
262  Positional arguments passed to the Task constructor. None used at this
263  time.
264  **kwargs: `dict`
265  Keyword arguments passed on to the Task constructor. None used at this
266  time.
267 
268  """
269 
270  RunnerClass = PairedVisitListTaskRunner
271  ConfigClass = MeasurePhotonTransferCurveTaskConfig
272  _DefaultName = "measurePhotonTransferCurve"
273 
274  def __init__(self, *args, **kwargs):
275  pipeBase.CmdLineTask.__init__(self, *args, **kwargs)
276  self.makeSubtask("isr")
277  plt.interactive(False) # stop windows popping up when plotting. When headless, use 'agg' backend too
278  validateIsrConfig(self.isr, self.config.isrMandatorySteps,
279  self.config.isrForbiddenSteps, self.config.isrDesirableSteps, checkTrim=False)
280  self.config.validate()
281  self.config.freeze()
282 
283  @classmethod
284  def _makeArgumentParser(cls):
285  """Augment argument parser for the MeasurePhotonTransferCurveTask."""
286  parser = pipeBase.ArgumentParser(name=cls._DefaultName)
287  parser.add_argument("--visit-pairs", dest="visitPairs", nargs="*",
288  help="Visit pairs to use. Each pair must be of the form INT,INT e.g. 123,456")
289  parser.add_id_argument("--id", datasetType="photonTransferCurveDataset",
290  ContainerClass=NonexistentDatasetTaskDataIdContainer,
291  help="The ccds to use, e.g. --id ccd=0..100")
292  return parser
293 
294  @pipeBase.timeMethod
295  def runDataRef(self, dataRef, visitPairs):
296  """Run the Photon Transfer Curve (PTC) measurement task.
297 
298  For a dataRef (which is each detector here),
299  and given a list of visit pairs at different exposure times,
300  measure the PTC.
301 
302  Parameters
303  ----------
304  dataRef : list of lsst.daf.persistence.ButlerDataRef
305  dataRef for the detector for the visits to be fit.
306  visitPairs : `iterable` of `tuple` of `int`
307  Pairs of visit numbers to be processed together
308  """
309 
310  # setup necessary objects
311  detNum = dataRef.dataId[self.config.ccdKey]
312  detector = dataRef.get('camera')[dataRef.dataId[self.config.ccdKey]]
313  # expand some missing fields that we need for lsstCam. This is a work-around
314  # for Gen2 problems that I (RHL) don't feel like solving. The calibs pipelines
315  # (which inherit from CalibTask) use addMissingKeys() to do basically the same thing
316  #
317  # Basically, the butler's trying to look up the fields in `raw_visit` which won't work
318  for name in dataRef.getButler().getKeys('bias'):
319  if name not in dataRef.dataId:
320  try:
321  dataRef.dataId[name] = \
322  dataRef.getButler().queryMetadata('raw', [name], detector=detNum)[0]
323  except OperationalError:
324  pass
325 
326  amps = detector.getAmplifiers()
327  ampNames = [amp.getName() for amp in amps]
328  dataset = PhotonTransferCurveDataset(ampNames)
329 
330  self.log.info('Measuring PTC using %s visits for detector %s' % (visitPairs, detNum))
331 
332  for (v1, v2) in visitPairs:
333  # Perform ISR on each exposure
334  dataRef.dataId['expId'] = v1
335  exp1 = self.isr.runDataRef(dataRef).exposure
336  dataRef.dataId['expId'] = v2
337  exp2 = self.isr.runDataRef(dataRef).exposure
338  del dataRef.dataId['expId']
339 
340  checkExpLengthEqual(exp1, exp2, v1, v2, raiseWithMessage=True)
341  expTime = exp1.getInfo().getVisitInfo().getExposureTime()
342 
343  for amp in detector:
344  mu, varDiff = self.measureMeanVarPair(exp1, exp2, region=amp.getBBox())
345  ampName = amp.getName()
346 
347  dataset.rawExpTimes[ampName].append(expTime)
348  dataset.rawMeans[ampName].append(mu)
349  dataset.rawVars[ampName].append(varDiff)
350  dataset.inputVisitPairs[ampName].append((v1, v2))
351 
352  numberAmps = len(detector.getAmplifiers())
353  numberAduValues = self.config.maxAduForLookupTableLinearizer
354  lookupTableArray = np.zeros((numberAmps, numberAduValues), dtype=np.float32)
355 
356  # Fit PTC and (non)linearity of signal vs time curve.
357  # Fill up PhotonTransferCurveDataset object.
358  # Fill up array for LUT linearizer.
359  dataset = self.fitPtcAndNonLinearity(dataset, lookupTableArray, ptcFitType=self.config.ptcFitType)
360 
361  if self.config.makePlots:
362  self.plot(dataRef, dataset, ptcFitType=self.config.ptcFitType)
363 
364  # Save data, PTC fit, and NL fit dictionaries
365  self.log.info(f"Writing PTC and NL data to {dataRef.getUri(write=True)}")
366  dataRef.put(dataset, datasetType="photonTransferCurveDataset")
367 
368  self.log.info('Finished measuring PTC for in detector %s' % detNum)
369 
370  return pipeBase.Struct(exitStatus=0)
371 
372  def measureMeanVarPair(self, exposure1, exposure2, region=None):
373  """Calculate the mean signal of two exposures and the variance of their difference.
374 
375  Parameters
376  ----------
377  exposure1 : `lsst.afw.image.exposure.exposure.ExposureF`
378  First exposure of flat field pair.
379 
380  exposure2 : `lsst.afw.image.exposure.exposure.ExposureF`
381  Second exposure of flat field pair.
382 
383  region : `lsst.geom.Box2I`
384  Region of each exposure where to perform the calculations (e.g, an amplifier).
385 
386  Return
387  ------
388 
389  mu : `np.float`
390  0.5*(mu1 + mu2), where mu1, and mu2 are the clipped means of the regions in
391  both exposures.
392 
393  varDiff : `np.float`
394  Half of the clipped variance of the difference of the regions inthe two input
395  exposures.
396  """
397 
398  if region is not None:
399  im1Area = exposure1.maskedImage[region]
400  im2Area = exposure2.maskedImage[region]
401  else:
402  im1Area = exposure1.maskedImage
403  im2Area = exposure2.maskedImage
404 
405  im1Area = afwMath.binImage(im1Area, self.config.binSize)
406  im2Area = afwMath.binImage(im2Area, self.config.binSize)
407 
408  # Clipped mean of images; then average of mean.
409  mu1 = afwMath.makeStatistics(im1Area, afwMath.MEANCLIP).getValue()
410  mu2 = afwMath.makeStatistics(im2Area, afwMath.MEANCLIP).getValue()
411  mu = 0.5*(mu1 + mu2)
412 
413  # Take difference of pairs
414  # symmetric formula: diff = (mu2*im1-mu1*im2)/(0.5*(mu1+mu2))
415  temp = im2Area.clone()
416  temp *= mu1
417  diffIm = im1Area.clone()
418  diffIm *= mu2
419  diffIm -= temp
420  diffIm /= mu
421 
422  varDiff = 0.5*(afwMath.makeStatistics(diffIm, afwMath.VARIANCECLIP).getValue())
423 
424  return mu, varDiff
425 
426  def _fitLeastSq(self, initialParams, dataX, dataY, function):
427  """Do a fit and estimate the parameter errors using using scipy.optimize.leastq.
428 
429  optimize.leastsq returns the fractional covariance matrix. To estimate the
430  standard deviation of the fit parameters, multiply the entries of this matrix
431  by the reduced chi squared and take the square root of the diagonal elements.
432 
433  Parameters
434  ----------
435  initialParams : list of np.float
436  initial values for fit parameters. For ptcFitType=POLYNOMIAL, its length
437  determines the degree of the polynomial.
438 
439  dataX : np.array of np.float
440  Data in the abscissa axis.
441 
442  dataY : np.array of np.float
443  Data in the ordinate axis.
444 
445  function : callable object (function)
446  Function to fit the data with.
447 
448  Return
449  ------
450  pFitSingleLeastSquares : list of np.float
451  List with fitted parameters.
452 
453  pErrSingleLeastSquares : list of np.float
454  List with errors for fitted parameters.
455  """
456 
457  def errFunc(p, x, y):
458  return function(p, x) - y
459 
460  pFit, pCov, infoDict, errMessage, success = leastsq(errFunc, initialParams,
461  args=(dataX, dataY), full_output=1, epsfcn=0.0001)
462 
463  if (len(dataY) > len(initialParams)) and pCov is not None:
464  reducedChiSq = (errFunc(pFit, dataX, dataY)**2).sum()/(len(dataY)-len(initialParams))
465  pCov *= reducedChiSq
466  else:
467  pCov[:, :] = np.inf
468 
469  errorVec = []
470  for i in range(len(pFit)):
471  errorVec.append(np.fabs(pCov[i][i])**0.5)
472 
473  pFitSingleLeastSquares = pFit
474  pErrSingleLeastSquares = np.array(errorVec)
475 
476  return pFitSingleLeastSquares, pErrSingleLeastSquares
477 
478  def _fitBootstrap(self, initialParams, dataX, dataY, function, confidenceSigma=1.):
479  """Do a fit using least squares and bootstrap to estimate parameter errors.
480 
481  The bootstrap error bars are calculated by fitting 100 random data sets.
482 
483  Parameters
484  ----------
485  initialParams : list of np.float
486  initial values for fit parameters. For ptcFitType=POLYNOMIAL, its length
487  determines the degree of the polynomial.
488 
489  dataX : np.array of np.float
490  Data in the abscissa axis.
491 
492  dataY : np.array of np.float
493  Data in the ordinate axis.
494 
495  function : callable object (function)
496  Function to fit the data with.
497 
498  confidenceSigma : np.float
499  Number of sigmas that determine confidence interval for the bootstrap errors.
500 
501  Return
502  ------
503  pFitBootstrap : list of np.float
504  List with fitted parameters.
505 
506  pErrBootstrap : list of np.float
507  List with errors for fitted parameters.
508  """
509 
510  def errFunc(p, x, y):
511  return function(p, x) - y
512 
513  # Fit first time
514  pFit, _ = leastsq(errFunc, initialParams, args=(dataX, dataY), full_output=0)
515 
516  # Get the stdev of the residuals
517  residuals = errFunc(pFit, dataX, dataY)
518  sigmaErrTotal = np.std(residuals)
519 
520  # 100 random data sets are generated and fitted
521  pars = []
522  for i in range(100):
523  randomDelta = np.random.normal(0., sigmaErrTotal, len(dataY))
524  randomDataY = dataY + randomDelta
525  randomFit, _ = leastsq(errFunc, initialParams,
526  args=(dataX, randomDataY), full_output=0)
527  pars.append(randomFit)
528  pars = np.array(pars)
529  meanPfit = np.mean(pars, 0)
530 
531  # confidence interval for parameter estimates
532  nSigma = confidenceSigma
533  errPfit = nSigma*np.std(pars, 0)
534  pFitBootstrap = meanPfit
535  pErrBootstrap = errPfit
536  return pFitBootstrap, pErrBootstrap
537 
538  def funcPolynomial(self, pars, x):
539  """Polynomial function definition"""
540  return poly.polyval(x, [*pars])
541 
542  def funcAstier(self, pars, x):
543  """Single brighter-fatter parameter model for PTC; Equation 16 of Astier+19"""
544  a00, gain, noise = pars
545  return 0.5/(a00*gain*gain)*(np.exp(2*a00*x*gain)-1) + noise/(gain*gain)
546 
547  @staticmethod
548  def _initialParsForPolynomial(order):
549  assert(order >= 2)
550  pars = np.zeros(order, dtype=np.float)
551  pars[0] = 10
552  pars[1] = 1
553  pars[2:] = 0.0001
554  return pars
555 
556  @staticmethod
557  def _boundsForPolynomial(initialPars):
558  lowers = [np.NINF for p in initialPars]
559  uppers = [np.inf for p in initialPars]
560  lowers[1] = 0 # no negative gains
561  return (lowers, uppers)
562 
563  @staticmethod
564  def _boundsForAstier(initialPars):
565  lowers = [np.NINF for p in initialPars]
566  uppers = [np.inf for p in initialPars]
567  return (lowers, uppers)
568 
569  @staticmethod
570  def _getInitialGoodPoints(means, variances, maxDeviationPositive, maxDeviationNegative):
571  """Return a boolean array to mask bad points.
572 
573  A linear function has a constant ratio, so find the median
574  value of the ratios, and exclude the points that deviate
575  from that by more than a factor of maxDeviationPositive/negative.
576  Asymmetric deviations are supported as we expect the PTC to turn
577  down as the flux increases, but sometimes it anomalously turns
578  upwards just before turning over, which ruins the fits, so it
579  is wise to be stricter about restricting positive outliers than
580  negative ones.
581 
582  Too high and points that are so bad that fit will fail will be included
583  Too low and the non-linear points will be excluded, biasing the NL fit."""
584  ratios = [b/a for (a, b) in zip(means, variances)]
585  medianRatio = np.median(ratios)
586  ratioDeviations = [(r/medianRatio)-1 for r in ratios]
587 
588  # so that it doesn't matter if the deviation is expressed as positive or negative
589  maxDeviationPositive = abs(maxDeviationPositive)
590  maxDeviationNegative = -1. * abs(maxDeviationNegative)
591 
592  goodPoints = np.array([True if (r < maxDeviationPositive and r > maxDeviationNegative)
593  else False for r in ratioDeviations])
594  return goodPoints
595 
596  def _makeZeroSafe(self, array, warn=True, substituteValue=1e-9):
597  """"""
598  nBad = Counter(array)[0]
599  if nBad == 0:
600  return array
601 
602  if warn:
603  msg = f"Found {nBad} zeros in array at elements {[x for x in np.where(array==0)[0]]}"
604  self.log.warn(msg)
605 
606  array[array == 0] = substituteValue
607  return array
608 
609  def calculateLinearityResidualAndLinearizers(self, exposureTimeVector, meanSignalVector):
610  """Calculate linearity residual and fit an n-order polynomial to the mean vs time curve
611  to produce corrections (deviation from linear part of polynomial) for a particular amplifier
612  to populate LinearizeLookupTable. Use quadratic and linear parts of this polynomial to approximate
613  c0 for LinearizeSquared."
614 
615  Parameters
616  ---------
617 
618  exposureTimeVector: `list` of `np.float`
619  List of exposure times for each flat pair
620 
621  meanSignalVector: `list` of `np.float`
622  List of mean signal from diference image of flat pairs
623 
624  Returns
625  -------
626  c0: `np.float`
627  Coefficient for LinearizeSquared, where corrImage = uncorrImage + c0*uncorrImage^2.
628  c0 ~ -k2/(k1^2), where k1 and k2 are fit from
629  meanSignalVector = k0 + k1*exposureTimeVector + k2*exposureTimeVector^2 +...
630  + kn*exposureTimeVector^n, with n = "polynomialFitDegreeNonLinearity".
631 
632  linearizerTableRow: list of `np.float`
633  One dimensional array with deviation from linear part of n-order polynomial fit
634  to mean vs time curve. This array will be one row (for the particular amplifier at hand)
635  of the table array for LinearizeLookupTable.
636 
637  linResidual: list of `np.float`
638  Linearity residual from the mean vs time curve, defined as
639  100*(1 - meanSignalReference/expTimeReference/(meanSignal/expTime).
640 
641  parsFit: list of `np.float`
642  Parameters from n-order polynomial fit to mean vs time curve.
643 
644  parsFitErr: list of `np.float`
645  Parameters from n-order polynomial fit to mean vs time curve.
646 
647  """
648 
649  # Lookup table linearizer
650  parsIniNonLinearity = self._initialParsForPolynomial(self.config.polynomialFitDegreeNonLinearity + 1)
651  if self.config.doFitBootstrap:
652  parsFit, parsFitErr = self._fitBootstrap(parsIniNonLinearity, exposureTimeVector,
653  meanSignalVector, self.funcPolynomial)
654  else:
655  parsFit, parsFitErr = self._fitLeastSq(parsIniNonLinearity, exposureTimeVector, meanSignalVector,
656  self.funcPolynomial)
657 
658  # Use linear part to get time at wich signal is maxAduForLookupTableLinearizer ADU
659  tMax = (self.config.maxAduForLookupTableLinearizer - parsFit[0])/parsFit[1]
660  timeRange = np.linspace(0, tMax, self.config.maxAduForLookupTableLinearizer)
661  signalIdeal = parsFit[0] + parsFit[1]*timeRange
662  signalUncorrected = self.funcPolynomial(parsFit, timeRange)
663  linearizerTableRow = signalIdeal - signalUncorrected # LinearizerLookupTable has corrections
664 
665  # Use quadratic and linear part of fit to produce c0 for LinearizeSquared
666  # Check that magnitude of higher order (>= 3) coefficents of the polyFit are small,
667  # i.e., less than threshold = 1e-10 (typical quadratic and cubic coefficents are ~1e-6
668  # and ~1e-12).
669  k1, k2 = parsFit[1], parsFit[2]
670  c0 = -k2/(k1**2) # c0 coefficient for LinearizeSquared
671  for coefficient in parsFit[3:]:
672  if np.fabs(coefficient) > 1e-10:
673  msg = f"Coefficient {coefficient} in polynomial fit larger than threshold 1e-10."
674  self.log.warn(msg)
675 
676  # Linearity residual
677  linResidualTimeIndex = self.config.linResidualTimeIndex
678  if exposureTimeVector[linResidualTimeIndex] == 0.0:
679  raise RuntimeError("Reference time for linearity residual can't be 0.0")
680  linResidual = 100*(1 - ((meanSignalVector[linResidualTimeIndex] /
681  exposureTimeVector[linResidualTimeIndex]) /
682  (meanSignalVector/exposureTimeVector)))
683 
684  return c0, linearizerTableRow, linResidual, parsFit, parsFitErr
685 
686  def fitPtcAndNonLinearity(self, dataset, tableArray, ptcFitType):
687  """Fit the photon transfer curve and calculate linearity and residuals.
688 
689  Fit the photon transfer curve with either a polynomial of the order
690  specified in the task config, or using the Astier approximation.
691 
692  Sigma clipping is performed iteratively for the fit, as well as an
693  initial clipping of data points that are more than
694  config.initialNonLinearityExclusionThreshold away from lying on a
695  straight line. This other step is necessary because the photon transfer
696  curve turns over catastrophically at very high flux (because saturation
697  drops the variance to ~0) and these far outliers cause the initial fit
698  to fail, meaning the sigma cannot be calculated to perform the
699  sigma-clipping.
700 
701  Parameters
702  ----------
703  dataset : `lsst.cp.pipe.ptc.PhotonTransferCurveDataset`
704  The dataset containing the means, variances and exposure times
705  ptcFitType : `str`
706  Fit a 'POLYNOMIAL' (degree: 'polynomialFitDegree') or
707  'ASTIERAPPROXIMATION' to the PTC
708  tableArray : `np.array`
709  Look-up table array with size rows=nAmps and columns=ADU values
710 
711  Returns
712  -------
713  dataset: `lsst.cp.pipe.ptc.PhotonTransferCurveDataset`
714  This is the same dataset as the input paramter, however, it has been modified
715  to include information such as the fit vectors and the fit parameters. See
716  the class `PhotonTransferCurveDatase`.
717  """
718 
719  def errFunc(p, x, y):
720  return ptcFunc(p, x) - y
721 
722  sigmaCutPtcOutliers = self.config.sigmaCutPtcOutliers
723  maxIterationsPtcOutliers = self.config.maxIterationsPtcOutliers
724 
725  for i, ampName in enumerate(dataset.ampNames):
726  timeVecOriginal = np.array(dataset.rawExpTimes[ampName])
727  meanVecOriginal = np.array(dataset.rawMeans[ampName])
728  varVecOriginal = np.array(dataset.rawVars[ampName])
729  varVecOriginal = self._makeZeroSafe(varVecOriginal)
730 
731  mask = ((meanVecOriginal >= self.config.minMeanSignal) &
732  (meanVecOriginal <= self.config.maxMeanSignal))
733 
734  goodPoints = self._getInitialGoodPoints(meanVecOriginal, varVecOriginal,
735  self.config.initialNonLinearityExclusionThresholdPositive,
736  self.config.initialNonLinearityExclusionThresholdNegative)
737  mask = mask & goodPoints
738 
739  if ptcFitType == 'ASTIERAPPROXIMATION':
740  ptcFunc = self.funcAstier
741  parsIniPtc = [-1e-9, 1.0, 10.] # a00, gain, noise
742  bounds = self._boundsForAstier(parsIniPtc)
743  if ptcFitType == 'POLYNOMIAL':
744  ptcFunc = self.funcPolynomial
745  parsIniPtc = self._initialParsForPolynomial(self.config.polynomialFitDegree + 1)
746  bounds = self._boundsForPolynomial(parsIniPtc)
747 
748  # Before bootstrap fit, do an iterative fit to get rid of outliers
749  count = 1
750  while count <= maxIterationsPtcOutliers:
751  # Note that application of the mask actually shrinks the array
752  # to size rather than setting elements to zero (as we want) so
753  # always update mask itself and re-apply to the original data
754  meanTempVec = meanVecOriginal[mask]
755  varTempVec = varVecOriginal[mask]
756  res = least_squares(errFunc, parsIniPtc, bounds=bounds, args=(meanTempVec, varTempVec))
757  pars = res.x
758 
759  # change this to the original from the temp because the masks are ANDed
760  # meaning once a point is masked it's always masked, and the masks must
761  # always be the same length for broadcasting
762  sigResids = (varVecOriginal - ptcFunc(pars, meanVecOriginal))/np.sqrt(varVecOriginal)
763  newMask = np.array([True if np.abs(r) < sigmaCutPtcOutliers else False for r in sigResids])
764  mask = mask & newMask
765 
766  nDroppedTotal = Counter(mask)[False]
767  self.log.debug(f"Iteration {count}: discarded {nDroppedTotal} points in total for {ampName}")
768  count += 1
769  # objects should never shrink
770  assert (len(mask) == len(timeVecOriginal) == len(meanVecOriginal) == len(varVecOriginal))
771 
772  dataset.visitMask[ampName] = mask # store the final mask
773 
774  parsIniPtc = pars
775  timeVecFinal = timeVecOriginal[mask]
776  meanVecFinal = meanVecOriginal[mask]
777  varVecFinal = varVecOriginal[mask]
778 
779  if Counter(mask)[False] > 0:
780  self.log.info((f"Number of points discarded in PTC of amplifier {ampName}:" +
781  f" {Counter(mask)[False]} out of {len(meanVecOriginal)}"))
782 
783  if (len(meanVecFinal) < len(parsIniPtc)):
784  msg = (f"\nSERIOUS: Not enough data points ({len(meanVecFinal)}) compared to the number of"
785  f"parameters of the PTC model({len(parsIniPtc)}). Setting {ampName} to BAD.")
786  self.log.warn(msg)
787  dataset.badAmps.append(ampName)
788  dataset.gain[ampName] = np.nan
789  dataset.gainErr[ampName] = np.nan
790  dataset.noise[ampName] = np.nan
791  dataset.noiseErr[ampName] = np.nan
792  dataset.nonLinearity[ampName] = np.nan
793  dataset.nonLinearityError[ampName] = np.nan
794  dataset.nonLinearityResiduals[ampName] = np.nan
795  continue
796 
797  # Fit the PTC
798  if self.config.doFitBootstrap:
799  parsFit, parsFitErr = self._fitBootstrap(parsIniPtc, meanVecFinal, varVecFinal, ptcFunc)
800  else:
801  parsFit, parsFitErr = self._fitLeastSq(parsIniPtc, meanVecFinal, varVecFinal, ptcFunc)
802 
803  dataset.ptcFitPars[ampName] = parsFit
804  dataset.ptcFitParsError[ampName] = parsFitErr
805 
806  if ptcFitType == 'ASTIERAPPROXIMATION':
807  ptcGain = parsFit[1]
808  ptcGainErr = parsFitErr[1]
809  ptcNoise = np.sqrt(np.fabs(parsFit[2]))
810  ptcNoiseErr = 0.5*(parsFitErr[2]/np.fabs(parsFit[2]))*np.sqrt(np.fabs(parsFit[2]))
811  if ptcFitType == 'POLYNOMIAL':
812  ptcGain = 1./parsFit[1]
813  ptcGainErr = np.fabs(1./parsFit[1])*(parsFitErr[1]/parsFit[1])
814  ptcNoise = np.sqrt(np.fabs(parsFit[0]))*ptcGain
815  ptcNoiseErr = (0.5*(parsFitErr[0]/np.fabs(parsFit[0]))*(np.sqrt(np.fabs(parsFit[0]))))*ptcGain
816 
817  dataset.gain[ampName] = ptcGain
818  dataset.gainErr[ampName] = ptcGainErr
819  dataset.noise[ampName] = ptcNoise
820  dataset.noiseErr[ampName] = ptcNoiseErr
821  dataset.ptcFitType[ampName] = ptcFitType
822 
823  # Non-linearity residuals (NL of mean vs time curve): percentage, and fit to a quadratic function
824  # In this case, len(parsIniNonLinearity) = 3 indicates that we want a quadratic fit
825 
826  (c0, linearizerTableRow, linResidualNonLinearity, parsFitNonLinearity,
827  parsFitErrNonLinearity) = self.calculateLinearityResidualAndLinearizers(timeVecFinal,
828  meanVecFinal)
829  # LinearizerLookupTable
830  tableArray[i, :] = linearizerTableRow
831 
832  dataset.nonLinearity[ampName] = parsFitNonLinearity
833  dataset.nonLinearityError[ampName] = parsFitErrNonLinearity
834  dataset.nonLinearityResiduals[ampName] = linResidualNonLinearity
835  dataset.coefficientLinearizeSquared[ampName] = c0
836 
837  return dataset
838 
839  def plot(self, dataRef, dataset, ptcFitType):
840  dirname = dataRef.getUri(datasetType='cpPipePlotRoot', write=True)
841  if not os.path.exists(dirname):
842  os.makedirs(dirname)
843 
844  detNum = dataRef.dataId[self.config.ccdKey]
845  filename = f"PTC_det{detNum}.pdf"
846  filenameFull = os.path.join(dirname, filename)
847  with PdfPages(filenameFull) as pdfPages:
848  self._plotPtc(dataset, ptcFitType, pdfPages)
849 
850  def _plotPtc(self, dataset, ptcFitType, pdfPages):
851  """Plot PTC, linearity, and linearity residual per amplifier"""
852 
853  if ptcFitType == 'ASTIERAPPROXIMATION':
854  ptcFunc = self.funcAstier
855  stringTitle = r"Var = $\frac{1}{2g^2a_{00}}(\exp (2a_{00} \mu g) - 1) + \frac{n_{00}}{g^2}$"
856 
857  if ptcFitType == 'POLYNOMIAL':
858  ptcFunc = self.funcPolynomial
859  stringTitle = f"Polynomial (degree: {self.config.polynomialFitDegree})"
860 
861  legendFontSize = 7.5
862  labelFontSize = 8
863  titleFontSize = 10
864  supTitleFontSize = 18
865  markerSize = 25
866 
867  # General determination of the size of the plot grid
868  nAmps = len(dataset.ampNames)
869  if nAmps == 2:
870  nRows, nCols = 2, 1
871  nRows = np.sqrt(nAmps)
872  mantissa, _ = np.modf(nRows)
873  if mantissa > 0:
874  nRows = int(nRows) + 1
875  nCols = nRows
876  else:
877  nRows = int(nRows)
878  nCols = nRows
879 
880  f, ax = plt.subplots(nrows=nRows, ncols=nCols, sharex='col', sharey='row', figsize=(13, 10))
881  f2, ax2 = plt.subplots(nrows=nRows, ncols=nCols, sharex='col', sharey='row', figsize=(13, 10))
882 
883  for i, (amp, a, a2) in enumerate(zip(dataset.ampNames, ax.flatten(), ax2.flatten())):
884  meanVecOriginal = np.array(dataset.rawMeans[amp])
885  varVecOriginal = np.array(dataset.rawVars[amp])
886  mask = dataset.visitMask[amp]
887  meanVecFinal = meanVecOriginal[mask]
888  varVecFinal = varVecOriginal[mask]
889  meanVecOutliers = meanVecOriginal[np.invert(mask)]
890  varVecOutliers = varVecOriginal[np.invert(mask)]
891  pars, parsErr = dataset.ptcFitPars[amp], dataset.ptcFitParsError[amp]
892 
893  if ptcFitType == 'ASTIERAPPROXIMATION':
894  ptcA00, ptcA00error = pars[0], parsErr[0]
895  ptcGain, ptcGainError = pars[1], parsErr[1]
896  ptcNoise = np.sqrt(np.fabs(pars[2]))
897  ptcNoiseError = 0.5*(parsErr[2]/np.fabs(pars[2]))*np.sqrt(np.fabs(pars[2]))
898  stringLegend = (f"a00: {ptcA00:.2e}+/-{ptcA00error:.2e}"
899  f"\n Gain: {ptcGain:.4}+/-{ptcGainError:.2e}"
900  f"\n Noise: {ptcNoise:.4}+/-{ptcNoiseError:.2e}")
901 
902  if ptcFitType == 'POLYNOMIAL':
903  ptcGain, ptcGainError = 1./pars[1], np.fabs(1./pars[1])*(parsErr[1]/pars[1])
904  ptcNoise = np.sqrt(np.fabs(pars[0]))*ptcGain
905  ptcNoiseError = (0.5*(parsErr[0]/np.fabs(pars[0]))*(np.sqrt(np.fabs(pars[0]))))*ptcGain
906  stringLegend = (f"Noise: {ptcNoise:.4}+/-{ptcNoiseError:.2e} \n"
907  f"Gain: {ptcGain:.4}+/-{ptcGainError:.2e}")
908 
909  minMeanVecFinal = np.min(meanVecFinal)
910  maxMeanVecFinal = np.max(meanVecFinal)
911  meanVecFit = np.linspace(minMeanVecFinal, maxMeanVecFinal, 100*len(meanVecFinal))
912  minMeanVecOriginal = np.min(meanVecOriginal)
913  maxMeanVecOriginal = np.max(meanVecOriginal)
914  deltaXlim = maxMeanVecOriginal - minMeanVecOriginal
915 
916  a.plot(meanVecFit, ptcFunc(pars, meanVecFit), color='red')
917  a.plot(meanVecFinal, pars[0] + pars[1]*meanVecFinal, color='green', linestyle='--')
918  a.scatter(meanVecFinal, varVecFinal, c='blue', marker='o', s=markerSize)
919  a.scatter(meanVecOutliers, varVecOutliers, c='magenta', marker='s', s=markerSize)
920  a.set_xlabel(r'Mean signal ($\mu$, ADU)', fontsize=labelFontSize)
921  a.set_xticks(meanVecOriginal)
922  a.set_ylabel(r'Variance (ADU$^2$)', fontsize=labelFontSize)
923  a.tick_params(labelsize=11)
924  a.text(0.03, 0.8, stringLegend, transform=a.transAxes, fontsize=legendFontSize)
925  a.set_xscale('linear', fontsize=labelFontSize)
926  a.set_yscale('linear', fontsize=labelFontSize)
927  a.set_title(amp, fontsize=titleFontSize)
928  a.set_xlim([minMeanVecOriginal - 0.2*deltaXlim, maxMeanVecOriginal + 0.2*deltaXlim])
929 
930  # Same, but in log-scale
931  a2.plot(meanVecFit, ptcFunc(pars, meanVecFit), color='red')
932  a2.scatter(meanVecFinal, varVecFinal, c='blue', marker='o', s=markerSize)
933  a2.scatter(meanVecOutliers, varVecOutliers, c='magenta', marker='s', s=markerSize)
934  a2.set_xlabel(r'Mean Signal ($\mu$, ADU)', fontsize=labelFontSize)
935  a2.set_ylabel(r'Variance (ADU$^2$)', fontsize=labelFontSize)
936  a2.tick_params(labelsize=11)
937  a2.text(0.03, 0.8, stringLegend, transform=a2.transAxes, fontsize=legendFontSize)
938  a2.set_xscale('log')
939  a2.set_yscale('log')
940  a2.set_title(amp, fontsize=titleFontSize)
941  a2.set_xlim([minMeanVecOriginal, maxMeanVecOriginal])
942 
943  f.suptitle(f"PTC \n Fit: " + stringTitle, fontsize=20)
944  pdfPages.savefig(f)
945  f2.suptitle(f"PTC (log-log)", fontsize=20)
946  pdfPages.savefig(f2)
947 
948  # Plot mean vs time
949  f, ax = plt.subplots(nrows=4, ncols=4, sharex='col', sharey='row', figsize=(13, 10))
950  for i, (amp, a) in enumerate(zip(dataset.ampNames, ax.flatten())):
951  meanVecFinal = np.array(dataset.rawMeans[amp])[dataset.visitMask[amp]]
952  timeVecFinal = np.array(dataset.rawExpTimes[amp])[dataset.visitMask[amp]]
953 
954  pars, parsErr = dataset.nonLinearity[amp], dataset.nonLinearityError[amp]
955  c0, c0Error = pars[0], parsErr[0]
956  c1, c1Error = pars[1], parsErr[1]
957  c2, c2Error = pars[2], parsErr[2]
958  stringLegend = f"c0: {c0:.4}+/-{c0Error:.2e}\n c1: {c1:.4}+/-{c1Error:.2e}" \
959  + f"\n c2(NL): {c2:.2e}+/-{c2Error:.2e}"
960  a.scatter(timeVecFinal, meanVecFinal)
961  a.plot(timeVecFinal, self.funcPolynomial(pars, timeVecFinal), color='red')
962  a.set_xlabel('Time (sec)', fontsize=labelFontSize)
963  a.set_xticks(timeVecFinal)
964  a.set_ylabel(r'Mean signal ($\mu$, ADU)', fontsize=labelFontSize)
965  a.tick_params(labelsize=labelFontSize)
966  a.text(0.03, 0.75, stringLegend, transform=a.transAxes, fontsize=legendFontSize)
967  a.set_xscale('linear', fontsize=labelFontSize)
968  a.set_yscale('linear', fontsize=labelFontSize)
969  a.set_title(amp, fontsize=titleFontSize)
970 
971  f.suptitle("Linearity \n Fit: " + r"$\mu = c_0 + c_1 t + c_2 t^2$", fontsize=supTitleFontSize)
972  pdfPages.savefig()
973 
974  # Plot linearity residual
975  f, ax = plt.subplots(nrows=4, ncols=4, sharex='col', sharey='row', figsize=(13, 10))
976  for i, (amp, a) in enumerate(zip(dataset.ampNames, ax.flatten())):
977  meanVecFinal = np.array(dataset.rawMeans[amp])[dataset.visitMask[amp]]
978  linRes = np.array(dataset.nonLinearityResiduals[amp])
979 
980  a.scatter(meanVecFinal, linRes)
981  a.axhline(y=0, color='k')
982  a.axvline(x=timeVecFinal[self.config.linResidualTimeIndex], color='g', linestyle='--')
983  a.set_xlabel(r'Mean signal ($\mu$, ADU)', fontsize=labelFontSize)
984  a.set_xticks(meanVecFinal)
985  a.set_ylabel('LR (%)', fontsize=labelFontSize)
986  a.tick_params(labelsize=labelFontSize)
987  a.set_xscale('linear', fontsize=labelFontSize)
988  a.set_yscale('linear', fontsize=labelFontSize)
989  a.set_title(amp, fontsize=titleFontSize)
990 
991  f.suptitle(r"Linearity Residual: $100(1 - \mu_{\rm{ref}}/t_{\rm{ref}})/(\mu / t))$" + "\n" +
992  r"$t_{\rm{ref}}$: " + f"{timeVecFinal[2]} s", fontsize=supTitleFontSize)
993  pdfPages.savefig()
994 
995  return
def _plotPtc(self, dataset, ptcFitType, pdfPages)
Definition: ptc.py:850
def runDataRef(self, dataRef, visitPairs)
Definition: ptc.py:295
def calculateLinearityResidualAndLinearizers(self, exposureTimeVector, meanSignalVector)
Definition: ptc.py:609
def __setattr__(self, attribute, value)
Definition: ptc.py:216
def checkExpLengthEqual(exp1, exp2, v1=None, v2=None, raiseWithMessage=False)
Definition: utils.py:162
def fitPtcAndNonLinearity(self, dataset, tableArray, ptcFitType)
Definition: ptc.py:686
def validateIsrConfig(isrTask, mandatory=None, forbidden=None, desirable=None, undesirable=None, checkTrim=True, logName=None)
Definition: utils.py:200
def _getInitialGoodPoints(means, variances, maxDeviationPositive, maxDeviationNegative)
Definition: ptc.py:570
def _fitBootstrap(self, initialParams, dataX, dataY, function, confidenceSigma=1.)
Definition: ptc.py:478
def __init__(self, args, kwargs)
Definition: ptc.py:274
def measureMeanVarPair(self, exposure1, exposure2, region=None)
Definition: ptc.py:372
def _makeZeroSafe(self, array, warn=True, substituteValue=1e-9)
Definition: ptc.py:596
def _fitLeastSq(self, initialParams, dataX, dataY, function)
Definition: ptc.py:426
def plot(self, dataRef, dataset, ptcFitType)
Definition: ptc.py:839