lsst.cp.pipe  19.0.0-15-g7d47663
ptc.py
Go to the documentation of this file.
1 # This file is part of cp_pipe.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 #
22 
23 __all__ = ['MeasurePhotonTransferCurveTask',
24  'MeasurePhotonTransferCurveTaskConfig',
25  'PhotonTransferCurveDataset']
26 
27 import numpy as np
28 import matplotlib.pyplot as plt
29 import os
30 from matplotlib.backends.backend_pdf import PdfPages
31 from sqlite3 import OperationalError
32 from collections import Counter
33 from dataclasses import dataclass
34 
35 import lsst.afw.math as afwMath
36 import lsst.pex.config as pexConfig
37 import lsst.pipe.base as pipeBase
38 from lsst.ip.isr import IsrTask
39 from .utils import (NonexistentDatasetTaskDataIdContainer, PairedVisitListTaskRunner,
40  checkExpLengthEqual, validateIsrConfig)
41 from scipy.optimize import leastsq, least_squares
42 import numpy.polynomial.polynomial as poly
43 
44 from lsst.ip.isr.linearize import Linearizer
45 import datetime
46 
47 
48 class MeasurePhotonTransferCurveTaskConfig(pexConfig.Config):
49  """Config class for photon transfer curve measurement task"""
50  isr = pexConfig.ConfigurableField(
51  target=IsrTask,
52  doc="""Task to perform instrumental signature removal.""",
53  )
54  isrMandatorySteps = pexConfig.ListField(
55  dtype=str,
56  doc="isr operations that must be performed for valid results. Raises if any of these are False.",
57  default=['doAssembleCcd']
58  )
59  isrForbiddenSteps = pexConfig.ListField(
60  dtype=str,
61  doc="isr operations that must NOT be performed for valid results. Raises if any of these are True",
62  default=['doFlat', 'doFringe', 'doBrighterFatter', 'doUseOpticsTransmission',
63  'doUseFilterTransmission', 'doUseSensorTransmission', 'doUseAtmosphereTransmission']
64  )
65  isrDesirableSteps = pexConfig.ListField(
66  dtype=str,
67  doc="isr operations that it is advisable to perform, but are not mission-critical." +
68  " WARNs are logged for any of these found to be False.",
69  default=['doBias', 'doDark', 'doCrosstalk', 'doDefect']
70  )
71  isrUndesirableSteps = pexConfig.ListField(
72  dtype=str,
73  doc="isr operations that it is *not* advisable to perform in the general case, but are not" +
74  " forbidden as some use-cases might warrant them." +
75  " WARNs are logged for any of these found to be True.",
76  default=['doLinearize']
77  )
78  ccdKey = pexConfig.Field(
79  dtype=str,
80  doc="The key by which to pull a detector from a dataId, e.g. 'ccd' or 'detector'.",
81  default='ccd',
82  )
83  makePlots = pexConfig.Field(
84  dtype=bool,
85  doc="Plot the PTC curves?",
86  default=False,
87  )
88  ptcFitType = pexConfig.ChoiceField(
89  dtype=str,
90  doc="Fit PTC to approximation in Astier+19 (Equation 16) or to a polynomial.",
91  default="POLYNOMIAL",
92  allowed={
93  "POLYNOMIAL": "n-degree polynomial (use 'polynomialFitDegree' to set 'n').",
94  "ASTIERAPPROXIMATION": "Approximation in Astier+19 (Eq. 16)."
95  }
96  )
97  polynomialFitDegree = pexConfig.Field(
98  dtype=int,
99  doc="Degree of polynomial to fit the PTC, when 'ptcFitType'=POLYNOMIAL.",
100  default=2,
101  )
102  polynomialFitDegreeNonLinearity = pexConfig.Field(
103  dtype=int,
104  doc="Degree of polynomial to fit the meanSignal vs exposureTime curve to produce" +
105  " the table for LinearizeLookupTable.",
106  default=3,
107  )
108  binSize = pexConfig.Field(
109  dtype=int,
110  doc="Bin the image by this factor in both dimensions.",
111  default=1,
112  )
113  minMeanSignal = pexConfig.Field(
114  dtype=float,
115  doc="Minimum value (inclusive) of mean signal (in DN) above which to consider.",
116  default=0,
117  )
118  maxMeanSignal = pexConfig.Field(
119  dtype=float,
120  doc="Maximum value (inclusive) of mean signal (in DN) below which to consider.",
121  default=9e6,
122  )
123  initialNonLinearityExclusionThresholdPositive = pexConfig.RangeField(
124  dtype=float,
125  doc="Initially exclude data points with a variance that are more than a factor of this from being"
126  " linear in the positive direction, from the PTC fit. Note that these points will also be"
127  " excluded from the non-linearity fit. This is done before the iterative outlier rejection,"
128  " to allow an accurate determination of the sigmas for said iterative fit.",
129  default=0.12,
130  min=0.0,
131  max=1.0,
132  )
133  initialNonLinearityExclusionThresholdNegative = pexConfig.RangeField(
134  dtype=float,
135  doc="Initially exclude data points with a variance that are more than a factor of this from being"
136  " linear in the negative direction, from the PTC fit. Note that these points will also be"
137  " excluded from the non-linearity fit. This is done before the iterative outlier rejection,"
138  " to allow an accurate determination of the sigmas for said iterative fit.",
139  default=0.25,
140  min=0.0,
141  max=1.0,
142  )
143  sigmaCutPtcOutliers = pexConfig.Field(
144  dtype=float,
145  doc="Sigma cut for outlier rejection in PTC.",
146  default=5.0,
147  )
148  maxIterationsPtcOutliers = pexConfig.Field(
149  dtype=int,
150  doc="Maximum number of iterations for outlier rejection in PTC.",
151  default=2,
152  )
153  doFitBootstrap = pexConfig.Field(
154  dtype=bool,
155  doc="Use bootstrap for the PTC fit parameters and errors?.",
156  default=False,
157  )
158  linResidualTimeIndex = pexConfig.Field(
159  dtype=int,
160  doc="Index position in time array for reference time in linearity residual calculation.",
161  default=2,
162  )
163  maxAduForLookupTableLinearizer = pexConfig.Field(
164  dtype=int,
165  doc="Maximum DN value for the LookupTable linearizer.",
166  default=2**18,
167  )
168  instrumentName = pexConfig.Field(
169  dtype=str,
170  doc="Instrument name.",
171  default='',
172  )
173 
174 
175 @dataclass
177  """A simple class to hold the output from the
178  `calculateLinearityResidualAndLinearizers` function.
179  """
180  # Normalized coefficients for polynomial NL correction
181  polynomialLinearizerCoefficients: list
182  # Normalized coefficient for quadratic polynomial NL correction (c0)
183  quadraticPolynomialLinearizerCoefficient: float
184  # LUT array row for the amplifier at hand
185  linearizerTableRow: list
186  # Linearity residual, Eq. 2.2. of Janesick (2001)
187  linearityResidual: list
188  meanSignalVsTimePolyFitPars: list
189  meanSignalVsTimePolyFitParsErr: list
190  fractionalNonLinearityResidual: list
191  meanSignalVsTimePolyFitReducedChiSq: float
192 
193 
195  """A simple class to hold the output data from the PTC task.
196 
197  The dataset is made up of a dictionary for each item, keyed by the
198  amplifiers' names, which much be supplied at construction time.
199 
200  New items cannot be added to the class to save accidentally saving to the
201  wrong property, and the class can be frozen if desired.
202 
203  inputVisitPairs records the visits used to produce the data.
204  When fitPtcAndNonLinearity() is run, a mask is built up, which is by definition
205  always the same length as inputVisitPairs, rawExpTimes, rawMeans
206  and rawVars, and is a list of bools, which are incrementally set to False
207  as points are discarded from the fits.
208 
209  PTC fit parameters for polynomials are stored in a list in ascending order
210  of polynomial term, i.e. par[0]*x^0 + par[1]*x + par[2]*x^2 etc
211  with the length of the list corresponding to the order of the polynomial
212  plus one.
213  """
214  def __init__(self, ampNames):
215  # add items to __dict__ directly because __setattr__ is overridden
216 
217  # instance variables
218  self.__dict__["ampNames"] = ampNames
219  self.__dict__["badAmps"] = []
220 
221  # raw data variables
222  self.__dict__["inputVisitPairs"] = {ampName: [] for ampName in ampNames}
223  self.__dict__["visitMask"] = {ampName: [] for ampName in ampNames}
224  self.__dict__["rawExpTimes"] = {ampName: [] for ampName in ampNames}
225  self.__dict__["rawMeans"] = {ampName: [] for ampName in ampNames}
226  self.__dict__["rawVars"] = {ampName: [] for ampName in ampNames}
227 
228  # fit information
229  self.__dict__["ptcFitType"] = {ampName: "" for ampName in ampNames}
230  self.__dict__["ptcFitPars"] = {ampName: [] for ampName in ampNames}
231  self.__dict__["ptcFitParsError"] = {ampName: [] for ampName in ampNames}
232  self.__dict__["ptcFitReducedChiSquared"] = {ampName: [] for ampName in ampNames}
233  self.__dict__["nonLinearity"] = {ampName: [] for ampName in ampNames}
234  self.__dict__["nonLinearityError"] = {ampName: [] for ampName in ampNames}
235  self.__dict__["nonLinearityResiduals"] = {ampName: [] for ampName in ampNames}
236  self.__dict__["fractionalNonLinearityResiduals"] = {ampName: [] for ampName in ampNames}
237  self.__dict__["nonLinearityReducedChiSquared"] = {ampName: [] for ampName in ampNames}
238 
239  # final results
240  self.__dict__["gain"] = {ampName: -1. for ampName in ampNames}
241  self.__dict__["gainErr"] = {ampName: -1. for ampName in ampNames}
242  self.__dict__["noise"] = {ampName: -1. for ampName in ampNames}
243  self.__dict__["noiseErr"] = {ampName: -1. for ampName in ampNames}
244  self.__dict__["coefficientsLinearizePolynomial"] = {ampName: [] for ampName in ampNames}
245  self.__dict__["coefficientLinearizeSquared"] = {ampName: [] for ampName in ampNames}
246 
247  def __setattr__(self, attribute, value):
248  """Protect class attributes"""
249  if attribute not in self.__dict__:
250  raise AttributeError(f"{attribute} is not already a member of PhotonTransferCurveDataset, which"
251  " does not support setting of new attributes.")
252  else:
253  self.__dict__[attribute] = value
254 
255  def getVisitsUsed(self, ampName):
256  """Get the visits used, i.e. not discarded, for a given amp.
257 
258  If no mask has been created yet, all visits are returned.
259  """
260  if self.visitMask[ampName] == []:
261  return self.inputVisitPairs[ampName]
262 
263  # if the mask exists it had better be the same length as the visitPairs
264  assert len(self.visitMask[ampName]) == len(self.inputVisitPairs[ampName])
265 
266  pairs = self.inputVisitPairs[ampName]
267  mask = self.visitMask[ampName]
268  # cast to bool required because numpy
269  return [(v1, v2) for ((v1, v2), m) in zip(pairs, mask) if bool(m) is True]
270 
271  def getGoodAmps(self):
272  return [amp for amp in self.ampNames if amp not in self.badAmps]
273 
274 
275 class MeasurePhotonTransferCurveTask(pipeBase.CmdLineTask):
276  """A class to calculate, fit, and plot a PTC from a set of flat pairs.
277 
278  The Photon Transfer Curve (var(signal) vs mean(signal)) is a standard tool
279  used in astronomical detectors characterization (e.g., Janesick 2001,
280  Janesick 2007). This task calculates the PTC from a series of pairs of
281  flat-field images; each pair taken at identical exposure times. The
282  difference image of each pair is formed to eliminate fixed pattern noise,
283  and then the variance of the difference image and the mean of the average image
284  are used to produce the PTC. An n-degree polynomial or the approximation in Equation
285  16 of Astier+19 ("The Shape of the Photon Transfer Curve of CCD sensors",
286  arXiv:1905.08677) can be fitted to the PTC curve. These models include
287  parameters such as the gain (e/DN) and readout noise.
288 
289  Linearizers to correct for signal-chain non-linearity are also calculated.
290  The `Linearizer` class, in general, can support per-amp linearizers, but in this
291  task this is not supported.
292  Parameters
293  ----------
294 
295  *args: `list`
296  Positional arguments passed to the Task constructor. None used at this
297  time.
298  **kwargs: `dict`
299  Keyword arguments passed on to the Task constructor. None used at this
300  time.
301 
302  """
303 
304  RunnerClass = PairedVisitListTaskRunner
305  ConfigClass = MeasurePhotonTransferCurveTaskConfig
306  _DefaultName = "measurePhotonTransferCurve"
307 
308  def __init__(self, *args, **kwargs):
309  pipeBase.CmdLineTask.__init__(self, *args, **kwargs)
310  self.makeSubtask("isr")
311  plt.interactive(False) # stop windows popping up when plotting. When headless, use 'agg' backend too
312  validateIsrConfig(self.isr, self.config.isrMandatorySteps,
313  self.config.isrForbiddenSteps, self.config.isrDesirableSteps, checkTrim=False)
314  self.config.validate()
315  self.config.freeze()
316 
317  @classmethod
318  def _makeArgumentParser(cls):
319  """Augment argument parser for the MeasurePhotonTransferCurveTask."""
320  parser = pipeBase.ArgumentParser(name=cls._DefaultName)
321  parser.add_argument("--visit-pairs", dest="visitPairs", nargs="*",
322  help="Visit pairs to use. Each pair must be of the form INT,INT e.g. 123,456")
323  parser.add_id_argument("--id", datasetType="photonTransferCurveDataset",
324  ContainerClass=NonexistentDatasetTaskDataIdContainer,
325  help="The ccds to use, e.g. --id ccd=0..100")
326  return parser
327 
328  @pipeBase.timeMethod
329  def runDataRef(self, dataRef, visitPairs):
330  """Run the Photon Transfer Curve (PTC) measurement task.
331 
332  For a dataRef (which is each detector here),
333  and given a list of visit pairs at different exposure times,
334  measure the PTC.
335 
336  Parameters
337  ----------
338  dataRef : list of lsst.daf.persistence.ButlerDataRef
339  dataRef for the detector for the visits to be fit.
340  visitPairs : `iterable` of `tuple` of `int`
341  Pairs of visit numbers to be processed together
342  """
343 
344  # setup necessary objects
345  detNum = dataRef.dataId[self.config.ccdKey]
346  detector = dataRef.get('camera')[dataRef.dataId[self.config.ccdKey]]
347  # expand some missing fields that we need for lsstCam. This is a work-around
348  # for Gen2 problems that I (RHL) don't feel like solving. The calibs pipelines
349  # (which inherit from CalibTask) use addMissingKeys() to do basically the same thing
350  #
351  # Basically, the butler's trying to look up the fields in `raw_visit` which won't work
352  for name in dataRef.getButler().getKeys('bias'):
353  if name not in dataRef.dataId:
354  try:
355  dataRef.dataId[name] = \
356  dataRef.getButler().queryMetadata('raw', [name], detector=detNum)[0]
357  except OperationalError:
358  pass
359 
360  amps = detector.getAmplifiers()
361  ampNames = [amp.getName() for amp in amps]
362  dataset = PhotonTransferCurveDataset(ampNames)
363 
364  self.log.info('Measuring PTC using %s visits for detector %s' % (visitPairs, detNum))
365 
366  for (v1, v2) in visitPairs:
367  # Perform ISR on each exposure
368  dataRef.dataId['expId'] = v1
369  exp1 = self.isr.runDataRef(dataRef).exposure
370  dataRef.dataId['expId'] = v2
371  exp2 = self.isr.runDataRef(dataRef).exposure
372  del dataRef.dataId['expId']
373 
374  checkExpLengthEqual(exp1, exp2, v1, v2, raiseWithMessage=True)
375  expTime = exp1.getInfo().getVisitInfo().getExposureTime()
376 
377  for amp in detector:
378  mu, varDiff = self.measureMeanVarPair(exp1, exp2, region=amp.getBBox())
379  ampName = amp.getName()
380 
381  dataset.rawExpTimes[ampName].append(expTime)
382  dataset.rawMeans[ampName].append(mu)
383  dataset.rawVars[ampName].append(varDiff)
384  dataset.inputVisitPairs[ampName].append((v1, v2))
385 
386  numberAmps = len(detector.getAmplifiers())
387  numberAduValues = self.config.maxAduForLookupTableLinearizer
388  lookupTableArray = np.zeros((numberAmps, numberAduValues), dtype=np.float32)
389 
390  # Fit PTC and (non)linearity of signal vs time curve.
391  # Fill up PhotonTransferCurveDataset object.
392  # Fill up array for LUT linearizer.
393  # Produce coefficients for Polynomial ans Squared linearizers.
394  dataset = self.fitPtcAndNonLinearity(dataset, self.config.ptcFitType,
395  tableArray=lookupTableArray)
396 
397  if self.config.makePlots:
398  self.plot(dataRef, dataset, ptcFitType=self.config.ptcFitType)
399 
400  # Save data, PTC fit, and NL fit dictionaries
401  self.log.info(f"Writing PTC and NL data to {dataRef.getUri(write=True)}")
402  dataRef.put(dataset, datasetType="photonTransferCurveDataset")
403 
404  butler = dataRef.getButler()
405  self.log.info(f"Writing linearizers: \n "
406  "lookup table (linear component of polynomial fit), \n "
407  "polynomial (coefficients for a polynomial correction), \n "
408  "and squared linearizer (quadratic coefficient from polynomial)")
409 
410  detName = detector.getName()
411  now = datetime.datetime.utcnow()
412  calibDate = now.strftime("%Y-%m-%d")
413 
414  for linType, dataType in [("LOOKUPTABLE", 'linearizeLut'),
415  ("LINEARIZEPOLYNOMIAL", 'linearizePolynomial'),
416  ("LINEARIZESQUARED", 'linearizeSquared')]:
417 
418  if linType == "LOOKUPTABLE":
419  tableArray = lookupTableArray
420  else:
421  tableArray = None
422 
423  linearizer = self.buildLinearizerObject(dataset, detector, calibDate, linType,
424  instruName=self.config.instrumentName,
425  tableArray=tableArray,
426  log=self.log)
427  butler.put(linearizer, datasetType=dataType, dataId={'detector': detNum,
428  'detectorName': detName, 'calibDate': calibDate})
429 
430  self.log.info('Finished measuring PTC for in detector %s' % detNum)
431 
432  return pipeBase.Struct(exitStatus=0)
433 
434  def buildLinearizerObject(self, dataset, detector, calibDate, linearizerType, instruName='',
435  tableArray=None, log=None):
436  """Build linearizer object to persist.
437 
438  Parameters
439  ----------
440  dataset : `lsst.cp.pipe.ptc.PhotonTransferCurveDataset`
441  The dataset containing the means, variances, and exposure times
442  detector : `lsst.afw.cameraGeom.Detector`
443  Detector object
444  calibDate : `datetime.datetime`
445  Calibration date
446  linearizerType : `str`
447  'LOOKUPTABLE', 'LINEARIZESQUARED', or 'LINEARIZEPOLYNOMIAL'
448  instruName : `str`, optional
449  Instrument name
450  tableArray : `np.array`, optional
451  Look-up table array with size rows=nAmps and columns=DN values
452  log : `lsst.log.Log`, optional
453  Logger to handle messages
454 
455  Returns
456  -------
457  linearizer : `lsst.ip.isr.Linearizer`
458  Linearizer object
459  """
460  detName = detector.getName()
461  detNum = detector.getId()
462  if linearizerType == "LOOKUPTABLE":
463  if tableArray is not None:
464  linearizer = Linearizer(detector=detector, table=tableArray, log=log)
465  else:
466  raise RuntimeError("tableArray must be provided when creating a LookupTable linearizer")
467  elif linearizerType in ("LINEARIZESQUARED", "LINEARIZEPOLYNOMIAL"):
468  linearizer = Linearizer(log=log)
469  else:
470  raise RuntimeError("Invalid linearizerType {linearizerType} to build a Linearizer object. "
471  "Supported: 'LOOKUPTABLE', 'LINEARIZESQUARED', or 'LINEARIZEPOLYNOMIAL'")
472  for i, amp in enumerate(detector.getAmplifiers()):
473  ampName = amp.getName()
474  if linearizerType == "LOOKUPTABLE":
475  linearizer.linearityCoeffs[ampName] = [i, 0]
476  linearizer.linearityType[ampName] = "LookupTable"
477  elif linearizerType == "LINEARIZESQUARED":
478  linearizer.linearityCoeffs[ampName] = [dataset.coefficientLinearizeSquared[ampName]]
479  linearizer.linearityType[ampName] = "Squared"
480  elif linearizerType == "LINEARIZEPOLYNOMIAL":
481  linearizer.linearityCoeffs[ampName] = dataset.coefficientsLinearizePolynomial[ampName]
482  linearizer.linearityType[ampName] = "Polynomial"
483  linearizer.linearityBBox[ampName] = amp.getBBox()
484 
485  linearizer.validate()
486  calibId = f"detectorName={detName} detector={detNum} calibDate={calibDate} ccd={detNum} filter=NONE"
487 
488  try:
489  raftName = detName.split("_")[0]
490  calibId += f" raftName={raftName}"
491  except Exception:
492  raftname = "NONE"
493  calibId += f" raftName={raftname}"
494 
495  serial = detector.getSerial()
496  linearizer.updateMetadata(instrumentName=instruName, detectorId=f"{detNum}",
497  calibId=calibId, serial=serial, detectorName=f"{detName}")
498 
499  return linearizer
500 
501  def measureMeanVarPair(self, exposure1, exposure2, region=None):
502  """Calculate the mean signal of two exposures and the variance of their difference.
503 
504  Parameters
505  ----------
506  exposure1 : `lsst.afw.image.exposure.exposure.ExposureF`
507  First exposure of flat field pair.
508 
509  exposure2 : `lsst.afw.image.exposure.exposure.ExposureF`
510  Second exposure of flat field pair.
511 
512  region : `lsst.geom.Box2I`
513  Region of each exposure where to perform the calculations (e.g, an amplifier).
514 
515  Return
516  ------
517 
518  mu : `float`
519  0.5*(mu1 + mu2), where mu1, and mu2 are the clipped means of the regions in
520  both exposures.
521 
522  varDiff : `float`
523  Half of the clipped variance of the difference of the regions inthe two input
524  exposures.
525  """
526 
527  if region is not None:
528  im1Area = exposure1.maskedImage[region]
529  im2Area = exposure2.maskedImage[region]
530  else:
531  im1Area = exposure1.maskedImage
532  im2Area = exposure2.maskedImage
533 
534  im1Area = afwMath.binImage(im1Area, self.config.binSize)
535  im2Area = afwMath.binImage(im2Area, self.config.binSize)
536 
537  # Clipped mean of images; then average of mean.
538  mu1 = afwMath.makeStatistics(im1Area, afwMath.MEANCLIP).getValue()
539  mu2 = afwMath.makeStatistics(im2Area, afwMath.MEANCLIP).getValue()
540  mu = 0.5*(mu1 + mu2)
541 
542  # Take difference of pairs
543  # symmetric formula: diff = (mu2*im1-mu1*im2)/(0.5*(mu1+mu2))
544  temp = im2Area.clone()
545  temp *= mu1
546  diffIm = im1Area.clone()
547  diffIm *= mu2
548  diffIm -= temp
549  diffIm /= mu
550 
551  varDiff = 0.5*(afwMath.makeStatistics(diffIm, afwMath.VARIANCECLIP).getValue())
552 
553  return mu, varDiff
554 
555  def _fitLeastSq(self, initialParams, dataX, dataY, function):
556  """Do a fit and estimate the parameter errors using using scipy.optimize.leastq.
557 
558  optimize.leastsq returns the fractional covariance matrix. To estimate the
559  standard deviation of the fit parameters, multiply the entries of this matrix
560  by the unweighted reduced chi squared and take the square root of the diagonal elements.
561 
562  Parameters
563  ----------
564  initialParams : `list` of `float`
565  initial values for fit parameters. For ptcFitType=POLYNOMIAL, its length
566  determines the degree of the polynomial.
567 
568  dataX : `numpy.array` of `float`
569  Data in the abscissa axis.
570 
571  dataY : `numpy.array` of `float`
572  Data in the ordinate axis.
573 
574  function : callable object (function)
575  Function to fit the data with.
576 
577  Return
578  ------
579  pFitSingleLeastSquares : `list` of `float`
580  List with fitted parameters.
581 
582  pErrSingleLeastSquares : `list` of `float`
583  List with errors for fitted parameters.
584 
585  reducedChiSqSingleLeastSquares : `float`
586  Unweighted reduced chi squared
587  """
588 
589  def errFunc(p, x, y):
590  return function(p, x) - y
591 
592  pFit, pCov, infoDict, errMessage, success = leastsq(errFunc, initialParams,
593  args=(dataX, dataY), full_output=1, epsfcn=0.0001)
594 
595  if (len(dataY) > len(initialParams)) and pCov is not None:
596  reducedChiSq = (errFunc(pFit, dataX, dataY)**2).sum()/(len(dataY)-len(initialParams))
597  pCov *= reducedChiSq
598  else:
599  pCov = np.zeros((len(initialParams), len(initialParams)))
600  pCov[:, :] = np.inf
601  reducedChiSq = np.inf
602 
603  errorVec = []
604  for i in range(len(pFit)):
605  errorVec.append(np.fabs(pCov[i][i])**0.5)
606 
607  pFitSingleLeastSquares = pFit
608  pErrSingleLeastSquares = np.array(errorVec)
609 
610  return pFitSingleLeastSquares, pErrSingleLeastSquares, reducedChiSq
611 
612  def _fitBootstrap(self, initialParams, dataX, dataY, function, confidenceSigma=1.):
613  """Do a fit using least squares and bootstrap to estimate parameter errors.
614 
615  The bootstrap error bars are calculated by fitting 100 random data sets.
616 
617  Parameters
618  ----------
619  initialParams : `list` of `float`
620  initial values for fit parameters. For ptcFitType=POLYNOMIAL, its length
621  determines the degree of the polynomial.
622 
623  dataX : `numpy.array` of `float`
624  Data in the abscissa axis.
625 
626  dataY : `numpy.array` of `float`
627  Data in the ordinate axis.
628 
629  function : callable object (function)
630  Function to fit the data with.
631 
632  confidenceSigma : `float`
633  Number of sigmas that determine confidence interval for the bootstrap errors.
634 
635  Return
636  ------
637  pFitBootstrap : `list` of `float`
638  List with fitted parameters.
639 
640  pErrBootstrap : `list` of `float`
641  List with errors for fitted parameters.
642 
643  reducedChiSqBootstrap : `float`
644  Reduced chi squared.
645  """
646 
647  def errFunc(p, x, y):
648  return function(p, x) - y
649 
650  # Fit first time
651  pFit, _ = leastsq(errFunc, initialParams, args=(dataX, dataY), full_output=0)
652 
653  # Get the stdev of the residuals
654  residuals = errFunc(pFit, dataX, dataY)
655  sigmaErrTotal = np.std(residuals)
656 
657  # 100 random data sets are generated and fitted
658  pars = []
659  for i in range(100):
660  randomDelta = np.random.normal(0., sigmaErrTotal, len(dataY))
661  randomDataY = dataY + randomDelta
662  randomFit, _ = leastsq(errFunc, initialParams,
663  args=(dataX, randomDataY), full_output=0)
664  pars.append(randomFit)
665  pars = np.array(pars)
666  meanPfit = np.mean(pars, 0)
667 
668  # confidence interval for parameter estimates
669  nSigma = confidenceSigma
670  errPfit = nSigma*np.std(pars, 0)
671  pFitBootstrap = meanPfit
672  pErrBootstrap = errPfit
673 
674  reducedChiSq = (errFunc(pFitBootstrap, dataX, dataY)**2).sum()/(len(dataY)-len(initialParams))
675  return pFitBootstrap, pErrBootstrap, reducedChiSq
676 
677  def funcPolynomial(self, pars, x):
678  """Polynomial function definition"""
679  return poly.polyval(x, [*pars])
680 
681  def funcAstier(self, pars, x):
682  """Single brighter-fatter parameter model for PTC; Equation 16 of Astier+19"""
683  a00, gain, noise = pars
684  return 0.5/(a00*gain*gain)*(np.exp(2*a00*x*gain)-1) + noise/(gain*gain)
685 
686  @staticmethod
687  def _initialParsForPolynomial(order):
688  assert(order >= 2)
689  pars = np.zeros(order, dtype=np.float)
690  pars[0] = 10
691  pars[1] = 1
692  pars[2:] = 0.0001
693  return pars
694 
695  @staticmethod
696  def _boundsForPolynomial(initialPars):
697  lowers = [np.NINF for p in initialPars]
698  uppers = [np.inf for p in initialPars]
699  lowers[1] = 0 # no negative gains
700  return (lowers, uppers)
701 
702  @staticmethod
703  def _boundsForAstier(initialPars):
704  lowers = [np.NINF for p in initialPars]
705  uppers = [np.inf for p in initialPars]
706  return (lowers, uppers)
707 
708  @staticmethod
709  def _getInitialGoodPoints(means, variances, maxDeviationPositive, maxDeviationNegative):
710  """Return a boolean array to mask bad points.
711 
712  A linear function has a constant ratio, so find the median
713  value of the ratios, and exclude the points that deviate
714  from that by more than a factor of maxDeviationPositive/negative.
715  Asymmetric deviations are supported as we expect the PTC to turn
716  down as the flux increases, but sometimes it anomalously turns
717  upwards just before turning over, which ruins the fits, so it
718  is wise to be stricter about restricting positive outliers than
719  negative ones.
720 
721  Too high and points that are so bad that fit will fail will be included
722  Too low and the non-linear points will be excluded, biasing the NL fit."""
723  ratios = [b/a for (a, b) in zip(means, variances)]
724  medianRatio = np.median(ratios)
725  ratioDeviations = [(r/medianRatio)-1 for r in ratios]
726 
727  # so that it doesn't matter if the deviation is expressed as positive or negative
728  maxDeviationPositive = abs(maxDeviationPositive)
729  maxDeviationNegative = -1. * abs(maxDeviationNegative)
730 
731  goodPoints = np.array([True if (r < maxDeviationPositive and r > maxDeviationNegative)
732  else False for r in ratioDeviations])
733  return goodPoints
734 
735  def _makeZeroSafe(self, array, warn=True, substituteValue=1e-9):
736  """"""
737  nBad = Counter(array)[0]
738  if nBad == 0:
739  return array
740 
741  if warn:
742  msg = f"Found {nBad} zeros in array at elements {[x for x in np.where(array==0)[0]]}"
743  self.log.warn(msg)
744 
745  array[array == 0] = substituteValue
746  return array
747 
748  def calculateLinearityResidualAndLinearizers(self, exposureTimeVector, meanSignalVector):
749  """Calculate linearity residual and fit an n-order polynomial to the mean vs time curve
750  to produce corrections (deviation from linear part of polynomial) for a particular amplifier
751  to populate LinearizeLookupTable.
752  Use the coefficients of this fit to calculate the correction coefficients for LinearizePolynomial
753  and LinearizeSquared."
754 
755  Parameters
756  ---------
757 
758  exposureTimeVector: `list` of `float`
759  List of exposure times for each flat pair
760 
761  meanSignalVector: `list` of `float`
762  List of mean signal from diference image of flat pairs
763 
764  Returns
765  -------
766  dataset : `lsst.cp.pipe.ptc.LinearityResidualsAndLinearizersDataset`
767  The dataset containing the fit parameters, the NL correction coefficients, and the
768  LUT row for the amplifier at hand. Explicitly:
769 
770  dataset.polynomialLinearizerCoefficients : `list` of `float`
771  Coefficients for LinearizePolynomial, where corrImage = uncorrImage + sum_i c_i uncorrImage^(2 +
772  i).
773  c_(j-2) = -k_j/(k_1^j) with units (DN^(1-j)). The units of k_j are DN/t^j, and they are fit from
774  meanSignalVector = k0 + k1*exposureTimeVector + k2*exposureTimeVector^2 +...
775  + kn*exposureTimeVector^n, with n = "polynomialFitDegreeNonLinearity".
776  k_0 and k_1 and degenerate with bias level and gain, and are not used by the non-linearity
777  correction. Therefore, j = 2...n in the above expression (see `LinearizePolynomial` class in
778  `linearize.py`.)
779 
780  dataset.quadraticPolynomialLinearizerCoefficient : `float`
781  Coefficient for LinearizeSquared, where corrImage = uncorrImage + c0*uncorrImage^2.
782  c0 = -k2/(k1^2), where k1 and k2 are fit from
783  meanSignalVector = k0 + k1*exposureTimeVector + k2*exposureTimeVector^2 +...
784  + kn*exposureTimeVector^n, with n = "polynomialFitDegreeNonLinearity".
785 
786  dataset.linearizerTableRow : `list` of `float`
787  One dimensional array with deviation from linear part of n-order polynomial fit
788  to mean vs time curve. This array will be one row (for the particular amplifier at hand)
789  of the table array for LinearizeLookupTable.
790 
791  dataset.linearityResidual : `list` of `float`
792  Linearity residual from the mean vs time curve, defined as
793  100*(1 - meanSignalReference/expTimeReference/(meanSignal/expTime).
794 
795  dataset.meanSignalVsTimePolyFitPars : `list` of `float`
796  Parameters from n-order polynomial fit to meanSignalVector vs exposureTimeVector.
797 
798  dataset.meanSignalVsTimePolyFitParsErr : `list` of `float`
799  Parameters from n-order polynomial fit to meanSignalVector vs exposureTimeVector.
800 
801  dataset.fractionalNonLinearityResidual : `list` of `float`
802  Fractional residuals from the meanSignal vs exposureTime curve with respect to linear part of
803  polynomial fit: 100*(linearPart - meanSignal)/linearPart, where
804  linearPart = k0 + k1*exposureTimeVector.
805 
806  dataset.meanSignalVsTimePolyFitReducedChiSq : `float`
807  Reduced unweighted chi squared from polynomial fit to meanSignalVector vs exposureTimeVector.
808  """
809 
810  # Lookup table linearizer
811  parsIniNonLinearity = self._initialParsForPolynomial(self.config.polynomialFitDegreeNonLinearity + 1)
812  if self.config.doFitBootstrap:
813  parsFit, parsFitErr, reducedChiSquaredNonLinearityFit = self._fitBootstrap(parsIniNonLinearity,
814  exposureTimeVector,
815  meanSignalVector,
816  self.funcPolynomial)
817  else:
818  parsFit, parsFitErr, reducedChiSquaredNonLinearityFit = self._fitLeastSq(parsIniNonLinearity,
819  exposureTimeVector,
820  meanSignalVector,
821  self.funcPolynomial)
822 
823  # LinearizeLookupTable:
824  # Use linear part to get time at wich signal is maxAduForLookupTableLinearizer DN
825  tMax = (self.config.maxAduForLookupTableLinearizer - parsFit[0])/parsFit[1]
826  timeRange = np.linspace(0, tMax, self.config.maxAduForLookupTableLinearizer)
827  signalIdeal = parsFit[0] + parsFit[1]*timeRange
828  signalUncorrected = self.funcPolynomial(parsFit, timeRange)
829  linearizerTableRow = signalIdeal - signalUncorrected # LinearizerLookupTable has corrections
830  # LinearizePolynomial and LinearizeSquared:
831  # Check that magnitude of higher order (>= 3) coefficents of the polyFit are small,
832  # i.e., less than threshold = 1e-10 (typical quadratic and cubic coefficents are ~1e-6
833  # and ~1e-12).
834  k1 = parsFit[1]
835  polynomialLinearizerCoefficients = []
836  for i, coefficient in enumerate(parsFit):
837  c = -coefficient/(k1**i)
838  polynomialLinearizerCoefficients.append(c)
839  if np.fabs(c) > 1e-10:
840  msg = f"Coefficient {c} in polynomial fit larger than threshold 1e-10."
841  self.log.warn(msg)
842  # Coefficient for LinearizedSquared. Called "c0" in linearize.py
843  c0 = polynomialLinearizerCoefficients[2]
844 
845  # Linearity residual
846  linResidualTimeIndex = self.config.linResidualTimeIndex
847  if exposureTimeVector[linResidualTimeIndex] == 0.0:
848  raise RuntimeError("Reference time for linearity residual can't be 0.0")
849  linResidual = 100*(1 - ((meanSignalVector[linResidualTimeIndex] /
850  exposureTimeVector[linResidualTimeIndex]) /
851  (meanSignalVector/exposureTimeVector)))
852 
853  # Fractional non-linearity residual, w.r.t linear part of polynomial fit
854  linearPart = parsFit[0] + k1*exposureTimeVector
855  fracNonLinearityResidual = 100*(linearPart - meanSignalVector)/linearPart
856 
857  dataset = LinearityResidualsAndLinearizersDataset([], None, [], [], [], [], [], None)
858  dataset.polynomialLinearizerCoefficients = polynomialLinearizerCoefficients
859  dataset.quadraticPolynomialLinearizerCoefficient = c0
860  dataset.linearizerTableRow = linearizerTableRow
861  dataset.linearityResidual = linResidual
862  dataset.meanSignalVsTimePolyFitPars = parsFit
863  dataset.meanSignalVsTimePolyFitParsErr = parsFitErr
864  dataset.fractionalNonLinearityResidual = fracNonLinearityResidual
865  dataset.meanSignalVsTimePolyFitReducedChiSq = reducedChiSquaredNonLinearityFit
866 
867  return dataset
868 
869  def fitPtcAndNonLinearity(self, dataset, ptcFitType, tableArray=None):
870  """Fit the photon transfer curve and calculate linearity and residuals.
871 
872  Fit the photon transfer curve with either a polynomial of the order
873  specified in the task config, or using the Astier approximation.
874 
875  Sigma clipping is performed iteratively for the fit, as well as an
876  initial clipping of data points that are more than
877  config.initialNonLinearityExclusionThreshold away from lying on a
878  straight line. This other step is necessary because the photon transfer
879  curve turns over catastrophically at very high flux (because saturation
880  drops the variance to ~0) and these far outliers cause the initial fit
881  to fail, meaning the sigma cannot be calculated to perform the
882  sigma-clipping.
883 
884  Parameters
885  ----------
886  dataset : `lsst.cp.pipe.ptc.PhotonTransferCurveDataset`
887  The dataset containing the means, variances and exposure times
888  ptcFitType : `str`
889  Fit a 'POLYNOMIAL' (degree: 'polynomialFitDegree') or
890  'ASTIERAPPROXIMATION' to the PTC
891  tableArray : `np.array`
892  Optional. Look-up table array with size rows=nAmps and columns=DN values.
893  It will be modified in-place if supplied.
894 
895  Returns
896  -------
897  dataset: `lsst.cp.pipe.ptc.PhotonTransferCurveDataset`
898  This is the same dataset as the input paramter, however, it has been modified
899  to include information such as the fit vectors and the fit parameters. See
900  the class `PhotonTransferCurveDatase`.
901  """
902 
903  def errFunc(p, x, y):
904  return ptcFunc(p, x) - y
905 
906  sigmaCutPtcOutliers = self.config.sigmaCutPtcOutliers
907  maxIterationsPtcOutliers = self.config.maxIterationsPtcOutliers
908 
909  for i, ampName in enumerate(dataset.ampNames):
910  timeVecOriginal = np.array(dataset.rawExpTimes[ampName])
911  meanVecOriginal = np.array(dataset.rawMeans[ampName])
912  varVecOriginal = np.array(dataset.rawVars[ampName])
913  varVecOriginal = self._makeZeroSafe(varVecOriginal)
914 
915  mask = ((meanVecOriginal >= self.config.minMeanSignal) &
916  (meanVecOriginal <= self.config.maxMeanSignal))
917 
918  goodPoints = self._getInitialGoodPoints(meanVecOriginal, varVecOriginal,
919  self.config.initialNonLinearityExclusionThresholdPositive,
920  self.config.initialNonLinearityExclusionThresholdNegative)
921  mask = mask & goodPoints
922 
923  if ptcFitType == 'ASTIERAPPROXIMATION':
924  ptcFunc = self.funcAstier
925  parsIniPtc = [-1e-9, 1.0, 10.] # a00, gain, noise
926  bounds = self._boundsForAstier(parsIniPtc)
927  if ptcFitType == 'POLYNOMIAL':
928  ptcFunc = self.funcPolynomial
929  parsIniPtc = self._initialParsForPolynomial(self.config.polynomialFitDegree + 1)
930  bounds = self._boundsForPolynomial(parsIniPtc)
931 
932  # Before bootstrap fit, do an iterative fit to get rid of outliers
933  count = 1
934  while count <= maxIterationsPtcOutliers:
935  # Note that application of the mask actually shrinks the array
936  # to size rather than setting elements to zero (as we want) so
937  # always update mask itself and re-apply to the original data
938  meanTempVec = meanVecOriginal[mask]
939  varTempVec = varVecOriginal[mask]
940  res = least_squares(errFunc, parsIniPtc, bounds=bounds, args=(meanTempVec, varTempVec))
941  pars = res.x
942 
943  # change this to the original from the temp because the masks are ANDed
944  # meaning once a point is masked it's always masked, and the masks must
945  # always be the same length for broadcasting
946  sigResids = (varVecOriginal - ptcFunc(pars, meanVecOriginal))/np.sqrt(varVecOriginal)
947  newMask = np.array([True if np.abs(r) < sigmaCutPtcOutliers else False for r in sigResids])
948  mask = mask & newMask
949 
950  nDroppedTotal = Counter(mask)[False]
951  self.log.debug(f"Iteration {count}: discarded {nDroppedTotal} points in total for {ampName}")
952  count += 1
953  # objects should never shrink
954  assert (len(mask) == len(timeVecOriginal) == len(meanVecOriginal) == len(varVecOriginal))
955 
956  dataset.visitMask[ampName] = mask # store the final mask
957 
958  parsIniPtc = pars
959  timeVecFinal = timeVecOriginal[mask]
960  meanVecFinal = meanVecOriginal[mask]
961  varVecFinal = varVecOriginal[mask]
962 
963  if Counter(mask)[False] > 0:
964  self.log.info((f"Number of points discarded in PTC of amplifier {ampName}:" +
965  f" {Counter(mask)[False]} out of {len(meanVecOriginal)}"))
966 
967  if (len(meanVecFinal) < len(parsIniPtc)):
968  msg = (f"\nSERIOUS: Not enough data points ({len(meanVecFinal)}) compared to the number of"
969  f"parameters of the PTC model({len(parsIniPtc)}). Setting {ampName} to BAD.")
970  self.log.warn(msg)
971  # The first and second parameters of initial fit are discarded (bias and gain)
972  # for the final NL coefficients
973  lenNonLinPars = self.config.polynomialFitDegreeNonLinearity - 1
974  dataset.badAmps.append(ampName)
975  dataset.gain[ampName] = np.nan
976  dataset.gainErr[ampName] = np.nan
977  dataset.noise[ampName] = np.nan
978  dataset.noiseErr[ampName] = np.nan
979  dataset.nonLinearity[ampName] = np.nan
980  dataset.nonLinearityError[ampName] = np.nan
981  dataset.nonLinearityResiduals[ampName] = np.nan
982  dataset.fractionalNonLinearityResiduals[ampName] = np.nan
983  dataset.coefficientLinearizeSquared[ampName] = np.nan
984  dataset.ptcFitPars[ampName] = np.nan
985  dataset.ptcFitParsError[ampName] = np.nan
986  dataset.ptcFitReducedChiSquared[ampName] = np.nan
987  dataset.coefficientsLinearizePolynomial[ampName] = [np.nan]*lenNonLinPars
988  tableArray[i, :] = [np.nan]*self.config.maxAduForLookupTableLinearizer
989  continue
990 
991  # Fit the PTC
992  if self.config.doFitBootstrap:
993  parsFit, parsFitErr, reducedChiSqPtc = self._fitBootstrap(parsIniPtc, meanVecFinal,
994  varVecFinal, ptcFunc)
995  else:
996  parsFit, parsFitErr, reducedChiSqPtc = self._fitLeastSq(parsIniPtc, meanVecFinal,
997  varVecFinal, ptcFunc)
998 
999  dataset.ptcFitPars[ampName] = parsFit
1000  dataset.ptcFitParsError[ampName] = parsFitErr
1001  dataset.ptcFitReducedChiSquared[ampName] = reducedChiSqPtc
1002 
1003  if ptcFitType == 'ASTIERAPPROXIMATION':
1004  ptcGain = parsFit[1]
1005  ptcGainErr = parsFitErr[1]
1006  ptcNoise = np.sqrt(np.fabs(parsFit[2]))
1007  ptcNoiseErr = 0.5*(parsFitErr[2]/np.fabs(parsFit[2]))*np.sqrt(np.fabs(parsFit[2]))
1008  if ptcFitType == 'POLYNOMIAL':
1009  ptcGain = 1./parsFit[1]
1010  ptcGainErr = np.fabs(1./parsFit[1])*(parsFitErr[1]/parsFit[1])
1011  ptcNoise = np.sqrt(np.fabs(parsFit[0]))*ptcGain
1012  ptcNoiseErr = (0.5*(parsFitErr[0]/np.fabs(parsFit[0]))*(np.sqrt(np.fabs(parsFit[0]))))*ptcGain
1013 
1014  dataset.gain[ampName] = ptcGain
1015  dataset.gainErr[ampName] = ptcGainErr
1016  dataset.noise[ampName] = ptcNoise
1017  dataset.noiseErr[ampName] = ptcNoiseErr
1018  dataset.ptcFitType[ampName] = ptcFitType
1019 
1020  # Non-linearity residuals (NL of mean vs time curve): percentage, and fit to a quadratic function
1021  # In this case, len(parsIniNonLinearity) = 3 indicates that we want a quadratic fit
1022 
1023  datasetLinRes = self.calculateLinearityResidualAndLinearizers(timeVecFinal, meanVecFinal)
1024 
1025  # LinearizerLookupTable
1026  if tableArray is not None:
1027  tableArray[i, :] = datasetLinRes.linearizerTableRow
1028  dataset.nonLinearity[ampName] = datasetLinRes.meanSignalVsTimePolyFitPars
1029  dataset.nonLinearityError[ampName] = datasetLinRes.meanSignalVsTimePolyFitParsErr
1030  dataset.nonLinearityResiduals[ampName] = datasetLinRes.linearityResidual
1031  dataset.fractionalNonLinearityResiduals[ampName] = datasetLinRes.fractionalNonLinearityResidual
1032  dataset.nonLinearityReducedChiSquared[ampName] = datasetLinRes.meanSignalVsTimePolyFitReducedChiSq
1033  # Slice correction coefficients (starting at 2) for polynomial linearizer. The first
1034  # and second are reduntant with the bias and gain, respectively,
1035  # and are not used by LinearizerPolynomial.
1036  polyLinCoeffs = np.array(datasetLinRes.polynomialLinearizerCoefficients[2:])
1037  dataset.coefficientsLinearizePolynomial[ampName] = polyLinCoeffs
1038  quadPolyLinCoeff = datasetLinRes.quadraticPolynomialLinearizerCoefficient
1039  dataset.coefficientLinearizeSquared[ampName] = quadPolyLinCoeff
1040 
1041  return dataset
1042 
1043  def plot(self, dataRef, dataset, ptcFitType):
1044  dirname = dataRef.getUri(datasetType='cpPipePlotRoot', write=True)
1045  if not os.path.exists(dirname):
1046  os.makedirs(dirname)
1047 
1048  detNum = dataRef.dataId[self.config.ccdKey]
1049  filename = f"PTC_det{detNum}.pdf"
1050  filenameFull = os.path.join(dirname, filename)
1051  with PdfPages(filenameFull) as pdfPages:
1052  self._plotPtc(dataset, ptcFitType, pdfPages)
1053 
1054  def _plotPtc(self, dataset, ptcFitType, pdfPages):
1055  """Plot PTC, linearity, and linearity residual per amplifier"""
1056 
1057  reducedChiSqPtc = dataset.ptcFitReducedChiSquared
1058  if ptcFitType == 'ASTIERAPPROXIMATION':
1059  ptcFunc = self.funcAstier
1060  stringTitle = (r"Var = $\frac{1}{2g^2a_{00}}(\exp (2a_{00} \mu g) - 1) + \frac{n_{00}}{g^2}$ "
1061  r" ($chi^2$/dof = %g)" % (reducedChiSqPtc))
1062  if ptcFitType == 'POLYNOMIAL':
1063  ptcFunc = self.funcPolynomial
1064  stringTitle = r"Polynomial (degree: %g)" % (self.config.polynomialFitDegree)
1065 
1066  legendFontSize = 7
1067  labelFontSize = 7
1068  titleFontSize = 9
1069  supTitleFontSize = 18
1070  markerSize = 25
1071 
1072  # General determination of the size of the plot grid
1073  nAmps = len(dataset.ampNames)
1074  if nAmps == 2:
1075  nRows, nCols = 2, 1
1076  nRows = np.sqrt(nAmps)
1077  mantissa, _ = np.modf(nRows)
1078  if mantissa > 0:
1079  nRows = int(nRows) + 1
1080  nCols = nRows
1081  else:
1082  nRows = int(nRows)
1083  nCols = nRows
1084 
1085  f, ax = plt.subplots(nrows=nRows, ncols=nCols, sharex='col', sharey='row', figsize=(13, 10))
1086  f2, ax2 = plt.subplots(nrows=nRows, ncols=nCols, sharex='col', sharey='row', figsize=(13, 10))
1087 
1088  for i, (amp, a, a2) in enumerate(zip(dataset.ampNames, ax.flatten(), ax2.flatten())):
1089  meanVecOriginal = np.array(dataset.rawMeans[amp])
1090  varVecOriginal = np.array(dataset.rawVars[amp])
1091  mask = dataset.visitMask[amp]
1092  meanVecFinal = meanVecOriginal[mask]
1093  varVecFinal = varVecOriginal[mask]
1094  meanVecOutliers = meanVecOriginal[np.invert(mask)]
1095  varVecOutliers = varVecOriginal[np.invert(mask)]
1096  pars, parsErr = dataset.ptcFitPars[amp], dataset.ptcFitParsError[amp]
1097 
1098  if ptcFitType == 'ASTIERAPPROXIMATION':
1099  if len(meanVecFinal):
1100  ptcA00, ptcA00error = pars[0], parsErr[0]
1101  ptcGain, ptcGainError = pars[1], parsErr[1]
1102  ptcNoise = np.sqrt(np.fabs(pars[2]))
1103  ptcNoiseError = 0.5*(parsErr[2]/np.fabs(pars[2]))*np.sqrt(np.fabs(pars[2]))
1104  stringLegend = (f"a00: {ptcA00:.2e}+/-{ptcA00error:.2e} 1/e"
1105  f"\n Gain: {ptcGain:.4}+/-{ptcGainError:.2e} e/DN"
1106  f"\n Noise: {ptcNoise:.4}+/-{ptcNoiseError:.2e} e \n")
1107 
1108  if ptcFitType == 'POLYNOMIAL':
1109  if len(meanVecFinal):
1110  ptcGain, ptcGainError = 1./pars[1], np.fabs(1./pars[1])*(parsErr[1]/pars[1])
1111  ptcNoise = np.sqrt(np.fabs(pars[0]))*ptcGain
1112  ptcNoiseError = (0.5*(parsErr[0]/np.fabs(pars[0]))*(np.sqrt(np.fabs(pars[0]))))*ptcGain
1113  stringLegend = (f"Noise: {ptcNoise:.4}+/-{ptcNoiseError:.2e} e \n"
1114  f"Gain: {ptcGain:.4}+/-{ptcGainError:.2e} e/DN \n")
1115 
1116  a.set_xlabel(r'Mean signal ($\mu$, DN)', fontsize=labelFontSize)
1117  a.set_ylabel(r'Variance (DN$^2$)', fontsize=labelFontSize)
1118  a.tick_params(labelsize=11)
1119  a.set_xscale('linear', fontsize=labelFontSize)
1120  a.set_yscale('linear', fontsize=labelFontSize)
1121 
1122  a2.set_xlabel(r'Mean Signal ($\mu$, DN)', fontsize=labelFontSize)
1123  a2.set_ylabel(r'Variance (DN$^2$)', fontsize=labelFontSize)
1124  a2.tick_params(labelsize=11)
1125  a2.set_xscale('log')
1126  a2.set_yscale('log')
1127 
1128  if len(meanVecFinal): # Empty if the whole amp is bad, for example.
1129  minMeanVecFinal = np.min(meanVecFinal)
1130  maxMeanVecFinal = np.max(meanVecFinal)
1131  meanVecFit = np.linspace(minMeanVecFinal, maxMeanVecFinal, 100*len(meanVecFinal))
1132  minMeanVecOriginal = np.min(meanVecOriginal)
1133  maxMeanVecOriginal = np.max(meanVecOriginal)
1134  deltaXlim = maxMeanVecOriginal - minMeanVecOriginal
1135 
1136  a.plot(meanVecFit, ptcFunc(pars, meanVecFit), color='red')
1137  a.plot(meanVecFinal, pars[0] + pars[1]*meanVecFinal, color='green', linestyle='--')
1138  a.scatter(meanVecFinal, varVecFinal, c='blue', marker='o', s=markerSize)
1139  a.scatter(meanVecOutliers, varVecOutliers, c='magenta', marker='s', s=markerSize)
1140  a.text(0.03, 0.8, stringLegend, transform=a.transAxes, fontsize=legendFontSize)
1141  a.set_title(amp, fontsize=titleFontSize)
1142  a.set_xlim([minMeanVecOriginal - 0.2*deltaXlim, maxMeanVecOriginal + 0.2*deltaXlim])
1143 
1144  # Same, but in log-scale
1145  a2.plot(meanVecFit, ptcFunc(pars, meanVecFit), color='red')
1146  a2.scatter(meanVecFinal, varVecFinal, c='blue', marker='o', s=markerSize)
1147  a2.scatter(meanVecOutliers, varVecOutliers, c='magenta', marker='s', s=markerSize)
1148  a2.text(0.03, 0.8, stringLegend, transform=a2.transAxes, fontsize=legendFontSize)
1149  a2.set_title(amp, fontsize=titleFontSize)
1150  a2.set_xlim([minMeanVecOriginal, maxMeanVecOriginal])
1151  else:
1152  a.set_title(f"{amp} (BAD)", fontsize=titleFontSize)
1153  a2.set_title(f"{amp} (BAD)", fontsize=titleFontSize)
1154 
1155  f.suptitle(f"PTC \n Fit: " + stringTitle, fontsize=20)
1156  pdfPages.savefig(f)
1157  f2.suptitle(f"PTC (log-log)", fontsize=20)
1158  pdfPages.savefig(f2)
1159 
1160  # Plot mean vs time
1161  f, ax = plt.subplots(nrows=4, ncols=4, sharex='col', sharey='row', figsize=(13, 10))
1162  for i, (amp, a) in enumerate(zip(dataset.ampNames, ax.flatten())):
1163  meanVecFinal = np.array(dataset.rawMeans[amp])[dataset.visitMask[amp]]
1164  timeVecFinal = np.array(dataset.rawExpTimes[amp])[dataset.visitMask[amp]]
1165 
1166  a.set_xlabel('Time (sec)', fontsize=labelFontSize)
1167  a.set_ylabel(r'Mean signal ($\mu$, DN)', fontsize=labelFontSize)
1168  a.tick_params(labelsize=labelFontSize)
1169  a.set_xscale('linear', fontsize=labelFontSize)
1170  a.set_yscale('linear', fontsize=labelFontSize)
1171 
1172  if len(meanVecFinal):
1173  pars, parsErr = dataset.nonLinearity[amp], dataset.nonLinearityError[amp]
1174  k0, k0Error = pars[0], parsErr[0]
1175  k1, k1Error = pars[1], parsErr[1]
1176  k2, k2Error = pars[2], parsErr[2]
1177  stringLegend = (f"k0: {k0:.4}+/-{k0Error:.2e} DN\n k1: {k1:.4}+/-{k1Error:.2e} DN/t"
1178  f"\n k2: {k2:.2e}+/-{k2Error:.2e} DN/t^2 \n")
1179  a.scatter(timeVecFinal, meanVecFinal)
1180  a.plot(timeVecFinal, self.funcPolynomial(pars, timeVecFinal), color='red')
1181  a.text(0.03, 0.75, stringLegend, transform=a.transAxes, fontsize=legendFontSize)
1182  a.set_title(f"{amp}", fontsize=titleFontSize)
1183  else:
1184  a.set_title(f"{amp} (BAD)", fontsize=titleFontSize)
1185 
1186  f.suptitle("Linearity \n Fit: Polynomial (degree: %g)"
1187  % (self.config.polynomialFitDegreeNonLinearity),
1188  fontsize=supTitleFontSize)
1189  pdfPages.savefig(f)
1190 
1191  # Plot linearity residual
1192  f, ax = plt.subplots(nrows=4, ncols=4, sharex='col', sharey='row', figsize=(13, 10))
1193  for i, (amp, a) in enumerate(zip(dataset.ampNames, ax.flatten())):
1194  meanVecFinal = np.array(dataset.rawMeans[amp])[dataset.visitMask[amp]]
1195  linRes = np.array(dataset.nonLinearityResiduals[amp])
1196 
1197  a.set_xlabel(r'Mean signal ($\mu$, DN)', fontsize=labelFontSize)
1198  a.set_ylabel('LR (%)', fontsize=labelFontSize)
1199  a.set_xscale('linear', fontsize=labelFontSize)
1200  a.set_yscale('linear', fontsize=labelFontSize)
1201  if len(meanVecFinal):
1202  a.axvline(x=timeVecFinal[self.config.linResidualTimeIndex], color='g', linestyle='--')
1203  a.scatter(meanVecFinal, linRes)
1204  a.set_title(f"{amp}", fontsize=titleFontSize)
1205  a.axhline(y=0, color='k')
1206  a.tick_params(labelsize=labelFontSize)
1207  else:
1208  a.set_title(f"{amp} (BAD)", fontsize=titleFontSize)
1209 
1210  f.suptitle(r"Linearity Residual: $100\times(1 - \mu_{\rm{ref}}/t_{\rm{ref}})/(\mu / t))$" + "\n" +
1211  r"$t_{\rm{ref}}$: " + f"{timeVecFinal[2]} s", fontsize=supTitleFontSize)
1212  pdfPages.savefig(f)
1213 
1214  # Plot fractional non-linearity residual (w.r.t linear part of polynomial fit)
1215  f, ax = plt.subplots(nrows=4, ncols=4, sharex='col', sharey='row', figsize=(13, 10))
1216  for i, (amp, a) in enumerate(zip(dataset.ampNames, ax.flatten())):
1217  meanVecFinal = np.array(dataset.rawMeans[amp])[dataset.visitMask[amp]]
1218  fracLinRes = np.array(dataset.fractionalNonLinearityResiduals[amp])
1219 
1220  a.axhline(y=0, color='k')
1221  a.axvline(x=0, color='k', linestyle='-')
1222  a.set_xlabel(r'Mean signal ($\mu$, DN)', fontsize=labelFontSize)
1223  a.set_ylabel('Fractional nonlinearity (%)', fontsize=labelFontSize)
1224  a.tick_params(labelsize=labelFontSize)
1225  a.set_xscale('linear', fontsize=labelFontSize)
1226  a.set_yscale('linear', fontsize=labelFontSize)
1227 
1228  if len(meanVecFinal):
1229  a.scatter(meanVecFinal, fracLinRes, c='g')
1230  a.set_title(amp, fontsize=titleFontSize)
1231  else:
1232  a.set_title(f"{amp} (BAD)", fontsize=titleFontSize)
1233 
1234  f.suptitle(r"Fractional NL residual" + "\n" +
1235  r"$100\times \frac{(k_0 + k_1*Time-\mu)}{k_0+k_1*Time}$",
1236  fontsize=supTitleFontSize)
1237  pdfPages.savefig(f)
1238 
1239  return
lsst::ip::isr::linearize::Linearizer
lsst.cp.pipe.ptc.PhotonTransferCurveDataset.__setattr__
def __setattr__(self, attribute, value)
Definition: ptc.py:247
lsst.cp.pipe.utils.validateIsrConfig
def validateIsrConfig(isrTask, mandatory=None, forbidden=None, desirable=None, undesirable=None, checkTrim=True, logName=None)
Definition: utils.py:199
lsst.cp.pipe.ptc.MeasurePhotonTransferCurveTask.fitPtcAndNonLinearity
def fitPtcAndNonLinearity(self, dataset, ptcFitType, tableArray=None)
Definition: ptc.py:869
lsst.cp.pipe.ptc.PhotonTransferCurveDataset
Definition: ptc.py:194
lsst.cp.pipe.ptc.MeasurePhotonTransferCurveTask._boundsForAstier
def _boundsForAstier(initialPars)
Definition: ptc.py:703
lsst.cp.pipe.ptc.MeasurePhotonTransferCurveTask.__init__
def __init__(self, *args, **kwargs)
Definition: ptc.py:308
lsst.cp.pipe.ptc.MeasurePhotonTransferCurveTask._boundsForPolynomial
def _boundsForPolynomial(initialPars)
Definition: ptc.py:696
lsst.cp.pipe.ptc.MeasurePhotonTransferCurveTask._plotPtc
def _plotPtc(self, dataset, ptcFitType, pdfPages)
Definition: ptc.py:1054
lsst.cp.pipe.ptc.MeasurePhotonTransferCurveTask.buildLinearizerObject
def buildLinearizerObject(self, dataset, detector, calibDate, linearizerType, instruName='', tableArray=None, log=None)
Definition: ptc.py:434
lsst.cp.pipe.ptc.MeasurePhotonTransferCurveTaskConfig
Definition: ptc.py:48
lsst.cp.pipe.ptc.PhotonTransferCurveDataset.getGoodAmps
def getGoodAmps(self)
Definition: ptc.py:271
lsst.cp.pipe.ptc.MeasurePhotonTransferCurveTask._fitBootstrap
def _fitBootstrap(self, initialParams, dataX, dataY, function, confidenceSigma=1.)
Definition: ptc.py:612
lsst.cp.pipe.ptc.PhotonTransferCurveDataset.getVisitsUsed
def getVisitsUsed(self, ampName)
Definition: ptc.py:255
lsst.cp.pipe.ptc.MeasurePhotonTransferCurveTask.measureMeanVarPair
def measureMeanVarPair(self, exposure1, exposure2, region=None)
Definition: ptc.py:501
lsst.cp.pipe.ptc.MeasurePhotonTransferCurveTask._DefaultName
string _DefaultName
Definition: ptc.py:306
lsst.cp.pipe.ptc.PhotonTransferCurveDataset.__init__
def __init__(self, ampNames)
Definition: ptc.py:214
lsst.cp.pipe.utils.checkExpLengthEqual
def checkExpLengthEqual(exp1, exp2, v1=None, v2=None, raiseWithMessage=False)
Definition: utils.py:162
lsst.cp.pipe.ptc.MeasurePhotonTransferCurveTask
Definition: ptc.py:275
lsst.cp.pipe.ptc.MeasurePhotonTransferCurveTask._makeZeroSafe
def _makeZeroSafe(self, array, warn=True, substituteValue=1e-9)
Definition: ptc.py:735
lsst.cp.pipe.ptc.MeasurePhotonTransferCurveTask._getInitialGoodPoints
def _getInitialGoodPoints(means, variances, maxDeviationPositive, maxDeviationNegative)
Definition: ptc.py:709
lsst::ip::isr::linearize
lsst.cp.pipe.ptc.MeasurePhotonTransferCurveTask.runDataRef
def runDataRef(self, dataRef, visitPairs)
Definition: ptc.py:329
lsst::ip::isr
lsst::afw::math
lsst.cp.pipe.ptc.LinearityResidualsAndLinearizersDataset
Definition: ptc.py:176
lsst.cp.pipe.ptc.MeasurePhotonTransferCurveTask._initialParsForPolynomial
def _initialParsForPolynomial(order)
Definition: ptc.py:687
lsst.cp.pipe.ptc.MeasurePhotonTransferCurveTask.plot
def plot(self, dataRef, dataset, ptcFitType)
Definition: ptc.py:1043
lsst.cp.pipe.ptc.MeasurePhotonTransferCurveTask._fitLeastSq
def _fitLeastSq(self, initialParams, dataX, dataY, function)
Definition: ptc.py:555
lsst.cp.pipe.ptc.MeasurePhotonTransferCurveTask.funcAstier
def funcAstier(self, pars, x)
Definition: ptc.py:681
lsst.cp.pipe.ptc.MeasurePhotonTransferCurveTask.funcPolynomial
def funcPolynomial(self, pars, x)
Definition: ptc.py:677
lsst::pipe::base
lsst.cp.pipe.ptc.MeasurePhotonTransferCurveTask.calculateLinearityResidualAndLinearizers
def calculateLinearityResidualAndLinearizers(self, exposureTimeVector, meanSignalVector)
Definition: ptc.py:748