lsst.cp.pipe  20.0.0-9-gd915bd2+2065d37f6a
ptc.py
Go to the documentation of this file.
1 # This file is part of cp_pipe.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 #
22 
23 __all__ = ['MeasurePhotonTransferCurveTask',
24  'MeasurePhotonTransferCurveTaskConfig',
25  'PhotonTransferCurveDataset']
26 
27 import numpy as np
28 import matplotlib.pyplot as plt
29 from sqlite3 import OperationalError
30 from collections import Counter
31 from dataclasses import dataclass
32 
33 import lsst.afw.math as afwMath
34 import lsst.pex.config as pexConfig
35 import lsst.pipe.base as pipeBase
36 from .utils import (NonexistentDatasetTaskDataIdContainer, PairedVisitListTaskRunner,
37  checkExpLengthEqual, fitLeastSq, fitBootstrap, funcPolynomial, funcAstier)
38 from scipy.optimize import least_squares
39 
40 from lsst.ip.isr.linearize import Linearizer
41 import datetime
42 
43 from .astierCovPtcUtils import (fftSize, CovFft, computeCovDirect, fitData)
44 
45 
46 class MeasurePhotonTransferCurveTaskConfig(pexConfig.Config):
47  """Config class for photon transfer curve measurement task"""
48  ccdKey = pexConfig.Field(
49  dtype=str,
50  doc="The key by which to pull a detector from a dataId, e.g. 'ccd' or 'detector'.",
51  default='ccd',
52  )
53  ptcFitType = pexConfig.ChoiceField(
54  dtype=str,
55  doc="Fit PTC to approximation in Astier+19 (Equation 16) or to a polynomial.",
56  default="POLYNOMIAL",
57  allowed={
58  "POLYNOMIAL": "n-degree polynomial (use 'polynomialFitDegree' to set 'n').",
59  "EXPAPPROXIMATION": "Approximation in Astier+19 (Eq. 16).",
60  "FULLCOVARIANCE": "Full covariances model in Astier+19 (Eq. 20)"
61  }
62  )
63  sigmaClipFullFitCovariancesAstier = pexConfig.Field(
64  dtype=float,
65  doc="sigma clip for full model fit for FULLCOVARIANCE ptcFitType ",
66  default=5.0,
67  )
68  maxIterFullFitCovariancesAstier = pexConfig.Field(
69  dtype=int,
70  doc="Maximum number of iterations in full model fit for FULLCOVARIANCE ptcFitType",
71  default=3,
72  )
73  maximumRangeCovariancesAstier = pexConfig.Field(
74  dtype=int,
75  doc="Maximum range of covariances as in Astier+19",
76  default=8,
77  )
78  covAstierRealSpace = pexConfig.Field(
79  dtype=bool,
80  doc="Calculate covariances in real space or via FFT? (see appendix A of Astier+19).",
81  default=False,
82  )
83  polynomialFitDegree = pexConfig.Field(
84  dtype=int,
85  doc="Degree of polynomial to fit the PTC, when 'ptcFitType'=POLYNOMIAL.",
86  default=3,
87  )
88  doCreateLinearizer = pexConfig.Field(
89  dtype=bool,
90  doc="Calculate non-linearity and persist linearizer?",
91  default=False,
92  )
93  linearizerType = pexConfig.ChoiceField(
94  dtype=str,
95  doc="Linearizer type, if doCreateLinearizer=True",
96  default="LINEARIZEPOLYNOMIAL",
97  allowed={
98  "LINEARIZEPOLYNOMIAL": "n-degree polynomial (use 'polynomialFitDegreeNonLinearity' to set 'n').",
99  "LINEARIZESQUARED": "c0 quadratic coefficient derived from coefficients of polynomiual fit",
100  "LOOKUPTABLE": "Loouk table formed from linear part of polynomial fit."
101  }
102  )
103  polynomialFitDegreeNonLinearity = pexConfig.Field(
104  dtype=int,
105  doc="If doCreateLinearizer, degree of polynomial to fit the meanSignal vs exposureTime" +
106  " curve to produce the table for LinearizeLookupTable.",
107  default=3,
108  )
109  binSize = pexConfig.Field(
110  dtype=int,
111  doc="Bin the image by this factor in both dimensions.",
112  default=1,
113  )
114  minMeanSignal = pexConfig.Field(
115  dtype=float,
116  doc="Minimum value (inclusive) of mean signal (in DN) above which to consider.",
117  default=0,
118  )
119  maxMeanSignal = pexConfig.Field(
120  dtype=float,
121  doc="Maximum value (inclusive) of mean signal (in DN) below which to consider.",
122  default=9e6,
123  )
124  initialNonLinearityExclusionThresholdPositive = pexConfig.RangeField(
125  dtype=float,
126  doc="Initially exclude data points with a variance that are more than a factor of this from being"
127  " linear in the positive direction, from the PTC fit. Note that these points will also be"
128  " excluded from the non-linearity fit. This is done before the iterative outlier rejection,"
129  " to allow an accurate determination of the sigmas for said iterative fit.",
130  default=0.12,
131  min=0.0,
132  max=1.0,
133  )
134  initialNonLinearityExclusionThresholdNegative = pexConfig.RangeField(
135  dtype=float,
136  doc="Initially exclude data points with a variance that are more than a factor of this from being"
137  " linear in the negative direction, from the PTC fit. Note that these points will also be"
138  " excluded from the non-linearity fit. This is done before the iterative outlier rejection,"
139  " to allow an accurate determination of the sigmas for said iterative fit.",
140  default=0.25,
141  min=0.0,
142  max=1.0,
143  )
144  sigmaCutPtcOutliers = pexConfig.Field(
145  dtype=float,
146  doc="Sigma cut for outlier rejection in PTC.",
147  default=5.0,
148  )
149  maskNameList = pexConfig.ListField(
150  dtype=str,
151  doc="Mask list to exclude from statistics calculations.",
152  default=['SUSPECT', 'BAD', 'NO_DATA'],
153  )
154  nSigmaClipPtc = pexConfig.Field(
155  dtype=float,
156  doc="Sigma cut for afwMath.StatisticsControl()",
157  default=5.5,
158  )
159  nIterSigmaClipPtc = pexConfig.Field(
160  dtype=int,
161  doc="Number of sigma-clipping iterations for afwMath.StatisticsControl()",
162  default=1,
163  )
164  maxIterationsPtcOutliers = pexConfig.Field(
165  dtype=int,
166  doc="Maximum number of iterations for outlier rejection in PTC.",
167  default=2,
168  )
169  doFitBootstrap = pexConfig.Field(
170  dtype=bool,
171  doc="Use bootstrap for the PTC fit parameters and errors?.",
172  default=False,
173  )
174  maxAduForLookupTableLinearizer = pexConfig.Field(
175  dtype=int,
176  doc="Maximum DN value for the LookupTable linearizer.",
177  default=2**18,
178  )
179  instrumentName = pexConfig.Field(
180  dtype=str,
181  doc="Instrument name.",
182  default='',
183  )
184 
185 
186 @dataclass
188  """A simple class to hold the output from the
189  `calculateLinearityResidualAndLinearizers` function.
190  """
191  # Normalized coefficients for polynomial NL correction
192  polynomialLinearizerCoefficients: list
193  # Normalized coefficient for quadratic polynomial NL correction (c0)
194  quadraticPolynomialLinearizerCoefficient: float
195  # LUT array row for the amplifier at hand
196  linearizerTableRow: list
197  meanSignalVsTimePolyFitPars: list
198  meanSignalVsTimePolyFitParsErr: list
199  meanSignalVsTimePolyFitReducedChiSq: float
200 
201 
203  """A simple class to hold the output data from the PTC task.
204 
205  The dataset is made up of a dictionary for each item, keyed by the
206  amplifiers' names, which much be supplied at construction time.
207 
208  New items cannot be added to the class to save accidentally saving to the
209  wrong property, and the class can be frozen if desired.
210 
211  inputVisitPairs records the visits used to produce the data.
212  When fitPtc() or fitCovariancesAstier() is run, a mask is built up, which is by definition
213  always the same length as inputVisitPairs, rawExpTimes, rawMeans
214  and rawVars, and is a list of bools, which are incrementally set to False
215  as points are discarded from the fits.
216 
217  PTC fit parameters for polynomials are stored in a list in ascending order
218  of polynomial term, i.e. par[0]*x^0 + par[1]*x + par[2]*x^2 etc
219  with the length of the list corresponding to the order of the polynomial
220  plus one.
221 
222  Parameters
223  ----------
224  ampNames : `list`
225  List with the names of the amplifiers of the detector at hand.
226 
227  ptcFitType : `str`
228  Type of model fitted to the PTC: "POLYNOMIAL", "EXPAPPROXIMATION", or "FULLCOVARIANCE".
229 
230  Returns
231  -------
232  `lsst.cp.pipe.ptc.PhotonTransferCurveDataset`
233  Output dataset from MeasurePhotonTransferCurveTask.
234  """
235 
236  def __init__(self, ampNames, ptcFitType):
237  # add items to __dict__ directly because __setattr__ is overridden
238 
239  # instance variables
240  self.__dict__["ptcFitType"] = ptcFitType
241  self.__dict__["ampNames"] = ampNames
242  self.__dict__["badAmps"] = []
243 
244  # raw data variables
245  # visitMask is the mask produced after outlier rejection. The mask produced by "FULLCOVARIANCE"
246  # may differ from the one produced in the other two PTC fit types.
247  self.__dict__["inputVisitPairs"] = {ampName: [] for ampName in ampNames}
248  self.__dict__["visitMask"] = {ampName: [] for ampName in ampNames}
249  self.__dict__["rawExpTimes"] = {ampName: [] for ampName in ampNames}
250  self.__dict__["rawMeans"] = {ampName: [] for ampName in ampNames}
251  self.__dict__["rawVars"] = {ampName: [] for ampName in ampNames}
252 
253  # Gain and noise
254  self.__dict__["gain"] = {ampName: -1. for ampName in ampNames}
255  self.__dict__["gainErr"] = {ampName: -1. for ampName in ampNames}
256  self.__dict__["noise"] = {ampName: -1. for ampName in ampNames}
257  self.__dict__["noiseErr"] = {ampName: -1. for ampName in ampNames}
258 
259  # if ptcFitTye in ["POLYNOMIAL", "EXPAPPROXIMATION"]
260  # fit information
261  self.__dict__["ptcFitPars"] = {ampName: [] for ampName in ampNames}
262  self.__dict__["ptcFitParsError"] = {ampName: [] for ampName in ampNames}
263  self.__dict__["ptcFitReducedChiSquared"] = {ampName: [] for ampName in ampNames}
264 
265  # if ptcFitTye in ["FULLCOVARIANCE"]
266  # "covariancesTuple" is a numpy recarray with entries of the form
267  # ['mu', 'i', 'j', 'var', 'cov', 'npix', 'ext', 'expTime', 'ampName']
268  # "covariancesFits" has CovFit objects that fit the measured covariances to Eq. 20 of Astier+19.
269  # In "covariancesFitsWithNoB", "b"=0 in the model described by Eq. 20 of Astier+19.
270  self.__dict__["covariancesTuple"] = {ampName: [] for ampName in ampNames}
271  self.__dict__["covariancesFitsWithNoB"] = {ampName: [] for ampName in ampNames}
272  self.__dict__["covariancesFits"] = {ampName: [] for ampName in ampNames}
273  self.__dict__["aMatrix"] = {ampName: [] for ampName in ampNames}
274  self.__dict__["bMatrix"] = {ampName: [] for ampName in ampNames}
275 
276  # "final" means that the "raw" vectors above had "visitMask" applied.
277  self.__dict__["finalVars"] = {ampName: [] for ampName in ampNames}
278  self.__dict__["finalModelVars"] = {ampName: [] for ampName in ampNames}
279  self.__dict__["finalMeans"] = {ampName: [] for ampName in ampNames}
280 
281  def __setattr__(self, attribute, value):
282  """Protect class attributes"""
283  if attribute not in self.__dict__:
284  raise AttributeError(f"{attribute} is not already a member of PhotonTransferCurveDataset, which"
285  " does not support setting of new attributes.")
286  else:
287  self.__dict__[attribute] = value
288 
289  def getVisitsUsed(self, ampName):
290  """Get the visits used, i.e. not discarded, for a given amp.
291 
292  If no mask has been created yet, all visits are returned.
293  """
294  if len(self.visitMask[ampName]) == 0:
295  return self.inputVisitPairs[ampName]
296 
297  # if the mask exists it had better be the same length as the visitPairs
298  assert len(self.visitMask[ampName]) == len(self.inputVisitPairs[ampName])
299 
300  pairs = self.inputVisitPairs[ampName]
301  mask = self.visitMask[ampName]
302  # cast to bool required because numpy
303  return [(v1, v2) for ((v1, v2), m) in zip(pairs, mask) if bool(m) is True]
304 
305  def getGoodAmps(self):
306  return [amp for amp in self.ampNames if amp not in self.badAmps]
307 
308 
309 class MeasurePhotonTransferCurveTask(pipeBase.CmdLineTask):
310  """A class to calculate, fit, and plot a PTC from a set of flat pairs.
311 
312  The Photon Transfer Curve (var(signal) vs mean(signal)) is a standard tool
313  used in astronomical detectors characterization (e.g., Janesick 2001,
314  Janesick 2007). If ptcFitType is "EXPAPPROXIMATION" or "POLYNOMIAL", this task calculates the
315  PTC from a series of pairs of flat-field images; each pair taken at identical exposure
316  times. The difference image of each pair is formed to eliminate fixed pattern noise,
317  and then the variance of the difference image and the mean of the average image
318  are used to produce the PTC. An n-degree polynomial or the approximation in Equation
319  16 of Astier+19 ("The Shape of the Photon Transfer Curve of CCD sensors",
320  arXiv:1905.08677) can be fitted to the PTC curve. These models include
321  parameters such as the gain (e/DN) and readout noise.
322 
323  Linearizers to correct for signal-chain non-linearity are also calculated.
324  The `Linearizer` class, in general, can support per-amp linearizers, but in this
325  task this is not supported.
326 
327  If ptcFitType is "FULLCOVARIANCE", the covariances of the difference images are calculated via the
328  DFT methods described in Astier+19 and the variances for the PTC are given by the cov[0,0] elements
329  at each signal level. The full model in Equation 20 of Astier+19 is fit to the PTC to get the gain
330  and the noise.
331 
332  Parameters
333  ----------
334 
335  *args: `list`
336  Positional arguments passed to the Task constructor. None used at this
337  time.
338  **kwargs: `dict`
339  Keyword arguments passed on to the Task constructor. None used at this
340  time.
341 
342  """
343 
344  RunnerClass = PairedVisitListTaskRunner
345  ConfigClass = MeasurePhotonTransferCurveTaskConfig
346  _DefaultName = "measurePhotonTransferCurve"
347 
348  def __init__(self, *args, **kwargs):
349  pipeBase.CmdLineTask.__init__(self, *args, **kwargs)
350  plt.interactive(False) # stop windows popping up when plotting. When headless, use 'agg' backend too
351  self.config.validate()
352  self.config.freeze()
353 
354  @classmethod
355  def _makeArgumentParser(cls):
356  """Augment argument parser for the MeasurePhotonTransferCurveTask."""
357  parser = pipeBase.ArgumentParser(name=cls._DefaultName)
358  parser.add_argument("--visit-pairs", dest="visitPairs", nargs="*",
359  help="Visit pairs to use. Each pair must be of the form INT,INT e.g. 123,456")
360  parser.add_id_argument("--id", datasetType="photonTransferCurveDataset",
361  ContainerClass=NonexistentDatasetTaskDataIdContainer,
362  help="The ccds to use, e.g. --id ccd=0..100")
363  return parser
364 
365  @pipeBase.timeMethod
366  def runDataRef(self, dataRef, visitPairs):
367  """Run the Photon Transfer Curve (PTC) measurement task.
368 
369  For a dataRef (which is each detector here),
370  and given a list of visit pairs (postISR) at different exposure times,
371  measure the PTC.
372 
373  Parameters
374  ----------
375  dataRef : list of lsst.daf.persistence.ButlerDataRef
376  dataRef for the detector for the visits to be fit.
377 
378  visitPairs : `iterable` of `tuple` of `int`
379  Pairs of visit numbers to be processed together
380  """
381 
382  # setup necessary objects
383  detNum = dataRef.dataId[self.config.ccdKey]
384  detector = dataRef.get('camera')[dataRef.dataId[self.config.ccdKey]]
385  # expand some missing fields that we need for lsstCam. This is a work-around
386  # for Gen2 problems that I (RHL) don't feel like solving. The calibs pipelines
387  # (which inherit from CalibTask) use addMissingKeys() to do basically the same thing
388  #
389  # Basically, the butler's trying to look up the fields in `raw_visit` which won't work
390  for name in dataRef.getButler().getKeys('bias'):
391  if name not in dataRef.dataId:
392  try:
393  dataRef.dataId[name] = \
394  dataRef.getButler().queryMetadata('raw', [name], detector=detNum)[0]
395  except OperationalError:
396  pass
397 
398  amps = detector.getAmplifiers()
399  ampNames = [amp.getName() for amp in amps]
400  datasetPtc = PhotonTransferCurveDataset(ampNames, self.config.ptcFitType)
401  self.log.info('Measuring PTC using %s visits for detector %s' % (visitPairs, detector.getId()))
402 
403  tupleRecords = []
404  allTags = []
405  for (v1, v2) in visitPairs:
406  # Get postISR exposures.
407  dataRef.dataId['expId'] = v1
408  exp1 = dataRef.get("postISRCCD", immediate=True)
409  dataRef.dataId['expId'] = v2
410  exp2 = dataRef.get("postISRCCD", immediate=True)
411  del dataRef.dataId['expId']
412 
413  checkExpLengthEqual(exp1, exp2, v1, v2, raiseWithMessage=True)
414  expTime = exp1.getInfo().getVisitInfo().getExposureTime()
415  tupleRows = []
416  nAmpsNan = 0
417  for ampNumber, amp in enumerate(detector):
418  ampName = amp.getName()
419  # covAstier: (i, j, var (cov[0,0]), cov, npix)
420  doRealSpace = self.config.covAstierRealSpace
421  muDiff, varDiff, covAstier = self.measureMeanVarCov(exp1, exp2, region=amp.getBBox(),
422  covAstierRealSpace=doRealSpace)
423  if np.isnan(muDiff) or np.isnan(varDiff) or (covAstier is None):
424  msg = (f"NaN mean or var, or None cov in amp {ampNumber} in visit pair {v1}, {v2} "
425  "of detector {detNum}.")
426  self.log.warn(msg)
427  nAmpsNan += 1
428  continue
429  tags = ['mu', 'i', 'j', 'var', 'cov', 'npix', 'ext', 'expTime', 'ampName']
430  if (muDiff <= self.config.minMeanSignal) or (muDiff >= self.config.maxMeanSignal):
431  continue
432  datasetPtc.rawExpTimes[ampName].append(expTime)
433  datasetPtc.rawMeans[ampName].append(muDiff)
434  datasetPtc.rawVars[ampName].append(varDiff)
435  datasetPtc.inputVisitPairs[ampName].append((v1, v2))
436 
437  tupleRows += [(muDiff, ) + covRow + (ampNumber, expTime, ampName) for covRow in covAstier]
438  if nAmpsNan == len(ampNames):
439  msg = f"NaN mean in all amps of visit pair {v1}, {v2} of detector {detNum}."
440  self.log.warn(msg)
441  continue
442  allTags += tags
443  tupleRecords += tupleRows
444  covariancesWithTags = np.core.records.fromrecords(tupleRecords, names=allTags)
445 
446  if self.config.ptcFitType in ["FULLCOVARIANCE", ]:
447  # Calculate covariances and fit them, including the PTC, to Astier+19 full model (Eq. 20)
448  datasetPtc = self.fitCovariancesAstier(datasetPtc, covariancesWithTags)
449  elif self.config.ptcFitType in ["EXPAPPROXIMATION", "POLYNOMIAL"]:
450  # Fit the PTC to a polynomial or to Astier+19 exponential approximation (Eq. 16)
451  # Fill up PhotonTransferCurveDataset object.
452  datasetPtc = self.fitPtc(datasetPtc, self.config.ptcFitType)
453 
454  # Fit a poynomial to calculate non-linearity and persist linearizer.
455  if self.config.doCreateLinearizer:
456  numberAmps = len(amps)
457  numberAduValues = self.config.maxAduForLookupTableLinearizer
458  lookupTableArray = np.zeros((numberAmps, numberAduValues), dtype=np.float32)
459 
460  # Fit (non)linearity of signal vs time curve.
461  # Fill up PhotonTransferCurveDataset object.
462  # Fill up array for LUT linearizer (tableArray).
463  # Produce coefficients for Polynomial ans Squared linearizers.
464  # Build linearizer objects.
465  linearizer = self.fitNonLinearityAndBuildLinearizers(datasetPtc, detector,
466  tableArray=lookupTableArray,
467  log=self.log)
468 
469  if self.config.linearizerType == "LINEARIZEPOLYNOMIAL":
470  linDataType = 'linearizePolynomial'
471  linMsg = "polynomial (coefficients for a polynomial correction)."
472  elif self.config.linearizerType == "LINEARIZESQUARED":
473  linDataType = 'linearizePolynomial'
474  linMsg = "squared (c0, derived from k_i coefficients of a polynomial fit)."
475  elif self.config.linearizerType == "LOOKUPTABLE":
476  linDataType = 'linearizePolynomial'
477  linMsg = "lookup table (linear component of polynomial fit)."
478  else:
479  raise RuntimeError("Invalid config.linearizerType {selg.config.linearizerType}. "
480  "Supported: 'LOOKUPTABLE', 'LINEARIZESQUARED', or 'LINEARIZEPOLYNOMIAL'")
481 
482  butler = dataRef.getButler()
483  self.log.info(f"Writing linearizer: \n {linMsg}")
484 
485  detName = detector.getName()
486  now = datetime.datetime.utcnow()
487  calibDate = now.strftime("%Y-%m-%d")
488 
489  butler.put(linearizer, datasetType=linDataType, dataId={'detector': detNum,
490  'detectorName': detName, 'calibDate': calibDate})
491 
492  self.log.info(f"Writing PTC data to {dataRef.getUri(write=True)}")
493  dataRef.put(datasetPtc, datasetType="photonTransferCurveDataset")
494 
495  return pipeBase.Struct(exitStatus=0)
496 
497  def fitCovariancesAstier(self, dataset, covariancesWithTagsArray):
498  """Fit measured flat covariances to full model in Astier+19.
499 
500  Parameters
501  ----------
502  dataset : `lsst.cp.pipe.ptc.PhotonTransferCurveDataset`
503  The dataset containing information such as the means, variances and exposure times.
504 
505  covariancesWithTagsArray : `numpy.recarray`
506  Tuple with at least (mu, cov, var, i, j, npix), where:
507  mu : 0.5*(m1 + m2), where:
508  mu1: mean value of flat1
509  mu2: mean value of flat2
510  cov: covariance value at lag(i, j)
511  var: variance(covariance value at lag(0, 0))
512  i: lag dimension
513  j: lag dimension
514  npix: number of pixels used for covariance calculation.
515 
516  Returns
517  -------
518  dataset: `lsst.cp.pipe.ptc.PhotonTransferCurveDataset`
519  This is the same dataset as the input paramter, however, it has been modified
520  to include information such as the fit vectors and the fit parameters. See
521  the class `PhotonTransferCurveDatase`.
522  """
523 
524  covFits, covFitsNoB = fitData(covariancesWithTagsArray, maxMu=self.config.maxMeanSignal,
525  r=self.config.maximumRangeCovariancesAstier,
526  nSigmaFullFit=self.config.sigmaClipFullFitCovariancesAstier,
527  maxIterFullFit=self.config.maxIterFullFitCovariancesAstier)
528 
529  dataset.covariancesTuple = covariancesWithTagsArray
530  dataset.covariancesFits = covFits
531  dataset.covariancesFitsWithNoB = covFitsNoB
532  dataset = self.getOutputPtcDataCovAstier(dataset, covFits)
533 
534  return dataset
535 
536  def getOutputPtcDataCovAstier(self, dataset, covFits):
537  """Get output data for PhotonTransferCurveCovAstierDataset from CovFit objects.
538 
539  Parameters
540  ----------
541  dataset : `lsst.cp.pipe.ptc.PhotonTransferCurveDataset`
542  The dataset containing information such as the means, variances and exposure times.
543 
544  covFits: `dict`
545  Dictionary of CovFit objects, with amp names as keys.
546 
547  Returns
548  -------
549  dataset : `lsst.cp.pipe.ptc.PhotonTransferCurveDataset`
550  This is the same dataset as the input paramter, however, it has been modified
551  to include extra information such as the mask 1D array, gains, reoudout noise, measured signal,
552  measured variance, modeled variance, a, and b coefficient matrices (see Astier+19) per amplifier.
553  See the class `PhotonTransferCurveDatase`.
554  """
555 
556  for i, amp in enumerate(covFits):
557  fit = covFits[amp]
558  (meanVecFinal, varVecFinal, varVecModel,
559  wc, varMask) = fit.getFitData(0, 0, divideByMu=False, returnMasked=True)
560  gain = fit.getGain()
561  dataset.visitMask[amp] = varMask
562  dataset.gain[amp] = gain
563  dataset.gainErr[amp] = fit.getGainErr()
564  dataset.noise[amp] = np.sqrt(np.fabs(fit.getRon()))
565  dataset.noiseErr[amp] = fit.getRonErr()
566  dataset.finalVars[amp].append(varVecFinal/(gain**2))
567  dataset.finalModelVars[amp].append(varVecModel/(gain**2))
568  dataset.finalMeans[amp].append(meanVecFinal/gain)
569  dataset.aMatrix[amp].append(fit.getA())
570  dataset.bMatrix[amp].append(fit.getB())
571 
572  return dataset
573 
574  def measureMeanVarCov(self, exposure1, exposure2, region=None, covAstierRealSpace=False):
575  """Calculate the mean of each of two exposures and the variance and covariance of their difference.
576 
577  The variance is calculated via afwMath, and the covariance via the methods in Astier+19 (appendix A).
578  In theory, var = covariance[0,0]. This should be validated, and in the future, we may decide to just
579  keep one (covariance).
580 
581  Parameters
582  ----------
583  exposure1 : `lsst.afw.image.exposure.exposure.ExposureF`
584  First exposure of flat field pair.
585 
586  exposure2 : `lsst.afw.image.exposure.exposure.ExposureF`
587  Second exposure of flat field pair.
588 
589  region : `lsst.geom.Box2I`, optional
590  Region of each exposure where to perform the calculations (e.g, an amplifier).
591 
592  covAstierRealSpace : `bool`, optional
593  Should the covariannces in Astier+19 be calculated in real space or via FFT?
594  See Appendix A of Astier+19.
595 
596  Returns
597  -------
598  mu : `float` or `NaN`
599  0.5*(mu1 + mu2), where mu1, and mu2 are the clipped means of the regions in
600  both exposures. If either mu1 or m2 are NaN's, the returned value is NaN.
601 
602  varDiff : `float` or `NaN`
603  Half of the clipped variance of the difference of the regions inthe two input
604  exposures. If either mu1 or m2 are NaN's, the returned value is NaN.
605 
606  covDiffAstier : `list` or `NaN`
607  List with tuples of the form (dx, dy, var, cov, npix), where:
608  dx : `int`
609  Lag in x
610  dy : `int`
611  Lag in y
612  var : `float`
613  Variance at (dx, dy).
614  cov : `float`
615  Covariance at (dx, dy).
616  nPix : `int`
617  Number of pixel pairs used to evaluate var and cov.
618  If either mu1 or m2 are NaN's, the returned value is NaN.
619  """
620 
621  if region is not None:
622  im1Area = exposure1.maskedImage[region]
623  im2Area = exposure2.maskedImage[region]
624  else:
625  im1Area = exposure1.maskedImage
626  im2Area = exposure2.maskedImage
627 
628  if self.config.binSize > 1:
629  im1Area = afwMath.binImage(im1Area, self.config.binSize)
630  im2Area = afwMath.binImage(im2Area, self.config.binSize)
631 
632  im1MaskVal = exposure1.getMask().getPlaneBitMask(self.config.maskNameList)
633  im1StatsCtrl = afwMath.StatisticsControl(self.config.nSigmaClipPtc,
634  self.config.nIterSigmaClipPtc,
635  im1MaskVal)
636  im1StatsCtrl.setNanSafe(True)
637  im1StatsCtrl.setAndMask(im1MaskVal)
638 
639  im2MaskVal = exposure2.getMask().getPlaneBitMask(self.config.maskNameList)
640  im2StatsCtrl = afwMath.StatisticsControl(self.config.nSigmaClipPtc,
641  self.config.nIterSigmaClipPtc,
642  im2MaskVal)
643  im2StatsCtrl.setNanSafe(True)
644  im2StatsCtrl.setAndMask(im2MaskVal)
645 
646  # Clipped mean of images; then average of mean.
647  mu1 = afwMath.makeStatistics(im1Area, afwMath.MEANCLIP, im1StatsCtrl).getValue()
648  mu2 = afwMath.makeStatistics(im2Area, afwMath.MEANCLIP, im2StatsCtrl).getValue()
649  if np.isnan(mu1) or np.isnan(mu2):
650  return np.nan, np.nan, None
651  mu = 0.5*(mu1 + mu2)
652 
653  # Take difference of pairs
654  # symmetric formula: diff = (mu2*im1-mu1*im2)/(0.5*(mu1+mu2))
655  temp = im2Area.clone()
656  temp *= mu1
657  diffIm = im1Area.clone()
658  diffIm *= mu2
659  diffIm -= temp
660  diffIm /= mu
661 
662  diffImMaskVal = diffIm.getMask().getPlaneBitMask(self.config.maskNameList)
663  diffImStatsCtrl = afwMath.StatisticsControl(self.config.nSigmaClipPtc,
664  self.config.nIterSigmaClipPtc,
665  diffImMaskVal)
666  diffImStatsCtrl.setNanSafe(True)
667  diffImStatsCtrl.setAndMask(diffImMaskVal)
668 
669  varDiff = 0.5*(afwMath.makeStatistics(diffIm, afwMath.VARIANCECLIP, diffImStatsCtrl).getValue())
670 
671  # Get the mask and identify good pixels as '1', and the rest as '0'.
672  w1 = np.where(im1Area.getMask().getArray() == 0, 1, 0)
673  w2 = np.where(im2Area.getMask().getArray() == 0, 1, 0)
674 
675  w12 = w1*w2
676  wDiff = np.where(diffIm.getMask().getArray() == 0, 1, 0)
677  w = w12*wDiff
678 
679  maxRangeCov = self.config.maximumRangeCovariancesAstier
680  if covAstierRealSpace:
681  covDiffAstier = computeCovDirect(diffIm.getImage().getArray(), w, maxRangeCov)
682  else:
683  shapeDiff = diffIm.getImage().getArray().shape
684  fftShape = (fftSize(shapeDiff[0] + maxRangeCov), fftSize(shapeDiff[1]+maxRangeCov))
685  c = CovFft(diffIm.getImage().getArray(), w, fftShape, maxRangeCov)
686  covDiffAstier = c.reportCovFft(maxRangeCov)
687 
688  return mu, varDiff, covDiffAstier
689 
690  def computeCovDirect(self, diffImage, weightImage, maxRange):
691  """Compute covariances of diffImage in real space.
692 
693  For lags larger than ~25, it is slower than the FFT way.
694  Taken from https://github.com/PierreAstier/bfptc/
695 
696  Parameters
697  ----------
698  diffImage : `numpy.array`
699  Image to compute the covariance of.
700 
701  weightImage : `numpy.array`
702  Weight image of diffImage (1's and 0's for good and bad pixels, respectively).
703 
704  maxRange : `int`
705  Last index of the covariance to be computed.
706 
707  Returns
708  -------
709  outList : `list`
710  List with tuples of the form (dx, dy, var, cov, npix), where:
711  dx : `int`
712  Lag in x
713  dy : `int`
714  Lag in y
715  var : `float`
716  Variance at (dx, dy).
717  cov : `float`
718  Covariance at (dx, dy).
719  nPix : `int`
720  Number of pixel pairs used to evaluate var and cov.
721  """
722  outList = []
723  var = 0
724  # (dy,dx) = (0,0) has to be first
725  for dy in range(maxRange + 1):
726  for dx in range(0, maxRange + 1):
727  if (dx*dy > 0):
728  cov1, nPix1 = self.covDirectValue(diffImage, weightImage, dx, dy)
729  cov2, nPix2 = self.covDirectValue(diffImage, weightImage, dx, -dy)
730  cov = 0.5*(cov1 + cov2)
731  nPix = nPix1 + nPix2
732  else:
733  cov, nPix = self.covDirectValue(diffImage, weightImage, dx, dy)
734  if (dx == 0 and dy == 0):
735  var = cov
736  outList.append((dx, dy, var, cov, nPix))
737 
738  return outList
739 
740  def covDirectValue(self, diffImage, weightImage, dx, dy):
741  """Compute covariances of diffImage in real space at lag (dx, dy).
742 
743  Taken from https://github.com/PierreAstier/bfptc/ (c.f., appendix of Astier+19).
744 
745  Parameters
746  ----------
747  diffImage : `numpy.array`
748  Image to compute the covariance of.
749 
750  weightImage : `numpy.array`
751  Weight image of diffImage (1's and 0's for good and bad pixels, respectively).
752 
753  dx : `int`
754  Lag in x.
755 
756  dy : `int`
757  Lag in y.
758 
759  Returns
760  -------
761  cov : `float`
762  Covariance at (dx, dy)
763 
764  nPix : `int`
765  Number of pixel pairs used to evaluate var and cov.
766  """
767  (nCols, nRows) = diffImage.shape
768  # switching both signs does not change anything:
769  # it just swaps im1 and im2 below
770  if (dx < 0):
771  (dx, dy) = (-dx, -dy)
772  # now, we have dx >0. We have to distinguish two cases
773  # depending on the sign of dy
774  if dy >= 0:
775  im1 = diffImage[dy:, dx:]
776  w1 = weightImage[dy:, dx:]
777  im2 = diffImage[:nCols - dy, :nRows - dx]
778  w2 = weightImage[:nCols - dy, :nRows - dx]
779  else:
780  im1 = diffImage[:nCols + dy, dx:]
781  w1 = weightImage[:nCols + dy, dx:]
782  im2 = diffImage[-dy:, :nRows - dx]
783  w2 = weightImage[-dy:, :nRows - dx]
784  # use the same mask for all 3 calculations
785  wAll = w1*w2
786  # do not use mean() because weightImage=0 pixels would then count
787  nPix = wAll.sum()
788  im1TimesW = im1*wAll
789  s1 = im1TimesW.sum()/nPix
790  s2 = (im2*wAll).sum()/nPix
791  p = (im1TimesW*im2).sum()/nPix
792  cov = p - s1*s2
793 
794  return cov, nPix
795 
796  def fitNonLinearityAndBuildLinearizers(self, datasetPtc, detector, tableArray=None, log=None):
797  """Fit non-linearity function and build linearizer objects.
798 
799  Parameters
800  ----------
801  datasePtct : `lsst.cp.pipe.ptc.PhotonTransferCurveDataset`
802  The dataset containing information such as the means, variances and exposure times.
803  nLinearity
804 
805  detector : `lsst.afw.cameraGeom.Detector`
806  Detector object.
807 
808  tableArray : `np.array`, optional
809  Optional. Look-up table array with size rows=nAmps and columns=DN values.
810  It will be modified in-place if supplied.
811 
812  log : `lsst.log.Log`, optional
813  Logger to handle messages.
814 
815  Returns
816  -------
817  linearizer : `lsst.ip.isr.Linearizer`
818  Linearizer object
819  """
820 
821  # Fit NonLinearity
822  datasetNonLinearity = self.fitNonLinearity(datasetPtc, tableArray=tableArray)
823 
824  # Produce linearizer
825  now = datetime.datetime.utcnow()
826  calibDate = now.strftime("%Y-%m-%d")
827  linType = self.config.linearizerType
828 
829  if linType == "LOOKUPTABLE":
830  tableArray = tableArray
831  else:
832  tableArray = None
833 
834  linearizer = self.buildLinearizerObject(datasetNonLinearity, detector, calibDate, linType,
835  instruName=self.config.instrumentName,
836  tableArray=tableArray,
837  log=log)
838 
839  return linearizer
840 
841  def fitNonLinearity(self, datasetPtc, tableArray=None):
842  """Fit a polynomial to signal vs effective time curve to calculate linearity and residuals.
843 
844  Parameters
845  ----------
846  datasetPtc : `lsst.cp.pipe.ptc.PhotonTransferCurveDataset`
847  The dataset containing the means, variances and exposure times.
848 
849  tableArray : `np.array`
850  Optional. Look-up table array with size rows=nAmps and columns=DN values.
851  It will be modified in-place if supplied.
852 
853  Returns
854  -------
855  datasetNonLinearity : `dict`
856  Dictionary of `lsst.cp.pipe.ptc.LinearityResidualsAndLinearizersDataset`
857  dataclasses. Each one holds the output of `calculateLinearityResidualAndLinearizers` per
858  amplifier.
859  """
860  datasetNonLinearity = {ampName: [] for ampName in datasetPtc.ampNames}
861  for i, ampName in enumerate(datasetPtc.ampNames):
862  # If a mask is not found, use all points.
863  if (len(datasetPtc.visitMask[ampName]) == 0):
864  self.log.warn(f"Mask not found for {ampName} in non-linearity fit. Using all points.")
865  mask = np.repeat(True, len(datasetPtc.rawExpTimes[ampName]))
866  else:
867  mask = datasetPtc.visitMask[ampName]
868 
869  timeVecFinal = np.array(datasetPtc.rawExpTimes[ampName])[mask]
870  meanVecFinal = np.array(datasetPtc.rawMeans[ampName])[mask]
871 
872  # Non-linearity residuals (NL of mean vs time curve): percentage, and fit to a quadratic function
873  # In this case, len(parsIniNonLinearity) = 3 indicates that we want a quadratic fit
874  datasetLinRes = self.calculateLinearityResidualAndLinearizers(timeVecFinal, meanVecFinal)
875 
876  # LinearizerLookupTable
877  if tableArray is not None:
878  tableArray[i, :] = datasetLinRes.linearizerTableRow
879 
880  datasetNonLinearity[ampName] = datasetLinRes
881 
882  return datasetNonLinearity
883 
884  def calculateLinearityResidualAndLinearizers(self, exposureTimeVector, meanSignalVector):
885  """Calculate linearity residual and fit an n-order polynomial to the mean vs time curve
886  to produce corrections (deviation from linear part of polynomial) for a particular amplifier
887  to populate LinearizeLookupTable.
888  Use the coefficients of this fit to calculate the correction coefficients for LinearizePolynomial
889  and LinearizeSquared."
890 
891  Parameters
892  ---------
893 
894  exposureTimeVector: `list` of `float`
895  List of exposure times for each flat pair
896 
897  meanSignalVector: `list` of `float`
898  List of mean signal from diference image of flat pairs
899 
900  Returns
901  -------
902  dataset : `lsst.cp.pipe.ptc.LinearityResidualsAndLinearizersDataset`
903  The dataset containing the fit parameters, the NL correction coefficients, and the
904  LUT row for the amplifier at hand.
905 
906  Notes
907  -----
908  datase members:
909 
910  dataset.polynomialLinearizerCoefficients : `list` of `float`
911  Coefficients for LinearizePolynomial, where corrImage = uncorrImage + sum_i c_i uncorrImage^(2 +
912  i).
913  c_(j-2) = -k_j/(k_1^j) with units DN^(1-j) (c.f., Eq. 37 of 2003.05978). The units of k_j are
914  DN/t^j, and they are fit from meanSignalVector = k0 + k1*exposureTimeVector +
915  k2*exposureTimeVector^2 + ... + kn*exposureTimeVector^n, with
916  n = "polynomialFitDegreeNonLinearity". k_0 and k_1 and degenerate with bias level and gain,
917  and are not used by the non-linearity correction. Therefore, j = 2...n in the above expression
918  (see `LinearizePolynomial` class in `linearize.py`.)
919 
920  dataset.quadraticPolynomialLinearizerCoefficient : `float`
921  Coefficient for LinearizeSquared, where corrImage = uncorrImage + c0*uncorrImage^2.
922  c0 = -k2/(k1^2), where k1 and k2 are fit from
923  meanSignalVector = k0 + k1*exposureTimeVector + k2*exposureTimeVector^2 +...
924  + kn*exposureTimeVector^n, with n = "polynomialFitDegreeNonLinearity".
925 
926  dataset.linearizerTableRow : `list` of `float`
927  One dimensional array with deviation from linear part of n-order polynomial fit
928  to mean vs time curve. This array will be one row (for the particular amplifier at hand)
929  of the table array for LinearizeLookupTable.
930 
931  dataset.meanSignalVsTimePolyFitPars : `list` of `float`
932  Parameters from n-order polynomial fit to meanSignalVector vs exposureTimeVector.
933 
934  dataset.meanSignalVsTimePolyFitParsErr : `list` of `float`
935  Parameters from n-order polynomial fit to meanSignalVector vs exposureTimeVector.
936 
937  dataset.meanSignalVsTimePolyFitReducedChiSq : `float`
938  Reduced unweighted chi squared from polynomial fit to meanSignalVector vs exposureTimeVector.
939  """
940 
941  # Lookup table linearizer
942  parsIniNonLinearity = self._initialParsForPolynomial(self.config.polynomialFitDegreeNonLinearity + 1)
943  if self.config.doFitBootstrap:
944  (parsFit, parsFitErr,
945  reducedChiSquaredNonLinearityFit) = fitBootstrap(parsIniNonLinearity,
946  exposureTimeVector,
947  meanSignalVector,
948  funcPolynomial,
949  weightsY=1./np.sqrt(meanSignalVector))
950  else:
951  (parsFit, parsFitErr,
952  reducedChiSquaredNonLinearityFit) = fitLeastSq(parsIniNonLinearity,
953  exposureTimeVector,
954  meanSignalVector,
955  funcPolynomial,
956  weightsY=1./np.sqrt(meanSignalVector))
957 
958  # LinearizeLookupTable:
959  # Use linear part to get time at wich signal is maxAduForLookupTableLinearizer DN
960  tMax = (self.config.maxAduForLookupTableLinearizer - parsFit[0])/parsFit[1]
961  timeRange = np.linspace(0, tMax, self.config.maxAduForLookupTableLinearizer)
962  signalIdeal = parsFit[0] + parsFit[1]*timeRange
963  signalUncorrected = funcPolynomial(parsFit, timeRange)
964  linearizerTableRow = signalIdeal - signalUncorrected # LinearizerLookupTable has corrections
965  # LinearizePolynomial and LinearizeSquared:
966  # Check that magnitude of higher order (>= 3) coefficents of the polyFit are small,
967  # i.e., less than threshold = 1e-10 (typical quadratic and cubic coefficents are ~1e-6
968  # and ~1e-12).
969  k1 = parsFit[1]
970  polynomialLinearizerCoefficients = []
971  for i, coefficient in enumerate(parsFit):
972  c = -coefficient/(k1**i)
973  polynomialLinearizerCoefficients.append(c)
974  if np.fabs(c) > 1e-10:
975  msg = f"Coefficient {c} in polynomial fit larger than threshold 1e-10."
976  self.log.warn(msg)
977  # Coefficient for LinearizedSquared. Called "c0" in linearize.py
978  c0 = polynomialLinearizerCoefficients[2]
979 
980  dataset = LinearityResidualsAndLinearizersDataset([], None, [], [], [], None)
981  dataset.polynomialLinearizerCoefficients = polynomialLinearizerCoefficients
982  dataset.quadraticPolynomialLinearizerCoefficient = c0
983  dataset.linearizerTableRow = linearizerTableRow
984  dataset.meanSignalVsTimePolyFitPars = parsFit
985  dataset.meanSignalVsTimePolyFitParsErr = parsFitErr
986  dataset.meanSignalVsTimePolyFitReducedChiSq = reducedChiSquaredNonLinearityFit
987 
988  return dataset
989 
990  def buildLinearizerObject(self, datasetNonLinearity, detector, calibDate, linearizerType, instruName='',
991  tableArray=None, log=None):
992  """Build linearizer object to persist.
993 
994  Parameters
995  ----------
996  datasetNonLinearity : `dict`
997  Dictionary of `lsst.cp.pipe.ptc.LinearityResidualsAndLinearizersDataset` objects.
998 
999  detector : `lsst.afw.cameraGeom.Detector`
1000  Detector object
1001 
1002  calibDate : `datetime.datetime`
1003  Calibration date
1004 
1005  linearizerType : `str`
1006  'LOOKUPTABLE', 'LINEARIZESQUARED', or 'LINEARIZEPOLYNOMIAL'
1007 
1008  instruName : `str`, optional
1009  Instrument name
1010 
1011  tableArray : `np.array`, optional
1012  Look-up table array with size rows=nAmps and columns=DN values
1013 
1014  log : `lsst.log.Log`, optional
1015  Logger to handle messages
1016 
1017  Returns
1018  -------
1019  linearizer : `lsst.ip.isr.Linearizer`
1020  Linearizer object
1021  """
1022  detName = detector.getName()
1023  detNum = detector.getId()
1024  if linearizerType == "LOOKUPTABLE":
1025  if tableArray is not None:
1026  linearizer = Linearizer(detector=detector, table=tableArray, log=log)
1027  else:
1028  raise RuntimeError("tableArray must be provided when creating a LookupTable linearizer")
1029  elif linearizerType in ("LINEARIZESQUARED", "LINEARIZEPOLYNOMIAL"):
1030  linearizer = Linearizer(log=log)
1031  else:
1032  raise RuntimeError("Invalid linearizerType {linearizerType} to build a Linearizer object. "
1033  "Supported: 'LOOKUPTABLE', 'LINEARIZESQUARED', or 'LINEARIZEPOLYNOMIAL'")
1034  for i, amp in enumerate(detector.getAmplifiers()):
1035  ampName = amp.getName()
1036  datasetNonLinAmp = datasetNonLinearity[ampName]
1037  if linearizerType == "LOOKUPTABLE":
1038  linearizer.linearityCoeffs[ampName] = [i, 0]
1039  linearizer.linearityType[ampName] = "LookupTable"
1040  elif linearizerType == "LINEARIZESQUARED":
1041  linearizer.fitParams[ampName] = datasetNonLinAmp.meanSignalVsTimePolyFitPars
1042  linearizer.fitParamsErr[ampName] = datasetNonLinAmp.meanSignalVsTimePolyFitParsErr
1043  linearizer.linearityFitReducedChiSquared[ampName] = (
1044  datasetNonLinAmp.meanSignalVsTimePolyFitReducedChiSq)
1045  linearizer.linearityCoeffs[ampName] = [
1046  datasetNonLinAmp.quadraticPolynomialLinearizerCoefficient]
1047  linearizer.linearityType[ampName] = "Squared"
1048  elif linearizerType == "LINEARIZEPOLYNOMIAL":
1049  linearizer.fitParams[ampName] = datasetNonLinAmp.meanSignalVsTimePolyFitPars
1050  linearizer.fitParamsErr[ampName] = datasetNonLinAmp.meanSignalVsTimePolyFitParsErr
1051  linearizer.linearityFitReducedChiSquared[ampName] = (
1052  datasetNonLinAmp.meanSignalVsTimePolyFitReducedChiSq)
1053  # Slice correction coefficients (starting at 2) for polynomial linearizer
1054  # (and squared linearizer above). The first and second are reduntant with
1055  # the bias and gain, respectively, and are not used by LinearizerPolynomial.
1056  polyLinCoeffs = np.array(datasetNonLinAmp.polynomialLinearizerCoefficients[2:])
1057  linearizer.linearityCoeffs[ampName] = polyLinCoeffs
1058  linearizer.linearityType[ampName] = "Polynomial"
1059  linearizer.linearityBBox[ampName] = amp.getBBox()
1060  linearizer.validate()
1061  calibId = f"detectorName={detName} detector={detNum} calibDate={calibDate} ccd={detNum} filter=NONE"
1062 
1063  try:
1064  raftName = detName.split("_")[0]
1065  calibId += f" raftName={raftName}"
1066  except Exception:
1067  raftname = "NONE"
1068  calibId += f" raftName={raftname}"
1069 
1070  serial = detector.getSerial()
1071  linearizer.updateMetadata(instrumentName=instruName, detectorId=f"{detNum}",
1072  calibId=calibId, serial=serial, detectorName=f"{detName}")
1073 
1074  return linearizer
1075 
1076  @staticmethod
1077  def _initialParsForPolynomial(order):
1078  assert(order >= 2)
1079  pars = np.zeros(order, dtype=np.float)
1080  pars[0] = 10
1081  pars[1] = 1
1082  pars[2:] = 0.0001
1083  return pars
1084 
1085  @staticmethod
1086  def _boundsForPolynomial(initialPars):
1087  lowers = [np.NINF for p in initialPars]
1088  uppers = [np.inf for p in initialPars]
1089  lowers[1] = 0 # no negative gains
1090  return (lowers, uppers)
1091 
1092  @staticmethod
1093  def _boundsForAstier(initialPars):
1094  lowers = [np.NINF for p in initialPars]
1095  uppers = [np.inf for p in initialPars]
1096  return (lowers, uppers)
1097 
1098  @staticmethod
1099  def _getInitialGoodPoints(means, variances, maxDeviationPositive, maxDeviationNegative):
1100  """Return a boolean array to mask bad points.
1101 
1102  A linear function has a constant ratio, so find the median
1103  value of the ratios, and exclude the points that deviate
1104  from that by more than a factor of maxDeviationPositive/negative.
1105  Asymmetric deviations are supported as we expect the PTC to turn
1106  down as the flux increases, but sometimes it anomalously turns
1107  upwards just before turning over, which ruins the fits, so it
1108  is wise to be stricter about restricting positive outliers than
1109  negative ones.
1110 
1111  Too high and points that are so bad that fit will fail will be included
1112  Too low and the non-linear points will be excluded, biasing the NL fit."""
1113  ratios = [b/a for (a, b) in zip(means, variances)]
1114  medianRatio = np.median(ratios)
1115  ratioDeviations = [(r/medianRatio)-1 for r in ratios]
1116 
1117  # so that it doesn't matter if the deviation is expressed as positive or negative
1118  maxDeviationPositive = abs(maxDeviationPositive)
1119  maxDeviationNegative = -1. * abs(maxDeviationNegative)
1120 
1121  goodPoints = np.array([True if (r < maxDeviationPositive and r > maxDeviationNegative)
1122  else False for r in ratioDeviations])
1123  return goodPoints
1124 
1125  def _makeZeroSafe(self, array, warn=True, substituteValue=1e-9):
1126  """"""
1127  nBad = Counter(array)[0]
1128  if nBad == 0:
1129  return array
1130 
1131  if warn:
1132  msg = f"Found {nBad} zeros in array at elements {[x for x in np.where(array==0)[0]]}"
1133  self.log.warn(msg)
1134 
1135  array[array == 0] = substituteValue
1136  return array
1137 
1138  def fitPtc(self, dataset, ptcFitType):
1139  """Fit the photon transfer curve to a polynimial or to Astier+19 approximation.
1140 
1141  Fit the photon transfer curve with either a polynomial of the order
1142  specified in the task config, or using the Astier approximation.
1143 
1144  Sigma clipping is performed iteratively for the fit, as well as an
1145  initial clipping of data points that are more than
1146  config.initialNonLinearityExclusionThreshold away from lying on a
1147  straight line. This other step is necessary because the photon transfer
1148  curve turns over catastrophically at very high flux (because saturation
1149  drops the variance to ~0) and these far outliers cause the initial fit
1150  to fail, meaning the sigma cannot be calculated to perform the
1151  sigma-clipping.
1152 
1153  Parameters
1154  ----------
1155  dataset : `lsst.cp.pipe.ptc.PhotonTransferCurveDataset`
1156  The dataset containing the means, variances and exposure times
1157 
1158  ptcFitType : `str`
1159  Fit a 'POLYNOMIAL' (degree: 'polynomialFitDegree') or
1160  'EXPAPPROXIMATION' (Eq. 16 of Astier+19) to the PTC
1161 
1162  Returns
1163  -------
1164  dataset: `lsst.cp.pipe.ptc.PhotonTransferCurveDataset`
1165  This is the same dataset as the input paramter, however, it has been modified
1166  to include information such as the fit vectors and the fit parameters. See
1167  the class `PhotonTransferCurveDatase`.
1168  """
1169 
1170  def errFunc(p, x, y):
1171  return ptcFunc(p, x) - y
1172 
1173  sigmaCutPtcOutliers = self.config.sigmaCutPtcOutliers
1174  maxIterationsPtcOutliers = self.config.maxIterationsPtcOutliers
1175 
1176  for i, ampName in enumerate(dataset.ampNames):
1177  timeVecOriginal = np.array(dataset.rawExpTimes[ampName])
1178  meanVecOriginal = np.array(dataset.rawMeans[ampName])
1179  varVecOriginal = np.array(dataset.rawVars[ampName])
1180  varVecOriginal = self._makeZeroSafe(varVecOriginal)
1181 
1182  mask = ((meanVecOriginal >= self.config.minMeanSignal) &
1183  (meanVecOriginal <= self.config.maxMeanSignal))
1184 
1185  goodPoints = self._getInitialGoodPoints(meanVecOriginal, varVecOriginal,
1186  self.config.initialNonLinearityExclusionThresholdPositive,
1187  self.config.initialNonLinearityExclusionThresholdNegative)
1188  mask = mask & goodPoints
1189 
1190  if ptcFitType == 'EXPAPPROXIMATION':
1191  ptcFunc = funcAstier
1192  parsIniPtc = [-1e-9, 1.0, 10.] # a00, gain, noise
1193  bounds = self._boundsForAstier(parsIniPtc)
1194  if ptcFitType == 'POLYNOMIAL':
1195  ptcFunc = funcPolynomial
1196  parsIniPtc = self._initialParsForPolynomial(self.config.polynomialFitDegree + 1)
1197  bounds = self._boundsForPolynomial(parsIniPtc)
1198 
1199  # Before bootstrap fit, do an iterative fit to get rid of outliers
1200  count = 1
1201  while count <= maxIterationsPtcOutliers:
1202  # Note that application of the mask actually shrinks the array
1203  # to size rather than setting elements to zero (as we want) so
1204  # always update mask itself and re-apply to the original data
1205  meanTempVec = meanVecOriginal[mask]
1206  varTempVec = varVecOriginal[mask]
1207  res = least_squares(errFunc, parsIniPtc, bounds=bounds, args=(meanTempVec, varTempVec))
1208  pars = res.x
1209 
1210  # change this to the original from the temp because the masks are ANDed
1211  # meaning once a point is masked it's always masked, and the masks must
1212  # always be the same length for broadcasting
1213  sigResids = (varVecOriginal - ptcFunc(pars, meanVecOriginal))/np.sqrt(varVecOriginal)
1214  newMask = np.array([True if np.abs(r) < sigmaCutPtcOutliers else False for r in sigResids])
1215  mask = mask & newMask
1216 
1217  nDroppedTotal = Counter(mask)[False]
1218  self.log.debug(f"Iteration {count}: discarded {nDroppedTotal} points in total for {ampName}")
1219  count += 1
1220  # objects should never shrink
1221  assert (len(mask) == len(timeVecOriginal) == len(meanVecOriginal) == len(varVecOriginal))
1222 
1223  dataset.visitMask[ampName] = mask # store the final mask
1224  parsIniPtc = pars
1225  meanVecFinal = meanVecOriginal[mask]
1226  varVecFinal = varVecOriginal[mask]
1227 
1228  if Counter(mask)[False] > 0:
1229  self.log.info((f"Number of points discarded in PTC of amplifier {ampName}:" +
1230  f" {Counter(mask)[False]} out of {len(meanVecOriginal)}"))
1231 
1232  if (len(meanVecFinal) < len(parsIniPtc)):
1233  msg = (f"\nSERIOUS: Not enough data points ({len(meanVecFinal)}) compared to the number of"
1234  f"parameters of the PTC model({len(parsIniPtc)}). Setting {ampName} to BAD.")
1235  self.log.warn(msg)
1236  # The first and second parameters of initial fit are discarded (bias and gain)
1237  # for the final NL coefficients
1238  dataset.badAmps.append(ampName)
1239  dataset.gain[ampName] = np.nan
1240  dataset.gainErr[ampName] = np.nan
1241  dataset.noise[ampName] = np.nan
1242  dataset.noiseErr[ampName] = np.nan
1243  dataset.ptcFitPars[ampName] = np.nan
1244  dataset.ptcFitParsError[ampName] = np.nan
1245  dataset.ptcFitReducedChiSquared[ampName] = np.nan
1246  continue
1247 
1248  # Fit the PTC
1249  if self.config.doFitBootstrap:
1250  parsFit, parsFitErr, reducedChiSqPtc = fitBootstrap(parsIniPtc, meanVecFinal,
1251  varVecFinal, ptcFunc,
1252  weightsY=1./np.sqrt(varVecFinal))
1253  else:
1254  parsFit, parsFitErr, reducedChiSqPtc = fitLeastSq(parsIniPtc, meanVecFinal,
1255  varVecFinal, ptcFunc,
1256  weightsY=1./np.sqrt(varVecFinal))
1257  dataset.ptcFitPars[ampName] = parsFit
1258  dataset.ptcFitParsError[ampName] = parsFitErr
1259  dataset.ptcFitReducedChiSquared[ampName] = reducedChiSqPtc
1260 
1261  if ptcFitType == 'EXPAPPROXIMATION':
1262  ptcGain = parsFit[1]
1263  ptcGainErr = parsFitErr[1]
1264  ptcNoise = np.sqrt(np.fabs(parsFit[2]))
1265  ptcNoiseErr = 0.5*(parsFitErr[2]/np.fabs(parsFit[2]))*np.sqrt(np.fabs(parsFit[2]))
1266  if ptcFitType == 'POLYNOMIAL':
1267  ptcGain = 1./parsFit[1]
1268  ptcGainErr = np.fabs(1./parsFit[1])*(parsFitErr[1]/parsFit[1])
1269  ptcNoise = np.sqrt(np.fabs(parsFit[0]))*ptcGain
1270  ptcNoiseErr = (0.5*(parsFitErr[0]/np.fabs(parsFit[0]))*(np.sqrt(np.fabs(parsFit[0]))))*ptcGain
1271  dataset.gain[ampName] = ptcGain
1272  dataset.gainErr[ampName] = ptcGainErr
1273  dataset.noise[ampName] = ptcNoise
1274  dataset.noiseErr[ampName] = ptcNoiseErr
1275  if not len(dataset.ptcFitType) == 0:
1276  dataset.ptcFitType = ptcFitType
1277 
1278  return dataset
lsst::ip::isr::linearize::Linearizer
lsst.cp.pipe.ptc.PhotonTransferCurveDataset.__setattr__
def __setattr__(self, attribute, value)
Definition: ptc.py:281
lsst.cp.pipe.ptc.MeasurePhotonTransferCurveTask.fitCovariancesAstier
def fitCovariancesAstier(self, dataset, covariancesWithTagsArray)
Definition: ptc.py:497
lsst.cp.pipe.ptc.PhotonTransferCurveDataset
Definition: ptc.py:202
lsst.cp.pipe.ptc.MeasurePhotonTransferCurveTask.fitPtc
def fitPtc(self, dataset, ptcFitType)
Definition: ptc.py:1138
lsst.cp.pipe.ptc.MeasurePhotonTransferCurveTask.fitNonLinearityAndBuildLinearizers
def fitNonLinearityAndBuildLinearizers(self, datasetPtc, detector, tableArray=None, log=None)
Definition: ptc.py:796
lsst.cp.pipe.astierCovPtcUtils.fftSize
def fftSize(s)
Definition: astierCovPtcUtils.py:122
lsst.cp.pipe.utils.funcPolynomial
def funcPolynomial(pars, x)
Definition: utils.py:423
lsst.cp.pipe.ptc.MeasurePhotonTransferCurveTask._boundsForAstier
def _boundsForAstier(initialPars)
Definition: ptc.py:1093
lsst.cp.pipe.ptc.MeasurePhotonTransferCurveTask.measureMeanVarCov
def measureMeanVarCov(self, exposure1, exposure2, region=None, covAstierRealSpace=False)
Definition: ptc.py:574
lsst.cp.pipe.astierCovPtcUtils.fitData
def fitData(tupleName, maxMu=1e9, r=8, nSigmaFullFit=5.5, maxIterFullFit=3)
Definition: astierCovPtcUtils.py:323
lsst.cp.pipe.ptc.MeasurePhotonTransferCurveTask.__init__
def __init__(self, *args, **kwargs)
Definition: ptc.py:348
lsst.cp.pipe.ptc.MeasurePhotonTransferCurveTask._boundsForPolynomial
def _boundsForPolynomial(initialPars)
Definition: ptc.py:1086
lsst.cp.pipe.ptc.MeasurePhotonTransferCurveTask.getOutputPtcDataCovAstier
def getOutputPtcDataCovAstier(self, dataset, covFits)
Definition: ptc.py:536
lsst.cp.pipe.ptc.MeasurePhotonTransferCurveTaskConfig
Definition: ptc.py:46
lsst.cp.pipe.ptc.PhotonTransferCurveDataset.getGoodAmps
def getGoodAmps(self)
Definition: ptc.py:305
lsst.cp.pipe.ptc.PhotonTransferCurveDataset.getVisitsUsed
def getVisitsUsed(self, ampName)
Definition: ptc.py:289
lsst.cp.pipe.ptc.MeasurePhotonTransferCurveTask.covDirectValue
def covDirectValue(self, diffImage, weightImage, dx, dy)
Definition: ptc.py:740
lsst.cp.pipe.ptc.MeasurePhotonTransferCurveTask._DefaultName
string _DefaultName
Definition: ptc.py:346
lsst::pex::config
lsst.cp.pipe.astierCovPtcUtils.CovFft
Definition: astierCovPtcUtils.py:28
lsst.cp.pipe.utils.fitLeastSq
def fitLeastSq(initialParams, dataX, dataY, function, weightsY=None)
Definition: utils.py:285
lsst.cp.pipe.utils.checkExpLengthEqual
def checkExpLengthEqual(exp1, exp2, v1=None, v2=None, raiseWithMessage=False)
Definition: utils.py:459
lsst.cp.pipe.ptc.MeasurePhotonTransferCurveTask
Definition: ptc.py:309
lsst.cp.pipe.ptc.MeasurePhotonTransferCurveTask._makeZeroSafe
def _makeZeroSafe(self, array, warn=True, substituteValue=1e-9)
Definition: ptc.py:1125
lsst.cp.pipe.ptc.MeasurePhotonTransferCurveTask._getInitialGoodPoints
def _getInitialGoodPoints(means, variances, maxDeviationPositive, maxDeviationNegative)
Definition: ptc.py:1099
lsst::ip::isr::linearize
lsst.cp.pipe.ptc.MeasurePhotonTransferCurveTask.runDataRef
def runDataRef(self, dataRef, visitPairs)
Definition: ptc.py:366
lsst::afw::math
lsst.cp.pipe.ptc.LinearityResidualsAndLinearizersDataset
Definition: ptc.py:187
lsst.cp.pipe.ptc.MeasurePhotonTransferCurveTask._initialParsForPolynomial
def _initialParsForPolynomial(order)
Definition: ptc.py:1077
lsst.cp.pipe.utils.fitBootstrap
def fitBootstrap(initialParams, dataX, dataY, function, weightsY=None, confidenceSigma=1.)
Definition: utils.py:352
lsst.cp.pipe.ptc.MeasurePhotonTransferCurveTask.computeCovDirect
def computeCovDirect(self, diffImage, weightImage, maxRange)
Definition: ptc.py:690
lsst.cp.pipe.ptc.MeasurePhotonTransferCurveTask.fitNonLinearity
def fitNonLinearity(self, datasetPtc, tableArray=None)
Definition: ptc.py:841
lsst::pipe::base
lsst.cp.pipe.ptc.MeasurePhotonTransferCurveTask.calculateLinearityResidualAndLinearizers
def calculateLinearityResidualAndLinearizers(self, exposureTimeVector, meanSignalVector)
Definition: ptc.py:884
lsst.cp.pipe.ptc.MeasurePhotonTransferCurveTask.buildLinearizerObject
def buildLinearizerObject(self, datasetNonLinearity, detector, calibDate, linearizerType, instruName='', tableArray=None, log=None)
Definition: ptc.py:990
lsst.cp.pipe.ptc.PhotonTransferCurveDataset.__init__
def __init__(self, ampNames, ptcFitType)
Definition: ptc.py:236