lsst.cp.pipe  20.0.0-22-gc512666+9eba1c4719
ptc.py
Go to the documentation of this file.
1 # This file is part of cp_pipe.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 #
22 import numpy as np
23 import matplotlib.pyplot as plt
24 from collections import Counter
25 
26 import lsst.afw.math as afwMath
27 import lsst.pex.config as pexConfig
28 import lsst.pipe.base as pipeBase
29 from .utils import (fitLeastSq, fitBootstrap, funcPolynomial, funcAstier)
30 from scipy.optimize import least_squares
31 
32 import datetime
33 
34 from .astierCovPtcUtils import (fftSize, CovFft, computeCovDirect, fitData)
35 from .linearity import LinearitySolveTask
36 from .photodiode import getBOTphotodiodeData
37 
38 from lsst.pipe.tasks.getRepositoryData import DataRefListRunner
39 from lsst.ip.isr import PhotonTransferCurveDataset
40 
41 __all__ = ['MeasurePhotonTransferCurveTask',
42  'MeasurePhotonTransferCurveTaskConfig']
43 
44 
45 class MeasurePhotonTransferCurveTaskConfig(pexConfig.Config):
46  """Config class for photon transfer curve measurement task"""
47  ccdKey = pexConfig.Field(
48  dtype=str,
49  doc="The key by which to pull a detector from a dataId, e.g. 'ccd' or 'detector'.",
50  default='ccd',
51  )
52  ptcFitType = pexConfig.ChoiceField(
53  dtype=str,
54  doc="Fit PTC to Eq. 16, Eq. 20 in Astier+19, or to a polynomial.",
55  default="POLYNOMIAL",
56  allowed={
57  "POLYNOMIAL": "n-degree polynomial (use 'polynomialFitDegree' to set 'n').",
58  "EXPAPPROXIMATION": "Approximation in Astier+19 (Eq. 16).",
59  "FULLCOVARIANCE": "Full covariances model in Astier+19 (Eq. 20)"
60  }
61  )
62  sigmaClipFullFitCovariancesAstier = pexConfig.Field(
63  dtype=float,
64  doc="sigma clip for full model fit for FULLCOVARIANCE ptcFitType ",
65  default=5.0,
66  )
67  maxIterFullFitCovariancesAstier = pexConfig.Field(
68  dtype=int,
69  doc="Maximum number of iterations in full model fit for FULLCOVARIANCE ptcFitType",
70  default=3,
71  )
72  maximumRangeCovariancesAstier = pexConfig.Field(
73  dtype=int,
74  doc="Maximum range of covariances as in Astier+19",
75  default=8,
76  )
77  covAstierRealSpace = pexConfig.Field(
78  dtype=bool,
79  doc="Calculate covariances in real space or via FFT? (see appendix A of Astier+19).",
80  default=False,
81  )
82  polynomialFitDegree = pexConfig.Field(
83  dtype=int,
84  doc="Degree of polynomial to fit the PTC, when 'ptcFitType'=POLYNOMIAL.",
85  default=3,
86  )
87  linearity = pexConfig.ConfigurableField(
88  target=LinearitySolveTask,
89  doc="Task to solve the linearity."
90  )
91 
92  doCreateLinearizer = pexConfig.Field(
93  dtype=bool,
94  doc="Calculate non-linearity and persist linearizer?",
95  default=False,
96  )
97 
98  binSize = pexConfig.Field(
99  dtype=int,
100  doc="Bin the image by this factor in both dimensions.",
101  default=1,
102  )
103  minMeanSignal = pexConfig.Field(
104  dtype=float,
105  doc="Minimum value (inclusive) of mean signal (in DN) above which to consider.",
106  default=0,
107  )
108  maxMeanSignal = pexConfig.Field(
109  dtype=float,
110  doc="Maximum value (inclusive) of mean signal (in DN) below which to consider.",
111  default=9e6,
112  )
113  initialNonLinearityExclusionThresholdPositive = pexConfig.RangeField(
114  dtype=float,
115  doc="Initially exclude data points with a variance that are more than a factor of this from being"
116  " linear in the positive direction, from the PTC fit. Note that these points will also be"
117  " excluded from the non-linearity fit. This is done before the iterative outlier rejection,"
118  " to allow an accurate determination of the sigmas for said iterative fit.",
119  default=0.12,
120  min=0.0,
121  max=1.0,
122  )
123  initialNonLinearityExclusionThresholdNegative = pexConfig.RangeField(
124  dtype=float,
125  doc="Initially exclude data points with a variance that are more than a factor of this from being"
126  " linear in the negative direction, from the PTC fit. Note that these points will also be"
127  " excluded from the non-linearity fit. This is done before the iterative outlier rejection,"
128  " to allow an accurate determination of the sigmas for said iterative fit.",
129  default=0.25,
130  min=0.0,
131  max=1.0,
132  )
133  sigmaCutPtcOutliers = pexConfig.Field(
134  dtype=float,
135  doc="Sigma cut for outlier rejection in PTC.",
136  default=5.0,
137  )
138  maskNameList = pexConfig.ListField(
139  dtype=str,
140  doc="Mask list to exclude from statistics calculations.",
141  default=['SUSPECT', 'BAD', 'NO_DATA'],
142  )
143  nSigmaClipPtc = pexConfig.Field(
144  dtype=float,
145  doc="Sigma cut for afwMath.StatisticsControl()",
146  default=5.5,
147  )
148  nIterSigmaClipPtc = pexConfig.Field(
149  dtype=int,
150  doc="Number of sigma-clipping iterations for afwMath.StatisticsControl()",
151  default=1,
152  )
153  maxIterationsPtcOutliers = pexConfig.Field(
154  dtype=int,
155  doc="Maximum number of iterations for outlier rejection in PTC.",
156  default=2,
157  )
158  doFitBootstrap = pexConfig.Field(
159  dtype=bool,
160  doc="Use bootstrap for the PTC fit parameters and errors?.",
161  default=False,
162  )
163  doPhotodiode = pexConfig.Field(
164  dtype=bool,
165  doc="Apply a correction based on the photodiode readings if available?",
166  default=True,
167  )
168  photodiodeDataPath = pexConfig.Field(
169  dtype=str,
170  doc="Gen2 only: path to locate the data photodiode data files.",
171  default=""
172  )
173  instrumentName = pexConfig.Field(
174  dtype=str,
175  doc="Instrument name.",
176  default='',
177  )
178 
179 
180 class MeasurePhotonTransferCurveTask(pipeBase.CmdLineTask):
181  """A class to calculate, fit, and plot a PTC from a set of flat pairs.
182 
183  The Photon Transfer Curve (var(signal) vs mean(signal)) is a standard tool
184  used in astronomical detectors characterization (e.g., Janesick 2001,
185  Janesick 2007). If ptcFitType is "EXPAPPROXIMATION" or "POLYNOMIAL", this task calculates the
186  PTC from a series of pairs of flat-field images; each pair taken at identical exposure
187  times. The difference image of each pair is formed to eliminate fixed pattern noise,
188  and then the variance of the difference image and the mean of the average image
189  are used to produce the PTC. An n-degree polynomial or the approximation in Equation
190  16 of Astier+19 ("The Shape of the Photon Transfer Curve of CCD sensors",
191  arXiv:1905.08677) can be fitted to the PTC curve. These models include
192  parameters such as the gain (e/DN) and readout noise.
193 
194  Linearizers to correct for signal-chain non-linearity are also calculated.
195  The `Linearizer` class, in general, can support per-amp linearizers, but in this
196  task this is not supported.
197 
198  If ptcFitType is "FULLCOVARIANCE", the covariances of the difference images are calculated via the
199  DFT methods described in Astier+19 and the variances for the PTC are given by the cov[0,0] elements
200  at each signal level. The full model in Equation 20 of Astier+19 is fit to the PTC to get the gain
201  and the noise.
202 
203  Parameters
204  ----------
205 
206  *args: `list`
207  Positional arguments passed to the Task constructor. None used at this
208  time.
209  **kwargs: `dict`
210  Keyword arguments passed on to the Task constructor. None used at this
211  time.
212 
213  """
214 
215  RunnerClass = DataRefListRunner
216  ConfigClass = MeasurePhotonTransferCurveTaskConfig
217  _DefaultName = "measurePhotonTransferCurve"
218 
219  def __init__(self, *args, **kwargs):
220  pipeBase.CmdLineTask.__init__(self, *args, **kwargs)
221  self.makeSubtask("linearity")
222  plt.interactive(False) # stop windows popping up when plotting. When headless, use 'agg' backend too
223  self.config.validate()
224  self.config.freeze()
225 
226  @pipeBase.timeMethod
227  def runDataRef(self, dataRefList):
228  """Run the Photon Transfer Curve (PTC) measurement task.
229 
230  For a dataRef (which is each detector here),
231  and given a list of exposure pairs (postISR) at different exposure times,
232  measure the PTC.
233 
234  Parameters
235  ----------
236  dataRefList : `list` [`lsst.daf.peristence.ButlerDataRef`]
237  Data references for exposures for detectors to process.
238  """
239  if len(dataRefList) < 2:
240  raise RuntimeError("Insufficient inputs to combine.")
241 
242  # setup necessary objects
243  dataRef = dataRefList[0]
244 
245  detNum = dataRef.dataId[self.config.ccdKey]
246  camera = dataRef.get('camera')
247  detector = camera[dataRef.dataId[self.config.ccdKey]]
248 
249  amps = detector.getAmplifiers()
250  ampNames = [amp.getName() for amp in amps]
251  datasetPtc = PhotonTransferCurveDataset(ampNames, self.config.ptcFitType)
252 
253  # Get the pairs of flat indexed by expTime
254  expPairs = self.makePairs(dataRefList)
255  expIds = []
256  for (exp1, exp2) in expPairs.values():
257  id1 = exp1.getInfo().getVisitInfo().getExposureId()
258  id2 = exp2.getInfo().getVisitInfo().getExposureId()
259  expIds.append((id1, id2))
260  self.log.info(f"Measuring PTC using {expIds} exposures for detector {detector.getId()}")
261 
262  # get photodiode data early so that logic can be put in to only use the
263  # data if all files are found, as partial corrections are not possible
264  # or at least require significant logic to deal with
265  if self.config.doPhotodiode:
266  for (expId1, expId2) in expIds:
267  charges = [-1, -1] # necessary to have a not-found value to keep lists in step
268  for i, expId in enumerate([expId1, expId2]):
269  # //1000 is a Gen2 only hack, working around the fact an
270  # exposure's ID is not the same as the expId in the
271  # registry. Currently expId is concatenated with the
272  # zero-padded detector ID. This will all go away in Gen3.
273  dataRef.dataId['expId'] = expId//1000
274  if self.config.photodiodeDataPath:
275  photodiodeData = getBOTphotodiodeData(dataRef, self.config.photodiodeDataPath)
276  else:
277  photodiodeData = getBOTphotodiodeData(dataRef)
278  if photodiodeData: # default path stored in function def to keep task clean
279  charges[i] = photodiodeData.getCharge()
280  else:
281  # full expId (not //1000) here, as that encodes the
282  # the detector number as so is fully qualifying
283  self.log.warn(f"No photodiode data found for {expId}")
284 
285  for ampName in ampNames:
286  datasetPtc.photoCharge[ampName].append((charges[0], charges[1]))
287  else:
288  # Can't be an empty list, as initialized, because astropy.Table won't allow it
289  # when saving as fits
290  for ampName in ampNames:
291  datasetPtc.photoCharge[ampName] = np.repeat(np.nan, len(expIds))
292 
293  for ampName in ampNames:
294  datasetPtc.inputExpIdPairs[ampName] = expIds
295 
296  tupleRecords = []
297  allTags = []
298  for expTime, (exp1, exp2) in expPairs.items():
299  expId1 = exp1.getInfo().getVisitInfo().getExposureId()
300  expId2 = exp2.getInfo().getVisitInfo().getExposureId()
301  tupleRows = []
302  nAmpsNan = 0
303  for ampNumber, amp in enumerate(detector):
304  ampName = amp.getName()
305  # covAstier: (i, j, var (cov[0,0]), cov, npix)
306  doRealSpace = self.config.covAstierRealSpace
307  muDiff, varDiff, covAstier = self.measureMeanVarCov(exp1, exp2, region=amp.getBBox(),
308  covAstierRealSpace=doRealSpace)
309  datasetPtc.rawExpTimes[ampName].append(expTime)
310  datasetPtc.rawMeans[ampName].append(muDiff)
311  datasetPtc.rawVars[ampName].append(varDiff)
312 
313  if np.isnan(muDiff) or np.isnan(varDiff) or (covAstier is None):
314  msg = (f"NaN mean or var, or None cov in amp {ampName} in exposure pair {expId1},"
315  f" {expId2} of detector {detNum}.")
316  self.log.warn(msg)
317  nAmpsNan += 1
318  continue
319  tags = ['mu', 'i', 'j', 'var', 'cov', 'npix', 'ext', 'expTime', 'ampName']
320  if (muDiff <= self.config.minMeanSignal) or (muDiff >= self.config.maxMeanSignal):
321  continue
322 
323  tupleRows += [(muDiff, ) + covRow + (ampNumber, expTime, ampName) for covRow in covAstier]
324  if nAmpsNan == len(ampNames):
325  msg = f"NaN mean in all amps of exposure pair {expId1}, {expId2} of detector {detNum}."
326  self.log.warn(msg)
327  continue
328  allTags += tags
329  tupleRecords += tupleRows
330  covariancesWithTags = np.core.records.fromrecords(tupleRecords, names=allTags)
331 
332  if self.config.ptcFitType in ["FULLCOVARIANCE", ]:
333  # Calculate covariances and fit them, including the PTC, to Astier+19 full model (Eq. 20)
334  datasetPtc = self.fitCovariancesAstier(datasetPtc, covariancesWithTags)
335  elif self.config.ptcFitType in ["EXPAPPROXIMATION", "POLYNOMIAL"]:
336  # Fit the PTC to a polynomial or to Astier+19 exponential approximation (Eq. 16)
337  # Fill up PhotonTransferCurveDataset object.
338  datasetPtc = self.fitPtc(datasetPtc, self.config.ptcFitType)
339 
340  detName = detector.getName()
341  now = datetime.datetime.utcnow()
342  calibDate = now.strftime("%Y-%m-%d")
343  butler = dataRef.getButler()
344 
345  datasetPtc.updateMetadata(setDate=True, camera=camera, detector=detector)
346 
347  # Fit a poynomial to calculate non-linearity and persist linearizer.
348  if self.config.doCreateLinearizer:
349  # Fit (non)linearity of signal vs time curve.
350  # Fill up PhotonTransferCurveDataset object.
351  # Fill up array for LUT linearizer (tableArray).
352  # Produce coefficients for Polynomial and Squared linearizers.
353  # Build linearizer objects.
354  dimensions = {'camera': camera.getName(), 'detector': detector.getId()}
355  linearityResults = self.linearity.run(datasetPtc, camera, dimensions)
356  linearizer = linearityResults.outputLinearizer
357 
358  self.log.info("Writing linearizer:")
359 
360  detName = detector.getName()
361  now = datetime.datetime.utcnow()
362  calibDate = now.strftime("%Y-%m-%d")
363 
364  butler.put(linearizer, datasetType='linearizer',
365  dataId={'detector': detNum, 'detectorName': detName, 'calibDate': calibDate})
366 
367  self.log.info(f"Writing PTC data.")
368  butler.put(datasetPtc, datasetType='photonTransferCurveDataset', dataId={'detector': detNum,
369  'detectorName': detName, 'calibDate': calibDate})
370 
371  return pipeBase.Struct(exitStatus=0)
372 
373  def makePairs(self, dataRefList):
374  """Produce a list of flat pairs indexed by exposure time.
375 
376  Parameters
377  ----------
378  dataRefList : `list` [`lsst.daf.peristence.ButlerDataRef`]
379  Data references for exposures for detectors to process.
380 
381  Return
382  ------
383  flatPairs : `dict` [`float`, `lsst.afw.image.exposure.exposure.ExposureF`]
384  Dictionary that groups flat-field exposures that have the same exposure time (seconds).
385 
386  Notes
387  -----
388  We use the difference of one pair of flat-field images taken at the same exposure time when
389  calculating the PTC to reduce Fixed Pattern Noise. If there are > 2 flat-field images with the
390  same exposure time, the first two are kept and the rest discarded.
391  """
392 
393  # Organize exposures by observation date.
394  expDict = {}
395  for dataRef in dataRefList:
396  try:
397  tempFlat = dataRef.get("postISRCCD")
398  except RuntimeError:
399  self.log.warn("postISR exposure could not be retrieved. Ignoring flat.")
400  continue
401  expDate = tempFlat.getInfo().getVisitInfo().getDate().get()
402  expDict.setdefault(expDate, tempFlat)
403  sortedExps = {k: expDict[k] for k in sorted(expDict)}
404 
405  flatPairs = {}
406  for exp in sortedExps:
407  tempFlat = sortedExps[exp]
408  expTime = tempFlat.getInfo().getVisitInfo().getExposureTime()
409  listAtExpTime = flatPairs.setdefault(expTime, [])
410  if len(listAtExpTime) >= 2:
411  self.log.warn(f"Already found 2 exposures at expTime {expTime}. "
412  f"Ignoring exposure {tempFlat.getInfo().getVisitInfo().getExposureId()}")
413  else:
414  listAtExpTime.append(tempFlat)
415 
416  keysToDrop = []
417  for (key, value) in flatPairs.items():
418  if len(value) < 2:
419  keysToDrop.append(key)
420 
421  if len(keysToDrop):
422  for key in keysToDrop:
423  self.log.warn(f"Only one exposure found at expTime {key}. Dropping exposure "
424  f"{flatPairs[key][0].getInfo().getVisitInfo().getExposureId()}.")
425  flatPairs.pop(key)
426  return flatPairs
427 
428  def fitCovariancesAstier(self, dataset, covariancesWithTagsArray):
429  """Fit measured flat covariances to full model in Astier+19.
430 
431  Parameters
432  ----------
433  dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset`
434  The dataset containing information such as the means, variances and exposure times.
435 
436  covariancesWithTagsArray : `numpy.recarray`
437  Tuple with at least (mu, cov, var, i, j, npix), where:
438  mu : 0.5*(m1 + m2), where:
439  mu1: mean value of flat1
440  mu2: mean value of flat2
441  cov: covariance value at lag(i, j)
442  var: variance(covariance value at lag(0, 0))
443  i: lag dimension
444  j: lag dimension
445  npix: number of pixels used for covariance calculation.
446 
447  Returns
448  -------
449  dataset: `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset`
450  This is the same dataset as the input paramter, however, it has been modified
451  to include information such as the fit vectors and the fit parameters. See
452  the class `PhotonTransferCurveDatase`.
453  """
454 
455  covFits, covFitsNoB = fitData(covariancesWithTagsArray, maxMu=self.config.maxMeanSignal,
456  r=self.config.maximumRangeCovariancesAstier,
457  nSigmaFullFit=self.config.sigmaClipFullFitCovariancesAstier,
458  maxIterFullFit=self.config.maxIterFullFitCovariancesAstier)
459 
460  dataset = self.getOutputPtcDataCovAstier(dataset, covFits, covFitsNoB)
461 
462  return dataset
463 
464  def getOutputPtcDataCovAstier(self, dataset, covFits, covFitsNoB):
465  """Get output data for PhotonTransferCurveCovAstierDataset from CovFit objects.
466 
467  Parameters
468  ----------
469  dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset`
470  The dataset containing information such as the means, variances and exposure times.
471 
472  covFits: `dict`
473  Dictionary of CovFit objects, with amp names as keys.
474 
475  covFitsNoB : `dict`
476  Dictionary of CovFit objects, with amp names as keys, and 'b=0' in Eq. 20 of Astier+19.
477 
478  Returns
479  -------
480  dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset`
481  This is the same dataset as the input paramter, however, it has been modified
482  to include extra information such as the mask 1D array, gains, reoudout noise, measured signal,
483  measured variance, modeled variance, a, and b coefficient matrices (see Astier+19) per amplifier.
484  See the class `PhotonTransferCurveDatase`.
485  """
486  assert(len(covFits) == len(covFitsNoB))
487 
488  for i, amp in enumerate(dataset.ampNames):
489  lenInputTimes = len(dataset.rawExpTimes[amp])
490  # Not used when ptcFitType is 'FULLCOVARIANCE'
491  dataset.ptcFitPars[amp] = np.nan
492  dataset.ptcFitParsError[amp] = np.nan
493  dataset.ptcFitChiSq[amp] = np.nan
494  if amp in covFits:
495  fit = covFits[amp]
496  fitNoB = covFitsNoB[amp]
497  # Save full covariances, covariances models, and their weights
498  dataset.covariances[amp] = fit.cov
499  dataset.covariancesModel[amp] = fit.evalCovModel()
500  dataset.covariancesSqrtWeights[amp] = fit.sqrtW
501  dataset.aMatrix[amp] = fit.getA()
502  dataset.bMatrix[amp] = fit.getB()
503  dataset.covariancesNoB[amp] = fitNoB.cov
504  dataset.covariancesModelNoB[amp] = fitNoB.evalCovModel()
505  dataset.covariancesSqrtWeightsNoB[amp] = fitNoB.sqrtW
506  dataset.aMatrixNoB[amp] = fitNoB.getA()
507 
508  (meanVecFinal, varVecFinal, varVecModel,
509  wc, varMask) = fit.getFitData(0, 0, divideByMu=False, returnMasked=True)
510  gain = fit.getGain()
511  dataset.expIdMask[amp] = varMask
512  dataset.gain[amp] = gain
513  dataset.gainErr[amp] = fit.getGainErr()
514  dataset.noise[amp] = np.sqrt(fit.getRon())
515  dataset.noiseErr[amp] = fit.getRonErr()
516 
517  padLength = lenInputTimes - len(varVecFinal)
518  dataset.finalVars[amp] = np.pad(varVecFinal/(gain**2), (0, padLength), 'constant',
519  constant_values=np.nan)
520  dataset.finalModelVars[amp] = np.pad(varVecModel/(gain**2), (0, padLength), 'constant',
521  constant_values=np.nan)
522  dataset.finalMeans[amp] = np.pad(meanVecFinal/gain, (0, padLength), 'constant',
523  constant_values=np.nan)
524  else:
525  # Bad amp
526  # Entries need to have proper dimensions so read/write with astropy.Table works.
527  matrixSide = self.config.maximumRangeCovariancesAstier
528  nanMatrix = np.full((matrixSide, matrixSide), np.nan)
529  listNanMatrix = np.full((lenInputTimes, matrixSide, matrixSide), np.nan)
530 
531  dataset.covariances[amp] = listNanMatrix
532  dataset.covariancesModel[amp] = listNanMatrix
533  dataset.covariancesSqrtWeights[amp] = listNanMatrix
534  dataset.aMatrix[amp] = nanMatrix
535  dataset.bMatrix[amp] = nanMatrix
536  dataset.covariancesNoB[amp] = listNanMatrix
537  dataset.covariancesModelNoB[amp] = listNanMatrix
538  dataset.covariancesSqrtWeightsNoB[amp] = listNanMatrix
539  dataset.aMatrixNoB[amp] = nanMatrix
540 
541  dataset.expIdMask[amp] = np.repeat(np.nan, lenInputTimes)
542  dataset.gain[amp] = np.nan
543  dataset.gainErr[amp] = np.nan
544  dataset.noise[amp] = np.nan
545  dataset.noiseErr[amp] = np.nan
546  dataset.finalVars[amp] = np.repeat(np.nan, lenInputTimes)
547  dataset.finalModelVars[amp] = np.repeat(np.nan, lenInputTimes)
548  dataset.finalMeans[amp] = np.repeat(np.nan, lenInputTimes)
549 
550  return dataset
551 
552  def measureMeanVarCov(self, exposure1, exposure2, region=None, covAstierRealSpace=False):
553  """Calculate the mean of each of two exposures and the variance and covariance of their difference.
554 
555  The variance is calculated via afwMath, and the covariance via the methods in Astier+19 (appendix A).
556  In theory, var = covariance[0,0]. This should be validated, and in the future, we may decide to just
557  keep one (covariance).
558 
559  Parameters
560  ----------
561  exposure1 : `lsst.afw.image.exposure.exposure.ExposureF`
562  First exposure of flat field pair.
563 
564  exposure2 : `lsst.afw.image.exposure.exposure.ExposureF`
565  Second exposure of flat field pair.
566 
567  region : `lsst.geom.Box2I`, optional
568  Region of each exposure where to perform the calculations (e.g, an amplifier).
569 
570  covAstierRealSpace : `bool`, optional
571  Should the covariannces in Astier+19 be calculated in real space or via FFT?
572  See Appendix A of Astier+19.
573 
574  Returns
575  -------
576  mu : `float` or `NaN`
577  0.5*(mu1 + mu2), where mu1, and mu2 are the clipped means of the regions in
578  both exposures. If either mu1 or m2 are NaN's, the returned value is NaN.
579 
580  varDiff : `float` or `NaN`
581  Half of the clipped variance of the difference of the regions inthe two input
582  exposures. If either mu1 or m2 are NaN's, the returned value is NaN.
583 
584  covDiffAstier : `list` or `NaN`
585  List with tuples of the form (dx, dy, var, cov, npix), where:
586  dx : `int`
587  Lag in x
588  dy : `int`
589  Lag in y
590  var : `float`
591  Variance at (dx, dy).
592  cov : `float`
593  Covariance at (dx, dy).
594  nPix : `int`
595  Number of pixel pairs used to evaluate var and cov.
596  If either mu1 or m2 are NaN's, the returned value is NaN.
597  """
598 
599  if region is not None:
600  im1Area = exposure1.maskedImage[region]
601  im2Area = exposure2.maskedImage[region]
602  else:
603  im1Area = exposure1.maskedImage
604  im2Area = exposure2.maskedImage
605 
606  if self.config.binSize > 1:
607  im1Area = afwMath.binImage(im1Area, self.config.binSize)
608  im2Area = afwMath.binImage(im2Area, self.config.binSize)
609 
610  im1MaskVal = exposure1.getMask().getPlaneBitMask(self.config.maskNameList)
611  im1StatsCtrl = afwMath.StatisticsControl(self.config.nSigmaClipPtc,
612  self.config.nIterSigmaClipPtc,
613  im1MaskVal)
614  im1StatsCtrl.setNanSafe(True)
615  im1StatsCtrl.setAndMask(im1MaskVal)
616 
617  im2MaskVal = exposure2.getMask().getPlaneBitMask(self.config.maskNameList)
618  im2StatsCtrl = afwMath.StatisticsControl(self.config.nSigmaClipPtc,
619  self.config.nIterSigmaClipPtc,
620  im2MaskVal)
621  im2StatsCtrl.setNanSafe(True)
622  im2StatsCtrl.setAndMask(im2MaskVal)
623 
624  # Clipped mean of images; then average of mean.
625  mu1 = afwMath.makeStatistics(im1Area, afwMath.MEANCLIP, im1StatsCtrl).getValue()
626  mu2 = afwMath.makeStatistics(im2Area, afwMath.MEANCLIP, im2StatsCtrl).getValue()
627  if np.isnan(mu1) or np.isnan(mu2):
628  return np.nan, np.nan, None
629  mu = 0.5*(mu1 + mu2)
630 
631  # Take difference of pairs
632  # symmetric formula: diff = (mu2*im1-mu1*im2)/(0.5*(mu1+mu2))
633  temp = im2Area.clone()
634  temp *= mu1
635  diffIm = im1Area.clone()
636  diffIm *= mu2
637  diffIm -= temp
638  diffIm /= mu
639 
640  diffImMaskVal = diffIm.getMask().getPlaneBitMask(self.config.maskNameList)
641  diffImStatsCtrl = afwMath.StatisticsControl(self.config.nSigmaClipPtc,
642  self.config.nIterSigmaClipPtc,
643  diffImMaskVal)
644  diffImStatsCtrl.setNanSafe(True)
645  diffImStatsCtrl.setAndMask(diffImMaskVal)
646 
647  varDiff = 0.5*(afwMath.makeStatistics(diffIm, afwMath.VARIANCECLIP, diffImStatsCtrl).getValue())
648 
649  # Get the mask and identify good pixels as '1', and the rest as '0'.
650  w1 = np.where(im1Area.getMask().getArray() == 0, 1, 0)
651  w2 = np.where(im2Area.getMask().getArray() == 0, 1, 0)
652 
653  w12 = w1*w2
654  wDiff = np.where(diffIm.getMask().getArray() == 0, 1, 0)
655  w = w12*wDiff
656 
657  maxRangeCov = self.config.maximumRangeCovariancesAstier
658  if covAstierRealSpace:
659  covDiffAstier = computeCovDirect(diffIm.getImage().getArray(), w, maxRangeCov)
660  else:
661  shapeDiff = diffIm.getImage().getArray().shape
662  fftShape = (fftSize(shapeDiff[0] + maxRangeCov), fftSize(shapeDiff[1]+maxRangeCov))
663  c = CovFft(diffIm.getImage().getArray(), w, fftShape, maxRangeCov)
664  covDiffAstier = c.reportCovFft(maxRangeCov)
665 
666  return mu, varDiff, covDiffAstier
667 
668  def computeCovDirect(self, diffImage, weightImage, maxRange):
669  """Compute covariances of diffImage in real space.
670 
671  For lags larger than ~25, it is slower than the FFT way.
672  Taken from https://github.com/PierreAstier/bfptc/
673 
674  Parameters
675  ----------
676  diffImage : `numpy.array`
677  Image to compute the covariance of.
678 
679  weightImage : `numpy.array`
680  Weight image of diffImage (1's and 0's for good and bad pixels, respectively).
681 
682  maxRange : `int`
683  Last index of the covariance to be computed.
684 
685  Returns
686  -------
687  outList : `list`
688  List with tuples of the form (dx, dy, var, cov, npix), where:
689  dx : `int`
690  Lag in x
691  dy : `int`
692  Lag in y
693  var : `float`
694  Variance at (dx, dy).
695  cov : `float`
696  Covariance at (dx, dy).
697  nPix : `int`
698  Number of pixel pairs used to evaluate var and cov.
699  """
700  outList = []
701  var = 0
702  # (dy,dx) = (0,0) has to be first
703  for dy in range(maxRange + 1):
704  for dx in range(0, maxRange + 1):
705  if (dx*dy > 0):
706  cov1, nPix1 = self.covDirectValue(diffImage, weightImage, dx, dy)
707  cov2, nPix2 = self.covDirectValue(diffImage, weightImage, dx, -dy)
708  cov = 0.5*(cov1 + cov2)
709  nPix = nPix1 + nPix2
710  else:
711  cov, nPix = self.covDirectValue(diffImage, weightImage, dx, dy)
712  if (dx == 0 and dy == 0):
713  var = cov
714  outList.append((dx, dy, var, cov, nPix))
715 
716  return outList
717 
718  def covDirectValue(self, diffImage, weightImage, dx, dy):
719  """Compute covariances of diffImage in real space at lag (dx, dy).
720 
721  Taken from https://github.com/PierreAstier/bfptc/ (c.f., appendix of Astier+19).
722 
723  Parameters
724  ----------
725  diffImage : `numpy.array`
726  Image to compute the covariance of.
727 
728  weightImage : `numpy.array`
729  Weight image of diffImage (1's and 0's for good and bad pixels, respectively).
730 
731  dx : `int`
732  Lag in x.
733 
734  dy : `int`
735  Lag in y.
736 
737  Returns
738  -------
739  cov : `float`
740  Covariance at (dx, dy)
741 
742  nPix : `int`
743  Number of pixel pairs used to evaluate var and cov.
744  """
745  (nCols, nRows) = diffImage.shape
746  # switching both signs does not change anything:
747  # it just swaps im1 and im2 below
748  if (dx < 0):
749  (dx, dy) = (-dx, -dy)
750  # now, we have dx >0. We have to distinguish two cases
751  # depending on the sign of dy
752  if dy >= 0:
753  im1 = diffImage[dy:, dx:]
754  w1 = weightImage[dy:, dx:]
755  im2 = diffImage[:nCols - dy, :nRows - dx]
756  w2 = weightImage[:nCols - dy, :nRows - dx]
757  else:
758  im1 = diffImage[:nCols + dy, dx:]
759  w1 = weightImage[:nCols + dy, dx:]
760  im2 = diffImage[-dy:, :nRows - dx]
761  w2 = weightImage[-dy:, :nRows - dx]
762  # use the same mask for all 3 calculations
763  wAll = w1*w2
764  # do not use mean() because weightImage=0 pixels would then count
765  nPix = wAll.sum()
766  im1TimesW = im1*wAll
767  s1 = im1TimesW.sum()/nPix
768  s2 = (im2*wAll).sum()/nPix
769  p = (im1TimesW*im2).sum()/nPix
770  cov = p - s1*s2
771 
772  return cov, nPix
773 
774  @staticmethod
775  def _initialParsForPolynomial(order):
776  assert(order >= 2)
777  pars = np.zeros(order, dtype=np.float)
778  pars[0] = 10
779  pars[1] = 1
780  pars[2:] = 0.0001
781  return pars
782 
783  @staticmethod
784  def _boundsForPolynomial(initialPars):
785  lowers = [np.NINF for p in initialPars]
786  uppers = [np.inf for p in initialPars]
787  lowers[1] = 0 # no negative gains
788  return (lowers, uppers)
789 
790  @staticmethod
791  def _boundsForAstier(initialPars):
792  lowers = [np.NINF for p in initialPars]
793  uppers = [np.inf for p in initialPars]
794  return (lowers, uppers)
795 
796  @staticmethod
797  def _getInitialGoodPoints(means, variances, maxDeviationPositive, maxDeviationNegative):
798  """Return a boolean array to mask bad points.
799 
800  Parameters
801  ----------
802  means : `numpy.array`
803  Input array with mean signal values.
804 
805  variances : `numpy.array`
806  Input array with variances at each mean value.
807 
808  maxDeviationPositive : `float`
809  Maximum deviation from being constant for the variance/mean
810  ratio, in the positive direction.
811 
812  maxDeviationNegative : `float`
813  Maximum deviation from being constant for the variance/mean
814  ratio, in the negative direction.
815 
816  Return
817  ------
818  goodPoints : `numpy.array` [`bool`]
819  Boolean array to select good (`True`) and bad (`False`)
820  points.
821 
822  Notes
823  -----
824  A linear function has a constant ratio, so find the median
825  value of the ratios, and exclude the points that deviate
826  from that by more than a factor of maxDeviationPositive/negative.
827  Asymmetric deviations are supported as we expect the PTC to turn
828  down as the flux increases, but sometimes it anomalously turns
829  upwards just before turning over, which ruins the fits, so it
830  is wise to be stricter about restricting positive outliers than
831  negative ones.
832 
833  Too high and points that are so bad that fit will fail will be included
834  Too low and the non-linear points will be excluded, biasing the NL fit."""
835 
836  assert(len(means) == len(variances))
837  ratios = [b/a for (a, b) in zip(means, variances)]
838  medianRatio = np.nanmedian(ratios)
839  ratioDeviations = [(r/medianRatio)-1 for r in ratios]
840 
841  # so that it doesn't matter if the deviation is expressed as positive or negative
842  maxDeviationPositive = abs(maxDeviationPositive)
843  maxDeviationNegative = -1. * abs(maxDeviationNegative)
844 
845  goodPoints = np.array([True if (r < maxDeviationPositive and r > maxDeviationNegative)
846  else False for r in ratioDeviations])
847  return goodPoints
848 
849  def _makeZeroSafe(self, array, warn=True, substituteValue=1e-9):
850  """"""
851  nBad = Counter(array)[0]
852  if nBad == 0:
853  return array
854 
855  if warn:
856  msg = f"Found {nBad} zeros in array at elements {[x for x in np.where(array==0)[0]]}"
857  self.log.warn(msg)
858 
859  array[array == 0] = substituteValue
860  return array
861 
862  def fitPtc(self, dataset, ptcFitType):
863  """Fit the photon transfer curve to a polynimial or to Astier+19 approximation.
864 
865  Fit the photon transfer curve with either a polynomial of the order
866  specified in the task config, or using the Astier approximation.
867 
868  Sigma clipping is performed iteratively for the fit, as well as an
869  initial clipping of data points that are more than
870  config.initialNonLinearityExclusionThreshold away from lying on a
871  straight line. This other step is necessary because the photon transfer
872  curve turns over catastrophically at very high flux (because saturation
873  drops the variance to ~0) and these far outliers cause the initial fit
874  to fail, meaning the sigma cannot be calculated to perform the
875  sigma-clipping.
876 
877  Parameters
878  ----------
879  dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset`
880  The dataset containing the means, variances and exposure times
881 
882  ptcFitType : `str`
883  Fit a 'POLYNOMIAL' (degree: 'polynomialFitDegree') or
884  'EXPAPPROXIMATION' (Eq. 16 of Astier+19) to the PTC
885 
886  Returns
887  -------
888  dataset: `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset`
889  This is the same dataset as the input paramter, however, it has been modified
890  to include information such as the fit vectors and the fit parameters. See
891  the class `PhotonTransferCurveDatase`.
892  """
893 
894  matrixSide = self.config.maximumRangeCovariancesAstier
895  nanMatrix = np.empty((matrixSide, matrixSide))
896  nanMatrix[:] = np.nan
897 
898  for amp in dataset.ampNames:
899  lenInputTimes = len(dataset.rawExpTimes[amp])
900  listNanMatrix = np.empty((lenInputTimes, matrixSide, matrixSide))
901  listNanMatrix[:] = np.nan
902 
903  dataset.covariances[amp] = listNanMatrix
904  dataset.covariancesModel[amp] = listNanMatrix
905  dataset.covariancesSqrtWeights[amp] = listNanMatrix
906  dataset.aMatrix[amp] = nanMatrix
907  dataset.bMatrix[amp] = nanMatrix
908  dataset.covariancesNoB[amp] = listNanMatrix
909  dataset.covariancesModelNoB[amp] = listNanMatrix
910  dataset.covariancesSqrtWeightsNoB[amp] = listNanMatrix
911  dataset.aMatrixNoB[amp] = nanMatrix
912 
913  def errFunc(p, x, y):
914  return ptcFunc(p, x) - y
915 
916  sigmaCutPtcOutliers = self.config.sigmaCutPtcOutliers
917  maxIterationsPtcOutliers = self.config.maxIterationsPtcOutliers
918 
919  for i, ampName in enumerate(dataset.ampNames):
920  timeVecOriginal = np.array(dataset.rawExpTimes[ampName])
921  meanVecOriginal = np.array(dataset.rawMeans[ampName])
922  varVecOriginal = np.array(dataset.rawVars[ampName])
923  varVecOriginal = self._makeZeroSafe(varVecOriginal)
924 
925  mask = ((meanVecOriginal >= self.config.minMeanSignal) &
926  (meanVecOriginal <= self.config.maxMeanSignal))
927 
928  goodPoints = self._getInitialGoodPoints(meanVecOriginal, varVecOriginal,
929  self.config.initialNonLinearityExclusionThresholdPositive,
930  self.config.initialNonLinearityExclusionThresholdNegative)
931  if not (mask.any() and goodPoints.any()):
932  msg = (f"\nSERIOUS: All points in either mask: {mask} or goodPoints: {goodPoints} are bad."
933  f"Setting {ampName} to BAD.")
934  self.log.warn(msg)
935  # The first and second parameters of initial fit are discarded (bias and gain)
936  # for the final NL coefficients
937  dataset.badAmps.append(ampName)
938  dataset.expIdMask[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName]))
939  dataset.gain[ampName] = np.nan
940  dataset.gainErr[ampName] = np.nan
941  dataset.noise[ampName] = np.nan
942  dataset.noiseErr[ampName] = np.nan
943  dataset.ptcFitPars[ampName] = (np.repeat(np.nan, self.config.polynomialFitDegree + 1) if
944  ptcFitType in ["POLYNOMIAL", ] else np.repeat(np.nan, 3))
945  dataset.ptcFitParsError[ampName] = (np.repeat(np.nan, self.config.polynomialFitDegree + 1) if
946  ptcFitType in ["POLYNOMIAL", ] else np.repeat(np.nan, 3))
947  dataset.ptcFitChiSq[ampName] = np.nan
948  dataset.finalVars[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName]))
949  dataset.finalModelVars[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName]))
950  dataset.finalMeans[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName]))
951  continue
952 
953  mask = mask & goodPoints
954 
955  if ptcFitType == 'EXPAPPROXIMATION':
956  ptcFunc = funcAstier
957  parsIniPtc = [-1e-9, 1.0, 10.] # a00, gain, noise
958  bounds = self._boundsForAstier(parsIniPtc)
959  if ptcFitType == 'POLYNOMIAL':
960  ptcFunc = funcPolynomial
961  parsIniPtc = self._initialParsForPolynomial(self.config.polynomialFitDegree + 1)
962  bounds = self._boundsForPolynomial(parsIniPtc)
963 
964  # Before bootstrap fit, do an iterative fit to get rid of outliers
965  count = 1
966  while count <= maxIterationsPtcOutliers:
967  # Note that application of the mask actually shrinks the array
968  # to size rather than setting elements to zero (as we want) so
969  # always update mask itself and re-apply to the original data
970  meanTempVec = meanVecOriginal[mask]
971  varTempVec = varVecOriginal[mask]
972  res = least_squares(errFunc, parsIniPtc, bounds=bounds, args=(meanTempVec, varTempVec))
973  pars = res.x
974 
975  # change this to the original from the temp because the masks are ANDed
976  # meaning once a point is masked it's always masked, and the masks must
977  # always be the same length for broadcasting
978  sigResids = (varVecOriginal - ptcFunc(pars, meanVecOriginal))/np.sqrt(varVecOriginal)
979  newMask = np.array([True if np.abs(r) < sigmaCutPtcOutliers else False for r in sigResids])
980  mask = mask & newMask
981  if not (mask.any() and newMask.any()):
982  msg = (f"\nSERIOUS: All points in either mask: {mask} or newMask: {newMask} are bad. "
983  f"Setting {ampName} to BAD.")
984  self.log.warn(msg)
985  # The first and second parameters of initial fit are discarded (bias and gain)
986  # for the final NL coefficients
987  dataset.badAmps.append(ampName)
988  dataset.expIdMask[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName]))
989  dataset.gain[ampName] = np.nan
990  dataset.gainErr[ampName] = np.nan
991  dataset.noise[ampName] = np.nan
992  dataset.noiseErr[ampName] = np.nan
993  dataset.ptcFitPars[ampName] = (np.repeat(np.nan, self.config.polynomialFitDegree + 1)
994  if ptcFitType in ["POLYNOMIAL", ] else
995  np.repeat(np.nan, 3))
996  dataset.ptcFitParsError[ampName] = (np.repeat(np.nan, self.config.polynomialFitDegree + 1)
997  if ptcFitType in ["POLYNOMIAL", ] else
998  np.repeat(np.nan, 3))
999  dataset.ptcFitChiSq[ampName] = np.nan
1000  dataset.finalVars[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName]))
1001  dataset.finalModelVars[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName]))
1002  dataset.finalMeans[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName]))
1003  break
1004  nDroppedTotal = Counter(mask)[False]
1005  self.log.debug(f"Iteration {count}: discarded {nDroppedTotal} points in total for {ampName}")
1006  count += 1
1007  # objects should never shrink
1008  assert (len(mask) == len(timeVecOriginal) == len(meanVecOriginal) == len(varVecOriginal))
1009 
1010  if not (mask.any() and newMask.any()):
1011  continue
1012  dataset.expIdMask[ampName] = mask # store the final mask
1013  parsIniPtc = pars
1014  meanVecFinal = meanVecOriginal[mask]
1015  varVecFinal = varVecOriginal[mask]
1016 
1017  if Counter(mask)[False] > 0:
1018  self.log.info((f"Number of points discarded in PTC of amplifier {ampName}:" +
1019  f" {Counter(mask)[False]} out of {len(meanVecOriginal)}"))
1020 
1021  if (len(meanVecFinal) < len(parsIniPtc)):
1022  msg = (f"\nSERIOUS: Not enough data points ({len(meanVecFinal)}) compared to the number of"
1023  f"parameters of the PTC model({len(parsIniPtc)}). Setting {ampName} to BAD.")
1024  self.log.warn(msg)
1025  # The first and second parameters of initial fit are discarded (bias and gain)
1026  # for the final NL coefficients
1027  dataset.badAmps.append(ampName)
1028  dataset.expIdMask[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName]))
1029  dataset.gain[ampName] = np.nan
1030  dataset.gainErr[ampName] = np.nan
1031  dataset.noise[ampName] = np.nan
1032  dataset.noiseErr[ampName] = np.nan
1033  dataset.ptcFitPars[ampName] = (np.repeat(np.nan, self.config.polynomialFitDegree + 1) if
1034  ptcFitType in ["POLYNOMIAL", ] else np.repeat(np.nan, 3))
1035  dataset.ptcFitParsError[ampName] = (np.repeat(np.nan, self.config.polynomialFitDegree + 1) if
1036  ptcFitType in ["POLYNOMIAL", ] else np.repeat(np.nan, 3))
1037  dataset.ptcFitChiSq[ampName] = np.nan
1038  dataset.finalVars[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName]))
1039  dataset.finalModelVars[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName]))
1040  dataset.finalMeans[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName]))
1041  continue
1042 
1043  # Fit the PTC
1044  if self.config.doFitBootstrap:
1045  parsFit, parsFitErr, reducedChiSqPtc = fitBootstrap(parsIniPtc, meanVecFinal,
1046  varVecFinal, ptcFunc,
1047  weightsY=1./np.sqrt(varVecFinal))
1048  else:
1049  parsFit, parsFitErr, reducedChiSqPtc = fitLeastSq(parsIniPtc, meanVecFinal,
1050  varVecFinal, ptcFunc,
1051  weightsY=1./np.sqrt(varVecFinal))
1052  dataset.ptcFitPars[ampName] = parsFit
1053  dataset.ptcFitParsError[ampName] = parsFitErr
1054  dataset.ptcFitChiSq[ampName] = reducedChiSqPtc
1055  # Masked variances (measured and modeled) and means. Need to pad the array so astropy.Table does
1056  # not crash (the mask may vary per amp).
1057  padLength = len(dataset.rawExpTimes[ampName]) - len(varVecFinal)
1058  dataset.finalVars[ampName] = np.pad(varVecFinal, (0, padLength), 'constant',
1059  constant_values=np.nan)
1060  dataset.finalModelVars[ampName] = np.pad(ptcFunc(parsFit, meanVecFinal), (0, padLength),
1061  'constant', constant_values=np.nan)
1062  dataset.finalMeans[ampName] = np.pad(meanVecFinal, (0, padLength), 'constant',
1063  constant_values=np.nan)
1064 
1065  if ptcFitType == 'EXPAPPROXIMATION':
1066  ptcGain = parsFit[1]
1067  ptcGainErr = parsFitErr[1]
1068  ptcNoise = np.sqrt(np.fabs(parsFit[2]))
1069  ptcNoiseErr = 0.5*(parsFitErr[2]/np.fabs(parsFit[2]))*np.sqrt(np.fabs(parsFit[2]))
1070  if ptcFitType == 'POLYNOMIAL':
1071  ptcGain = 1./parsFit[1]
1072  ptcGainErr = np.fabs(1./parsFit[1])*(parsFitErr[1]/parsFit[1])
1073  ptcNoise = np.sqrt(np.fabs(parsFit[0]))*ptcGain
1074  ptcNoiseErr = (0.5*(parsFitErr[0]/np.fabs(parsFit[0]))*(np.sqrt(np.fabs(parsFit[0]))))*ptcGain
1075  dataset.gain[ampName] = ptcGain
1076  dataset.gainErr[ampName] = ptcGainErr
1077  dataset.noise[ampName] = ptcNoise
1078  dataset.noiseErr[ampName] = ptcNoiseErr
1079  if not len(dataset.ptcFitType) == 0:
1080  dataset.ptcFitType = ptcFitType
1081  if len(dataset.badAmps) == 0:
1082  dataset.badAmps = np.repeat(np.nan, len(list(dataset.rawExpTimes.values())[0]))
1083 
1084  return dataset
lsst.cp.pipe.ptc.MeasurePhotonTransferCurveTask.runDataRef
def runDataRef(self, dataRefList)
Definition: ptc.py:227
lsst.cp.pipe.ptc.MeasurePhotonTransferCurveTask.fitCovariancesAstier
def fitCovariancesAstier(self, dataset, covariancesWithTagsArray)
Definition: ptc.py:428
lsst.cp.pipe.ptc.MeasurePhotonTransferCurveTask.fitPtc
def fitPtc(self, dataset, ptcFitType)
Definition: ptc.py:862
lsst.cp.pipe.astierCovPtcUtils.fftSize
def fftSize(s)
Definition: astierCovPtcUtils.py:122
lsst.cp.pipe.ptc.MeasurePhotonTransferCurveTask._boundsForAstier
def _boundsForAstier(initialPars)
Definition: ptc.py:791
lsst.cp.pipe.ptc.MeasurePhotonTransferCurveTask.measureMeanVarCov
def measureMeanVarCov(self, exposure1, exposure2, region=None, covAstierRealSpace=False)
Definition: ptc.py:552
lsst.cp.pipe.astierCovPtcUtils.fitData
def fitData(tupleName, maxMu=1e9, r=8, nSigmaFullFit=5.5, maxIterFullFit=3)
Definition: astierCovPtcUtils.py:323
lsst.cp.pipe.ptc.MeasurePhotonTransferCurveTask.__init__
def __init__(self, *args, **kwargs)
Definition: ptc.py:219
lsst.cp.pipe.photodiode.getBOTphotodiodeData
def getBOTphotodiodeData(dataRef, dataPath='/project/shared/BOT/_parent/raw/photodiode_data/', logger=None)
Definition: photodiode.py:30
lsst.cp.pipe.ptc.MeasurePhotonTransferCurveTask._boundsForPolynomial
def _boundsForPolynomial(initialPars)
Definition: ptc.py:784
lsst.cp.pipe.ptc.MeasurePhotonTransferCurveTaskConfig
Definition: ptc.py:45
lsst.cp.pipe.ptc.MeasurePhotonTransferCurveTask.getOutputPtcDataCovAstier
def getOutputPtcDataCovAstier(self, dataset, covFits, covFitsNoB)
Definition: ptc.py:464
lsst.cp.pipe.ptc.MeasurePhotonTransferCurveTask.makePairs
def makePairs(self, dataRefList)
Definition: ptc.py:373
lsst.cp.pipe.ptc.MeasurePhotonTransferCurveTask.covDirectValue
def covDirectValue(self, diffImage, weightImage, dx, dy)
Definition: ptc.py:718
lsst::pex::config
lsst.cp.pipe.astierCovPtcUtils.CovFft
Definition: astierCovPtcUtils.py:28
lsst.cp.pipe.utils.fitLeastSq
def fitLeastSq(initialParams, dataX, dataY, function, weightsY=None)
Definition: utils.py:285
lsst.cp.pipe.ptc.MeasurePhotonTransferCurveTask
Definition: ptc.py:180
lsst.cp.pipe.ptc.MeasurePhotonTransferCurveTask._makeZeroSafe
def _makeZeroSafe(self, array, warn=True, substituteValue=1e-9)
Definition: ptc.py:849
lsst.cp.pipe.ptc.MeasurePhotonTransferCurveTask._getInitialGoodPoints
def _getInitialGoodPoints(means, variances, maxDeviationPositive, maxDeviationNegative)
Definition: ptc.py:797
lsst::ip::isr
lsst::afw::math
lsst.cp.pipe.ptc.MeasurePhotonTransferCurveTask._initialParsForPolynomial
def _initialParsForPolynomial(order)
Definition: ptc.py:775
lsst.cp.pipe.utils.fitBootstrap
def fitBootstrap(initialParams, dataX, dataY, function, weightsY=None, confidenceSigma=1.)
Definition: utils.py:352
lsst.cp.pipe.ptc.MeasurePhotonTransferCurveTask.computeCovDirect
def computeCovDirect(self, diffImage, weightImage, maxRange)
Definition: ptc.py:668
lsst::pipe::base