lsst.cp.pipe  20.0.0-29-g1235a2f+df63e48dfd
ptc.py
Go to the documentation of this file.
1 # This file is part of cp_pipe.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 #
22 import numpy as np
23 import matplotlib.pyplot as plt
24 from collections import Counter
25 
26 import lsst.afw.math as afwMath
27 import lsst.pex.config as pexConfig
28 import lsst.pipe.base as pipeBase
29 from .utils import (fitLeastSq, fitBootstrap, funcPolynomial, funcAstier)
30 from scipy.optimize import least_squares
31 
32 import datetime
33 
34 from .astierCovPtcUtils import (fftSize, CovFft, computeCovDirect, fitData)
35 from .linearity import LinearitySolveTask
36 from .photodiode import getBOTphotodiodeData
37 
38 from lsst.pipe.tasks.getRepositoryData import DataRefListRunner
39 from lsst.ip.isr import PhotonTransferCurveDataset
40 
41 import copy
42 
43 __all__ = ['MeasurePhotonTransferCurveTask',
44  'MeasurePhotonTransferCurveTaskConfig']
45 
46 
47 class MeasurePhotonTransferCurveTaskConfig(pexConfig.Config):
48  """Config class for photon transfer curve measurement task"""
49  ccdKey = pexConfig.Field(
50  dtype=str,
51  doc="The key by which to pull a detector from a dataId, e.g. 'ccd' or 'detector'.",
52  default='ccd',
53  )
54  ptcFitType = pexConfig.ChoiceField(
55  dtype=str,
56  doc="Fit PTC to Eq. 16, Eq. 20 in Astier+19, or to a polynomial.",
57  default="POLYNOMIAL",
58  allowed={
59  "POLYNOMIAL": "n-degree polynomial (use 'polynomialFitDegree' to set 'n').",
60  "EXPAPPROXIMATION": "Approximation in Astier+19 (Eq. 16).",
61  "FULLCOVARIANCE": "Full covariances model in Astier+19 (Eq. 20)"
62  }
63  )
64  maximumRangeCovariancesAstier = pexConfig.Field(
65  dtype=int,
66  doc="Maximum range of covariances as in Astier+19",
67  default=8,
68  )
69  covAstierRealSpace = pexConfig.Field(
70  dtype=bool,
71  doc="Calculate covariances in real space or via FFT? (see appendix A of Astier+19).",
72  default=False,
73  )
74  polynomialFitDegree = pexConfig.Field(
75  dtype=int,
76  doc="Degree of polynomial to fit the PTC, when 'ptcFitType'=POLYNOMIAL.",
77  default=3,
78  )
79  linearity = pexConfig.ConfigurableField(
80  target=LinearitySolveTask,
81  doc="Task to solve the linearity."
82  )
83 
84  doCreateLinearizer = pexConfig.Field(
85  dtype=bool,
86  doc="Calculate non-linearity and persist linearizer?",
87  default=False,
88  )
89 
90  binSize = pexConfig.Field(
91  dtype=int,
92  doc="Bin the image by this factor in both dimensions.",
93  default=1,
94  )
95  minMeanSignal = pexConfig.DictField(
96  keytype=str,
97  itemtype=float,
98  doc="Minimum values (inclusive) of mean signal (in ADU) above which to consider, per amp."
99  " The same cut is applied to all amps if this dictionary is of the form"
100  " {'ALL_AMPS': value}",
101  default={'ALL_AMPS': 0.0},
102  )
103  maxMeanSignal = pexConfig.DictField(
104  keytype=str,
105  itemtype=float,
106  doc="Maximum values (inclusive) of mean signal (in ADU) below which to consider, per amp."
107  " The same cut is applied to all amps if this dictionary is of the form"
108  " {'ALL_AMPS': value}",
109  default={'ALL_AMPS': 1e6},
110  )
111  initialNonLinearityExclusionThresholdPositive = pexConfig.RangeField(
112  dtype=float,
113  doc="Initially exclude data points with a variance that are more than a factor of this from being"
114  " linear in the positive direction, from the PTC fit. Note that these points will also be"
115  " excluded from the non-linearity fit. This is done before the iterative outlier rejection,"
116  " to allow an accurate determination of the sigmas for said iterative fit.",
117  default=0.12,
118  min=0.0,
119  max=1.0,
120  )
121  initialNonLinearityExclusionThresholdNegative = pexConfig.RangeField(
122  dtype=float,
123  doc="Initially exclude data points with a variance that are more than a factor of this from being"
124  " linear in the negative direction, from the PTC fit. Note that these points will also be"
125  " excluded from the non-linearity fit. This is done before the iterative outlier rejection,"
126  " to allow an accurate determination of the sigmas for said iterative fit.",
127  default=0.25,
128  min=0.0,
129  max=1.0,
130  )
131  sigmaCutPtcOutliers = pexConfig.Field(
132  dtype=float,
133  doc="Sigma cut for outlier rejection in PTC.",
134  default=5.0,
135  )
136  maskNameList = pexConfig.ListField(
137  dtype=str,
138  doc="Mask list to exclude from statistics calculations.",
139  default=['SUSPECT', 'BAD', 'NO_DATA'],
140  )
141  nSigmaClipPtc = pexConfig.Field(
142  dtype=float,
143  doc="Sigma cut for afwMath.StatisticsControl()",
144  default=5.5,
145  )
146  nIterSigmaClipPtc = pexConfig.Field(
147  dtype=int,
148  doc="Number of sigma-clipping iterations for afwMath.StatisticsControl()",
149  default=1,
150  )
151  minNumberGoodPixelsForFft = pexConfig.Field(
152  dtype=int,
153  doc="Minimum number of acceptable good pixels per amp to calculate the covariances via FFT.",
154  default=10000,
155  )
156  maxIterationsPtcOutliers = pexConfig.Field(
157  dtype=int,
158  doc="Maximum number of iterations for outlier rejection in PTC.",
159  default=2,
160  )
161  doFitBootstrap = pexConfig.Field(
162  dtype=bool,
163  doc="Use bootstrap for the PTC fit parameters and errors?.",
164  default=False,
165  )
166  doPhotodiode = pexConfig.Field(
167  dtype=bool,
168  doc="Apply a correction based on the photodiode readings if available?",
169  default=True,
170  )
171  photodiodeDataPath = pexConfig.Field(
172  dtype=str,
173  doc="Gen2 only: path to locate the data photodiode data files.",
174  default=""
175  )
176  instrumentName = pexConfig.Field(
177  dtype=str,
178  doc="Instrument name.",
179  default='',
180  )
181 
182 
183 class MeasurePhotonTransferCurveTask(pipeBase.CmdLineTask):
184  """A class to calculate, fit, and plot a PTC from a set of flat pairs.
185 
186  The Photon Transfer Curve (var(signal) vs mean(signal)) is a standard tool
187  used in astronomical detectors characterization (e.g., Janesick 2001,
188  Janesick 2007). If ptcFitType is "EXPAPPROXIMATION" or "POLYNOMIAL", this task calculates the
189  PTC from a series of pairs of flat-field images; each pair taken at identical exposure
190  times. The difference image of each pair is formed to eliminate fixed pattern noise,
191  and then the variance of the difference image and the mean of the average image
192  are used to produce the PTC. An n-degree polynomial or the approximation in Equation
193  16 of Astier+19 ("The Shape of the Photon Transfer Curve of CCD sensors",
194  arXiv:1905.08677) can be fitted to the PTC curve. These models include
195  parameters such as the gain (e/DN) and readout noise.
196 
197  Linearizers to correct for signal-chain non-linearity are also calculated.
198  The `Linearizer` class, in general, can support per-amp linearizers, but in this
199  task this is not supported.
200 
201  If ptcFitType is "FULLCOVARIANCE", the covariances of the difference images are calculated via the
202  DFT methods described in Astier+19 and the variances for the PTC are given by the cov[0,0] elements
203  at each signal level. The full model in Equation 20 of Astier+19 is fit to the PTC to get the gain
204  and the noise.
205 
206  Parameters
207  ----------
208 
209  *args: `list`
210  Positional arguments passed to the Task constructor. None used at this
211  time.
212  **kwargs: `dict`
213  Keyword arguments passed on to the Task constructor. None used at this
214  time.
215 
216  """
217 
218  RunnerClass = DataRefListRunner
219  ConfigClass = MeasurePhotonTransferCurveTaskConfig
220  _DefaultName = "measurePhotonTransferCurve"
221 
222  def __init__(self, *args, **kwargs):
223  pipeBase.CmdLineTask.__init__(self, *args, **kwargs)
224  self.makeSubtask("linearity")
225  plt.interactive(False) # stop windows popping up when plotting. When headless, use 'agg' backend too
226  self.config.validate()
227  self.config.freeze()
228 
229  @pipeBase.timeMethod
230  def runDataRef(self, dataRefList):
231  """Run the Photon Transfer Curve (PTC) measurement task.
232 
233  For a dataRef (which is each detector here),
234  and given a list of exposure pairs (postISR) at different exposure times,
235  measure the PTC.
236 
237  Parameters
238  ----------
239  dataRefList : `list` [`lsst.daf.peristence.ButlerDataRef`]
240  Data references for exposures for detectors to process.
241  """
242  if len(dataRefList) < 2:
243  raise RuntimeError("Insufficient inputs to combine.")
244 
245  # setup necessary objects
246  dataRef = dataRefList[0]
247 
248  detNum = dataRef.dataId[self.config.ccdKey]
249  camera = dataRef.get('camera')
250  detector = camera[dataRef.dataId[self.config.ccdKey]]
251 
252  amps = detector.getAmplifiers()
253  ampNames = [amp.getName() for amp in amps]
254  datasetPtc = PhotonTransferCurveDataset(ampNames, self.config.ptcFitType)
255 
256  # Get the pairs of flat indexed by expTime
257  expPairs = self.makePairs(dataRefList)
258  expIds = []
259  for (exp1, exp2) in expPairs.values():
260  id1 = exp1.getInfo().getVisitInfo().getExposureId()
261  id2 = exp2.getInfo().getVisitInfo().getExposureId()
262  expIds.append((id1, id2))
263  self.log.info(f"Measuring PTC using {expIds} exposures for detector {detector.getId()}")
264 
265  # get photodiode data early so that logic can be put in to only use the
266  # data if all files are found, as partial corrections are not possible
267  # or at least require significant logic to deal with
268  if self.config.doPhotodiode:
269  for (expId1, expId2) in expIds:
270  charges = [-1, -1] # necessary to have a not-found value to keep lists in step
271  for i, expId in enumerate([expId1, expId2]):
272  # //1000 is a Gen2 only hack, working around the fact an
273  # exposure's ID is not the same as the expId in the
274  # registry. Currently expId is concatenated with the
275  # zero-padded detector ID. This will all go away in Gen3.
276  dataRef.dataId['expId'] = expId//1000
277  if self.config.photodiodeDataPath:
278  photodiodeData = getBOTphotodiodeData(dataRef, self.config.photodiodeDataPath)
279  else:
280  photodiodeData = getBOTphotodiodeData(dataRef)
281  if photodiodeData: # default path stored in function def to keep task clean
282  charges[i] = photodiodeData.getCharge()
283  else:
284  # full expId (not //1000) here, as that encodes the
285  # the detector number as so is fully qualifying
286  self.log.warn(f"No photodiode data found for {expId}")
287 
288  for ampName in ampNames:
289  datasetPtc.photoCharge[ampName].append((charges[0], charges[1]))
290  else:
291  # Can't be an empty list, as initialized, because astropy.Table won't allow it
292  # when saving as fits
293  for ampName in ampNames:
294  datasetPtc.photoCharge[ampName] = np.repeat(np.nan, len(expIds))
295 
296  for ampName in ampNames:
297  datasetPtc.inputExpIdPairs[ampName] = expIds
298 
299  maxMeanSignalDict = {ampName: 1e6 for ampName in ampNames}
300  minMeanSignalDict = {ampName: 0.0 for ampName in ampNames}
301  for ampName in ampNames:
302  if 'ALL_AMPS' in self.config.maxMeanSignal:
303  maxMeanSignalDict[ampName] = self.config.maxMeanSignal['ALL_AMPS']
304  elif ampName in self.config.maxMeanSignal:
305  maxMeanSignalDict[ampName] = self.config.maxMeanSignal[ampName]
306 
307  if 'ALL_AMPS' in self.config.minMeanSignal:
308  minMeanSignalDict[ampName] = self.config.minMeanSignal['ALL_AMPS']
309  elif ampName in self.config.minMeanSignal:
310  minMeanSignalDict[ampName] = self.config.minMeanSignal[ampName]
311 
312  tupleRecords = []
313  allTags = []
314  for expTime, (exp1, exp2) in expPairs.items():
315  expId1 = exp1.getInfo().getVisitInfo().getExposureId()
316  expId2 = exp2.getInfo().getVisitInfo().getExposureId()
317  tupleRows = []
318  nAmpsNan = 0
319  tags = ['mu', 'i', 'j', 'var', 'cov', 'npix', 'ext', 'expTime', 'ampName']
320  for ampNumber, amp in enumerate(detector):
321  ampName = amp.getName()
322  # covAstier: (i, j, var (cov[0,0]), cov, npix)
323  doRealSpace = self.config.covAstierRealSpace
324  muDiff, varDiff, covAstier = self.measureMeanVarCov(exp1, exp2, region=amp.getBBox(),
325  covAstierRealSpace=doRealSpace)
326 
327  if np.isnan(muDiff) or np.isnan(varDiff) or (covAstier is None):
328  msg = (f"NaN mean or var, or None cov in amp {ampName} in exposure pair {expId1},"
329  f" {expId2} of detector {detNum}.")
330  self.log.warn(msg)
331  nAmpsNan += 1
332  continue
333  if (muDiff <= minMeanSignalDict[ampName]) or (muDiff >= maxMeanSignalDict[ampName]):
334  continue
335 
336  datasetPtc.rawExpTimes[ampName].append(expTime)
337  datasetPtc.rawMeans[ampName].append(muDiff)
338  datasetPtc.rawVars[ampName].append(varDiff)
339 
340  tupleRows += [(muDiff, ) + covRow + (ampNumber, expTime, ampName) for covRow in covAstier]
341  if nAmpsNan == len(ampNames):
342  msg = f"NaN mean in all amps of exposure pair {expId1}, {expId2} of detector {detNum}."
343  self.log.warn(msg)
344  continue
345  allTags += tags
346  tupleRecords += tupleRows
347  covariancesWithTags = np.core.records.fromrecords(tupleRecords, names=allTags)
348 
349  for ampName in datasetPtc.ampNames:
350  # Sort raw vectors by rawMeans index
351  index = np.argsort(datasetPtc.rawMeans[ampName])
352  datasetPtc.rawExpTimes[ampName] = np.array(datasetPtc.rawExpTimes[ampName])[index]
353  datasetPtc.rawMeans[ampName] = np.array(datasetPtc.rawMeans[ampName])[index]
354  datasetPtc.rawVars[ampName] = np.array(datasetPtc.rawVars[ampName])[index]
355 
356  if self.config.ptcFitType in ["FULLCOVARIANCE", ]:
357  # Calculate covariances and fit them, including the PTC, to Astier+19 full model (Eq. 20)
358  # First, fit get the flat pairs that are masked, according to the regular PTC (C_00 vs mu)
359  # The points at these fluxes will also be masked when calculating the other covariances, C_ij)
360  newDatasetPtc = copy.copy(datasetPtc)
361  newDatasetPtc = self.fitPtc(newDatasetPtc, 'EXPAPPROXIMATION')
362  for ampName in datasetPtc.ampNames:
363  datasetPtc.expIdMask[ampName] = newDatasetPtc.expIdMask[ampName]
364 
365  datasetPtc.fitType = "FULLCOVARIANCE"
366  datasetPtc = self.fitCovariancesAstier(datasetPtc, covariancesWithTags)
367  elif self.config.ptcFitType in ["EXPAPPROXIMATION", "POLYNOMIAL"]:
368  # Fit the PTC to a polynomial or to Astier+19 exponential approximation (Eq. 16)
369  # Fill up PhotonTransferCurveDataset object.
370  datasetPtc = self.fitPtc(datasetPtc, self.config.ptcFitType)
371 
372  detName = detector.getName()
373  now = datetime.datetime.utcnow()
374  calibDate = now.strftime("%Y-%m-%d")
375  butler = dataRef.getButler()
376 
377  datasetPtc.updateMetadata(setDate=True, camera=camera, detector=detector)
378 
379  # Fit a poynomial to calculate non-linearity and persist linearizer.
380  if self.config.doCreateLinearizer:
381  # Fit (non)linearity of signal vs time curve.
382  # Fill up PhotonTransferCurveDataset object.
383  # Fill up array for LUT linearizer (tableArray).
384  # Produce coefficients for Polynomial and Squared linearizers.
385  # Build linearizer objects.
386  dimensions = {'camera': camera.getName(), 'detector': detector.getId()}
387  linearityResults = self.linearity.run(datasetPtc, camera, dimensions)
388  linearizer = linearityResults.outputLinearizer
389 
390  self.log.info("Writing linearizer:")
391 
392  detName = detector.getName()
393  now = datetime.datetime.utcnow()
394  calibDate = now.strftime("%Y-%m-%d")
395 
396  butler.put(linearizer, datasetType='linearizer',
397  dataId={'detector': detNum, 'detectorName': detName, 'calibDate': calibDate})
398 
399  self.log.info(f"Writing PTC data.")
400  butler.put(datasetPtc, datasetType='photonTransferCurveDataset', dataId={'detector': detNum,
401  'detectorName': detName, 'calibDate': calibDate})
402 
403  return pipeBase.Struct(exitStatus=0)
404 
405  def makePairs(self, dataRefList):
406  """Produce a list of flat pairs indexed by exposure time.
407 
408  Parameters
409  ----------
410  dataRefList : `list` [`lsst.daf.peristence.ButlerDataRef`]
411  Data references for exposures for detectors to process.
412 
413  Return
414  ------
415  flatPairs : `dict` [`float`, `lsst.afw.image.exposure.exposure.ExposureF`]
416  Dictionary that groups flat-field exposures that have the same exposure time (seconds).
417 
418  Notes
419  -----
420  We use the difference of one pair of flat-field images taken at the same exposure time when
421  calculating the PTC to reduce Fixed Pattern Noise. If there are > 2 flat-field images with the
422  same exposure time, the first two are kept and the rest discarded.
423  """
424 
425  # Organize exposures by observation date.
426  expDict = {}
427  for dataRef in dataRefList:
428  try:
429  tempFlat = dataRef.get("postISRCCD")
430  except RuntimeError:
431  self.log.warn("postISR exposure could not be retrieved. Ignoring flat.")
432  continue
433  expDate = tempFlat.getInfo().getVisitInfo().getDate().get()
434  expDict.setdefault(expDate, tempFlat)
435  sortedExps = {k: expDict[k] for k in sorted(expDict)}
436 
437  flatPairs = {}
438  for exp in sortedExps:
439  tempFlat = sortedExps[exp]
440  expTime = tempFlat.getInfo().getVisitInfo().getExposureTime()
441  listAtExpTime = flatPairs.setdefault(expTime, [])
442  if len(listAtExpTime) >= 2:
443  self.log.warn(f"Already found 2 exposures at expTime {expTime}. "
444  f"Ignoring exposure {tempFlat.getInfo().getVisitInfo().getExposureId()}")
445  else:
446  listAtExpTime.append(tempFlat)
447 
448  keysToDrop = []
449  for (key, value) in flatPairs.items():
450  if len(value) < 2:
451  keysToDrop.append(key)
452 
453  if len(keysToDrop):
454  for key in keysToDrop:
455  self.log.warn(f"Only one exposure found at expTime {key}. Dropping exposure "
456  f"{flatPairs[key][0].getInfo().getVisitInfo().getExposureId()}.")
457  flatPairs.pop(key)
458  sortedFlatPairs = {k: flatPairs[k] for k in sorted(flatPairs)}
459  return sortedFlatPairs
460 
461  def fitCovariancesAstier(self, dataset, covariancesWithTagsArray):
462  """Fit measured flat covariances to full model in Astier+19.
463 
464  Parameters
465  ----------
466  dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset`
467  The dataset containing information such as the means, variances and exposure times.
468 
469  covariancesWithTagsArray : `numpy.recarray`
470  Tuple with at least (mu, cov, var, i, j, npix), where:
471  mu : 0.5*(m1 + m2), where:
472  mu1: mean value of flat1
473  mu2: mean value of flat2
474  cov: covariance value at lag(i, j)
475  var: variance(covariance value at lag(0, 0))
476  i: lag dimension
477  j: lag dimension
478  npix: number of pixels used for covariance calculation.
479 
480  Returns
481  -------
482  dataset: `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset`
483  This is the same dataset as the input paramter, however, it has been modified
484  to include information such as the fit vectors and the fit parameters. See
485  the class `PhotonTransferCurveDatase`.
486  """
487 
488  covFits, covFitsNoB = fitData(covariancesWithTagsArray,
489  r=self.config.maximumRangeCovariancesAstier,
490  expIdMask=dataset.expIdMask)
491  dataset = self.getOutputPtcDataCovAstier(dataset, covFits, covFitsNoB)
492  return dataset
493 
494  def getOutputPtcDataCovAstier(self, dataset, covFits, covFitsNoB):
495  """Get output data for PhotonTransferCurveCovAstierDataset from CovFit objects.
496 
497  Parameters
498  ----------
499  dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset`
500  The dataset containing information such as the means, variances and exposure times.
501 
502  covFits: `dict`
503  Dictionary of CovFit objects, with amp names as keys.
504 
505  covFitsNoB : `dict`
506  Dictionary of CovFit objects, with amp names as keys, and 'b=0' in Eq. 20 of Astier+19.
507 
508  Returns
509  -------
510  dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset`
511  This is the same dataset as the input paramter, however, it has been modified
512  to include extra information such as the mask 1D array, gains, reoudout noise, measured signal,
513  measured variance, modeled variance, a, and b coefficient matrices (see Astier+19) per amplifier.
514  See the class `PhotonTransferCurveDatase`.
515  """
516  assert(len(covFits) == len(covFitsNoB))
517 
518  for i, amp in enumerate(dataset.ampNames):
519  lenInputTimes = len(dataset.rawExpTimes[amp])
520  # Not used when ptcFitType is 'FULLCOVARIANCE'
521  dataset.ptcFitPars[amp] = np.nan
522  dataset.ptcFitParsError[amp] = np.nan
523  dataset.ptcFitChiSq[amp] = np.nan
524  if (amp in covFits and (covFits[amp].covParams is not None) and
525  (covFitsNoB[amp].covParams is not None)):
526  fit = covFits[amp]
527  fitNoB = covFitsNoB[amp]
528  # Save full covariances, covariances models, and their weights
529  dataset.covariances[amp] = fit.cov
530  dataset.covariancesModel[amp] = fit.evalCovModel()
531  dataset.covariancesSqrtWeights[amp] = fit.sqrtW
532  dataset.aMatrix[amp] = fit.getA()
533  dataset.bMatrix[amp] = fit.getB()
534  dataset.covariancesNoB[amp] = fitNoB.cov
535  dataset.covariancesModelNoB[amp] = fitNoB.evalCovModel()
536  dataset.covariancesSqrtWeightsNoB[amp] = fitNoB.sqrtW
537  dataset.aMatrixNoB[amp] = fitNoB.getA()
538 
539  (meanVecFinal, varVecFinal, varVecModel,
540  wc, varMask) = fit.getFitData(0, 0, divideByMu=False)
541  gain = fit.getGain()
542 
543  dataset.gain[amp] = gain
544  dataset.gainErr[amp] = fit.getGainErr()
545  dataset.noise[amp] = np.sqrt(fit.getRon())
546  dataset.noiseErr[amp] = fit.getRonErr()
547 
548  padLength = lenInputTimes - len(varVecFinal)
549  dataset.finalVars[amp] = np.pad(varVecFinal/(gain**2), (0, padLength), 'constant',
550  constant_values=np.nan)
551  dataset.finalModelVars[amp] = np.pad(varVecModel/(gain**2), (0, padLength), 'constant',
552  constant_values=np.nan)
553  dataset.finalMeans[amp] = np.pad(meanVecFinal/gain, (0, padLength), 'constant',
554  constant_values=np.nan)
555  else:
556  # Bad amp
557  # Entries need to have proper dimensions so read/write with astropy.Table works.
558  matrixSide = self.config.maximumRangeCovariancesAstier
559  nanMatrix = np.full((matrixSide, matrixSide), np.nan)
560  listNanMatrix = np.full((lenInputTimes, matrixSide, matrixSide), np.nan)
561 
562  dataset.covariances[amp] = listNanMatrix
563  dataset.covariancesModel[amp] = listNanMatrix
564  dataset.covariancesSqrtWeights[amp] = listNanMatrix
565  dataset.aMatrix[amp] = nanMatrix
566  dataset.bMatrix[amp] = nanMatrix
567  dataset.covariancesNoB[amp] = listNanMatrix
568  dataset.covariancesModelNoB[amp] = listNanMatrix
569  dataset.covariancesSqrtWeightsNoB[amp] = listNanMatrix
570  dataset.aMatrixNoB[amp] = nanMatrix
571 
572  dataset.expIdMask[amp] = np.repeat(np.nan, lenInputTimes)
573  dataset.gain[amp] = np.nan
574  dataset.gainErr[amp] = np.nan
575  dataset.noise[amp] = np.nan
576  dataset.noiseErr[amp] = np.nan
577  dataset.finalVars[amp] = np.repeat(np.nan, lenInputTimes)
578  dataset.finalModelVars[amp] = np.repeat(np.nan, lenInputTimes)
579  dataset.finalMeans[amp] = np.repeat(np.nan, lenInputTimes)
580 
581  return dataset
582 
583  def measureMeanVarCov(self, exposure1, exposure2, region=None, covAstierRealSpace=False):
584  """Calculate the mean of each of two exposures and the variance and covariance of their difference.
585 
586  The variance is calculated via afwMath, and the covariance via the methods in Astier+19 (appendix A).
587  In theory, var = covariance[0,0]. This should be validated, and in the future, we may decide to just
588  keep one (covariance).
589 
590  Parameters
591  ----------
592  exposure1 : `lsst.afw.image.exposure.exposure.ExposureF`
593  First exposure of flat field pair.
594 
595  exposure2 : `lsst.afw.image.exposure.exposure.ExposureF`
596  Second exposure of flat field pair.
597 
598  region : `lsst.geom.Box2I`, optional
599  Region of each exposure where to perform the calculations (e.g, an amplifier).
600 
601  covAstierRealSpace : `bool`, optional
602  Should the covariannces in Astier+19 be calculated in real space or via FFT?
603  See Appendix A of Astier+19.
604 
605  Returns
606  -------
607  mu : `float` or `NaN`
608  0.5*(mu1 + mu2), where mu1, and mu2 are the clipped means of the regions in
609  both exposures. If either mu1 or m2 are NaN's, the returned value is NaN.
610 
611  varDiff : `float` or `NaN`
612  Half of the clipped variance of the difference of the regions inthe two input
613  exposures. If either mu1 or m2 are NaN's, the returned value is NaN.
614 
615  covDiffAstier : `list` or `None`
616  List with tuples of the form (dx, dy, var, cov, npix), where:
617  dx : `int`
618  Lag in x
619  dy : `int`
620  Lag in y
621  var : `float`
622  Variance at (dx, dy).
623  cov : `float`
624  Covariance at (dx, dy).
625  nPix : `int`
626  Number of pixel pairs used to evaluate var and cov.
627  If either mu1 or m2 are NaN's, the returned value is None.
628  """
629 
630  if region is not None:
631  im1Area = exposure1.maskedImage[region]
632  im2Area = exposure2.maskedImage[region]
633  else:
634  im1Area = exposure1.maskedImage
635  im2Area = exposure2.maskedImage
636 
637  if self.config.binSize > 1:
638  im1Area = afwMath.binImage(im1Area, self.config.binSize)
639  im2Area = afwMath.binImage(im2Area, self.config.binSize)
640 
641  im1MaskVal = exposure1.getMask().getPlaneBitMask(self.config.maskNameList)
642  im1StatsCtrl = afwMath.StatisticsControl(self.config.nSigmaClipPtc,
643  self.config.nIterSigmaClipPtc,
644  im1MaskVal)
645  im1StatsCtrl.setNanSafe(True)
646  im1StatsCtrl.setAndMask(im1MaskVal)
647 
648  im2MaskVal = exposure2.getMask().getPlaneBitMask(self.config.maskNameList)
649  im2StatsCtrl = afwMath.StatisticsControl(self.config.nSigmaClipPtc,
650  self.config.nIterSigmaClipPtc,
651  im2MaskVal)
652  im2StatsCtrl.setNanSafe(True)
653  im2StatsCtrl.setAndMask(im2MaskVal)
654 
655  # Clipped mean of images; then average of mean.
656  mu1 = afwMath.makeStatistics(im1Area, afwMath.MEANCLIP, im1StatsCtrl).getValue()
657  mu2 = afwMath.makeStatistics(im2Area, afwMath.MEANCLIP, im2StatsCtrl).getValue()
658  if np.isnan(mu1) or np.isnan(mu2):
659  self.log.warn(f"Mean of amp in image 1 or 2 is NaN: {mu1}, {mu2}.")
660  return np.nan, np.nan, None
661  mu = 0.5*(mu1 + mu2)
662 
663  # Take difference of pairs
664  # symmetric formula: diff = (mu2*im1-mu1*im2)/(0.5*(mu1+mu2))
665  temp = im2Area.clone()
666  temp *= mu1
667  diffIm = im1Area.clone()
668  diffIm *= mu2
669  diffIm -= temp
670  diffIm /= mu
671 
672  diffImMaskVal = diffIm.getMask().getPlaneBitMask(self.config.maskNameList)
673  diffImStatsCtrl = afwMath.StatisticsControl(self.config.nSigmaClipPtc,
674  self.config.nIterSigmaClipPtc,
675  diffImMaskVal)
676  diffImStatsCtrl.setNanSafe(True)
677  diffImStatsCtrl.setAndMask(diffImMaskVal)
678 
679  varDiff = 0.5*(afwMath.makeStatistics(diffIm, afwMath.VARIANCECLIP, diffImStatsCtrl).getValue())
680 
681  # Get the mask and identify good pixels as '1', and the rest as '0'.
682  w1 = np.where(im1Area.getMask().getArray() == 0, 1, 0)
683  w2 = np.where(im2Area.getMask().getArray() == 0, 1, 0)
684 
685  w12 = w1*w2
686  wDiff = np.where(diffIm.getMask().getArray() == 0, 1, 0)
687  w = w12*wDiff
688 
689  if np.sum(w) < self.config.minNumberGoodPixelsForFft:
690  self.log.warn(f"Number of good points for FFT ({np.sum(w)}) is less than threshold "
691  f"({self.config.minNumberGoodPixelsForFft})")
692  return np.nan, np.nan, None
693 
694  maxRangeCov = self.config.maximumRangeCovariancesAstier
695  if covAstierRealSpace:
696  covDiffAstier = computeCovDirect(diffIm.getImage().getArray(), w, maxRangeCov)
697  else:
698  shapeDiff = diffIm.getImage().getArray().shape
699  fftShape = (fftSize(shapeDiff[0] + maxRangeCov), fftSize(shapeDiff[1]+maxRangeCov))
700  c = CovFft(diffIm.getImage().getArray(), w, fftShape, maxRangeCov)
701  covDiffAstier = c.reportCovFft(maxRangeCov)
702 
703  return mu, varDiff, covDiffAstier
704 
705  def computeCovDirect(self, diffImage, weightImage, maxRange):
706  """Compute covariances of diffImage in real space.
707 
708  For lags larger than ~25, it is slower than the FFT way.
709  Taken from https://github.com/PierreAstier/bfptc/
710 
711  Parameters
712  ----------
713  diffImage : `numpy.array`
714  Image to compute the covariance of.
715 
716  weightImage : `numpy.array`
717  Weight image of diffImage (1's and 0's for good and bad pixels, respectively).
718 
719  maxRange : `int`
720  Last index of the covariance to be computed.
721 
722  Returns
723  -------
724  outList : `list`
725  List with tuples of the form (dx, dy, var, cov, npix), where:
726  dx : `int`
727  Lag in x
728  dy : `int`
729  Lag in y
730  var : `float`
731  Variance at (dx, dy).
732  cov : `float`
733  Covariance at (dx, dy).
734  nPix : `int`
735  Number of pixel pairs used to evaluate var and cov.
736  """
737  outList = []
738  var = 0
739  # (dy,dx) = (0,0) has to be first
740  for dy in range(maxRange + 1):
741  for dx in range(0, maxRange + 1):
742  if (dx*dy > 0):
743  cov1, nPix1 = self.covDirectValue(diffImage, weightImage, dx, dy)
744  cov2, nPix2 = self.covDirectValue(diffImage, weightImage, dx, -dy)
745  cov = 0.5*(cov1 + cov2)
746  nPix = nPix1 + nPix2
747  else:
748  cov, nPix = self.covDirectValue(diffImage, weightImage, dx, dy)
749  if (dx == 0 and dy == 0):
750  var = cov
751  outList.append((dx, dy, var, cov, nPix))
752 
753  return outList
754 
755  def covDirectValue(self, diffImage, weightImage, dx, dy):
756  """Compute covariances of diffImage in real space at lag (dx, dy).
757 
758  Taken from https://github.com/PierreAstier/bfptc/ (c.f., appendix of Astier+19).
759 
760  Parameters
761  ----------
762  diffImage : `numpy.array`
763  Image to compute the covariance of.
764 
765  weightImage : `numpy.array`
766  Weight image of diffImage (1's and 0's for good and bad pixels, respectively).
767 
768  dx : `int`
769  Lag in x.
770 
771  dy : `int`
772  Lag in y.
773 
774  Returns
775  -------
776  cov : `float`
777  Covariance at (dx, dy)
778 
779  nPix : `int`
780  Number of pixel pairs used to evaluate var and cov.
781  """
782  (nCols, nRows) = diffImage.shape
783  # switching both signs does not change anything:
784  # it just swaps im1 and im2 below
785  if (dx < 0):
786  (dx, dy) = (-dx, -dy)
787  # now, we have dx >0. We have to distinguish two cases
788  # depending on the sign of dy
789  if dy >= 0:
790  im1 = diffImage[dy:, dx:]
791  w1 = weightImage[dy:, dx:]
792  im2 = diffImage[:nCols - dy, :nRows - dx]
793  w2 = weightImage[:nCols - dy, :nRows - dx]
794  else:
795  im1 = diffImage[:nCols + dy, dx:]
796  w1 = weightImage[:nCols + dy, dx:]
797  im2 = diffImage[-dy:, :nRows - dx]
798  w2 = weightImage[-dy:, :nRows - dx]
799  # use the same mask for all 3 calculations
800  wAll = w1*w2
801  # do not use mean() because weightImage=0 pixels would then count
802  nPix = wAll.sum()
803  im1TimesW = im1*wAll
804  s1 = im1TimesW.sum()/nPix
805  s2 = (im2*wAll).sum()/nPix
806  p = (im1TimesW*im2).sum()/nPix
807  cov = p - s1*s2
808 
809  return cov, nPix
810 
811  @staticmethod
812  def _initialParsForPolynomial(order):
813  assert(order >= 2)
814  pars = np.zeros(order, dtype=np.float)
815  pars[0] = 10
816  pars[1] = 1
817  pars[2:] = 0.0001
818  return pars
819 
820  @staticmethod
821  def _boundsForPolynomial(initialPars, lowers=[], uppers=[]):
822  if not len(lowers):
823  lowers = [np.NINF for p in initialPars]
824  if not len(uppers):
825  uppers = [np.inf for p in initialPars]
826  lowers[1] = 0 # no negative gains
827  return (lowers, uppers)
828 
829  @staticmethod
830  def _boundsForAstier(initialPars, lowers=[], uppers=[]):
831  if not len(lowers):
832  lowers = [np.NINF for p in initialPars]
833  if not len(uppers):
834  uppers = [np.inf for p in initialPars]
835  return (lowers, uppers)
836 
837  @staticmethod
838  def _getInitialGoodPoints(means, variances, maxDeviationPositive, maxDeviationNegative):
839  """Return a boolean array to mask bad points.
840 
841  Parameters
842  ----------
843  means : `numpy.array`
844  Input array with mean signal values.
845 
846  variances : `numpy.array`
847  Input array with variances at each mean value.
848 
849  maxDeviationPositive : `float`
850  Maximum deviation from being constant for the variance/mean
851  ratio, in the positive direction.
852 
853  maxDeviationNegative : `float`
854  Maximum deviation from being constant for the variance/mean
855  ratio, in the negative direction.
856 
857  Return
858  ------
859  goodPoints : `numpy.array` [`bool`]
860  Boolean array to select good (`True`) and bad (`False`)
861  points.
862 
863  Notes
864  -----
865  A linear function has a constant ratio, so find the median
866  value of the ratios, and exclude the points that deviate
867  from that by more than a factor of maxDeviationPositive/negative.
868  Asymmetric deviations are supported as we expect the PTC to turn
869  down as the flux increases, but sometimes it anomalously turns
870  upwards just before turning over, which ruins the fits, so it
871  is wise to be stricter about restricting positive outliers than
872  negative ones.
873 
874  Too high and points that are so bad that fit will fail will be included
875  Too low and the non-linear points will be excluded, biasing the NL fit.
876 
877  This function also masks points after the variance starts decreasing.
878  """
879 
880  assert(len(means) == len(variances))
881  ratios = [b/a for (a, b) in zip(means, variances)]
882  medianRatio = np.nanmedian(ratios)
883  ratioDeviations = [(r/medianRatio)-1 for r in ratios]
884 
885  # so that it doesn't matter if the deviation is expressed as positive or negative
886  maxDeviationPositive = abs(maxDeviationPositive)
887  maxDeviationNegative = -1. * abs(maxDeviationNegative)
888 
889  goodPoints = np.array([True if (r < maxDeviationPositive and r > maxDeviationNegative)
890  else False for r in ratioDeviations])
891 
892  # Discard points when variance starts decreasing
893  pivot = np.where(np.array(np.diff(variances)) < 0)[0]
894  if len(pivot) == 0:
895  return goodPoints
896  else:
897  pivot = np.min(pivot)
898  goodPoints[pivot:len(goodPoints)] = False
899  return goodPoints
900 
901  def _makeZeroSafe(self, array, warn=True, substituteValue=1e-9):
902  """"""
903  nBad = Counter(array)[0]
904  if nBad == 0:
905  return array
906 
907  if warn:
908  msg = f"Found {nBad} zeros in array at elements {[x for x in np.where(array==0)[0]]}"
909  self.log.warn(msg)
910 
911  array[array == 0] = substituteValue
912  return array
913 
914  def fitPtc(self, dataset, ptcFitType):
915  """Fit the photon transfer curve to a polynimial or to Astier+19 approximation.
916 
917  Fit the photon transfer curve with either a polynomial of the order
918  specified in the task config, or using the Astier approximation.
919 
920  Sigma clipping is performed iteratively for the fit, as well as an
921  initial clipping of data points that are more than
922  config.initialNonLinearityExclusionThreshold away from lying on a
923  straight line. This other step is necessary because the photon transfer
924  curve turns over catastrophically at very high flux (because saturation
925  drops the variance to ~0) and these far outliers cause the initial fit
926  to fail, meaning the sigma cannot be calculated to perform the
927  sigma-clipping.
928 
929  Parameters
930  ----------
931  dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset`
932  The dataset containing the means, variances and exposure times
933 
934  ptcFitType : `str`
935  Fit a 'POLYNOMIAL' (degree: 'polynomialFitDegree') or
936  'EXPAPPROXIMATION' (Eq. 16 of Astier+19) to the PTC
937 
938  Returns
939  -------
940  dataset: `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset`
941  This is the same dataset as the input paramter, however, it has been modified
942  to include information such as the fit vectors and the fit parameters. See
943  the class `PhotonTransferCurveDatase`.
944  """
945 
946  matrixSide = self.config.maximumRangeCovariancesAstier
947  nanMatrix = np.empty((matrixSide, matrixSide))
948  nanMatrix[:] = np.nan
949 
950  for amp in dataset.ampNames:
951  lenInputTimes = len(dataset.rawExpTimes[amp])
952  listNanMatrix = np.empty((lenInputTimes, matrixSide, matrixSide))
953  listNanMatrix[:] = np.nan
954 
955  dataset.covariances[amp] = listNanMatrix
956  dataset.covariancesModel[amp] = listNanMatrix
957  dataset.covariancesSqrtWeights[amp] = listNanMatrix
958  dataset.aMatrix[amp] = nanMatrix
959  dataset.bMatrix[amp] = nanMatrix
960  dataset.covariancesNoB[amp] = listNanMatrix
961  dataset.covariancesModelNoB[amp] = listNanMatrix
962  dataset.covariancesSqrtWeightsNoB[amp] = listNanMatrix
963  dataset.aMatrixNoB[amp] = nanMatrix
964 
965  def errFunc(p, x, y):
966  return ptcFunc(p, x) - y
967 
968  sigmaCutPtcOutliers = self.config.sigmaCutPtcOutliers
969  maxIterationsPtcOutliers = self.config.maxIterationsPtcOutliers
970 
971  for i, ampName in enumerate(dataset.ampNames):
972  timeVecOriginal = np.array(dataset.rawExpTimes[ampName])
973  meanVecOriginal = np.array(dataset.rawMeans[ampName])
974  varVecOriginal = np.array(dataset.rawVars[ampName])
975  varVecOriginal = self._makeZeroSafe(varVecOriginal)
976 
977  goodPoints = self._getInitialGoodPoints(meanVecOriginal, varVecOriginal,
978  self.config.initialNonLinearityExclusionThresholdPositive,
979  self.config.initialNonLinearityExclusionThresholdNegative)
980  if not (goodPoints.any()):
981  msg = (f"\nSERIOUS: All points in goodPoints: {goodPoints} are bad."
982  f"Setting {ampName} to BAD.")
983  self.log.warn(msg)
984  # The first and second parameters of initial fit are discarded (bias and gain)
985  # for the final NL coefficients
986  dataset.badAmps.append(ampName)
987  dataset.expIdMask[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName]))
988  dataset.gain[ampName] = np.nan
989  dataset.gainErr[ampName] = np.nan
990  dataset.noise[ampName] = np.nan
991  dataset.noiseErr[ampName] = np.nan
992  dataset.ptcFitPars[ampName] = (np.repeat(np.nan, self.config.polynomialFitDegree + 1) if
993  ptcFitType in ["POLYNOMIAL", ] else np.repeat(np.nan, 3))
994  dataset.ptcFitParsError[ampName] = (np.repeat(np.nan, self.config.polynomialFitDegree + 1) if
995  ptcFitType in ["POLYNOMIAL", ] else np.repeat(np.nan, 3))
996  dataset.ptcFitChiSq[ampName] = np.nan
997  dataset.finalVars[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName]))
998  dataset.finalModelVars[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName]))
999  dataset.finalMeans[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName]))
1000  continue
1001 
1002  mask = goodPoints
1003 
1004  if ptcFitType == 'EXPAPPROXIMATION':
1005  ptcFunc = funcAstier
1006  parsIniPtc = [-1e-9, 1.0, 10.] # a00, gain, noise
1007  # lowers and uppers obtained from studies by C. Lage (UC Davis, 11/2020).
1008  bounds = self._boundsForAstier(parsIniPtc, lowers=[-1e-4, 0.5, -2000],
1009  uppers=[1e-4, 2.5, 2000])
1010  if ptcFitType == 'POLYNOMIAL':
1011  ptcFunc = funcPolynomial
1012  parsIniPtc = self._initialParsForPolynomial(self.config.polynomialFitDegree + 1)
1013  bounds = self._boundsForPolynomial(parsIniPtc)
1014 
1015  # Before bootstrap fit, do an iterative fit to get rid of outliers
1016  count = 1
1017  while count <= maxIterationsPtcOutliers:
1018  # Note that application of the mask actually shrinks the array
1019  # to size rather than setting elements to zero (as we want) so
1020  # always update mask itself and re-apply to the original data
1021  meanTempVec = meanVecOriginal[mask]
1022  varTempVec = varVecOriginal[mask]
1023  res = least_squares(errFunc, parsIniPtc, bounds=bounds, args=(meanTempVec, varTempVec))
1024  pars = res.x
1025 
1026  # change this to the original from the temp because the masks are ANDed
1027  # meaning once a point is masked it's always masked, and the masks must
1028  # always be the same length for broadcasting
1029  sigResids = (varVecOriginal - ptcFunc(pars, meanVecOriginal))/np.sqrt(varVecOriginal)
1030  newMask = np.array([True if np.abs(r) < sigmaCutPtcOutliers else False for r in sigResids])
1031  mask = mask & newMask
1032  if not (mask.any() and newMask.any()):
1033  msg = (f"\nSERIOUS: All points in either mask: {mask} or newMask: {newMask} are bad. "
1034  f"Setting {ampName} to BAD.")
1035  self.log.warn(msg)
1036  # The first and second parameters of initial fit are discarded (bias and gain)
1037  # for the final NL coefficients
1038  dataset.badAmps.append(ampName)
1039  dataset.expIdMask[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName]))
1040  dataset.gain[ampName] = np.nan
1041  dataset.gainErr[ampName] = np.nan
1042  dataset.noise[ampName] = np.nan
1043  dataset.noiseErr[ampName] = np.nan
1044  dataset.ptcFitPars[ampName] = (np.repeat(np.nan, self.config.polynomialFitDegree + 1)
1045  if ptcFitType in ["POLYNOMIAL", ] else
1046  np.repeat(np.nan, 3))
1047  dataset.ptcFitParsError[ampName] = (np.repeat(np.nan, self.config.polynomialFitDegree + 1)
1048  if ptcFitType in ["POLYNOMIAL", ] else
1049  np.repeat(np.nan, 3))
1050  dataset.ptcFitChiSq[ampName] = np.nan
1051  dataset.finalVars[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName]))
1052  dataset.finalModelVars[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName]))
1053  dataset.finalMeans[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName]))
1054  break
1055  nDroppedTotal = Counter(mask)[False]
1056  self.log.debug(f"Iteration {count}: discarded {nDroppedTotal} points in total for {ampName}")
1057  count += 1
1058  # objects should never shrink
1059  assert (len(mask) == len(timeVecOriginal) == len(meanVecOriginal) == len(varVecOriginal))
1060 
1061  if not (mask.any() and newMask.any()):
1062  continue
1063  dataset.expIdMask[ampName] = mask # store the final mask
1064  parsIniPtc = pars
1065  meanVecFinal = meanVecOriginal[mask]
1066  varVecFinal = varVecOriginal[mask]
1067 
1068  if Counter(mask)[False] > 0:
1069  self.log.info((f"Number of points discarded in PTC of amplifier {ampName}:" +
1070  f" {Counter(mask)[False]} out of {len(meanVecOriginal)}"))
1071 
1072  if (len(meanVecFinal) < len(parsIniPtc)):
1073  msg = (f"\nSERIOUS: Not enough data points ({len(meanVecFinal)}) compared to the number of"
1074  f"parameters of the PTC model({len(parsIniPtc)}). Setting {ampName} to BAD.")
1075  self.log.warn(msg)
1076  # The first and second parameters of initial fit are discarded (bias and gain)
1077  # for the final NL coefficients
1078  dataset.badAmps.append(ampName)
1079  dataset.expIdMask[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName]))
1080  dataset.gain[ampName] = np.nan
1081  dataset.gainErr[ampName] = np.nan
1082  dataset.noise[ampName] = np.nan
1083  dataset.noiseErr[ampName] = np.nan
1084  dataset.ptcFitPars[ampName] = (np.repeat(np.nan, self.config.polynomialFitDegree + 1) if
1085  ptcFitType in ["POLYNOMIAL", ] else np.repeat(np.nan, 3))
1086  dataset.ptcFitParsError[ampName] = (np.repeat(np.nan, self.config.polynomialFitDegree + 1) if
1087  ptcFitType in ["POLYNOMIAL", ] else np.repeat(np.nan, 3))
1088  dataset.ptcFitChiSq[ampName] = np.nan
1089  dataset.finalVars[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName]))
1090  dataset.finalModelVars[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName]))
1091  dataset.finalMeans[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName]))
1092  continue
1093 
1094  # Fit the PTC
1095  if self.config.doFitBootstrap:
1096  parsFit, parsFitErr, reducedChiSqPtc = fitBootstrap(parsIniPtc, meanVecFinal,
1097  varVecFinal, ptcFunc,
1098  weightsY=1./np.sqrt(varVecFinal))
1099  else:
1100  parsFit, parsFitErr, reducedChiSqPtc = fitLeastSq(parsIniPtc, meanVecFinal,
1101  varVecFinal, ptcFunc,
1102  weightsY=1./np.sqrt(varVecFinal))
1103  dataset.ptcFitPars[ampName] = parsFit
1104  dataset.ptcFitParsError[ampName] = parsFitErr
1105  dataset.ptcFitChiSq[ampName] = reducedChiSqPtc
1106  # Masked variances (measured and modeled) and means. Need to pad the array so astropy.Table does
1107  # not crash (the mask may vary per amp).
1108  padLength = len(dataset.rawExpTimes[ampName]) - len(varVecFinal)
1109  dataset.finalVars[ampName] = np.pad(varVecFinal, (0, padLength), 'constant',
1110  constant_values=np.nan)
1111  dataset.finalModelVars[ampName] = np.pad(ptcFunc(parsFit, meanVecFinal), (0, padLength),
1112  'constant', constant_values=np.nan)
1113  dataset.finalMeans[ampName] = np.pad(meanVecFinal, (0, padLength), 'constant',
1114  constant_values=np.nan)
1115 
1116  if ptcFitType == 'EXPAPPROXIMATION':
1117  ptcGain = parsFit[1]
1118  ptcGainErr = parsFitErr[1]
1119  ptcNoise = np.sqrt(np.fabs(parsFit[2]))
1120  ptcNoiseErr = 0.5*(parsFitErr[2]/np.fabs(parsFit[2]))*np.sqrt(np.fabs(parsFit[2]))
1121  if ptcFitType == 'POLYNOMIAL':
1122  ptcGain = 1./parsFit[1]
1123  ptcGainErr = np.fabs(1./parsFit[1])*(parsFitErr[1]/parsFit[1])
1124  ptcNoise = np.sqrt(np.fabs(parsFit[0]))*ptcGain
1125  ptcNoiseErr = (0.5*(parsFitErr[0]/np.fabs(parsFit[0]))*(np.sqrt(np.fabs(parsFit[0]))))*ptcGain
1126  dataset.gain[ampName] = ptcGain
1127  dataset.gainErr[ampName] = ptcGainErr
1128  dataset.noise[ampName] = ptcNoise
1129  dataset.noiseErr[ampName] = ptcNoiseErr
1130  if not len(dataset.ptcFitType) == 0:
1131  dataset.ptcFitType = ptcFitType
1132  if len(dataset.badAmps) == 0:
1133  dataset.badAmps = np.repeat(np.nan, len(list(dataset.rawExpTimes.values())[0]))
1134 
1135  return dataset
lsst.cp.pipe.ptc.MeasurePhotonTransferCurveTask.runDataRef
def runDataRef(self, dataRefList)
Definition: ptc.py:230
lsst.cp.pipe.ptc.MeasurePhotonTransferCurveTask.fitCovariancesAstier
def fitCovariancesAstier(self, dataset, covariancesWithTagsArray)
Definition: ptc.py:461
lsst.cp.pipe.ptc.MeasurePhotonTransferCurveTask.fitPtc
def fitPtc(self, dataset, ptcFitType)
Definition: ptc.py:914
lsst.cp.pipe.astierCovPtcUtils.fftSize
def fftSize(s)
Definition: astierCovPtcUtils.py:122
lsst.cp.pipe.ptc.MeasurePhotonTransferCurveTask.measureMeanVarCov
def measureMeanVarCov(self, exposure1, exposure2, region=None, covAstierRealSpace=False)
Definition: ptc.py:583
lsst.cp.pipe.ptc.MeasurePhotonTransferCurveTask.__init__
def __init__(self, *args, **kwargs)
Definition: ptc.py:222
lsst.cp.pipe.photodiode.getBOTphotodiodeData
def getBOTphotodiodeData(dataRef, dataPath='/project/shared/BOT/_parent/raw/photodiode_data/', logger=None)
Definition: photodiode.py:30
lsst.cp.pipe.ptc.MeasurePhotonTransferCurveTaskConfig
Definition: ptc.py:47
lsst.cp.pipe.ptc.MeasurePhotonTransferCurveTask.getOutputPtcDataCovAstier
def getOutputPtcDataCovAstier(self, dataset, covFits, covFitsNoB)
Definition: ptc.py:494
lsst.cp.pipe.astierCovPtcUtils.fitData
def fitData(tupleName, expIdMask, r=8)
Definition: astierCovPtcUtils.py:310
lsst.cp.pipe.ptc.MeasurePhotonTransferCurveTask._boundsForAstier
def _boundsForAstier(initialPars, lowers=[], uppers=[])
Definition: ptc.py:830
lsst.cp.pipe.ptc.MeasurePhotonTransferCurveTask.makePairs
def makePairs(self, dataRefList)
Definition: ptc.py:405
lsst.cp.pipe.ptc.MeasurePhotonTransferCurveTask.covDirectValue
def covDirectValue(self, diffImage, weightImage, dx, dy)
Definition: ptc.py:755
lsst::pex::config
lsst.cp.pipe.astierCovPtcUtils.CovFft
Definition: astierCovPtcUtils.py:28
lsst.cp.pipe.utils.fitLeastSq
def fitLeastSq(initialParams, dataX, dataY, function, weightsY=None)
Definition: utils.py:330
lsst.cp.pipe.ptc.MeasurePhotonTransferCurveTask
Definition: ptc.py:183
lsst.cp.pipe.ptc.MeasurePhotonTransferCurveTask._makeZeroSafe
def _makeZeroSafe(self, array, warn=True, substituteValue=1e-9)
Definition: ptc.py:901
lsst.cp.pipe.ptc.MeasurePhotonTransferCurveTask._getInitialGoodPoints
def _getInitialGoodPoints(means, variances, maxDeviationPositive, maxDeviationNegative)
Definition: ptc.py:838
lsst::ip::isr
lsst::afw::math
lsst.cp.pipe.ptc.MeasurePhotonTransferCurveTask._initialParsForPolynomial
def _initialParsForPolynomial(order)
Definition: ptc.py:812
lsst.cp.pipe.utils.fitBootstrap
def fitBootstrap(initialParams, dataX, dataY, function, weightsY=None, confidenceSigma=1.)
Definition: utils.py:397
lsst.cp.pipe.ptc.MeasurePhotonTransferCurveTask.computeCovDirect
def computeCovDirect(self, diffImage, weightImage, maxRange)
Definition: ptc.py:705
lsst::pipe::base
lsst.cp.pipe.ptc.MeasurePhotonTransferCurveTask._boundsForPolynomial
def _boundsForPolynomial(initialPars, lowers=[], uppers=[])
Definition: ptc.py:821