lsst.cp.pipe  20.0.0-28-g28eb519+ebc4a5d2c6
ptc.py
Go to the documentation of this file.
1 # This file is part of cp_pipe.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 #
22 import numpy as np
23 import matplotlib.pyplot as plt
24 from collections import Counter
25 
26 import lsst.afw.math as afwMath
27 import lsst.pex.config as pexConfig
28 import lsst.pipe.base as pipeBase
29 from .utils import (fitLeastSq, fitBootstrap, funcPolynomial, funcAstier)
30 from scipy.optimize import least_squares
31 
32 import datetime
33 
34 from .astierCovPtcUtils import (fftSize, CovFft, computeCovDirect, fitData)
35 from .linearity import LinearitySolveTask
36 from .photodiode import getBOTphotodiodeData
37 
38 from lsst.pipe.tasks.getRepositoryData import DataRefListRunner
39 from lsst.ip.isr import PhotonTransferCurveDataset
40 
41 import copy
42 
43 __all__ = ['MeasurePhotonTransferCurveTask',
44  'MeasurePhotonTransferCurveTaskConfig']
45 
46 
47 class MeasurePhotonTransferCurveTaskConfig(pexConfig.Config):
48  """Config class for photon transfer curve measurement task"""
49  ccdKey = pexConfig.Field(
50  dtype=str,
51  doc="The key by which to pull a detector from a dataId, e.g. 'ccd' or 'detector'.",
52  default='ccd',
53  )
54  ptcFitType = pexConfig.ChoiceField(
55  dtype=str,
56  doc="Fit PTC to Eq. 16, Eq. 20 in Astier+19, or to a polynomial.",
57  default="POLYNOMIAL",
58  allowed={
59  "POLYNOMIAL": "n-degree polynomial (use 'polynomialFitDegree' to set 'n').",
60  "EXPAPPROXIMATION": "Approximation in Astier+19 (Eq. 16).",
61  "FULLCOVARIANCE": "Full covariances model in Astier+19 (Eq. 20)"
62  }
63  )
64  maximumRangeCovariancesAstier = pexConfig.Field(
65  dtype=int,
66  doc="Maximum range of covariances as in Astier+19",
67  default=8,
68  )
69  covAstierRealSpace = pexConfig.Field(
70  dtype=bool,
71  doc="Calculate covariances in real space or via FFT? (see appendix A of Astier+19).",
72  default=False,
73  )
74  polynomialFitDegree = pexConfig.Field(
75  dtype=int,
76  doc="Degree of polynomial to fit the PTC, when 'ptcFitType'=POLYNOMIAL.",
77  default=3,
78  )
79  linearity = pexConfig.ConfigurableField(
80  target=LinearitySolveTask,
81  doc="Task to solve the linearity."
82  )
83 
84  doCreateLinearizer = pexConfig.Field(
85  dtype=bool,
86  doc="Calculate non-linearity and persist linearizer?",
87  default=False,
88  )
89 
90  binSize = pexConfig.Field(
91  dtype=int,
92  doc="Bin the image by this factor in both dimensions.",
93  default=1,
94  )
95  minMeanSignal = pexConfig.DictField(
96  keytype=str,
97  itemtype=float,
98  doc="Minimum values (inclusive) of mean signal (in ADU) above which to consider, per amp."
99  " The same cut is applied to all amps if this dictionary is of the form"
100  " {'ALL_AMPS': value}",
101  default={'ALL_AMPS': 0.0},
102  )
103  maxMeanSignal = pexConfig.DictField(
104  keytype=str,
105  itemtype=float,
106  doc="Maximum values (inclusive) of mean signal (in ADU) below which to consider, per amp."
107  " The same cut is applied to all amps if this dictionary is of the form"
108  " {'ALL_AMPS': value}",
109  default={'ALL_AMPS': 1e6},
110  )
111  initialNonLinearityExclusionThresholdPositive = pexConfig.RangeField(
112  dtype=float,
113  doc="Initially exclude data points with a variance that are more than a factor of this from being"
114  " linear in the positive direction, from the PTC fit. Note that these points will also be"
115  " excluded from the non-linearity fit. This is done before the iterative outlier rejection,"
116  " to allow an accurate determination of the sigmas for said iterative fit.",
117  default=0.12,
118  min=0.0,
119  max=1.0,
120  )
121  initialNonLinearityExclusionThresholdNegative = pexConfig.RangeField(
122  dtype=float,
123  doc="Initially exclude data points with a variance that are more than a factor of this from being"
124  " linear in the negative direction, from the PTC fit. Note that these points will also be"
125  " excluded from the non-linearity fit. This is done before the iterative outlier rejection,"
126  " to allow an accurate determination of the sigmas for said iterative fit.",
127  default=0.25,
128  min=0.0,
129  max=1.0,
130  )
131  sigmaCutPtcOutliers = pexConfig.Field(
132  dtype=float,
133  doc="Sigma cut for outlier rejection in PTC.",
134  default=5.0,
135  )
136  maskNameList = pexConfig.ListField(
137  dtype=str,
138  doc="Mask list to exclude from statistics calculations.",
139  default=['SUSPECT', 'BAD', 'NO_DATA'],
140  )
141  nSigmaClipPtc = pexConfig.Field(
142  dtype=float,
143  doc="Sigma cut for afwMath.StatisticsControl()",
144  default=5.5,
145  )
146  nIterSigmaClipPtc = pexConfig.Field(
147  dtype=int,
148  doc="Number of sigma-clipping iterations for afwMath.StatisticsControl()",
149  default=1,
150  )
151  maxIterationsPtcOutliers = pexConfig.Field(
152  dtype=int,
153  doc="Maximum number of iterations for outlier rejection in PTC.",
154  default=2,
155  )
156  doFitBootstrap = pexConfig.Field(
157  dtype=bool,
158  doc="Use bootstrap for the PTC fit parameters and errors?.",
159  default=False,
160  )
161  doPhotodiode = pexConfig.Field(
162  dtype=bool,
163  doc="Apply a correction based on the photodiode readings if available?",
164  default=True,
165  )
166  photodiodeDataPath = pexConfig.Field(
167  dtype=str,
168  doc="Gen2 only: path to locate the data photodiode data files.",
169  default=""
170  )
171  instrumentName = pexConfig.Field(
172  dtype=str,
173  doc="Instrument name.",
174  default='',
175  )
176 
177 
178 class MeasurePhotonTransferCurveTask(pipeBase.CmdLineTask):
179  """A class to calculate, fit, and plot a PTC from a set of flat pairs.
180 
181  The Photon Transfer Curve (var(signal) vs mean(signal)) is a standard tool
182  used in astronomical detectors characterization (e.g., Janesick 2001,
183  Janesick 2007). If ptcFitType is "EXPAPPROXIMATION" or "POLYNOMIAL", this task calculates the
184  PTC from a series of pairs of flat-field images; each pair taken at identical exposure
185  times. The difference image of each pair is formed to eliminate fixed pattern noise,
186  and then the variance of the difference image and the mean of the average image
187  are used to produce the PTC. An n-degree polynomial or the approximation in Equation
188  16 of Astier+19 ("The Shape of the Photon Transfer Curve of CCD sensors",
189  arXiv:1905.08677) can be fitted to the PTC curve. These models include
190  parameters such as the gain (e/DN) and readout noise.
191 
192  Linearizers to correct for signal-chain non-linearity are also calculated.
193  The `Linearizer` class, in general, can support per-amp linearizers, but in this
194  task this is not supported.
195 
196  If ptcFitType is "FULLCOVARIANCE", the covariances of the difference images are calculated via the
197  DFT methods described in Astier+19 and the variances for the PTC are given by the cov[0,0] elements
198  at each signal level. The full model in Equation 20 of Astier+19 is fit to the PTC to get the gain
199  and the noise.
200 
201  Parameters
202  ----------
203 
204  *args: `list`
205  Positional arguments passed to the Task constructor. None used at this
206  time.
207  **kwargs: `dict`
208  Keyword arguments passed on to the Task constructor. None used at this
209  time.
210 
211  """
212 
213  RunnerClass = DataRefListRunner
214  ConfigClass = MeasurePhotonTransferCurveTaskConfig
215  _DefaultName = "measurePhotonTransferCurve"
216 
217  def __init__(self, *args, **kwargs):
218  pipeBase.CmdLineTask.__init__(self, *args, **kwargs)
219  self.makeSubtask("linearity")
220  plt.interactive(False) # stop windows popping up when plotting. When headless, use 'agg' backend too
221  self.config.validate()
222  self.config.freeze()
223 
224  @pipeBase.timeMethod
225  def runDataRef(self, dataRefList):
226  """Run the Photon Transfer Curve (PTC) measurement task.
227 
228  For a dataRef (which is each detector here),
229  and given a list of exposure pairs (postISR) at different exposure times,
230  measure the PTC.
231 
232  Parameters
233  ----------
234  dataRefList : `list` [`lsst.daf.peristence.ButlerDataRef`]
235  Data references for exposures for detectors to process.
236  """
237  if len(dataRefList) < 2:
238  raise RuntimeError("Insufficient inputs to combine.")
239 
240  # setup necessary objects
241  dataRef = dataRefList[0]
242 
243  detNum = dataRef.dataId[self.config.ccdKey]
244  camera = dataRef.get('camera')
245  detector = camera[dataRef.dataId[self.config.ccdKey]]
246 
247  amps = detector.getAmplifiers()
248  ampNames = [amp.getName() for amp in amps]
249  datasetPtc = PhotonTransferCurveDataset(ampNames, self.config.ptcFitType)
250 
251  # Get the pairs of flat indexed by expTime
252  expPairs = self.makePairs(dataRefList)
253  expIds = []
254  for (exp1, exp2) in expPairs.values():
255  id1 = exp1.getInfo().getVisitInfo().getExposureId()
256  id2 = exp2.getInfo().getVisitInfo().getExposureId()
257  expIds.append((id1, id2))
258  self.log.info(f"Measuring PTC using {expIds} exposures for detector {detector.getId()}")
259 
260  # get photodiode data early so that logic can be put in to only use the
261  # data if all files are found, as partial corrections are not possible
262  # or at least require significant logic to deal with
263  if self.config.doPhotodiode:
264  for (expId1, expId2) in expIds:
265  charges = [-1, -1] # necessary to have a not-found value to keep lists in step
266  for i, expId in enumerate([expId1, expId2]):
267  # //1000 is a Gen2 only hack, working around the fact an
268  # exposure's ID is not the same as the expId in the
269  # registry. Currently expId is concatenated with the
270  # zero-padded detector ID. This will all go away in Gen3.
271  dataRef.dataId['expId'] = expId//1000
272  if self.config.photodiodeDataPath:
273  photodiodeData = getBOTphotodiodeData(dataRef, self.config.photodiodeDataPath)
274  else:
275  photodiodeData = getBOTphotodiodeData(dataRef)
276  if photodiodeData: # default path stored in function def to keep task clean
277  charges[i] = photodiodeData.getCharge()
278  else:
279  # full expId (not //1000) here, as that encodes the
280  # the detector number as so is fully qualifying
281  self.log.warn(f"No photodiode data found for {expId}")
282 
283  for ampName in ampNames:
284  datasetPtc.photoCharge[ampName].append((charges[0], charges[1]))
285  else:
286  # Can't be an empty list, as initialized, because astropy.Table won't allow it
287  # when saving as fits
288  for ampName in ampNames:
289  datasetPtc.photoCharge[ampName] = np.repeat(np.nan, len(expIds))
290 
291  for ampName in ampNames:
292  datasetPtc.inputExpIdPairs[ampName] = expIds
293 
294  maxMeanSignalDict = {ampName: 1e6 for ampName in ampNames}
295  minMeanSignalDict = {ampName: 0.0 for ampName in ampNames}
296  for ampName in ampNames:
297  if 'ALL_AMPS' in self.config.maxMeanSignal:
298  maxMeanSignalDict[ampName] = self.config.maxMeanSignal['ALL_AMPS']
299  elif ampName in self.config.maxMeanSignal:
300  maxMeanSignalDict[ampName] = self.config.maxMeanSignal[ampName]
301 
302  if 'ALL_AMPS' in self.config.minMeanSignal:
303  minMeanSignalDict[ampName] = self.config.minMeanSignal['ALL_AMPS']
304  elif ampName in self.config.minMeanSignal:
305  minMeanSignalDict[ampName] = self.config.minMeanSignal[ampName]
306 
307  tupleRecords = []
308  allTags = []
309  for expTime, (exp1, exp2) in expPairs.items():
310  expId1 = exp1.getInfo().getVisitInfo().getExposureId()
311  expId2 = exp2.getInfo().getVisitInfo().getExposureId()
312  tupleRows = []
313  nAmpsNan = 0
314  tags = ['mu', 'i', 'j', 'var', 'cov', 'npix', 'ext', 'expTime', 'ampName']
315  for ampNumber, amp in enumerate(detector):
316  ampName = amp.getName()
317  # covAstier: (i, j, var (cov[0,0]), cov, npix)
318  doRealSpace = self.config.covAstierRealSpace
319  muDiff, varDiff, covAstier = self.measureMeanVarCov(exp1, exp2, region=amp.getBBox(),
320  covAstierRealSpace=doRealSpace)
321 
322  if np.isnan(muDiff) or np.isnan(varDiff) or (covAstier is None):
323  msg = (f"NaN mean or var, or None cov in amp {ampName} in exposure pair {expId1},"
324  f" {expId2} of detector {detNum}.")
325  self.log.warn(msg)
326  nAmpsNan += 1
327  continue
328  if (muDiff <= minMeanSignalDict[ampName]) or (muDiff >= maxMeanSignalDict[ampName]):
329  continue
330 
331  datasetPtc.rawExpTimes[ampName].append(expTime)
332  datasetPtc.rawMeans[ampName].append(muDiff)
333  datasetPtc.rawVars[ampName].append(varDiff)
334 
335  tupleRows += [(muDiff, ) + covRow + (ampNumber, expTime, ampName) for covRow in covAstier]
336  if nAmpsNan == len(ampNames):
337  msg = f"NaN mean in all amps of exposure pair {expId1}, {expId2} of detector {detNum}."
338  self.log.warn(msg)
339  continue
340  allTags += tags
341  tupleRecords += tupleRows
342  covariancesWithTags = np.core.records.fromrecords(tupleRecords, names=allTags)
343 
344  for ampName in datasetPtc.ampNames:
345  # Sort raw vectors by rawMeans index
346  index = np.argsort(datasetPtc.rawMeans[ampName])
347  datasetPtc.rawExpTimes[ampName] = np.array(datasetPtc.rawExpTimes[ampName])[index]
348  datasetPtc.rawMeans[ampName] = np.array(datasetPtc.rawMeans[ampName])[index]
349  datasetPtc.rawVars[ampName] = np.array(datasetPtc.rawVars[ampName])[index]
350 
351  if self.config.ptcFitType in ["FULLCOVARIANCE", ]:
352  # Calculate covariances and fit them, including the PTC, to Astier+19 full model (Eq. 20)
353  # First, fit get the flat pairs that are masked, according to the regular PTC (C_00 vs mu)
354  # The points at these fluxes will also be masked when calculating the other covariances, C_ij)
355  newDatasetPtc = copy.copy(datasetPtc)
356  newDatasetPtc = self.fitPtc(newDatasetPtc, 'EXPAPPROXIMATION')
357  for ampName in datasetPtc.ampNames:
358  datasetPtc.expIdMask[ampName] = newDatasetPtc.expIdMask[ampName]
359 
360  datasetPtc.fitType = "FULLCOVARIANCE"
361  datasetPtc = self.fitCovariancesAstier(datasetPtc, covariancesWithTags)
362  elif self.config.ptcFitType in ["EXPAPPROXIMATION", "POLYNOMIAL"]:
363  # Fit the PTC to a polynomial or to Astier+19 exponential approximation (Eq. 16)
364  # Fill up PhotonTransferCurveDataset object.
365  datasetPtc = self.fitPtc(datasetPtc, self.config.ptcFitType)
366 
367  detName = detector.getName()
368  now = datetime.datetime.utcnow()
369  calibDate = now.strftime("%Y-%m-%d")
370  butler = dataRef.getButler()
371 
372  datasetPtc.updateMetadata(setDate=True, camera=camera, detector=detector)
373 
374  # Fit a poynomial to calculate non-linearity and persist linearizer.
375  if self.config.doCreateLinearizer:
376  # Fit (non)linearity of signal vs time curve.
377  # Fill up PhotonTransferCurveDataset object.
378  # Fill up array for LUT linearizer (tableArray).
379  # Produce coefficients for Polynomial and Squared linearizers.
380  # Build linearizer objects.
381  dimensions = {'camera': camera.getName(), 'detector': detector.getId()}
382  linearityResults = self.linearity.run(datasetPtc, camera, dimensions)
383  linearizer = linearityResults.outputLinearizer
384 
385  self.log.info("Writing linearizer:")
386 
387  detName = detector.getName()
388  now = datetime.datetime.utcnow()
389  calibDate = now.strftime("%Y-%m-%d")
390 
391  butler.put(linearizer, datasetType='linearizer',
392  dataId={'detector': detNum, 'detectorName': detName, 'calibDate': calibDate})
393 
394  self.log.info(f"Writing PTC data.")
395  butler.put(datasetPtc, datasetType='photonTransferCurveDataset', dataId={'detector': detNum,
396  'detectorName': detName, 'calibDate': calibDate})
397 
398  return pipeBase.Struct(exitStatus=0)
399 
400  def makePairs(self, dataRefList):
401  """Produce a list of flat pairs indexed by exposure time.
402 
403  Parameters
404  ----------
405  dataRefList : `list` [`lsst.daf.peristence.ButlerDataRef`]
406  Data references for exposures for detectors to process.
407 
408  Return
409  ------
410  flatPairs : `dict` [`float`, `lsst.afw.image.exposure.exposure.ExposureF`]
411  Dictionary that groups flat-field exposures that have the same exposure time (seconds).
412 
413  Notes
414  -----
415  We use the difference of one pair of flat-field images taken at the same exposure time when
416  calculating the PTC to reduce Fixed Pattern Noise. If there are > 2 flat-field images with the
417  same exposure time, the first two are kept and the rest discarded.
418  """
419 
420  # Organize exposures by observation date.
421  expDict = {}
422  for dataRef in dataRefList:
423  try:
424  tempFlat = dataRef.get("postISRCCD")
425  except RuntimeError:
426  self.log.warn("postISR exposure could not be retrieved. Ignoring flat.")
427  continue
428  expDate = tempFlat.getInfo().getVisitInfo().getDate().get()
429  expDict.setdefault(expDate, tempFlat)
430  sortedExps = {k: expDict[k] for k in sorted(expDict)}
431 
432  flatPairs = {}
433  for exp in sortedExps:
434  tempFlat = sortedExps[exp]
435  expTime = tempFlat.getInfo().getVisitInfo().getExposureTime()
436  listAtExpTime = flatPairs.setdefault(expTime, [])
437  if len(listAtExpTime) >= 2:
438  self.log.warn(f"Already found 2 exposures at expTime {expTime}. "
439  f"Ignoring exposure {tempFlat.getInfo().getVisitInfo().getExposureId()}")
440  else:
441  listAtExpTime.append(tempFlat)
442 
443  keysToDrop = []
444  for (key, value) in flatPairs.items():
445  if len(value) < 2:
446  keysToDrop.append(key)
447 
448  if len(keysToDrop):
449  for key in keysToDrop:
450  self.log.warn(f"Only one exposure found at expTime {key}. Dropping exposure "
451  f"{flatPairs[key][0].getInfo().getVisitInfo().getExposureId()}.")
452  flatPairs.pop(key)
453  sortedFlatPairs = {k: flatPairs[k] for k in sorted(flatPairs)}
454  return sortedFlatPairs
455 
456  def fitCovariancesAstier(self, dataset, covariancesWithTagsArray):
457  """Fit measured flat covariances to full model in Astier+19.
458 
459  Parameters
460  ----------
461  dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset`
462  The dataset containing information such as the means, variances and exposure times.
463 
464  covariancesWithTagsArray : `numpy.recarray`
465  Tuple with at least (mu, cov, var, i, j, npix), where:
466  mu : 0.5*(m1 + m2), where:
467  mu1: mean value of flat1
468  mu2: mean value of flat2
469  cov: covariance value at lag(i, j)
470  var: variance(covariance value at lag(0, 0))
471  i: lag dimension
472  j: lag dimension
473  npix: number of pixels used for covariance calculation.
474 
475  Returns
476  -------
477  dataset: `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset`
478  This is the same dataset as the input paramter, however, it has been modified
479  to include information such as the fit vectors and the fit parameters. See
480  the class `PhotonTransferCurveDatase`.
481  """
482 
483  covFits, covFitsNoB = fitData(covariancesWithTagsArray,
484  r=self.config.maximumRangeCovariancesAstier,
485  expIdMask=dataset.expIdMask)
486  dataset = self.getOutputPtcDataCovAstier(dataset, covFits, covFitsNoB)
487  return dataset
488 
489  def getOutputPtcDataCovAstier(self, dataset, covFits, covFitsNoB):
490  """Get output data for PhotonTransferCurveCovAstierDataset from CovFit objects.
491 
492  Parameters
493  ----------
494  dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset`
495  The dataset containing information such as the means, variances and exposure times.
496 
497  covFits: `dict`
498  Dictionary of CovFit objects, with amp names as keys.
499 
500  covFitsNoB : `dict`
501  Dictionary of CovFit objects, with amp names as keys, and 'b=0' in Eq. 20 of Astier+19.
502 
503  Returns
504  -------
505  dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset`
506  This is the same dataset as the input paramter, however, it has been modified
507  to include extra information such as the mask 1D array, gains, reoudout noise, measured signal,
508  measured variance, modeled variance, a, and b coefficient matrices (see Astier+19) per amplifier.
509  See the class `PhotonTransferCurveDatase`.
510  """
511  assert(len(covFits) == len(covFitsNoB))
512 
513  for i, amp in enumerate(dataset.ampNames):
514  lenInputTimes = len(dataset.rawExpTimes[amp])
515  # Not used when ptcFitType is 'FULLCOVARIANCE'
516  dataset.ptcFitPars[amp] = np.nan
517  dataset.ptcFitParsError[amp] = np.nan
518  dataset.ptcFitChiSq[amp] = np.nan
519  if (amp in covFits and (covFits[amp].covParams is not None) and
520  (covFitsNoB[amp].covParams is not None)):
521  fit = covFits[amp]
522  fitNoB = covFitsNoB[amp]
523  # Save full covariances, covariances models, and their weights
524  dataset.covariances[amp] = fit.cov
525  dataset.covariancesModel[amp] = fit.evalCovModel()
526  dataset.covariancesSqrtWeights[amp] = fit.sqrtW
527  dataset.aMatrix[amp] = fit.getA()
528  dataset.bMatrix[amp] = fit.getB()
529  dataset.covariancesNoB[amp] = fitNoB.cov
530  dataset.covariancesModelNoB[amp] = fitNoB.evalCovModel()
531  dataset.covariancesSqrtWeightsNoB[amp] = fitNoB.sqrtW
532  dataset.aMatrixNoB[amp] = fitNoB.getA()
533 
534  (meanVecFinal, varVecFinal, varVecModel,
535  wc, varMask) = fit.getFitData(0, 0, divideByMu=False)
536  gain = fit.getGain()
537 
538  dataset.gain[amp] = gain
539  dataset.gainErr[amp] = fit.getGainErr()
540  dataset.noise[amp] = np.sqrt(fit.getRon())
541  dataset.noiseErr[amp] = fit.getRonErr()
542 
543  padLength = lenInputTimes - len(varVecFinal)
544  dataset.finalVars[amp] = np.pad(varVecFinal/(gain**2), (0, padLength), 'constant',
545  constant_values=np.nan)
546  dataset.finalModelVars[amp] = np.pad(varVecModel/(gain**2), (0, padLength), 'constant',
547  constant_values=np.nan)
548  dataset.finalMeans[amp] = np.pad(meanVecFinal/gain, (0, padLength), 'constant',
549  constant_values=np.nan)
550  else:
551  # Bad amp
552  # Entries need to have proper dimensions so read/write with astropy.Table works.
553  matrixSide = self.config.maximumRangeCovariancesAstier
554  nanMatrix = np.full((matrixSide, matrixSide), np.nan)
555  listNanMatrix = np.full((lenInputTimes, matrixSide, matrixSide), np.nan)
556 
557  dataset.covariances[amp] = listNanMatrix
558  dataset.covariancesModel[amp] = listNanMatrix
559  dataset.covariancesSqrtWeights[amp] = listNanMatrix
560  dataset.aMatrix[amp] = nanMatrix
561  dataset.bMatrix[amp] = nanMatrix
562  dataset.covariancesNoB[amp] = listNanMatrix
563  dataset.covariancesModelNoB[amp] = listNanMatrix
564  dataset.covariancesSqrtWeightsNoB[amp] = listNanMatrix
565  dataset.aMatrixNoB[amp] = nanMatrix
566 
567  dataset.expIdMask[amp] = np.repeat(np.nan, lenInputTimes)
568  dataset.gain[amp] = np.nan
569  dataset.gainErr[amp] = np.nan
570  dataset.noise[amp] = np.nan
571  dataset.noiseErr[amp] = np.nan
572  dataset.finalVars[amp] = np.repeat(np.nan, lenInputTimes)
573  dataset.finalModelVars[amp] = np.repeat(np.nan, lenInputTimes)
574  dataset.finalMeans[amp] = np.repeat(np.nan, lenInputTimes)
575 
576  return dataset
577 
578  def measureMeanVarCov(self, exposure1, exposure2, region=None, covAstierRealSpace=False):
579  """Calculate the mean of each of two exposures and the variance and covariance of their difference.
580 
581  The variance is calculated via afwMath, and the covariance via the methods in Astier+19 (appendix A).
582  In theory, var = covariance[0,0]. This should be validated, and in the future, we may decide to just
583  keep one (covariance).
584 
585  Parameters
586  ----------
587  exposure1 : `lsst.afw.image.exposure.exposure.ExposureF`
588  First exposure of flat field pair.
589 
590  exposure2 : `lsst.afw.image.exposure.exposure.ExposureF`
591  Second exposure of flat field pair.
592 
593  region : `lsst.geom.Box2I`, optional
594  Region of each exposure where to perform the calculations (e.g, an amplifier).
595 
596  covAstierRealSpace : `bool`, optional
597  Should the covariannces in Astier+19 be calculated in real space or via FFT?
598  See Appendix A of Astier+19.
599 
600  Returns
601  -------
602  mu : `float` or `NaN`
603  0.5*(mu1 + mu2), where mu1, and mu2 are the clipped means of the regions in
604  both exposures. If either mu1 or m2 are NaN's, the returned value is NaN.
605 
606  varDiff : `float` or `NaN`
607  Half of the clipped variance of the difference of the regions inthe two input
608  exposures. If either mu1 or m2 are NaN's, the returned value is NaN.
609 
610  covDiffAstier : `list` or `None`
611  List with tuples of the form (dx, dy, var, cov, npix), where:
612  dx : `int`
613  Lag in x
614  dy : `int`
615  Lag in y
616  var : `float`
617  Variance at (dx, dy).
618  cov : `float`
619  Covariance at (dx, dy).
620  nPix : `int`
621  Number of pixel pairs used to evaluate var and cov.
622  If either mu1 or m2 are NaN's, the returned value is None.
623  """
624 
625  if region is not None:
626  im1Area = exposure1.maskedImage[region]
627  im2Area = exposure2.maskedImage[region]
628  else:
629  im1Area = exposure1.maskedImage
630  im2Area = exposure2.maskedImage
631 
632  if self.config.binSize > 1:
633  im1Area = afwMath.binImage(im1Area, self.config.binSize)
634  im2Area = afwMath.binImage(im2Area, self.config.binSize)
635 
636  im1MaskVal = exposure1.getMask().getPlaneBitMask(self.config.maskNameList)
637  im1StatsCtrl = afwMath.StatisticsControl(self.config.nSigmaClipPtc,
638  self.config.nIterSigmaClipPtc,
639  im1MaskVal)
640  im1StatsCtrl.setNanSafe(True)
641  im1StatsCtrl.setAndMask(im1MaskVal)
642 
643  im2MaskVal = exposure2.getMask().getPlaneBitMask(self.config.maskNameList)
644  im2StatsCtrl = afwMath.StatisticsControl(self.config.nSigmaClipPtc,
645  self.config.nIterSigmaClipPtc,
646  im2MaskVal)
647  im2StatsCtrl.setNanSafe(True)
648  im2StatsCtrl.setAndMask(im2MaskVal)
649 
650  # Clipped mean of images; then average of mean.
651  mu1 = afwMath.makeStatistics(im1Area, afwMath.MEANCLIP, im1StatsCtrl).getValue()
652  mu2 = afwMath.makeStatistics(im2Area, afwMath.MEANCLIP, im2StatsCtrl).getValue()
653  if np.isnan(mu1) or np.isnan(mu2):
654  return np.nan, np.nan, None
655  mu = 0.5*(mu1 + mu2)
656 
657  # Take difference of pairs
658  # symmetric formula: diff = (mu2*im1-mu1*im2)/(0.5*(mu1+mu2))
659  temp = im2Area.clone()
660  temp *= mu1
661  diffIm = im1Area.clone()
662  diffIm *= mu2
663  diffIm -= temp
664  diffIm /= mu
665 
666  diffImMaskVal = diffIm.getMask().getPlaneBitMask(self.config.maskNameList)
667  diffImStatsCtrl = afwMath.StatisticsControl(self.config.nSigmaClipPtc,
668  self.config.nIterSigmaClipPtc,
669  diffImMaskVal)
670  diffImStatsCtrl.setNanSafe(True)
671  diffImStatsCtrl.setAndMask(diffImMaskVal)
672 
673  varDiff = 0.5*(afwMath.makeStatistics(diffIm, afwMath.VARIANCECLIP, diffImStatsCtrl).getValue())
674 
675  # Get the mask and identify good pixels as '1', and the rest as '0'.
676  w1 = np.where(im1Area.getMask().getArray() == 0, 1, 0)
677  w2 = np.where(im2Area.getMask().getArray() == 0, 1, 0)
678 
679  w12 = w1*w2
680  wDiff = np.where(diffIm.getMask().getArray() == 0, 1, 0)
681  w = w12*wDiff
682 
683  if np.sum(w) == 0:
684  return np.nan, np.nan, None
685 
686  maxRangeCov = self.config.maximumRangeCovariancesAstier
687  if covAstierRealSpace:
688  covDiffAstier = computeCovDirect(diffIm.getImage().getArray(), w, maxRangeCov)
689  else:
690  shapeDiff = diffIm.getImage().getArray().shape
691  fftShape = (fftSize(shapeDiff[0] + maxRangeCov), fftSize(shapeDiff[1]+maxRangeCov))
692  c = CovFft(diffIm.getImage().getArray(), w, fftShape, maxRangeCov)
693  covDiffAstier = c.reportCovFft(maxRangeCov)
694 
695  return mu, varDiff, covDiffAstier
696 
697  def computeCovDirect(self, diffImage, weightImage, maxRange):
698  """Compute covariances of diffImage in real space.
699 
700  For lags larger than ~25, it is slower than the FFT way.
701  Taken from https://github.com/PierreAstier/bfptc/
702 
703  Parameters
704  ----------
705  diffImage : `numpy.array`
706  Image to compute the covariance of.
707 
708  weightImage : `numpy.array`
709  Weight image of diffImage (1's and 0's for good and bad pixels, respectively).
710 
711  maxRange : `int`
712  Last index of the covariance to be computed.
713 
714  Returns
715  -------
716  outList : `list`
717  List with tuples of the form (dx, dy, var, cov, npix), where:
718  dx : `int`
719  Lag in x
720  dy : `int`
721  Lag in y
722  var : `float`
723  Variance at (dx, dy).
724  cov : `float`
725  Covariance at (dx, dy).
726  nPix : `int`
727  Number of pixel pairs used to evaluate var and cov.
728  """
729  outList = []
730  var = 0
731  # (dy,dx) = (0,0) has to be first
732  for dy in range(maxRange + 1):
733  for dx in range(0, maxRange + 1):
734  if (dx*dy > 0):
735  cov1, nPix1 = self.covDirectValue(diffImage, weightImage, dx, dy)
736  cov2, nPix2 = self.covDirectValue(diffImage, weightImage, dx, -dy)
737  cov = 0.5*(cov1 + cov2)
738  nPix = nPix1 + nPix2
739  else:
740  cov, nPix = self.covDirectValue(diffImage, weightImage, dx, dy)
741  if (dx == 0 and dy == 0):
742  var = cov
743  outList.append((dx, dy, var, cov, nPix))
744 
745  return outList
746 
747  def covDirectValue(self, diffImage, weightImage, dx, dy):
748  """Compute covariances of diffImage in real space at lag (dx, dy).
749 
750  Taken from https://github.com/PierreAstier/bfptc/ (c.f., appendix of Astier+19).
751 
752  Parameters
753  ----------
754  diffImage : `numpy.array`
755  Image to compute the covariance of.
756 
757  weightImage : `numpy.array`
758  Weight image of diffImage (1's and 0's for good and bad pixels, respectively).
759 
760  dx : `int`
761  Lag in x.
762 
763  dy : `int`
764  Lag in y.
765 
766  Returns
767  -------
768  cov : `float`
769  Covariance at (dx, dy)
770 
771  nPix : `int`
772  Number of pixel pairs used to evaluate var and cov.
773  """
774  (nCols, nRows) = diffImage.shape
775  # switching both signs does not change anything:
776  # it just swaps im1 and im2 below
777  if (dx < 0):
778  (dx, dy) = (-dx, -dy)
779  # now, we have dx >0. We have to distinguish two cases
780  # depending on the sign of dy
781  if dy >= 0:
782  im1 = diffImage[dy:, dx:]
783  w1 = weightImage[dy:, dx:]
784  im2 = diffImage[:nCols - dy, :nRows - dx]
785  w2 = weightImage[:nCols - dy, :nRows - dx]
786  else:
787  im1 = diffImage[:nCols + dy, dx:]
788  w1 = weightImage[:nCols + dy, dx:]
789  im2 = diffImage[-dy:, :nRows - dx]
790  w2 = weightImage[-dy:, :nRows - dx]
791  # use the same mask for all 3 calculations
792  wAll = w1*w2
793  # do not use mean() because weightImage=0 pixels would then count
794  nPix = wAll.sum()
795  im1TimesW = im1*wAll
796  s1 = im1TimesW.sum()/nPix
797  s2 = (im2*wAll).sum()/nPix
798  p = (im1TimesW*im2).sum()/nPix
799  cov = p - s1*s2
800 
801  return cov, nPix
802 
803  @staticmethod
804  def _initialParsForPolynomial(order):
805  assert(order >= 2)
806  pars = np.zeros(order, dtype=np.float)
807  pars[0] = 10
808  pars[1] = 1
809  pars[2:] = 0.0001
810  return pars
811 
812  @staticmethod
813  def _boundsForPolynomial(initialPars, lowers=[], uppers=[]):
814  if not len(lowers):
815  lowers = [np.NINF for p in initialPars]
816  if not len(uppers):
817  uppers = [np.inf for p in initialPars]
818  lowers[1] = 0 # no negative gains
819  return (lowers, uppers)
820 
821  @staticmethod
822  def _boundsForAstier(initialPars, lowers=[], uppers=[]):
823  if not len(lowers):
824  lowers = [np.NINF for p in initialPars]
825  if not len(uppers):
826  uppers = [np.inf for p in initialPars]
827  return (lowers, uppers)
828 
829  @staticmethod
830  def _getInitialGoodPoints(means, variances, maxDeviationPositive, maxDeviationNegative):
831  """Return a boolean array to mask bad points.
832 
833  Parameters
834  ----------
835  means : `numpy.array`
836  Input array with mean signal values.
837 
838  variances : `numpy.array`
839  Input array with variances at each mean value.
840 
841  maxDeviationPositive : `float`
842  Maximum deviation from being constant for the variance/mean
843  ratio, in the positive direction.
844 
845  maxDeviationNegative : `float`
846  Maximum deviation from being constant for the variance/mean
847  ratio, in the negative direction.
848 
849  Return
850  ------
851  goodPoints : `numpy.array` [`bool`]
852  Boolean array to select good (`True`) and bad (`False`)
853  points.
854 
855  Notes
856  -----
857  A linear function has a constant ratio, so find the median
858  value of the ratios, and exclude the points that deviate
859  from that by more than a factor of maxDeviationPositive/negative.
860  Asymmetric deviations are supported as we expect the PTC to turn
861  down as the flux increases, but sometimes it anomalously turns
862  upwards just before turning over, which ruins the fits, so it
863  is wise to be stricter about restricting positive outliers than
864  negative ones.
865 
866  Too high and points that are so bad that fit will fail will be included
867  Too low and the non-linear points will be excluded, biasing the NL fit.
868 
869  This function also masks points after the variance starts decreasing.
870  """
871 
872  assert(len(means) == len(variances))
873  ratios = [b/a for (a, b) in zip(means, variances)]
874  medianRatio = np.nanmedian(ratios)
875  ratioDeviations = [(r/medianRatio)-1 for r in ratios]
876 
877  # so that it doesn't matter if the deviation is expressed as positive or negative
878  maxDeviationPositive = abs(maxDeviationPositive)
879  maxDeviationNegative = -1. * abs(maxDeviationNegative)
880 
881  goodPoints = np.array([True if (r < maxDeviationPositive and r > maxDeviationNegative)
882  else False for r in ratioDeviations])
883 
884  # Discard points when variance starts decreasing
885  pivot = np.where(np.array(np.diff(variances)) < 0)[0]
886  if len(pivot) == 0:
887  return goodPoints
888  else:
889  pivot = np.min(pivot)
890  goodPoints[pivot:len(goodPoints)] = False
891  return goodPoints
892 
893  def _makeZeroSafe(self, array, warn=True, substituteValue=1e-9):
894  """"""
895  nBad = Counter(array)[0]
896  if nBad == 0:
897  return array
898 
899  if warn:
900  msg = f"Found {nBad} zeros in array at elements {[x for x in np.where(array==0)[0]]}"
901  self.log.warn(msg)
902 
903  array[array == 0] = substituteValue
904  return array
905 
906  def fitPtc(self, dataset, ptcFitType):
907  """Fit the photon transfer curve to a polynimial or to Astier+19 approximation.
908 
909  Fit the photon transfer curve with either a polynomial of the order
910  specified in the task config, or using the Astier approximation.
911 
912  Sigma clipping is performed iteratively for the fit, as well as an
913  initial clipping of data points that are more than
914  config.initialNonLinearityExclusionThreshold away from lying on a
915  straight line. This other step is necessary because the photon transfer
916  curve turns over catastrophically at very high flux (because saturation
917  drops the variance to ~0) and these far outliers cause the initial fit
918  to fail, meaning the sigma cannot be calculated to perform the
919  sigma-clipping.
920 
921  Parameters
922  ----------
923  dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset`
924  The dataset containing the means, variances and exposure times
925 
926  ptcFitType : `str`
927  Fit a 'POLYNOMIAL' (degree: 'polynomialFitDegree') or
928  'EXPAPPROXIMATION' (Eq. 16 of Astier+19) to the PTC
929 
930  Returns
931  -------
932  dataset: `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset`
933  This is the same dataset as the input paramter, however, it has been modified
934  to include information such as the fit vectors and the fit parameters. See
935  the class `PhotonTransferCurveDatase`.
936  """
937 
938  matrixSide = self.config.maximumRangeCovariancesAstier
939  nanMatrix = np.empty((matrixSide, matrixSide))
940  nanMatrix[:] = np.nan
941 
942  for amp in dataset.ampNames:
943  lenInputTimes = len(dataset.rawExpTimes[amp])
944  listNanMatrix = np.empty((lenInputTimes, matrixSide, matrixSide))
945  listNanMatrix[:] = np.nan
946 
947  dataset.covariances[amp] = listNanMatrix
948  dataset.covariancesModel[amp] = listNanMatrix
949  dataset.covariancesSqrtWeights[amp] = listNanMatrix
950  dataset.aMatrix[amp] = nanMatrix
951  dataset.bMatrix[amp] = nanMatrix
952  dataset.covariancesNoB[amp] = listNanMatrix
953  dataset.covariancesModelNoB[amp] = listNanMatrix
954  dataset.covariancesSqrtWeightsNoB[amp] = listNanMatrix
955  dataset.aMatrixNoB[amp] = nanMatrix
956 
957  def errFunc(p, x, y):
958  return ptcFunc(p, x) - y
959 
960  sigmaCutPtcOutliers = self.config.sigmaCutPtcOutliers
961  maxIterationsPtcOutliers = self.config.maxIterationsPtcOutliers
962 
963  for i, ampName in enumerate(dataset.ampNames):
964  timeVecOriginal = np.array(dataset.rawExpTimes[ampName])
965  meanVecOriginal = np.array(dataset.rawMeans[ampName])
966  varVecOriginal = np.array(dataset.rawVars[ampName])
967  varVecOriginal = self._makeZeroSafe(varVecOriginal)
968 
969  goodPoints = self._getInitialGoodPoints(meanVecOriginal, varVecOriginal,
970  self.config.initialNonLinearityExclusionThresholdPositive,
971  self.config.initialNonLinearityExclusionThresholdNegative)
972  if not (goodPoints.any()):
973  msg = (f"\nSERIOUS: All points in goodPoints: {goodPoints} are bad."
974  f"Setting {ampName} to BAD.")
975  self.log.warn(msg)
976  # The first and second parameters of initial fit are discarded (bias and gain)
977  # for the final NL coefficients
978  dataset.badAmps.append(ampName)
979  dataset.expIdMask[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName]))
980  dataset.gain[ampName] = np.nan
981  dataset.gainErr[ampName] = np.nan
982  dataset.noise[ampName] = np.nan
983  dataset.noiseErr[ampName] = np.nan
984  dataset.ptcFitPars[ampName] = (np.repeat(np.nan, self.config.polynomialFitDegree + 1) if
985  ptcFitType in ["POLYNOMIAL", ] else np.repeat(np.nan, 3))
986  dataset.ptcFitParsError[ampName] = (np.repeat(np.nan, self.config.polynomialFitDegree + 1) if
987  ptcFitType in ["POLYNOMIAL", ] else np.repeat(np.nan, 3))
988  dataset.ptcFitChiSq[ampName] = np.nan
989  dataset.finalVars[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName]))
990  dataset.finalModelVars[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName]))
991  dataset.finalMeans[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName]))
992  continue
993 
994  mask = goodPoints
995 
996  if ptcFitType == 'EXPAPPROXIMATION':
997  ptcFunc = funcAstier
998  parsIniPtc = [-1e-9, 1.0, 10.] # a00, gain, noise
999  # lowers and uppers obtained from studies by C. Lage (UC Davis, 11/2020).
1000  bounds = self._boundsForAstier(parsIniPtc, lowers=[-1e-4, 0.5, -100],
1001  uppers=[1e-4, 2.5, 100])
1002  if ptcFitType == 'POLYNOMIAL':
1003  ptcFunc = funcPolynomial
1004  parsIniPtc = self._initialParsForPolynomial(self.config.polynomialFitDegree + 1)
1005  bounds = self._boundsForPolynomial(parsIniPtc)
1006 
1007  # Before bootstrap fit, do an iterative fit to get rid of outliers
1008  count = 1
1009  while count <= maxIterationsPtcOutliers:
1010  # Note that application of the mask actually shrinks the array
1011  # to size rather than setting elements to zero (as we want) so
1012  # always update mask itself and re-apply to the original data
1013  meanTempVec = meanVecOriginal[mask]
1014  varTempVec = varVecOriginal[mask]
1015  res = least_squares(errFunc, parsIniPtc, bounds=bounds, args=(meanTempVec, varTempVec))
1016  pars = res.x
1017 
1018  # change this to the original from the temp because the masks are ANDed
1019  # meaning once a point is masked it's always masked, and the masks must
1020  # always be the same length for broadcasting
1021  sigResids = (varVecOriginal - ptcFunc(pars, meanVecOriginal))/np.sqrt(varVecOriginal)
1022  newMask = np.array([True if np.abs(r) < sigmaCutPtcOutliers else False for r in sigResids])
1023  mask = mask & newMask
1024  if not (mask.any() and newMask.any()):
1025  msg = (f"\nSERIOUS: All points in either mask: {mask} or newMask: {newMask} are bad. "
1026  f"Setting {ampName} to BAD.")
1027  self.log.warn(msg)
1028  # The first and second parameters of initial fit are discarded (bias and gain)
1029  # for the final NL coefficients
1030  dataset.badAmps.append(ampName)
1031  dataset.expIdMask[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName]))
1032  dataset.gain[ampName] = np.nan
1033  dataset.gainErr[ampName] = np.nan
1034  dataset.noise[ampName] = np.nan
1035  dataset.noiseErr[ampName] = np.nan
1036  dataset.ptcFitPars[ampName] = (np.repeat(np.nan, self.config.polynomialFitDegree + 1)
1037  if ptcFitType in ["POLYNOMIAL", ] else
1038  np.repeat(np.nan, 3))
1039  dataset.ptcFitParsError[ampName] = (np.repeat(np.nan, self.config.polynomialFitDegree + 1)
1040  if ptcFitType in ["POLYNOMIAL", ] else
1041  np.repeat(np.nan, 3))
1042  dataset.ptcFitChiSq[ampName] = np.nan
1043  dataset.finalVars[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName]))
1044  dataset.finalModelVars[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName]))
1045  dataset.finalMeans[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName]))
1046  break
1047  nDroppedTotal = Counter(mask)[False]
1048  self.log.debug(f"Iteration {count}: discarded {nDroppedTotal} points in total for {ampName}")
1049  count += 1
1050  # objects should never shrink
1051  assert (len(mask) == len(timeVecOriginal) == len(meanVecOriginal) == len(varVecOriginal))
1052 
1053  if not (mask.any() and newMask.any()):
1054  continue
1055  dataset.expIdMask[ampName] = mask # store the final mask
1056  parsIniPtc = pars
1057  meanVecFinal = meanVecOriginal[mask]
1058  varVecFinal = varVecOriginal[mask]
1059 
1060  if Counter(mask)[False] > 0:
1061  self.log.info((f"Number of points discarded in PTC of amplifier {ampName}:" +
1062  f" {Counter(mask)[False]} out of {len(meanVecOriginal)}"))
1063 
1064  if (len(meanVecFinal) < len(parsIniPtc)):
1065  msg = (f"\nSERIOUS: Not enough data points ({len(meanVecFinal)}) compared to the number of"
1066  f"parameters of the PTC model({len(parsIniPtc)}). Setting {ampName} to BAD.")
1067  self.log.warn(msg)
1068  # The first and second parameters of initial fit are discarded (bias and gain)
1069  # for the final NL coefficients
1070  dataset.badAmps.append(ampName)
1071  dataset.expIdMask[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName]))
1072  dataset.gain[ampName] = np.nan
1073  dataset.gainErr[ampName] = np.nan
1074  dataset.noise[ampName] = np.nan
1075  dataset.noiseErr[ampName] = np.nan
1076  dataset.ptcFitPars[ampName] = (np.repeat(np.nan, self.config.polynomialFitDegree + 1) if
1077  ptcFitType in ["POLYNOMIAL", ] else np.repeat(np.nan, 3))
1078  dataset.ptcFitParsError[ampName] = (np.repeat(np.nan, self.config.polynomialFitDegree + 1) if
1079  ptcFitType in ["POLYNOMIAL", ] else np.repeat(np.nan, 3))
1080  dataset.ptcFitChiSq[ampName] = np.nan
1081  dataset.finalVars[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName]))
1082  dataset.finalModelVars[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName]))
1083  dataset.finalMeans[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName]))
1084  continue
1085 
1086  # Fit the PTC
1087  if self.config.doFitBootstrap:
1088  parsFit, parsFitErr, reducedChiSqPtc = fitBootstrap(parsIniPtc, meanVecFinal,
1089  varVecFinal, ptcFunc,
1090  weightsY=1./np.sqrt(varVecFinal))
1091  else:
1092  parsFit, parsFitErr, reducedChiSqPtc = fitLeastSq(parsIniPtc, meanVecFinal,
1093  varVecFinal, ptcFunc,
1094  weightsY=1./np.sqrt(varVecFinal))
1095  dataset.ptcFitPars[ampName] = parsFit
1096  dataset.ptcFitParsError[ampName] = parsFitErr
1097  dataset.ptcFitChiSq[ampName] = reducedChiSqPtc
1098  # Masked variances (measured and modeled) and means. Need to pad the array so astropy.Table does
1099  # not crash (the mask may vary per amp).
1100  padLength = len(dataset.rawExpTimes[ampName]) - len(varVecFinal)
1101  dataset.finalVars[ampName] = np.pad(varVecFinal, (0, padLength), 'constant',
1102  constant_values=np.nan)
1103  dataset.finalModelVars[ampName] = np.pad(ptcFunc(parsFit, meanVecFinal), (0, padLength),
1104  'constant', constant_values=np.nan)
1105  dataset.finalMeans[ampName] = np.pad(meanVecFinal, (0, padLength), 'constant',
1106  constant_values=np.nan)
1107 
1108  if ptcFitType == 'EXPAPPROXIMATION':
1109  ptcGain = parsFit[1]
1110  ptcGainErr = parsFitErr[1]
1111  ptcNoise = np.sqrt(np.fabs(parsFit[2]))
1112  ptcNoiseErr = 0.5*(parsFitErr[2]/np.fabs(parsFit[2]))*np.sqrt(np.fabs(parsFit[2]))
1113  if ptcFitType == 'POLYNOMIAL':
1114  ptcGain = 1./parsFit[1]
1115  ptcGainErr = np.fabs(1./parsFit[1])*(parsFitErr[1]/parsFit[1])
1116  ptcNoise = np.sqrt(np.fabs(parsFit[0]))*ptcGain
1117  ptcNoiseErr = (0.5*(parsFitErr[0]/np.fabs(parsFit[0]))*(np.sqrt(np.fabs(parsFit[0]))))*ptcGain
1118  dataset.gain[ampName] = ptcGain
1119  dataset.gainErr[ampName] = ptcGainErr
1120  dataset.noise[ampName] = ptcNoise
1121  dataset.noiseErr[ampName] = ptcNoiseErr
1122  if not len(dataset.ptcFitType) == 0:
1123  dataset.ptcFitType = ptcFitType
1124  if len(dataset.badAmps) == 0:
1125  dataset.badAmps = np.repeat(np.nan, len(list(dataset.rawExpTimes.values())[0]))
1126 
1127  return dataset
lsst.cp.pipe.ptc.MeasurePhotonTransferCurveTask.runDataRef
def runDataRef(self, dataRefList)
Definition: ptc.py:225
lsst.cp.pipe.ptc.MeasurePhotonTransferCurveTask.fitCovariancesAstier
def fitCovariancesAstier(self, dataset, covariancesWithTagsArray)
Definition: ptc.py:456
lsst.cp.pipe.ptc.MeasurePhotonTransferCurveTask.fitPtc
def fitPtc(self, dataset, ptcFitType)
Definition: ptc.py:906
lsst.cp.pipe.astierCovPtcUtils.fftSize
def fftSize(s)
Definition: astierCovPtcUtils.py:122
lsst.cp.pipe.ptc.MeasurePhotonTransferCurveTask.measureMeanVarCov
def measureMeanVarCov(self, exposure1, exposure2, region=None, covAstierRealSpace=False)
Definition: ptc.py:578
lsst.cp.pipe.ptc.MeasurePhotonTransferCurveTask.__init__
def __init__(self, *args, **kwargs)
Definition: ptc.py:217
lsst.cp.pipe.photodiode.getBOTphotodiodeData
def getBOTphotodiodeData(dataRef, dataPath='/project/shared/BOT/_parent/raw/photodiode_data/', logger=None)
Definition: photodiode.py:30
lsst.cp.pipe.ptc.MeasurePhotonTransferCurveTaskConfig
Definition: ptc.py:47
lsst.cp.pipe.ptc.MeasurePhotonTransferCurveTask.getOutputPtcDataCovAstier
def getOutputPtcDataCovAstier(self, dataset, covFits, covFitsNoB)
Definition: ptc.py:489
lsst.cp.pipe.astierCovPtcUtils.fitData
def fitData(tupleName, expIdMask, r=8)
Definition: astierCovPtcUtils.py:310
lsst.cp.pipe.ptc.MeasurePhotonTransferCurveTask._boundsForAstier
def _boundsForAstier(initialPars, lowers=[], uppers=[])
Definition: ptc.py:822
lsst.cp.pipe.ptc.MeasurePhotonTransferCurveTask.makePairs
def makePairs(self, dataRefList)
Definition: ptc.py:400
lsst.cp.pipe.ptc.MeasurePhotonTransferCurveTask.covDirectValue
def covDirectValue(self, diffImage, weightImage, dx, dy)
Definition: ptc.py:747
lsst::pex::config
lsst.cp.pipe.astierCovPtcUtils.CovFft
Definition: astierCovPtcUtils.py:28
lsst.cp.pipe.utils.fitLeastSq
def fitLeastSq(initialParams, dataX, dataY, function, weightsY=None)
Definition: utils.py:330
lsst.cp.pipe.ptc.MeasurePhotonTransferCurveTask
Definition: ptc.py:178
lsst.cp.pipe.ptc.MeasurePhotonTransferCurveTask._makeZeroSafe
def _makeZeroSafe(self, array, warn=True, substituteValue=1e-9)
Definition: ptc.py:893
lsst.cp.pipe.ptc.MeasurePhotonTransferCurveTask._getInitialGoodPoints
def _getInitialGoodPoints(means, variances, maxDeviationPositive, maxDeviationNegative)
Definition: ptc.py:830
lsst::ip::isr
lsst::afw::math
lsst.cp.pipe.ptc.MeasurePhotonTransferCurveTask._initialParsForPolynomial
def _initialParsForPolynomial(order)
Definition: ptc.py:804
lsst.cp.pipe.utils.fitBootstrap
def fitBootstrap(initialParams, dataX, dataY, function, weightsY=None, confidenceSigma=1.)
Definition: utils.py:397
lsst.cp.pipe.ptc.MeasurePhotonTransferCurveTask.computeCovDirect
def computeCovDirect(self, diffImage, weightImage, maxRange)
Definition: ptc.py:697
lsst::pipe::base
lsst.cp.pipe.ptc.MeasurePhotonTransferCurveTask._boundsForPolynomial
def _boundsForPolynomial(initialPars, lowers=[], uppers=[])
Definition: ptc.py:813