Coverage for python/lsst/cp/pipe/ptc/cpSolvePtcTask.py : 12%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of cp_pipe.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21#
22import numpy as np
23from collections import Counter
25import lsst.pex.config as pexConfig
26import lsst.pipe.base as pipeBase
27from lsst.cp.pipe.utils import (fitLeastSq, fitBootstrap, funcPolynomial, funcAstier)
29from scipy.optimize import least_squares
31import lsst.pipe.base.connectionTypes as cT
33from .astierCovPtcUtils import fitDataFullCovariance
35from lsst.ip.isr import PhotonTransferCurveDataset
37from lsst.cp.pipe._lookupStaticCalibration import lookupStaticCalibration
39import copy
42__all__ = ['PhotonTransferCurveSolveConfig', 'PhotonTransferCurveSolveTask']
45class PhotonTransferCurveSolveConnections(pipeBase.PipelineTaskConnections,
46 dimensions=("instrument", "detector")):
47 inputCovariances = cT.Input(
48 name="ptcCovariances",
49 doc="Tuple with measured covariances from flats.",
50 storageClass="PhotonTransferCurveDataset",
51 dimensions=("instrument", "exposure", "detector"),
52 multiple=True,
53 )
54 camera = cT.PrerequisiteInput(
55 name="camera",
56 doc="Camera the input data comes from.",
57 storageClass="Camera",
58 dimensions=("instrument",),
59 isCalibration=True,
60 lookupFunction=lookupStaticCalibration,
61 )
62 outputPtcDataset = cT.Output(
63 name="ptcDatsetProposal",
64 doc="Output proposed ptc dataset.",
65 storageClass="PhotonTransferCurveDataset",
66 dimensions=("instrument", "detector"),
67 multiple=False,
68 isCalibration=True,
69 )
72class PhotonTransferCurveSolveConfig(pipeBase.PipelineTaskConfig,
73 pipelineConnections=PhotonTransferCurveSolveConnections):
74 """Configuration for fitting measured covariances.
75 """
76 ptcFitType = pexConfig.ChoiceField(
77 dtype=str,
78 doc="Fit PTC to Eq. 16, Eq. 20 in Astier+19, or to a polynomial.",
79 default="POLYNOMIAL",
80 allowed={
81 "POLYNOMIAL": "n-degree polynomial (use 'polynomialFitDegree' to set 'n').",
82 "EXPAPPROXIMATION": "Approximation in Astier+19 (Eq. 16).",
83 "FULLCOVARIANCE": "Full covariances model in Astier+19 (Eq. 20)"
84 }
85 )
86 maximumRangeCovariancesAstier = pexConfig.Field(
87 dtype=int,
88 doc="Maximum range of covariances as in Astier+19",
89 default=8,
90 )
91 sigmaClipFullFitCovariancesAstier = pexConfig.Field(
92 dtype=float,
93 doc="sigma clip for full model fit for FULLCOVARIANCE ptcFitType ",
94 default=5.0,
95 )
96 maxIterFullFitCovariancesAstier = pexConfig.Field(
97 dtype=int,
98 doc="Maximum number of iterations in full model fit for FULLCOVARIANCE ptcFitType",
99 default=3,
100 )
101 polynomialFitDegree = pexConfig.Field(
102 dtype=int,
103 doc="Degree of polynomial to fit the PTC, when 'ptcFitType'=POLYNOMIAL.",
104 default=3,
105 )
106 sigmaCutPtcOutliers = pexConfig.Field(
107 dtype=float,
108 doc="Sigma cut for outlier rejection in PTC.",
109 default=5.0,
110 )
111 maxIterationsPtcOutliers = pexConfig.Field(
112 dtype=int,
113 doc="Maximum number of iterations for outlier rejection in PTC.",
114 default=2,
115 )
116 initialNonLinearityExclusionThresholdPositive = pexConfig.RangeField(
117 dtype=float,
118 doc="Initially exclude data points with a variance that are more than a factor of this from being"
119 " linear in the positive direction, from the PTC fit. Note that these points will also be"
120 " excluded from the non-linearity fit. This is done before the iterative outlier rejection,"
121 " to allow an accurate determination of the sigmas for said iterative fit.",
122 default=0.05,
123 min=0.0,
124 max=1.0,
125 )
126 initialNonLinearityExclusionThresholdNegative = pexConfig.RangeField(
127 dtype=float,
128 doc="Initially exclude data points with a variance that are more than a factor of this from being"
129 " linear in the negative direction, from the PTC fit. Note that these points will also be"
130 " excluded from the non-linearity fit. This is done before the iterative outlier rejection,"
131 " to allow an accurate determination of the sigmas for said iterative fit.",
132 default=0.25,
133 min=0.0,
134 max=1.0,
135 )
136 minMeanRatioTest = pexConfig.Field(
137 dtype=float,
138 doc="In the initial test to screen out bad points with a ratio test, points with low"
139 " flux can get inadvertantly screened. This test only screens out points with flux"
140 " above this value.",
141 default=20000,
142 )
143 minVarPivotSearch = pexConfig.Field(
144 dtype=float,
145 doc="The code looks for a pivot signal point after which the variance starts decreasing at high-flux"
146 " to exclude then form the PTC model fit. However, sometimes at low fluxes, the variance"
147 " decreases slightly. Set this variable for the variance value, in ADU^2, after which the pivot "
148 " should be sought.",
149 default=10000,
150 )
151 doFitBootstrap = pexConfig.Field(
152 dtype=bool,
153 doc="Use bootstrap for the PTC fit parameters and errors?.",
154 default=False,
155 )
158class PhotonTransferCurveSolveTask(pipeBase.PipelineTask,
159 pipeBase.CmdLineTask):
160 """Task to fit the PTC from flat covariances.
161 This task assembles the list of individual PTC datasets produced
162 by `PhotonTransferCurveSolveTask` into one single final PTC dataset.
163 The task fits the measured (co)variances to a polynomial model or to
164 the models described in equations 16 and 20 of Astier+19
165 (referred to as `POLYNOMIAL`, `EXPAPPROXIMATION`, and `FULLCOVARIANCE`
166 in the configuration options of the task, respectively). Parameters
167 of interest such as tghe gain and noise are derived from the fits.
169 Astier+19: "The Shape of the Photon Transfer Curve
170 of CCD sensors", arXiv:1905.08677
171 """
172 ConfigClass = PhotonTransferCurveSolveConfig
173 _DefaultName = 'cpPhotonTransferCurveSolve'
175 def runQuantum(self, butlerQC, inputRefs, outputRefs):
176 """Ensure that the input and output dimensions are passed along.
178 Parameters
179 ----------
180 butlerQC : `~lsst.daf.butler.butlerQuantumContext.ButlerQuantumContext`
181 Butler to operate on.
182 inputRefs : `~lsst.pipe.base.connections.InputQuantizedConnection`
183 Input data refs to load.
184 ouptutRefs : `~lsst.pipe.base.connections.OutputQuantizedConnection`
185 Output data refs to persist.
186 """
187 inputs = butlerQC.get(inputRefs)
188 outputs = self.run(inputCovariances=inputs['inputCovariances'], camera=inputs['camera'])
189 butlerQC.put(outputs, outputRefs)
191 def run(self, inputCovariances, camera=None, inputExpList=None):
192 """Fit measure covariances to different models.
194 Parameters
195 ----------
196 inputCovariances : `list` [`lsst.ip.isr.PhotonTransferCurveDataset`]
197 List of lsst.ip.isr.PhotonTransferCurveDataset datasets.
199 camera : `lsst.afw.cameraGeom.Camera`, optional
200 Input camera.
202 inputExpList : `list` [`~lsst.afw.image.exposure.exposure.ExposureF`], optional
203 List of exposures.
205 Returns
206 -------
207 results : `lsst.pipe.base.Struct`
208 The results struct containing:
209 ``outputPtcDatset`` : `lsst.ip.isr.PhotonTransferCurveDataset`
210 Final PTC dataset, containing information such as the means, variances,
211 and exposure times.
212 """
213 # Assemble partial PTC datasets into a single dataset.
214 ampNames = np.unique(inputCovariances[0].ampNames)
215 datasetPtc = PhotonTransferCurveDataset(ampNames, self.config.ptcFitType,
216 self.config.maximumRangeCovariancesAstier)
217 for partialPtcDataset in inputCovariances:
218 if partialPtcDataset.ptcFitType == 'DUMMY':
219 continue
220 for ampName in ampNames:
221 datasetPtc.inputExpIdPairs[ampName].append(partialPtcDataset.inputExpIdPairs[ampName])
222 if type(partialPtcDataset.rawExpTimes[ampName]) is list:
223 datasetPtc.rawExpTimes[ampName].append(partialPtcDataset.rawExpTimes[ampName][0])
224 else:
225 datasetPtc.rawExpTimes[ampName].append(partialPtcDataset.rawExpTimes[ampName])
226 if type(partialPtcDataset.rawMeans[ampName]) is list:
227 datasetPtc.rawMeans[ampName].append(partialPtcDataset.rawMeans[ampName][0])
228 else:
229 datasetPtc.rawMeans[ampName].append(partialPtcDataset.rawMeans[ampName])
230 if type(partialPtcDataset.rawVars[ampName]) is list:
231 datasetPtc.rawVars[ampName].append(partialPtcDataset.rawVars[ampName][0])
232 else:
233 datasetPtc.rawVars[ampName].append(partialPtcDataset.rawVars[ampName])
234 datasetPtc.covariances[ampName].append(np.array(partialPtcDataset.covariances[ampName][0]))
235 datasetPtc.covariancesSqrtWeights[ampName].append(
236 np.array(partialPtcDataset.covariancesSqrtWeights[ampName][0]))
237 # Sort arrays that are filled so far in the final dataset by rawMeans index
238 for ampName in ampNames:
239 index = np.argsort(np.ravel(np.array(datasetPtc.rawMeans[ampName])))
240 datasetPtc.inputExpIdPairs[ampName] = np.array(datasetPtc.inputExpIdPairs[ampName])[index]
241 datasetPtc.rawExpTimes[ampName] = np.array(datasetPtc.rawExpTimes[ampName])[index]
242 datasetPtc.rawMeans[ampName] = np.array(datasetPtc.rawMeans[ampName])[index]
243 datasetPtc.rawVars[ampName] = np.array(datasetPtc.rawVars[ampName])[index]
244 datasetPtc.covariances[ampName] = np.array(datasetPtc.covariances[ampName])[index]
245 datasetPtc.covariancesSqrtWeights[ampName] = np.array(
246 datasetPtc.covariancesSqrtWeights[ampName])[index]
248 if self.config.ptcFitType == "FULLCOVARIANCE":
249 # Calculate covariances and fit them, including the PTC, to Astier+19 full model (Eq. 20)
250 # First, fit get the flat pairs that are masked, fitting C_00 vs mu to
251 # the EXPAPPROXIMATION model (Eq. 16 in Astier+19).
252 # The points at these fluxes will also be masked when calculating the other covariances, C_ij)
253 tempDatasetPtc = copy.copy(datasetPtc)
254 tempDatasetPtc.ptcFitType = "EXPAPPROXIMATION"
255 tempDatasetPtc = self.fitPtc(tempDatasetPtc)
256 for ampName in datasetPtc.ampNames:
257 datasetPtc.expIdMask[ampName] = tempDatasetPtc.expIdMask[ampName]
258 datasetPtc.fitType = "FULLCOVARIANCE"
259 datasetPtc = self.fitCovariancesAstier(datasetPtc)
260 # The other options are: self.config.ptcFitType in ("EXPAPPROXIMATION", "POLYNOMIAL")
261 else:
262 # Fit the PTC to a polynomial or to Astier+19 exponential approximation (Eq. 16).
263 # Fill up PhotonTransferCurveDataset object.
264 datasetPtc = self.fitPtc(datasetPtc)
265 if inputExpList is not None:
266 # It should be a list of exposures, to get the detector.
267 detector = inputExpList[0].getDetector()
268 else:
269 detector = None
270 datasetPtc.updateMetadata(setDate=True, camera=camera, detector=detector)
272 return pipeBase.Struct(
273 outputPtcDataset=datasetPtc,
274 )
276 def fitCovariancesAstier(self, dataset):
277 """Fit measured flat covariances to full model in Astier+19.
279 Parameters
280 ----------
281 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset`
282 The dataset containing information such as the means, (co)variances,
283 and exposure times.
285 Returns
286 -------
287 dataset: `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset`
288 This is the same dataset as the input paramter, however, it has been modified
289 to include information such as the fit vectors and the fit parameters. See
290 the class `PhotonTransferCurveDatase`.
291 """
293 covFits, covFitsNoB = fitDataFullCovariance(dataset)
294 dataset = self.getOutputPtcDataCovAstier(dataset, covFits, covFitsNoB)
296 return dataset
298 def getOutputPtcDataCovAstier(self, dataset, covFits, covFitsNoB):
299 """Get output data for PhotonTransferCurveCovAstierDataset from CovFit objects.
301 Parameters
302 ----------
303 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset`
304 The dataset containing information such as the means, variances and exposure times.
305 covFits: `dict`
306 Dictionary of CovFit objects, with amp names as keys.
307 covFitsNoB : `dict`
308 Dictionary of CovFit objects, with amp names as keys, and 'b=0' in Eq. 20 of Astier+19.
310 Returns
311 -------
312 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset`
313 This is the same dataset as the input paramter, however, it has been modified
314 to include extra information such as the mask 1D array, gains, reoudout noise, measured signal,
315 measured variance, modeled variance, a, and b coefficient matrices (see Astier+19) per amplifier.
316 See the class `PhotonTransferCurveDatase`.
317 """
318 assert(len(covFits) == len(covFitsNoB))
320 for i, amp in enumerate(dataset.ampNames):
321 lenInputTimes = len(dataset.rawExpTimes[amp])
322 # Not used when ptcFitType is 'FULLCOVARIANCE'
323 dataset.ptcFitPars[amp] = [np.nan]
324 dataset.ptcFitParsError[amp] = [np.nan]
325 dataset.ptcFitChiSq[amp] = np.nan
326 if amp in covFits:
327 fit = covFits[amp]
328 fitNoB = covFitsNoB[amp]
329 # Save full covariances, covariances models, and their weights
330 # dataset.expIdMask is already full
331 dataset.covariances[amp] = fit.cov
332 dataset.covariancesModel[amp] = fit.evalCovModel()
333 dataset.covariancesSqrtWeights[amp] = fit.sqrtW
334 dataset.aMatrix[amp] = fit.getA()
335 dataset.bMatrix[amp] = fit.getB()
336 dataset.covariancesModelNoB[amp] = fitNoB.evalCovModel()
337 dataset.aMatrixNoB[amp] = fitNoB.getA()
339 (meanVecFinal, varVecFinal, varVecModel,
340 wc, varMask) = fit.getFitData(0, 0, divideByMu=False)
341 gain = fit.getGain()
343 dataset.gain[amp] = gain
344 dataset.gainErr[amp] = fit.getGainErr()
345 dataset.noise[amp] = np.sqrt(fit.getRon())
346 dataset.noiseErr[amp] = fit.getRonErr()
347 dataset.finalVars[amp] = varVecFinal
348 dataset.finalModelVars[amp] = varVecModel
349 dataset.finalMeans[amp] = meanVecFinal
351 else:
352 # Bad amp
353 # Entries need to have proper dimensions so read/write with astropy.Table works.
354 matrixSide = self.config.maximumRangeCovariancesAstier
355 nanMatrix = np.full((matrixSide, matrixSide), np.nan)
356 listNanMatrix = np.full((lenInputTimes, matrixSide, matrixSide), np.nan)
358 dataset.covariances[amp] = listNanMatrix
359 dataset.covariancesModel[amp] = listNanMatrix
360 dataset.covariancesSqrtWeights[amp] = listNanMatrix
361 dataset.aMatrix[amp] = nanMatrix
362 dataset.bMatrix[amp] = nanMatrix
363 dataset.covariancesModelNoB[amp] = listNanMatrix
364 dataset.aMatrixNoB[amp] = nanMatrix
366 dataset.expIdMask[amp] = np.repeat(np.nan, lenInputTimes)
367 dataset.gain[amp] = np.nan
368 dataset.gainErr[amp] = np.nan
369 dataset.noise[amp] = np.nan
370 dataset.noiseErr[amp] = np.nan
371 dataset.finalVars[amp] = np.repeat(np.nan, lenInputTimes)
372 dataset.finalModelVars[amp] = np.repeat(np.nan, lenInputTimes)
373 dataset.finalMeans[amp] = np.repeat(np.nan, lenInputTimes)
375 return dataset
377 @staticmethod
378 def _initialParsForPolynomial(order):
379 assert(order >= 2)
380 pars = np.zeros(order, dtype=np.float)
381 pars[0] = 10
382 pars[1] = 1
383 pars[2:] = 0.0001
384 return pars
386 @staticmethod
387 def _boundsForPolynomial(initialPars, lowers=[], uppers=[]):
388 if not len(lowers):
389 lowers = [np.NINF for p in initialPars]
390 if not len(uppers):
391 uppers = [np.inf for p in initialPars]
392 lowers[1] = 0 # no negative gains
393 return (lowers, uppers)
395 @staticmethod
396 def _boundsForAstier(initialPars, lowers=[], uppers=[]):
397 if not len(lowers):
398 lowers = [np.NINF for p in initialPars]
399 if not len(uppers):
400 uppers = [np.inf for p in initialPars]
401 return (lowers, uppers)
403 @staticmethod
404 def _getInitialGoodPoints(means, variances, maxDeviationPositive, maxDeviationNegative,
405 minMeanRatioTest, minVarPivotSearch):
406 """Return a boolean array to mask bad points.
408 Parameters
409 ----------
410 means : `numpy.array`
411 Input array with mean signal values.
412 variances : `numpy.array`
413 Input array with variances at each mean value.
414 maxDeviationPositive : `float`
415 Maximum deviation from being constant for the variance/mean
416 ratio, in the positive direction.
417 maxDeviationNegative : `float`
418 Maximum deviation from being constant for the variance/mean
419 ratio, in the negative direction.
420 minMeanRatioTest : `float`
421 Minimum signal value (in ADU) after which to start examining
422 the ratios var/mean.
423 minVarPivotSearch : `float`
424 Minimum variance point (in ADU^2) after which the pivot point
425 wher the variance starts decreasing should be sought.
427 Returns
428 ------
429 goodPoints : `numpy.array` [`bool`]
430 Boolean array to select good (`True`) and bad (`False`)
431 points.
433 Notes
434 -----
435 A linear function has a constant ratio, so find the median
436 value of the ratios, and exclude the points that deviate
437 from that by more than a factor of maxDeviationPositive/negative.
438 Asymmetric deviations are supported as we expect the PTC to turn
439 down as the flux increases, but sometimes it anomalously turns
440 upwards just before turning over, which ruins the fits, so it
441 is wise to be stricter about restricting positive outliers than
442 negative ones.
443 Too high and points that are so bad that fit will fail will be included
444 Too low and the non-linear points will be excluded, biasing the NL fit.
445 This function also masks points after the variance starts decreasing.
446 """
448 assert(len(means) == len(variances))
449 ratios = [b/a for (a, b) in zip(means, variances)]
450 medianRatio = np.nanmedian(ratios)
451 ratioDeviations = [0.0 if a < minMeanRatioTest else (r/medianRatio)-1
452 for (a, r) in zip(means, ratios)]
454 # so that it doesn't matter if the deviation is expressed as positive or negative
455 maxDeviationPositive = abs(maxDeviationPositive)
456 maxDeviationNegative = -1. * abs(maxDeviationNegative)
458 goodPoints = np.array([True if (r < maxDeviationPositive and r > maxDeviationNegative)
459 else False for r in ratioDeviations])
461 # Eliminate points beyond which the variance decreases
462 pivot = np.where(np.array(np.diff(variances)) < 0)[0]
463 if len(pivot) > 0:
464 # For small values, sometimes the variance decreases slightly
465 # Only look when var > self.config.minVarPivotSearch
466 pivot = [p for p in pivot if variances[p] > minVarPivotSearch]
467 if len(pivot) > 0:
468 pivot = np.min(pivot)
469 goodPoints[pivot+1:len(goodPoints)] = False
471 return goodPoints
473 def _makeZeroSafe(self, array, substituteValue=1e-9):
474 """"""
475 array = np.array(array)
476 nBad = Counter(np.ravel(array))[0]
477 if nBad == 0:
478 return array
480 index, = np.where(array == 0)
481 if len(index):
482 msg = f"Found {nBad} zeros in array at elements {index}"
483 self.log.warn(msg)
485 array[index] = substituteValue
487 return array
489 def fitPtc(self, dataset):
490 """Fit the photon transfer curve to a polynomial or to Astier+19 approximation.
492 Fit the photon transfer curve with either a polynomial of the order
493 specified in the task config, or using the exponential approximation
494 in Astier+19 (Eq. 16).
496 Sigma clipping is performed iteratively for the fit, as well as an
497 initial clipping of data points that are more than
498 config.initialNonLinearityExclusionThreshold away from lying on a
499 straight line. This other step is necessary because the photon transfer
500 curve turns over catastrophically at very high flux (because saturation
501 drops the variance to ~0) and these far outliers cause the initial fit
502 to fail, meaning the sigma cannot be calculated to perform the
503 sigma-clipping.
505 Parameters
506 ----------
507 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset`
508 The dataset containing the means, variances and exposure times.
510 Returns
511 -------
512 dataset: `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset`
513 This is the same dataset as the input parameter, however, it has been modified
514 to include information such as the fit vectors and the fit parameters. See
515 the class `PhotonTransferCurveDatase`.
517 Raises
518 ------
519 RuntimeError:
520 Raises if dataset.ptcFitType is None or empty.
521 """
522 if dataset.ptcFitType:
523 ptcFitType = dataset.ptcFitType
524 else:
525 raise RuntimeError("ptcFitType is None of empty in PTC dataset.")
526 matrixSide = self.config.maximumRangeCovariancesAstier
527 nanMatrix = np.empty((matrixSide, matrixSide))
528 nanMatrix[:] = np.nan
530 for amp in dataset.ampNames:
531 lenInputTimes = len(dataset.rawExpTimes[amp])
532 listNanMatrix = np.empty((lenInputTimes, matrixSide, matrixSide))
533 listNanMatrix[:] = np.nan
535 dataset.covariancesModel[amp] = listNanMatrix
536 dataset.aMatrix[amp] = nanMatrix
537 dataset.bMatrix[amp] = nanMatrix
538 dataset.covariancesModelNoB[amp] = listNanMatrix
539 dataset.aMatrixNoB[amp] = nanMatrix
541 def errFunc(p, x, y):
542 return ptcFunc(p, x) - y
544 sigmaCutPtcOutliers = self.config.sigmaCutPtcOutliers
545 maxIterationsPtcOutliers = self.config.maxIterationsPtcOutliers
547 for i, ampName in enumerate(dataset.ampNames):
548 timeVecOriginal = np.ravel(np.array(dataset.rawExpTimes[ampName]))
549 meanVecOriginal = np.ravel(np.array(dataset.rawMeans[ampName]))
550 varVecOriginal = np.ravel(np.array(dataset.rawVars[ampName]))
551 varVecOriginal = self._makeZeroSafe(varVecOriginal)
553 goodPoints = self._getInitialGoodPoints(meanVecOriginal, varVecOriginal,
554 self.config.initialNonLinearityExclusionThresholdPositive,
555 self.config.initialNonLinearityExclusionThresholdNegative,
556 self.config.minMeanRatioTest,
557 self.config.minVarPivotSearch)
558 if not (goodPoints.any()):
559 msg = (f"SERIOUS: All points in goodPoints: {goodPoints} are bad."
560 f"Setting {ampName} to BAD.")
561 self.log.warn(msg)
562 # Fill entries with NaNs
563 self.fillBadAmp(dataset, ptcFitType, ampName)
564 continue
566 mask = goodPoints
568 if ptcFitType == 'EXPAPPROXIMATION':
569 ptcFunc = funcAstier
570 parsIniPtc = [-1e-9, 1.0, 10.] # a00, gain, noisei^2
571 # lowers and uppers obtained from BOT data studies by C. Lage (UC Davis, 11/2020).
572 bounds = self._boundsForAstier(parsIniPtc, lowers=[-1e-4, 0.5, -2000],
573 uppers=[1e-4, 2.5, 2000])
574 if ptcFitType == 'POLYNOMIAL':
575 ptcFunc = funcPolynomial
576 parsIniPtc = self._initialParsForPolynomial(self.config.polynomialFitDegree + 1)
577 bounds = self._boundsForPolynomial(parsIniPtc)
579 # Before bootstrap fit, do an iterative fit to get rid of outliers
580 count = 1
581 while count <= maxIterationsPtcOutliers:
582 # Note that application of the mask actually shrinks the array
583 # to size rather than setting elements to zero (as we want) so
584 # always update mask itself and re-apply to the original data
585 meanTempVec = meanVecOriginal[mask]
586 varTempVec = varVecOriginal[mask]
587 res = least_squares(errFunc, parsIniPtc, bounds=bounds, args=(meanTempVec, varTempVec))
588 pars = res.x
590 # change this to the original from the temp because the masks are ANDed
591 # meaning once a point is masked it's always masked, and the masks must
592 # always be the same length for broadcasting
593 sigResids = (varVecOriginal - ptcFunc(pars, meanVecOriginal))/np.sqrt(varVecOriginal)
594 newMask = np.array([True if np.abs(r) < sigmaCutPtcOutliers else False for r in sigResids])
595 mask = mask & newMask
596 if not (mask.any() and newMask.any()):
597 msg = (f"SERIOUS: All points in either mask: {mask} or newMask: {newMask} are bad. "
598 f"Setting {ampName} to BAD.")
599 self.log.warn(msg)
600 # Fill entries with NaNs
601 self.fillBadAmp(dataset, ptcFitType, ampName)
602 break
603 nDroppedTotal = Counter(mask)[False]
604 self.log.debug(f"Iteration {count}: discarded {nDroppedTotal} points in total for {ampName}")
605 count += 1
606 # objects should never shrink
607 assert (len(mask) == len(timeVecOriginal) == len(meanVecOriginal) == len(varVecOriginal))
608 if not (mask.any() and newMask.any()):
609 continue
610 dataset.expIdMask[ampName] = mask # store the final mask
611 parsIniPtc = pars
612 meanVecFinal = meanVecOriginal[mask]
613 varVecFinal = varVecOriginal[mask]
615 if Counter(mask)[False] > 0:
616 self.log.info((f"Number of points discarded in PTC of amplifier {ampName}:"
617 f" {Counter(mask)[False]} out of {len(meanVecOriginal)}"))
619 if (len(meanVecFinal) < len(parsIniPtc)):
620 msg = (f"SERIOUS: Not enough data points ({len(meanVecFinal)}) compared to the number of "
621 f"parameters of the PTC model({len(parsIniPtc)}). Setting {ampName} to BAD.")
622 self.log.warn(msg)
623 # Fill entries with NaNs
624 self.fillBadAmp(dataset, ptcFitType, ampName)
625 continue
626 # Fit the PTC
627 if self.config.doFitBootstrap:
628 parsFit, parsFitErr, reducedChiSqPtc = fitBootstrap(parsIniPtc, meanVecFinal,
629 varVecFinal, ptcFunc,
630 weightsY=1./np.sqrt(varVecFinal))
631 else:
632 parsFit, parsFitErr, reducedChiSqPtc = fitLeastSq(parsIniPtc, meanVecFinal,
633 varVecFinal, ptcFunc,
634 weightsY=1./np.sqrt(varVecFinal))
635 dataset.ptcFitPars[ampName] = parsFit
636 dataset.ptcFitParsError[ampName] = parsFitErr
637 dataset.ptcFitChiSq[ampName] = reducedChiSqPtc
638 # Masked variances (measured and modeled) and means. Need to pad the array so astropy.Table does
639 # not crash (the mask may vary per amp).
640 padLength = len(dataset.rawExpTimes[ampName]) - len(varVecFinal)
641 dataset.finalVars[ampName] = np.pad(varVecFinal, (0, padLength), 'constant',
642 constant_values=np.nan)
643 dataset.finalModelVars[ampName] = np.pad(ptcFunc(parsFit, meanVecFinal), (0, padLength),
644 'constant', constant_values=np.nan)
645 dataset.finalMeans[ampName] = np.pad(meanVecFinal, (0, padLength), 'constant',
646 constant_values=np.nan)
647 if ptcFitType == 'EXPAPPROXIMATION':
648 ptcGain = parsFit[1]
649 ptcGainErr = parsFitErr[1]
650 ptcNoise = np.sqrt(np.fabs(parsFit[2]))
651 ptcNoiseErr = 0.5*(parsFitErr[2]/np.fabs(parsFit[2]))*np.sqrt(np.fabs(parsFit[2]))
652 if ptcFitType == 'POLYNOMIAL':
653 ptcGain = 1./parsFit[1]
654 ptcGainErr = np.fabs(1./parsFit[1])*(parsFitErr[1]/parsFit[1])
655 ptcNoise = np.sqrt(np.fabs(parsFit[0]))*ptcGain
656 ptcNoiseErr = (0.5*(parsFitErr[0]/np.fabs(parsFit[0]))*(np.sqrt(np.fabs(parsFit[0]))))*ptcGain
657 dataset.gain[ampName] = ptcGain
658 dataset.gainErr[ampName] = ptcGainErr
659 dataset.noise[ampName] = ptcNoise
660 dataset.noiseErr[ampName] = ptcNoiseErr
662 if not len(dataset.ptcFitType) == 0:
663 dataset.ptcFitType = ptcFitType
664 if len(dataset.badAmps) == 0:
665 dataset.badAmps = np.repeat(np.nan, len(list(dataset.rawExpTimes.values())[0]))
667 return dataset
669 def fillBadAmp(self, dataset, ptcFitType, ampName):
670 """Fill the dataset with NaNs if there are not enough good points.
672 Parameters
673 ----------
674 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset`
675 The dataset containing the means, variances and exposure times.
676 ptcFitType : `str`
677 Fit a 'POLYNOMIAL' (degree: 'polynomialFitDegree') or
678 'EXPAPPROXIMATION' (Eq. 16 of Astier+19) to the PTC.
679 ampName : `str`
680 Amplifier name.
681 """
682 dataset.badAmps.append(ampName)
683 dataset.expIdMask[ampName] = np.repeat(False, len(dataset.rawExpTimes[ampName]))
684 dataset.gain[ampName] = np.nan
685 dataset.gainErr[ampName] = np.nan
686 dataset.noise[ampName] = np.nan
687 dataset.noiseErr[ampName] = np.nan
688 dataset.ptcFitPars[ampName] = (np.repeat(np.nan, self.config.polynomialFitDegree + 1) if
689 ptcFitType in ["POLYNOMIAL", ] else np.repeat(np.nan, 3))
690 dataset.ptcFitParsError[ampName] = (np.repeat(np.nan, self.config.polynomialFitDegree + 1) if
691 ptcFitType in ["POLYNOMIAL", ] else np.repeat(np.nan, 3))
692 dataset.ptcFitChiSq[ampName] = np.nan
693 dataset.finalVars[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName]))
694 dataset.finalModelVars[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName]))
695 dataset.finalMeans[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName]))
697 return