Coverage for python/lsst/cp/pipe/ptc/cpSolvePtcTask.py: 11%
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of cp_pipe.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21#
22import numpy as np
23from collections import Counter
25import lsst.pex.config as pexConfig
26import lsst.pipe.base as pipeBase
27from lsst.cp.pipe.utils import (fitLeastSq, fitBootstrap, funcPolynomial, funcAstier, symmetrize)
29from scipy.signal import fftconvolve
30from scipy.optimize import least_squares
31from itertools import groupby
32from operator import itemgetter
34import lsst.pipe.base.connectionTypes as cT
36from lsst.ip.isr import PhotonTransferCurveDataset
38from lsst.cp.pipe._lookupStaticCalibration import lookupStaticCalibration
40import copy
43__all__ = ['PhotonTransferCurveSolveConfig', 'PhotonTransferCurveSolveTask']
46class PhotonTransferCurveSolveConnections(pipeBase.PipelineTaskConnections,
47 dimensions=("instrument", "detector")):
48 inputCovariances = cT.Input(
49 name="ptcCovariances",
50 doc="Tuple with measured covariances from flats.",
51 storageClass="PhotonTransferCurveDataset",
52 dimensions=("instrument", "exposure", "detector"),
53 multiple=True,
54 )
55 camera = cT.PrerequisiteInput(
56 name="camera",
57 doc="Camera the input data comes from.",
58 storageClass="Camera",
59 dimensions=("instrument",),
60 isCalibration=True,
61 lookupFunction=lookupStaticCalibration,
62 )
63 outputPtcDataset = cT.Output(
64 name="ptcDatsetProposal",
65 doc="Output proposed ptc dataset.",
66 storageClass="PhotonTransferCurveDataset",
67 dimensions=("instrument", "detector"),
68 multiple=False,
69 isCalibration=True,
70 )
73class PhotonTransferCurveSolveConfig(pipeBase.PipelineTaskConfig,
74 pipelineConnections=PhotonTransferCurveSolveConnections):
75 """Configuration for fitting measured covariances.
76 """
78 ptcFitType = pexConfig.ChoiceField(
79 dtype=str,
80 doc="Fit PTC to Eq. 16, Eq. 20 in Astier+19, or to a polynomial.",
81 default="POLYNOMIAL",
82 allowed={
83 "POLYNOMIAL": "n-degree polynomial (use 'polynomialFitDegree' to set 'n').",
84 "EXPAPPROXIMATION": "Approximation in Astier+19 (Eq. 16).",
85 "FULLCOVARIANCE": "Full covariances model in Astier+19 (Eq. 20)"
86 }
87 )
88 maximumRangeCovariancesAstier = pexConfig.Field(
89 dtype=int,
90 doc="Maximum range of covariances as in Astier+19",
91 default=8,
92 )
93 sigmaClipFullFitCovariancesAstier = pexConfig.Field(
94 dtype=float,
95 doc="sigma clip for full model fit for FULLCOVARIANCE ptcFitType ",
96 default=5.0,
97 )
98 maxIterFullFitCovariancesAstier = pexConfig.Field(
99 dtype=int,
100 doc="Maximum number of iterations in full model fit for FULLCOVARIANCE ptcFitType",
101 default=3,
102 )
103 polynomialFitDegree = pexConfig.Field(
104 dtype=int,
105 doc="Degree of polynomial to fit the PTC, when 'ptcFitType'=POLYNOMIAL.",
106 default=3,
107 )
108 sigmaCutPtcOutliers = pexConfig.Field(
109 dtype=float,
110 doc="Sigma cut for outlier rejection in PTC.",
111 default=5.0,
112 )
113 maxIterationsPtcOutliers = pexConfig.RangeField(
114 dtype=int,
115 doc="Maximum number of iterations for outlier rejection in PTC.",
116 default=2,
117 min=0
118 )
119 minVarPivotSearch = pexConfig.Field(
120 dtype=float,
121 doc="The code looks for a pivot signal point after which the variance starts decreasing at high-flux"
122 " to exclude then from the PTC model fit. However, sometimes at low fluxes, the variance"
123 " decreases slightly. Set this variable for the variance value, in ADU^2, after which the pivot "
124 " should be sought.",
125 default=10000,
126 )
127 consecutivePointsVarDecreases = pexConfig.RangeField(
128 dtype=int,
129 doc="Required number of consecutive points/fluxes in the PTC where the variance "
130 "decreases in order to find a first estimate of the PTC turn-off. ",
131 default=2,
132 min=2
133 )
134 doFitBootstrap = pexConfig.Field(
135 dtype=bool,
136 doc="Use bootstrap for the PTC fit parameters and errors?.",
137 default=False,
138 )
141class PhotonTransferCurveSolveTask(pipeBase.PipelineTask,
142 pipeBase.CmdLineTask):
143 """Task to fit the PTC from flat covariances.
145 The first task of the PTC measurement pipeline,
146 ``PhotonTransferCurveMeasureTask`` (and assumed to have been run
147 before this task), produced a list of
148 `~lsst.ip.isr.PhotonTransferCurveDataset` objects. Each dataset
149 contains the mean signal and covariances of the
150 difference image of the flat-field images taken at
151 the same exposure time. The list also contains dummy
152 datasets (with no measurements), whose purpose is to have
153 the input and output dimensions of ``PhotonTransferCurveMeasureTask``
154 match.
156 This task, ``PhotonTransferCurveSolveTask``, assembles the list
157 of individual PTC datasets produced
158 by ``PhotonTransferCurveMeasureTask`` into one single final PTC
159 dataset, discarding the dummy datset as appropiate.
160 The task fits the measured (co)variances to one of three models:
161 a polynomial model of a given order, or the models described
162 in equations 16 and 20 of Astier+19. These options are referred
163 to as ``POLYNOMIAL``, ``EXPAPPROXIMATION``, and ``FULLCOVARIANCE``
164 in the configuration options of the task, respectively).
165 Parameters of interest such as the gain and noise are derived
166 from the fits. The ``FULLCOVARIANCE`` model is fitted to the
167 full covariance data (as oppossed to the other two models, which
168 are fit to the variance vs mean measurements only).
170 Astier+19: "The Shape of the Photon Transfer Curve
171 of CCD sensors", arXiv:1905.08677
172 """
174 ConfigClass = PhotonTransferCurveSolveConfig
175 _DefaultName = 'cpPhotonTransferCurveSolve'
177 def runQuantum(self, butlerQC, inputRefs, outputRefs):
178 """Ensure that the input and output dimensions are passed along.
180 Parameters
181 ----------
182 butlerQC : `~lsst.daf.butler.butlerQuantumContext.ButlerQuantumContext`
183 Butler to operate on.
184 inputRefs : `~lsst.pipe.base.connections.InputQuantizedConnection`
185 Input data refs to load.
186 ouptutRefs : `~lsst.pipe.base.connections.OutputQuantizedConnection`
187 Output data refs to persist.
188 """
189 inputs = butlerQC.get(inputRefs)
190 outputs = self.run(inputCovariances=inputs['inputCovariances'], camera=inputs['camera'])
191 butlerQC.put(outputs, outputRefs)
193 def run(self, inputCovariances, camera=None, inputExpList=None):
194 """Fit measured covariances to different models.
196 Parameters
197 ----------
198 inputCovariances : `list` [`lsst.ip.isr.PhotonTransferCurveDataset`]
199 List of lsst.ip.isr.PhotonTransferCurveDataset datasets.
200 camera : `lsst.afw.cameraGeom.Camera`, optional
201 Input camera.
202 inputExpList : `list` [`~lsst.afw.image.ExposureF`], optional
203 List of exposures.
205 Returns
206 -------
207 results : `lsst.pipe.base.Struct`
208 The resultins structure contains:
209 ``outputPtcDatset``
210 Final PTC dataset, containing information such as the
211 means, variances, and exposure times
212 (`lsst.ip.isr.PhotonTransferCurveDataset`).
213 """
214 # Assemble individual PTC datasets into a single PTC dataset.
215 ampNames = np.unique(inputCovariances[0].ampNames)
216 datasetPtc = PhotonTransferCurveDataset(ampNames, self.config.ptcFitType,
217 self.config.maximumRangeCovariancesAstier)
218 for partialPtcDataset in inputCovariances:
219 # Ignore dummy datasets
220 if partialPtcDataset.ptcFitType == 'DUMMY':
221 continue
222 for ampName in ampNames:
223 datasetPtc.inputExpIdPairs[ampName].append(partialPtcDataset.inputExpIdPairs[ampName])
224 if type(partialPtcDataset.rawExpTimes[ampName]) is list:
225 datasetPtc.rawExpTimes[ampName].append(partialPtcDataset.rawExpTimes[ampName][0])
226 else:
227 datasetPtc.rawExpTimes[ampName].append(partialPtcDataset.rawExpTimes[ampName])
228 if type(partialPtcDataset.rawMeans[ampName]) is list:
229 datasetPtc.rawMeans[ampName].append(partialPtcDataset.rawMeans[ampName][0])
230 else:
231 datasetPtc.rawMeans[ampName].append(partialPtcDataset.rawMeans[ampName])
232 if type(partialPtcDataset.rawVars[ampName]) is list:
233 datasetPtc.rawVars[ampName].append(partialPtcDataset.rawVars[ampName][0])
234 else:
235 datasetPtc.rawVars[ampName].append(partialPtcDataset.rawVars[ampName])
236 if type(partialPtcDataset.expIdMask[ampName]) is list:
237 datasetPtc.expIdMask[ampName].append(partialPtcDataset.expIdMask[ampName][0])
238 else:
239 datasetPtc.expIdMask[ampName].append(partialPtcDataset.expIdMask[ampName])
240 datasetPtc.covariances[ampName].append(np.array(partialPtcDataset.covariances[ampName][0]))
241 datasetPtc.covariancesSqrtWeights[ampName].append(
242 np.array(partialPtcDataset.covariancesSqrtWeights[ampName][0]))
243 # Sort arrays that are filled so far in the final dataset by
244 # rawMeans index
245 for ampName in ampNames:
246 index = np.argsort(np.ravel(np.array(datasetPtc.rawMeans[ampName])))
247 datasetPtc.inputExpIdPairs[ampName] = np.array(datasetPtc.inputExpIdPairs[ampName])[index]
248 datasetPtc.rawExpTimes[ampName] = np.array(datasetPtc.rawExpTimes[ampName])[index]
249 datasetPtc.rawMeans[ampName] = np.array(datasetPtc.rawMeans[ampName])[index]
250 datasetPtc.rawVars[ampName] = np.array(datasetPtc.rawVars[ampName])[index]
251 datasetPtc.expIdMask[ampName] = np.array(datasetPtc.expIdMask[ampName])[index]
252 datasetPtc.covariances[ampName] = np.array(datasetPtc.covariances[ampName])[index]
253 datasetPtc.covariancesSqrtWeights[ampName] = np.array(
254 datasetPtc.covariancesSqrtWeights[ampName])[index]
255 if self.config.ptcFitType == "FULLCOVARIANCE":
256 # Fit the measured covariances vs mean signal to
257 # the Astier+19 full model (Eq. 20). Before that
258 # do a preliminary fit to the variance (C_00) vs mean
259 # signal (mu) curve using the EXPAPPROXIMATION model
260 # (Eq. 16 in Astier+19) in order to
261 # get the flat pairs that are masked. The
262 # points at these fluxes will also be masked when
263 # calculating the other elements of the covariance
264 # matrix, C_ij, i!=j).
266 # Preliminary fit, usign a temp dataset to get the mask
267 tempDatasetPtc = copy.copy(datasetPtc)
268 tempDatasetPtc.ptcFitType = "EXPAPPROXIMATION"
269 tempDatasetPtc = self.fitMeasurementsToModel(tempDatasetPtc)
271 # "FULLCOVARIANCE", using the mask obtained from the
272 # previous fit.
273 for ampName in datasetPtc.ampNames:
274 datasetPtc.expIdMask[ampName] = tempDatasetPtc.expIdMask[ampName]
275 datasetPtc.fitType = "FULLCOVARIANCE"
276 datasetPtc = self.fitMeasurementsToModel(datasetPtc)
277 # The other options are: self.config.ptcFitType in
278 # ("EXPAPPROXIMATION", "POLYNOMIAL")
279 else:
280 # Fit the PTC to a polynomial or to Astier+19 exponential
281 # approximation (Eq. 16). Fill up
282 # PhotonTransferCurveDataset object.
283 datasetPtc = self.fitMeasurementsToModel(datasetPtc)
284 if inputExpList is not None:
285 # It should be a list of exposures, to get the detector.
286 detector = inputExpList[0].getDetector()
287 else:
288 detector = None
289 datasetPtc.updateMetadata(setDate=True, camera=camera, detector=detector)
291 return pipeBase.Struct(
292 outputPtcDataset=datasetPtc,
293 )
295 def fitMeasurementsToModel(self, dataset):
296 """Fit the measured covariances vs mean signal to a
297 polynomial or one of the models in Astier+19
298 (Eq. 16 or Eq.20).
300 Parameters
301 ----------
302 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset`
303 The dataset containing information such as the means,
304 (co)variances, and exposure times.
306 Returns
307 -------
308 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset`
309 This is the same dataset as the input parameter, however,
310 it has been modified to include information such as the
311 fit vectors and the fit parameters. See the class
312 `PhotonTransferCurveDatase`.
313 """
314 fitType = dataset.ptcFitType
315 if fitType in ["FULLCOVARIANCE", ]:
316 # This model uses the full covariance matrix in the fit.
317 # The PTC is technically defined as variance vs signal,
318 # with variance = Cov_00
319 dataset = self.fitDataFullCovariance(dataset)
320 elif fitType in ["POLYNOMIAL", "EXPAPPROXIMATION"]:
321 # The PTC is technically defined as variance vs signal
322 dataset = self.fitPtc(dataset)
323 else:
324 raise RuntimeError(
325 f"Fitting option {fitType} not one of "
326 "'POLYNOMIAL', 'EXPAPPROXIMATION', or 'FULLCOVARIANCE'"
327 )
329 return dataset
331 def fitDataFullCovariance(self, dataset):
332 """Fit measured flat covariances to the full model in
333 Astier+19 (Eq. 20).
335 Parameters
336 ----------
337 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset`
338 The dataset containing information such as the means,
339 (co)variances, and exposure times.
341 Returns
342 -------
343 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset`
344 This is the same dataset as the input parameter, however,
345 it has been modified to include information such as the
346 fit vectors and the fit parameters. See the class
347 `PhotonTransferCurveDatase`.
349 Notes
350 -----
351 The parameters of the full model for C_ij(mu) ("C_ij" and "mu"
352 in ADU^2 and ADU, respectively) in Astier+19 (Eq. 20) are:
353 "a" coefficients (r by r matrix), units: 1/e
354 "b" coefficients (r by r matrix), units: 1/e
355 noise matrix (r by r matrix), units: e^2
356 gain, units: e/ADU
357 "b" appears in Eq. 20 only through the "ab" combination, which
358 is defined in this code as "c=ab".
360 Total number of parameters: #entries(a) + #entries(c) + #entries(noise)
361 + 1. This is equivalent to r^2 + r^2 + r^2 + 1, where "r" is the
362 maximum lag considered for the covariances calculation, and the
363 extra "1" is the gain. If "b" is 0, then "c" is 0, and len(pInit) will
364 have r^2 fewer entries.
365 """
366 matrixSide = self.config.maximumRangeCovariancesAstier
367 lenParams = matrixSide*matrixSide
369 for ampName in dataset.ampNames:
370 lenInputTimes = len(dataset.rawExpTimes[ampName])
371 # Not used when ptcFitType is 'FULLCOVARIANCE'
372 dataset.ptcFitPars[ampName] = [np.nan]
373 dataset.ptcFitParsError[ampName] = [np.nan]
374 dataset.ptcFitChiSq[ampName] = np.nan
376 if ampName in dataset.badAmps:
377 # Bad amp
378 # Entries need to have proper dimensions so read/write
379 # with astropy.Table works.
380 nanMatrix = np.full((matrixSide, matrixSide), np.nan)
381 listNanMatrix = np.full((lenInputTimes, matrixSide, matrixSide), np.nan)
382 dataset.covariancesModel[ampName] = listNanMatrix
383 dataset.covariancesSqrtWeights[ampName] = listNanMatrix
384 dataset.aMatrix[ampName] = nanMatrix
385 dataset.bMatrix[ampName] = nanMatrix
386 dataset.covariancesModelNoB[ampName] = listNanMatrix
387 dataset.aMatrixNoB[ampName] = nanMatrix
389 dataset.expIdMask[ampName] = np.repeat(np.nan, lenInputTimes)
390 dataset.gain[ampName] = np.nan
391 dataset.gainErr[ampName] = np.nan
392 dataset.noise[ampName] = np.nan
393 dataset.noiseErr[ampName] = np.nan
394 dataset.finalVars[ampName] = np.repeat(np.nan, lenInputTimes)
395 dataset.finalModelVars[ampName] = np.repeat(np.nan, lenInputTimes)
396 dataset.finalMeans[ampName] = np.repeat(np.nan, lenInputTimes)
397 continue
399 muAtAmp = dataset.rawMeans[ampName]
400 maskAtAmp = dataset.expIdMask[ampName]
401 if len(maskAtAmp) == 0:
402 maskAtAmp = np.repeat(True, len(muAtAmp))
404 muAtAmp = muAtAmp[maskAtAmp]
405 covAtAmp = np.nan_to_num(dataset.covariances[ampName])[maskAtAmp]
406 covSqrtWeightsAtAmp = np.nan_to_num(dataset.covariancesSqrtWeights[ampName])[maskAtAmp]
408 # Initial fit, to approximate parameters, with c=0
409 a0, c0, noise0, gain0 = self.initialFitFullCovariance(muAtAmp, covAtAmp, covSqrtWeightsAtAmp)
411 # Fit full model (Eq. 20 of Astier+19) and same model with
412 # b=0 (c=0 in this code)
413 pInit = np.concatenate((a0.flatten(), c0.flatten(), noise0.flatten(), np.array(gain0)), axis=None)
414 functionsDict = {'fullModel': self.funcFullCovarianceModel,
415 'fullModelNoB': self.funcFullCovarianceModelNoB}
416 fitResults = {'fullModel': {'a': [], 'c': [], 'noise': [], 'gain': [], 'paramsErr': []},
417 'fullModelNoB': {'a': [], 'c': [], 'noise': [], 'gain': [], 'paramsErr': []}}
418 for key in functionsDict:
419 params, paramsErr, _ = fitLeastSq(pInit, muAtAmp,
420 covAtAmp.flatten(), functionsDict[key],
421 weightsY=covSqrtWeightsAtAmp.flatten())
422 a = params[:lenParams].reshape((matrixSide, matrixSide))
423 c = params[lenParams:2*lenParams].reshape((matrixSide, matrixSide))
424 noise = params[2*lenParams:3*lenParams].reshape((matrixSide, matrixSide))
425 gain = params[-1]
427 fitResults[key]['a'] = a
428 fitResults[key]['c'] = c
429 fitResults[key]['noise'] = noise
430 fitResults[key]['gain'] = gain
431 fitResults[key]['paramsErr'] = paramsErr
433 # Put the information in the PTC dataset
435 # Not used when ptcFitType is 'FULLCOVARIANCE'
436 dataset.ptcFitPars[ampName] = [np.nan]
437 dataset.ptcFitParsError[ampName] = [np.nan]
438 dataset.ptcFitChiSq[ampName] = np.nan
440 # Save full covariances, covariances models, and their weights
441 # dataset.expIdMask is already full
442 dataset.covariances[ampName] = covAtAmp
443 dataset.covariancesModel[ampName] = self.evalCovModel(muAtAmp,
444 fitResults['fullModel']['a'],
445 fitResults['fullModel']['c'],
446 fitResults['fullModel']['noise'],
447 fitResults['fullModel']['gain'])
448 dataset.covariancesSqrtWeights[ampName] = covSqrtWeightsAtAmp
449 dataset.aMatrix[ampName] = fitResults['fullModel']['a']
450 dataset.bMatrix[ampName] = fitResults['fullModel']['c']/fitResults['fullModel']['a']
451 dataset.covariancesModelNoB[ampName] = self.evalCovModel(muAtAmp,
452 fitResults['fullModelNoB']['a'],
453 fitResults['fullModelNoB']['c'],
454 fitResults['fullModelNoB']['noise'],
455 fitResults['fullModelNoB']['gain'],
456 setBtoZero=True)
457 dataset.aMatrixNoB[ampName] = fitResults['fullModelNoB']['a']
458 dataset.gain[ampName] = fitResults['fullModel']['gain']
459 dataset.gainErr[ampName] = fitResults['fullModel']['paramsErr'][-1]
460 readoutNoise = fitResults['fullModel']['noise'][0][0]
461 readoutNoiseSqrt = np.sqrt(np.fabs(readoutNoise))
462 dataset.noise[ampName] = readoutNoise
463 readoutNoiseSigma = fitResults['fullModel']['paramsErr'][2*lenParams]
464 dataset.noiseErr[ampName] = 0.5*(readoutNoiseSigma/np.fabs(readoutNoise))*readoutNoiseSqrt
465 dataset.finalVars[ampName] = covAtAmp[:, 0, 0]
466 dataset.finalModelVars[ampName] = dataset.covariancesModel[ampName][:, 0, 0]
467 dataset.finalMeans[ampName] = muAtAmp
469 return dataset
471 def initialFitFullCovariance(self, mu, cov, sqrtW):
472 """ Performs a crude parabolic fit of the data in order to start
473 the full fit close to the solution, setting b=0 (c=0) in Eq. 20
474 of Astier+19.
476 Parameters
477 ----------
478 mu : `numpy.array`, (N,)
479 Signal `mu` (ADU)
480 cov : `numpy.array`, (N, M, M)
481 Covariance arrays of size `(M, M)` (with
482 `M = config.maximumRangeCovariancesAstier`),
483 indexed by mean signal `mu`.
484 sqrtW : `numpy.array`, (N,)
485 Covariance weights, defined as 1./sqrt(Variances)
487 Returns
488 -------
489 a : `numpy.array`, (M, M)
490 "a" parameter per flux in Eq. 20 of Astier+19.
491 c : `numpy.array`, (M, M)
492 "c"="ab" parameter per flux in Eq. 20 of Astier+19.
493 noise : `numpy.array`, (M, M)
494 "noise" parameter per flux in Eq. 20 of Astier+19.
495 gain : `float`
496 Amplifier gain (e/ADU)
497 """
498 matrixSide = self.config.maximumRangeCovariancesAstier
500 # Initialize fit parameters
501 a = np.zeros((matrixSide, matrixSide))
502 c = np.zeros((matrixSide, matrixSide))
503 noise = np.zeros((matrixSide, matrixSide))
504 gain = 1.
506 # iterate the fit to account for higher orders
507 # the chi2 does not necessarily go down, so one could
508 # stop when it increases
509 oldChi2 = 1e30
510 for _ in range(5):
511 model = np.nan_to_num(self.evalCovModel(mu, a, c, noise, gain, setBtoZero=True))
512 # loop on lags
513 for i in range(matrixSide):
514 for j in range(matrixSide):
515 # fit a parabola for a given lag
516 parsFit = np.polyfit(mu, cov[:, i, j] - model[:, i, j],
517 2, w=sqrtW[:, i, j])
518 # model equation (Eq. 20) in Astier+19, with c=a*b=0:
519 a[i, j] += parsFit[0]
520 noise[i, j] += parsFit[2]
521 if(i + j == 0):
522 gain = 1./(1/gain+parsFit[1])
523 weightedRes = (model - cov)*sqrtW
524 chi2 = (weightedRes.flatten()**2).sum()
525 if chi2 > oldChi2:
526 break
527 oldChi2 = chi2
529 return a, c, noise, gain
531 def funcFullCovarianceModel(self, params, x):
532 """Model to fit covariances from flat fields; Equation 20 of
533 Astier+19.
535 Parameters
536 ----------
537 params : `list`
538 Parameters of the model: aMatrix, CMatrix, noiseMatrix,
539 gain (e/ADU).
540 x : `numpy.array`, (N,)
541 Signal `mu` (ADU)
543 Returns
544 -------
545 y : `numpy.array`, (N,)
546 Covariance matrix.
547 """
548 matrixSide = self.config.maximumRangeCovariancesAstier
549 lenParams = matrixSide*matrixSide
550 aMatrix = params[:lenParams].reshape((matrixSide, matrixSide))
551 cMatrix = params[lenParams:2*lenParams].reshape((matrixSide, matrixSide))
552 noiseMatrix = params[2*lenParams:3*lenParams].reshape((matrixSide, matrixSide))
553 gain = params[-1]
555 return self.evalCovModel(x, aMatrix, cMatrix, noiseMatrix, gain).flatten()
557 def funcFullCovarianceModelNoB(self, params, x):
558 """Model to fit covariances from flat fields; Equation 20 of
559 Astier+19, with b=0 (equivalent to c=a*b=0 in this code).
561 Parameters
562 ----------
563 params : `list`
564 Parameters of the model: aMatrix, CMatrix, noiseMatrix,
565 gain (e/ADU).
566 x : `numpy.array`, (N,)
567 Signal mu (ADU)
569 Returns
570 -------
571 y : `numpy.array`, (N,)
572 Covariance matrix.
573 """
574 matrixSide = self.config.maximumRangeCovariancesAstier
575 lenParams = matrixSide*matrixSide
576 aMatrix = params[:lenParams].reshape((matrixSide, matrixSide))
577 cMatrix = params[lenParams:2*lenParams].reshape((matrixSide, matrixSide))
578 noiseMatrix = params[2*lenParams:3*lenParams].reshape((matrixSide, matrixSide))
579 gain = params[-1]
581 return self.evalCovModel(x, aMatrix, cMatrix, noiseMatrix, gain, setBtoZero=True).flatten()
583 def evalCovModel(self, mu, aMatrix, cMatrix, noiseMatrix, gain, setBtoZero=False):
584 """Computes full covariances model (Eq. 20 of Astier+19).
586 Parameters
587 ----------
588 mu : `numpy.array`, (N,)
589 List of mean signals.
590 aMatrix : `numpy.array`, (M, M)
591 "a" parameter per flux in Eq. 20 of Astier+19.
592 cMatrix : `numpy.array`, (M, M)
593 "c"="ab" parameter per flux in Eq. 20 of Astier+19.
594 noiseMatrix : `numpy.array`, (M, M)
595 "noise" parameter per flux in Eq. 20 of Astier+19.
596 gain : `float`
597 Amplifier gain (e/ADU)
598 setBtoZero=False : `bool`, optional
599 Set "b" parameter in full model (see Astier+19) to zero.
601 Returns
602 -------
603 covModel : `numpy.array`, (N, M, M)
604 Covariances model.
606 Notes
607 -----
608 By default, computes the covModel for the mu's stored(self.mu).
609 Returns cov[Nmu, M, M]. The variance for the PTC is
610 cov[:, 0, 0]. mu and cov are in ADUs and ADUs squared. To use
611 electrons for both, the gain should be set to 1. This routine
612 implements the model in Astier+19 (1905.08677).
613 The parameters of the full model for C_ij(mu) ("C_ij" and "mu"
614 in ADU^2 and ADU, respectively) in Astier+19 (Eq. 20) are:
615 "a" coefficients (M by M matrix), units: 1/e
616 "b" coefficients (M by M matrix), units: 1/e
617 noise matrix (M by M matrix), units: e^2
618 gain, units: e/ADU
619 "b" appears in Eq. 20 only through the "ab" combination, which
620 is defined in this code as "c=ab".
621 """
622 matrixSide = self.config.maximumRangeCovariancesAstier
623 sa = (matrixSide, matrixSide)
624 # pad a with zeros and symmetrize
625 aEnlarged = np.zeros((int(sa[0]*1.5)+1, int(sa[1]*1.5)+1))
626 aEnlarged[0:sa[0], 0:sa[1]] = aMatrix
627 aSym = symmetrize(aEnlarged)
628 # pad c with zeros and symmetrize
629 cEnlarged = np.zeros((int(sa[0]*1.5)+1, int(sa[1]*1.5)+1))
630 cEnlarged[0:sa[0], 0:sa[1]] = cMatrix
631 cSym = symmetrize(cEnlarged)
632 a2 = fftconvolve(aSym, aSym, mode='same')
633 a3 = fftconvolve(a2, aSym, mode='same')
634 ac = fftconvolve(aSym, cSym, mode='same')
635 (xc, yc) = np.unravel_index(np.abs(aSym).argmax(), a2.shape)
637 a1 = aMatrix[np.newaxis, :, :]
638 a2 = a2[np.newaxis, xc:xc + matrixSide, yc:yc + matrixSide]
639 a3 = a3[np.newaxis, xc:xc + matrixSide, yc:yc + matrixSide]
640 ac = ac[np.newaxis, xc:xc + matrixSide, yc:yc + matrixSide]
641 c1 = cMatrix[np.newaxis, ::]
643 # assumes that mu is 1d
644 bigMu = mu[:, np.newaxis, np.newaxis]*gain
645 # c(=a*b in Astier+19) also has a contribution to the last
646 # term, that is absent for now.
647 if setBtoZero:
648 c1 = np.zeros_like(c1)
649 ac = np.zeros_like(ac)
650 covModel = (bigMu/(gain*gain)*(a1*bigMu+2./3.*(bigMu*bigMu)*(a2 + c1)
651 + (1./3.*a3 + 5./6.*ac)*(bigMu*bigMu*bigMu)) + noiseMatrix[np.newaxis, :, :]/gain**2)
652 # add the Poisson term, and the read out noise (variance)
653 covModel[:, 0, 0] += mu/gain
655 return covModel
657 # EXPAPPROXIMATION and POLYNOMIAL fit methods
658 @staticmethod
659 def _initialParsForPolynomial(order):
660 assert(order >= 2)
661 pars = np.zeros(order, dtype=float)
662 pars[0] = 10
663 pars[1] = 1
664 pars[2:] = 0.0001
665 return pars
667 @staticmethod
668 def _boundsForPolynomial(initialPars, lowers=[], uppers=[]):
669 if not len(lowers):
670 lowers = [np.NINF for p in initialPars]
671 if not len(uppers):
672 uppers = [np.inf for p in initialPars]
673 lowers[1] = 0 # no negative gains
674 return (lowers, uppers)
676 @staticmethod
677 def _boundsForAstier(initialPars, lowers=[], uppers=[]):
678 if not len(lowers):
679 lowers = [np.NINF for p in initialPars]
680 if not len(uppers):
681 uppers = [np.inf for p in initialPars]
682 return (lowers, uppers)
684 @staticmethod
685 def _getInitialGoodPoints(means, variances, minVarPivotSearch, consecutivePointsVarDecreases):
686 """Return a boolean array to mask bad points.
688 Parameters
689 ----------
690 means : `numpy.array`
691 Input array with mean signal values.
692 variances : `numpy.array`
693 Input array with variances at each mean value.
694 minVarPivotSearch : `float`
695 The variance (in ADU^2), above which, the point
696 of decreasing variance should be sought.
697 consecutivePointsVarDecreases : `int`
698 Required number of consecutive points/fluxes
699 in the PTC where the variance
700 decreases in order to find a first
701 estimate of the PTC turn-off.
703 Returns
704 ------
705 goodPoints : `numpy.array` [`bool`]
706 Boolean array to select good (`True`) and bad (`False`)
707 points.
709 Notes
710 -----
711 Eliminate points beyond which the variance decreases.
712 """
713 goodPoints = np.ones_like(means, dtype=bool)
714 # Variances are sorted and should monotonically increase
715 pivotList = np.where(np.array(np.diff(variances)) < 0)[0]
716 if len(pivotList) > 0:
717 # For small values, sometimes the variance decreases slightly
718 # Only look when var > self.config.minVarPivotSearch
719 pivotList = [p for p in pivotList if variances[p] > minVarPivotSearch]
720 # Require that the varince decreases during
721 # consecutivePointsVarDecreases
722 # consecutive points. This will give a first
723 # estimate of the PTC turn-off, which
724 # may be updated (reduced) further in the code.
725 if len(pivotList) > 1:
726 # enumerate(pivotList) creates tuples (index, value), for
727 # each value in pivotList. The lambda function subtracts
728 # each value from the index.
729 # groupby groups elements by equal key value.
730 for k, g in groupby(enumerate(pivotList), lambda x: x[0]-x[1]):
731 group = (map(itemgetter(1), g))
732 # Form groups of consecute values from pivotList
733 group = list(map(int, group))
734 # values in pivotList are indices where np.diff(variances)
735 # is negative, i.e., where the variance starts decreasing.
736 # Find the first group of consecutive numbers when
737 # variance decreases.
738 if len(group) >= consecutivePointsVarDecreases:
739 pivotIndex = np.min(group)
740 goodPoints[pivotIndex+1:] = False
741 break
743 return goodPoints
745 def _makeZeroSafe(self, array, substituteValue=1e-9):
746 """"""
747 array = np.array(array)
748 nBad = Counter(np.ravel(array))[0]
749 if nBad == 0:
750 return array
752 index, = np.where(array == 0)
753 if len(index):
754 msg = f"Found {nBad} zeros in array at elements {index}"
755 self.log.warning(msg)
757 array[index] = substituteValue
759 return array
761 def fitPtc(self, dataset):
762 """Fit the photon transfer curve to a polynomial or to the
763 Astier+19 approximation (Eq. 16).
765 Fit the photon transfer curve with either a polynomial of
766 the order specified in the task config, or using the
767 exponential approximation in Astier+19 (Eq. 16).
769 Sigma clipping is performed iteratively for the fit, as
770 well as an initial clipping of data points that are more
771 than `config.initialNonLinearityExclusionThreshold` away
772 from lying on a straight line. This other step is necessary
773 because the photon transfer curve turns over catastrophically
774 at very high flux (because saturation
775 drops the variance to ~0) and these far outliers cause the
776 initial fit to fail, meaning the sigma cannot be calculated
777 to perform the sigma-clipping.
779 Parameters
780 ----------
781 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset`
782 The dataset containing the means, variances and
783 exposure times.
785 Returns
786 -------
787 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset`
788 This is the same dataset as the input parameter, however,
789 it has been modified to include information such as the
790 fit vectors and the fit parameters. See the class
791 `PhotonTransferCurveDatase`.
793 Raises
794 ------
795 RuntimeError:
796 Raises if dataset.ptcFitType is None or empty.
797 """
798 if dataset.ptcFitType:
799 ptcFitType = dataset.ptcFitType
800 else:
801 raise RuntimeError("ptcFitType is None of empty in PTC dataset.")
802 matrixSide = self.config.maximumRangeCovariancesAstier
803 nanMatrix = np.empty((matrixSide, matrixSide))
804 nanMatrix[:] = np.nan
806 for amp in dataset.ampNames:
807 lenInputTimes = len(dataset.rawExpTimes[amp])
808 listNanMatrix = np.empty((lenInputTimes, matrixSide, matrixSide))
809 listNanMatrix[:] = np.nan
811 dataset.covariancesModel[amp] = listNanMatrix
812 dataset.aMatrix[amp] = nanMatrix
813 dataset.bMatrix[amp] = nanMatrix
814 dataset.covariancesModelNoB[amp] = listNanMatrix
815 dataset.aMatrixNoB[amp] = nanMatrix
817 def errFunc(p, x, y):
818 return ptcFunc(p, x) - y
820 sigmaCutPtcOutliers = self.config.sigmaCutPtcOutliers
821 maxIterationsPtcOutliers = self.config.maxIterationsPtcOutliers
823 for i, ampName in enumerate(dataset.ampNames):
824 timeVecOriginal = np.ravel(np.array(dataset.rawExpTimes[ampName]))
825 meanVecOriginal = np.ravel(np.array(dataset.rawMeans[ampName]))
826 varVecOriginal = np.ravel(np.array(dataset.rawVars[ampName]))
827 varVecOriginal = self._makeZeroSafe(varVecOriginal)
829 # Discard points when the variance starts to decrease after two
830 # consecutive signal levels
831 goodPoints = self._getInitialGoodPoints(meanVecOriginal, varVecOriginal,
832 self.config.minVarPivotSearch,
833 self.config.consecutivePointsVarDecreases)
834 # Check if all points are bad from the 'cpExtractPtcTask'
835 initialExpIdMask = np.ravel(np.array(dataset.expIdMask[ampName]))
837 if not (goodPoints.any() and initialExpIdMask.any()):
838 msg = (f"SERIOUS: All points in goodPoints: {goodPoints} or "
839 f"in initialExpIdMask: {initialExpIdMask} are bad."
840 f"Setting {ampName} to BAD.")
841 self.log.warning(msg)
842 # Fill entries with NaNs
843 self.fillBadAmp(dataset, ptcFitType, ampName)
844 continue
846 mask = goodPoints
848 if ptcFitType == 'EXPAPPROXIMATION':
849 ptcFunc = funcAstier
850 parsIniPtc = [-1e-9, 1.0, 10.] # a00, gain, noise^2
851 # lowers and uppers obtained from BOT data studies by
852 # C. Lage (UC Davis, 11/2020).
853 bounds = self._boundsForAstier(parsIniPtc, lowers=[-1e-4, 0.5, -2000],
854 uppers=[1e-4, 2.5, 2000])
855 if ptcFitType == 'POLYNOMIAL':
856 ptcFunc = funcPolynomial
857 parsIniPtc = self._initialParsForPolynomial(self.config.polynomialFitDegree + 1)
858 bounds = self._boundsForPolynomial(parsIniPtc)
860 # Before bootstrap fit, do an iterative fit to get rid of outliers.
861 # This further process of outlier rejection be skipped
862 # if self.config.maxIterationsPtcOutliers = 0.
863 # We already did some initial outlier rejection above in
864 # self._getInitialGoodPoints.
865 count = 1
866 newMask = np.ones_like(meanVecOriginal, dtype=bool)
867 pars = parsIniPtc
868 while count <= maxIterationsPtcOutliers:
869 # Note that application of the mask actually shrinks the array
870 # to size rather than setting elements to zero (as we want) so
871 # always update mask itself and re-apply to the original data
872 meanTempVec = meanVecOriginal[mask]
873 varTempVec = varVecOriginal[mask]
874 res = least_squares(errFunc, parsIniPtc, bounds=bounds, args=(meanTempVec, varTempVec))
875 pars = res.x
877 # change this to the original from the temp because
878 # the masks are ANDed meaning once a point is masked
879 # it's always masked, and the masks must always be the
880 # same length for broadcasting
881 sigResids = (varVecOriginal - ptcFunc(pars, meanVecOriginal))/np.sqrt(varVecOriginal)
882 newMask = np.array([True if np.abs(r) < sigmaCutPtcOutliers else False for r in sigResids])
883 mask = mask & newMask
884 if not (mask.any() and newMask.any()):
885 msg = (f"SERIOUS: All points in either mask: {mask} or newMask: {newMask} are bad. "
886 f"Setting {ampName} to BAD.")
887 self.log.warning(msg)
888 # Fill entries with NaNs
889 self.fillBadAmp(dataset, ptcFitType, ampName)
890 break
891 nDroppedTotal = Counter(mask)[False]
892 self.log.debug("Iteration %d: discarded %d points in total for %s",
893 count, nDroppedTotal, ampName)
894 count += 1
895 # objects should never shrink
896 assert (len(mask) == len(timeVecOriginal) == len(meanVecOriginal) == len(varVecOriginal))
897 if not (mask.any() and newMask.any()):
898 continue
899 dataset.expIdMask[ampName] = np.array(dataset.expIdMask[ampName])
900 # store the final mask
901 if len(dataset.expIdMask[ampName]):
902 dataset.expIdMask[ampName] &= mask # bitwise_and if there is already a mask
903 else:
904 dataset.expIdMask[ampName] = mask
905 parsIniPtc = pars
906 meanVecFinal = meanVecOriginal[mask]
907 varVecFinal = varVecOriginal[mask]
909 if Counter(mask)[False] > 0:
910 self.log.info("Number of points discarded in PTC of amplifier %s:"
911 " %d out of %d", ampName, Counter(mask)[False], len(meanVecOriginal))
913 if (len(meanVecFinal) < len(parsIniPtc)):
914 msg = (f"SERIOUS: Not enough data points ({len(meanVecFinal)}) compared to the number of "
915 f"parameters of the PTC model({len(parsIniPtc)}). Setting {ampName} to BAD.")
916 self.log.warning(msg)
917 # Fill entries with NaNs
918 self.fillBadAmp(dataset, ptcFitType, ampName)
919 continue
920 # Fit the PTC
921 if self.config.doFitBootstrap:
922 parsFit, parsFitErr, reducedChiSqPtc = fitBootstrap(parsIniPtc, meanVecFinal,
923 varVecFinal, ptcFunc,
924 weightsY=1./np.sqrt(varVecFinal))
925 else:
926 parsFit, parsFitErr, reducedChiSqPtc = fitLeastSq(parsIniPtc, meanVecFinal,
927 varVecFinal, ptcFunc,
928 weightsY=1./np.sqrt(varVecFinal))
929 dataset.ptcFitPars[ampName] = parsFit
930 dataset.ptcFitParsError[ampName] = parsFitErr
931 dataset.ptcFitChiSq[ampName] = reducedChiSqPtc
932 # Masked variances (measured and modeled) and means. Need
933 # to pad the array so astropy.Table does not crash (the
934 # mask may vary per amp).
935 padLength = len(dataset.rawExpTimes[ampName]) - len(varVecFinal)
936 dataset.finalVars[ampName] = np.pad(varVecFinal, (0, padLength), 'constant',
937 constant_values=np.nan)
938 dataset.finalModelVars[ampName] = np.pad(ptcFunc(parsFit, meanVecFinal), (0, padLength),
939 'constant', constant_values=np.nan)
940 dataset.finalMeans[ampName] = np.pad(meanVecFinal, (0, padLength), 'constant',
941 constant_values=np.nan)
942 if ptcFitType == 'EXPAPPROXIMATION':
943 ptcGain = parsFit[1]
944 ptcGainErr = parsFitErr[1]
945 ptcNoise = np.sqrt(np.fabs(parsFit[2]))
946 ptcNoiseErr = 0.5*(parsFitErr[2]/np.fabs(parsFit[2]))*np.sqrt(np.fabs(parsFit[2]))
947 if ptcFitType == 'POLYNOMIAL':
948 ptcGain = 1./parsFit[1]
949 ptcGainErr = np.fabs(1./parsFit[1])*(parsFitErr[1]/parsFit[1])
950 ptcNoise = np.sqrt(np.fabs(parsFit[0]))*ptcGain
951 ptcNoiseErr = (0.5*(parsFitErr[0]/np.fabs(parsFit[0]))*(np.sqrt(np.fabs(parsFit[0]))))*ptcGain
952 dataset.gain[ampName] = ptcGain
953 dataset.gainErr[ampName] = ptcGainErr
954 dataset.noise[ampName] = ptcNoise
955 dataset.noiseErr[ampName] = ptcNoiseErr
957 if not len(dataset.ptcFitType) == 0:
958 dataset.ptcFitType = ptcFitType
959 if len(dataset.badAmps) == 0:
960 dataset.badAmps = np.repeat(np.nan, len(list(dataset.rawExpTimes.values())[0]))
962 return dataset
964 def fillBadAmp(self, dataset, ptcFitType, ampName):
965 """Fill the dataset with NaNs if there are not enough
966 good points.
968 Parameters
969 ----------
970 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset`
971 The dataset containing the means, variances and
972 exposure times.
973 ptcFitType : {'POLYNOMIAL', 'EXPAPPROXIMATION'}
974 Fit a 'POLYNOMIAL' (degree: 'polynomialFitDegree') or
975 'EXPAPPROXIMATION' (Eq. 16 of Astier+19) to the PTC.
976 ampName : `str`
977 Amplifier name.
978 """
979 dataset.badAmps.append(ampName)
980 dataset.expIdMask[ampName] = np.repeat(False, len(dataset.rawExpTimes[ampName]))
981 dataset.gain[ampName] = np.nan
982 dataset.gainErr[ampName] = np.nan
983 dataset.noise[ampName] = np.nan
984 dataset.noiseErr[ampName] = np.nan
985 dataset.ptcFitPars[ampName] = (np.repeat(np.nan, self.config.polynomialFitDegree + 1) if
986 ptcFitType in ["POLYNOMIAL", ] else np.repeat(np.nan, 3))
987 dataset.ptcFitParsError[ampName] = (np.repeat(np.nan, self.config.polynomialFitDegree + 1) if
988 ptcFitType in ["POLYNOMIAL", ] else np.repeat(np.nan, 3))
989 dataset.ptcFitChiSq[ampName] = np.nan
990 dataset.finalVars[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName]))
991 dataset.finalModelVars[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName]))
992 dataset.finalMeans[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName]))
994 return