Coverage for python/lsst/cp/pipe/ptc/cpSolvePtcTask.py: 11%
397 statements
« prev ^ index » next coverage.py v6.4.4, created at 2022-08-25 01:44 -0700
« prev ^ index » next coverage.py v6.4.4, created at 2022-08-25 01:44 -0700
1# This file is part of cp_pipe.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21#
22import numpy as np
23from collections import Counter
25import lsst.pex.config as pexConfig
26import lsst.pipe.base as pipeBase
27from lsst.cp.pipe.utils import (fitLeastSq, fitBootstrap, funcPolynomial, funcAstier, symmetrize)
29from scipy.signal import fftconvolve
30from scipy.optimize import least_squares
31from itertools import groupby
32from operator import itemgetter
34import lsst.pipe.base.connectionTypes as cT
36from lsst.ip.isr import PhotonTransferCurveDataset
38from lsst.cp.pipe._lookupStaticCalibration import lookupStaticCalibration
40import copy
43__all__ = ['PhotonTransferCurveSolveConfig', 'PhotonTransferCurveSolveTask']
46class PhotonTransferCurveSolveConnections(pipeBase.PipelineTaskConnections,
47 dimensions=("instrument", "detector")):
48 inputCovariances = cT.Input(
49 name="ptcCovariances",
50 doc="Tuple with measured covariances from flats.",
51 storageClass="PhotonTransferCurveDataset",
52 dimensions=("instrument", "exposure", "detector"),
53 isCalibration=True,
54 multiple=True,
55 )
56 camera = cT.PrerequisiteInput(
57 name="camera",
58 doc="Camera the input data comes from.",
59 storageClass="Camera",
60 dimensions=("instrument",),
61 isCalibration=True,
62 lookupFunction=lookupStaticCalibration,
63 )
64 outputPtcDataset = cT.Output(
65 name="ptcDatsetProposal",
66 doc="Output proposed ptc dataset.",
67 storageClass="PhotonTransferCurveDataset",
68 dimensions=("instrument", "detector"),
69 multiple=False,
70 isCalibration=True,
71 )
74class PhotonTransferCurveSolveConfig(pipeBase.PipelineTaskConfig,
75 pipelineConnections=PhotonTransferCurveSolveConnections):
76 """Configuration for fitting measured covariances.
77 """
79 ptcFitType = pexConfig.ChoiceField(
80 dtype=str,
81 doc="Fit PTC to Eq. 16, Eq. 20 in Astier+19, or to a polynomial.",
82 default="POLYNOMIAL",
83 allowed={
84 "POLYNOMIAL": "n-degree polynomial (use 'polynomialFitDegree' to set 'n').",
85 "EXPAPPROXIMATION": "Approximation in Astier+19 (Eq. 16).",
86 "FULLCOVARIANCE": "Full covariances model in Astier+19 (Eq. 20)"
87 }
88 )
89 maximumRangeCovariancesAstier = pexConfig.Field(
90 dtype=int,
91 doc="Maximum range of covariances as in Astier+19",
92 default=8,
93 )
94 sigmaClipFullFitCovariancesAstier = pexConfig.Field(
95 dtype=float,
96 doc="sigma clip for full model fit for FULLCOVARIANCE ptcFitType ",
97 default=5.0,
98 )
99 maxIterFullFitCovariancesAstier = pexConfig.Field(
100 dtype=int,
101 doc="Maximum number of iterations in full model fit for FULLCOVARIANCE ptcFitType",
102 default=3,
103 )
104 polynomialFitDegree = pexConfig.Field(
105 dtype=int,
106 doc="Degree of polynomial to fit the PTC, when 'ptcFitType'=POLYNOMIAL.",
107 default=3,
108 )
109 sigmaCutPtcOutliers = pexConfig.Field(
110 dtype=float,
111 doc="Sigma cut for outlier rejection in PTC.",
112 default=5.0,
113 )
114 maxIterationsPtcOutliers = pexConfig.RangeField(
115 dtype=int,
116 doc="Maximum number of iterations for outlier rejection in PTC.",
117 default=2,
118 min=0
119 )
120 minVarPivotSearch = pexConfig.Field(
121 dtype=float,
122 doc="The code looks for a pivot signal point after which the variance starts decreasing at high-flux"
123 " to exclude then from the PTC model fit. However, sometimes at low fluxes, the variance"
124 " decreases slightly. Set this variable for the variance value, in ADU^2, after which the pivot "
125 " should be sought.",
126 default=10000,
127 )
128 consecutivePointsVarDecreases = pexConfig.RangeField(
129 dtype=int,
130 doc="Required number of consecutive points/fluxes in the PTC where the variance "
131 "decreases in order to find a first estimate of the PTC turn-off. ",
132 default=2,
133 min=2
134 )
135 doFitBootstrap = pexConfig.Field(
136 dtype=bool,
137 doc="Use bootstrap for the PTC fit parameters and errors?.",
138 default=False,
139 )
142class PhotonTransferCurveSolveTask(pipeBase.PipelineTask):
143 """Task to fit the PTC from flat covariances.
145 The first task of the PTC measurement pipeline,
146 ``PhotonTransferCurveMeasureTask`` (and assumed to have been run
147 before this task), produced a list of
148 `~lsst.ip.isr.PhotonTransferCurveDataset` objects. Each dataset
149 contains the mean signal and covariances of the
150 difference image of the flat-field images taken at
151 the same exposure time. The list also contains dummy
152 datasets (with no measurements), whose purpose is to have
153 the input and output dimensions of ``PhotonTransferCurveMeasureTask``
154 match.
156 This task, ``PhotonTransferCurveSolveTask``, assembles the list
157 of individual PTC datasets produced
158 by ``PhotonTransferCurveMeasureTask`` into one single final PTC
159 dataset, discarding the dummy datset as appropiate.
160 The task fits the measured (co)variances to one of three models:
161 a polynomial model of a given order, or the models described
162 in equations 16 and 20 of Astier+19. These options are referred
163 to as ``POLYNOMIAL``, ``EXPAPPROXIMATION``, and ``FULLCOVARIANCE``
164 in the configuration options of the task, respectively).
165 Parameters of interest such as the gain and noise are derived
166 from the fits. The ``FULLCOVARIANCE`` model is fitted to the
167 full covariance data (as oppossed to the other two models, which
168 are fit to the variance vs mean measurements only).
170 Astier+19: "The Shape of the Photon Transfer Curve
171 of CCD sensors", arXiv:1905.08677
172 """
174 ConfigClass = PhotonTransferCurveSolveConfig
175 _DefaultName = 'cpPhotonTransferCurveSolve'
177 def runQuantum(self, butlerQC, inputRefs, outputRefs):
178 """Ensure that the input and output dimensions are passed along.
180 Parameters
181 ----------
182 butlerQC : `~lsst.daf.butler.butlerQuantumContext.ButlerQuantumContext`
183 Butler to operate on.
184 inputRefs : `~lsst.pipe.base.connections.InputQuantizedConnection`
185 Input data refs to load.
186 ouptutRefs : `~lsst.pipe.base.connections.OutputQuantizedConnection`
187 Output data refs to persist.
188 """
189 inputs = butlerQC.get(inputRefs)
190 detId = inputRefs.inputCovariances[0].dataId['detector']
191 outputs = self.run(inputCovariances=inputs['inputCovariances'], camera=inputs['camera'], detId=detId)
192 butlerQC.put(outputs, outputRefs)
194 def run(self, inputCovariances, camera=None, detId=0):
195 """Fit measured covariances to different models.
197 Parameters
198 ----------
199 inputCovariances : `list` [`lsst.ip.isr.PhotonTransferCurveDataset`]
200 List of lsst.ip.isr.PhotonTransferCurveDataset datasets.
201 camera : `lsst.afw.cameraGeom.Camera`, optional
202 Input camera.
203 detId : `int`
204 Detector ID to locate the detector in the camera and
205 populate the `lsst.ip.isr.PhotonTransferCurveDataset`
206 metadata.
207 Returns
208 -------
209 results : `lsst.pipe.base.Struct`
210 The resultins structure contains:
211 ``outputPtcDatset``
212 Final PTC dataset, containing information such as the
213 means, variances, and exposure times
214 (`lsst.ip.isr.PhotonTransferCurveDataset`).
215 """
216 # Assemble individual PTC datasets into a single PTC dataset.
217 ampNames = np.unique(inputCovariances[0].ampNames)
218 datasetPtc = PhotonTransferCurveDataset(ampNames=ampNames,
219 ptcFitType=self.config.ptcFitType,
220 covMatrixSide=self.config.maximumRangeCovariancesAstier)
221 for partialPtcDataset in inputCovariances:
222 # Ignore dummy datasets
223 if partialPtcDataset.ptcFitType == 'DUMMY':
224 continue
225 for ampName in ampNames:
226 datasetPtc.inputExpIdPairs[ampName].append(partialPtcDataset.inputExpIdPairs[ampName])
227 if type(partialPtcDataset.rawExpTimes[ampName]) is list:
228 datasetPtc.rawExpTimes[ampName].append(partialPtcDataset.rawExpTimes[ampName][0])
229 else:
230 datasetPtc.rawExpTimes[ampName].append(partialPtcDataset.rawExpTimes[ampName])
231 if type(partialPtcDataset.rawMeans[ampName]) is list:
232 datasetPtc.rawMeans[ampName].append(partialPtcDataset.rawMeans[ampName][0])
233 else:
234 datasetPtc.rawMeans[ampName].append(partialPtcDataset.rawMeans[ampName])
235 if type(partialPtcDataset.rawVars[ampName]) is list:
236 datasetPtc.rawVars[ampName].append(partialPtcDataset.rawVars[ampName][0])
237 else:
238 datasetPtc.rawVars[ampName].append(partialPtcDataset.rawVars[ampName])
239 if type(partialPtcDataset.expIdMask[ampName]) is list:
240 datasetPtc.expIdMask[ampName].append(partialPtcDataset.expIdMask[ampName][0])
241 else:
242 datasetPtc.expIdMask[ampName].append(partialPtcDataset.expIdMask[ampName])
243 datasetPtc.covariances[ampName].append(np.array(partialPtcDataset.covariances[ampName][0]))
244 datasetPtc.covariancesSqrtWeights[ampName].append(
245 np.array(partialPtcDataset.covariancesSqrtWeights[ampName][0]))
246 # Sort arrays that are filled so far in the final dataset by
247 # rawMeans index
248 for ampName in ampNames:
249 index = np.argsort(np.ravel(np.array(datasetPtc.rawMeans[ampName])))
250 datasetPtc.inputExpIdPairs[ampName] = np.array(datasetPtc.inputExpIdPairs[ampName])[index]
251 datasetPtc.rawExpTimes[ampName] = np.array(datasetPtc.rawExpTimes[ampName])[index]
252 datasetPtc.rawMeans[ampName] = np.array(datasetPtc.rawMeans[ampName])[index]
253 datasetPtc.rawVars[ampName] = np.array(datasetPtc.rawVars[ampName])[index]
254 datasetPtc.expIdMask[ampName] = np.array(datasetPtc.expIdMask[ampName])[index]
255 datasetPtc.covariances[ampName] = np.array(datasetPtc.covariances[ampName])[index]
256 datasetPtc.covariancesSqrtWeights[ampName] = np.array(
257 datasetPtc.covariancesSqrtWeights[ampName])[index]
258 if self.config.ptcFitType == "FULLCOVARIANCE":
259 # Fit the measured covariances vs mean signal to
260 # the Astier+19 full model (Eq. 20). Before that
261 # do a preliminary fit to the variance (C_00) vs mean
262 # signal (mu) curve using the EXPAPPROXIMATION model
263 # (Eq. 16 in Astier+19) in order to
264 # get the flat pairs that are masked. The
265 # points at these fluxes will also be masked when
266 # calculating the other elements of the covariance
267 # matrix, C_ij, i!=j).
269 # Preliminary fit, usign a temp dataset to get the mask
270 tempDatasetPtc = copy.copy(datasetPtc)
271 tempDatasetPtc.ptcFitType = "EXPAPPROXIMATION"
272 tempDatasetPtc = self.fitMeasurementsToModel(tempDatasetPtc)
274 # "FULLCOVARIANCE", using the mask obtained from the
275 # previous fit.
276 for ampName in datasetPtc.ampNames:
277 datasetPtc.expIdMask[ampName] = tempDatasetPtc.expIdMask[ampName]
278 datasetPtc.fitType = "FULLCOVARIANCE"
279 datasetPtc = self.fitMeasurementsToModel(datasetPtc)
280 # The other options are: self.config.ptcFitType in
281 # ("EXPAPPROXIMATION", "POLYNOMIAL")
282 else:
283 # Fit the PTC to a polynomial or to Astier+19 exponential
284 # approximation (Eq. 16). Fill up
285 # PhotonTransferCurveDataset object.
286 datasetPtc = self.fitMeasurementsToModel(datasetPtc)
288 if camera:
289 detector = camera[detId]
290 else:
291 detector = None
292 datasetPtc.updateMetadata(setDate=True, camera=camera, detector=detector)
294 return pipeBase.Struct(
295 outputPtcDataset=datasetPtc,
296 )
298 def fitMeasurementsToModel(self, dataset):
299 """Fit the measured covariances vs mean signal to a
300 polynomial or one of the models in Astier+19
301 (Eq. 16 or Eq.20).
303 Parameters
304 ----------
305 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset`
306 The dataset containing information such as the means,
307 (co)variances, and exposure times.
309 Returns
310 -------
311 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset`
312 This is the same dataset as the input parameter, however,
313 it has been modified to include information such as the
314 fit vectors and the fit parameters. See the class
315 `PhotonTransferCurveDatase`.
316 """
317 fitType = dataset.ptcFitType
318 if fitType in ["FULLCOVARIANCE", ]:
319 # This model uses the full covariance matrix in the fit.
320 # The PTC is technically defined as variance vs signal,
321 # with variance = Cov_00
322 dataset = self.fitDataFullCovariance(dataset)
323 elif fitType in ["POLYNOMIAL", "EXPAPPROXIMATION"]:
324 # The PTC is technically defined as variance vs signal
325 dataset = self.fitPtc(dataset)
326 else:
327 raise RuntimeError(
328 f"Fitting option {fitType} not one of "
329 "'POLYNOMIAL', 'EXPAPPROXIMATION', or 'FULLCOVARIANCE'"
330 )
332 return dataset
334 def fitDataFullCovariance(self, dataset):
335 """Fit measured flat covariances to the full model in
336 Astier+19 (Eq. 20).
338 Parameters
339 ----------
340 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset`
341 The dataset containing information such as the means,
342 (co)variances, and exposure times.
344 Returns
345 -------
346 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset`
347 This is the same dataset as the input parameter, however,
348 it has been modified to include information such as the
349 fit vectors and the fit parameters. See the class
350 `PhotonTransferCurveDatase`.
352 Notes
353 -----
354 The parameters of the full model for C_ij(mu) ("C_ij" and "mu"
355 in ADU^2 and ADU, respectively) in Astier+19 (Eq. 20) are:
356 "a" coefficients (r by r matrix), units: 1/e
357 "b" coefficients (r by r matrix), units: 1/e
358 noise matrix (r by r matrix), units: e^2
359 gain, units: e/ADU
360 "b" appears in Eq. 20 only through the "ab" combination, which
361 is defined in this code as "c=ab".
363 Total number of parameters: #entries(a) + #entries(c) + #entries(noise)
364 + 1. This is equivalent to r^2 + r^2 + r^2 + 1, where "r" is the
365 maximum lag considered for the covariances calculation, and the
366 extra "1" is the gain. If "b" is 0, then "c" is 0, and len(pInit) will
367 have r^2 fewer entries.
368 """
369 matrixSide = self.config.maximumRangeCovariancesAstier
370 lenParams = matrixSide*matrixSide
372 for ampName in dataset.ampNames:
373 lenInputTimes = len(dataset.rawExpTimes[ampName])
374 # Not used when ptcFitType is 'FULLCOVARIANCE'
375 dataset.ptcFitPars[ampName] = [np.nan]
376 dataset.ptcFitParsError[ampName] = [np.nan]
377 dataset.ptcFitChiSq[ampName] = np.nan
379 if ampName in dataset.badAmps:
380 # Bad amp
381 # Entries need to have proper dimensions so read/write
382 # with astropy.Table works.
383 nanMatrix = np.full((matrixSide, matrixSide), np.nan)
384 listNanMatrix = np.full((lenInputTimes, matrixSide, matrixSide), np.nan)
385 dataset.covariancesModel[ampName] = listNanMatrix
386 dataset.covariancesSqrtWeights[ampName] = listNanMatrix
387 dataset.aMatrix[ampName] = nanMatrix
388 dataset.bMatrix[ampName] = nanMatrix
389 dataset.covariancesModelNoB[ampName] = listNanMatrix
390 dataset.aMatrixNoB[ampName] = nanMatrix
392 dataset.expIdMask[ampName] = np.repeat(False, lenInputTimes)
393 dataset.gain[ampName] = np.nan
394 dataset.gainErr[ampName] = np.nan
395 dataset.noise[ampName] = np.nan
396 dataset.noiseErr[ampName] = np.nan
397 dataset.finalVars[ampName] = np.repeat(np.nan, lenInputTimes)
398 dataset.finalModelVars[ampName] = np.repeat(np.nan, lenInputTimes)
399 dataset.finalMeans[ampName] = np.repeat(np.nan, lenInputTimes)
400 continue
402 muAtAmp = dataset.rawMeans[ampName]
403 maskAtAmp = dataset.expIdMask[ampName]
404 if len(maskAtAmp) == 0:
405 maskAtAmp = np.repeat(True, len(muAtAmp))
407 muAtAmp = muAtAmp[maskAtAmp]
408 covAtAmp = np.nan_to_num(dataset.covariances[ampName])[maskAtAmp]
409 covSqrtWeightsAtAmp = np.nan_to_num(dataset.covariancesSqrtWeights[ampName])[maskAtAmp]
411 # Initial fit, to approximate parameters, with c=0
412 a0, c0, noise0, gain0 = self.initialFitFullCovariance(muAtAmp, covAtAmp, covSqrtWeightsAtAmp)
414 # Fit full model (Eq. 20 of Astier+19) and same model with
415 # b=0 (c=0 in this code)
416 pInit = np.concatenate((a0.flatten(), c0.flatten(), noise0.flatten(), np.array(gain0)), axis=None)
417 functionsDict = {'fullModel': self.funcFullCovarianceModel,
418 'fullModelNoB': self.funcFullCovarianceModelNoB}
419 fitResults = {'fullModel': {'a': [], 'c': [], 'noise': [], 'gain': [], 'paramsErr': []},
420 'fullModelNoB': {'a': [], 'c': [], 'noise': [], 'gain': [], 'paramsErr': []}}
421 for key in functionsDict:
422 params, paramsErr, _ = fitLeastSq(pInit, muAtAmp,
423 covAtAmp.flatten(), functionsDict[key],
424 weightsY=covSqrtWeightsAtAmp.flatten())
425 a = params[:lenParams].reshape((matrixSide, matrixSide))
426 c = params[lenParams:2*lenParams].reshape((matrixSide, matrixSide))
427 noise = params[2*lenParams:3*lenParams].reshape((matrixSide, matrixSide))
428 gain = params[-1]
430 fitResults[key]['a'] = a
431 fitResults[key]['c'] = c
432 fitResults[key]['noise'] = noise
433 fitResults[key]['gain'] = gain
434 fitResults[key]['paramsErr'] = paramsErr
436 # Put the information in the PTC dataset
438 # Not used when ptcFitType is 'FULLCOVARIANCE'
439 dataset.ptcFitPars[ampName] = [np.nan]
440 dataset.ptcFitParsError[ampName] = [np.nan]
441 dataset.ptcFitChiSq[ampName] = np.nan
443 # Save full covariances, covariances models, and their weights.
444 # dataset.expIdMask is already full, but needs to be
445 # converted to bool.
446 dataset.expIdMask[ampName] = np.array(dataset.expIdMask[ampName], dtype=bool)
447 dataset.covariances[ampName] = covAtAmp
448 dataset.covariancesModel[ampName] = self.evalCovModel(muAtAmp,
449 fitResults['fullModel']['a'],
450 fitResults['fullModel']['c'],
451 fitResults['fullModel']['noise'],
452 fitResults['fullModel']['gain'])
453 dataset.covariancesSqrtWeights[ampName] = covSqrtWeightsAtAmp
454 dataset.aMatrix[ampName] = fitResults['fullModel']['a']
455 dataset.bMatrix[ampName] = fitResults['fullModel']['c']/fitResults['fullModel']['a']
456 dataset.covariancesModelNoB[ampName] = self.evalCovModel(muAtAmp,
457 fitResults['fullModelNoB']['a'],
458 fitResults['fullModelNoB']['c'],
459 fitResults['fullModelNoB']['noise'],
460 fitResults['fullModelNoB']['gain'],
461 setBtoZero=True)
462 dataset.aMatrixNoB[ampName] = fitResults['fullModelNoB']['a']
463 dataset.gain[ampName] = fitResults['fullModel']['gain']
464 dataset.gainErr[ampName] = fitResults['fullModel']['paramsErr'][-1]
465 readoutNoise = fitResults['fullModel']['noise'][0][0]
466 readoutNoiseSqrt = np.sqrt(np.fabs(readoutNoise))
467 dataset.noise[ampName] = readoutNoise
468 readoutNoiseSigma = fitResults['fullModel']['paramsErr'][2*lenParams]
469 dataset.noiseErr[ampName] = 0.5*(readoutNoiseSigma/np.fabs(readoutNoise))*readoutNoiseSqrt
470 dataset.finalVars[ampName] = covAtAmp[:, 0, 0]
471 dataset.finalModelVars[ampName] = dataset.covariancesModel[ampName][:, 0, 0]
472 dataset.finalMeans[ampName] = muAtAmp
474 return dataset
476 def initialFitFullCovariance(self, mu, cov, sqrtW):
477 """ Performs a crude parabolic fit of the data in order to start
478 the full fit close to the solution, setting b=0 (c=0) in Eq. 20
479 of Astier+19.
481 Parameters
482 ----------
483 mu : `numpy.array`, (N,)
484 Signal `mu` (ADU)
485 cov : `numpy.array`, (N, M, M)
486 Covariance arrays of size `(M, M)` (with
487 `M = config.maximumRangeCovariancesAstier`),
488 indexed by mean signal `mu`.
489 sqrtW : `numpy.array`, (N,)
490 Covariance weights, defined as 1./sqrt(Variances)
492 Returns
493 -------
494 a : `numpy.array`, (M, M)
495 "a" parameter per flux in Eq. 20 of Astier+19.
496 c : `numpy.array`, (M, M)
497 "c"="ab" parameter per flux in Eq. 20 of Astier+19.
498 noise : `numpy.array`, (M, M)
499 "noise" parameter per flux in Eq. 20 of Astier+19.
500 gain : `float`
501 Amplifier gain (e/ADU)
502 """
503 matrixSide = self.config.maximumRangeCovariancesAstier
505 # Initialize fit parameters
506 a = np.zeros((matrixSide, matrixSide))
507 c = np.zeros((matrixSide, matrixSide))
508 noise = np.zeros((matrixSide, matrixSide))
509 gain = 1.
511 # iterate the fit to account for higher orders
512 # the chi2 does not necessarily go down, so one could
513 # stop when it increases
514 oldChi2 = 1e30
515 for _ in range(5):
516 model = np.nan_to_num(self.evalCovModel(mu, a, c, noise, gain, setBtoZero=True))
517 # loop on lags
518 for i in range(matrixSide):
519 for j in range(matrixSide):
520 # fit a parabola for a given lag
521 parsFit = np.polyfit(mu, cov[:, i, j] - model[:, i, j],
522 2, w=sqrtW[:, i, j])
523 # model equation (Eq. 20) in Astier+19, with c=a*b=0:
524 a[i, j] += parsFit[0]
525 noise[i, j] += parsFit[2]
526 if(i + j == 0):
527 gain = 1./(1/gain+parsFit[1])
528 weightedRes = (model - cov)*sqrtW
529 chi2 = (weightedRes.flatten()**2).sum()
530 if chi2 > oldChi2:
531 break
532 oldChi2 = chi2
534 return a, c, noise, gain
536 def funcFullCovarianceModel(self, params, x):
537 """Model to fit covariances from flat fields; Equation 20 of
538 Astier+19.
540 Parameters
541 ----------
542 params : `list`
543 Parameters of the model: aMatrix, CMatrix, noiseMatrix,
544 gain (e/ADU).
545 x : `numpy.array`, (N,)
546 Signal `mu` (ADU)
548 Returns
549 -------
550 y : `numpy.array`, (N,)
551 Covariance matrix.
552 """
553 matrixSide = self.config.maximumRangeCovariancesAstier
554 lenParams = matrixSide*matrixSide
555 aMatrix = params[:lenParams].reshape((matrixSide, matrixSide))
556 cMatrix = params[lenParams:2*lenParams].reshape((matrixSide, matrixSide))
557 noiseMatrix = params[2*lenParams:3*lenParams].reshape((matrixSide, matrixSide))
558 gain = params[-1]
560 return self.evalCovModel(x, aMatrix, cMatrix, noiseMatrix, gain).flatten()
562 def funcFullCovarianceModelNoB(self, params, x):
563 """Model to fit covariances from flat fields; Equation 20 of
564 Astier+19, with b=0 (equivalent to c=a*b=0 in this code).
566 Parameters
567 ----------
568 params : `list`
569 Parameters of the model: aMatrix, CMatrix, noiseMatrix,
570 gain (e/ADU).
571 x : `numpy.array`, (N,)
572 Signal mu (ADU)
574 Returns
575 -------
576 y : `numpy.array`, (N,)
577 Covariance matrix.
578 """
579 matrixSide = self.config.maximumRangeCovariancesAstier
580 lenParams = matrixSide*matrixSide
581 aMatrix = params[:lenParams].reshape((matrixSide, matrixSide))
582 cMatrix = params[lenParams:2*lenParams].reshape((matrixSide, matrixSide))
583 noiseMatrix = params[2*lenParams:3*lenParams].reshape((matrixSide, matrixSide))
584 gain = params[-1]
586 return self.evalCovModel(x, aMatrix, cMatrix, noiseMatrix, gain, setBtoZero=True).flatten()
588 def evalCovModel(self, mu, aMatrix, cMatrix, noiseMatrix, gain, setBtoZero=False):
589 """Computes full covariances model (Eq. 20 of Astier+19).
591 Parameters
592 ----------
593 mu : `numpy.array`, (N,)
594 List of mean signals.
595 aMatrix : `numpy.array`, (M, M)
596 "a" parameter per flux in Eq. 20 of Astier+19.
597 cMatrix : `numpy.array`, (M, M)
598 "c"="ab" parameter per flux in Eq. 20 of Astier+19.
599 noiseMatrix : `numpy.array`, (M, M)
600 "noise" parameter per flux in Eq. 20 of Astier+19.
601 gain : `float`
602 Amplifier gain (e/ADU)
603 setBtoZero=False : `bool`, optional
604 Set "b" parameter in full model (see Astier+19) to zero.
606 Returns
607 -------
608 covModel : `numpy.array`, (N, M, M)
609 Covariances model.
611 Notes
612 -----
613 By default, computes the covModel for the mu's stored(self.mu).
614 Returns cov[Nmu, M, M]. The variance for the PTC is
615 cov[:, 0, 0]. mu and cov are in ADUs and ADUs squared. To use
616 electrons for both, the gain should be set to 1. This routine
617 implements the model in Astier+19 (1905.08677).
618 The parameters of the full model for C_ij(mu) ("C_ij" and "mu"
619 in ADU^2 and ADU, respectively) in Astier+19 (Eq. 20) are:
620 "a" coefficients (M by M matrix), units: 1/e
621 "b" coefficients (M by M matrix), units: 1/e
622 noise matrix (M by M matrix), units: e^2
623 gain, units: e/ADU
624 "b" appears in Eq. 20 only through the "ab" combination, which
625 is defined in this code as "c=ab".
626 """
627 matrixSide = self.config.maximumRangeCovariancesAstier
628 sa = (matrixSide, matrixSide)
629 # pad a with zeros and symmetrize
630 aEnlarged = np.zeros((int(sa[0]*1.5)+1, int(sa[1]*1.5)+1))
631 aEnlarged[0:sa[0], 0:sa[1]] = aMatrix
632 aSym = symmetrize(aEnlarged)
633 # pad c with zeros and symmetrize
634 cEnlarged = np.zeros((int(sa[0]*1.5)+1, int(sa[1]*1.5)+1))
635 cEnlarged[0:sa[0], 0:sa[1]] = cMatrix
636 cSym = symmetrize(cEnlarged)
637 a2 = fftconvolve(aSym, aSym, mode='same')
638 a3 = fftconvolve(a2, aSym, mode='same')
639 ac = fftconvolve(aSym, cSym, mode='same')
640 (xc, yc) = np.unravel_index(np.abs(aSym).argmax(), a2.shape)
642 a1 = aMatrix[np.newaxis, :, :]
643 a2 = a2[np.newaxis, xc:xc + matrixSide, yc:yc + matrixSide]
644 a3 = a3[np.newaxis, xc:xc + matrixSide, yc:yc + matrixSide]
645 ac = ac[np.newaxis, xc:xc + matrixSide, yc:yc + matrixSide]
646 c1 = cMatrix[np.newaxis, ::]
648 # assumes that mu is 1d
649 bigMu = mu[:, np.newaxis, np.newaxis]*gain
650 # c(=a*b in Astier+19) also has a contribution to the last
651 # term, that is absent for now.
652 if setBtoZero:
653 c1 = np.zeros_like(c1)
654 ac = np.zeros_like(ac)
655 covModel = (bigMu/(gain*gain)*(a1*bigMu+2./3.*(bigMu*bigMu)*(a2 + c1)
656 + (1./3.*a3 + 5./6.*ac)*(bigMu*bigMu*bigMu)) + noiseMatrix[np.newaxis, :, :]/gain**2)
657 # add the Poisson term, and the read out noise (variance)
658 covModel[:, 0, 0] += mu/gain
660 return covModel
662 # EXPAPPROXIMATION and POLYNOMIAL fit methods
663 @staticmethod
664 def _initialParsForPolynomial(order):
665 assert(order >= 2)
666 pars = np.zeros(order, dtype=float)
667 pars[0] = 10
668 pars[1] = 1
669 pars[2:] = 0.0001
670 return pars
672 @staticmethod
673 def _boundsForPolynomial(initialPars, lowers=[], uppers=[]):
674 if not len(lowers):
675 lowers = [np.NINF for p in initialPars]
676 if not len(uppers):
677 uppers = [np.inf for p in initialPars]
678 lowers[1] = 0 # no negative gains
679 return (lowers, uppers)
681 @staticmethod
682 def _boundsForAstier(initialPars, lowers=[], uppers=[]):
683 if not len(lowers):
684 lowers = [np.NINF for p in initialPars]
685 if not len(uppers):
686 uppers = [np.inf for p in initialPars]
687 return (lowers, uppers)
689 @staticmethod
690 def _getInitialGoodPoints(means, variances, minVarPivotSearch, consecutivePointsVarDecreases):
691 """Return a boolean array to mask bad points.
693 Parameters
694 ----------
695 means : `numpy.array`
696 Input array with mean signal values.
697 variances : `numpy.array`
698 Input array with variances at each mean value.
699 minVarPivotSearch : `float`
700 The variance (in ADU^2), above which, the point
701 of decreasing variance should be sought.
702 consecutivePointsVarDecreases : `int`
703 Required number of consecutive points/fluxes
704 in the PTC where the variance
705 decreases in order to find a first
706 estimate of the PTC turn-off.
708 Returns
709 ------
710 goodPoints : `numpy.array` [`bool`]
711 Boolean array to select good (`True`) and bad (`False`)
712 points.
714 Notes
715 -----
716 Eliminate points beyond which the variance decreases.
717 """
718 goodPoints = np.ones_like(means, dtype=bool)
719 # Variances are sorted and should monotonically increase
720 pivotList = np.where(np.array(np.diff(variances)) < 0)[0]
721 if len(pivotList) > 0:
722 # For small values, sometimes the variance decreases slightly
723 # Only look when var > self.config.minVarPivotSearch
724 pivotList = [p for p in pivotList if variances[p] > minVarPivotSearch]
725 # Require that the varince decreases during
726 # consecutivePointsVarDecreases
727 # consecutive points. This will give a first
728 # estimate of the PTC turn-off, which
729 # may be updated (reduced) further in the code.
730 if len(pivotList) > 1:
731 # enumerate(pivotList) creates tuples (index, value), for
732 # each value in pivotList. The lambda function subtracts
733 # each value from the index.
734 # groupby groups elements by equal key value.
735 for k, g in groupby(enumerate(pivotList), lambda x: x[0]-x[1]):
736 group = (map(itemgetter(1), g))
737 # Form groups of consecute values from pivotList
738 group = list(map(int, group))
739 # values in pivotList are indices where np.diff(variances)
740 # is negative, i.e., where the variance starts decreasing.
741 # Find the first group of consecutive numbers when
742 # variance decreases.
743 if len(group) >= consecutivePointsVarDecreases:
744 pivotIndex = np.min(group)
745 goodPoints[pivotIndex+1:] = False
746 break
748 return goodPoints
750 def _makeZeroSafe(self, array, substituteValue=1e-9):
751 """"""
752 array = np.array(array)
753 nBad = Counter(np.ravel(array))[0]
754 if nBad == 0:
755 return array
757 index, = np.where(array == 0)
758 if len(index):
759 msg = f"Found {nBad} zeros in array at elements {index}"
760 self.log.warning(msg)
762 array[index] = substituteValue
764 return array
766 def fitPtc(self, dataset):
767 """Fit the photon transfer curve to a polynomial or to the
768 Astier+19 approximation (Eq. 16).
770 Fit the photon transfer curve with either a polynomial of
771 the order specified in the task config, or using the
772 exponential approximation in Astier+19 (Eq. 16).
774 Sigma clipping is performed iteratively for the fit, as
775 well as an initial clipping of data points that are more
776 than `config.initialNonLinearityExclusionThreshold` away
777 from lying on a straight line. This other step is necessary
778 because the photon transfer curve turns over catastrophically
779 at very high flux (because saturation
780 drops the variance to ~0) and these far outliers cause the
781 initial fit to fail, meaning the sigma cannot be calculated
782 to perform the sigma-clipping.
784 Parameters
785 ----------
786 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset`
787 The dataset containing the means, variances and
788 exposure times.
790 Returns
791 -------
792 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset`
793 This is the same dataset as the input parameter, however,
794 it has been modified to include information such as the
795 fit vectors and the fit parameters. See the class
796 `PhotonTransferCurveDatase`.
798 Raises
799 ------
800 RuntimeError:
801 Raises if dataset.ptcFitType is None or empty.
802 """
803 if dataset.ptcFitType:
804 ptcFitType = dataset.ptcFitType
805 else:
806 raise RuntimeError("ptcFitType is None of empty in PTC dataset.")
807 matrixSide = self.config.maximumRangeCovariancesAstier
808 nanMatrix = np.empty((matrixSide, matrixSide))
809 nanMatrix[:] = np.nan
811 for amp in dataset.ampNames:
812 lenInputTimes = len(dataset.rawExpTimes[amp])
813 listNanMatrix = np.empty((lenInputTimes, matrixSide, matrixSide))
814 listNanMatrix[:] = np.nan
816 dataset.covariancesModel[amp] = listNanMatrix
817 dataset.aMatrix[amp] = nanMatrix
818 dataset.bMatrix[amp] = nanMatrix
819 dataset.covariancesModelNoB[amp] = listNanMatrix
820 dataset.aMatrixNoB[amp] = nanMatrix
822 def errFunc(p, x, y):
823 return ptcFunc(p, x) - y
825 sigmaCutPtcOutliers = self.config.sigmaCutPtcOutliers
826 maxIterationsPtcOutliers = self.config.maxIterationsPtcOutliers
828 for i, ampName in enumerate(dataset.ampNames):
829 timeVecOriginal = np.ravel(np.array(dataset.rawExpTimes[ampName]))
830 meanVecOriginal = np.ravel(np.array(dataset.rawMeans[ampName]))
831 varVecOriginal = np.ravel(np.array(dataset.rawVars[ampName]))
832 varVecOriginal = self._makeZeroSafe(varVecOriginal)
834 # Discard points when the variance starts to decrease after two
835 # consecutive signal levels
836 goodPoints = self._getInitialGoodPoints(meanVecOriginal, varVecOriginal,
837 self.config.minVarPivotSearch,
838 self.config.consecutivePointsVarDecreases)
840 # Check if all points are bad from the 'cpExtractPtcTask'
841 initialExpIdMask = np.ravel(np.array(dataset.expIdMask[ampName]))
843 if not (goodPoints.any() and initialExpIdMask.any()):
844 msg = (f"SERIOUS: All points in goodPoints: {goodPoints} or "
845 f"in initialExpIdMask: {initialExpIdMask} are bad."
846 f"Setting {ampName} to BAD.")
847 self.log.warning(msg)
848 # Fill entries with NaNs
849 self.fillBadAmp(dataset, ptcFitType, ampName)
850 continue
852 # Save the point where the variance starts decreasing as the
853 # PTC turnoff point
854 ptcTurnoff = meanVecOriginal[goodPoints][-1]
855 dataset.ptcTurnoff[ampName] = ptcTurnoff
857 mask = goodPoints
859 if ptcFitType == 'EXPAPPROXIMATION':
860 ptcFunc = funcAstier
861 parsIniPtc = [-1e-9, 1.0, 10.] # a00, gain, noise^2
862 # lowers and uppers obtained from BOT data studies by
863 # C. Lage (UC Davis, 11/2020).
864 bounds = self._boundsForAstier(parsIniPtc, lowers=[-1e-4, 0.5, -2000],
865 uppers=[1e-4, 2.5, 2000])
866 if ptcFitType == 'POLYNOMIAL':
867 ptcFunc = funcPolynomial
868 parsIniPtc = self._initialParsForPolynomial(self.config.polynomialFitDegree + 1)
869 bounds = self._boundsForPolynomial(parsIniPtc)
871 # Before bootstrap fit, do an iterative fit to get rid of outliers.
872 # This further process of outlier rejection be skipped
873 # if self.config.maxIterationsPtcOutliers = 0.
874 # We already did some initial outlier rejection above in
875 # self._getInitialGoodPoints.
876 count = 1
877 newMask = np.ones_like(meanVecOriginal, dtype=bool)
878 pars = parsIniPtc
879 while count <= maxIterationsPtcOutliers:
880 # Note that application of the mask actually shrinks the array
881 # to size rather than setting elements to zero (as we want) so
882 # always update mask itself and re-apply to the original data
883 meanTempVec = meanVecOriginal[mask]
884 varTempVec = varVecOriginal[mask]
885 res = least_squares(errFunc, parsIniPtc, bounds=bounds, args=(meanTempVec, varTempVec))
886 pars = res.x
888 # change this to the original from the temp because
889 # the masks are ANDed meaning once a point is masked
890 # it's always masked, and the masks must always be the
891 # same length for broadcasting
892 sigResids = (varVecOriginal - ptcFunc(pars, meanVecOriginal))/np.sqrt(varVecOriginal)
893 newMask = np.array([True if np.abs(r) < sigmaCutPtcOutliers else False for r in sigResids])
894 mask = mask & newMask
895 if not (mask.any() and newMask.any()):
896 msg = (f"SERIOUS: All points in either mask: {mask} or newMask: {newMask} are bad. "
897 f"Setting {ampName} to BAD.")
898 self.log.warning(msg)
899 # Fill entries with NaNs
900 self.fillBadAmp(dataset, ptcFitType, ampName)
901 break
902 nDroppedTotal = Counter(mask)[False]
903 self.log.debug("Iteration %d: discarded %d points in total for %s",
904 count, nDroppedTotal, ampName)
905 count += 1
906 # objects should never shrink
907 assert (len(mask) == len(timeVecOriginal) == len(meanVecOriginal) == len(varVecOriginal))
908 if not (mask.any() and newMask.any()):
909 continue
910 dataset.expIdMask[ampName] = np.array(dataset.expIdMask[ampName])
911 # store the final mask
912 if len(dataset.expIdMask[ampName]):
913 dataset.expIdMask[ampName] &= mask # bitwise_and if there is already a mask
914 else:
915 dataset.expIdMask[ampName] = mask
916 # In case there was a previous mask stored
917 mask = dataset.expIdMask[ampName]
918 parsIniPtc = pars
919 meanVecFinal = meanVecOriginal[mask]
920 varVecFinal = varVecOriginal[mask]
922 if Counter(mask)[False] > 0:
923 self.log.info("Number of points discarded in PTC of amplifier %s:"
924 " %d out of %d", ampName, Counter(mask)[False], len(meanVecOriginal))
926 if (len(meanVecFinal) < len(parsIniPtc)):
927 msg = (f"SERIOUS: Not enough data points ({len(meanVecFinal)}) compared to the number of "
928 f"parameters of the PTC model({len(parsIniPtc)}). Setting {ampName} to BAD.")
929 self.log.warning(msg)
930 # Fill entries with NaNs
931 self.fillBadAmp(dataset, ptcFitType, ampName)
932 continue
933 # Fit the PTC.
934 # The variance of the variance is Var(v)=2*v^2/Npix. This is
935 # already calculated in `makeCovArray` of CpPtcExtract.
936 # dataset.covariancesSqrtWeights[ampName][:,0,0]
937 # has 1/sqrt(Var(v)).
938 weightsY = dataset.covariancesSqrtWeights[ampName][:, 0, 0][mask]
939 if self.config.doFitBootstrap:
940 parsFit, parsFitErr, reducedChiSqPtc = fitBootstrap(parsIniPtc, meanVecFinal,
941 varVecFinal, ptcFunc,
942 weightsY=weightsY)
943 else:
944 parsFit, parsFitErr, reducedChiSqPtc = fitLeastSq(parsIniPtc, meanVecFinal,
945 varVecFinal, ptcFunc,
946 weightsY=weightsY)
947 dataset.ptcFitPars[ampName] = parsFit
948 dataset.ptcFitParsError[ampName] = parsFitErr
949 dataset.ptcFitChiSq[ampName] = reducedChiSqPtc
950 # Masked variances (measured and modeled) and means. Need
951 # to pad the array so astropy.Table does not crash (the
952 # mask may vary per amp).
953 padLength = len(dataset.rawExpTimes[ampName]) - len(varVecFinal)
954 dataset.finalVars[ampName] = np.pad(varVecFinal, (0, padLength), 'constant',
955 constant_values=np.nan)
956 dataset.finalModelVars[ampName] = np.pad(ptcFunc(parsFit, meanVecFinal), (0, padLength),
957 'constant', constant_values=np.nan)
958 dataset.finalMeans[ampName] = np.pad(meanVecFinal, (0, padLength), 'constant',
959 constant_values=np.nan)
960 if ptcFitType == 'EXPAPPROXIMATION':
961 ptcGain = parsFit[1]
962 ptcGainErr = parsFitErr[1]
963 ptcNoise = np.sqrt(np.fabs(parsFit[2]))
964 ptcNoiseErr = 0.5*(parsFitErr[2]/np.fabs(parsFit[2]))*np.sqrt(np.fabs(parsFit[2]))
965 if ptcFitType == 'POLYNOMIAL':
966 ptcGain = 1./parsFit[1]
967 ptcGainErr = np.fabs(1./parsFit[1])*(parsFitErr[1]/parsFit[1])
968 ptcNoise = np.sqrt(np.fabs(parsFit[0]))*ptcGain
969 ptcNoiseErr = (0.5*(parsFitErr[0]/np.fabs(parsFit[0]))*(np.sqrt(np.fabs(parsFit[0]))))*ptcGain
970 dataset.gain[ampName] = ptcGain
971 dataset.gainErr[ampName] = ptcGainErr
972 dataset.noise[ampName] = ptcNoise
973 dataset.noiseErr[ampName] = ptcNoiseErr
975 if not len(dataset.ptcFitType) == 0:
976 dataset.ptcFitType = ptcFitType
977 if len(dataset.badAmps) == 0:
978 dataset.badAmps = np.repeat(np.nan, len(list(dataset.rawExpTimes.values())[0]))
980 return dataset
982 def fillBadAmp(self, dataset, ptcFitType, ampName):
983 """Fill the dataset with NaNs if there are not enough
984 good points.
986 Parameters
987 ----------
988 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset`
989 The dataset containing the means, variances and
990 exposure times.
991 ptcFitType : {'POLYNOMIAL', 'EXPAPPROXIMATION'}
992 Fit a 'POLYNOMIAL' (degree: 'polynomialFitDegree') or
993 'EXPAPPROXIMATION' (Eq. 16 of Astier+19) to the PTC.
994 ampName : `str`
995 Amplifier name.
996 """
997 dataset.badAmps.append(ampName)
998 dataset.expIdMask[ampName] = np.repeat(False, len(dataset.rawExpTimes[ampName]))
999 dataset.gain[ampName] = np.nan
1000 dataset.gainErr[ampName] = np.nan
1001 dataset.noise[ampName] = np.nan
1002 dataset.noiseErr[ampName] = np.nan
1003 dataset.ptcFitPars[ampName] = (np.repeat(np.nan, self.config.polynomialFitDegree + 1) if
1004 ptcFitType in ["POLYNOMIAL", ] else np.repeat(np.nan, 3))
1005 dataset.ptcFitParsError[ampName] = (np.repeat(np.nan, self.config.polynomialFitDegree + 1) if
1006 ptcFitType in ["POLYNOMIAL", ] else np.repeat(np.nan, 3))
1007 dataset.ptcFitChiSq[ampName] = np.nan
1008 dataset.ptcTurnoff[ampName] = np.nan
1009 dataset.finalVars[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName]))
1010 dataset.finalModelVars[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName]))
1011 dataset.finalMeans[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName]))
1013 return