Coverage for python/lsst/cp/pipe/ptc/cpSolvePtcTask.py: 10%
400 statements
« prev ^ index » next coverage.py v6.5.0, created at 2023-02-15 03:29 -0800
« prev ^ index » next coverage.py v6.5.0, created at 2023-02-15 03:29 -0800
1# This file is part of cp_pipe.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21#
22import numpy as np
23from collections import Counter
25import lsst.pex.config as pexConfig
26import lsst.pipe.base as pipeBase
27from lsst.cp.pipe.utils import (fitLeastSq, fitBootstrap, funcPolynomial, funcAstier, symmetrize)
29from scipy.signal import fftconvolve
30from scipy.optimize import least_squares
31from itertools import groupby
32from operator import itemgetter
34import lsst.pipe.base.connectionTypes as cT
36from lsst.ip.isr import PhotonTransferCurveDataset
38from lsst.cp.pipe._lookupStaticCalibration import lookupStaticCalibration
40import copy
43__all__ = ['PhotonTransferCurveSolveConfig', 'PhotonTransferCurveSolveTask']
46class PhotonTransferCurveSolveConnections(pipeBase.PipelineTaskConnections,
47 dimensions=("instrument", "detector")):
48 inputCovariances = cT.Input(
49 name="ptcCovariances",
50 doc="Tuple with measured covariances from flats.",
51 storageClass="PhotonTransferCurveDataset",
52 dimensions=("instrument", "exposure", "detector"),
53 isCalibration=True,
54 multiple=True,
55 )
56 camera = cT.PrerequisiteInput(
57 name="camera",
58 doc="Camera the input data comes from.",
59 storageClass="Camera",
60 dimensions=("instrument",),
61 isCalibration=True,
62 lookupFunction=lookupStaticCalibration,
63 )
64 outputPtcDataset = cT.Output(
65 name="ptcDatsetProposal",
66 doc="Output proposed ptc dataset.",
67 storageClass="PhotonTransferCurveDataset",
68 dimensions=("instrument", "detector"),
69 multiple=False,
70 isCalibration=True,
71 )
74class PhotonTransferCurveSolveConfig(pipeBase.PipelineTaskConfig,
75 pipelineConnections=PhotonTransferCurveSolveConnections):
76 """Configuration for fitting measured covariances.
77 """
79 ptcFitType = pexConfig.ChoiceField(
80 dtype=str,
81 doc="Fit PTC to Eq. 16, Eq. 20 in Astier+19, or to a polynomial.",
82 default="POLYNOMIAL",
83 allowed={
84 "POLYNOMIAL": "n-degree polynomial (use 'polynomialFitDegree' to set 'n').",
85 "EXPAPPROXIMATION": "Approximation in Astier+19 (Eq. 16).",
86 "FULLCOVARIANCE": "Full covariances model in Astier+19 (Eq. 20)"
87 }
88 )
89 maximumRangeCovariancesAstier = pexConfig.Field(
90 dtype=int,
91 doc="Maximum range of covariances as in Astier+19",
92 default=8,
93 )
94 sigmaClipFullFitCovariancesAstier = pexConfig.Field(
95 dtype=float,
96 doc="sigma clip for full model fit for FULLCOVARIANCE ptcFitType ",
97 default=5.0,
98 )
99 maxIterFullFitCovariancesAstier = pexConfig.Field(
100 dtype=int,
101 doc="Maximum number of iterations in full model fit for FULLCOVARIANCE ptcFitType",
102 default=3,
103 )
104 polynomialFitDegree = pexConfig.Field(
105 dtype=int,
106 doc="Degree of polynomial to fit the PTC, when 'ptcFitType'=POLYNOMIAL.",
107 default=3,
108 )
109 sigmaCutPtcOutliers = pexConfig.Field(
110 dtype=float,
111 doc="Sigma cut for outlier rejection in PTC.",
112 default=5.0,
113 )
114 maxIterationsPtcOutliers = pexConfig.RangeField(
115 dtype=int,
116 doc="Maximum number of iterations for outlier rejection in PTC.",
117 default=2,
118 min=0
119 )
120 minVarPivotSearch = pexConfig.Field(
121 dtype=float,
122 doc="The code looks for a pivot signal point after which the variance starts decreasing at high-flux"
123 " to exclude then from the PTC model fit. However, sometimes at low fluxes, the variance"
124 " decreases slightly. Set this variable for the variance value, in ADU^2, after which the pivot "
125 " should be sought.",
126 default=10000,
127 )
128 consecutivePointsVarDecreases = pexConfig.RangeField(
129 dtype=int,
130 doc="Required number of consecutive points/fluxes in the PTC where the variance "
131 "decreases in order to find a first estimate of the PTC turn-off. ",
132 default=2,
133 min=2
134 )
135 doFitBootstrap = pexConfig.Field(
136 dtype=bool,
137 doc="Use bootstrap for the PTC fit parameters and errors?.",
138 default=False,
139 )
140 binSize = pexConfig.Field(
141 dtype=int,
142 doc="Bin the image by this factor in both dimensions.",
143 default=1,
144 )
147class PhotonTransferCurveSolveTask(pipeBase.PipelineTask):
148 """Task to fit the PTC from flat covariances.
150 The first task of the PTC measurement pipeline,
151 ``PhotonTransferCurveMeasureTask`` (and assumed to have been run
152 before this task), produced a list of
153 `~lsst.ip.isr.PhotonTransferCurveDataset` objects. Each dataset
154 contains the mean signal and covariances of the
155 difference image of the flat-field images taken at
156 the same exposure time. The list also contains dummy
157 datasets (with no measurements), whose purpose is to have
158 the input and output dimensions of ``PhotonTransferCurveMeasureTask``
159 match.
161 This task, ``PhotonTransferCurveSolveTask``, assembles the list
162 of individual PTC datasets produced
163 by ``PhotonTransferCurveMeasureTask`` into one single final PTC
164 dataset, discarding the dummy datset as appropiate.
165 The task fits the measured (co)variances to one of three models:
166 a polynomial model of a given order, or the models described
167 in equations 16 and 20 of Astier+19. These options are referred
168 to as ``POLYNOMIAL``, ``EXPAPPROXIMATION``, and ``FULLCOVARIANCE``
169 in the configuration options of the task, respectively).
170 Parameters of interest such as the gain and noise are derived
171 from the fits. The ``FULLCOVARIANCE`` model is fitted to the
172 full covariance data (as oppossed to the other two models, which
173 are fit to the variance vs mean measurements only).
175 Astier+19: "The Shape of the Photon Transfer Curve
176 of CCD sensors", arXiv:1905.08677
177 """
179 ConfigClass = PhotonTransferCurveSolveConfig
180 _DefaultName = 'cpPhotonTransferCurveSolve'
182 def runQuantum(self, butlerQC, inputRefs, outputRefs):
183 """Ensure that the input and output dimensions are passed along.
185 Parameters
186 ----------
187 butlerQC : `~lsst.daf.butler.butlerQuantumContext.ButlerQuantumContext`
188 Butler to operate on.
189 inputRefs : `~lsst.pipe.base.connections.InputQuantizedConnection`
190 Input data refs to load.
191 ouptutRefs : `~lsst.pipe.base.connections.OutputQuantizedConnection`
192 Output data refs to persist.
193 """
194 inputs = butlerQC.get(inputRefs)
195 detId = inputRefs.inputCovariances[0].dataId['detector']
196 outputs = self.run(inputCovariances=inputs['inputCovariances'], camera=inputs['camera'], detId=detId)
197 butlerQC.put(outputs, outputRefs)
199 def run(self, inputCovariances, camera=None, detId=0):
200 """Fit measured covariances to different models.
202 Parameters
203 ----------
204 inputCovariances : `list` [`lsst.ip.isr.PhotonTransferCurveDataset`]
205 List of lsst.ip.isr.PhotonTransferCurveDataset datasets.
206 camera : `lsst.afw.cameraGeom.Camera`, optional
207 Input camera.
208 detId : `int`
209 Detector ID to locate the detector in the camera and
210 populate the `lsst.ip.isr.PhotonTransferCurveDataset`
211 metadata.
212 Returns
213 -------
214 results : `lsst.pipe.base.Struct`
215 The resultins structure contains:
217 ``outputPtcDatset``
218 Final PTC dataset, containing information such as the
219 means, variances, and exposure times
220 (`lsst.ip.isr.PhotonTransferCurveDataset`).
221 """
222 # Assemble individual PTC datasets into a single PTC dataset.
223 ampNames = np.unique(inputCovariances[0].ampNames)
224 datasetPtc = PhotonTransferCurveDataset(ampNames=ampNames,
225 ptcFitType=self.config.ptcFitType,
226 covMatrixSide=self.config.maximumRangeCovariancesAstier)
227 for partialPtcDataset in inputCovariances:
228 # Ignore dummy datasets
229 if partialPtcDataset.ptcFitType == 'DUMMY':
230 continue
231 for ampName in ampNames:
232 datasetPtc.inputExpIdPairs[ampName].append(partialPtcDataset.inputExpIdPairs[ampName])
233 if type(partialPtcDataset.rawExpTimes[ampName]) is list:
234 datasetPtc.rawExpTimes[ampName].append(partialPtcDataset.rawExpTimes[ampName][0])
235 else:
236 datasetPtc.rawExpTimes[ampName].append(partialPtcDataset.rawExpTimes[ampName])
237 if type(partialPtcDataset.rawMeans[ampName]) is list:
238 datasetPtc.rawMeans[ampName].append(partialPtcDataset.rawMeans[ampName][0])
239 else:
240 datasetPtc.rawMeans[ampName].append(partialPtcDataset.rawMeans[ampName])
241 if type(partialPtcDataset.rawVars[ampName]) is list:
242 datasetPtc.rawVars[ampName].append(partialPtcDataset.rawVars[ampName][0])
243 else:
244 datasetPtc.rawVars[ampName].append(partialPtcDataset.rawVars[ampName])
245 if type(partialPtcDataset.expIdMask[ampName]) is list:
246 datasetPtc.expIdMask[ampName].append(partialPtcDataset.expIdMask[ampName][0])
247 else:
248 datasetPtc.expIdMask[ampName].append(partialPtcDataset.expIdMask[ampName])
249 datasetPtc.covariances[ampName].append(np.array(partialPtcDataset.covariances[ampName][0]))
250 datasetPtc.covariancesSqrtWeights[ampName].append(
251 np.array(partialPtcDataset.covariancesSqrtWeights[ampName][0]))
252 # Sort arrays that are filled so far in the final dataset by
253 # rawMeans index
254 for ampName in ampNames:
255 index = np.argsort(np.ravel(np.array(datasetPtc.rawMeans[ampName])))
256 datasetPtc.inputExpIdPairs[ampName] = np.array(datasetPtc.inputExpIdPairs[ampName])[index]
257 datasetPtc.rawExpTimes[ampName] = np.array(datasetPtc.rawExpTimes[ampName])[index]
258 datasetPtc.rawMeans[ampName] = np.array(datasetPtc.rawMeans[ampName])[index]
259 datasetPtc.rawVars[ampName] = np.array(datasetPtc.rawVars[ampName])[index]
260 datasetPtc.expIdMask[ampName] = np.array(datasetPtc.expIdMask[ampName])[index]
261 datasetPtc.covariances[ampName] = np.array(datasetPtc.covariances[ampName])[index]
262 datasetPtc.covariancesSqrtWeights[ampName] = np.array(
263 datasetPtc.covariancesSqrtWeights[ampName])[index]
264 if self.config.ptcFitType == "FULLCOVARIANCE":
265 # Fit the measured covariances vs mean signal to
266 # the Astier+19 full model (Eq. 20). Before that
267 # do a preliminary fit to the variance (C_00) vs mean
268 # signal (mu) curve using the EXPAPPROXIMATION model
269 # (Eq. 16 in Astier+19) in order to
270 # get the flat pairs that are masked. The
271 # points at these fluxes will also be masked when
272 # calculating the other elements of the covariance
273 # matrix, C_ij, i!=j).
275 # Preliminary fit, usign a temp dataset to get the mask
276 tempDatasetPtc = copy.copy(datasetPtc)
277 tempDatasetPtc.ptcFitType = "EXPAPPROXIMATION"
278 tempDatasetPtc = self.fitMeasurementsToModel(tempDatasetPtc)
280 # "FULLCOVARIANCE", using the mask obtained from the
281 # previous fit.
282 for ampName in datasetPtc.ampNames:
283 datasetPtc.expIdMask[ampName] = tempDatasetPtc.expIdMask[ampName]
284 datasetPtc.fitType = "FULLCOVARIANCE"
285 datasetPtc = self.fitMeasurementsToModel(datasetPtc)
286 # The other options are: self.config.ptcFitType in
287 # ("EXPAPPROXIMATION", "POLYNOMIAL")
288 else:
289 # Fit the PTC to a polynomial or to Astier+19 exponential
290 # approximation (Eq. 16). Fill up
291 # PhotonTransferCurveDataset object.
292 datasetPtc = self.fitMeasurementsToModel(datasetPtc)
294 if camera:
295 detector = camera[detId]
296 else:
297 detector = None
298 datasetPtc.updateMetadata(setDate=True, camera=camera, detector=detector)
300 return pipeBase.Struct(
301 outputPtcDataset=datasetPtc,
302 )
304 def fitMeasurementsToModel(self, dataset):
305 """Fit the measured covariances vs mean signal to a
306 polynomial or one of the models in Astier+19
307 (Eq. 16 or Eq.20).
309 Parameters
310 ----------
311 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset`
312 The dataset containing information such as the means,
313 (co)variances, and exposure times.
315 Returns
316 -------
317 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset`
318 This is the same dataset as the input parameter, however,
319 it has been modified to include information such as the
320 fit vectors and the fit parameters. See the class
321 `PhotonTransferCurveDatase`.
322 """
323 fitType = dataset.ptcFitType
324 if fitType in ["FULLCOVARIANCE", ]:
325 # This model uses the full covariance matrix in the fit.
326 # The PTC is technically defined as variance vs signal,
327 # with variance = Cov_00
328 dataset = self.fitDataFullCovariance(dataset)
329 elif fitType in ["POLYNOMIAL", "EXPAPPROXIMATION"]:
330 # The PTC is technically defined as variance vs signal
331 dataset = self.fitPtc(dataset)
332 else:
333 raise RuntimeError(
334 f"Fitting option {fitType} not one of "
335 "'POLYNOMIAL', 'EXPAPPROXIMATION', or 'FULLCOVARIANCE'"
336 )
338 return dataset
340 def fitDataFullCovariance(self, dataset):
341 """Fit measured flat covariances to the full model in
342 Astier+19 (Eq. 20).
344 Parameters
345 ----------
346 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset`
347 The dataset containing information such as the means,
348 (co)variances, and exposure times.
350 Returns
351 -------
352 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset`
353 This is the same dataset as the input parameter, however,
354 it has been modified to include information such as the
355 fit vectors and the fit parameters. See the class
356 `PhotonTransferCurveDatase`.
358 Notes
359 -----
360 The parameters of the full model for C_ij(mu) ("C_ij" and "mu"
361 in ADU^2 and ADU, respectively) in Astier+19 (Eq. 20) are:
363 - "a" coefficients (r by r matrix), units: 1/e
364 - "b" coefficients (r by r matrix), units: 1/e
365 - noise matrix (r by r matrix), units: e^2
366 - gain, units: e/ADU
368 "b" appears in Eq. 20 only through the "ab" combination, which
369 is defined in this code as "c=ab".
371 Total number of parameters: #entries(a) + #entries(c) + #entries(noise)
372 + 1. This is equivalent to r^2 + r^2 + r^2 + 1, where "r" is the
373 maximum lag considered for the covariances calculation, and the
374 extra "1" is the gain. If "b" is 0, then "c" is 0, and len(pInit) will
375 have r^2 fewer entries.
376 """
377 matrixSide = self.config.maximumRangeCovariancesAstier
378 lenParams = matrixSide*matrixSide
380 for ampName in dataset.ampNames:
381 lenInputTimes = len(dataset.rawExpTimes[ampName])
382 # Not used when ptcFitType is 'FULLCOVARIANCE'
383 dataset.ptcFitPars[ampName] = [np.nan]
384 dataset.ptcFitParsError[ampName] = [np.nan]
385 dataset.ptcFitChiSq[ampName] = np.nan
387 if ampName in dataset.badAmps:
388 # Bad amp
389 # Entries need to have proper dimensions so read/write
390 # with astropy.Table works.
391 nanMatrix = np.full((matrixSide, matrixSide), np.nan)
392 listNanMatrix = np.full((lenInputTimes, matrixSide, matrixSide), np.nan)
393 dataset.covariancesModel[ampName] = listNanMatrix
394 dataset.covariancesSqrtWeights[ampName] = listNanMatrix
395 dataset.aMatrix[ampName] = nanMatrix
396 dataset.bMatrix[ampName] = nanMatrix
397 dataset.covariancesModelNoB[ampName] = listNanMatrix
398 dataset.aMatrixNoB[ampName] = nanMatrix
400 dataset.expIdMask[ampName] = np.repeat(False, lenInputTimes)
401 dataset.gain[ampName] = np.nan
402 dataset.gainErr[ampName] = np.nan
403 dataset.noise[ampName] = np.nan
404 dataset.noiseErr[ampName] = np.nan
405 dataset.finalVars[ampName] = np.repeat(np.nan, lenInputTimes)
406 dataset.finalModelVars[ampName] = np.repeat(np.nan, lenInputTimes)
407 dataset.finalMeans[ampName] = np.repeat(np.nan, lenInputTimes)
408 continue
410 muAtAmp = dataset.rawMeans[ampName]
411 maskAtAmp = dataset.expIdMask[ampName]
412 if len(maskAtAmp) == 0:
413 maskAtAmp = np.repeat(True, len(muAtAmp))
415 muAtAmp = muAtAmp[maskAtAmp]
416 covAtAmp = np.nan_to_num(dataset.covariances[ampName])[maskAtAmp]
417 covSqrtWeightsAtAmp = np.nan_to_num(dataset.covariancesSqrtWeights[ampName])[maskAtAmp]
419 # Initial fit, to approximate parameters, with c=0
420 a0, c0, noise0, gain0 = self.initialFitFullCovariance(muAtAmp, covAtAmp, covSqrtWeightsAtAmp)
422 # Fit full model (Eq. 20 of Astier+19) and same model with
423 # b=0 (c=0 in this code)
424 pInit = np.concatenate((a0.flatten(), c0.flatten(), noise0.flatten(), np.array(gain0)), axis=None)
425 functionsDict = {'fullModel': self.funcFullCovarianceModel,
426 'fullModelNoB': self.funcFullCovarianceModelNoB}
427 fitResults = {'fullModel': {'a': [], 'c': [], 'noise': [], 'gain': [], 'paramsErr': []},
428 'fullModelNoB': {'a': [], 'c': [], 'noise': [], 'gain': [], 'paramsErr': []}}
429 for key in functionsDict:
430 params, paramsErr, _ = fitLeastSq(pInit, muAtAmp,
431 covAtAmp.flatten(), functionsDict[key],
432 weightsY=covSqrtWeightsAtAmp.flatten())
433 a = params[:lenParams].reshape((matrixSide, matrixSide))
434 c = params[lenParams:2*lenParams].reshape((matrixSide, matrixSide))
435 noise = params[2*lenParams:3*lenParams].reshape((matrixSide, matrixSide))
436 gain = params[-1]
438 fitResults[key]['a'] = a
439 fitResults[key]['c'] = c
440 fitResults[key]['noise'] = noise
441 fitResults[key]['gain'] = gain
442 fitResults[key]['paramsErr'] = paramsErr
444 # Put the information in the PTC dataset
446 # Not used when ptcFitType is 'FULLCOVARIANCE'
447 dataset.ptcFitPars[ampName] = [np.nan]
448 dataset.ptcFitParsError[ampName] = [np.nan]
449 dataset.ptcFitChiSq[ampName] = np.nan
451 # Save full covariances, covariances models, and their weights.
452 # dataset.expIdMask is already full, but needs to be
453 # converted to bool.
454 dataset.expIdMask[ampName] = np.array(dataset.expIdMask[ampName], dtype=bool)
455 dataset.covariances[ampName] = covAtAmp
456 dataset.covariancesModel[ampName] = self.evalCovModel(muAtAmp,
457 fitResults['fullModel']['a'],
458 fitResults['fullModel']['c'],
459 fitResults['fullModel']['noise'],
460 fitResults['fullModel']['gain'])
461 dataset.covariancesSqrtWeights[ampName] = covSqrtWeightsAtAmp
462 dataset.aMatrix[ampName] = fitResults['fullModel']['a']
463 dataset.bMatrix[ampName] = fitResults['fullModel']['c']/fitResults['fullModel']['a']
464 dataset.covariancesModelNoB[ampName] = self.evalCovModel(muAtAmp,
465 fitResults['fullModelNoB']['a'],
466 fitResults['fullModelNoB']['c'],
467 fitResults['fullModelNoB']['noise'],
468 fitResults['fullModelNoB']['gain'],
469 setBtoZero=True)
470 dataset.aMatrixNoB[ampName] = fitResults['fullModelNoB']['a']
471 dataset.gain[ampName] = fitResults['fullModel']['gain']
472 dataset.gainErr[ampName] = fitResults['fullModel']['paramsErr'][-1]
473 readoutNoise = fitResults['fullModel']['noise'][0][0]
474 readoutNoiseSqrt = np.sqrt(np.fabs(readoutNoise))
475 dataset.noise[ampName] = readoutNoise
476 readoutNoiseSigma = fitResults['fullModel']['paramsErr'][2*lenParams]
477 dataset.noiseErr[ampName] = 0.5*(readoutNoiseSigma/np.fabs(readoutNoise))*readoutNoiseSqrt
478 dataset.finalVars[ampName] = covAtAmp[:, 0, 0]
479 dataset.finalModelVars[ampName] = dataset.covariancesModel[ampName][:, 0, 0]
480 dataset.finalMeans[ampName] = muAtAmp
482 return dataset
484 def initialFitFullCovariance(self, mu, cov, sqrtW):
485 """ Performs a crude parabolic fit of the data in order to start
486 the full fit close to the solution, setting b=0 (c=0) in Eq. 20
487 of Astier+19.
489 Parameters
490 ----------
491 mu : `numpy.array`, (N,)
492 Signal `mu` (ADU)
493 cov : `numpy.array`, (N, M, M)
494 Covariance arrays of size `(M, M)` (with
495 `M = config.maximumRangeCovariancesAstier`),
496 indexed by mean signal `mu`.
497 sqrtW : `numpy.array`, (N,)
498 Covariance weights, defined as 1./sqrt(Variances)
500 Returns
501 -------
502 a : `numpy.array`, (M, M)
503 "a" parameter per flux in Eq. 20 of Astier+19.
504 c : `numpy.array`, (M, M)
505 "c"="ab" parameter per flux in Eq. 20 of Astier+19.
506 noise : `numpy.array`, (M, M)
507 "noise" parameter per flux in Eq. 20 of Astier+19.
508 gain : `float`
509 Amplifier gain (e/ADU)
510 """
511 matrixSide = self.config.maximumRangeCovariancesAstier
513 # Initialize fit parameters
514 a = np.zeros((matrixSide, matrixSide))
515 c = np.zeros((matrixSide, matrixSide))
516 noise = np.zeros((matrixSide, matrixSide))
517 gain = 1.
519 # iterate the fit to account for higher orders
520 # the chi2 does not necessarily go down, so one could
521 # stop when it increases
522 oldChi2 = 1e30
523 for _ in range(5):
524 model = np.nan_to_num(self.evalCovModel(mu, a, c, noise, gain, setBtoZero=True))
525 # loop on lags
526 for i in range(matrixSide):
527 for j in range(matrixSide):
528 # fit a parabola for a given lag
529 parsFit = np.polyfit(mu, cov[:, i, j] - model[:, i, j],
530 2, w=sqrtW[:, i, j])
531 # model equation (Eq. 20) in Astier+19, with c=a*b=0:
532 a[i, j] += parsFit[0]
533 noise[i, j] += parsFit[2]
534 if(i + j == 0):
535 gain = 1./(1/gain+parsFit[1])
536 weightedRes = (model - cov)*sqrtW
537 chi2 = (weightedRes.flatten()**2).sum()
538 if chi2 > oldChi2:
539 break
540 oldChi2 = chi2
542 return a, c, noise, gain
544 def funcFullCovarianceModel(self, params, x):
545 """Model to fit covariances from flat fields; Equation 20 of
546 Astier+19.
548 Parameters
549 ----------
550 params : `list`
551 Parameters of the model: aMatrix, CMatrix, noiseMatrix,
552 gain (e/ADU).
553 x : `numpy.array`, (N,)
554 Signal `mu` (ADU)
556 Returns
557 -------
558 y : `numpy.array`, (N,)
559 Covariance matrix.
560 """
561 matrixSide = self.config.maximumRangeCovariancesAstier
562 lenParams = matrixSide*matrixSide
563 aMatrix = params[:lenParams].reshape((matrixSide, matrixSide))
564 cMatrix = params[lenParams:2*lenParams].reshape((matrixSide, matrixSide))
565 noiseMatrix = params[2*lenParams:3*lenParams].reshape((matrixSide, matrixSide))
566 gain = params[-1]
568 return self.evalCovModel(x, aMatrix, cMatrix, noiseMatrix, gain).flatten()
570 def funcFullCovarianceModelNoB(self, params, x):
571 """Model to fit covariances from flat fields; Equation 20 of
572 Astier+19, with b=0 (equivalent to c=a*b=0 in this code).
574 Parameters
575 ----------
576 params : `list`
577 Parameters of the model: aMatrix, CMatrix, noiseMatrix,
578 gain (e/ADU).
579 x : `numpy.array`, (N,)
580 Signal mu (ADU)
582 Returns
583 -------
584 y : `numpy.array`, (N,)
585 Covariance matrix.
586 """
587 matrixSide = self.config.maximumRangeCovariancesAstier
588 lenParams = matrixSide*matrixSide
589 aMatrix = params[:lenParams].reshape((matrixSide, matrixSide))
590 cMatrix = params[lenParams:2*lenParams].reshape((matrixSide, matrixSide))
591 noiseMatrix = params[2*lenParams:3*lenParams].reshape((matrixSide, matrixSide))
592 gain = params[-1]
594 return self.evalCovModel(x, aMatrix, cMatrix, noiseMatrix, gain, setBtoZero=True).flatten()
596 def evalCovModel(self, mu, aMatrix, cMatrix, noiseMatrix, gain, setBtoZero=False):
597 """Computes full covariances model (Eq. 20 of Astier+19).
599 Parameters
600 ----------
601 mu : `numpy.array`, (N,)
602 List of mean signals.
603 aMatrix : `numpy.array`, (M, M)
604 "a" parameter per flux in Eq. 20 of Astier+19.
605 cMatrix : `numpy.array`, (M, M)
606 "c"="ab" parameter per flux in Eq. 20 of Astier+19.
607 noiseMatrix : `numpy.array`, (M, M)
608 "noise" parameter per flux in Eq. 20 of Astier+19.
609 gain : `float`
610 Amplifier gain (e/ADU)
611 setBtoZero=False : `bool`, optional
612 Set "b" parameter in full model (see Astier+19) to zero.
614 Returns
615 -------
616 covModel : `numpy.array`, (N, M, M)
617 Covariances model.
619 Notes
620 -----
621 By default, computes the covModel for the mu's stored(self.mu).
622 Returns cov[Nmu, M, M]. The variance for the PTC is
623 cov[:, 0, 0]. mu and cov are in ADUs and ADUs squared. To use
624 electrons for both, the gain should be set to 1. This routine
625 implements the model in Astier+19 (1905.08677).
626 The parameters of the full model for C_ij(mu) ("C_ij" and "mu"
627 in ADU^2 and ADU, respectively) in Astier+19 (Eq. 20) are:
629 - "a" coefficients (M by M matrix), units: 1/e
630 - "b" coefficients (M by M matrix), units: 1/e
631 - noise matrix (M by M matrix), units: e^2
632 - gain, units: e/ADU
634 "b" appears in Eq. 20 only through the "ab" combination, which
635 is defined in this code as "c=ab".
636 """
637 matrixSide = self.config.maximumRangeCovariancesAstier
638 sa = (matrixSide, matrixSide)
639 # pad a with zeros and symmetrize
640 aEnlarged = np.zeros((int(sa[0]*1.5)+1, int(sa[1]*1.5)+1))
641 aEnlarged[0:sa[0], 0:sa[1]] = aMatrix
642 aSym = symmetrize(aEnlarged)
643 # pad c with zeros and symmetrize
644 cEnlarged = np.zeros((int(sa[0]*1.5)+1, int(sa[1]*1.5)+1))
645 cEnlarged[0:sa[0], 0:sa[1]] = cMatrix
646 cSym = symmetrize(cEnlarged)
647 a2 = fftconvolve(aSym, aSym, mode='same')
648 a3 = fftconvolve(a2, aSym, mode='same')
649 ac = fftconvolve(aSym, cSym, mode='same')
650 (xc, yc) = np.unravel_index(np.abs(aSym).argmax(), a2.shape)
652 a1 = aMatrix[np.newaxis, :, :]
653 a2 = a2[np.newaxis, xc:xc + matrixSide, yc:yc + matrixSide]
654 a3 = a3[np.newaxis, xc:xc + matrixSide, yc:yc + matrixSide]
655 ac = ac[np.newaxis, xc:xc + matrixSide, yc:yc + matrixSide]
656 c1 = cMatrix[np.newaxis, ::]
658 # assumes that mu is 1d
659 bigMu = mu[:, np.newaxis, np.newaxis]*gain
660 # c(=a*b in Astier+19) also has a contribution to the last
661 # term, that is absent for now.
662 if setBtoZero:
663 c1 = np.zeros_like(c1)
664 ac = np.zeros_like(ac)
665 covModel = (bigMu/(gain*gain)*(a1*bigMu+2./3.*(bigMu*bigMu)*(a2 + c1)
666 + (1./3.*a3 + 5./6.*ac)*(bigMu*bigMu*bigMu)) + noiseMatrix[np.newaxis, :, :]/gain**2)
667 # add the Poisson term, and the read out noise (variance)
668 covModel[:, 0, 0] += mu/gain
670 return covModel
672 # EXPAPPROXIMATION and POLYNOMIAL fit methods
673 @staticmethod
674 def _initialParsForPolynomial(order):
675 assert(order >= 2)
676 pars = np.zeros(order, dtype=float)
677 pars[0] = 10
678 pars[1] = 1
679 pars[2:] = 0.0001
680 return pars
682 @staticmethod
683 def _boundsForPolynomial(initialPars, lowers=[], uppers=[]):
684 if not len(lowers):
685 lowers = [np.NINF for p in initialPars]
686 if not len(uppers):
687 uppers = [np.inf for p in initialPars]
688 lowers[1] = 0 # no negative gains
689 return (lowers, uppers)
691 @staticmethod
692 def _boundsForAstier(initialPars, lowers=[], uppers=[]):
693 if not len(lowers):
694 lowers = [np.NINF for p in initialPars]
695 if not len(uppers):
696 uppers = [np.inf for p in initialPars]
697 return (lowers, uppers)
699 @staticmethod
700 def _getInitialGoodPoints(means, variances, minVarPivotSearch, consecutivePointsVarDecreases):
701 """Return a boolean array to mask bad points.
703 Parameters
704 ----------
705 means : `numpy.array`
706 Input array with mean signal values.
707 variances : `numpy.array`
708 Input array with variances at each mean value.
709 minVarPivotSearch : `float`
710 The variance (in ADU^2), above which, the point
711 of decreasing variance should be sought.
712 consecutivePointsVarDecreases : `int`
713 Required number of consecutive points/fluxes
714 in the PTC where the variance
715 decreases in order to find a first
716 estimate of the PTC turn-off.
718 Returns
719 ------
720 goodPoints : `numpy.array` [`bool`]
721 Boolean array to select good (`True`) and bad (`False`)
722 points.
724 Notes
725 -----
726 Eliminate points beyond which the variance decreases.
727 """
728 goodPoints = np.ones_like(means, dtype=bool)
729 # Variances are sorted and should monotonically increase
730 pivotList = np.where(np.array(np.diff(variances)) < 0)[0]
731 if len(pivotList) > 0:
732 # For small values, sometimes the variance decreases slightly
733 # Only look when var > self.config.minVarPivotSearch
734 pivotList = [p for p in pivotList if variances[p] > minVarPivotSearch]
735 # Require that the varince decreases during
736 # consecutivePointsVarDecreases
737 # consecutive points. This will give a first
738 # estimate of the PTC turn-off, which
739 # may be updated (reduced) further in the code.
740 if len(pivotList) > 1:
741 # enumerate(pivotList) creates tuples (index, value), for
742 # each value in pivotList. The lambda function subtracts
743 # each value from the index.
744 # groupby groups elements by equal key value.
745 for k, g in groupby(enumerate(pivotList), lambda x: x[0]-x[1]):
746 group = (map(itemgetter(1), g))
747 # Form groups of consecute values from pivotList
748 group = list(map(int, group))
749 # values in pivotList are indices where np.diff(variances)
750 # is negative, i.e., where the variance starts decreasing.
751 # Find the first group of consecutive numbers when
752 # variance decreases.
753 if len(group) >= consecutivePointsVarDecreases:
754 pivotIndex = np.min(group)
755 goodPoints[pivotIndex+1:] = False
756 break
758 return goodPoints
760 def _makeZeroSafe(self, array, substituteValue=1e-9):
761 """"""
762 array = np.array(array)
763 nBad = Counter(np.ravel(array))[0]
764 if nBad == 0:
765 return array
767 index, = np.where(array == 0)
768 if len(index):
769 msg = f"Found {nBad} zeros in array at elements {index}"
770 self.log.warning(msg)
772 array[index] = substituteValue
774 return array
776 def fitPtc(self, dataset):
777 """Fit the photon transfer curve to a polynomial or to the
778 Astier+19 approximation (Eq. 16).
780 Fit the photon transfer curve with either a polynomial of
781 the order specified in the task config, or using the
782 exponential approximation in Astier+19 (Eq. 16).
784 Sigma clipping is performed iteratively for the fit, as
785 well as an initial clipping of data points that are more
786 than `config.initialNonLinearityExclusionThreshold` away
787 from lying on a straight line. This other step is necessary
788 because the photon transfer curve turns over catastrophically
789 at very high flux (because saturation
790 drops the variance to ~0) and these far outliers cause the
791 initial fit to fail, meaning the sigma cannot be calculated
792 to perform the sigma-clipping.
794 Parameters
795 ----------
796 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset`
797 The dataset containing the means, variances and
798 exposure times.
800 Returns
801 -------
802 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset`
803 This is the same dataset as the input parameter, however,
804 it has been modified to include information such as the
805 fit vectors and the fit parameters. See the class
806 `PhotonTransferCurveDatase`.
808 Raises
809 ------
810 RuntimeError
811 Raised if dataset.ptcFitType is None or empty.
812 """
813 if dataset.ptcFitType:
814 ptcFitType = dataset.ptcFitType
815 else:
816 raise RuntimeError("ptcFitType is None of empty in PTC dataset.")
817 matrixSide = self.config.maximumRangeCovariancesAstier
818 nanMatrix = np.empty((matrixSide, matrixSide))
819 nanMatrix[:] = np.nan
821 for amp in dataset.ampNames:
822 lenInputTimes = len(dataset.rawExpTimes[amp])
823 listNanMatrix = np.empty((lenInputTimes, matrixSide, matrixSide))
824 listNanMatrix[:] = np.nan
826 dataset.covariancesModel[amp] = listNanMatrix
827 dataset.aMatrix[amp] = nanMatrix
828 dataset.bMatrix[amp] = nanMatrix
829 dataset.covariancesModelNoB[amp] = listNanMatrix
830 dataset.aMatrixNoB[amp] = nanMatrix
832 def errFunc(p, x, y):
833 return ptcFunc(p, x) - y
835 sigmaCutPtcOutliers = self.config.sigmaCutPtcOutliers
836 maxIterationsPtcOutliers = self.config.maxIterationsPtcOutliers
838 for i, ampName in enumerate(dataset.ampNames):
839 timeVecOriginal = np.ravel(np.array(dataset.rawExpTimes[ampName]))
840 meanVecOriginal = np.ravel(np.array(dataset.rawMeans[ampName]))
841 varVecOriginal = np.ravel(np.array(dataset.rawVars[ampName]))
842 varVecOriginal = self._makeZeroSafe(varVecOriginal)
844 # Discard points when the variance starts to decrease after two
845 # consecutive signal levels
846 goodPoints = self._getInitialGoodPoints(meanVecOriginal, varVecOriginal,
847 self.config.minVarPivotSearch,
848 self.config.consecutivePointsVarDecreases)
850 # Check if all points are bad from the 'cpExtractPtcTask'
851 initialExpIdMask = np.ravel(np.array(dataset.expIdMask[ampName]))
853 if not (goodPoints.any() and initialExpIdMask.any()):
854 msg = (f"SERIOUS: All points in goodPoints: {goodPoints} or "
855 f"in initialExpIdMask: {initialExpIdMask} are bad."
856 f"Setting {ampName} to BAD.")
857 self.log.warning(msg)
858 # Fill entries with NaNs
859 self.fillBadAmp(dataset, ptcFitType, ampName)
860 continue
862 # Save the point where the variance starts decreasing as the
863 # PTC turnoff point
864 ptcTurnoff = meanVecOriginal[goodPoints][-1]
865 dataset.ptcTurnoff[ampName] = ptcTurnoff
867 mask = goodPoints
869 if ptcFitType == 'EXPAPPROXIMATION':
870 ptcFunc = funcAstier
871 parsIniPtc = [-1e-9, 1.0, 10.] # a00, gain, noise^2
872 # lowers and uppers obtained from BOT data studies by
873 # C. Lage (UC Davis, 11/2020).
874 if self.config.binSize > 1:
875 bounds = self._boundsForAstier(parsIniPtc)
876 else:
877 bounds = self._boundsForAstier(parsIniPtc, lowers=[-1e-4, 0.5, -2000],
878 uppers=[1e-4, 2.5, 2000])
879 if ptcFitType == 'POLYNOMIAL':
880 ptcFunc = funcPolynomial
881 parsIniPtc = self._initialParsForPolynomial(self.config.polynomialFitDegree + 1)
882 bounds = self._boundsForPolynomial(parsIniPtc)
884 # Before bootstrap fit, do an iterative fit to get rid of outliers.
885 # This further process of outlier rejection be skipped
886 # if self.config.maxIterationsPtcOutliers = 0.
887 # We already did some initial outlier rejection above in
888 # self._getInitialGoodPoints.
889 count = 1
890 newMask = np.ones_like(meanVecOriginal, dtype=bool)
891 pars = parsIniPtc
892 while count <= maxIterationsPtcOutliers:
893 # Note that application of the mask actually shrinks the array
894 # to size rather than setting elements to zero (as we want) so
895 # always update mask itself and re-apply to the original data
896 meanTempVec = meanVecOriginal[mask]
897 varTempVec = varVecOriginal[mask]
898 res = least_squares(errFunc, parsIniPtc, bounds=bounds, args=(meanTempVec, varTempVec))
899 pars = res.x
901 # change this to the original from the temp because
902 # the masks are ANDed meaning once a point is masked
903 # it's always masked, and the masks must always be the
904 # same length for broadcasting
905 sigResids = (varVecOriginal - ptcFunc(pars, meanVecOriginal))/np.sqrt(varVecOriginal)
906 newMask = np.array([True if np.abs(r) < sigmaCutPtcOutliers else False for r in sigResids])
907 mask = mask & newMask
908 if not (mask.any() and newMask.any()):
909 msg = (f"SERIOUS: All points in either mask: {mask} or newMask: {newMask} are bad. "
910 f"Setting {ampName} to BAD.")
911 self.log.warning(msg)
912 # Fill entries with NaNs
913 self.fillBadAmp(dataset, ptcFitType, ampName)
914 break
915 nDroppedTotal = Counter(mask)[False]
916 self.log.debug("Iteration %d: discarded %d points in total for %s",
917 count, nDroppedTotal, ampName)
918 count += 1
919 # objects should never shrink
920 assert (len(mask) == len(timeVecOriginal) == len(meanVecOriginal) == len(varVecOriginal))
921 if not (mask.any() and newMask.any()):
922 continue
923 dataset.expIdMask[ampName] = np.array(dataset.expIdMask[ampName])
924 # store the final mask
925 if len(dataset.expIdMask[ampName]):
926 dataset.expIdMask[ampName] &= mask # bitwise_and if there is already a mask
927 else:
928 dataset.expIdMask[ampName] = mask
929 # In case there was a previous mask stored
930 mask = dataset.expIdMask[ampName]
931 parsIniPtc = pars
932 meanVecFinal = meanVecOriginal[mask]
933 varVecFinal = varVecOriginal[mask]
935 if Counter(mask)[False] > 0:
936 self.log.info("Number of points discarded in PTC of amplifier %s:"
937 " %d out of %d", ampName, Counter(mask)[False], len(meanVecOriginal))
939 if (len(meanVecFinal) < len(parsIniPtc)):
940 msg = (f"SERIOUS: Not enough data points ({len(meanVecFinal)}) compared to the number of "
941 f"parameters of the PTC model({len(parsIniPtc)}). Setting {ampName} to BAD.")
942 self.log.warning(msg)
943 # Fill entries with NaNs
944 self.fillBadAmp(dataset, ptcFitType, ampName)
945 continue
946 # Fit the PTC.
947 # The variance of the variance is Var(v)=2*v^2/Npix. This is
948 # already calculated in `makeCovArray` of CpPtcExtract.
949 # dataset.covariancesSqrtWeights[ampName][:,0,0]
950 # has 1/sqrt(Var(v)).
951 weightsY = dataset.covariancesSqrtWeights[ampName][:, 0, 0][mask]
952 if self.config.doFitBootstrap:
953 parsFit, parsFitErr, reducedChiSqPtc = fitBootstrap(parsIniPtc, meanVecFinal,
954 varVecFinal, ptcFunc,
955 weightsY=weightsY)
956 else:
957 parsFit, parsFitErr, reducedChiSqPtc = fitLeastSq(parsIniPtc, meanVecFinal,
958 varVecFinal, ptcFunc,
959 weightsY=weightsY)
960 dataset.ptcFitPars[ampName] = parsFit
961 dataset.ptcFitParsError[ampName] = parsFitErr
962 dataset.ptcFitChiSq[ampName] = reducedChiSqPtc
963 # Masked variances (measured and modeled) and means. Need
964 # to pad the array so astropy.Table does not crash (the
965 # mask may vary per amp).
966 padLength = len(dataset.rawExpTimes[ampName]) - len(varVecFinal)
967 dataset.finalVars[ampName] = np.pad(varVecFinal, (0, padLength), 'constant',
968 constant_values=np.nan)
969 dataset.finalModelVars[ampName] = np.pad(ptcFunc(parsFit, meanVecFinal), (0, padLength),
970 'constant', constant_values=np.nan)
971 dataset.finalMeans[ampName] = np.pad(meanVecFinal, (0, padLength), 'constant',
972 constant_values=np.nan)
973 if ptcFitType == 'EXPAPPROXIMATION':
974 ptcGain = parsFit[1]
975 ptcGainErr = parsFitErr[1]
976 ptcNoise = np.sqrt(np.fabs(parsFit[2]))
977 ptcNoiseErr = 0.5*(parsFitErr[2]/np.fabs(parsFit[2]))*np.sqrt(np.fabs(parsFit[2]))
978 if ptcFitType == 'POLYNOMIAL':
979 ptcGain = 1./parsFit[1]
980 ptcGainErr = np.fabs(1./parsFit[1])*(parsFitErr[1]/parsFit[1])
981 ptcNoise = np.sqrt(np.fabs(parsFit[0]))*ptcGain
982 ptcNoiseErr = (0.5*(parsFitErr[0]/np.fabs(parsFit[0]))*(np.sqrt(np.fabs(parsFit[0]))))*ptcGain
983 dataset.gain[ampName] = ptcGain
984 dataset.gainErr[ampName] = ptcGainErr
985 dataset.noise[ampName] = ptcNoise
986 dataset.noiseErr[ampName] = ptcNoiseErr
988 if not len(dataset.ptcFitType) == 0:
989 dataset.ptcFitType = ptcFitType
990 if len(dataset.badAmps) == 0:
991 dataset.badAmps = np.repeat(np.nan, len(list(dataset.rawExpTimes.values())[0]))
993 return dataset
995 def fillBadAmp(self, dataset, ptcFitType, ampName):
996 """Fill the dataset with NaNs if there are not enough
997 good points.
999 Parameters
1000 ----------
1001 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset`
1002 The dataset containing the means, variances and
1003 exposure times.
1004 ptcFitType : {'POLYNOMIAL', 'EXPAPPROXIMATION'}
1005 Fit a 'POLYNOMIAL' (degree: 'polynomialFitDegree') or
1006 'EXPAPPROXIMATION' (Eq. 16 of Astier+19) to the PTC.
1007 ampName : `str`
1008 Amplifier name.
1009 """
1010 dataset.badAmps.append(ampName)
1011 dataset.expIdMask[ampName] = np.repeat(False, len(dataset.rawExpTimes[ampName]))
1012 dataset.gain[ampName] = np.nan
1013 dataset.gainErr[ampName] = np.nan
1014 dataset.noise[ampName] = np.nan
1015 dataset.noiseErr[ampName] = np.nan
1016 dataset.ptcFitPars[ampName] = (np.repeat(np.nan, self.config.polynomialFitDegree + 1) if
1017 ptcFitType in ["POLYNOMIAL", ] else np.repeat(np.nan, 3))
1018 dataset.ptcFitParsError[ampName] = (np.repeat(np.nan, self.config.polynomialFitDegree + 1) if
1019 ptcFitType in ["POLYNOMIAL", ] else np.repeat(np.nan, 3))
1020 dataset.ptcFitChiSq[ampName] = np.nan
1021 dataset.ptcTurnoff[ampName] = np.nan
1022 dataset.finalVars[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName]))
1023 dataset.finalModelVars[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName]))
1024 dataset.finalMeans[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName]))
1026 return