Coverage for python/lsst/cp/pipe/ptc/cpSolvePtcTask.py: 10%
401 statements
« prev ^ index » next coverage.py v6.5.0, created at 2023-02-17 01:33 -0800
« prev ^ index » next coverage.py v6.5.0, created at 2023-02-17 01:33 -0800
1# This file is part of cp_pipe.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21#
22import numpy as np
23from collections import Counter
25import lsst.pex.config as pexConfig
26import lsst.pipe.base as pipeBase
27from lsst.cp.pipe.utils import (fitLeastSq, fitBootstrap, funcPolynomial, funcAstier, symmetrize)
29from scipy.signal import fftconvolve
30from scipy.optimize import least_squares
31from itertools import groupby
32from operator import itemgetter
34import lsst.pipe.base.connectionTypes as cT
36from lsst.ip.isr import PhotonTransferCurveDataset
38from lsst.cp.pipe._lookupStaticCalibration import lookupStaticCalibration
40import copy
43__all__ = ['PhotonTransferCurveSolveConfig', 'PhotonTransferCurveSolveTask']
46class PhotonTransferCurveSolveConnections(pipeBase.PipelineTaskConnections,
47 dimensions=("instrument", "detector")):
48 inputCovariances = cT.Input(
49 name="ptcCovariances",
50 doc="Tuple with measured covariances from flats.",
51 storageClass="PhotonTransferCurveDataset",
52 dimensions=("instrument", "exposure", "detector"),
53 isCalibration=True,
54 multiple=True,
55 )
56 camera = cT.PrerequisiteInput(
57 name="camera",
58 doc="Camera the input data comes from.",
59 storageClass="Camera",
60 dimensions=("instrument",),
61 isCalibration=True,
62 lookupFunction=lookupStaticCalibration,
63 )
64 outputPtcDataset = cT.Output(
65 name="ptcDatsetProposal",
66 doc="Output proposed ptc dataset.",
67 storageClass="PhotonTransferCurveDataset",
68 dimensions=("instrument", "detector"),
69 multiple=False,
70 isCalibration=True,
71 )
74class PhotonTransferCurveSolveConfig(pipeBase.PipelineTaskConfig,
75 pipelineConnections=PhotonTransferCurveSolveConnections):
76 """Configuration for fitting measured covariances.
77 """
79 ptcFitType = pexConfig.ChoiceField(
80 dtype=str,
81 doc="Fit PTC to Eq. 16, Eq. 20 in Astier+19, or to a polynomial.",
82 default="POLYNOMIAL",
83 allowed={
84 "POLYNOMIAL": "n-degree polynomial (use 'polynomialFitDegree' to set 'n').",
85 "EXPAPPROXIMATION": "Approximation in Astier+19 (Eq. 16).",
86 "FULLCOVARIANCE": "Full covariances model in Astier+19 (Eq. 20)"
87 }
88 )
89 maximumRangeCovariancesAstier = pexConfig.Field(
90 dtype=int,
91 doc="Maximum range of covariances as in Astier+19",
92 default=8,
93 )
94 sigmaClipFullFitCovariancesAstier = pexConfig.Field(
95 dtype=float,
96 doc="sigma clip for full model fit for FULLCOVARIANCE ptcFitType ",
97 default=5.0,
98 )
99 maxIterFullFitCovariancesAstier = pexConfig.Field(
100 dtype=int,
101 doc="Maximum number of iterations in full model fit for FULLCOVARIANCE ptcFitType",
102 default=3,
103 )
104 polynomialFitDegree = pexConfig.Field(
105 dtype=int,
106 doc="Degree of polynomial to fit the PTC, when 'ptcFitType'=POLYNOMIAL.",
107 default=3,
108 )
109 sigmaCutPtcOutliers = pexConfig.Field(
110 dtype=float,
111 doc="Sigma cut for outlier rejection in PTC.",
112 default=5.0,
113 )
114 maxIterationsPtcOutliers = pexConfig.RangeField(
115 dtype=int,
116 doc="Maximum number of iterations for outlier rejection in PTC.",
117 default=2,
118 min=0
119 )
120 minVarPivotSearch = pexConfig.Field(
121 dtype=float,
122 doc="The code looks for a pivot signal point after which the variance starts decreasing at high-flux"
123 " to exclude then from the PTC model fit. However, sometimes at low fluxes, the variance"
124 " decreases slightly. Set this variable for the variance value, in ADU^2, after which the pivot "
125 " should be sought.",
126 default=10000,
127 )
128 consecutivePointsVarDecreases = pexConfig.RangeField(
129 dtype=int,
130 doc="Required number of consecutive points/fluxes in the PTC where the variance "
131 "decreases in order to find a first estimate of the PTC turn-off. ",
132 default=2,
133 min=2
134 )
135 doFitBootstrap = pexConfig.Field(
136 dtype=bool,
137 doc="Use bootstrap for the PTC fit parameters and errors?.",
138 default=False,
139 )
140 binSize = pexConfig.Field(
141 dtype=int,
142 doc="Bin the image by this factor in both dimensions.",
143 default=1,
144 )
147class PhotonTransferCurveSolveTask(pipeBase.PipelineTask):
148 """Task to fit the PTC from flat covariances.
150 The first task of the PTC measurement pipeline,
151 ``PhotonTransferCurveMeasureTask`` (and assumed to have been run
152 before this task), produced a list of
153 `~lsst.ip.isr.PhotonTransferCurveDataset` objects. Each dataset
154 contains the mean signal and covariances of the
155 difference image of the flat-field images taken at
156 the same exposure time. The list also contains dummy
157 datasets (with no measurements), whose purpose is to have
158 the input and output dimensions of ``PhotonTransferCurveMeasureTask``
159 match.
161 This task, ``PhotonTransferCurveSolveTask``, assembles the list
162 of individual PTC datasets produced
163 by ``PhotonTransferCurveMeasureTask`` into one single final PTC
164 dataset, discarding the dummy datset as appropiate.
165 The task fits the measured (co)variances to one of three models:
166 a polynomial model of a given order, or the models described
167 in equations 16 and 20 of Astier+19. These options are referred
168 to as ``POLYNOMIAL``, ``EXPAPPROXIMATION``, and ``FULLCOVARIANCE``
169 in the configuration options of the task, respectively).
170 Parameters of interest such as the gain and noise are derived
171 from the fits. The ``FULLCOVARIANCE`` model is fitted to the
172 full covariance data (as oppossed to the other two models, which
173 are fit to the variance vs mean measurements only).
175 Astier+19: "The Shape of the Photon Transfer Curve
176 of CCD sensors", arXiv:1905.08677
177 """
179 ConfigClass = PhotonTransferCurveSolveConfig
180 _DefaultName = 'cpPhotonTransferCurveSolve'
182 def runQuantum(self, butlerQC, inputRefs, outputRefs):
183 """Ensure that the input and output dimensions are passed along.
185 Parameters
186 ----------
187 butlerQC : `~lsst.daf.butler.butlerQuantumContext.ButlerQuantumContext`
188 Butler to operate on.
189 inputRefs : `~lsst.pipe.base.connections.InputQuantizedConnection`
190 Input data refs to load.
191 ouptutRefs : `~lsst.pipe.base.connections.OutputQuantizedConnection`
192 Output data refs to persist.
193 """
194 inputs = butlerQC.get(inputRefs)
195 detId = inputRefs.inputCovariances[0].dataId['detector']
196 outputs = self.run(inputCovariances=inputs['inputCovariances'], camera=inputs['camera'], detId=detId)
197 butlerQC.put(outputs, outputRefs)
199 def run(self, inputCovariances, camera=None, detId=0):
200 """Fit measured covariances to different models.
202 Parameters
203 ----------
204 inputCovariances : `list` [`lsst.ip.isr.PhotonTransferCurveDataset`]
205 List of lsst.ip.isr.PhotonTransferCurveDataset datasets.
206 camera : `lsst.afw.cameraGeom.Camera`, optional
207 Input camera.
208 detId : `int`
209 Detector ID to locate the detector in the camera and
210 populate the `lsst.ip.isr.PhotonTransferCurveDataset`
211 metadata.
212 Returns
213 -------
214 results : `lsst.pipe.base.Struct`
215 The resultins structure contains:
217 ``outputPtcDatset``
218 Final PTC dataset, containing information such as the
219 means, variances, and exposure times
220 (`lsst.ip.isr.PhotonTransferCurveDataset`).
221 """
222 # Assemble individual PTC datasets into a single PTC dataset.
223 ampNames = np.unique(inputCovariances[0].ampNames)
224 datasetPtc = PhotonTransferCurveDataset(ampNames=ampNames,
225 ptcFitType=self.config.ptcFitType,
226 covMatrixSide=self.config.maximumRangeCovariancesAstier)
227 for partialPtcDataset in inputCovariances:
228 # Ignore dummy datasets
229 if partialPtcDataset.ptcFitType == 'DUMMY':
230 continue
231 for ampName in ampNames:
232 datasetPtc.inputExpIdPairs[ampName].append(partialPtcDataset.inputExpIdPairs[ampName])
233 if type(partialPtcDataset.rawExpTimes[ampName]) is list:
234 datasetPtc.rawExpTimes[ampName].append(partialPtcDataset.rawExpTimes[ampName][0])
235 else:
236 datasetPtc.rawExpTimes[ampName].append(partialPtcDataset.rawExpTimes[ampName])
237 if type(partialPtcDataset.rawMeans[ampName]) is list:
238 datasetPtc.rawMeans[ampName].append(partialPtcDataset.rawMeans[ampName][0])
239 else:
240 datasetPtc.rawMeans[ampName].append(partialPtcDataset.rawMeans[ampName])
241 if type(partialPtcDataset.rawVars[ampName]) is list:
242 datasetPtc.rawVars[ampName].append(partialPtcDataset.rawVars[ampName][0])
243 else:
244 datasetPtc.rawVars[ampName].append(partialPtcDataset.rawVars[ampName])
245 if type(partialPtcDataset.expIdMask[ampName]) is list:
246 datasetPtc.expIdMask[ampName].append(partialPtcDataset.expIdMask[ampName][0])
247 else:
248 datasetPtc.expIdMask[ampName].append(partialPtcDataset.expIdMask[ampName])
249 datasetPtc.covariances[ampName].append(np.array(partialPtcDataset.covariances[ampName][0]))
250 datasetPtc.covariancesSqrtWeights[ampName].append(
251 np.array(partialPtcDataset.covariancesSqrtWeights[ampName][0]))
252 # Sort arrays that are filled so far in the final dataset by
253 # rawMeans index
254 for ampName in ampNames:
255 index = np.argsort(np.ravel(np.array(datasetPtc.rawMeans[ampName])))
256 datasetPtc.inputExpIdPairs[ampName] = np.array(datasetPtc.inputExpIdPairs[ampName])[index]
257 datasetPtc.rawExpTimes[ampName] = np.array(datasetPtc.rawExpTimes[ampName])[index]
258 datasetPtc.rawMeans[ampName] = np.array(datasetPtc.rawMeans[ampName])[index]
259 datasetPtc.rawVars[ampName] = np.array(datasetPtc.rawVars[ampName])[index]
260 datasetPtc.expIdMask[ampName] = np.array(datasetPtc.expIdMask[ampName])[index]
261 datasetPtc.covariances[ampName] = np.array(datasetPtc.covariances[ampName])[index]
262 datasetPtc.covariancesSqrtWeights[ampName] = np.array(
263 datasetPtc.covariancesSqrtWeights[ampName])[index]
264 if self.config.ptcFitType == "FULLCOVARIANCE":
265 # Fit the measured covariances vs mean signal to
266 # the Astier+19 full model (Eq. 20). Before that
267 # do a preliminary fit to the variance (C_00) vs mean
268 # signal (mu) curve using the EXPAPPROXIMATION model
269 # (Eq. 16 in Astier+19) in order to
270 # get the flat pairs that are masked. The
271 # points at these fluxes will also be masked when
272 # calculating the other elements of the covariance
273 # matrix, C_ij, i!=j).
275 # Preliminary fit, usign a temp dataset to get the mask
276 tempDatasetPtc = copy.copy(datasetPtc)
277 tempDatasetPtc.ptcFitType = "EXPAPPROXIMATION"
278 tempDatasetPtc = self.fitMeasurementsToModel(tempDatasetPtc)
280 # "FULLCOVARIANCE", using the mask obtained from the
281 # previous fit.
282 for ampName in datasetPtc.ampNames:
283 datasetPtc.expIdMask[ampName] = tempDatasetPtc.expIdMask[ampName]
284 datasetPtc.fitType = "FULLCOVARIANCE"
285 datasetPtc = self.fitMeasurementsToModel(datasetPtc)
286 # The other options are: self.config.ptcFitType in
287 # ("EXPAPPROXIMATION", "POLYNOMIAL")
288 else:
289 # Fit the PTC to a polynomial or to Astier+19 exponential
290 # approximation (Eq. 16). Fill up
291 # PhotonTransferCurveDataset object.
292 datasetPtc = self.fitMeasurementsToModel(datasetPtc)
294 if camera:
295 detector = camera[detId]
296 else:
297 detector = None
298 datasetPtc.updateMetadata(setDate=True, camera=camera, detector=detector)
300 return pipeBase.Struct(
301 outputPtcDataset=datasetPtc,
302 )
304 def fitMeasurementsToModel(self, dataset):
305 """Fit the measured covariances vs mean signal to a
306 polynomial or one of the models in Astier+19
307 (Eq. 16 or Eq.20).
309 Parameters
310 ----------
311 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset`
312 The dataset containing information such as the means,
313 (co)variances, and exposure times.
315 Returns
316 -------
317 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset`
318 This is the same dataset as the input parameter, however,
319 it has been modified to include information such as the
320 fit vectors and the fit parameters. See the class
321 `PhotonTransferCurveDatase`.
322 """
323 fitType = dataset.ptcFitType
324 if fitType in ["FULLCOVARIANCE", ]:
325 # This model uses the full covariance matrix in the fit.
326 # The PTC is technically defined as variance vs signal,
327 # with variance = Cov_00
328 dataset = self.fitDataFullCovariance(dataset)
329 elif fitType in ["POLYNOMIAL", "EXPAPPROXIMATION"]:
330 # The PTC is technically defined as variance vs signal
331 dataset = self.fitPtc(dataset)
332 else:
333 raise RuntimeError(
334 f"Fitting option {fitType} not one of "
335 "'POLYNOMIAL', 'EXPAPPROXIMATION', or 'FULLCOVARIANCE'"
336 )
338 return dataset
340 def fitDataFullCovariance(self, dataset):
341 """Fit measured flat covariances to the full model in
342 Astier+19 (Eq. 20).
344 Parameters
345 ----------
346 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset`
347 The dataset containing information such as the means,
348 (co)variances, and exposure times.
350 Returns
351 -------
352 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset`
353 This is the same dataset as the input parameter, however,
354 it has been modified to include information such as the
355 fit vectors and the fit parameters. See the class
356 `PhotonTransferCurveDatase`.
358 Notes
359 -----
360 The parameters of the full model for C_ij(mu) ("C_ij" and "mu"
361 in ADU^2 and ADU, respectively) in Astier+19 (Eq. 20) are:
363 - "a" coefficients (r by r matrix), units: 1/e
364 - "b" coefficients (r by r matrix), units: 1/e
365 - noise matrix (r by r matrix), units: e^2
366 - gain, units: e/ADU
368 "b" appears in Eq. 20 only through the "ab" combination, which
369 is defined in this code as "c=ab".
371 Total number of parameters: #entries(a) + #entries(c) + #entries(noise)
372 + 1. This is equivalent to r^2 + r^2 + r^2 + 1, where "r" is the
373 maximum lag considered for the covariances calculation, and the
374 extra "1" is the gain. If "b" is 0, then "c" is 0, and len(pInit) will
375 have r^2 fewer entries.
376 """
377 matrixSide = self.config.maximumRangeCovariancesAstier
378 lenParams = matrixSide*matrixSide
380 for ampName in dataset.ampNames:
381 lenInputTimes = len(dataset.rawExpTimes[ampName])
382 # Not used when ptcFitType is 'FULLCOVARIANCE'
383 dataset.ptcFitPars[ampName] = [np.nan]
384 dataset.ptcFitParsError[ampName] = [np.nan]
385 dataset.ptcFitChiSq[ampName] = np.nan
387 if ampName in dataset.badAmps:
388 # Bad amp
389 # Entries need to have proper dimensions so read/write
390 # with astropy.Table works.
391 nanMatrix = np.full((matrixSide, matrixSide), np.nan)
392 listNanMatrix = np.full((lenInputTimes, matrixSide, matrixSide), np.nan)
393 dataset.covariancesModel[ampName] = listNanMatrix
394 dataset.covariancesSqrtWeights[ampName] = listNanMatrix
395 dataset.aMatrix[ampName] = nanMatrix
396 dataset.bMatrix[ampName] = nanMatrix
397 dataset.covariancesModelNoB[ampName] = listNanMatrix
398 dataset.aMatrixNoB[ampName] = nanMatrix
400 dataset.expIdMask[ampName] = np.repeat(False, lenInputTimes)
401 dataset.gain[ampName] = np.nan
402 dataset.gainErr[ampName] = np.nan
403 dataset.noise[ampName] = np.nan
404 dataset.noiseErr[ampName] = np.nan
405 dataset.finalVars[ampName] = np.repeat(np.nan, lenInputTimes)
406 dataset.finalModelVars[ampName] = np.repeat(np.nan, lenInputTimes)
407 dataset.finalMeans[ampName] = np.repeat(np.nan, lenInputTimes)
408 continue
410 muAtAmp = dataset.rawMeans[ampName]
411 maskAtAmp = dataset.expIdMask[ampName]
412 if len(maskAtAmp) == 0:
413 maskAtAmp = np.repeat(True, len(muAtAmp))
415 muAtAmp = muAtAmp[maskAtAmp]
416 covAtAmp = np.nan_to_num(dataset.covariances[ampName])[maskAtAmp]
417 covSqrtWeightsAtAmp = np.nan_to_num(dataset.covariancesSqrtWeights[ampName])[maskAtAmp]
419 # Initial fit, to approximate parameters, with c=0
420 a0, c0, noise0, gain0 = self.initialFitFullCovariance(muAtAmp, covAtAmp, covSqrtWeightsAtAmp)
422 # Fit full model (Eq. 20 of Astier+19) and same model with
423 # b=0 (c=0 in this code)
424 pInit = np.concatenate((a0.flatten(), c0.flatten(), noise0.flatten(), np.array(gain0)), axis=None)
425 functionsDict = {'fullModel': self.funcFullCovarianceModel,
426 'fullModelNoB': self.funcFullCovarianceModelNoB}
427 fitResults = {'fullModel': {'a': [], 'c': [], 'noise': [], 'gain': [], 'paramsErr': []},
428 'fullModelNoB': {'a': [], 'c': [], 'noise': [], 'gain': [], 'paramsErr': []}}
429 for key in functionsDict:
430 params, paramsErr, _ = fitLeastSq(pInit, muAtAmp,
431 covAtAmp.flatten(), functionsDict[key],
432 weightsY=covSqrtWeightsAtAmp.flatten())
433 a = params[:lenParams].reshape((matrixSide, matrixSide))
434 c = params[lenParams:2*lenParams].reshape((matrixSide, matrixSide))
435 noise = params[2*lenParams:3*lenParams].reshape((matrixSide, matrixSide))
436 gain = params[-1]
438 fitResults[key]['a'] = a
439 fitResults[key]['c'] = c
440 fitResults[key]['noise'] = noise
441 fitResults[key]['gain'] = gain
442 fitResults[key]['paramsErr'] = paramsErr
444 # Put the information in the PTC dataset
446 # Not used when ptcFitType is 'FULLCOVARIANCE'
447 dataset.ptcFitPars[ampName] = [np.nan]
448 dataset.ptcFitParsError[ampName] = [np.nan]
449 dataset.ptcFitChiSq[ampName] = np.nan
451 # Save full covariances, covariances models, and their weights.
452 # dataset.expIdMask is already full, but needs to be
453 # converted to bool.
454 dataset.expIdMask[ampName] = np.array(dataset.expIdMask[ampName], dtype=bool)
455 dataset.covariances[ampName] = covAtAmp
456 dataset.covariancesModel[ampName] = self.evalCovModel(muAtAmp,
457 fitResults['fullModel']['a'],
458 fitResults['fullModel']['c'],
459 fitResults['fullModel']['noise'],
460 fitResults['fullModel']['gain'])
461 dataset.covariancesSqrtWeights[ampName] = covSqrtWeightsAtAmp
462 dataset.aMatrix[ampName] = fitResults['fullModel']['a']
463 dataset.bMatrix[ampName] = fitResults['fullModel']['c']/fitResults['fullModel']['a']
464 dataset.covariancesModelNoB[ampName] = self.evalCovModel(muAtAmp,
465 fitResults['fullModelNoB']['a'],
466 fitResults['fullModelNoB']['c'],
467 fitResults['fullModelNoB']['noise'],
468 fitResults['fullModelNoB']['gain'],
469 setBtoZero=True)
470 dataset.aMatrixNoB[ampName] = fitResults['fullModelNoB']['a']
471 dataset.gain[ampName] = fitResults['fullModel']['gain']
472 dataset.gainErr[ampName] = fitResults['fullModel']['paramsErr'][-1]
473 readoutNoise = fitResults['fullModel']['noise'][0][0]
474 readoutNoiseSqrt = np.sqrt(np.fabs(readoutNoise))
475 dataset.noise[ampName] = readoutNoise
476 readoutNoiseSigma = fitResults['fullModel']['paramsErr'][2*lenParams]
477 dataset.noiseErr[ampName] = 0.5*(readoutNoiseSigma/np.fabs(readoutNoise))*readoutNoiseSqrt
478 dataset.finalVars[ampName] = covAtAmp[:, 0, 0]
479 dataset.finalModelVars[ampName] = dataset.covariancesModel[ampName][:, 0, 0]
480 dataset.finalMeans[ampName] = muAtAmp
482 return dataset
484 def initialFitFullCovariance(self, mu, cov, sqrtW):
485 """ Performs a crude parabolic fit of the data in order to start
486 the full fit close to the solution, setting b=0 (c=0) in Eq. 20
487 of Astier+19.
489 Parameters
490 ----------
491 mu : `numpy.array`, (N,)
492 Signal `mu` (ADU)
493 cov : `numpy.array`, (N, M, M)
494 Covariance arrays of size `(M, M)` (with
495 `M = config.maximumRangeCovariancesAstier`),
496 indexed by mean signal `mu`.
497 sqrtW : `numpy.array`, (N,)
498 Covariance weights, defined as 1./sqrt(Variances)
500 Returns
501 -------
502 a : `numpy.array`, (M, M)
503 "a" parameter per flux in Eq. 20 of Astier+19.
504 c : `numpy.array`, (M, M)
505 "c"="ab" parameter per flux in Eq. 20 of Astier+19.
506 noise : `numpy.array`, (M, M)
507 "noise" parameter per flux in Eq. 20 of Astier+19.
508 gain : `float`
509 Amplifier gain (e/ADU)
510 """
511 matrixSide = self.config.maximumRangeCovariancesAstier
513 # Initialize fit parameters
514 a = np.zeros((matrixSide, matrixSide))
515 c = np.zeros((matrixSide, matrixSide))
516 noise = np.zeros((matrixSide, matrixSide))
517 gain = 1.
519 # iterate the fit to account for higher orders
520 # the chi2 does not necessarily go down, so one could
521 # stop when it increases
522 oldChi2 = 1e30
523 for _ in range(5):
524 model = np.nan_to_num(self.evalCovModel(mu, a, c, noise, gain, setBtoZero=True))
525 # loop on lags
526 for i in range(matrixSide):
527 for j in range(matrixSide):
528 # fit a parabola for a given lag
529 parsFit = np.polyfit(mu, cov[:, i, j] - model[:, i, j],
530 2, w=sqrtW[:, i, j])
531 # model equation (Eq. 20) in Astier+19, with c=a*b=0:
532 a[i, j] += parsFit[0]
533 noise[i, j] += parsFit[2]
534 if(i + j == 0):
535 gain = 1./(1/gain+parsFit[1])
536 weightedRes = (model - cov)*sqrtW
537 chi2 = (weightedRes.flatten()**2).sum()
538 if chi2 > oldChi2:
539 break
540 oldChi2 = chi2
542 return a, c, noise, gain
544 def funcFullCovarianceModel(self, params, x):
545 """Model to fit covariances from flat fields; Equation 20 of
546 Astier+19.
548 Parameters
549 ----------
550 params : `list`
551 Parameters of the model: aMatrix, CMatrix, noiseMatrix,
552 gain (e/ADU).
553 x : `numpy.array`, (N,)
554 Signal `mu` (ADU)
556 Returns
557 -------
558 y : `numpy.array`, (N,)
559 Covariance matrix.
560 """
561 matrixSide = self.config.maximumRangeCovariancesAstier
562 lenParams = matrixSide*matrixSide
563 aMatrix = params[:lenParams].reshape((matrixSide, matrixSide))
564 cMatrix = params[lenParams:2*lenParams].reshape((matrixSide, matrixSide))
565 noiseMatrix = params[2*lenParams:3*lenParams].reshape((matrixSide, matrixSide))
566 gain = params[-1]
568 return self.evalCovModel(x, aMatrix, cMatrix, noiseMatrix, gain).flatten()
570 def funcFullCovarianceModelNoB(self, params, x):
571 """Model to fit covariances from flat fields; Equation 20 of
572 Astier+19, with b=0 (equivalent to c=a*b=0 in this code).
574 Parameters
575 ----------
576 params : `list`
577 Parameters of the model: aMatrix, CMatrix, noiseMatrix,
578 gain (e/ADU).
579 x : `numpy.array`, (N,)
580 Signal mu (ADU)
582 Returns
583 -------
584 y : `numpy.array`, (N,)
585 Covariance matrix.
586 """
587 matrixSide = self.config.maximumRangeCovariancesAstier
588 lenParams = matrixSide*matrixSide
589 aMatrix = params[:lenParams].reshape((matrixSide, matrixSide))
590 cMatrix = params[lenParams:2*lenParams].reshape((matrixSide, matrixSide))
591 noiseMatrix = params[2*lenParams:3*lenParams].reshape((matrixSide, matrixSide))
592 gain = params[-1]
594 return self.evalCovModel(x, aMatrix, cMatrix, noiseMatrix, gain, setBtoZero=True).flatten()
596 def evalCovModel(self, mu, aMatrix, cMatrix, noiseMatrix, gain, setBtoZero=False):
597 """Computes full covariances model (Eq. 20 of Astier+19).
599 Parameters
600 ----------
601 mu : `numpy.array`, (N,)
602 List of mean signals.
603 aMatrix : `numpy.array`, (M, M)
604 "a" parameter per flux in Eq. 20 of Astier+19.
605 cMatrix : `numpy.array`, (M, M)
606 "c"="ab" parameter per flux in Eq. 20 of Astier+19.
607 noiseMatrix : `numpy.array`, (M, M)
608 "noise" parameter per flux in Eq. 20 of Astier+19.
609 gain : `float`
610 Amplifier gain (e/ADU)
611 setBtoZero=False : `bool`, optional
612 Set "b" parameter in full model (see Astier+19) to zero.
614 Returns
615 -------
616 covModel : `numpy.array`, (N, M, M)
617 Covariances model.
619 Notes
620 -----
621 By default, computes the covModel for the mu's stored(self.mu).
622 Returns cov[Nmu, M, M]. The variance for the PTC is
623 cov[:, 0, 0]. mu and cov are in ADUs and ADUs squared. To use
624 electrons for both, the gain should be set to 1. This routine
625 implements the model in Astier+19 (1905.08677).
626 The parameters of the full model for C_ij(mu) ("C_ij" and "mu"
627 in ADU^2 and ADU, respectively) in Astier+19 (Eq. 20) are:
629 - "a" coefficients (M by M matrix), units: 1/e
630 - "b" coefficients (M by M matrix), units: 1/e
631 - noise matrix (M by M matrix), units: e^2
632 - gain, units: e/ADU
634 "b" appears in Eq. 20 only through the "ab" combination, which
635 is defined in this code as "c=ab".
636 """
637 matrixSide = self.config.maximumRangeCovariancesAstier
638 sa = (matrixSide, matrixSide)
639 # pad a with zeros and symmetrize
640 aEnlarged = np.zeros((int(sa[0]*1.5)+1, int(sa[1]*1.5)+1))
641 aEnlarged[0:sa[0], 0:sa[1]] = aMatrix
642 aSym = symmetrize(aEnlarged)
643 # pad c with zeros and symmetrize
644 cEnlarged = np.zeros((int(sa[0]*1.5)+1, int(sa[1]*1.5)+1))
645 cEnlarged[0:sa[0], 0:sa[1]] = cMatrix
646 cSym = symmetrize(cEnlarged)
647 a2 = fftconvolve(aSym, aSym, mode='same')
648 a3 = fftconvolve(a2, aSym, mode='same')
649 ac = fftconvolve(aSym, cSym, mode='same')
650 (xc, yc) = np.unravel_index(np.abs(aSym).argmax(), a2.shape)
652 a1 = aMatrix[np.newaxis, :, :]
653 a2 = a2[np.newaxis, xc:xc + matrixSide, yc:yc + matrixSide]
654 a3 = a3[np.newaxis, xc:xc + matrixSide, yc:yc + matrixSide]
655 ac = ac[np.newaxis, xc:xc + matrixSide, yc:yc + matrixSide]
656 c1 = cMatrix[np.newaxis, ::]
658 # assumes that mu is 1d
659 bigMu = mu[:, np.newaxis, np.newaxis]*gain
660 # c(=a*b in Astier+19) also has a contribution to the last
661 # term, that is absent for now.
662 if setBtoZero:
663 c1 = np.zeros_like(c1)
664 ac = np.zeros_like(ac)
665 covModel = (bigMu/(gain*gain)*(a1*bigMu+2./3.*(bigMu*bigMu)*(a2 + c1)
666 + (1./3.*a3 + 5./6.*ac)*(bigMu*bigMu*bigMu)) + noiseMatrix[np.newaxis, :, :]/gain**2)
667 # add the Poisson term, and the read out noise (variance)
668 covModel[:, 0, 0] += mu/gain
670 return covModel
672 # EXPAPPROXIMATION and POLYNOMIAL fit methods
673 @staticmethod
674 def _initialParsForPolynomial(order):
675 assert(order >= 2)
676 pars = np.zeros(order, dtype=float)
677 pars[0] = 10
678 pars[1] = 1
679 pars[2:] = 0.0001
680 return pars
682 @staticmethod
683 def _boundsForPolynomial(initialPars, lowers=[], uppers=[]):
684 if not len(lowers):
685 lowers = [np.NINF for p in initialPars]
686 if not len(uppers):
687 uppers = [np.inf for p in initialPars]
688 lowers[1] = 0 # no negative gains
689 return (lowers, uppers)
691 @staticmethod
692 def _boundsForAstier(initialPars, lowers=[], uppers=[]):
693 if not len(lowers):
694 lowers = [np.NINF for p in initialPars]
695 if not len(uppers):
696 uppers = [np.inf for p in initialPars]
697 return (lowers, uppers)
699 @staticmethod
700 def _getInitialGoodPoints(means, variances, minVarPivotSearch, consecutivePointsVarDecreases):
701 """Return a boolean array to mask bad points.
703 Parameters
704 ----------
705 means : `numpy.array`
706 Input array with mean signal values.
707 variances : `numpy.array`
708 Input array with variances at each mean value.
709 minVarPivotSearch : `float`
710 The variance (in ADU^2), above which, the point
711 of decreasing variance should be sought.
712 consecutivePointsVarDecreases : `int`
713 Required number of consecutive points/fluxes
714 in the PTC where the variance
715 decreases in order to find a first
716 estimate of the PTC turn-off.
718 Returns
719 ------
720 goodPoints : `numpy.array` [`bool`]
721 Boolean array to select good (`True`) and bad (`False`)
722 points.
724 Notes
725 -----
726 Eliminate points beyond which the variance decreases.
727 """
728 goodPoints = np.ones_like(means, dtype=bool)
729 # Variances are sorted and should monotonically increase
730 pivotList = np.where(np.array(np.diff(variances)) < 0)[0]
731 if len(pivotList) > 0:
732 # For small values, sometimes the variance decreases slightly
733 # Only look when var > self.config.minVarPivotSearch
734 pivotList = [p for p in pivotList if variances[p] > minVarPivotSearch]
735 # Require that the varince decreases during
736 # consecutivePointsVarDecreases
737 # consecutive points. This will give a first
738 # estimate of the PTC turn-off, which
739 # may be updated (reduced) further in the code.
740 if len(pivotList) > 1:
741 # enumerate(pivotList) creates tuples (index, value), for
742 # each value in pivotList. The lambda function subtracts
743 # each value from the index.
744 # groupby groups elements by equal key value.
745 for k, g in groupby(enumerate(pivotList), lambda x: x[0]-x[1]):
746 group = (map(itemgetter(1), g))
747 # Form groups of consecute values from pivotList
748 group = list(map(int, group))
749 # values in pivotList are indices where np.diff(variances)
750 # is negative, i.e., where the variance starts decreasing.
751 # Find the first group of consecutive numbers when
752 # variance decreases.
753 if len(group) >= consecutivePointsVarDecreases:
754 pivotIndex = np.min(group)
755 goodPoints[pivotIndex+1:] = False
756 break
758 # Finally, we filter out any infinities or NaNs.
759 goodPoints[(~np.isfinite(means)) | (~np.isfinite(variances))] = False
761 return goodPoints
763 def _makeZeroSafe(self, array, substituteValue=1e-9):
764 """"""
765 array = np.array(array)
766 nBad = Counter(np.ravel(array))[0]
767 if nBad == 0:
768 return array
770 index, = np.where(array == 0)
771 if len(index):
772 msg = f"Found {nBad} zeros in array at elements {index}"
773 self.log.warning(msg)
775 array[index] = substituteValue
777 return array
779 def fitPtc(self, dataset):
780 """Fit the photon transfer curve to a polynomial or to the
781 Astier+19 approximation (Eq. 16).
783 Fit the photon transfer curve with either a polynomial of
784 the order specified in the task config, or using the
785 exponential approximation in Astier+19 (Eq. 16).
787 Sigma clipping is performed iteratively for the fit, as
788 well as an initial clipping of data points that are more
789 than `config.initialNonLinearityExclusionThreshold` away
790 from lying on a straight line. This other step is necessary
791 because the photon transfer curve turns over catastrophically
792 at very high flux (because saturation
793 drops the variance to ~0) and these far outliers cause the
794 initial fit to fail, meaning the sigma cannot be calculated
795 to perform the sigma-clipping.
797 Parameters
798 ----------
799 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset`
800 The dataset containing the means, variances and
801 exposure times.
803 Returns
804 -------
805 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset`
806 This is the same dataset as the input parameter, however,
807 it has been modified to include information such as the
808 fit vectors and the fit parameters. See the class
809 `PhotonTransferCurveDatase`.
811 Raises
812 ------
813 RuntimeError
814 Raised if dataset.ptcFitType is None or empty.
815 """
816 if dataset.ptcFitType:
817 ptcFitType = dataset.ptcFitType
818 else:
819 raise RuntimeError("ptcFitType is None of empty in PTC dataset.")
820 matrixSide = self.config.maximumRangeCovariancesAstier
821 nanMatrix = np.empty((matrixSide, matrixSide))
822 nanMatrix[:] = np.nan
824 for amp in dataset.ampNames:
825 lenInputTimes = len(dataset.rawExpTimes[amp])
826 listNanMatrix = np.empty((lenInputTimes, matrixSide, matrixSide))
827 listNanMatrix[:] = np.nan
829 dataset.covariancesModel[amp] = listNanMatrix
830 dataset.aMatrix[amp] = nanMatrix
831 dataset.bMatrix[amp] = nanMatrix
832 dataset.covariancesModelNoB[amp] = listNanMatrix
833 dataset.aMatrixNoB[amp] = nanMatrix
835 def errFunc(p, x, y):
836 return ptcFunc(p, x) - y
838 sigmaCutPtcOutliers = self.config.sigmaCutPtcOutliers
839 maxIterationsPtcOutliers = self.config.maxIterationsPtcOutliers
841 for i, ampName in enumerate(dataset.ampNames):
842 timeVecOriginal = np.ravel(np.array(dataset.rawExpTimes[ampName]))
843 meanVecOriginal = np.ravel(np.array(dataset.rawMeans[ampName]))
844 varVecOriginal = np.ravel(np.array(dataset.rawVars[ampName]))
845 varVecOriginal = self._makeZeroSafe(varVecOriginal)
847 # Discard points when the variance starts to decrease after two
848 # consecutive signal levels
849 goodPoints = self._getInitialGoodPoints(meanVecOriginal, varVecOriginal,
850 self.config.minVarPivotSearch,
851 self.config.consecutivePointsVarDecreases)
853 # Check if all points are bad from the 'cpExtractPtcTask'
854 initialExpIdMask = np.ravel(np.array(dataset.expIdMask[ampName]))
856 if not (goodPoints.any() and initialExpIdMask.any()):
857 msg = (f"SERIOUS: All points in goodPoints: {goodPoints} or "
858 f"in initialExpIdMask: {initialExpIdMask} are bad."
859 f"Setting {ampName} to BAD.")
860 self.log.warning(msg)
861 # Fill entries with NaNs
862 self.fillBadAmp(dataset, ptcFitType, ampName)
863 continue
865 # Save the point where the variance starts decreasing as the
866 # PTC turnoff point
867 ptcTurnoff = meanVecOriginal[goodPoints][-1]
868 dataset.ptcTurnoff[ampName] = ptcTurnoff
870 mask = goodPoints
872 if ptcFitType == 'EXPAPPROXIMATION':
873 ptcFunc = funcAstier
874 parsIniPtc = [-1e-9, 1.0, 10.] # a00, gain, noise^2
875 # lowers and uppers obtained from BOT data studies by
876 # C. Lage (UC Davis, 11/2020).
877 if self.config.binSize > 1:
878 bounds = self._boundsForAstier(parsIniPtc)
879 else:
880 bounds = self._boundsForAstier(parsIniPtc, lowers=[-1e-4, 0.5, -2000],
881 uppers=[1e-4, 2.5, 2000])
882 if ptcFitType == 'POLYNOMIAL':
883 ptcFunc = funcPolynomial
884 parsIniPtc = self._initialParsForPolynomial(self.config.polynomialFitDegree + 1)
885 bounds = self._boundsForPolynomial(parsIniPtc)
887 # Before bootstrap fit, do an iterative fit to get rid of outliers.
888 # This further process of outlier rejection be skipped
889 # if self.config.maxIterationsPtcOutliers = 0.
890 # We already did some initial outlier rejection above in
891 # self._getInitialGoodPoints.
892 count = 1
893 newMask = np.ones_like(meanVecOriginal, dtype=bool)
894 pars = parsIniPtc
895 while count <= maxIterationsPtcOutliers:
896 # Note that application of the mask actually shrinks the array
897 # to size rather than setting elements to zero (as we want) so
898 # always update mask itself and re-apply to the original data
899 meanTempVec = meanVecOriginal[mask]
900 varTempVec = varVecOriginal[mask]
901 res = least_squares(errFunc, parsIniPtc, bounds=bounds, args=(meanTempVec, varTempVec))
902 pars = res.x
904 # change this to the original from the temp because
905 # the masks are ANDed meaning once a point is masked
906 # it's always masked, and the masks must always be the
907 # same length for broadcasting
908 sigResids = (varVecOriginal - ptcFunc(pars, meanVecOriginal))/np.sqrt(varVecOriginal)
909 newMask = np.array([True if np.abs(r) < sigmaCutPtcOutliers else False for r in sigResids])
910 mask = mask & newMask
911 if not (mask.any() and newMask.any()):
912 msg = (f"SERIOUS: All points in either mask: {mask} or newMask: {newMask} are bad. "
913 f"Setting {ampName} to BAD.")
914 self.log.warning(msg)
915 # Fill entries with NaNs
916 self.fillBadAmp(dataset, ptcFitType, ampName)
917 break
918 nDroppedTotal = Counter(mask)[False]
919 self.log.debug("Iteration %d: discarded %d points in total for %s",
920 count, nDroppedTotal, ampName)
921 count += 1
922 # objects should never shrink
923 assert (len(mask) == len(timeVecOriginal) == len(meanVecOriginal) == len(varVecOriginal))
924 if not (mask.any() and newMask.any()):
925 continue
926 dataset.expIdMask[ampName] = np.array(dataset.expIdMask[ampName])
927 # store the final mask
928 if len(dataset.expIdMask[ampName]):
929 dataset.expIdMask[ampName] &= mask # bitwise_and if there is already a mask
930 else:
931 dataset.expIdMask[ampName] = mask
932 # In case there was a previous mask stored
933 mask = dataset.expIdMask[ampName]
934 parsIniPtc = pars
935 meanVecFinal = meanVecOriginal[mask]
936 varVecFinal = varVecOriginal[mask]
938 if Counter(mask)[False] > 0:
939 self.log.info("Number of points discarded in PTC of amplifier %s:"
940 " %d out of %d", ampName, Counter(mask)[False], len(meanVecOriginal))
942 if (len(meanVecFinal) < len(parsIniPtc)):
943 msg = (f"SERIOUS: Not enough data points ({len(meanVecFinal)}) compared to the number of "
944 f"parameters of the PTC model({len(parsIniPtc)}). Setting {ampName} to BAD.")
945 self.log.warning(msg)
946 # Fill entries with NaNs
947 self.fillBadAmp(dataset, ptcFitType, ampName)
948 continue
949 # Fit the PTC.
950 # The variance of the variance is Var(v)=2*v^2/Npix. This is
951 # already calculated in `makeCovArray` of CpPtcExtract.
952 # dataset.covariancesSqrtWeights[ampName][:,0,0]
953 # has 1/sqrt(Var(v)).
954 weightsY = dataset.covariancesSqrtWeights[ampName][:, 0, 0][mask]
955 if self.config.doFitBootstrap:
956 parsFit, parsFitErr, reducedChiSqPtc = fitBootstrap(parsIniPtc, meanVecFinal,
957 varVecFinal, ptcFunc,
958 weightsY=weightsY)
959 else:
960 parsFit, parsFitErr, reducedChiSqPtc = fitLeastSq(parsIniPtc, meanVecFinal,
961 varVecFinal, ptcFunc,
962 weightsY=weightsY)
963 dataset.ptcFitPars[ampName] = parsFit
964 dataset.ptcFitParsError[ampName] = parsFitErr
965 dataset.ptcFitChiSq[ampName] = reducedChiSqPtc
966 # Masked variances (measured and modeled) and means. Need
967 # to pad the array so astropy.Table does not crash (the
968 # mask may vary per amp).
969 padLength = len(dataset.rawExpTimes[ampName]) - len(varVecFinal)
970 dataset.finalVars[ampName] = np.pad(varVecFinal, (0, padLength), 'constant',
971 constant_values=np.nan)
972 dataset.finalModelVars[ampName] = np.pad(ptcFunc(parsFit, meanVecFinal), (0, padLength),
973 'constant', constant_values=np.nan)
974 dataset.finalMeans[ampName] = np.pad(meanVecFinal, (0, padLength), 'constant',
975 constant_values=np.nan)
976 if ptcFitType == 'EXPAPPROXIMATION':
977 ptcGain = parsFit[1]
978 ptcGainErr = parsFitErr[1]
979 ptcNoise = np.sqrt(np.fabs(parsFit[2]))
980 ptcNoiseErr = 0.5*(parsFitErr[2]/np.fabs(parsFit[2]))*np.sqrt(np.fabs(parsFit[2]))
981 if ptcFitType == 'POLYNOMIAL':
982 ptcGain = 1./parsFit[1]
983 ptcGainErr = np.fabs(1./parsFit[1])*(parsFitErr[1]/parsFit[1])
984 ptcNoise = np.sqrt(np.fabs(parsFit[0]))*ptcGain
985 ptcNoiseErr = (0.5*(parsFitErr[0]/np.fabs(parsFit[0]))*(np.sqrt(np.fabs(parsFit[0]))))*ptcGain
986 dataset.gain[ampName] = ptcGain
987 dataset.gainErr[ampName] = ptcGainErr
988 dataset.noise[ampName] = ptcNoise
989 dataset.noiseErr[ampName] = ptcNoiseErr
991 if not len(dataset.ptcFitType) == 0:
992 dataset.ptcFitType = ptcFitType
993 if len(dataset.badAmps) == 0:
994 dataset.badAmps = np.repeat(np.nan, len(list(dataset.rawExpTimes.values())[0]))
996 return dataset
998 def fillBadAmp(self, dataset, ptcFitType, ampName):
999 """Fill the dataset with NaNs if there are not enough
1000 good points.
1002 Parameters
1003 ----------
1004 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset`
1005 The dataset containing the means, variances and
1006 exposure times.
1007 ptcFitType : {'POLYNOMIAL', 'EXPAPPROXIMATION'}
1008 Fit a 'POLYNOMIAL' (degree: 'polynomialFitDegree') or
1009 'EXPAPPROXIMATION' (Eq. 16 of Astier+19) to the PTC.
1010 ampName : `str`
1011 Amplifier name.
1012 """
1013 dataset.badAmps.append(ampName)
1014 dataset.expIdMask[ampName] = np.repeat(False, len(dataset.rawExpTimes[ampName]))
1015 dataset.gain[ampName] = np.nan
1016 dataset.gainErr[ampName] = np.nan
1017 dataset.noise[ampName] = np.nan
1018 dataset.noiseErr[ampName] = np.nan
1019 dataset.ptcFitPars[ampName] = (np.repeat(np.nan, self.config.polynomialFitDegree + 1) if
1020 ptcFitType in ["POLYNOMIAL", ] else np.repeat(np.nan, 3))
1021 dataset.ptcFitParsError[ampName] = (np.repeat(np.nan, self.config.polynomialFitDegree + 1) if
1022 ptcFitType in ["POLYNOMIAL", ] else np.repeat(np.nan, 3))
1023 dataset.ptcFitChiSq[ampName] = np.nan
1024 dataset.ptcTurnoff[ampName] = np.nan
1025 dataset.finalVars[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName]))
1026 dataset.finalModelVars[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName]))
1027 dataset.finalMeans[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName]))
1029 return