Coverage for python/lsst/cp/pipe/ptc/cpSolvePtcTask.py: 11%
499 statements
« prev ^ index » next coverage.py v7.4.4, created at 2024-04-10 12:33 +0000
« prev ^ index » next coverage.py v7.4.4, created at 2024-04-10 12:33 +0000
1# This file is part of cp_pipe.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21#
22import numpy as np
23from collections import Counter
25import lsst.pex.config as pexConfig
26import lsst.pipe.base as pipeBase
27from lsst.cp.pipe.utils import (fitLeastSq, fitBootstrap, funcPolynomial,
28 funcAstier, symmetrize, Pol2D)
30from scipy.signal import fftconvolve
31from scipy.optimize import least_squares
32from itertools import groupby
33from operator import itemgetter
35import lsst.pipe.base.connectionTypes as cT
37from lsst.ip.isr import PhotonTransferCurveDataset
39import copy
42__all__ = ['PhotonTransferCurveSolveConfig', 'PhotonTransferCurveSolveTask']
45class PhotonTransferCurveSolveConnections(pipeBase.PipelineTaskConnections,
46 dimensions=("instrument", "detector")):
47 inputCovariances = cT.Input(
48 name="ptcCovariances",
49 doc="Tuple with measured covariances from flats.",
50 storageClass="PhotonTransferCurveDataset",
51 dimensions=("instrument", "exposure", "detector"),
52 isCalibration=True,
53 multiple=True,
54 )
55 camera = cT.PrerequisiteInput(
56 name="camera",
57 doc="Camera the input data comes from.",
58 storageClass="Camera",
59 dimensions=("instrument",),
60 isCalibration=True,
61 )
62 outputPtcDataset = cT.Output(
63 name="ptcDatsetProposal",
64 doc="Output proposed ptc dataset.",
65 storageClass="PhotonTransferCurveDataset",
66 dimensions=("instrument", "detector"),
67 multiple=False,
68 isCalibration=True,
69 )
72class PhotonTransferCurveSolveConfig(pipeBase.PipelineTaskConfig,
73 pipelineConnections=PhotonTransferCurveSolveConnections):
74 """Configuration for fitting measured covariances.
75 """
77 ptcFitType = pexConfig.ChoiceField(
78 dtype=str,
79 doc="Fit PTC to Eq. 16, Eq. 20 in Astier+19, or to a polynomial.",
80 default="POLYNOMIAL",
81 allowed={
82 "POLYNOMIAL": "n-degree polynomial (use 'polynomialFitDegree' to set 'n').",
83 "EXPAPPROXIMATION": "Approximation in Astier+19 (Eq. 16).",
84 "FULLCOVARIANCE": "Full covariances model in Astier+19 (Eq. 20)"
85 }
86 )
87 minMeanSignal = pexConfig.DictField(
88 keytype=str,
89 itemtype=float,
90 doc="Minimum values (inclusive) of mean signal (in ADU) per amp to use."
91 " The same cut is applied to all amps if this parameter [`dict`] is passed as "
92 " {'ALL_AMPS': value}",
93 default={'ALL_AMPS': 0.0},
94 )
95 maxMeanSignal = pexConfig.DictField(
96 keytype=str,
97 itemtype=float,
98 doc="Maximum values (inclusive) of mean signal (in ADU) below which to consider, per amp."
99 " The same cut is applied to all amps if this dictionary is of the form"
100 " {'ALL_AMPS': value}",
101 default={'ALL_AMPS': 1e6},
102 )
103 maximumRangeCovariancesAstier = pexConfig.Field(
104 dtype=int,
105 doc="Maximum range of measured covariances as in Astier+19",
106 default=8,
107 )
108 maximumRangeCovariancesAstierFullCovFit = pexConfig.Field(
109 dtype=int,
110 doc="Maximum range up to where to fit covariances as in Astier+19, "
111 "for the FULLCOVARIANCE model."
112 "This is different from maximumRangeCovariancesAstier."
113 "It should be less or equal than maximumRangeCovariancesAstier."
114 "The number of parameters for this model is "
115 "3*maximumRangeCovariancesAstierFullCovFit^2 + 1, so increase with care "
116 "so that the fit is not too slow.",
117 default=8,
118 )
119 doSubtractLongRangeCovariances = pexConfig.Field(
120 dtype=bool,
121 doc="Subtract long-range covariances before FULLCOVARIANCE fit, "
122 "beyond startLongRangeCovariances?",
123 default=False,
124 )
125 startLongRangeCovariances = pexConfig.Field(
126 dtype=int,
127 doc="If doSubtractLongRangeCovariances is True, subtract covariances "
128 "beyond this range. It should be less than maximumRangeCovariancesAstier. ",
129 default=4,
130 )
131 polyDegLongRangeCovariances = pexConfig.Field(
132 dtype=int,
133 doc="If doSubtractLongRangeCovariances is True, polynomial "
134 "degree to fit data beyond startLongRangeCovariances.",
135 default=1,
136 )
137 sigmaClipFullFitCovariancesAstier = pexConfig.Field(
138 dtype=float,
139 doc="sigma clip for full model fit for FULLCOVARIANCE ptcFitType ",
140 default=5.0,
141 )
142 maxIterFullFitCovariancesAstier = pexConfig.Field(
143 dtype=int,
144 doc="Maximum number of iterations in full model fit for FULLCOVARIANCE ptcFitType",
145 default=3,
146 )
147 polynomialFitDegree = pexConfig.Field(
148 dtype=int,
149 doc="Degree of polynomial to fit the PTC, when 'ptcFitType'=POLYNOMIAL.",
150 default=3,
151 )
152 doLegacyTurnoffSelection = pexConfig.Field(
153 dtype=bool,
154 doc="Use 'legacy' computation for PTC turnoff selection. If set "
155 "to False, then the KS test p-value selection will be used instead.",
156 default=False,
157 )
158 sigmaCutPtcOutliers = pexConfig.Field(
159 dtype=float,
160 doc="Sigma cut for outlier rejection in PTC.",
161 default=5.0,
162 )
163 maxIterationsPtcOutliers = pexConfig.RangeField(
164 dtype=int,
165 doc="Maximum number of iterations for outlier rejection in PTC.",
166 default=2,
167 min=0
168 )
169 maxSignalInitialPtcOutlierFit = pexConfig.Field(
170 dtype=float,
171 doc="Maximum signal considered for intial outlier fit. This should be below "
172 "the PTC turnoff to ensure accurate outlier rejection. If "
173 "scaleMaxSignalInitialPtcOutlierFit=True then the units are electrons; "
174 "otherwise ADU.",
175 default=50_000.,
176 )
177 scaleMaxSignalInitialPtcOutlierFit = pexConfig.Field(
178 dtype=bool,
179 doc="Scale maxSignalInitialPtcOutlierFit by approximate gain? If yes then "
180 "maxSignalInitialPtcOutlierFit is assumed to have units of electrons, "
181 "otherwise ADU.",
182 default=True,
183 )
184 minVarPivotSearch = pexConfig.Field(
185 dtype=float,
186 doc="The code looks for a pivot signal point after which the variance starts decreasing at high-flux"
187 " to exclude then from the PTC model fit. However, sometimes at low fluxes, the variance"
188 " decreases slightly. Set this variable for the variance value, in ADU^2, after which the pivot "
189 " should be sought. Only used if doLegacyTurnoffSelection is True.",
190 default=10000,
191 )
192 consecutivePointsVarDecreases = pexConfig.RangeField(
193 dtype=int,
194 doc="Required number of consecutive points/fluxes in the PTC where the variance "
195 "decreases in order to find a first estimate of the PTC turn-off. "
196 "Only used if doLegacyTurnoffSelection is True.",
197 default=2,
198 min=2
199 )
200 ksTestMinPvalue = pexConfig.Field(
201 dtype=float,
202 doc="Minimum value of the Gaussian histogram KS test p-value to be used in PTC fit. "
203 "Only used if doLegacyTurnoffSelection is False.",
204 default=0.01,
205 )
206 doFitBootstrap = pexConfig.Field(
207 dtype=bool,
208 doc="Use bootstrap for the PTC fit parameters and errors?.",
209 default=False,
210 )
211 binSize = pexConfig.Field(
212 dtype=int,
213 doc="Bin the image by this factor in both dimensions.",
214 default=1,
215 )
217 def validate(self):
218 super().validate()
219 fitMatrixSide = self.maximumRangeCovariancesAstierFullCovFit
220 measureMatrixSide = self.maximumRangeCovariancesAstier
221 if self.ptcFitType == "FULLCOVARIANCE":
222 if fitMatrixSide > measureMatrixSide:
223 raise RuntimeError("Covariance fit size %s is larger than"
224 "measurement size %s.",
225 fitMatrixSide, measureMatrixSide)
226 if self.doSubtractLongRangeCovariances:
227 startLag = self.startLongRangeCovariances
228 if measureMatrixSide < startLag:
229 raise RuntimeError("Covariance measure size %s is smaller than long"
230 "-range covariance starting point %s.",
231 measureMatrixSide, startLag)
234class PhotonTransferCurveSolveTask(pipeBase.PipelineTask):
235 """Task to fit the PTC from flat covariances.
237 The first task of the PTC measurement pipeline,
238 ``PhotonTransferCurveMeasureTask`` (and assumed to have been run
239 before this task), produced a list of
240 `~lsst.ip.isr.PhotonTransferCurveDataset` objects. Each dataset
241 contains the mean signal and covariances of the
242 difference image of the flat-field images taken at
243 the same exposure time. The list also contains dummy
244 datasets (with no measurements), whose purpose is to have
245 the input and output dimensions of ``PhotonTransferCurveMeasureTask``
246 match.
248 This task, ``PhotonTransferCurveSolveTask``, assembles the list
249 of individual PTC datasets produced
250 by ``PhotonTransferCurveMeasureTask`` into one single final PTC
251 dataset, discarding the dummy datset as appropiate.
252 The task fits the measured (co)variances to one of three models:
253 a polynomial model of a given order, or the models described
254 in equations 16 and 20 of Astier+19. These options are referred
255 to as ``POLYNOMIAL``, ``EXPAPPROXIMATION``, and ``FULLCOVARIANCE``
256 in the configuration options of the task, respectively).
257 Parameters of interest such as the gain and noise are derived
258 from the fits. The ``FULLCOVARIANCE`` model is fitted to the
259 full covariance data (as oppossed to the other two models, which
260 are fit to the variance vs mean measurements only).
262 Astier+19: "The Shape of the Photon Transfer Curve
263 of CCD sensors", arXiv:1905.08677
264 """
266 ConfigClass = PhotonTransferCurveSolveConfig
267 _DefaultName = 'cpPhotonTransferCurveSolve'
269 def runQuantum(self, butlerQC, inputRefs, outputRefs):
270 """Ensure that the input and output dimensions are passed along.
272 Parameters
273 ----------
274 butlerQC : `~lsst.daf.butler.QuantumContext`
275 Butler to operate on.
276 inputRefs : `~lsst.pipe.base.InputQuantizedConnection`
277 Input data refs to load.
278 ouptutRefs : `~lsst.pipe.base.OutputQuantizedConnection`
279 Output data refs to persist.
280 """
281 inputs = butlerQC.get(inputRefs)
282 detId = inputRefs.inputCovariances[0].dataId['detector']
283 outputs = self.run(inputCovariances=inputs['inputCovariances'], camera=inputs['camera'], detId=detId)
284 butlerQC.put(outputs, outputRefs)
286 def run(self, inputCovariances, camera=None, detId=0):
287 """Fit measured covariances to different models.
289 Parameters
290 ----------
291 inputCovariances : `list` [`lsst.ip.isr.PhotonTransferCurveDataset`]
292 List of lsst.ip.isr.PhotonTransferCurveDataset datasets.
293 camera : `lsst.afw.cameraGeom.Camera`, optional
294 Input camera.
295 detId : `int`
296 Detector ID to locate the detector in the camera and
297 populate the `lsst.ip.isr.PhotonTransferCurveDataset`
298 metadata.
299 Returns
300 -------
301 results : `lsst.pipe.base.Struct`
302 The resultins structure contains:
304 ``outputPtcDatset``
305 Final PTC dataset, containing information such as the
306 means, variances, and exposure times
307 (`lsst.ip.isr.PhotonTransferCurveDataset`).
308 """
309 # Find the ampNames from a non-dummy ptc.
310 ampNames = []
311 for partialPtcDataset in inputCovariances:
312 if partialPtcDataset.ptcFitType != 'DUMMY':
313 ampNames = partialPtcDataset.ampNames
314 break
316 # Each amp may have a different min and max ADU signal
317 # specified in the config.
318 maxMeanSignalDict = {ampName: 1e6 for ampName in ampNames}
319 minMeanSignalDict = {ampName: 0.0 for ampName in ampNames}
320 for ampName in ampNames:
321 if 'ALL_AMPS' in self.config.maxMeanSignal:
322 maxMeanSignalDict[ampName] = self.config.maxMeanSignal['ALL_AMPS']
323 elif ampName in self.config.maxMeanSignal:
324 maxMeanSignalDict[ampName] = self.config.maxMeanSignal[ampName]
326 if 'ALL_AMPS' in self.config.minMeanSignal:
327 minMeanSignalDict[ampName] = self.config.minMeanSignal['ALL_AMPS']
328 elif ampName in self.config.minMeanSignal:
329 minMeanSignalDict[ampName] = self.config.minMeanSignal[ampName]
331 # Assemble individual PTC datasets into a single PTC dataset.
332 datasetPtc = PhotonTransferCurveDataset(
333 ampNames=ampNames,
334 ptcFitType=self.config.ptcFitType,
335 covMatrixSide=self.config.maximumRangeCovariancesAstier,
336 covMatrixSideFullCovFit=self.config.maximumRangeCovariancesAstierFullCovFit)
338 for partialPtcDataset in inputCovariances:
339 # Ignore dummy datasets
340 if partialPtcDataset.ptcFitType == 'DUMMY':
341 continue
342 for ampName in ampNames:
343 # The partial dataset consists of lists of values for each
344 # quantity. In the case of the input exposure pairs, this is a
345 # list of tuples. In all cases we only want the first
346 # (and only) element of the list.
347 datasetPtc.inputExpIdPairs[ampName].append(partialPtcDataset.inputExpIdPairs[ampName][0])
348 datasetPtc.rawExpTimes[ampName] = np.append(datasetPtc.rawExpTimes[ampName],
349 partialPtcDataset.rawExpTimes[ampName][0])
350 datasetPtc.rawMeans[ampName] = np.append(datasetPtc.rawMeans[ampName],
351 partialPtcDataset.rawMeans[ampName][0])
352 datasetPtc.rawVars[ampName] = np.append(datasetPtc.rawVars[ampName],
353 partialPtcDataset.rawVars[ampName][0])
354 datasetPtc.rowMeanVariance[ampName] = np.append(datasetPtc.rowMeanVariance[ampName],
355 partialPtcDataset.rowMeanVariance[ampName][0])
356 datasetPtc.photoCharges[ampName] = np.append(datasetPtc.photoCharges[ampName],
357 partialPtcDataset.photoCharges[ampName][0])
358 datasetPtc.histVars[ampName] = np.append(datasetPtc.histVars[ampName],
359 partialPtcDataset.histVars[ampName][0])
360 datasetPtc.histChi2Dofs[ampName] = np.append(datasetPtc.histChi2Dofs[ampName],
361 partialPtcDataset.histChi2Dofs[ampName][0])
362 datasetPtc.kspValues[ampName] = np.append(datasetPtc.kspValues[ampName],
363 partialPtcDataset.kspValues[ampName][0])
364 datasetPtc.noiseList[ampName] = np.append(datasetPtc.noiseList[ampName],
365 partialPtcDataset.noise[ampName])
366 datasetPtc.covariances[ampName] = np.append(
367 datasetPtc.covariances[ampName].ravel(),
368 partialPtcDataset.covariances[ampName].ravel()
369 ).reshape(
370 (
371 len(datasetPtc.rawExpTimes[ampName]),
372 datasetPtc.covMatrixSide,
373 datasetPtc.covMatrixSide,
374 )
375 )
376 datasetPtc.covariancesSqrtWeights[ampName] = np.append(
377 datasetPtc.covariancesSqrtWeights[ampName].ravel(),
378 partialPtcDataset.covariancesSqrtWeights[ampName].ravel()
379 ).reshape(
380 (
381 len(datasetPtc.rawExpTimes[ampName]),
382 datasetPtc.covMatrixSide,
383 datasetPtc.covMatrixSide,
384 )
385 )
387 # Apply min/max masking.
388 rawMean = partialPtcDataset.rawMeans[ampName][0]
389 rawVar = partialPtcDataset.rawVars[ampName][0]
390 expIdMask = partialPtcDataset.expIdMask[ampName][0]
391 if (rawMean <= minMeanSignalDict[ampName]) or (rawMean >= maxMeanSignalDict[ampName]) \
392 or not np.isfinite(rawMean) or not np.isfinite(rawVar):
393 expIdMask = False
395 kspValue = partialPtcDataset.kspValues[ampName][0]
396 if not self.config.doLegacyTurnoffSelection and \
397 kspValue < self.config.ksTestMinPvalue:
398 expIdMask = False
400 datasetPtc.expIdMask[ampName] = np.append(datasetPtc.expIdMask[ampName], expIdMask)
402 for key, value in partialPtcDataset.auxValues.items():
403 if key in datasetPtc.auxValues:
404 datasetPtc.auxValues[key] = np.append(datasetPtc.auxValues[key], value)
405 else:
406 datasetPtc.auxValues[key] = value
408 # Sort arrays that are filled so far in the final dataset by
409 # rawMeans index.
410 # First compute the mean across all the amps to make sure that they are
411 # all sorted the same way.
412 detectorMeans = np.zeros(len(datasetPtc.inputExpIdPairs[ampNames[0]]))
414 for i in range(len(detectorMeans)):
415 arr = np.array([datasetPtc.rawMeans[ampName][i] for ampName in ampNames])
416 good, = (np.isfinite(arr)).nonzero()
417 if good.size == 0:
418 detectorMeans[i] = np.nan
419 else:
420 detectorMeans[i] = np.mean(arr[good])
422 index = np.argsort(detectorMeans)
424 for ampName in ampNames:
425 datasetPtc.inputExpIdPairs[ampName] = np.array(
426 datasetPtc.inputExpIdPairs[ampName]
427 )[index].tolist()
428 datasetPtc.rawExpTimes[ampName] = datasetPtc.rawExpTimes[ampName][index]
429 datasetPtc.rawMeans[ampName] = datasetPtc.rawMeans[ampName][index]
430 datasetPtc.rawVars[ampName] = datasetPtc.rawVars[ampName][index]
431 datasetPtc.rowMeanVariance[ampName] = datasetPtc.rowMeanVariance[ampName][index]
432 datasetPtc.photoCharges[ampName] = datasetPtc.photoCharges[ampName][index]
433 datasetPtc.histVars[ampName] = datasetPtc.histVars[ampName][index]
434 datasetPtc.histChi2Dofs[ampName] = datasetPtc.histChi2Dofs[ampName][index]
435 datasetPtc.kspValues[ampName] = datasetPtc.kspValues[ampName][index]
436 datasetPtc.expIdMask[ampName] = datasetPtc.expIdMask[ampName][index]
437 datasetPtc.covariances[ampName] = datasetPtc.covariances[ampName][index]
438 datasetPtc.covariancesSqrtWeights[ampName] = datasetPtc.covariancesSqrtWeights[ampName][index]
439 for key, value in datasetPtc.auxValues.items():
440 datasetPtc.auxValues[key] = value[index]
442 if self.config.ptcFitType == "FULLCOVARIANCE":
443 # Fit the measured covariances vs mean signal to
444 # the Astier+19 full model (Eq. 20). Before that
445 # do a preliminary fit to the variance (C_00) vs mean
446 # signal (mu) curve using the EXPAPPROXIMATION model
447 # (Eq. 16 in Astier+19) in order to
448 # get the flat pairs that are masked. The
449 # points at these fluxes will also be masked when
450 # calculating the other elements of the covariance
451 # matrix, C_ij, i!=j).
453 # Preliminary fit, usign a temp dataset to get the mask
454 tempDatasetPtc = copy.copy(datasetPtc)
455 tempDatasetPtc.ptcFitType = "EXPAPPROXIMATION"
456 tempDatasetPtc = self.fitMeasurementsToModel(tempDatasetPtc)
458 # "FULLCOVARIANCE", using the mask obtained from the
459 # previous fit.
460 for ampName in datasetPtc.ampNames:
461 datasetPtc.expIdMask[ampName] = tempDatasetPtc.expIdMask[ampName]
462 datasetPtc.fitType = "FULLCOVARIANCE"
463 datasetPtc = self.fitMeasurementsToModel(datasetPtc)
464 # The other options are: self.config.ptcFitType in
465 # ("EXPAPPROXIMATION", "POLYNOMIAL")
466 else:
467 # Fit the PTC to a polynomial or to Astier+19 exponential
468 # approximation (Eq. 16). Fill up
469 # PhotonTransferCurveDataset object.
470 datasetPtc = self.fitMeasurementsToModel(datasetPtc)
472 # Initial validation of PTC fit.
473 for ampName in ampNames:
474 noise = np.nanmedian(datasetPtc.noiseList[ampName])
475 noiseFitted = np.sqrt(datasetPtc.noise[ampName])
477 # Check if noise is close to noiseFitted
478 if not np.isclose(noiseFitted, noise, rtol=0.05, atol=0.0):
479 self.log.warning(f"Read noise from PTC fit ({noiseFitted}) is not consistent "
480 f"with read noise measured from overscan ({noise}) for "
481 f"amplifier {ampName}. Try adjusting the fit range.")
483 if camera:
484 detector = camera[detId]
485 else:
486 detector = None
487 datasetPtc.updateMetadataFromExposures(inputCovariances)
488 datasetPtc.updateMetadata(setDate=True, camera=camera, detector=detector)
490 return pipeBase.Struct(
491 outputPtcDataset=datasetPtc,
492 )
494 def fitMeasurementsToModel(self, dataset):
495 """Fit the measured covariances vs mean signal to a
496 polynomial or one of the models in Astier+19
497 (Eq. 16 or Eq.20).
499 Parameters
500 ----------
501 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset`
502 The dataset containing information such as the means,
503 (co)variances, and exposure times.
505 Returns
506 -------
507 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset`
508 This is the same dataset as the input parameter, however,
509 it has been modified to include information such as the
510 fit vectors and the fit parameters. See the class
511 `PhotonTransferCurveDatase`.
512 """
513 fitType = dataset.ptcFitType
514 if fitType in ["FULLCOVARIANCE", ]:
515 # This model uses the full covariance matrix in the fit.
516 # The PTC is technically defined as variance vs signal,
517 # with variance = Cov_00
518 dataset = self.fitDataFullCovariance(dataset)
519 elif fitType in ["POLYNOMIAL", "EXPAPPROXIMATION"]:
520 # The PTC is technically defined as variance vs signal
521 dataset = self.fitPtc(dataset)
522 else:
523 raise RuntimeError(
524 f"Fitting option {fitType} not one of "
525 "'POLYNOMIAL', 'EXPAPPROXIMATION', or 'FULLCOVARIANCE'"
526 )
528 return dataset
530 def fitDataFullCovariance(self, dataset):
531 """Fit measured flat covariances to the full model in
532 Astier+19 (Eq. 20).
534 Parameters
535 ----------
536 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset`
537 The dataset containing information such as the means,
538 (co)variances, and exposure times.
540 Returns
541 -------
542 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset`
543 This is the same dataset as the input parameter, however,
544 it has been modified to include information such as the
545 fit vectors and the fit parameters. See the class
546 `PhotonTransferCurveDatase`.
548 Notes
549 -----
550 The parameters of the full model for C_ij(mu) ("C_ij" and "mu"
551 in ADU^2 and ADU, respectively) in Astier+19 (Eq. 20) are:
553 - "a" coefficients (r by r matrix), units: 1/e
554 - "b" coefficients (r by r matrix), units: 1/e
555 - noise matrix (r by r matrix), units: e^2
556 - gain, units: e/ADU
558 "b" appears in Eq. 20 only through the "ab" combination, which
559 is defined in this code as "c=ab".
561 Total number of parameters: #entries(a) + #entries(c) + #entries(noise)
562 + 1. This is equivalent to r^2 + r^2 + r^2 + 1, where "r" is the
563 maximum lag considered for the covariances calculation, and the
564 extra "1" is the gain. If "b" is 0, then "c" is 0, and len(pInit) will
565 have r^2 fewer entries.
566 """
567 matrixSide = dataset.covMatrixSide
568 matrixSideFit = dataset.covMatrixSideFullCovFit
569 lenParams = matrixSideFit*matrixSideFit
571 for ampName in dataset.ampNames:
572 lenInputTimes = len(dataset.rawExpTimes[ampName])
573 # Not used when ptcFitType is 'FULLCOVARIANCE'
574 dataset.ptcFitPars[ampName] = np.array([np.nan])
575 dataset.ptcFitParsError[ampName] = np.array([np.nan])
576 dataset.ptcFitChiSq[ampName] = np.nan
578 if ampName in dataset.badAmps:
579 # Bad amp
580 # Entries need to have proper dimensions so read/write
581 # with astropy.Table works.
582 nanMatrixFit = np.full((matrixSideFit, matrixSideFit), np.nan)
583 listNanMatrix = np.full((lenInputTimes, matrixSide, matrixSide), np.nan)
584 listNanMatrixFit = np.full((lenInputTimes, matrixSideFit, matrixSideFit), np.nan)
585 dataset.covariancesModel[ampName] = listNanMatrixFit
586 dataset.covariancesSqrtWeights[ampName] = listNanMatrix
587 dataset.aMatrix[ampName] = nanMatrixFit
588 dataset.bMatrix[ampName] = nanMatrixFit
589 dataset.covariancesModelNoB[ampName] = listNanMatrixFit
590 dataset.aMatrixNoB[ampName] = nanMatrixFit
591 dataset.noiseMatrix[ampName] = nanMatrixFit
592 dataset.noiseMatrixNoB[ampName] = nanMatrixFit
594 dataset.expIdMask[ampName] = np.repeat(False, lenInputTimes)
595 dataset.gain[ampName] = np.nan
596 dataset.gainErr[ampName] = np.nan
597 dataset.noise[ampName] = np.nan
598 dataset.noiseErr[ampName] = np.nan
599 dataset.finalVars[ampName] = np.repeat(np.nan, lenInputTimes)
600 dataset.finalModelVars[ampName] = np.repeat(np.nan, lenInputTimes)
601 dataset.finalMeans[ampName] = np.repeat(np.nan, lenInputTimes)
602 continue
604 muAtAmp = dataset.rawMeans[ampName]
605 maskAtAmp = dataset.expIdMask[ampName]
606 if len(maskAtAmp) == 0:
607 maskAtAmp = np.repeat(True, len(muAtAmp))
609 muAtAmpMasked = muAtAmp[maskAtAmp]
610 covAtAmp = dataset.covariances[ampName]
611 covAtAmpMasked = np.nan_to_num(covAtAmp)[maskAtAmp]
612 covSqrtWeightsAtAmp = dataset.covariancesSqrtWeights[ampName]
613 covSqrtWeightsAtAmpMasked = np.nan_to_num(covSqrtWeightsAtAmp)[maskAtAmp]
615 # Subtract long-range covariances
616 if self.config.doSubtractLongRangeCovariances:
617 startLag = self.config.startLongRangeCovariances
618 covAtAmpMasked, covSqrtWeightsAtAmpMasked = self.subtractDistantOffset(
619 muAtAmpMasked, covAtAmpMasked,
620 covSqrtWeightsAtAmpMasked,
621 start=startLag,
622 degree=self.config.polyDegLongRangeCovariances)
624 # In principle, we could fit to a lag smaller than the measured
625 # covariances.
626 r = self.config.maximumRangeCovariancesAstierFullCovFit
627 covAtAmpForFitMasked = covAtAmpMasked[:, :r, :r]
628 covSqrtWeightsAtAmpForFitMasked = covSqrtWeightsAtAmpMasked[:, :r, :r]
630 # Initial fit, to approximate parameters, with c=0
631 a0, c0, noise0, gain0 = self.initialFitFullCovariance(
632 muAtAmpMasked,
633 covAtAmpForFitMasked,
634 covSqrtWeightsAtAmpForFitMasked
635 )
637 # Fit full model (Eq. 20 of Astier+19) and same model with
638 # b=0 (c=0 in this code)
639 pInit = np.concatenate((a0.ravel(), c0.ravel(), noise0.ravel(), np.array(gain0)), axis=None)
640 functionsDict = {'fullModel': self.funcFullCovarianceModel,
641 'fullModelNoB': self.funcFullCovarianceModelNoB}
642 fitResults = {'fullModel': {'a': [], 'c': [], 'noise': [], 'gain': [], 'paramsErr': []},
643 'fullModelNoB': {'a': [], 'c': [], 'noise': [], 'gain': [], 'paramsErr': []}}
644 for key in functionsDict:
645 params, paramsErr, _ = fitLeastSq(pInit, muAtAmpMasked,
646 covAtAmpForFitMasked.ravel(), functionsDict[key],
647 weightsY=covSqrtWeightsAtAmpForFitMasked.ravel())
648 a = params[:lenParams].reshape((matrixSideFit, matrixSideFit))
649 c = params[lenParams:2*lenParams].reshape((matrixSideFit, matrixSideFit))
650 noise = params[2*lenParams:3*lenParams].reshape((matrixSideFit, matrixSideFit))
651 gain = params[-1]
653 fitResults[key]['a'] = a
654 fitResults[key]['c'] = c
655 fitResults[key]['noise'] = noise
656 fitResults[key]['gain'] = gain
657 fitResults[key]['paramsErr'] = paramsErr
659 # Put the information in the PTC dataset
661 # Not used when ptcFitType is 'FULLCOVARIANCE'
662 dataset.ptcFitPars[ampName] = np.array([np.nan])
663 dataset.ptcFitParsError[ampName] = np.array([np.nan])
664 dataset.ptcFitChiSq[ampName] = np.nan
666 # Save full covariances, covariances models, and their weights.
667 # dataset.expIdMask is already full, but needs to be
668 # converted to bool.
669 dataset.expIdMask[ampName] = np.array(dataset.expIdMask[ampName], dtype=bool)
670 dataset.covariances[ampName] = covAtAmp
671 # We evaluate the covariance model everywhere, even the
672 # masked amps.
673 dataset.covariancesModel[ampName] = self.evalCovModel(muAtAmp,
674 fitResults['fullModel']['a'],
675 fitResults['fullModel']['c'],
676 fitResults['fullModel']['noise'],
677 fitResults['fullModel']['gain'])
678 dataset.covariancesSqrtWeights[ampName] = covSqrtWeightsAtAmp
679 dataset.aMatrix[ampName] = fitResults['fullModel']['a']
680 dataset.bMatrix[ampName] = fitResults['fullModel']['c']/fitResults['fullModel']['a']
681 dataset.covariancesModelNoB[ampName] = self.evalCovModel(muAtAmp,
682 fitResults['fullModelNoB']['a'],
683 fitResults['fullModelNoB']['c'],
684 fitResults['fullModelNoB']['noise'],
685 fitResults['fullModelNoB']['gain'],
686 setBtoZero=True)
687 dataset.aMatrixNoB[ampName] = fitResults['fullModelNoB']['a']
688 dataset.gain[ampName] = fitResults['fullModel']['gain']
689 dataset.gainErr[ampName] = fitResults['fullModel']['paramsErr'][-1]
690 readoutNoise = fitResults['fullModel']['noise'][0][0]
691 readoutNoiseSqrt = np.sqrt(np.fabs(readoutNoise))
692 dataset.noise[ampName] = readoutNoise
693 readoutNoiseSigma = fitResults['fullModel']['paramsErr'][2*lenParams]
694 dataset.noiseErr[ampName] = 0.5*(readoutNoiseSigma/np.fabs(readoutNoise))*readoutNoiseSqrt
695 dataset.noiseMatrix[ampName] = fitResults['fullModel']['noise']
696 dataset.noiseMatrixNoB[ampName] = fitResults['fullModelNoB']['noise']
698 dataset.finalVars[ampName] = covAtAmp[:, 0, 0]
699 dataset.finalModelVars[ampName] = dataset.covariancesModel[ampName][:, 0, 0]
700 dataset.finalMeans[ampName] = muAtAmp
702 return dataset
704 def initialFitFullCovariance(self, mu, cov, sqrtW):
705 """ Performs a crude parabolic fit of the data in order to start
706 the full fit close to the solution, setting b=0 (c=0) in Eq. 20
707 of Astier+19.
709 Parameters
710 ----------
711 mu : `numpy.array`, (N,)
712 Signal `mu` (ADU)
713 cov : `numpy.array`, (N, M, M)
714 Covariance arrays of size `(M, M)` (with
715 `M = config.maximumRangeCovariancesAstier`),
716 indexed by mean signal `mu`.
717 sqrtW : `numpy.array`, (N,)
718 Covariance weights, defined as 1./sqrt(Variances)
720 Returns
721 -------
722 a : `numpy.array`, (M, M)
723 "a" parameter per flux in Eq. 20 of Astier+19.
724 c : `numpy.array`, (M, M)
725 "c"="ab" parameter per flux in Eq. 20 of Astier+19.
726 noise : `numpy.array`, (M, M)
727 "noise" parameter per flux in Eq. 20 of Astier+19.
728 gain : `float`
729 Amplifier gain (e/ADU)
730 """
731 matrixSideFit = self.config.maximumRangeCovariancesAstierFullCovFit
733 # Initialize fit parameters
734 a = np.zeros((matrixSideFit, matrixSideFit))
735 c = np.zeros((matrixSideFit, matrixSideFit))
736 noise = np.zeros((matrixSideFit, matrixSideFit))
737 gain = 1.
739 # iterate the fit to account for higher orders
740 # the chi2 does not necessarily go down, so one could
741 # stop when it increases
742 oldChi2 = 1e30
743 for _ in range(5):
744 model = np.nan_to_num(self.evalCovModel(mu, a, c, noise, gain, setBtoZero=True))
745 # loop on lags
746 for i in range(matrixSideFit):
747 for j in range(matrixSideFit):
748 # fit a parabola for a given lag
749 parsFit = np.polyfit(mu, cov[:, i, j] - model[:, i, j],
750 2, w=sqrtW[:, i, j])
751 # model equation (Eq. 20) in Astier+19, with c=a*b=0:
752 a[i, j] += parsFit[0]
753 noise[i, j] += parsFit[2]
754 if i + j == 0:
755 gain = 1./(1/gain+parsFit[1])
756 weightedRes = (model - cov)*sqrtW
757 chi2 = (weightedRes.flatten()**2).sum()
758 if chi2 > oldChi2:
759 break
760 oldChi2 = chi2
762 return a, c, noise, gain
764 def funcFullCovarianceModel(self, params, x):
765 """Model to fit covariances from flat fields; Equation 20 of
766 Astier+19.
768 Parameters
769 ----------
770 params : `list`
771 Parameters of the model: aMatrix, CMatrix, noiseMatrix,
772 gain (e/ADU).
773 x : `numpy.array`, (N,)
774 Signal `mu` (ADU)
776 Returns
777 -------
778 y : `numpy.array`, (N,)
779 Covariance matrix.
780 """
781 matrixSideFit = self.config.maximumRangeCovariancesAstierFullCovFit
782 lenParams = matrixSideFit*matrixSideFit
783 aMatrix = params[:lenParams].reshape((matrixSideFit, matrixSideFit))
784 cMatrix = params[lenParams:2*lenParams].reshape((matrixSideFit, matrixSideFit))
785 noiseMatrix = params[2*lenParams:3*lenParams].reshape((matrixSideFit, matrixSideFit))
786 gain = params[-1]
788 return self.evalCovModel(x, aMatrix, cMatrix, noiseMatrix, gain).flatten()
790 def funcFullCovarianceModelNoB(self, params, x):
791 """Model to fit covariances from flat fields; Equation 20 of
792 Astier+19, with b=0 (equivalent to c=a*b=0 in this code).
794 Parameters
795 ----------
796 params : `list`
797 Parameters of the model: aMatrix, CMatrix, noiseMatrix,
798 gain (e/ADU).
799 x : `numpy.array`, (N,)
800 Signal mu (ADU)
802 Returns
803 -------
804 y : `numpy.array`, (N,)
805 Covariance matrix.
806 """
807 matrixSideFit = self.config.maximumRangeCovariancesAstierFullCovFit
808 lenParams = matrixSideFit*matrixSideFit
809 aMatrix = params[:lenParams].reshape((matrixSideFit, matrixSideFit))
810 cMatrix = params[lenParams:2*lenParams].reshape((matrixSideFit, matrixSideFit))
811 noiseMatrix = params[2*lenParams:3*lenParams].reshape((matrixSideFit, matrixSideFit))
812 gain = params[-1]
814 return self.evalCovModel(x, aMatrix, cMatrix, noiseMatrix, gain, setBtoZero=True).flatten()
816 def evalCovModel(self, mu, aMatrix, cMatrix, noiseMatrix, gain, setBtoZero=False):
817 """Computes full covariances model (Eq. 20 of Astier+19).
819 Parameters
820 ----------
821 mu : `numpy.array`, (N,)
822 List of mean signals.
823 aMatrix : `numpy.array`, (M, M)
824 "a" parameter per flux in Eq. 20 of Astier+19.
825 cMatrix : `numpy.array`, (M, M)
826 "c"="ab" parameter per flux in Eq. 20 of Astier+19.
827 noiseMatrix : `numpy.array`, (M, M)
828 "noise" parameter per flux in Eq. 20 of Astier+19.
829 gain : `float`
830 Amplifier gain (e/ADU)
831 setBtoZero=False : `bool`, optional
832 Set "b" parameter in full model (see Astier+19) to zero.
834 Returns
835 -------
836 covModel : `numpy.array`, (N, M, M)
837 Covariances model.
839 Notes
840 -----
841 By default, computes the covModel for the mu's stored(self.mu).
842 Returns cov[Nmu, M, M]. The variance for the PTC is
843 cov[:, 0, 0]. mu and cov are in ADUs and ADUs squared. To use
844 electrons for both, the gain should be set to 1. This routine
845 implements the model in Astier+19 (1905.08677).
846 The parameters of the full model for C_ij(mu) ("C_ij" and "mu"
847 in ADU^2 and ADU, respectively) in Astier+19 (Eq. 20) are:
849 - "a" coefficients (M by M matrix), units: 1/e
850 - "b" coefficients (M by M matrix), units: 1/e
851 - noise matrix (M by M matrix), units: e^2
852 - gain, units: e/ADU
854 "b" appears in Eq. 20 only through the "ab" combination, which
855 is defined in this code as "c=ab".
856 """
857 matrixSideFit = self.config.maximumRangeCovariancesAstierFullCovFit
858 sa = (matrixSideFit, matrixSideFit)
859 # pad a with zeros and symmetrize
860 aEnlarged = np.zeros((int(sa[0]*1.5)+1, int(sa[1]*1.5)+1))
861 aEnlarged[0:sa[0], 0:sa[1]] = aMatrix
862 aSym = symmetrize(aEnlarged)
863 # pad c with zeros and symmetrize
864 cEnlarged = np.zeros((int(sa[0]*1.5)+1, int(sa[1]*1.5)+1))
865 cEnlarged[0:sa[0], 0:sa[1]] = cMatrix
866 cSym = symmetrize(cEnlarged)
867 a2 = fftconvolve(aSym, aSym, mode='same')
868 a3 = fftconvolve(a2, aSym, mode='same')
869 ac = fftconvolve(aSym, cSym, mode='same')
870 (xc, yc) = np.unravel_index(np.abs(aSym).argmax(), a2.shape)
872 a1 = aMatrix[np.newaxis, :, :]
873 a2 = a2[np.newaxis, xc:xc + matrixSideFit, yc:yc + matrixSideFit]
874 a3 = a3[np.newaxis, xc:xc + matrixSideFit, yc:yc + matrixSideFit]
875 ac = ac[np.newaxis, xc:xc + matrixSideFit, yc:yc + matrixSideFit]
876 c1 = cMatrix[np.newaxis, ::]
878 # assumes that mu is 1d
879 bigMu = mu[:, np.newaxis, np.newaxis]*gain
880 # c(=a*b in Astier+19) also has a contribution to the last
881 # term, that is absent for now.
882 if setBtoZero:
883 c1 = np.zeros_like(c1)
884 ac = np.zeros_like(ac)
885 covModel = (bigMu/(gain*gain)*(a1*bigMu+2./3.*(bigMu*bigMu)*(a2 + c1)
886 + (1./3.*a3 + 5./6.*ac)*(bigMu*bigMu*bigMu)) + noiseMatrix[np.newaxis, :, :]/gain**2)
887 # add the Poisson term, and the read out noise (variance)
888 covModel[:, 0, 0] += mu/gain
890 return covModel
892 def subtractDistantOffset(self, muAtAmpMasked, covAtAmpMasked, covSqrtWeightsAtAmpMasked,
893 start, degree=1):
894 """Subtract distant offset from the covariance matrices.
896 Parameters
897 ----------
898 muAtAmpMasked : `numpy.array`
899 Masked mean flux array for a particular amplifier.
900 covAtAmpMasked : `numpy.array`
901 Masked measured covariances for a particular amplifier.
902 covSqrtWeightsAtAmpMasked : `numpy.array`
903 Masked inverse covariance weights for a particular amplifier.
904 start : int, optional
905 The starting index to eliminate the core for the fit.
906 degree : int, optional
907 Degree of the polynomial fit.
909 Returns
910 -------
911 covAtAmpMasked : `numpy.array`
912 Subtracted measured covariances for a particular amplifier.
913 covSqrtWeightsAtAmpMasked : `numpy.array`
914 Masked inverse covariance weights for a particular amplifier.
916 Notes
917 -----
918 Ported from https://gitlab.in2p3.fr/astier/bfptc by P. Astier.
920 This function subtracts a distant offset from the
921 covariance matrices using polynomial fitting. The core
922 of the matrices is eliminated for the fit.
924 The function modifies the internal state of the object, updating the
925 covariance matrices and related attributes.
926 """
927 for k in range(len(muAtAmpMasked)):
928 # Make a copy because it will be altered
929 w = np.copy(covSqrtWeightsAtAmpMasked[k, ...])
930 wShape = w.shape
931 i, j = np.meshgrid(range(wShape[0]), range(wShape[1]), indexing='ij')
933 # Eliminate the core for the fit
934 w[:start, :start] = 0
936 poly = Pol2D(i, j, covAtAmpMasked[k, ...], degree, w=w)
937 back = poly.eval(i, j)
939 covAtAmpMasked[k, ...] -= back
941 return covAtAmpMasked, covSqrtWeightsAtAmpMasked
943 # EXPAPPROXIMATION and POLYNOMIAL fit methods
944 @staticmethod
945 def _initialParsForPolynomial(order):
946 assert order >= 2
947 pars = np.zeros(order, dtype=float)
948 pars[0] = 10
949 pars[1] = 1
950 pars[2:] = 0.0001
951 return pars
953 @staticmethod
954 def _boundsForPolynomial(initialPars, lowers=[], uppers=[]):
955 if not len(lowers):
956 lowers = [np.NINF for p in initialPars]
957 if not len(uppers):
958 uppers = [np.inf for p in initialPars]
959 lowers[1] = 0 # no negative gains
960 return (lowers, uppers)
962 @staticmethod
963 def _boundsForAstier(initialPars, lowers=[], uppers=[]):
964 if not len(lowers):
965 lowers = [np.NINF for p in initialPars]
966 if not len(uppers):
967 uppers = [np.inf for p in initialPars]
968 return (lowers, uppers)
970 @staticmethod
971 def _getInitialGoodPoints(means, variances, minVarPivotSearch, consecutivePointsVarDecreases):
972 """Return a boolean array to mask bad points.
974 Parameters
975 ----------
976 means : `numpy.array`
977 Input array with mean signal values.
978 variances : `numpy.array`
979 Input array with variances at each mean value.
980 minVarPivotSearch : `float`
981 The variance (in ADU^2), above which, the point
982 of decreasing variance should be sought.
983 consecutivePointsVarDecreases : `int`
984 Required number of consecutive points/fluxes
985 in the PTC where the variance
986 decreases in order to find a first
987 estimate of the PTC turn-off.
989 Returns
990 ------
991 goodPoints : `numpy.array` [`bool`]
992 Boolean array to select good (`True`) and bad (`False`)
993 points.
995 Notes
996 -----
997 Eliminate points beyond which the variance decreases.
998 """
999 goodPoints = np.ones_like(means, dtype=bool)
1000 # Variances are sorted and should monotonically increase
1001 pivotList = np.where(np.array(np.diff(variances)) < 0)[0]
1002 if len(pivotList) > 0:
1003 # For small values, sometimes the variance decreases slightly
1004 # Only look when var > self.config.minVarPivotSearch
1005 pivotList = [p for p in pivotList if variances[p] > minVarPivotSearch]
1006 # Require that the varince decreases during
1007 # consecutivePointsVarDecreases
1008 # consecutive points. This will give a first
1009 # estimate of the PTC turn-off, which
1010 # may be updated (reduced) further in the code.
1011 if len(pivotList) > 1:
1012 # enumerate(pivotList) creates tuples (index, value), for
1013 # each value in pivotList. The lambda function subtracts
1014 # each value from the index.
1015 # groupby groups elements by equal key value.
1016 for k, g in groupby(enumerate(pivotList), lambda x: x[0]-x[1]):
1017 group = (map(itemgetter(1), g))
1018 # Form groups of consecute values from pivotList
1019 group = list(map(int, group))
1020 # values in pivotList are indices where np.diff(variances)
1021 # is negative, i.e., where the variance starts decreasing.
1022 # Find the first group of consecutive numbers when
1023 # variance decreases.
1024 if len(group) >= consecutivePointsVarDecreases:
1025 pivotIndex = np.min(group)
1026 goodPoints[pivotIndex+1:] = False
1027 break
1029 # Finally, we filter out any infinities or NaNs.
1030 goodPoints[(~np.isfinite(means)) | (~np.isfinite(variances))] = False
1032 return goodPoints
1034 def _makeZeroSafe(self, array, substituteValue=1e-9):
1035 """"""
1036 array = np.array(array)
1037 nBad = Counter(np.ravel(array))[0]
1038 if nBad == 0:
1039 return array
1041 index, = np.where(array == 0)
1042 if len(index):
1043 msg = f"Found {nBad} zeros in array at elements {index}"
1044 self.log.warning(msg)
1046 array[index] = substituteValue
1048 return array
1050 def fitPtc(self, dataset):
1051 """Fit the photon transfer curve to a polynomial or to the
1052 Astier+19 approximation (Eq. 16).
1054 Fit the photon transfer curve with either a polynomial of
1055 the order specified in the task config (POLNOMIAL),
1056 or using the exponential approximation in Astier+19
1057 (Eq. 16, FULLCOVARIANCE).
1059 Sigma clipping is performed iteratively for the fit, as
1060 well as an initial clipping of data points that are more
1061 than `config.initialNonLinearityExclusionThreshold` away
1062 from lying on a straight line. This other step is necessary
1063 because the photon transfer curve turns over catastrophically
1064 at very high flux (because saturation
1065 drops the variance to ~0) and these far outliers cause the
1066 initial fit to fail, meaning the sigma cannot be calculated
1067 to perform the sigma-clipping.
1069 Parameters
1070 ----------
1071 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset`
1072 The dataset containing the means, variances and
1073 exposure times.
1075 Returns
1076 -------
1077 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset`
1078 This is the same dataset as the input parameter, however,
1079 it has been modified to include information such as the
1080 fit vectors and the fit parameters. See the class
1081 `PhotonTransferCurveDatase`.
1083 Raises
1084 ------
1085 RuntimeError
1086 Raised if dataset.ptcFitType is None or empty.
1087 """
1088 if dataset.ptcFitType:
1089 ptcFitType = dataset.ptcFitType
1090 else:
1091 raise RuntimeError("ptcFitType is None of empty in PTC dataset.")
1092 # For FULLCOVARIANCE model fit
1093 matrixSideFit = dataset.covMatrixSideFullCovFit
1094 nanMatrixFit = np.empty((matrixSideFit, matrixSideFit))
1095 nanMatrixFit[:] = np.nan
1097 for amp in dataset.ampNames:
1098 lenInputTimes = len(dataset.rawExpTimes[amp])
1099 listNanMatrixFit = np.empty((lenInputTimes, matrixSideFit, matrixSideFit))
1100 listNanMatrixFit[:] = np.nan
1102 dataset.covariancesModel[amp] = listNanMatrixFit
1103 dataset.aMatrix[amp] = nanMatrixFit
1104 dataset.bMatrix[amp] = nanMatrixFit
1105 dataset.covariancesModelNoB[amp] = listNanMatrixFit
1106 dataset.aMatrixNoB[amp] = nanMatrixFit
1107 dataset.noiseMatrix[amp] = nanMatrixFit
1108 dataset.noiseMatrixNoB[amp] = nanMatrixFit
1110 def errFunc(p, x, y):
1111 return ptcFunc(p, x) - y
1113 sigmaCutPtcOutliers = self.config.sigmaCutPtcOutliers
1114 maxIterationsPtcOutliers = self.config.maxIterationsPtcOutliers
1116 for i, ampName in enumerate(dataset.ampNames):
1117 meanVecOriginal = dataset.rawMeans[ampName].copy()
1118 varVecOriginal = dataset.rawVars[ampName].copy()
1119 varVecOriginal = self._makeZeroSafe(varVecOriginal)
1121 if self.config.doLegacyTurnoffSelection:
1122 # Discard points when the variance starts to decrease after two
1123 # consecutive signal levels
1124 goodPoints = self._getInitialGoodPoints(meanVecOriginal, varVecOriginal,
1125 self.config.minVarPivotSearch,
1126 self.config.consecutivePointsVarDecreases)
1127 else:
1128 goodPoints = dataset.expIdMask[ampName]
1130 # Check if all points are bad from the 'cpExtractPtcTask'
1131 initialExpIdMask = dataset.expIdMask[ampName]
1133 if not (goodPoints.any() and initialExpIdMask.any()):
1134 msg = (f"SERIOUS: All points in goodPoints: {goodPoints} or "
1135 f"in initialExpIdMask: {initialExpIdMask} are bad."
1136 f"Setting {ampName} to BAD.")
1137 self.log.warning(msg)
1138 # Fill entries with NaNs
1139 self.fillBadAmp(dataset, ptcFitType, ampName)
1140 continue
1142 mask = goodPoints
1144 if ptcFitType == 'EXPAPPROXIMATION':
1145 ptcFunc = funcAstier
1146 parsIniPtc = [-1e-9, 1.0, 10.] # a00, gain, noise^2
1147 # lowers and uppers obtained from BOT data studies by
1148 # C. Lage (UC Davis, 11/2020).
1149 if self.config.binSize > 1:
1150 bounds = self._boundsForAstier(parsIniPtc)
1151 else:
1152 bounds = self._boundsForAstier(parsIniPtc, lowers=[-1e-4, 0.1, -2000],
1153 uppers=[1e-4, 10.0, 2000])
1154 if ptcFitType == 'POLYNOMIAL':
1155 ptcFunc = funcPolynomial
1156 parsIniPtc = self._initialParsForPolynomial(self.config.polynomialFitDegree + 1)
1157 bounds = self._boundsForPolynomial(parsIniPtc)
1159 # We perform an initial (unweighted) fit of variance vs signal
1160 # (after initial KS test or post-drop selection) to look for
1161 # outliers, particularly at the high-flux end. The initial fit
1162 # is performed only for points that are guaranteed to be below
1163 # the PTC turnoff and then extrapolated to ensure that high
1164 # flux points that have abnormal variance values can be properly
1165 # rejected in this phase without biasing the initial fit.
1166 # This algorithm was initially developed by Seth Digel for
1167 # the EO Testing pipeline.
1169 if self.config.scaleMaxSignalInitialPtcOutlierFit:
1170 approxGain = np.nanmedian(meanVecOriginal/varVecOriginal)
1171 maxADUInitialPtcOutlierFit = self.config.maxSignalInitialPtcOutlierFit/approxGain
1172 self.log.info(
1173 "Using approximate gain %.3f and ADU signal cutoff of %.1f for amplifier %s",
1174 approxGain,
1175 maxADUInitialPtcOutlierFit,
1176 ampName,
1177 )
1178 else:
1179 maxADUInitialPtcOutlierFit = self.config.maxSignalInitialPtcOutlierFit
1181 if maxIterationsPtcOutliers == 0:
1182 # We are not doing any outlier rejection here, but we do want
1183 # an initial fit.
1184 res = least_squares(
1185 errFunc,
1186 parsIniPtc,
1187 bounds=bounds,
1188 args=(meanVecOriginal[mask], varVecOriginal[mask]),
1189 )
1190 pars = res.x
1191 newMask = mask.copy()
1192 else:
1193 newMask = (mask & (meanVecOriginal <= maxADUInitialPtcOutlierFit))
1195 count = 0
1196 lastMask = mask.copy()
1197 while count < maxIterationsPtcOutliers:
1198 res = least_squares(
1199 errFunc,
1200 parsIniPtc,
1201 bounds=bounds,
1202 args=(meanVecOriginal[newMask], varVecOriginal[newMask]),
1203 )
1204 pars = res.x
1206 sigResids = (varVecOriginal - ptcFunc(pars, meanVecOriginal))/np.sqrt(varVecOriginal)
1207 # The new mask includes points where the residuals are
1208 # finite, are less than the cut, and include the original
1209 # mask of known points that should not be used.
1210 newMask = (
1211 np.isfinite(sigResids)
1212 & (np.abs(np.nan_to_num(sigResids)) < sigmaCutPtcOutliers)
1213 & mask
1214 )
1215 if np.count_nonzero(newMask) == 0:
1216 msg = (f"SERIOUS: All points after outlier rejection are bad. "
1217 f"Setting {ampName} to BAD.")
1218 self.log.warning(msg)
1219 # Fill entries with NaNs
1220 self.fillBadAmp(dataset, ptcFitType, ampName)
1221 break
1223 self.log.debug(
1224 "Iteration %d: Removed %d points in total for %s.",
1225 count,
1226 np.count_nonzero(mask) - np.count_nonzero(newMask),
1227 ampName,
1228 )
1230 # If the mask hasn't changed then break out.
1231 if np.all(newMask == lastMask):
1232 self.log.debug("Convergence at iteration %d; breaking loop for %s.", count, ampName)
1233 break
1235 lastMask = newMask.copy()
1237 count += 1
1239 # Set the mask to the new mask
1240 mask = newMask.copy()
1242 if not mask.any():
1243 # We hae already filled the bad amp above, so continue.
1244 continue
1246 dataset.expIdMask[ampName] = mask
1248 parsIniPtc = pars
1249 meanVecFinal = meanVecOriginal[mask]
1250 varVecFinal = varVecOriginal[mask]
1252 # Save the maximum point after outlier detection as the
1253 # PTC turnoff point.
1254 dataset.ptcTurnoff[ampName] = meanVecFinal[-1]
1256 if Counter(mask)[False] > 0:
1257 self.log.info("Number of points discarded in PTC of amplifier %s:"
1258 " %d out of %d", ampName, Counter(mask)[False], len(meanVecOriginal))
1260 if (len(meanVecFinal) < len(parsIniPtc)):
1261 msg = (f"SERIOUS: Not enough data points ({len(meanVecFinal)}) compared to the number of "
1262 f"parameters of the PTC model({len(parsIniPtc)}). Setting {ampName} to BAD.")
1263 self.log.warning(msg)
1264 # Fill entries with NaNs
1265 self.fillBadAmp(dataset, ptcFitType, ampName)
1266 continue
1267 # Fit the PTC.
1268 # The variance of the variance is Var(v)=2*v^2/Npix. This is
1269 # already calculated in `makeCovArray` of CpPtcExtract.
1270 # dataset.covariancesSqrtWeights[ampName][:,0,0]
1271 # has 1/sqrt(Var(v)).
1272 weightsY = dataset.covariancesSqrtWeights[ampName][:, 0, 0][mask]
1273 if self.config.doFitBootstrap:
1274 parsFit, parsFitErr, reducedChiSqPtc = fitBootstrap(parsIniPtc, meanVecFinal,
1275 varVecFinal, ptcFunc,
1276 weightsY=weightsY)
1277 else:
1278 parsFit, parsFitErr, reducedChiSqPtc = fitLeastSq(parsIniPtc, meanVecFinal,
1279 varVecFinal, ptcFunc,
1280 weightsY=weightsY)
1281 dataset.ptcFitPars[ampName] = parsFit
1282 dataset.ptcFitParsError[ampName] = parsFitErr
1283 dataset.ptcFitChiSq[ampName] = reducedChiSqPtc
1285 dataset.finalVars[ampName] = varVecOriginal
1286 dataset.finalVars[ampName][~mask] = np.nan
1287 dataset.finalModelVars[ampName] = ptcFunc(parsFit, meanVecOriginal)
1288 dataset.finalModelVars[ampName][~mask] = np.nan
1289 dataset.finalMeans[ampName] = meanVecOriginal
1290 dataset.finalMeans[ampName][~mask] = np.nan
1292 if ptcFitType == 'EXPAPPROXIMATION':
1293 ptcGain = parsFit[1]
1294 ptcGainErr = parsFitErr[1]
1295 ptcNoise = np.sqrt(np.fabs(parsFit[2]))
1296 ptcNoiseErr = 0.5*(parsFitErr[2]/np.fabs(parsFit[2]))*np.sqrt(np.fabs(parsFit[2]))
1297 if ptcFitType == 'POLYNOMIAL':
1298 ptcGain = 1./parsFit[1]
1299 ptcGainErr = np.fabs(1./parsFit[1])*(parsFitErr[1]/parsFit[1])
1300 ptcNoise = np.sqrt(np.fabs(parsFit[0]))*ptcGain
1301 ptcNoiseErr = (0.5*(parsFitErr[0]/np.fabs(parsFit[0]))*(np.sqrt(np.fabs(parsFit[0]))))*ptcGain
1302 dataset.gain[ampName] = ptcGain
1303 dataset.gainErr[ampName] = ptcGainErr
1304 dataset.noise[ampName] = ptcNoise
1305 dataset.noiseErr[ampName] = ptcNoiseErr
1307 if not len(dataset.ptcFitType) == 0:
1308 dataset.ptcFitType = ptcFitType
1309 if len(dataset.badAmps) == 0:
1310 dataset.badAmps = []
1312 return dataset
1314 def fillBadAmp(self, dataset, ptcFitType, ampName):
1315 """Fill the dataset with NaNs if there are not enough
1316 good points.
1318 Parameters
1319 ----------
1320 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset`
1321 The dataset containing the means, variances and
1322 exposure times.
1323 ptcFitType : {'POLYNOMIAL', 'EXPAPPROXIMATION'}
1324 Fit a 'POLYNOMIAL' (degree: 'polynomialFitDegree') or
1325 'EXPAPPROXIMATION' (Eq. 16 of Astier+19) to the PTC.
1326 ampName : `str`
1327 Amplifier name.
1328 """
1329 dataset.badAmps.append(ampName)
1330 dataset.expIdMask[ampName] = np.repeat(False, len(dataset.rawExpTimes[ampName]))
1331 dataset.gain[ampName] = np.nan
1332 dataset.gainErr[ampName] = np.nan
1333 dataset.noise[ampName] = np.nan
1334 dataset.noiseErr[ampName] = np.nan
1335 dataset.ptcFitPars[ampName] = (np.repeat(np.nan, self.config.polynomialFitDegree + 1) if
1336 ptcFitType in ["POLYNOMIAL", ] else np.repeat(np.nan, 3))
1337 dataset.ptcFitParsError[ampName] = (np.repeat(np.nan, self.config.polynomialFitDegree + 1) if
1338 ptcFitType in ["POLYNOMIAL", ] else np.repeat(np.nan, 3))
1339 dataset.ptcFitChiSq[ampName] = np.nan
1340 dataset.ptcTurnoff[ampName] = np.nan
1341 dataset.finalVars[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName]))
1342 dataset.finalModelVars[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName]))
1343 dataset.finalMeans[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName]))
1345 return