Coverage for python/lsst/cp/pipe/ptc/cpSolvePtcTask.py: 10%
532 statements
« prev ^ index » next coverage.py v7.5.1, created at 2024-05-15 02:24 -0700
« prev ^ index » next coverage.py v7.5.1, created at 2024-05-15 02:24 -0700
1# This file is part of cp_pipe.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21#
22import numpy as np
23from collections import Counter
24import warnings
26import lsst.pex.config as pexConfig
27import lsst.pipe.base as pipeBase
28from lsst.cp.pipe.utils import (fitLeastSq, fitBootstrap, funcPolynomial,
29 funcAstier, symmetrize, Pol2D)
31from scipy.signal import fftconvolve
32from scipy.optimize import least_squares
33from itertools import groupby
34from operator import itemgetter
36import lsst.pipe.base.connectionTypes as cT
38from lsst.ip.isr import PhotonTransferCurveDataset
40import copy
43__all__ = ['PhotonTransferCurveSolveConfig', 'PhotonTransferCurveSolveTask']
46class PhotonTransferCurveSolveConnections(pipeBase.PipelineTaskConnections,
47 dimensions=("instrument", "detector")):
48 inputCovariances = cT.Input(
49 name="ptcCovariances",
50 doc="Tuple with measured covariances from flats.",
51 storageClass="PhotonTransferCurveDataset",
52 dimensions=("instrument", "exposure", "detector"),
53 isCalibration=True,
54 multiple=True,
55 )
56 camera = cT.PrerequisiteInput(
57 name="camera",
58 doc="Camera the input data comes from.",
59 storageClass="Camera",
60 dimensions=("instrument",),
61 isCalibration=True,
62 )
63 outputPtcDataset = cT.Output(
64 name="ptcDatsetProposal",
65 doc="Output proposed ptc dataset.",
66 storageClass="PhotonTransferCurveDataset",
67 dimensions=("instrument", "detector"),
68 multiple=False,
69 isCalibration=True,
70 )
73class PhotonTransferCurveSolveConfig(pipeBase.PipelineTaskConfig,
74 pipelineConnections=PhotonTransferCurveSolveConnections):
75 """Configuration for fitting measured covariances.
76 """
78 ptcFitType = pexConfig.ChoiceField(
79 dtype=str,
80 doc="Fit PTC to Eq. 16, Eq. 20 in Astier+19, or to a polynomial.",
81 default="POLYNOMIAL",
82 allowed={
83 "POLYNOMIAL": "n-degree polynomial (use 'polynomialFitDegree' to set 'n').",
84 "EXPAPPROXIMATION": "Approximation in Astier+19 (Eq. 16).",
85 "FULLCOVARIANCE": "Full covariances model in Astier+19 (Eq. 20)"
86 }
87 )
88 minMeanSignal = pexConfig.DictField(
89 keytype=str,
90 itemtype=float,
91 doc="Minimum values (inclusive) of mean signal (in ADU) per amp to use."
92 " The same cut is applied to all amps if this parameter [`dict`] is passed as "
93 " {'ALL_AMPS': value}",
94 default={'ALL_AMPS': 0.0},
95 )
96 maxMeanSignal = pexConfig.DictField(
97 keytype=str,
98 itemtype=float,
99 doc="Maximum values (inclusive) of mean signal (in ADU) below which to consider, per amp."
100 " The same cut is applied to all amps if this dictionary is of the form"
101 " {'ALL_AMPS': value}",
102 default={'ALL_AMPS': 1e6},
103 )
104 maximumRangeCovariancesAstier = pexConfig.Field(
105 dtype=int,
106 doc="Maximum range of measured covariances as in Astier+19",
107 default=8,
108 )
109 maximumRangeCovariancesAstierFullCovFit = pexConfig.Field(
110 dtype=int,
111 doc="Maximum range up to where to fit covariances as in Astier+19, "
112 "for the FULLCOVARIANCE model."
113 "This is different from maximumRangeCovariancesAstier."
114 "It should be less or equal than maximumRangeCovariancesAstier."
115 "The number of parameters for this model is "
116 "3*maximumRangeCovariancesAstierFullCovFit^2 + 1, so increase with care "
117 "so that the fit is not too slow.",
118 default=8,
119 )
120 doSubtractLongRangeCovariances = pexConfig.Field(
121 dtype=bool,
122 doc="Subtract long-range covariances before FULLCOVARIANCE fit, "
123 "beyond startLongRangeCovariances?",
124 default=False,
125 )
126 startLongRangeCovariances = pexConfig.Field(
127 dtype=int,
128 doc="If doSubtractLongRangeCovariances is True, subtract covariances "
129 "beyond this range. It should be less than maximumRangeCovariancesAstier. ",
130 default=4,
131 )
132 polyDegLongRangeCovariances = pexConfig.Field(
133 dtype=int,
134 doc="If doSubtractLongRangeCovariances is True, polynomial "
135 "degree to fit data beyond startLongRangeCovariances.",
136 default=1,
137 )
138 sigmaClipFullFitCovariancesAstier = pexConfig.Field(
139 dtype=float,
140 doc="sigma clip for full model fit for FULLCOVARIANCE ptcFitType ",
141 default=5.0,
142 )
143 maxIterFullFitCovariancesAstier = pexConfig.Field(
144 dtype=int,
145 doc="Maximum number of iterations in full model fit for FULLCOVARIANCE ptcFitType",
146 default=3,
147 )
148 polynomialFitDegree = pexConfig.Field(
149 dtype=int,
150 doc="Degree of polynomial to fit the PTC, when 'ptcFitType'=POLYNOMIAL.",
151 default=3,
152 )
153 doLegacyTurnoffSelection = pexConfig.Field(
154 dtype=bool,
155 doc="Use 'legacy' computation for PTC turnoff selection. If set "
156 "to False, then the KS test p-value selection will be used instead.",
157 default=False,
158 )
159 sigmaCutPtcOutliers = pexConfig.Field(
160 dtype=float,
161 doc="Sigma cut for outlier rejection in PTC.",
162 default=5.0,
163 )
164 maxIterationsPtcOutliers = pexConfig.RangeField(
165 dtype=int,
166 doc="Maximum number of iterations for outlier rejection in PTC.",
167 default=2,
168 min=0
169 )
170 maxSignalInitialPtcOutlierFit = pexConfig.Field(
171 dtype=float,
172 doc="Maximum signal considered for intial outlier fit. This should be below "
173 "the PTC turnoff to ensure accurate outlier rejection. If "
174 "scaleMaxSignalInitialPtcOutlierFit=True then the units are electrons; "
175 "otherwise ADU.",
176 default=50_000.,
177 )
178 maxDeltaInitialPtcOutlierFit = pexConfig.Field(
179 dtype=float,
180 doc="If there are any outliers in the initial fit that have mean greater than "
181 "maxSignalInitialPtcOutlierFit, then no points that have this delta "
182 "mean from the previous ``good`` point are allowed. If "
183 "scaleMaxSignalInitialPtcOutlierFit=True then the units are electrons; "
184 "otherwise ADU.",
185 default=9_000.,
186 )
187 scaleMaxSignalInitialPtcOutlierFit = pexConfig.Field(
188 dtype=bool,
189 doc="Scale maxSignalInitialPtcOutlierFit and maxDeltaInitialPtcOutlierFit by "
190 "approximate gain? If yes then "
191 "maxSignalInitialPtcOutlierFit and maxDeltaInitialPtcOutlierFit are assumed "
192 "to have units of electrons, otherwise ADU.",
193 default=True,
194 )
195 minVarPivotSearch = pexConfig.Field(
196 dtype=float,
197 doc="The code looks for a pivot signal point after which the variance starts decreasing at high-flux"
198 " to exclude then from the PTC model fit. However, sometimes at low fluxes, the variance"
199 " decreases slightly. Set this variable for the variance value, in ADU^2, after which the pivot "
200 " should be sought. Only used if doLegacyTurnoffSelection is True.",
201 default=10000,
202 )
203 consecutivePointsVarDecreases = pexConfig.RangeField(
204 dtype=int,
205 doc="Required number of consecutive points/fluxes in the PTC where the variance "
206 "decreases in order to find a first estimate of the PTC turn-off. "
207 "Only used if doLegacyTurnoffSelection is True.",
208 default=2,
209 min=2
210 )
211 ksTestMinPvalue = pexConfig.Field(
212 dtype=float,
213 doc="Minimum value of the Gaussian histogram KS test p-value to be used in PTC fit. "
214 "Only used if doLegacyTurnoffSelection is False.",
215 default=0.01,
216 )
217 doFitBootstrap = pexConfig.Field(
218 dtype=bool,
219 doc="Use bootstrap for the PTC fit parameters and errors?.",
220 default=False,
221 )
222 binSize = pexConfig.Field(
223 dtype=int,
224 doc="Bin the image by this factor in both dimensions.",
225 default=1,
226 )
228 def validate(self):
229 super().validate()
230 fitMatrixSide = self.maximumRangeCovariancesAstierFullCovFit
231 measureMatrixSide = self.maximumRangeCovariancesAstier
232 if self.ptcFitType == "FULLCOVARIANCE":
233 if fitMatrixSide > measureMatrixSide:
234 raise RuntimeError("Covariance fit size %s is larger than"
235 "measurement size %s.",
236 fitMatrixSide, measureMatrixSide)
237 if self.doSubtractLongRangeCovariances:
238 startLag = self.startLongRangeCovariances
239 if measureMatrixSide < startLag:
240 raise RuntimeError("Covariance measure size %s is smaller than long"
241 "-range covariance starting point %s.",
242 measureMatrixSide, startLag)
245class PhotonTransferCurveSolveTask(pipeBase.PipelineTask):
246 """Task to fit the PTC from flat covariances.
248 The first task of the PTC measurement pipeline,
249 ``PhotonTransferCurveMeasureTask`` (and assumed to have been run
250 before this task), produced a list of
251 `~lsst.ip.isr.PhotonTransferCurveDataset` objects. Each dataset
252 contains the mean signal and covariances of the
253 difference image of the flat-field images taken at
254 the same exposure time. The list also contains dummy
255 datasets (with no measurements), whose purpose is to have
256 the input and output dimensions of ``PhotonTransferCurveMeasureTask``
257 match.
259 This task, ``PhotonTransferCurveSolveTask``, assembles the list
260 of individual PTC datasets produced
261 by ``PhotonTransferCurveMeasureTask`` into one single final PTC
262 dataset, discarding the dummy datset as appropiate.
263 The task fits the measured (co)variances to one of three models:
264 a polynomial model of a given order, or the models described
265 in equations 16 and 20 of Astier+19. These options are referred
266 to as ``POLYNOMIAL``, ``EXPAPPROXIMATION``, and ``FULLCOVARIANCE``
267 in the configuration options of the task, respectively).
268 Parameters of interest such as the gain and noise are derived
269 from the fits. The ``FULLCOVARIANCE`` model is fitted to the
270 full covariance data (as oppossed to the other two models, which
271 are fit to the variance vs mean measurements only).
273 Astier+19: "The Shape of the Photon Transfer Curve
274 of CCD sensors", arXiv:1905.08677
275 """
277 ConfigClass = PhotonTransferCurveSolveConfig
278 _DefaultName = 'cpPhotonTransferCurveSolve'
280 def runQuantum(self, butlerQC, inputRefs, outputRefs):
281 """Ensure that the input and output dimensions are passed along.
283 Parameters
284 ----------
285 butlerQC : `~lsst.daf.butler.QuantumContext`
286 Butler to operate on.
287 inputRefs : `~lsst.pipe.base.InputQuantizedConnection`
288 Input data refs to load.
289 ouptutRefs : `~lsst.pipe.base.OutputQuantizedConnection`
290 Output data refs to persist.
291 """
292 inputs = butlerQC.get(inputRefs)
293 detId = inputRefs.inputCovariances[0].dataId['detector']
294 outputs = self.run(inputCovariances=inputs['inputCovariances'], camera=inputs['camera'], detId=detId)
295 butlerQC.put(outputs, outputRefs)
297 def run(self, inputCovariances, camera=None, detId=0):
298 """Fit measured covariances to different models.
300 Parameters
301 ----------
302 inputCovariances : `list` [`lsst.ip.isr.PhotonTransferCurveDataset`]
303 List of lsst.ip.isr.PhotonTransferCurveDataset datasets.
304 camera : `lsst.afw.cameraGeom.Camera`, optional
305 Input camera.
306 detId : `int`
307 Detector ID to locate the detector in the camera and
308 populate the `lsst.ip.isr.PhotonTransferCurveDataset`
309 metadata.
310 Returns
311 -------
312 results : `lsst.pipe.base.Struct`
313 The resultins structure contains:
315 ``outputPtcDatset``
316 Final PTC dataset, containing information such as the
317 means, variances, and exposure times
318 (`lsst.ip.isr.PhotonTransferCurveDataset`).
319 """
320 # Find the ampNames from a non-dummy ptc.
321 ampNames = []
322 for partialPtcDataset in inputCovariances:
323 if partialPtcDataset.ptcFitType != 'DUMMY':
324 ampNames = partialPtcDataset.ampNames
325 break
327 # Each amp may have a different min and max ADU signal
328 # specified in the config.
329 maxMeanSignalDict = {ampName: 1e6 for ampName in ampNames}
330 minMeanSignalDict = {ampName: 0.0 for ampName in ampNames}
331 for ampName in ampNames:
332 if 'ALL_AMPS' in self.config.maxMeanSignal:
333 maxMeanSignalDict[ampName] = self.config.maxMeanSignal['ALL_AMPS']
334 elif ampName in self.config.maxMeanSignal:
335 maxMeanSignalDict[ampName] = self.config.maxMeanSignal[ampName]
337 if 'ALL_AMPS' in self.config.minMeanSignal:
338 minMeanSignalDict[ampName] = self.config.minMeanSignal['ALL_AMPS']
339 elif ampName in self.config.minMeanSignal:
340 minMeanSignalDict[ampName] = self.config.minMeanSignal[ampName]
342 # Assemble individual PTC datasets into a single PTC dataset.
343 datasetPtc = PhotonTransferCurveDataset(
344 ampNames=ampNames,
345 ptcFitType=self.config.ptcFitType,
346 covMatrixSide=self.config.maximumRangeCovariancesAstier,
347 covMatrixSideFullCovFit=self.config.maximumRangeCovariancesAstierFullCovFit)
349 for partialPtcDataset in inputCovariances:
350 # Ignore dummy datasets
351 if partialPtcDataset.ptcFitType == 'DUMMY':
352 continue
353 for ampName in ampNames:
354 # The partial dataset consists of lists of values for each
355 # quantity. In the case of the input exposure pairs, this is a
356 # list of tuples. In all cases we only want the first
357 # (and only) element of the list.
358 datasetPtc.inputExpIdPairs[ampName].append(partialPtcDataset.inputExpIdPairs[ampName][0])
359 datasetPtc.rawExpTimes[ampName] = np.append(datasetPtc.rawExpTimes[ampName],
360 partialPtcDataset.rawExpTimes[ampName][0])
361 datasetPtc.rawMeans[ampName] = np.append(datasetPtc.rawMeans[ampName],
362 partialPtcDataset.rawMeans[ampName][0])
363 datasetPtc.rawVars[ampName] = np.append(datasetPtc.rawVars[ampName],
364 partialPtcDataset.rawVars[ampName][0])
365 datasetPtc.rowMeanVariance[ampName] = np.append(datasetPtc.rowMeanVariance[ampName],
366 partialPtcDataset.rowMeanVariance[ampName][0])
367 datasetPtc.photoCharges[ampName] = np.append(datasetPtc.photoCharges[ampName],
368 partialPtcDataset.photoCharges[ampName][0])
369 datasetPtc.histVars[ampName] = np.append(datasetPtc.histVars[ampName],
370 partialPtcDataset.histVars[ampName][0])
371 datasetPtc.histChi2Dofs[ampName] = np.append(datasetPtc.histChi2Dofs[ampName],
372 partialPtcDataset.histChi2Dofs[ampName][0])
373 datasetPtc.kspValues[ampName] = np.append(datasetPtc.kspValues[ampName],
374 partialPtcDataset.kspValues[ampName][0])
375 datasetPtc.noiseList[ampName] = np.append(datasetPtc.noiseList[ampName],
376 partialPtcDataset.noise[ampName])
377 datasetPtc.covariances[ampName] = np.append(
378 datasetPtc.covariances[ampName].ravel(),
379 partialPtcDataset.covariances[ampName].ravel()
380 ).reshape(
381 (
382 len(datasetPtc.rawExpTimes[ampName]),
383 datasetPtc.covMatrixSide,
384 datasetPtc.covMatrixSide,
385 )
386 )
387 datasetPtc.covariancesSqrtWeights[ampName] = np.append(
388 datasetPtc.covariancesSqrtWeights[ampName].ravel(),
389 partialPtcDataset.covariancesSqrtWeights[ampName].ravel()
390 ).reshape(
391 (
392 len(datasetPtc.rawExpTimes[ampName]),
393 datasetPtc.covMatrixSide,
394 datasetPtc.covMatrixSide,
395 )
396 )
398 # Apply min/max masking.
399 rawMean = partialPtcDataset.rawMeans[ampName][0]
400 rawVar = partialPtcDataset.rawVars[ampName][0]
401 expIdMask = partialPtcDataset.expIdMask[ampName][0]
402 if (rawMean <= minMeanSignalDict[ampName]) or (rawMean >= maxMeanSignalDict[ampName]) \
403 or not np.isfinite(rawMean) or not np.isfinite(rawVar):
404 expIdMask = False
406 kspValue = partialPtcDataset.kspValues[ampName][0]
407 if not self.config.doLegacyTurnoffSelection and \
408 kspValue < self.config.ksTestMinPvalue:
409 expIdMask = False
411 datasetPtc.expIdMask[ampName] = np.append(datasetPtc.expIdMask[ampName], expIdMask)
413 for key, value in partialPtcDataset.auxValues.items():
414 if key in datasetPtc.auxValues:
415 datasetPtc.auxValues[key] = np.append(datasetPtc.auxValues[key], value)
416 else:
417 datasetPtc.auxValues[key] = value
419 # Sort arrays that are filled so far in the final dataset by
420 # rawMeans index.
421 # First compute the mean across all the amps to make sure that they are
422 # all sorted the same way.
423 detectorMeans = np.zeros(len(datasetPtc.inputExpIdPairs[ampNames[0]]))
425 for i in range(len(detectorMeans)):
426 arr = np.array([datasetPtc.rawMeans[ampName][i] for ampName in ampNames])
427 good, = (np.isfinite(arr)).nonzero()
428 if good.size == 0:
429 detectorMeans[i] = np.nan
430 else:
431 detectorMeans[i] = np.mean(arr[good])
433 index = np.argsort(detectorMeans)
435 for ampName in ampNames:
436 datasetPtc.inputExpIdPairs[ampName] = np.array(
437 datasetPtc.inputExpIdPairs[ampName]
438 )[index].tolist()
439 datasetPtc.rawExpTimes[ampName] = datasetPtc.rawExpTimes[ampName][index]
440 datasetPtc.rawMeans[ampName] = datasetPtc.rawMeans[ampName][index]
441 datasetPtc.rawVars[ampName] = datasetPtc.rawVars[ampName][index]
442 datasetPtc.rowMeanVariance[ampName] = datasetPtc.rowMeanVariance[ampName][index]
443 datasetPtc.photoCharges[ampName] = datasetPtc.photoCharges[ampName][index]
444 datasetPtc.histVars[ampName] = datasetPtc.histVars[ampName][index]
445 datasetPtc.histChi2Dofs[ampName] = datasetPtc.histChi2Dofs[ampName][index]
446 datasetPtc.kspValues[ampName] = datasetPtc.kspValues[ampName][index]
447 datasetPtc.expIdMask[ampName] = datasetPtc.expIdMask[ampName][index]
448 datasetPtc.covariances[ampName] = datasetPtc.covariances[ampName][index]
449 datasetPtc.covariancesSqrtWeights[ampName] = datasetPtc.covariancesSqrtWeights[ampName][index]
450 for key, value in datasetPtc.auxValues.items():
451 datasetPtc.auxValues[key] = value[index]
453 if self.config.ptcFitType == "FULLCOVARIANCE":
454 # Fit the measured covariances vs mean signal to
455 # the Astier+19 full model (Eq. 20). Before that
456 # do a preliminary fit to the variance (C_00) vs mean
457 # signal (mu) curve using the EXPAPPROXIMATION model
458 # (Eq. 16 in Astier+19) in order to
459 # get the flat pairs that are masked. The
460 # points at these fluxes will also be masked when
461 # calculating the other elements of the covariance
462 # matrix, C_ij, i!=j).
464 # Preliminary fit, usign a temp dataset to get the mask
465 tempDatasetPtc = copy.copy(datasetPtc)
466 tempDatasetPtc.ptcFitType = "EXPAPPROXIMATION"
467 tempDatasetPtc = self.fitMeasurementsToModel(tempDatasetPtc)
469 # "FULLCOVARIANCE", using the mask obtained from the
470 # previous fit.
471 for ampName in datasetPtc.ampNames:
472 datasetPtc.expIdMask[ampName] = tempDatasetPtc.expIdMask[ampName]
473 datasetPtc.fitType = "FULLCOVARIANCE"
474 datasetPtc = self.fitMeasurementsToModel(datasetPtc)
475 # The other options are: self.config.ptcFitType in
476 # ("EXPAPPROXIMATION", "POLYNOMIAL")
477 else:
478 # Fit the PTC to a polynomial or to Astier+19 exponential
479 # approximation (Eq. 16). Fill up
480 # PhotonTransferCurveDataset object.
481 datasetPtc = self.fitMeasurementsToModel(datasetPtc)
483 # Initial validation of PTC fit.
484 for ampName in ampNames:
485 # These may be all nan (esp. in tests) and will be filtered
486 # as appropriate later.
487 with warnings.catch_warnings():
488 warnings.simplefilter("ignore")
489 noise = np.nanmedian(datasetPtc.noiseList[ampName])
490 noiseFitted = np.sqrt(datasetPtc.noise[ampName])
492 # Check if noise is close to noiseFitted
493 if not np.isclose(noiseFitted, noise, rtol=0.05, atol=0.0, equal_nan=True):
494 self.log.warning(f"Read noise from PTC fit ({noiseFitted}) is not consistent "
495 f"with read noise measured from overscan ({noise}) for "
496 f"amplifier {ampName}. Try adjusting the fit range.")
498 if camera:
499 detector = camera[detId]
500 else:
501 detector = None
502 datasetPtc.updateMetadataFromExposures(inputCovariances)
503 datasetPtc.updateMetadata(setDate=True, camera=camera, detector=detector)
505 return pipeBase.Struct(
506 outputPtcDataset=datasetPtc,
507 )
509 def fitMeasurementsToModel(self, dataset):
510 """Fit the measured covariances vs mean signal to a
511 polynomial or one of the models in Astier+19
512 (Eq. 16 or Eq.20).
514 Parameters
515 ----------
516 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset`
517 The dataset containing information such as the means,
518 (co)variances, and exposure times.
520 Returns
521 -------
522 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset`
523 This is the same dataset as the input parameter, however,
524 it has been modified to include information such as the
525 fit vectors and the fit parameters. See the class
526 `PhotonTransferCurveDatase`.
527 """
528 fitType = dataset.ptcFitType
529 if fitType in ["FULLCOVARIANCE", ]:
530 # This model uses the full covariance matrix in the fit.
531 # The PTC is technically defined as variance vs signal,
532 # with variance = Cov_00
533 dataset = self.fitDataFullCovariance(dataset)
534 elif fitType in ["POLYNOMIAL", "EXPAPPROXIMATION"]:
535 # The PTC is technically defined as variance vs signal
536 dataset = self.fitPtc(dataset)
537 else:
538 raise RuntimeError(
539 f"Fitting option {fitType} not one of "
540 "'POLYNOMIAL', 'EXPAPPROXIMATION', or 'FULLCOVARIANCE'"
541 )
543 return dataset
545 def fitDataFullCovariance(self, dataset):
546 """Fit measured flat covariances to the full model in
547 Astier+19 (Eq. 20).
549 Parameters
550 ----------
551 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset`
552 The dataset containing information such as the means,
553 (co)variances, and exposure times.
555 Returns
556 -------
557 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset`
558 This is the same dataset as the input parameter, however,
559 it has been modified to include information such as the
560 fit vectors and the fit parameters. See the class
561 `PhotonTransferCurveDatase`.
563 Notes
564 -----
565 The parameters of the full model for C_ij(mu) ("C_ij" and "mu"
566 in ADU^2 and ADU, respectively) in Astier+19 (Eq. 20) are:
568 - "a" coefficients (r by r matrix), units: 1/e
569 - "b" coefficients (r by r matrix), units: 1/e
570 - noise matrix (r by r matrix), units: e^2
571 - gain, units: e/ADU
573 "b" appears in Eq. 20 only through the "ab" combination, which
574 is defined in this code as "c=ab".
576 Total number of parameters: #entries(a) + #entries(c) + #entries(noise)
577 + 1. This is equivalent to r^2 + r^2 + r^2 + 1, where "r" is the
578 maximum lag considered for the covariances calculation, and the
579 extra "1" is the gain. If "b" is 0, then "c" is 0, and len(pInit) will
580 have r^2 fewer entries.
581 """
582 matrixSide = dataset.covMatrixSide
583 matrixSideFit = dataset.covMatrixSideFullCovFit
584 lenParams = matrixSideFit*matrixSideFit
586 for ampName in dataset.ampNames:
587 lenInputTimes = len(dataset.rawExpTimes[ampName])
588 # Not used when ptcFitType is 'FULLCOVARIANCE'
589 dataset.ptcFitPars[ampName] = np.array([np.nan])
590 dataset.ptcFitParsError[ampName] = np.array([np.nan])
591 dataset.ptcFitChiSq[ampName] = np.nan
593 if ampName in dataset.badAmps:
594 # Bad amp
595 # Entries need to have proper dimensions so read/write
596 # with astropy.Table works.
597 nanMatrixFit = np.full((matrixSideFit, matrixSideFit), np.nan)
598 listNanMatrix = np.full((lenInputTimes, matrixSide, matrixSide), np.nan)
599 listNanMatrixFit = np.full((lenInputTimes, matrixSideFit, matrixSideFit), np.nan)
600 dataset.covariancesModel[ampName] = listNanMatrixFit
601 dataset.covariancesSqrtWeights[ampName] = listNanMatrix
602 dataset.aMatrix[ampName] = nanMatrixFit
603 dataset.bMatrix[ampName] = nanMatrixFit
604 dataset.covariancesModelNoB[ampName] = listNanMatrixFit
605 dataset.aMatrixNoB[ampName] = nanMatrixFit
606 dataset.noiseMatrix[ampName] = nanMatrixFit
607 dataset.noiseMatrixNoB[ampName] = nanMatrixFit
609 dataset.expIdMask[ampName] = np.repeat(False, lenInputTimes)
610 dataset.gain[ampName] = np.nan
611 dataset.gainErr[ampName] = np.nan
612 dataset.noise[ampName] = np.nan
613 dataset.noiseErr[ampName] = np.nan
614 dataset.finalVars[ampName] = np.repeat(np.nan, lenInputTimes)
615 dataset.finalModelVars[ampName] = np.repeat(np.nan, lenInputTimes)
616 dataset.finalMeans[ampName] = np.repeat(np.nan, lenInputTimes)
617 continue
619 muAtAmp = dataset.rawMeans[ampName]
620 maskAtAmp = dataset.expIdMask[ampName]
621 if len(maskAtAmp) == 0:
622 maskAtAmp = np.repeat(True, len(muAtAmp))
624 muAtAmpMasked = muAtAmp[maskAtAmp]
625 covAtAmp = dataset.covariances[ampName]
626 covAtAmpMasked = np.nan_to_num(covAtAmp)[maskAtAmp]
627 covSqrtWeightsAtAmp = dataset.covariancesSqrtWeights[ampName]
628 covSqrtWeightsAtAmpMasked = np.nan_to_num(covSqrtWeightsAtAmp)[maskAtAmp]
630 # Subtract long-range covariances
631 if self.config.doSubtractLongRangeCovariances:
632 startLag = self.config.startLongRangeCovariances
633 covAtAmpMasked, covSqrtWeightsAtAmpMasked = self.subtractDistantOffset(
634 muAtAmpMasked, covAtAmpMasked,
635 covSqrtWeightsAtAmpMasked,
636 start=startLag,
637 degree=self.config.polyDegLongRangeCovariances)
639 # In principle, we could fit to a lag smaller than the measured
640 # covariances.
641 r = self.config.maximumRangeCovariancesAstierFullCovFit
642 covAtAmpForFitMasked = covAtAmpMasked[:, :r, :r]
643 covSqrtWeightsAtAmpForFitMasked = covSqrtWeightsAtAmpMasked[:, :r, :r]
645 # Initial fit, to approximate parameters, with c=0
646 a0, c0, noise0, gain0 = self.initialFitFullCovariance(
647 muAtAmpMasked,
648 covAtAmpForFitMasked,
649 covSqrtWeightsAtAmpForFitMasked
650 )
652 # Fit full model (Eq. 20 of Astier+19) and same model with
653 # b=0 (c=0 in this code)
654 pInit = np.concatenate((a0.ravel(), c0.ravel(), noise0.ravel(), np.array(gain0)), axis=None)
655 functionsDict = {'fullModel': self.funcFullCovarianceModel,
656 'fullModelNoB': self.funcFullCovarianceModelNoB}
657 fitResults = {'fullModel': {'a': [], 'c': [], 'noise': [], 'gain': [], 'paramsErr': []},
658 'fullModelNoB': {'a': [], 'c': [], 'noise': [], 'gain': [], 'paramsErr': []}}
659 for key in functionsDict:
660 params, paramsErr, _ = fitLeastSq(pInit, muAtAmpMasked,
661 covAtAmpForFitMasked.ravel(), functionsDict[key],
662 weightsY=covSqrtWeightsAtAmpForFitMasked.ravel())
663 a = params[:lenParams].reshape((matrixSideFit, matrixSideFit))
664 c = params[lenParams:2*lenParams].reshape((matrixSideFit, matrixSideFit))
665 noise = params[2*lenParams:3*lenParams].reshape((matrixSideFit, matrixSideFit))
666 gain = params[-1]
668 fitResults[key]['a'] = a
669 fitResults[key]['c'] = c
670 fitResults[key]['noise'] = noise
671 fitResults[key]['gain'] = gain
672 fitResults[key]['paramsErr'] = paramsErr
674 # Put the information in the PTC dataset
676 # Not used when ptcFitType is 'FULLCOVARIANCE'
677 dataset.ptcFitPars[ampName] = np.array([np.nan])
678 dataset.ptcFitParsError[ampName] = np.array([np.nan])
679 dataset.ptcFitChiSq[ampName] = np.nan
681 # Save full covariances, covariances models, and their weights.
682 # dataset.expIdMask is already full, but needs to be
683 # converted to bool.
684 dataset.expIdMask[ampName] = np.array(dataset.expIdMask[ampName], dtype=bool)
685 dataset.covariances[ampName] = covAtAmp
686 # We evaluate the covariance model everywhere, even the
687 # masked amps.
688 dataset.covariancesModel[ampName] = self.evalCovModel(muAtAmp,
689 fitResults['fullModel']['a'],
690 fitResults['fullModel']['c'],
691 fitResults['fullModel']['noise'],
692 fitResults['fullModel']['gain'])
693 dataset.covariancesSqrtWeights[ampName] = covSqrtWeightsAtAmp
694 dataset.aMatrix[ampName] = fitResults['fullModel']['a']
695 dataset.bMatrix[ampName] = fitResults['fullModel']['c']/fitResults['fullModel']['a']
696 dataset.covariancesModelNoB[ampName] = self.evalCovModel(muAtAmp,
697 fitResults['fullModelNoB']['a'],
698 fitResults['fullModelNoB']['c'],
699 fitResults['fullModelNoB']['noise'],
700 fitResults['fullModelNoB']['gain'],
701 setBtoZero=True)
702 dataset.aMatrixNoB[ampName] = fitResults['fullModelNoB']['a']
703 dataset.gain[ampName] = fitResults['fullModel']['gain']
704 dataset.gainErr[ampName] = fitResults['fullModel']['paramsErr'][-1]
705 readoutNoise = fitResults['fullModel']['noise'][0][0]
706 readoutNoiseSqrt = np.sqrt(np.fabs(readoutNoise))
707 dataset.noise[ampName] = readoutNoise
708 readoutNoiseSigma = fitResults['fullModel']['paramsErr'][2*lenParams]
709 dataset.noiseErr[ampName] = 0.5*(readoutNoiseSigma/np.fabs(readoutNoise))*readoutNoiseSqrt
710 dataset.noiseMatrix[ampName] = fitResults['fullModel']['noise']
711 dataset.noiseMatrixNoB[ampName] = fitResults['fullModelNoB']['noise']
713 dataset.finalVars[ampName] = covAtAmp[:, 0, 0]
714 dataset.finalModelVars[ampName] = dataset.covariancesModel[ampName][:, 0, 0]
715 dataset.finalMeans[ampName] = muAtAmp
717 return dataset
719 def initialFitFullCovariance(self, mu, cov, sqrtW):
720 """ Performs a crude parabolic fit of the data in order to start
721 the full fit close to the solution, setting b=0 (c=0) in Eq. 20
722 of Astier+19.
724 Parameters
725 ----------
726 mu : `numpy.array`, (N,)
727 Signal `mu` (ADU)
728 cov : `numpy.array`, (N, M, M)
729 Covariance arrays of size `(M, M)` (with
730 `M = config.maximumRangeCovariancesAstier`),
731 indexed by mean signal `mu`.
732 sqrtW : `numpy.array`, (N,)
733 Covariance weights, defined as 1./sqrt(Variances)
735 Returns
736 -------
737 a : `numpy.array`, (M, M)
738 "a" parameter per flux in Eq. 20 of Astier+19.
739 c : `numpy.array`, (M, M)
740 "c"="ab" parameter per flux in Eq. 20 of Astier+19.
741 noise : `numpy.array`, (M, M)
742 "noise" parameter per flux in Eq. 20 of Astier+19.
743 gain : `float`
744 Amplifier gain (e/ADU)
745 """
746 matrixSideFit = self.config.maximumRangeCovariancesAstierFullCovFit
748 # Initialize fit parameters
749 a = np.zeros((matrixSideFit, matrixSideFit))
750 c = np.zeros((matrixSideFit, matrixSideFit))
751 noise = np.zeros((matrixSideFit, matrixSideFit))
752 gain = 1.
754 # iterate the fit to account for higher orders
755 # the chi2 does not necessarily go down, so one could
756 # stop when it increases
757 oldChi2 = 1e30
758 for _ in range(5):
759 model = np.nan_to_num(self.evalCovModel(mu, a, c, noise, gain, setBtoZero=True))
760 # loop on lags
761 for i in range(matrixSideFit):
762 for j in range(matrixSideFit):
763 # fit a parabola for a given lag
764 parsFit = np.polyfit(mu, cov[:, i, j] - model[:, i, j],
765 2, w=sqrtW[:, i, j])
766 # model equation (Eq. 20) in Astier+19, with c=a*b=0:
767 a[i, j] += parsFit[0]
768 noise[i, j] += parsFit[2]
769 if i + j == 0:
770 gain = 1./(1/gain+parsFit[1])
771 weightedRes = (model - cov)*sqrtW
772 chi2 = (weightedRes.flatten()**2).sum()
773 if chi2 > oldChi2:
774 break
775 oldChi2 = chi2
777 return a, c, noise, gain
779 def funcFullCovarianceModel(self, params, x):
780 """Model to fit covariances from flat fields; Equation 20 of
781 Astier+19.
783 Parameters
784 ----------
785 params : `list`
786 Parameters of the model: aMatrix, CMatrix, noiseMatrix,
787 gain (e/ADU).
788 x : `numpy.array`, (N,)
789 Signal `mu` (ADU)
791 Returns
792 -------
793 y : `numpy.array`, (N,)
794 Covariance matrix.
795 """
796 matrixSideFit = self.config.maximumRangeCovariancesAstierFullCovFit
797 lenParams = matrixSideFit*matrixSideFit
798 aMatrix = params[:lenParams].reshape((matrixSideFit, matrixSideFit))
799 cMatrix = params[lenParams:2*lenParams].reshape((matrixSideFit, matrixSideFit))
800 noiseMatrix = params[2*lenParams:3*lenParams].reshape((matrixSideFit, matrixSideFit))
801 gain = params[-1]
803 return self.evalCovModel(x, aMatrix, cMatrix, noiseMatrix, gain).flatten()
805 def funcFullCovarianceModelNoB(self, params, x):
806 """Model to fit covariances from flat fields; Equation 20 of
807 Astier+19, with b=0 (equivalent to c=a*b=0 in this code).
809 Parameters
810 ----------
811 params : `list`
812 Parameters of the model: aMatrix, CMatrix, noiseMatrix,
813 gain (e/ADU).
814 x : `numpy.array`, (N,)
815 Signal mu (ADU)
817 Returns
818 -------
819 y : `numpy.array`, (N,)
820 Covariance matrix.
821 """
822 matrixSideFit = self.config.maximumRangeCovariancesAstierFullCovFit
823 lenParams = matrixSideFit*matrixSideFit
824 aMatrix = params[:lenParams].reshape((matrixSideFit, matrixSideFit))
825 cMatrix = params[lenParams:2*lenParams].reshape((matrixSideFit, matrixSideFit))
826 noiseMatrix = params[2*lenParams:3*lenParams].reshape((matrixSideFit, matrixSideFit))
827 gain = params[-1]
829 return self.evalCovModel(x, aMatrix, cMatrix, noiseMatrix, gain, setBtoZero=True).flatten()
831 def evalCovModel(self, mu, aMatrix, cMatrix, noiseMatrix, gain, setBtoZero=False):
832 """Computes full covariances model (Eq. 20 of Astier+19).
834 Parameters
835 ----------
836 mu : `numpy.array`, (N,)
837 List of mean signals.
838 aMatrix : `numpy.array`, (M, M)
839 "a" parameter per flux in Eq. 20 of Astier+19.
840 cMatrix : `numpy.array`, (M, M)
841 "c"="ab" parameter per flux in Eq. 20 of Astier+19.
842 noiseMatrix : `numpy.array`, (M, M)
843 "noise" parameter per flux in Eq. 20 of Astier+19.
844 gain : `float`
845 Amplifier gain (e/ADU)
846 setBtoZero=False : `bool`, optional
847 Set "b" parameter in full model (see Astier+19) to zero.
849 Returns
850 -------
851 covModel : `numpy.array`, (N, M, M)
852 Covariances model.
854 Notes
855 -----
856 By default, computes the covModel for the mu's stored(self.mu).
857 Returns cov[Nmu, M, M]. The variance for the PTC is
858 cov[:, 0, 0]. mu and cov are in ADUs and ADUs squared. To use
859 electrons for both, the gain should be set to 1. This routine
860 implements the model in Astier+19 (1905.08677).
861 The parameters of the full model for C_ij(mu) ("C_ij" and "mu"
862 in ADU^2 and ADU, respectively) in Astier+19 (Eq. 20) are:
864 - "a" coefficients (M by M matrix), units: 1/e
865 - "b" coefficients (M by M matrix), units: 1/e
866 - noise matrix (M by M matrix), units: e^2
867 - gain, units: e/ADU
869 "b" appears in Eq. 20 only through the "ab" combination, which
870 is defined in this code as "c=ab".
871 """
872 matrixSideFit = self.config.maximumRangeCovariancesAstierFullCovFit
873 sa = (matrixSideFit, matrixSideFit)
874 # pad a with zeros and symmetrize
875 aEnlarged = np.zeros((int(sa[0]*1.5)+1, int(sa[1]*1.5)+1))
876 aEnlarged[0:sa[0], 0:sa[1]] = aMatrix
877 aSym = symmetrize(aEnlarged)
878 # pad c with zeros and symmetrize
879 cEnlarged = np.zeros((int(sa[0]*1.5)+1, int(sa[1]*1.5)+1))
880 cEnlarged[0:sa[0], 0:sa[1]] = cMatrix
881 cSym = symmetrize(cEnlarged)
882 a2 = fftconvolve(aSym, aSym, mode='same')
883 a3 = fftconvolve(a2, aSym, mode='same')
884 ac = fftconvolve(aSym, cSym, mode='same')
885 (xc, yc) = np.unravel_index(np.abs(aSym).argmax(), a2.shape)
887 a1 = aMatrix[np.newaxis, :, :]
888 a2 = a2[np.newaxis, xc:xc + matrixSideFit, yc:yc + matrixSideFit]
889 a3 = a3[np.newaxis, xc:xc + matrixSideFit, yc:yc + matrixSideFit]
890 ac = ac[np.newaxis, xc:xc + matrixSideFit, yc:yc + matrixSideFit]
891 c1 = cMatrix[np.newaxis, ::]
893 # assumes that mu is 1d
894 bigMu = mu[:, np.newaxis, np.newaxis]*gain
895 # c(=a*b in Astier+19) also has a contribution to the last
896 # term, that is absent for now.
897 if setBtoZero:
898 c1 = np.zeros_like(c1)
899 ac = np.zeros_like(ac)
900 covModel = (bigMu/(gain*gain)*(a1*bigMu+2./3.*(bigMu*bigMu)*(a2 + c1)
901 + (1./3.*a3 + 5./6.*ac)*(bigMu*bigMu*bigMu)) + noiseMatrix[np.newaxis, :, :]/gain**2)
902 # add the Poisson term, and the read out noise (variance)
903 covModel[:, 0, 0] += mu/gain
905 return covModel
907 def subtractDistantOffset(self, muAtAmpMasked, covAtAmpMasked, covSqrtWeightsAtAmpMasked,
908 start, degree=1):
909 """Subtract distant offset from the covariance matrices.
911 Parameters
912 ----------
913 muAtAmpMasked : `numpy.array`
914 Masked mean flux array for a particular amplifier.
915 covAtAmpMasked : `numpy.array`
916 Masked measured covariances for a particular amplifier.
917 covSqrtWeightsAtAmpMasked : `numpy.array`
918 Masked inverse covariance weights for a particular amplifier.
919 start : int, optional
920 The starting index to eliminate the core for the fit.
921 degree : int, optional
922 Degree of the polynomial fit.
924 Returns
925 -------
926 covAtAmpMasked : `numpy.array`
927 Subtracted measured covariances for a particular amplifier.
928 covSqrtWeightsAtAmpMasked : `numpy.array`
929 Masked inverse covariance weights for a particular amplifier.
931 Notes
932 -----
933 Ported from https://gitlab.in2p3.fr/astier/bfptc by P. Astier.
935 This function subtracts a distant offset from the
936 covariance matrices using polynomial fitting. The core
937 of the matrices is eliminated for the fit.
939 The function modifies the internal state of the object, updating the
940 covariance matrices and related attributes.
941 """
942 for k in range(len(muAtAmpMasked)):
943 # Make a copy because it will be altered
944 w = np.copy(covSqrtWeightsAtAmpMasked[k, ...])
945 wShape = w.shape
946 i, j = np.meshgrid(range(wShape[0]), range(wShape[1]), indexing='ij')
948 # Eliminate the core for the fit
949 w[:start, :start] = 0
951 poly = Pol2D(i, j, covAtAmpMasked[k, ...], degree, w=w)
952 back = poly.eval(i, j)
954 covAtAmpMasked[k, ...] -= back
956 return covAtAmpMasked, covSqrtWeightsAtAmpMasked
958 # EXPAPPROXIMATION and POLYNOMIAL fit methods
959 @staticmethod
960 def _initialParsForPolynomial(order):
961 assert order >= 2
962 pars = np.zeros(order, dtype=float)
963 pars[0] = 10
964 pars[1] = 1
965 pars[2:] = 0.0001
966 return pars
968 @staticmethod
969 def _boundsForPolynomial(initialPars, lowers=[], uppers=[]):
970 if not len(lowers):
971 lowers = [np.NINF for p in initialPars]
972 if not len(uppers):
973 uppers = [np.inf for p in initialPars]
974 lowers[1] = 0 # no negative gains
975 return (lowers, uppers)
977 @staticmethod
978 def _boundsForAstier(initialPars, lowers=[], uppers=[]):
979 if not len(lowers):
980 lowers = [np.NINF for p in initialPars]
981 if not len(uppers):
982 uppers = [np.inf for p in initialPars]
983 return (lowers, uppers)
985 @staticmethod
986 def _getInitialGoodPoints(means, variances, minVarPivotSearch, consecutivePointsVarDecreases):
987 """Return a boolean array to mask bad points.
989 Parameters
990 ----------
991 means : `numpy.array`
992 Input array with mean signal values.
993 variances : `numpy.array`
994 Input array with variances at each mean value.
995 minVarPivotSearch : `float`
996 The variance (in ADU^2), above which, the point
997 of decreasing variance should be sought.
998 consecutivePointsVarDecreases : `int`
999 Required number of consecutive points/fluxes
1000 in the PTC where the variance
1001 decreases in order to find a first
1002 estimate of the PTC turn-off.
1004 Returns
1005 ------
1006 goodPoints : `numpy.array` [`bool`]
1007 Boolean array to select good (`True`) and bad (`False`)
1008 points.
1010 Notes
1011 -----
1012 Eliminate points beyond which the variance decreases.
1013 """
1014 goodPoints = np.ones_like(means, dtype=bool)
1015 # Variances are sorted and should monotonically increase
1016 pivotList = np.where(np.array(np.diff(variances)) < 0)[0]
1017 if len(pivotList) > 0:
1018 # For small values, sometimes the variance decreases slightly
1019 # Only look when var > self.config.minVarPivotSearch
1020 pivotList = [p for p in pivotList if variances[p] > minVarPivotSearch]
1021 # Require that the varince decreases during
1022 # consecutivePointsVarDecreases
1023 # consecutive points. This will give a first
1024 # estimate of the PTC turn-off, which
1025 # may be updated (reduced) further in the code.
1026 if len(pivotList) > 1:
1027 # enumerate(pivotList) creates tuples (index, value), for
1028 # each value in pivotList. The lambda function subtracts
1029 # each value from the index.
1030 # groupby groups elements by equal key value.
1031 for k, g in groupby(enumerate(pivotList), lambda x: x[0]-x[1]):
1032 group = (map(itemgetter(1), g))
1033 # Form groups of consecute values from pivotList
1034 group = list(map(int, group))
1035 # values in pivotList are indices where np.diff(variances)
1036 # is negative, i.e., where the variance starts decreasing.
1037 # Find the first group of consecutive numbers when
1038 # variance decreases.
1039 if len(group) >= consecutivePointsVarDecreases:
1040 pivotIndex = np.min(group)
1041 goodPoints[pivotIndex+1:] = False
1042 break
1044 # Finally, we filter out any infinities or NaNs.
1045 goodPoints[(~np.isfinite(means)) | (~np.isfinite(variances))] = False
1047 return goodPoints
1049 def _makeZeroSafe(self, array, substituteValue=1e-9):
1050 """"""
1051 array = np.array(array)
1052 nBad = Counter(np.ravel(array))[0]
1053 if nBad == 0:
1054 return array
1056 index, = np.where(array == 0)
1057 if len(index):
1058 msg = f"Found {nBad} zeros in array at elements {index}"
1059 self.log.warning(msg)
1061 array[index] = substituteValue
1063 return array
1065 def fitPtc(self, dataset):
1066 """Fit the photon transfer curve to a polynomial or to the
1067 Astier+19 approximation (Eq. 16).
1069 Fit the photon transfer curve with either a polynomial of
1070 the order specified in the task config (POLNOMIAL),
1071 or using the exponential approximation in Astier+19
1072 (Eq. 16, FULLCOVARIANCE).
1074 Sigma clipping is performed iteratively for the fit, as
1075 well as an initial clipping of data points that are more
1076 than `config.initialNonLinearityExclusionThreshold` away
1077 from lying on a straight line. This other step is necessary
1078 because the photon transfer curve turns over catastrophically
1079 at very high flux (because saturation
1080 drops the variance to ~0) and these far outliers cause the
1081 initial fit to fail, meaning the sigma cannot be calculated
1082 to perform the sigma-clipping.
1084 Parameters
1085 ----------
1086 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset`
1087 The dataset containing the means, variances and
1088 exposure times.
1090 Returns
1091 -------
1092 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset`
1093 This is the same dataset as the input parameter, however,
1094 it has been modified to include information such as the
1095 fit vectors and the fit parameters. See the class
1096 `PhotonTransferCurveDatase`.
1098 Raises
1099 ------
1100 RuntimeError
1101 Raised if dataset.ptcFitType is None or empty.
1102 """
1103 if dataset.ptcFitType:
1104 ptcFitType = dataset.ptcFitType
1105 else:
1106 raise RuntimeError("ptcFitType is None of empty in PTC dataset.")
1107 # For FULLCOVARIANCE model fit
1108 matrixSideFit = dataset.covMatrixSideFullCovFit
1109 nanMatrixFit = np.empty((matrixSideFit, matrixSideFit))
1110 nanMatrixFit[:] = np.nan
1112 for amp in dataset.ampNames:
1113 lenInputTimes = len(dataset.rawExpTimes[amp])
1114 listNanMatrixFit = np.empty((lenInputTimes, matrixSideFit, matrixSideFit))
1115 listNanMatrixFit[:] = np.nan
1117 dataset.covariancesModel[amp] = listNanMatrixFit
1118 dataset.aMatrix[amp] = nanMatrixFit
1119 dataset.bMatrix[amp] = nanMatrixFit
1120 dataset.covariancesModelNoB[amp] = listNanMatrixFit
1121 dataset.aMatrixNoB[amp] = nanMatrixFit
1122 dataset.noiseMatrix[amp] = nanMatrixFit
1123 dataset.noiseMatrixNoB[amp] = nanMatrixFit
1125 def errFunc(p, x, y):
1126 return ptcFunc(p, x) - y
1128 sigmaCutPtcOutliers = self.config.sigmaCutPtcOutliers
1129 maxIterationsPtcOutliers = self.config.maxIterationsPtcOutliers
1131 for i, ampName in enumerate(dataset.ampNames):
1132 meanVecOriginal = dataset.rawMeans[ampName].copy()
1133 varVecOriginal = dataset.rawVars[ampName].copy()
1134 varVecOriginal = self._makeZeroSafe(varVecOriginal)
1136 # These must be sorted for the given amplifier.
1137 meanVecSort = np.argsort(meanVecOriginal)
1138 meanVecSorted = meanVecOriginal[meanVecSort]
1139 varVecSorted = varVecOriginal[meanVecSort]
1141 if self.config.doLegacyTurnoffSelection:
1142 # Discard points when the variance starts to decrease after two
1143 # consecutive signal levels
1144 goodPoints = self._getInitialGoodPoints(meanVecSorted, varVecSorted,
1145 self.config.minVarPivotSearch,
1146 self.config.consecutivePointsVarDecreases)
1147 else:
1148 # Make sure we have this properly sorted.
1149 goodPoints = dataset.expIdMask[ampName].copy()
1150 goodPoints = goodPoints[meanVecSort]
1152 # Check if all points are bad from the 'cpExtractPtcTask'
1153 initialExpIdMask = dataset.expIdMask[ampName].copy()
1155 if not (goodPoints.any() and initialExpIdMask.any()):
1156 msg = (f"SERIOUS: All points in goodPoints: {goodPoints} or "
1157 f"in initialExpIdMask: {initialExpIdMask} are bad."
1158 f"Setting {ampName} to BAD.")
1159 self.log.warning(msg)
1160 # Fill entries with NaNs
1161 self.fillBadAmp(dataset, ptcFitType, ampName)
1162 continue
1164 mask = goodPoints.copy()
1166 if ptcFitType == 'EXPAPPROXIMATION':
1167 ptcFunc = funcAstier
1168 parsIniPtc = [-1e-9, 1.0, 10.] # a00, gain, noise^2
1169 # lowers and uppers obtained from BOT data studies by
1170 # C. Lage (UC Davis, 11/2020).
1171 if self.config.binSize > 1:
1172 bounds = self._boundsForAstier(parsIniPtc)
1173 else:
1174 bounds = self._boundsForAstier(parsIniPtc, lowers=[-1e-4, 0.1, -2000],
1175 uppers=[1e-4, 10.0, 2000])
1176 if ptcFitType == 'POLYNOMIAL':
1177 ptcFunc = funcPolynomial
1178 parsIniPtc = self._initialParsForPolynomial(self.config.polynomialFitDegree + 1)
1179 bounds = self._boundsForPolynomial(parsIniPtc)
1181 # We perform an initial (unweighted) fit of variance vs signal
1182 # (after initial KS test or post-drop selection) to look for
1183 # outliers, particularly at the high-flux end. The initial fit
1184 # is performed only for points that are guaranteed to be below
1185 # the PTC turnoff and then extrapolated to ensure that high
1186 # flux points that have abnormal variance values can be properly
1187 # rejected in this phase without biasing the initial fit.
1188 # This algorithm was initially developed by Seth Digel for
1189 # the EO Testing pipeline.
1191 if self.config.scaleMaxSignalInitialPtcOutlierFit:
1192 approxGain = np.nanmedian(meanVecSorted/varVecSorted)
1193 maxADUInitialPtcOutlierFit = self.config.maxSignalInitialPtcOutlierFit/approxGain
1194 maxDeltaADUInitialPtcOutlierFit = self.config.maxDeltaInitialPtcOutlierFit/approxGain
1195 self.log.info(
1196 "Using approximate gain %.3f and ADU signal cutoff of %.1f and delta %.1f "
1197 "for amplifier %s",
1198 approxGain,
1199 maxADUInitialPtcOutlierFit,
1200 maxDeltaADUInitialPtcOutlierFit,
1201 ampName,
1202 )
1203 else:
1204 maxADUInitialPtcOutlierFit = self.config.maxSignalInitialPtcOutlierFit
1205 maxDeltaADUInitialPtcOutlierFit = self.config.maxDeltaInitialPtcOutlierFit
1207 if maxIterationsPtcOutliers == 0:
1208 # We are not doing any outlier rejection here, but we do want
1209 # an initial fit.
1210 res = least_squares(
1211 errFunc,
1212 parsIniPtc,
1213 bounds=bounds,
1214 args=(meanVecSorted[mask], varVecSorted[mask]),
1215 )
1216 pars = res.x
1217 newMask = mask.copy()
1218 else:
1219 newMask = (mask & (meanVecSorted <= maxADUInitialPtcOutlierFit))
1221 converged = False
1222 count = 0
1223 lastMask = mask.copy()
1224 while count < maxIterationsPtcOutliers:
1225 res = least_squares(
1226 errFunc,
1227 parsIniPtc,
1228 bounds=bounds,
1229 args=(meanVecSorted[newMask], varVecSorted[newMask]),
1230 )
1231 pars = res.x
1233 sigResids = (varVecSorted - ptcFunc(pars, meanVecSorted))/np.sqrt(varVecSorted)
1234 # The new mask includes points where the residuals are
1235 # finite, are less than the cut, and include the original
1236 # mask of known points that should not be used.
1237 newMask = (
1238 np.isfinite(sigResids)
1239 & (np.abs(np.nan_to_num(sigResids)) < sigmaCutPtcOutliers)
1240 & mask
1241 )
1242 # Demand at least 2 points to continue.
1243 if np.count_nonzero(newMask) < 2:
1244 msg = (f"SERIOUS: All points after outlier rejection are bad. "
1245 f"Setting {ampName} to BAD.")
1246 self.log.warning(msg)
1247 # Fill entries with NaNs
1248 self.fillBadAmp(dataset, ptcFitType, ampName)
1249 break
1251 self.log.debug(
1252 "Iteration %d: Removed %d points in total for %s.",
1253 count,
1254 np.count_nonzero(mask) - np.count_nonzero(newMask),
1255 ampName,
1256 )
1258 # Loop over all used (True) points. If one of them follows
1259 # a False point, then it must be within
1260 # maxDeltaADUInitialPtcOutlierFit of a True point. If it
1261 # is a large gap, everything above is marked False.
1262 useMask, = np.where(newMask)
1263 for useIndex, usePoint in enumerate(useMask):
1264 if useIndex == 0 or newMask[usePoint - 1]:
1265 # The previous point was good; continue.
1266 continue
1267 deltaADU = meanVecSorted[usePoint] - meanVecSorted[useMask[useIndex - 1]]
1268 if deltaADU < maxDeltaADUInitialPtcOutlierFit:
1269 # This jump is fine; continue.
1270 continue
1272 # Mark all further points bad.
1273 newMask[usePoint:] = False
1274 break
1276 # If the mask hasn't changed then break out.
1277 if np.all(newMask == lastMask):
1278 self.log.debug("Convergence at iteration %d; breaking loop for %s.", count, ampName)
1279 converged = True
1280 break
1282 lastMask = newMask.copy()
1284 count += 1
1286 if not converged:
1287 self.log.warning(
1288 "Outlier detection was not converged prior to %d iteration for %s",
1289 count,
1290 ampName
1291 )
1293 # Set the mask to the new mask, and reset the sorting.
1294 mask = np.zeros(len(meanVecSort), dtype=np.bool_)
1295 mask[meanVecSort[newMask]] = True
1296 maskSorted = newMask.copy()
1298 if not mask.any():
1299 # We hae already filled the bad amp above, so continue.
1300 continue
1302 dataset.expIdMask[ampName] = mask
1304 parsIniPtc = pars
1305 meanVecFinal = meanVecOriginal[mask]
1306 varVecFinal = varVecOriginal[mask]
1308 # Save the maximum point after outlier detection as the
1309 # PTC turnoff point. We need to pay attention to the sorting
1310 # here.
1311 dataset.ptcTurnoff[ampName] = np.max(meanVecFinal)
1312 # And compute the ptcTurnoffSamplingError as one half the
1313 # difference between the previous and next point.
1314 lastGoodIndex = np.where(maskSorted)[0][-1]
1315 ptcTurnoffLow = meanVecSorted[lastGoodIndex - 1]
1316 if lastGoodIndex == (len(meanVecSorted) - 1):
1317 # If it's the last index, just use the interval.
1318 ptcTurnoffSamplingError = dataset.ptcTurnoff[ampName] - ptcTurnoffLow
1319 elif not np.isfinite(meanVecSorted[lastGoodIndex + 1]):
1320 # If the next index is not finite, just use the interval.
1321 ptcTurnoffSamplingError = dataset.ptcTurnoff[ampName] - ptcTurnoffLow
1322 else:
1323 ptcTurnoffSamplingError = (meanVecSorted[lastGoodIndex + 1] - ptcTurnoffLow)/2.
1324 dataset.ptcTurnoffSamplingError[ampName] = ptcTurnoffSamplingError
1326 if Counter(mask)[False] > 0:
1327 self.log.info("Number of points discarded in PTC of amplifier %s:"
1328 " %d out of %d", ampName, Counter(mask)[False], len(meanVecOriginal))
1330 if (len(meanVecFinal) < len(parsIniPtc)):
1331 msg = (f"SERIOUS: Not enough data points ({len(meanVecFinal)}) compared to the number of "
1332 f"parameters of the PTC model({len(parsIniPtc)}). Setting {ampName} to BAD.")
1333 self.log.warning(msg)
1334 # Fill entries with NaNs
1335 self.fillBadAmp(dataset, ptcFitType, ampName)
1336 continue
1337 # Fit the PTC.
1338 # The variance of the variance is Var(v)=2*v^2/Npix. This is
1339 # already calculated in `makeCovArray` of CpPtcExtract.
1340 # dataset.covariancesSqrtWeights[ampName][:,0,0]
1341 # has 1/sqrt(Var(v)).
1342 weightsY = dataset.covariancesSqrtWeights[ampName][:, 0, 0][mask]
1343 if self.config.doFitBootstrap:
1344 parsFit, parsFitErr, reducedChiSqPtc = fitBootstrap(parsIniPtc, meanVecFinal,
1345 varVecFinal, ptcFunc,
1346 weightsY=weightsY)
1347 else:
1348 parsFit, parsFitErr, reducedChiSqPtc = fitLeastSq(parsIniPtc, meanVecFinal,
1349 varVecFinal, ptcFunc,
1350 weightsY=weightsY)
1351 dataset.ptcFitPars[ampName] = parsFit
1352 dataset.ptcFitParsError[ampName] = parsFitErr
1353 dataset.ptcFitChiSq[ampName] = reducedChiSqPtc
1355 dataset.finalVars[ampName] = varVecOriginal
1356 dataset.finalVars[ampName][~mask] = np.nan
1357 dataset.finalModelVars[ampName] = ptcFunc(parsFit, meanVecOriginal)
1358 dataset.finalModelVars[ampName][~mask] = np.nan
1359 dataset.finalMeans[ampName] = meanVecOriginal
1360 dataset.finalMeans[ampName][~mask] = np.nan
1362 if ptcFitType == 'EXPAPPROXIMATION':
1363 ptcGain = parsFit[1]
1364 ptcGainErr = parsFitErr[1]
1365 ptcNoise = np.sqrt(np.fabs(parsFit[2]))
1366 ptcNoiseErr = 0.5*(parsFitErr[2]/np.fabs(parsFit[2]))*np.sqrt(np.fabs(parsFit[2]))
1367 if ptcFitType == 'POLYNOMIAL':
1368 ptcGain = 1./parsFit[1]
1369 ptcGainErr = np.fabs(1./parsFit[1])*(parsFitErr[1]/parsFit[1])
1370 ptcNoise = np.sqrt(np.fabs(parsFit[0]))*ptcGain
1371 ptcNoiseErr = (0.5*(parsFitErr[0]/np.fabs(parsFit[0]))*(np.sqrt(np.fabs(parsFit[0]))))*ptcGain
1372 dataset.gain[ampName] = ptcGain
1373 dataset.gainErr[ampName] = ptcGainErr
1374 dataset.noise[ampName] = ptcNoise
1375 dataset.noiseErr[ampName] = ptcNoiseErr
1377 if not len(dataset.ptcFitType) == 0:
1378 dataset.ptcFitType = ptcFitType
1379 if len(dataset.badAmps) == 0:
1380 dataset.badAmps = []
1382 return dataset
1384 def fillBadAmp(self, dataset, ptcFitType, ampName):
1385 """Fill the dataset with NaNs if there are not enough
1386 good points.
1388 Parameters
1389 ----------
1390 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset`
1391 The dataset containing the means, variances and
1392 exposure times.
1393 ptcFitType : {'POLYNOMIAL', 'EXPAPPROXIMATION'}
1394 Fit a 'POLYNOMIAL' (degree: 'polynomialFitDegree') or
1395 'EXPAPPROXIMATION' (Eq. 16 of Astier+19) to the PTC.
1396 ampName : `str`
1397 Amplifier name.
1398 """
1399 dataset.badAmps.append(ampName)
1400 dataset.expIdMask[ampName] = np.repeat(False, len(dataset.rawExpTimes[ampName]))
1401 dataset.gain[ampName] = np.nan
1402 dataset.gainErr[ampName] = np.nan
1403 dataset.noise[ampName] = np.nan
1404 dataset.noiseErr[ampName] = np.nan
1405 dataset.ptcFitPars[ampName] = (np.repeat(np.nan, self.config.polynomialFitDegree + 1) if
1406 ptcFitType in ["POLYNOMIAL", ] else np.repeat(np.nan, 3))
1407 dataset.ptcFitParsError[ampName] = (np.repeat(np.nan, self.config.polynomialFitDegree + 1) if
1408 ptcFitType in ["POLYNOMIAL", ] else np.repeat(np.nan, 3))
1409 dataset.ptcFitChiSq[ampName] = np.nan
1410 dataset.ptcTurnoff[ampName] = np.nan
1411 dataset.finalVars[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName]))
1412 dataset.finalModelVars[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName]))
1413 dataset.finalMeans[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName]))
1415 return