Coverage for python/lsst/cp/pipe/ptc.py : 11%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of cp_pipe.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21#
23__all__ = ['MeasurePhotonTransferCurveTask',
24 'MeasurePhotonTransferCurveTaskConfig',
25 'PhotonTransferCurveDataset']
27import numpy as np
28import matplotlib.pyplot as plt
29from sqlite3 import OperationalError
30from collections import Counter
31from dataclasses import dataclass
33import lsst.afw.math as afwMath
34import lsst.pex.config as pexConfig
35import lsst.pipe.base as pipeBase
36from .utils import (NonexistentDatasetTaskDataIdContainer, PairedVisitListTaskRunner,
37 checkExpLengthEqual, fitLeastSq, fitBootstrap, funcPolynomial, funcAstier)
38from scipy.optimize import least_squares
40from lsst.ip.isr.linearize import Linearizer
41import datetime
43from .astierCovPtcUtils import (fftSize, CovFft, computeCovDirect, fitData)
46class MeasurePhotonTransferCurveTaskConfig(pexConfig.Config):
47 """Config class for photon transfer curve measurement task"""
48 ccdKey = pexConfig.Field(
49 dtype=str,
50 doc="The key by which to pull a detector from a dataId, e.g. 'ccd' or 'detector'.",
51 default='ccd',
52 )
53 ptcFitType = pexConfig.ChoiceField(
54 dtype=str,
55 doc="Fit PTC to approximation in Astier+19 (Equation 16) or to a polynomial.",
56 default="POLYNOMIAL",
57 allowed={
58 "POLYNOMIAL": "n-degree polynomial (use 'polynomialFitDegree' to set 'n').",
59 "EXPAPPROXIMATION": "Approximation in Astier+19 (Eq. 16).",
60 "FULLCOVARIANCE": "Full covariances model in Astier+19 (Eq. 20)"
61 }
62 )
63 maximumRangeCovariancesAstier = pexConfig.Field(
64 dtype=int,
65 doc="Maximum range of covariances as in Astier+19",
66 default=8,
67 )
68 covAstierRealSpace = pexConfig.Field(
69 dtype=bool,
70 doc="Calculate covariances in real space or via FFT? (see appendix A of Astier+19).",
71 default=False,
72 )
73 polynomialFitDegree = pexConfig.Field(
74 dtype=int,
75 doc="Degree of polynomial to fit the PTC, when 'ptcFitType'=POLYNOMIAL.",
76 default=3,
77 )
78 doCreateLinearizer = pexConfig.Field(
79 dtype=bool,
80 doc="Calculate non-linearity and persist linearizer?",
81 default=False,
82 )
83 linearizerType = pexConfig.ChoiceField(
84 dtype=str,
85 doc="Linearizer type, if doCreateLinearizer=True",
86 default="LINEARIZEPOLYNOMIAL",
87 allowed={
88 "LINEARIZEPOLYNOMIAL": "n-degree polynomial (use 'polynomialFitDegreeNonLinearity' to set 'n').",
89 "LINEARIZESQUARED": "c0 quadratic coefficient derived from coefficients of polynomiual fit",
90 "LOOKUPTABLE": "Loouk table formed from linear part of polynomial fit."
91 }
92 )
93 polynomialFitDegreeNonLinearity = pexConfig.Field(
94 dtype=int,
95 doc="If doCreateLinearizer, degree of polynomial to fit the meanSignal vs exposureTime" +
96 " curve to produce the table for LinearizeLookupTable.",
97 default=3,
98 )
99 binSize = pexConfig.Field(
100 dtype=int,
101 doc="Bin the image by this factor in both dimensions.",
102 default=1,
103 )
104 minMeanSignal = pexConfig.Field(
105 dtype=float,
106 doc="Minimum value (inclusive) of mean signal (in DN) above which to consider.",
107 default=0,
108 )
109 maxMeanSignal = pexConfig.Field(
110 dtype=float,
111 doc="Maximum value (inclusive) of mean signal (in DN) below which to consider.",
112 default=9e6,
113 )
114 initialNonLinearityExclusionThresholdPositive = pexConfig.RangeField(
115 dtype=float,
116 doc="Initially exclude data points with a variance that are more than a factor of this from being"
117 " linear in the positive direction, from the PTC fit. Note that these points will also be"
118 " excluded from the non-linearity fit. This is done before the iterative outlier rejection,"
119 " to allow an accurate determination of the sigmas for said iterative fit.",
120 default=0.12,
121 min=0.0,
122 max=1.0,
123 )
124 initialNonLinearityExclusionThresholdNegative = pexConfig.RangeField(
125 dtype=float,
126 doc="Initially exclude data points with a variance that are more than a factor of this from being"
127 " linear in the negative direction, from the PTC fit. Note that these points will also be"
128 " excluded from the non-linearity fit. This is done before the iterative outlier rejection,"
129 " to allow an accurate determination of the sigmas for said iterative fit.",
130 default=0.25,
131 min=0.0,
132 max=1.0,
133 )
134 sigmaCutPtcOutliers = pexConfig.Field(
135 dtype=float,
136 doc="Sigma cut for outlier rejection in PTC.",
137 default=5.0,
138 )
139 maskNameList = pexConfig.ListField(
140 dtype=str,
141 doc="Mask list to exclude from statistics calculations.",
142 default=['SUSPECT', 'BAD', 'NO_DATA'],
143 )
144 nSigmaClipPtc = pexConfig.Field(
145 dtype=float,
146 doc="Sigma cut for afwMath.StatisticsControl()",
147 default=5.5,
148 )
149 nIterSigmaClipPtc = pexConfig.Field(
150 dtype=int,
151 doc="Number of sigma-clipping iterations for afwMath.StatisticsControl()",
152 default=1,
153 )
154 maxIterationsPtcOutliers = pexConfig.Field(
155 dtype=int,
156 doc="Maximum number of iterations for outlier rejection in PTC.",
157 default=2,
158 )
159 doFitBootstrap = pexConfig.Field(
160 dtype=bool,
161 doc="Use bootstrap for the PTC fit parameters and errors?.",
162 default=False,
163 )
164 maxAduForLookupTableLinearizer = pexConfig.Field(
165 dtype=int,
166 doc="Maximum DN value for the LookupTable linearizer.",
167 default=2**18,
168 )
169 instrumentName = pexConfig.Field(
170 dtype=str,
171 doc="Instrument name.",
172 default='',
173 )
176@dataclass
177class LinearityResidualsAndLinearizersDataset:
178 """A simple class to hold the output from the
179 `calculateLinearityResidualAndLinearizers` function.
180 """
181 # Normalized coefficients for polynomial NL correction
182 polynomialLinearizerCoefficients: list
183 # Normalized coefficient for quadratic polynomial NL correction (c0)
184 quadraticPolynomialLinearizerCoefficient: float
185 # LUT array row for the amplifier at hand
186 linearizerTableRow: list
187 meanSignalVsTimePolyFitPars: list
188 meanSignalVsTimePolyFitParsErr: list
189 meanSignalVsTimePolyFitReducedChiSq: float
192class PhotonTransferCurveDataset:
193 """A simple class to hold the output data from the PTC task.
195 The dataset is made up of a dictionary for each item, keyed by the
196 amplifiers' names, which much be supplied at construction time.
198 New items cannot be added to the class to save accidentally saving to the
199 wrong property, and the class can be frozen if desired.
201 inputVisitPairs records the visits used to produce the data.
202 When fitPtc() or fitCovariancesAstier() is run, a mask is built up, which is by definition
203 always the same length as inputVisitPairs, rawExpTimes, rawMeans
204 and rawVars, and is a list of bools, which are incrementally set to False
205 as points are discarded from the fits.
207 PTC fit parameters for polynomials are stored in a list in ascending order
208 of polynomial term, i.e. par[0]*x^0 + par[1]*x + par[2]*x^2 etc
209 with the length of the list corresponding to the order of the polynomial
210 plus one.
212 Parameters
213 ----------
214 ampNames : `list`
215 List with the names of the amplifiers of the detector at hand.
217 ptcFitType : `str`
218 Type of model fitted to the PTC: "POLYNOMIAL", "EXPAPPROXIMATION", or "FULLCOVARIANCE".
220 Returns
221 -------
222 `lsst.cp.pipe.ptc.PhotonTransferCurveDataset`
223 Output dataset from MeasurePhotonTransferCurveTask.
224 """
226 def __init__(self, ampNames, ptcFitType):
227 # add items to __dict__ directly because __setattr__ is overridden
229 # instance variables
230 self.__dict__["ptcFitType"] = ptcFitType
231 self.__dict__["ampNames"] = ampNames
232 self.__dict__["badAmps"] = []
234 # raw data variables
235 # visitMask is the mask produced after outlier rejection. The mask produced by "FULLCOVARIANCE"
236 # may differ from the one produced in the other two PTC fit types.
237 self.__dict__["inputVisitPairs"] = {ampName: [] for ampName in ampNames}
238 self.__dict__["visitMask"] = {ampName: [] for ampName in ampNames}
239 self.__dict__["rawExpTimes"] = {ampName: [] for ampName in ampNames}
240 self.__dict__["rawMeans"] = {ampName: [] for ampName in ampNames}
241 self.__dict__["rawVars"] = {ampName: [] for ampName in ampNames}
243 # Gain and noise
244 self.__dict__["gain"] = {ampName: -1. for ampName in ampNames}
245 self.__dict__["gainErr"] = {ampName: -1. for ampName in ampNames}
246 self.__dict__["noise"] = {ampName: -1. for ampName in ampNames}
247 self.__dict__["noiseErr"] = {ampName: -1. for ampName in ampNames}
249 # if ptcFitTye in ["POLYNOMIAL", "EXPAPPROXIMATION"]
250 # fit information
251 self.__dict__["ptcFitPars"] = {ampName: [] for ampName in ampNames}
252 self.__dict__["ptcFitParsError"] = {ampName: [] for ampName in ampNames}
253 self.__dict__["ptcFitReducedChiSquared"] = {ampName: [] for ampName in ampNames}
255 # if ptcFitTye in ["FULLCOVARIANCE"]
256 # "covariancesTuple" is a numpy recarray with entries of the form
257 # ['mu', 'i', 'j', 'var', 'cov', 'npix', 'ext', 'expTime', 'ampName']
258 # "covariancesFits" has CovFit objects that fit the measured covariances to Eq. 20 of Astier+19.
259 # In "covariancesFitsWithNoB", "b"=0 in the model described by Eq. 20 of Astier+19.
260 self.__dict__["covariancesTuple"] = {ampName: [] for ampName in ampNames}
261 self.__dict__["covariancesFitsWithNoB"] = {ampName: [] for ampName in ampNames}
262 self.__dict__["covariancesFits"] = {ampName: [] for ampName in ampNames}
263 self.__dict__["aMatrix"] = {ampName: [] for ampName in ampNames}
264 self.__dict__["bMatrix"] = {ampName: [] for ampName in ampNames}
266 # "final" means that the "raw" vectors above had "visitMask" applied.
267 self.__dict__["finalVars"] = {ampName: [] for ampName in ampNames}
268 self.__dict__["finalModelVars"] = {ampName: [] for ampName in ampNames}
269 self.__dict__["finalMeans"] = {ampName: [] for ampName in ampNames}
271 def __setattr__(self, attribute, value):
272 """Protect class attributes"""
273 if attribute not in self.__dict__:
274 raise AttributeError(f"{attribute} is not already a member of PhotonTransferCurveDataset, which"
275 " does not support setting of new attributes.")
276 else:
277 self.__dict__[attribute] = value
279 def getVisitsUsed(self, ampName):
280 """Get the visits used, i.e. not discarded, for a given amp.
282 If no mask has been created yet, all visits are returned.
283 """
284 if len(self.visitMask[ampName]) == 0:
285 return self.inputVisitPairs[ampName]
287 # if the mask exists it had better be the same length as the visitPairs
288 assert len(self.visitMask[ampName]) == len(self.inputVisitPairs[ampName])
290 pairs = self.inputVisitPairs[ampName]
291 mask = self.visitMask[ampName]
292 # cast to bool required because numpy
293 return [(v1, v2) for ((v1, v2), m) in zip(pairs, mask) if bool(m) is True]
295 def getGoodAmps(self):
296 return [amp for amp in self.ampNames if amp not in self.badAmps]
299class MeasurePhotonTransferCurveTask(pipeBase.CmdLineTask):
300 """A class to calculate, fit, and plot a PTC from a set of flat pairs.
302 The Photon Transfer Curve (var(signal) vs mean(signal)) is a standard tool
303 used in astronomical detectors characterization (e.g., Janesick 2001,
304 Janesick 2007). If ptcFitType is "EXPAPPROXIMATION" or "POLYNOMIAL", this task calculates the
305 PTC from a series of pairs of flat-field images; each pair taken at identical exposure
306 times. The difference image of each pair is formed to eliminate fixed pattern noise,
307 and then the variance of the difference image and the mean of the average image
308 are used to produce the PTC. An n-degree polynomial or the approximation in Equation
309 16 of Astier+19 ("The Shape of the Photon Transfer Curve of CCD sensors",
310 arXiv:1905.08677) can be fitted to the PTC curve. These models include
311 parameters such as the gain (e/DN) and readout noise.
313 Linearizers to correct for signal-chain non-linearity are also calculated.
314 The `Linearizer` class, in general, can support per-amp linearizers, but in this
315 task this is not supported.
317 If ptcFitType is "FULLCOVARIANCE", the covariances of the difference images are calculated via the
318 DFT methods described in Astier+19 and the variances for the PTC are given by the cov[0,0] elements
319 at each signal level. The full model in Equation 20 of Astier+19 is fit to the PTC to get the gain
320 and the noise.
322 Parameters
323 ----------
325 *args: `list`
326 Positional arguments passed to the Task constructor. None used at this
327 time.
328 **kwargs: `dict`
329 Keyword arguments passed on to the Task constructor. None used at this
330 time.
332 """
334 RunnerClass = PairedVisitListTaskRunner
335 ConfigClass = MeasurePhotonTransferCurveTaskConfig
336 _DefaultName = "measurePhotonTransferCurve"
338 def __init__(self, *args, **kwargs):
339 pipeBase.CmdLineTask.__init__(self, *args, **kwargs)
340 plt.interactive(False) # stop windows popping up when plotting. When headless, use 'agg' backend too
341 self.config.validate()
342 self.config.freeze()
344 @classmethod
345 def _makeArgumentParser(cls):
346 """Augment argument parser for the MeasurePhotonTransferCurveTask."""
347 parser = pipeBase.ArgumentParser(name=cls._DefaultName)
348 parser.add_argument("--visit-pairs", dest="visitPairs", nargs="*",
349 help="Visit pairs to use. Each pair must be of the form INT,INT e.g. 123,456")
350 parser.add_id_argument("--id", datasetType="photonTransferCurveDataset",
351 ContainerClass=NonexistentDatasetTaskDataIdContainer,
352 help="The ccds to use, e.g. --id ccd=0..100")
353 return parser
355 @pipeBase.timeMethod
356 def runDataRef(self, dataRef, visitPairs):
357 """Run the Photon Transfer Curve (PTC) measurement task.
359 For a dataRef (which is each detector here),
360 and given a list of visit pairs (postISR) at different exposure times,
361 measure the PTC.
363 Parameters
364 ----------
365 dataRef : list of lsst.daf.persistence.ButlerDataRef
366 dataRef for the detector for the visits to be fit.
368 visitPairs : `iterable` of `tuple` of `int`
369 Pairs of visit numbers to be processed together
370 """
372 # setup necessary objects
373 detNum = dataRef.dataId[self.config.ccdKey]
374 detector = dataRef.get('camera')[dataRef.dataId[self.config.ccdKey]]
375 # expand some missing fields that we need for lsstCam. This is a work-around
376 # for Gen2 problems that I (RHL) don't feel like solving. The calibs pipelines
377 # (which inherit from CalibTask) use addMissingKeys() to do basically the same thing
378 #
379 # Basically, the butler's trying to look up the fields in `raw_visit` which won't work
380 for name in dataRef.getButler().getKeys('bias'):
381 if name not in dataRef.dataId:
382 try:
383 dataRef.dataId[name] = \
384 dataRef.getButler().queryMetadata('raw', [name], detector=detNum)[0]
385 except OperationalError:
386 pass
388 amps = detector.getAmplifiers()
389 ampNames = [amp.getName() for amp in amps]
390 datasetPtc = PhotonTransferCurveDataset(ampNames, self.config.ptcFitType)
391 self.log.info('Measuring PTC using %s visits for detector %s' % (visitPairs, detector.getId()))
393 tupleRecords = []
394 allTags = []
395 for (v1, v2) in visitPairs:
396 # Get postISR exposures.
397 dataRef.dataId['expId'] = v1
398 exp1 = dataRef.get("postISRCCD", immediate=True)
399 dataRef.dataId['expId'] = v2
400 exp2 = dataRef.get("postISRCCD", immediate=True)
401 del dataRef.dataId['expId']
403 checkExpLengthEqual(exp1, exp2, v1, v2, raiseWithMessage=True)
404 expTime = exp1.getInfo().getVisitInfo().getExposureTime()
406 tupleRows = []
407 nAmpsNan = 0
408 for ampNumber, amp in enumerate(detector):
409 ampName = amp.getName()
410 # covAstier: (i, j, var (cov[0,0]), cov, npix)
411 doRealSpace = self.config.covAstierRealSpace
412 muDiff, varDiff, covAstier = self.measureMeanVarCov(exp1, exp2, region=amp.getBBox(),
413 covAstierRealSpace=doRealSpace)
414 if np.isnan(muDiff) or np.isnan(varDiff) or np.isnan(covAstier):
415 msg = f"NaN mean or cov in amp {ampNumber} in visit pair {v1}, {v2} of detector {detNum}."
416 self.log.warn(msg)
417 nAmpsNan += 1
418 continue
420 datasetPtc.rawExpTimes[ampName].append(expTime)
421 datasetPtc.rawMeans[ampName].append(muDiff)
422 datasetPtc.rawVars[ampName].append(varDiff)
423 datasetPtc.inputVisitPairs[ampName].append((v1, v2))
425 tupleRows += [(muDiff, ) + covRow + (ampNumber, expTime, ampName) for covRow in covAstier]
426 tags = ['mu', 'i', 'j', 'var', 'cov', 'npix', 'ext', 'expTime', 'ampName']
427 if nAmpsNan == len(ampNames):
428 msg = f"NaN mean in all amps of visit pair {v1}, {v2} of detector {detNum}."
429 self.log.warn(msg)
430 continue
431 allTags += tags
432 tupleRecords += tupleRows
433 covariancesWithTags = np.core.records.fromrecords(tupleRecords, names=allTags)
435 if self.config.ptcFitType in ["FULLCOVARIANCE", ]:
436 # Calculate covariances and fit them, including the PTC, to Astier+19 full model (Eq. 20)
437 datasetPtc = self.fitCovariancesAstier(datasetPtc, covariancesWithTags)
438 elif self.config.ptcFitType in ["EXPAPPROXIMATION", "POLYNOMIAL"]:
439 # Fit the PTC to a polynomial or to Astier+19 exponential approximation (Eq. 16)
440 # Fill up PhotonTransferCurveDataset object.
441 datasetPtc = self.fitPtc(datasetPtc, self.config.ptcFitType)
443 # Fit a poynomial to calculate non-linearity and persist linearizer.
444 if self.config.doCreateLinearizer:
445 numberAmps = len(amps)
446 numberAduValues = self.config.maxAduForLookupTableLinearizer
447 lookupTableArray = np.zeros((numberAmps, numberAduValues), dtype=np.float32)
449 # Fit (non)linearity of signal vs time curve.
450 # Fill up PhotonTransferCurveDataset object.
451 # Fill up array for LUT linearizer (tableArray).
452 # Produce coefficients for Polynomial ans Squared linearizers.
453 # Build linearizer objects.
454 linearizer = self.fitNonLinearityAndBuildLinearizers(datasetPtc, detector,
455 tableArray=lookupTableArray,
456 log=self.log)
458 if self.config.linearizerType == "LINEARIZEPOLYNOMIAL":
459 linDataType = 'linearizePolynomial'
460 linMsg = "polynomial (coefficients for a polynomial correction)."
461 elif self.config.linearizerType == "LINEARIZESQUARED":
462 linDataType = 'linearizePolynomial'
463 linMsg = "squared (c0, derived from k_i coefficients of a polynomial fit)."
464 elif self.config.linearizerType == "LOOKUPTABLE":
465 linDataType = 'linearizePolynomial'
466 linMsg = "lookup table (linear component of polynomial fit)."
467 else:
468 raise RuntimeError("Invalid config.linearizerType {selg.config.linearizerType}. "
469 "Supported: 'LOOKUPTABLE', 'LINEARIZESQUARED', or 'LINEARIZEPOLYNOMIAL'")
471 butler = dataRef.getButler()
472 self.log.info(f"Writing linearizer: \n {linMsg}")
474 detName = detector.getName()
475 now = datetime.datetime.utcnow()
476 calibDate = now.strftime("%Y-%m-%d")
478 butler.put(linearizer, datasetType=linDataType, dataId={'detector': detNum,
479 'detectorName': detName, 'calibDate': calibDate})
481 self.log.info(f"Writing PTC data to {dataRef.getUri(write=True)}")
482 dataRef.put(datasetPtc, datasetType="photonTransferCurveDataset")
484 return pipeBase.Struct(exitStatus=0)
486 def fitCovariancesAstier(self, dataset, covariancesWithTagsArray):
487 """Fit measured flat covariances to full model in Astier+19.
489 Parameters
490 ----------
491 dataset : `lsst.cp.pipe.ptc.PhotonTransferCurveDataset`
492 The dataset containing information such as the means, variances and exposure times.
494 covariancesWithTagsArray : `numpy.recarray`
495 Tuple with at least (mu, cov, var, i, j, npix), where:
496 mu : 0.5*(m1 + m2), where:
497 mu1: mean value of flat1
498 mu2: mean value of flat2
499 cov: covariance value at lag(i, j)
500 var: variance(covariance value at lag(0, 0))
501 i: lag dimension
502 j: lag dimension
503 npix: number of pixels used for covariance calculation.
505 Returns
506 -------
507 dataset: `lsst.cp.pipe.ptc.PhotonTransferCurveDataset`
508 This is the same dataset as the input paramter, however, it has been modified
509 to include information such as the fit vectors and the fit parameters. See
510 the class `PhotonTransferCurveDatase`.
511 """
513 covFits, covFitsNoB = fitData(covariancesWithTagsArray, maxMu=self.config.maxMeanSignal,
514 r=self.config.maximumRangeCovariancesAstier)
516 dataset.covariancesTuple = covariancesWithTagsArray
517 dataset.covariancesFits = covFits
518 dataset.covariancesFitsWithNoB = covFitsNoB
519 dataset = self.getOutputPtcDataCovAstier(dataset, covFits)
521 return dataset
523 def getOutputPtcDataCovAstier(self, dataset, covFits):
524 """Get output data for PhotonTransferCurveCovAstierDataset from CovFit objects.
526 Parameters
527 ----------
528 dataset : `lsst.cp.pipe.ptc.PhotonTransferCurveDataset`
529 The dataset containing information such as the means, variances and exposure times.
531 covFits: `dict`
532 Dictionary of CovFit objects, with amp names as keys.
534 Returns
535 -------
536 dataset : `lsst.cp.pipe.ptc.PhotonTransferCurveDataset`
537 This is the same dataset as the input paramter, however, it has been modified
538 to include extra information such as the mask 1D array, gains, reoudout noise, measured signal,
539 measured variance, modeled variance, a, and b coefficient matrices (see Astier+19) per amplifier.
540 See the class `PhotonTransferCurveDatase`.
541 """
543 for i, amp in enumerate(covFits):
544 fit = covFits[amp]
545 meanVecFinal, varVecFinal, varVecModel, wc = fit.getNormalizedFitData(0, 0, divideByMu=False)
546 gain = fit.getGain()
547 dataset.visitMask[amp] = fit.getMaskVar()
548 dataset.gain[amp] = gain
549 dataset.gainErr[amp] = fit.getGainErr()
550 dataset.noise[amp] = np.sqrt(np.fabs(fit.getRon()))
551 dataset.noiseErr[amp] = fit.getRonErr()
552 dataset.finalVars[amp].append(varVecFinal/(gain**2))
553 dataset.finalModelVars[amp].append(varVecModel/(gain**2))
554 dataset.finalMeans[amp].append(meanVecFinal/gain)
555 dataset.aMatrix[amp].append(fit.getA())
556 dataset.bMatrix[amp].append(fit.getB())
558 return dataset
560 def measureMeanVarCov(self, exposure1, exposure2, region=None, covAstierRealSpace=False):
561 """Calculate the mean of each of two exposures and the variance and covariance of their difference.
563 The variance is calculated via afwMath, and the covariance via the methods in Astier+19 (appendix A).
564 In theory, var = covariance[0,0]. This should be validated, and in the future, we may decide to just
565 keep one (covariance).
567 Parameters
568 ----------
569 exposure1 : `lsst.afw.image.exposure.exposure.ExposureF`
570 First exposure of flat field pair.
572 exposure2 : `lsst.afw.image.exposure.exposure.ExposureF`
573 Second exposure of flat field pair.
575 region : `lsst.geom.Box2I`, optional
576 Region of each exposure where to perform the calculations (e.g, an amplifier).
578 covAstierRealSpace : `bool`, optional
579 Should the covariannces in Astier+19 be calculated in real space or via FFT?
580 See Appendix A of Astier+19.
582 Returns
583 -------
584 mu : `float` or `NaN`
585 0.5*(mu1 + mu2), where mu1, and mu2 are the clipped means of the regions in
586 both exposures. If either mu1 or m2 are NaN's, the returned value is NaN.
588 varDiff : `float` or `NaN`
589 Half of the clipped variance of the difference of the regions inthe two input
590 exposures. If either mu1 or m2 are NaN's, the returned value is NaN.
592 covDiffAstier : `list` or `NaN`
593 List with tuples of the form (dx, dy, var, cov, npix), where:
594 dx : `int`
595 Lag in x
596 dy : `int`
597 Lag in y
598 var : `float`
599 Variance at (dx, dy).
600 cov : `float`
601 Covariance at (dx, dy).
602 nPix : `int`
603 Number of pixel pairs used to evaluate var and cov.
604 If either mu1 or m2 are NaN's, the returned value is NaN.
605 """
607 if region is not None:
608 im1Area = exposure1.maskedImage[region]
609 im2Area = exposure2.maskedImage[region]
610 else:
611 im1Area = exposure1.maskedImage
612 im2Area = exposure2.maskedImage
614 im1Area = afwMath.binImage(im1Area, self.config.binSize)
615 im2Area = afwMath.binImage(im2Area, self.config.binSize)
617 im1MaskVal = exposure1.getMask().getPlaneBitMask(self.config.maskNameList)
618 im1StatsCtrl = afwMath.StatisticsControl(self.config.nSigmaClipPtc,
619 self.config.nIterSigmaClipPtc,
620 im1MaskVal)
621 im1StatsCtrl.setNanSafe(True)
622 im1StatsCtrl.setAndMask(im1MaskVal)
624 im2MaskVal = exposure2.getMask().getPlaneBitMask(self.config.maskNameList)
625 im2StatsCtrl = afwMath.StatisticsControl(self.config.nSigmaClipPtc,
626 self.config.nIterSigmaClipPtc,
627 im2MaskVal)
628 im2StatsCtrl.setNanSafe(True)
629 im2StatsCtrl.setAndMask(im2MaskVal)
631 # Clipped mean of images; then average of mean.
632 mu1 = afwMath.makeStatistics(im1Area, afwMath.MEANCLIP, im1StatsCtrl).getValue()
633 mu2 = afwMath.makeStatistics(im2Area, afwMath.MEANCLIP, im2StatsCtrl).getValue()
634 if np.isnan(mu1) or np.isnan(mu2):
635 return np.nan, np.nan, np.nan
636 mu = 0.5*(mu1 + mu2)
638 # Take difference of pairs
639 # symmetric formula: diff = (mu2*im1-mu1*im2)/(0.5*(mu1+mu2))
640 temp = im2Area.clone()
641 temp *= mu1
642 diffIm = im1Area.clone()
643 diffIm *= mu2
644 diffIm -= temp
645 diffIm /= mu
647 diffImMaskVal = diffIm.getMask().getPlaneBitMask(self.config.maskNameList)
648 diffImStatsCtrl = afwMath.StatisticsControl(self.config.nSigmaClipPtc,
649 self.config.nIterSigmaClipPtc,
650 diffImMaskVal)
651 diffImStatsCtrl.setNanSafe(True)
652 diffImStatsCtrl.setAndMask(diffImMaskVal)
654 varDiff = 0.5*(afwMath.makeStatistics(diffIm, afwMath.VARIANCECLIP, diffImStatsCtrl).getValue())
656 # Get the mask and identify good pixels as '1', and the rest as '0'.
657 w1 = np.where(im1Area.getMask().getArray() == 0, 1, 0)
658 w2 = np.where(im2Area.getMask().getArray() == 0, 1, 0)
660 w12 = w1*w2
661 wDiff = np.where(diffIm.getMask().getArray() == 0, 1, 0)
662 w = w12*wDiff
664 maxRangeCov = self.config.maximumRangeCovariancesAstier
665 if covAstierRealSpace:
666 covDiffAstier = computeCovDirect(diffIm.getImage().getArray(), w, maxRangeCov)
667 else:
668 shapeDiff = diffIm.getImage().getArray().shape
669 fftShape = (fftSize(shapeDiff[0] + maxRangeCov), fftSize(shapeDiff[1]+maxRangeCov))
670 c = CovFft(diffIm.getImage().getArray(), w, fftShape, maxRangeCov)
671 covDiffAstier = c.reportCovFft(maxRangeCov)
673 return mu, varDiff, covDiffAstier
675 def computeCovDirect(self, diffImage, weightImage, maxRange):
676 """Compute covariances of diffImage in real space.
678 For lags larger than ~25, it is slower than the FFT way.
679 Taken from https://github.com/PierreAstier/bfptc/
681 Parameters
682 ----------
683 diffImage : `numpy.array`
684 Image to compute the covariance of.
686 weightImage : `numpy.array`
687 Weight image of diffImage (1's and 0's for good and bad pixels, respectively).
689 maxRange : `int`
690 Last index of the covariance to be computed.
692 Returns
693 -------
694 outList : `list`
695 List with tuples of the form (dx, dy, var, cov, npix), where:
696 dx : `int`
697 Lag in x
698 dy : `int`
699 Lag in y
700 var : `float`
701 Variance at (dx, dy).
702 cov : `float`
703 Covariance at (dx, dy).
704 nPix : `int`
705 Number of pixel pairs used to evaluate var and cov.
706 """
707 outList = []
708 var = 0
709 # (dy,dx) = (0,0) has to be first
710 for dy in range(maxRange + 1):
711 for dx in range(0, maxRange + 1):
712 if (dx*dy > 0):
713 cov1, nPix1 = self.covDirectValue(diffImage, weightImage, dx, dy)
714 cov2, nPix2 = self.covDirectValue(diffImage, weightImage, dx, -dy)
715 cov = 0.5*(cov1 + cov2)
716 nPix = nPix1 + nPix2
717 else:
718 cov, nPix = self.covDirectValue(diffImage, weightImage, dx, dy)
719 if (dx == 0 and dy == 0):
720 var = cov
721 outList.append((dx, dy, var, cov, nPix))
723 return outList
725 def covDirectValue(self, diffImage, weightImage, dx, dy):
726 """Compute covariances of diffImage in real space at lag (dx, dy).
728 Taken from https://github.com/PierreAstier/bfptc/ (c.f., appendix of Astier+19).
730 Parameters
731 ----------
732 diffImage : `numpy.array`
733 Image to compute the covariance of.
735 weightImage : `numpy.array`
736 Weight image of diffImage (1's and 0's for good and bad pixels, respectively).
738 dx : `int`
739 Lag in x.
741 dy : `int`
742 Lag in y.
744 Returns
745 -------
746 cov : `float`
747 Covariance at (dx, dy)
749 nPix : `int`
750 Number of pixel pairs used to evaluate var and cov.
751 """
752 (nCols, nRows) = diffImage.shape
753 # switching both signs does not change anything:
754 # it just swaps im1 and im2 below
755 if (dx < 0):
756 (dx, dy) = (-dx, -dy)
757 # now, we have dx >0. We have to distinguish two cases
758 # depending on the sign of dy
759 if dy >= 0:
760 im1 = diffImage[dy:, dx:]
761 w1 = weightImage[dy:, dx:]
762 im2 = diffImage[:nCols - dy, :nRows - dx]
763 w2 = weightImage[:nCols - dy, :nRows - dx]
764 else:
765 im1 = diffImage[:nCols + dy, dx:]
766 w1 = weightImage[:nCols + dy, dx:]
767 im2 = diffImage[-dy:, :nRows - dx]
768 w2 = weightImage[-dy:, :nRows - dx]
769 # use the same mask for all 3 calculations
770 wAll = w1*w2
771 # do not use mean() because weightImage=0 pixels would then count
772 nPix = wAll.sum()
773 im1TimesW = im1*wAll
774 s1 = im1TimesW.sum()/nPix
775 s2 = (im2*wAll).sum()/nPix
776 p = (im1TimesW*im2).sum()/nPix
777 cov = p - s1*s2
779 return cov, nPix
781 def fitNonLinearityAndBuildLinearizers(self, datasetPtc, detector, tableArray=None, log=None):
782 """Fit non-linearity function and build linearizer objects.
784 Parameters
785 ----------
786 datasePtct : `lsst.cp.pipe.ptc.PhotonTransferCurveDataset`
787 The dataset containing information such as the means, variances and exposure times.
788 nLinearity
790 detector : `lsst.afw.cameraGeom.Detector`
791 Detector object.
793 tableArray : `np.array`, optional
794 Optional. Look-up table array with size rows=nAmps and columns=DN values.
795 It will be modified in-place if supplied.
797 log : `lsst.log.Log`, optional
798 Logger to handle messages.
800 Returns
801 -------
802 linearizer : `lsst.ip.isr.Linearizer`
803 Linearizer object
804 """
806 # Fit NonLinearity
807 datasetNonLinearity = self.fitNonLinearity(datasetPtc, tableArray=tableArray)
809 # Produce linearizer
810 now = datetime.datetime.utcnow()
811 calibDate = now.strftime("%Y-%m-%d")
812 linType = self.config.linearizerType
814 if linType == "LOOKUPTABLE":
815 tableArray = tableArray
816 else:
817 tableArray = None
819 linearizer = self.buildLinearizerObject(datasetNonLinearity, detector, calibDate, linType,
820 instruName=self.config.instrumentName,
821 tableArray=tableArray,
822 log=log)
824 return linearizer
826 def fitNonLinearity(self, datasetPtc, tableArray=None):
827 """Fit a polynomial to signal vs effective time curve to calculate linearity and residuals.
829 Parameters
830 ----------
831 datasetPtc : `lsst.cp.pipe.ptc.PhotonTransferCurveDataset`
832 The dataset containing the means, variances and exposure times.
834 tableArray : `np.array`
835 Optional. Look-up table array with size rows=nAmps and columns=DN values.
836 It will be modified in-place if supplied.
838 Returns
839 -------
840 datasetNonLinearity : `dict`
841 Dictionary of `lsst.cp.pipe.ptc.LinearityResidualsAndLinearizersDataset`
842 dataclasses. Each one holds the output of `calculateLinearityResidualAndLinearizers` per
843 amplifier.
844 """
845 datasetNonLinearity = {ampName: [] for ampName in datasetPtc.ampNames}
846 for i, ampName in enumerate(datasetPtc.ampNames):
847 # If a mask is not found, use all points.
848 if (len(datasetPtc.visitMask[ampName]) == 0):
849 self.log.warn(f"Mask not found for {ampName} in non-linearity fit. Using all points.")
850 mask = np.repeat(True, len(datasetPtc.rawExpTimes[ampName]))
851 else:
852 mask = datasetPtc.visitMask[ampName]
854 timeVecFinal = np.array(datasetPtc.rawExpTimes[ampName])[mask]
855 meanVecFinal = np.array(datasetPtc.rawMeans[ampName])[mask]
857 # Non-linearity residuals (NL of mean vs time curve): percentage, and fit to a quadratic function
858 # In this case, len(parsIniNonLinearity) = 3 indicates that we want a quadratic fit
859 datasetLinRes = self.calculateLinearityResidualAndLinearizers(timeVecFinal, meanVecFinal)
861 # LinearizerLookupTable
862 if tableArray is not None:
863 tableArray[i, :] = datasetLinRes.linearizerTableRow
865 datasetNonLinearity[ampName] = datasetLinRes
867 return datasetNonLinearity
869 def calculateLinearityResidualAndLinearizers(self, exposureTimeVector, meanSignalVector):
870 """Calculate linearity residual and fit an n-order polynomial to the mean vs time curve
871 to produce corrections (deviation from linear part of polynomial) for a particular amplifier
872 to populate LinearizeLookupTable.
873 Use the coefficients of this fit to calculate the correction coefficients for LinearizePolynomial
874 and LinearizeSquared."
876 Parameters
877 ---------
879 exposureTimeVector: `list` of `float`
880 List of exposure times for each flat pair
882 meanSignalVector: `list` of `float`
883 List of mean signal from diference image of flat pairs
885 Returns
886 -------
887 dataset : `lsst.cp.pipe.ptc.LinearityResidualsAndLinearizersDataset`
888 The dataset containing the fit parameters, the NL correction coefficients, and the
889 LUT row for the amplifier at hand.
891 Notes
892 -----
893 datase members:
895 dataset.polynomialLinearizerCoefficients : `list` of `float`
896 Coefficients for LinearizePolynomial, where corrImage = uncorrImage + sum_i c_i uncorrImage^(2 +
897 i).
898 c_(j-2) = -k_j/(k_1^j) with units DN^(1-j) (c.f., Eq. 37 of 2003.05978). The units of k_j are
899 DN/t^j, and they are fit from meanSignalVector = k0 + k1*exposureTimeVector +
900 k2*exposureTimeVector^2 + ... + kn*exposureTimeVector^n, with
901 n = "polynomialFitDegreeNonLinearity". k_0 and k_1 and degenerate with bias level and gain,
902 and are not used by the non-linearity correction. Therefore, j = 2...n in the above expression
903 (see `LinearizePolynomial` class in `linearize.py`.)
905 dataset.quadraticPolynomialLinearizerCoefficient : `float`
906 Coefficient for LinearizeSquared, where corrImage = uncorrImage + c0*uncorrImage^2.
907 c0 = -k2/(k1^2), where k1 and k2 are fit from
908 meanSignalVector = k0 + k1*exposureTimeVector + k2*exposureTimeVector^2 +...
909 + kn*exposureTimeVector^n, with n = "polynomialFitDegreeNonLinearity".
911 dataset.linearizerTableRow : `list` of `float`
912 One dimensional array with deviation from linear part of n-order polynomial fit
913 to mean vs time curve. This array will be one row (for the particular amplifier at hand)
914 of the table array for LinearizeLookupTable.
916 dataset.meanSignalVsTimePolyFitPars : `list` of `float`
917 Parameters from n-order polynomial fit to meanSignalVector vs exposureTimeVector.
919 dataset.meanSignalVsTimePolyFitParsErr : `list` of `float`
920 Parameters from n-order polynomial fit to meanSignalVector vs exposureTimeVector.
922 dataset.meanSignalVsTimePolyFitReducedChiSq : `float`
923 Reduced unweighted chi squared from polynomial fit to meanSignalVector vs exposureTimeVector.
924 """
926 # Lookup table linearizer
927 parsIniNonLinearity = self._initialParsForPolynomial(self.config.polynomialFitDegreeNonLinearity + 1)
928 if self.config.doFitBootstrap:
929 parsFit, parsFitErr, reducedChiSquaredNonLinearityFit = fitBootstrap(parsIniNonLinearity,
930 exposureTimeVector,
931 meanSignalVector,
932 funcPolynomial)
933 else:
934 parsFit, parsFitErr, reducedChiSquaredNonLinearityFit = fitLeastSq(parsIniNonLinearity,
935 exposureTimeVector,
936 meanSignalVector,
937 funcPolynomial)
939 # LinearizeLookupTable:
940 # Use linear part to get time at wich signal is maxAduForLookupTableLinearizer DN
941 tMax = (self.config.maxAduForLookupTableLinearizer - parsFit[0])/parsFit[1]
942 timeRange = np.linspace(0, tMax, self.config.maxAduForLookupTableLinearizer)
943 signalIdeal = parsFit[0] + parsFit[1]*timeRange
944 signalUncorrected = funcPolynomial(parsFit, timeRange)
945 linearizerTableRow = signalIdeal - signalUncorrected # LinearizerLookupTable has corrections
946 # LinearizePolynomial and LinearizeSquared:
947 # Check that magnitude of higher order (>= 3) coefficents of the polyFit are small,
948 # i.e., less than threshold = 1e-10 (typical quadratic and cubic coefficents are ~1e-6
949 # and ~1e-12).
950 k1 = parsFit[1]
951 polynomialLinearizerCoefficients = []
952 for i, coefficient in enumerate(parsFit):
953 c = -coefficient/(k1**i)
954 polynomialLinearizerCoefficients.append(c)
955 if np.fabs(c) > 1e-10:
956 msg = f"Coefficient {c} in polynomial fit larger than threshold 1e-10."
957 self.log.warn(msg)
958 # Coefficient for LinearizedSquared. Called "c0" in linearize.py
959 c0 = polynomialLinearizerCoefficients[2]
961 dataset = LinearityResidualsAndLinearizersDataset([], None, [], [], [], None)
962 dataset.polynomialLinearizerCoefficients = polynomialLinearizerCoefficients
963 dataset.quadraticPolynomialLinearizerCoefficient = c0
964 dataset.linearizerTableRow = linearizerTableRow
965 dataset.meanSignalVsTimePolyFitPars = parsFit
966 dataset.meanSignalVsTimePolyFitParsErr = parsFitErr
967 dataset.meanSignalVsTimePolyFitReducedChiSq = reducedChiSquaredNonLinearityFit
969 return dataset
971 def buildLinearizerObject(self, datasetNonLinearity, detector, calibDate, linearizerType, instruName='',
972 tableArray=None, log=None):
973 """Build linearizer object to persist.
975 Parameters
976 ----------
977 datasetNonLinearity : `dict`
978 Dictionary of `lsst.cp.pipe.ptc.LinearityResidualsAndLinearizersDataset` objects.
980 detector : `lsst.afw.cameraGeom.Detector`
981 Detector object
983 calibDate : `datetime.datetime`
984 Calibration date
986 linearizerType : `str`
987 'LOOKUPTABLE', 'LINEARIZESQUARED', or 'LINEARIZEPOLYNOMIAL'
989 instruName : `str`, optional
990 Instrument name
992 tableArray : `np.array`, optional
993 Look-up table array with size rows=nAmps and columns=DN values
995 log : `lsst.log.Log`, optional
996 Logger to handle messages
998 Returns
999 -------
1000 linearizer : `lsst.ip.isr.Linearizer`
1001 Linearizer object
1002 """
1003 detName = detector.getName()
1004 detNum = detector.getId()
1005 if linearizerType == "LOOKUPTABLE":
1006 if tableArray is not None:
1007 linearizer = Linearizer(detector=detector, table=tableArray, log=log)
1008 else:
1009 raise RuntimeError("tableArray must be provided when creating a LookupTable linearizer")
1010 elif linearizerType in ("LINEARIZESQUARED", "LINEARIZEPOLYNOMIAL"):
1011 linearizer = Linearizer(log=log)
1012 else:
1013 raise RuntimeError("Invalid linearizerType {linearizerType} to build a Linearizer object. "
1014 "Supported: 'LOOKUPTABLE', 'LINEARIZESQUARED', or 'LINEARIZEPOLYNOMIAL'")
1015 for i, amp in enumerate(detector.getAmplifiers()):
1016 ampName = amp.getName()
1017 datasetNonLinAmp = datasetNonLinearity[ampName]
1018 if linearizerType == "LOOKUPTABLE":
1019 linearizer.linearityCoeffs[ampName] = [i, 0]
1020 linearizer.linearityType[ampName] = "LookupTable"
1021 elif linearizerType == "LINEARIZESQUARED":
1022 linearizer.fitParams[ampName] = datasetNonLinAmp.meanSignalVsTimePolyFitPars
1023 linearizer.fitParamsErr[ampName] = datasetNonLinAmp.meanSignalVsTimePolyFitParsErr
1024 linearizer.linearityFitReducedChiSquared[ampName] = (
1025 datasetNonLinAmp.meanSignalVsTimePolyFitReducedChiSq)
1026 linearizer.linearityCoeffs[ampName] = [
1027 datasetNonLinAmp.quadraticPolynomialLinearizerCoefficient]
1028 linearizer.linearityType[ampName] = "Squared"
1029 elif linearizerType == "LINEARIZEPOLYNOMIAL":
1030 linearizer.fitParams[ampName] = datasetNonLinAmp.meanSignalVsTimePolyFitPars
1031 linearizer.fitParamsErr[ampName] = datasetNonLinAmp.meanSignalVsTimePolyFitParsErr
1032 linearizer.linearityFitReducedChiSquared[ampName] = (
1033 datasetNonLinAmp.meanSignalVsTimePolyFitReducedChiSq)
1034 # Slice correction coefficients (starting at 2) for polynomial linearizer
1035 # (and squared linearizer above). The first and second are reduntant with
1036 # the bias and gain, respectively, and are not used by LinearizerPolynomial.
1037 polyLinCoeffs = np.array(datasetNonLinAmp.polynomialLinearizerCoefficients[2:])
1038 linearizer.linearityCoeffs[ampName] = polyLinCoeffs
1039 linearizer.linearityType[ampName] = "Polynomial"
1040 linearizer.linearityBBox[ampName] = amp.getBBox()
1041 linearizer.validate()
1042 calibId = f"detectorName={detName} detector={detNum} calibDate={calibDate} ccd={detNum} filter=NONE"
1044 try:
1045 raftName = detName.split("_")[0]
1046 calibId += f" raftName={raftName}"
1047 except Exception:
1048 raftname = "NONE"
1049 calibId += f" raftName={raftname}"
1051 serial = detector.getSerial()
1052 linearizer.updateMetadata(instrumentName=instruName, detectorId=f"{detNum}",
1053 calibId=calibId, serial=serial, detectorName=f"{detName}")
1055 return linearizer
1057 @staticmethod
1058 def _initialParsForPolynomial(order):
1059 assert(order >= 2)
1060 pars = np.zeros(order, dtype=np.float)
1061 pars[0] = 10
1062 pars[1] = 1
1063 pars[2:] = 0.0001
1064 return pars
1066 @staticmethod
1067 def _boundsForPolynomial(initialPars):
1068 lowers = [np.NINF for p in initialPars]
1069 uppers = [np.inf for p in initialPars]
1070 lowers[1] = 0 # no negative gains
1071 return (lowers, uppers)
1073 @staticmethod
1074 def _boundsForAstier(initialPars):
1075 lowers = [np.NINF for p in initialPars]
1076 uppers = [np.inf for p in initialPars]
1077 return (lowers, uppers)
1079 @staticmethod
1080 def _getInitialGoodPoints(means, variances, maxDeviationPositive, maxDeviationNegative):
1081 """Return a boolean array to mask bad points.
1083 A linear function has a constant ratio, so find the median
1084 value of the ratios, and exclude the points that deviate
1085 from that by more than a factor of maxDeviationPositive/negative.
1086 Asymmetric deviations are supported as we expect the PTC to turn
1087 down as the flux increases, but sometimes it anomalously turns
1088 upwards just before turning over, which ruins the fits, so it
1089 is wise to be stricter about restricting positive outliers than
1090 negative ones.
1092 Too high and points that are so bad that fit will fail will be included
1093 Too low and the non-linear points will be excluded, biasing the NL fit."""
1094 ratios = [b/a for (a, b) in zip(means, variances)]
1095 medianRatio = np.median(ratios)
1096 ratioDeviations = [(r/medianRatio)-1 for r in ratios]
1098 # so that it doesn't matter if the deviation is expressed as positive or negative
1099 maxDeviationPositive = abs(maxDeviationPositive)
1100 maxDeviationNegative = -1. * abs(maxDeviationNegative)
1102 goodPoints = np.array([True if (r < maxDeviationPositive and r > maxDeviationNegative)
1103 else False for r in ratioDeviations])
1104 return goodPoints
1106 def _makeZeroSafe(self, array, warn=True, substituteValue=1e-9):
1107 """"""
1108 nBad = Counter(array)[0]
1109 if nBad == 0:
1110 return array
1112 if warn:
1113 msg = f"Found {nBad} zeros in array at elements {[x for x in np.where(array==0)[0]]}"
1114 self.log.warn(msg)
1116 array[array == 0] = substituteValue
1117 return array
1119 def fitPtc(self, dataset, ptcFitType):
1120 """Fit the photon transfer curve to a polynimial or to Astier+19 approximation.
1122 Fit the photon transfer curve with either a polynomial of the order
1123 specified in the task config, or using the Astier approximation.
1125 Sigma clipping is performed iteratively for the fit, as well as an
1126 initial clipping of data points that are more than
1127 config.initialNonLinearityExclusionThreshold away from lying on a
1128 straight line. This other step is necessary because the photon transfer
1129 curve turns over catastrophically at very high flux (because saturation
1130 drops the variance to ~0) and these far outliers cause the initial fit
1131 to fail, meaning the sigma cannot be calculated to perform the
1132 sigma-clipping.
1134 Parameters
1135 ----------
1136 dataset : `lsst.cp.pipe.ptc.PhotonTransferCurveDataset`
1137 The dataset containing the means, variances and exposure times
1139 ptcFitType : `str`
1140 Fit a 'POLYNOMIAL' (degree: 'polynomialFitDegree') or
1141 'EXPAPPROXIMATION' (Eq. 16 of Astier+19) to the PTC
1143 Returns
1144 -------
1145 dataset: `lsst.cp.pipe.ptc.PhotonTransferCurveDataset`
1146 This is the same dataset as the input paramter, however, it has been modified
1147 to include information such as the fit vectors and the fit parameters. See
1148 the class `PhotonTransferCurveDatase`.
1149 """
1151 def errFunc(p, x, y):
1152 return ptcFunc(p, x) - y
1154 sigmaCutPtcOutliers = self.config.sigmaCutPtcOutliers
1155 maxIterationsPtcOutliers = self.config.maxIterationsPtcOutliers
1157 for i, ampName in enumerate(dataset.ampNames):
1158 timeVecOriginal = np.array(dataset.rawExpTimes[ampName])
1159 meanVecOriginal = np.array(dataset.rawMeans[ampName])
1160 varVecOriginal = np.array(dataset.rawVars[ampName])
1161 varVecOriginal = self._makeZeroSafe(varVecOriginal)
1163 mask = ((meanVecOriginal >= self.config.minMeanSignal) &
1164 (meanVecOriginal <= self.config.maxMeanSignal))
1166 goodPoints = self._getInitialGoodPoints(meanVecOriginal, varVecOriginal,
1167 self.config.initialNonLinearityExclusionThresholdPositive,
1168 self.config.initialNonLinearityExclusionThresholdNegative)
1169 mask = mask & goodPoints
1171 if ptcFitType == 'EXPAPPROXIMATION':
1172 ptcFunc = funcAstier
1173 parsIniPtc = [-1e-9, 1.0, 10.] # a00, gain, noise
1174 bounds = self._boundsForAstier(parsIniPtc)
1175 if ptcFitType == 'POLYNOMIAL':
1176 ptcFunc = funcPolynomial
1177 parsIniPtc = self._initialParsForPolynomial(self.config.polynomialFitDegree + 1)
1178 bounds = self._boundsForPolynomial(parsIniPtc)
1180 # Before bootstrap fit, do an iterative fit to get rid of outliers
1181 count = 1
1182 while count <= maxIterationsPtcOutliers:
1183 # Note that application of the mask actually shrinks the array
1184 # to size rather than setting elements to zero (as we want) so
1185 # always update mask itself and re-apply to the original data
1186 meanTempVec = meanVecOriginal[mask]
1187 varTempVec = varVecOriginal[mask]
1188 res = least_squares(errFunc, parsIniPtc, bounds=bounds, args=(meanTempVec, varTempVec))
1189 pars = res.x
1191 # change this to the original from the temp because the masks are ANDed
1192 # meaning once a point is masked it's always masked, and the masks must
1193 # always be the same length for broadcasting
1194 sigResids = (varVecOriginal - ptcFunc(pars, meanVecOriginal))/np.sqrt(varVecOriginal)
1195 newMask = np.array([True if np.abs(r) < sigmaCutPtcOutliers else False for r in sigResids])
1196 mask = mask & newMask
1198 nDroppedTotal = Counter(mask)[False]
1199 self.log.debug(f"Iteration {count}: discarded {nDroppedTotal} points in total for {ampName}")
1200 count += 1
1201 # objects should never shrink
1202 assert (len(mask) == len(timeVecOriginal) == len(meanVecOriginal) == len(varVecOriginal))
1204 dataset.visitMask[ampName] = mask # store the final mask
1205 parsIniPtc = pars
1206 meanVecFinal = meanVecOriginal[mask]
1207 varVecFinal = varVecOriginal[mask]
1209 if Counter(mask)[False] > 0:
1210 self.log.info((f"Number of points discarded in PTC of amplifier {ampName}:" +
1211 f" {Counter(mask)[False]} out of {len(meanVecOriginal)}"))
1213 if (len(meanVecFinal) < len(parsIniPtc)):
1214 msg = (f"\nSERIOUS: Not enough data points ({len(meanVecFinal)}) compared to the number of"
1215 f"parameters of the PTC model({len(parsIniPtc)}). Setting {ampName} to BAD.")
1216 self.log.warn(msg)
1217 # The first and second parameters of initial fit are discarded (bias and gain)
1218 # for the final NL coefficients
1219 dataset.badAmps.append(ampName)
1220 dataset.gain[ampName] = np.nan
1221 dataset.gainErr[ampName] = np.nan
1222 dataset.noise[ampName] = np.nan
1223 dataset.noiseErr[ampName] = np.nan
1224 dataset.ptcFitPars[ampName] = np.nan
1225 dataset.ptcFitParsError[ampName] = np.nan
1226 dataset.ptcFitReducedChiSquared[ampName] = np.nan
1227 continue
1229 # Fit the PTC
1230 if self.config.doFitBootstrap:
1231 parsFit, parsFitErr, reducedChiSqPtc = fitBootstrap(parsIniPtc, meanVecFinal,
1232 varVecFinal, ptcFunc)
1233 else:
1234 parsFit, parsFitErr, reducedChiSqPtc = fitLeastSq(parsIniPtc, meanVecFinal,
1235 varVecFinal, ptcFunc)
1236 dataset.ptcFitPars[ampName] = parsFit
1237 dataset.ptcFitParsError[ampName] = parsFitErr
1238 dataset.ptcFitReducedChiSquared[ampName] = reducedChiSqPtc
1240 if ptcFitType == 'EXPAPPROXIMATION':
1241 ptcGain = parsFit[1]
1242 ptcGainErr = parsFitErr[1]
1243 ptcNoise = np.sqrt(np.fabs(parsFit[2]))
1244 ptcNoiseErr = 0.5*(parsFitErr[2]/np.fabs(parsFit[2]))*np.sqrt(np.fabs(parsFit[2]))
1245 if ptcFitType == 'POLYNOMIAL':
1246 ptcGain = 1./parsFit[1]
1247 ptcGainErr = np.fabs(1./parsFit[1])*(parsFitErr[1]/parsFit[1])
1248 ptcNoise = np.sqrt(np.fabs(parsFit[0]))*ptcGain
1249 ptcNoiseErr = (0.5*(parsFitErr[0]/np.fabs(parsFit[0]))*(np.sqrt(np.fabs(parsFit[0]))))*ptcGain
1250 dataset.gain[ampName] = ptcGain
1251 dataset.gainErr[ampName] = ptcGainErr
1252 dataset.noise[ampName] = ptcNoise
1253 dataset.noiseErr[ampName] = ptcNoiseErr
1254 if not len(dataset.ptcFitType) == 0:
1255 dataset.ptcFitType = ptcFitType
1257 return dataset