Coverage for python/lsst/cp/pipe/plotPtc.py : 6%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of cp_pipe.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21#
23__all__ = ['PlotPhotonTransferCurveTask']
25import numpy as np
26import matplotlib.pyplot as plt
27import matplotlib as mpl
28from matplotlib import gridspec
29import os
30from matplotlib.backends.backend_pdf import PdfPages
32import lsst.ip.isr as isr
33import lsst.pex.config as pexConfig
34import lsst.pipe.base as pipeBase
36from .utils import (funcAstier, funcPolynomial, NonexistentDatasetTaskDataIdContainer,
37 calculateWeightedReducedChi2)
38from matplotlib.ticker import MaxNLocator
40from .astierCovPtcFit import computeApproximateAcoeffs
41from .astierCovPtcUtils import getFitDataFromCovariances
43from lsst.ip.isr import PhotonTransferCurveDataset
46class PlotPhotonTransferCurveTaskConfig(pexConfig.Config):
47 """Config class for photon transfer curve measurement task"""
48 datasetFileName = pexConfig.Field(
49 dtype=str,
50 doc="datasetPtc file name (pkl)",
51 default="",
52 )
53 linearizerFileName = pexConfig.Field(
54 dtype=str,
55 doc="linearizer file name (fits)",
56 default="",
57 )
58 ccdKey = pexConfig.Field(
59 dtype=str,
60 doc="The key by which to pull a detector from a dataId, e.g. 'ccd' or 'detector'.",
61 default='detector',
62 )
63 signalElectronsRelativeA = pexConfig.Field(
64 dtype=float,
65 doc="Signal value for relative systematic bias between different methods of estimating a_ij "
66 "(Fig. 15 of Astier+19).",
67 default=75000,
68 )
69 plotNormalizedCovariancesNumberOfBins = pexConfig.Field(
70 dtype=int,
71 doc="Number of bins in `plotNormalizedCovariancesNumber` function "
72 "(Fig. 8, 10., of Astier+19).",
73 default=10,
74 )
77class PlotPhotonTransferCurveTask(pipeBase.CmdLineTask):
78 """A class to plot the dataset from MeasurePhotonTransferCurveTask.
80 Parameters
81 ----------
83 *args: `list`
84 Positional arguments passed to the Task constructor. None used at this
85 time.
86 **kwargs: `dict`
87 Keyword arguments passed on to the Task constructor. None used at this
88 time.
90 """
92 ConfigClass = PlotPhotonTransferCurveTaskConfig
93 _DefaultName = "plotPhotonTransferCurve"
95 def __init__(self, *args, **kwargs):
96 pipeBase.CmdLineTask.__init__(self, *args, **kwargs)
97 plt.interactive(False) # stop windows popping up when plotting. When headless, use 'agg' backend too
98 self.config.validate()
99 self.config.freeze()
101 @classmethod
102 def _makeArgumentParser(cls):
103 """Augment argument parser for the MeasurePhotonTransferCurveTask."""
104 parser = pipeBase.ArgumentParser(name=cls._DefaultName)
105 parser.add_id_argument("--id", datasetType="photonTransferCurveDataset",
106 ContainerClass=NonexistentDatasetTaskDataIdContainer,
107 help="The ccds to use, e.g. --id ccd=0..100")
108 return parser
110 @pipeBase.timeMethod
111 def runDataRef(self, dataRef):
112 """Run the Photon Transfer Curve (PTC) plotting measurement task.
114 Parameters
115 ----------
116 dataRef : list of lsst.daf.persistence.ButlerDataRef
117 dataRef for the detector for the expIds to be fit.
118 """
120 datasetFile = self.config.datasetFileName
121 datasetPtc = PhotonTransferCurveDataset.readFits(datasetFile)
123 dirname = dataRef.getUri(datasetType='cpPipePlotRoot', write=True)
124 if not os.path.exists(dirname):
125 os.makedirs(dirname)
127 detNum = dataRef.dataId[self.config.ccdKey]
128 filename = f"PTC_det{detNum}.pdf"
129 filenameFull = os.path.join(dirname, filename)
131 if self.config.linearizerFileName:
132 linearizer = isr.linearize.Linearizer.readFits(self.config.linearizerFileName)
133 else:
134 linearizer = None
135 self.run(filenameFull, datasetPtc, linearizer=linearizer, log=self.log)
137 return pipeBase.Struct(exitStatus=0)
139 def run(self, filenameFull, datasetPtc, linearizer=None, log=None):
140 """Make the plots for the PTC task"""
141 ptcFitType = datasetPtc.ptcFitType
142 with PdfPages(filenameFull) as pdfPages:
143 if ptcFitType in ["FULLCOVARIANCE", ]:
144 self.covAstierMakeAllPlots(datasetPtc, pdfPages, log=log)
145 elif ptcFitType in ["EXPAPPROXIMATION", "POLYNOMIAL"]:
146 self._plotStandardPtc(datasetPtc, ptcFitType, pdfPages)
147 else:
148 raise RuntimeError(f"The input dataset had an invalid dataset.ptcFitType: {ptcFitType}. \n" +
149 "Options: 'FULLCOVARIANCE', EXPAPPROXIMATION, or 'POLYNOMIAL'.")
150 if linearizer:
151 self._plotLinearizer(datasetPtc, linearizer, pdfPages)
153 return
155 def covAstierMakeAllPlots(self, dataset, pdfPages,
156 log=None):
157 """Make plots for MeasurePhotonTransferCurve task when doCovariancesAstier=True.
159 This function call other functions that mostly reproduce the plots in Astier+19.
160 Most of the code is ported from Pierre Astier's repository https://github.com/PierreAstier/bfptc
162 Parameters
163 ----------
164 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset`
165 The dataset containing the necessary information to produce the plots.
167 pdfPages: `matplotlib.backends.backend_pdf.PdfPages`
168 PDF file where the plots will be saved.
170 log : `lsst.log.Log`, optional
171 Logger to handle messages
172 """
173 mu = dataset.rawMeans
174 # dictionaries with ampNames as keys
175 fullCovs = dataset.covariances
176 fullCovsModel = dataset.covariancesModel
177 fullCovWeights = dataset.covariancesSqrtWeights
178 aDict = dataset.aMatrix
179 bDict = dataset.bMatrix
180 fullCovsNoB = dataset.covariancesNoB
181 fullCovsModelNoB = dataset.covariancesModelNoB
182 fullCovWeightsNoB = dataset.covariancesSqrtWeightsNoB
183 aDictNoB = dataset.aMatrixNoB
184 gainDict = dataset.gain
185 noiseDict = dataset.noise
187 self.plotCovariances(mu, fullCovs, fullCovsModel, fullCovWeights, fullCovsNoB, fullCovsModelNoB,
188 fullCovWeightsNoB, gainDict, noiseDict, aDict, bDict, pdfPages)
189 self.plotNormalizedCovariances(0, 0, mu, fullCovs, fullCovsModel, fullCovWeights, fullCovsNoB,
190 fullCovsModelNoB, fullCovWeightsNoB, pdfPages, offset=0.01,
191 topPlot=True,
192 numberOfBins=self.config.plotNormalizedCovariancesNumberOfBins,
193 log=log)
194 self.plotNormalizedCovariances(0, 1, mu, fullCovs, fullCovsModel, fullCovWeights, fullCovsNoB,
195 fullCovsModelNoB, fullCovWeightsNoB, pdfPages,
196 numberOfBins=self.config.plotNormalizedCovariancesNumberOfBins,
197 log=log)
198 self.plotNormalizedCovariances(1, 0, mu, fullCovs, fullCovsModel, fullCovWeights, fullCovsNoB,
199 fullCovsModelNoB, fullCovWeightsNoB, pdfPages,
200 numberOfBins=self.config.plotNormalizedCovariancesNumberOfBins,
201 log=log)
202 self.plot_a_b(aDict, bDict, pdfPages)
203 self.ab_vs_dist(aDict, bDict, pdfPages, bRange=4)
204 self.plotAcoeffsSum(aDict, bDict, pdfPages)
205 self.plotRelativeBiasACoeffs(aDict, aDictNoB, fullCovsModel, fullCovsModelNoB,
206 self.config.signalElectronsRelativeA, gainDict, pdfPages, maxr=4)
208 return
210 @staticmethod
211 def plotCovariances(mu, covs, covsModel, covsWeights, covsNoB, covsModelNoB, covsWeightsNoB,
212 gainDict, noiseDict, aDict, bDict, pdfPages):
213 """Plot covariances and models: Cov00, Cov10, Cov01.
215 Figs. 6 and 7 of Astier+19
217 Parameters
218 ----------
219 mu : `dict`, [`str`, `list`]
220 Dictionary keyed by amp name with mean signal values.
222 covs : `dict`, [`str`, `list`]
223 Dictionary keyed by amp names containing a list of measued covariances per mean flux.
225 covsModel : `dict`, [`str`, `list`]
226 Dictionary keyed by amp names containinging covariances model (Eq. 20 of Astier+19) per mean flux.
228 covsWeights : `dict`, [`str`, `list`]
229 Dictionary keyed by amp names containinging sqrt. of covariances weights.
231 covsNoB : `dict`, [`str`, `list`]
232 Dictionary keyed by amp names containing a list of measued covariances per mean flux ('b'=0 in
233 Astier+19).
235 covsModelNoB : `dict`, [`str`, `list`]
236 Dictionary keyed by amp names containing covariances model (with 'b'=0 in Eq. 20 of Astier+19)
237 per mean flux.
239 covsWeightsNoB : `dict`, [`str`, `list`]
240 Dictionary keyed by amp names containing sqrt. of covariances weights ('b' = 0 in Eq. 20 of
241 Astier+19).
243 gainDict : `dict`, [`str`, `float`]
244 Dictionary keyed by amp names containing the gains in e-/ADU.
246 noiseDict : `dict`, [`str`, `float`]
247 Dictionary keyed by amp names containing the rms redout noise in e-.
249 aDict : `dict`, [`str`, `numpy.array`]
250 Dictionary keyed by amp names containing 'a' coefficients (Eq. 20 of Astier+19).
252 bDict : `dict`, [`str`, `numpy.array`]
253 Dictionary keyed by amp names containing 'b' coefficients (Eq. 20 of Astier+19).
255 pdfPages: `matplotlib.backends.backend_pdf.PdfPages`
256 PDF file where the plots will be saved.
257 """
259 legendFontSize = 6.5
260 labelFontSize = 7
261 titleFontSize = 9
262 supTitleFontSize = 18
263 markerSize = 25
265 nAmps = len(covs)
266 if nAmps == 2:
267 nRows, nCols = 2, 1
268 nRows = np.sqrt(nAmps)
269 mantissa, _ = np.modf(nRows)
270 if mantissa > 0:
271 nRows = int(nRows) + 1
272 nCols = nRows
273 else:
274 nRows = int(nRows)
275 nCols = nRows
277 f, ax = plt.subplots(nrows=nRows, ncols=nCols, sharex='col', sharey='row', figsize=(13, 10))
278 f2, ax2 = plt.subplots(nrows=nRows, ncols=nCols, sharex='col', sharey='row', figsize=(13, 10))
279 fResCov00, axResCov00 = plt.subplots(nrows=nRows, ncols=nCols, sharex='col', sharey='row',
280 figsize=(13, 10))
281 fCov01, axCov01 = plt.subplots(nrows=nRows, ncols=nCols, sharex='col', sharey='row', figsize=(13, 10))
282 fCov10, axCov10 = plt.subplots(nrows=nRows, ncols=nCols, sharex='col', sharey='row', figsize=(13, 10))
284 assert(len(covsModel) == nAmps)
285 assert(len(covsWeights) == nAmps)
287 assert(len(covsNoB) == nAmps)
288 assert(len(covsModelNoB) == nAmps)
289 assert(len(covsWeightsNoB) == nAmps)
291 for i, (amp, a, a2, aResVar, a3, a4) in enumerate(zip(covs, ax.flatten(),
292 ax2.flatten(), axResCov00.flatten(),
293 axCov01.flatten(), axCov10.flatten())):
295 muAmp, cov, model, weight = mu[amp], covs[amp], covsModel[amp], covsWeights[amp]
296 if not np.isnan(np.array(cov)).all(): # If all the entries are np.nan, this is a bad amp.
297 aCoeffs, bCoeffs = np.array(aDict[amp]), np.array(bDict[amp])
298 gain, noise = gainDict[amp], noiseDict[amp]
299 (meanVecOriginal, varVecOriginal, varVecModelOriginal,
300 weightsOriginal, varMask) = getFitDataFromCovariances(0, 0, muAmp, cov, model, weight)
301 meanVecFinal, varVecFinal = meanVecOriginal[varMask], varVecOriginal[varMask]
302 varVecModelFinal = varVecModelOriginal[varMask]
303 meanVecOutliers = meanVecOriginal[np.invert(varMask)]
304 varVecOutliers = varVecOriginal[np.invert(varMask)]
305 varWeightsFinal = weightsOriginal[varMask]
306 # Get weighted reduced chi2
307 chi2FullModelVar = calculateWeightedReducedChi2(varVecFinal, varVecModelFinal,
308 varWeightsFinal, len(meanVecFinal), 4)
310 (meanVecOrigCov01, varVecOrigCov01, varVecModelOrigCov01,
311 _, maskCov01) = getFitDataFromCovariances(0, 0, muAmp, cov, model, weight)
312 meanVecFinalCov01, varVecFinalCov01 = meanVecOrigCov01[maskCov01], varVecOrigCov01[maskCov01]
313 varVecModelFinalCov01 = varVecModelOrigCov01[maskCov01]
314 meanVecOutliersCov01 = meanVecOrigCov01[np.invert(maskCov01)]
315 varVecOutliersCov01 = varVecOrigCov01[np.invert(maskCov01)]
317 (meanVecOrigCov10, varVecOrigCov10, varVecModelOrigCov10,
318 _, maskCov10) = getFitDataFromCovariances(1, 0, muAmp, cov, model, weight)
319 meanVecFinalCov10, varVecFinalCov10 = meanVecOrigCov10[maskCov10], varVecOrigCov10[maskCov10]
320 varVecModelFinalCov10 = varVecModelOrigCov10[maskCov10]
321 meanVecOutliersCov10 = meanVecOrigCov10[np.invert(maskCov10)]
322 varVecOutliersCov10 = varVecOrigCov10[np.invert(maskCov10)]
324 # cuadratic fit for residuals below
325 par2 = np.polyfit(meanVecFinal, varVecFinal, 2, w=varWeightsFinal)
326 varModelFinalQuadratic = np.polyval(par2, meanVecFinal)
327 chi2QuadModelVar = calculateWeightedReducedChi2(varVecFinal, varModelFinalQuadratic,
328 varWeightsFinal, len(meanVecFinal), 3)
330 # fit with no 'b' coefficient (c = a*b in Eq. 20 of Astier+19)
331 covNoB, modelNoB, weightNoB = covsNoB[amp], covsModelNoB[amp], covsWeightsNoB[amp]
332 (meanVecFinalNoB, varVecFinalNoB, varVecModelFinalNoB,
333 varWeightsFinalNoB, maskNoB) = getFitDataFromCovariances(0, 0, muAmp, covNoB, modelNoB,
334 weightNoB, returnMasked=True)
335 chi2FullModelNoBVar = calculateWeightedReducedChi2(varVecFinalNoB, varVecModelFinalNoB,
336 varWeightsFinalNoB, len(meanVecFinalNoB),
337 3)
338 stringLegend = (f"Gain: {gain:.4} e/ADU \n" +
339 f"Noise: {noise:.4} e \n" +
340 r"$a_{00}$: %.3e 1/e"%aCoeffs[0, 0] +
341 "\n" + r"$b_{00}$: %.3e 1/e"%bCoeffs[0, 0] +
342 f"\nLast in fit: {meanVecFinal[-1]:.7} ADU ")
343 minMeanVecFinal = np.nanmin(meanVecFinal)
344 maxMeanVecFinal = np.nanmax(meanVecFinal)
345 deltaXlim = maxMeanVecFinal - minMeanVecFinal
347 a.set_xlabel(r'Mean signal ($\mu$, ADU)', fontsize=labelFontSize)
348 a.set_ylabel(r'Variance (ADU$^2$)', fontsize=labelFontSize)
349 a.tick_params(labelsize=11)
350 a.set_xscale('linear', fontsize=labelFontSize)
351 a.set_yscale('linear', fontsize=labelFontSize)
352 a.scatter(meanVecFinal, varVecFinal, c='blue', marker='o', s=markerSize)
353 a.scatter(meanVecOutliers, varVecOutliers, c='magenta', marker='s', s=markerSize)
354 a.plot(meanVecFinal, varVecModelFinal, color='red', lineStyle='-')
355 a.text(0.03, 0.7, stringLegend, transform=a.transAxes, fontsize=legendFontSize)
356 a.set_title(amp, fontsize=titleFontSize)
357 a.set_xlim([minMeanVecFinal - 0.2*deltaXlim, maxMeanVecFinal + 0.2*deltaXlim])
359 # Same as above, but in log-scale
360 a2.set_xlabel(r'Mean Signal ($\mu$, ADU)', fontsize=labelFontSize)
361 a2.set_ylabel(r'Variance (ADU$^2$)', fontsize=labelFontSize)
362 a2.tick_params(labelsize=11)
363 a2.set_xscale('log')
364 a2.set_yscale('log')
365 a2.plot(meanVecFinal, varVecModelFinal, color='red', lineStyle='-')
366 a2.scatter(meanVecFinal, varVecFinal, c='blue', marker='o', s=markerSize)
367 a2.scatter(meanVecOutliers, varVecOutliers, c='magenta', marker='s', s=markerSize)
368 a2.text(0.03, 0.7, stringLegend, transform=a2.transAxes, fontsize=legendFontSize)
369 a2.set_title(amp, fontsize=titleFontSize)
370 a2.set_xlim([minMeanVecFinal, maxMeanVecFinal])
372 # Residuals var - model
373 aResVar.set_xlabel(r'Mean signal ($\mu$, ADU)', fontsize=labelFontSize)
374 aResVar.set_ylabel(r'Residuals (ADU$^2$)', fontsize=labelFontSize)
375 aResVar.tick_params(labelsize=11)
376 aResVar.set_xscale('linear', fontsize=labelFontSize)
377 aResVar.set_yscale('linear', fontsize=labelFontSize)
378 aResVar.plot(meanVecFinal, varVecFinal - varVecModelFinal, color='blue', lineStyle='-',
379 label=r'Full fit ($\chi_{\rm{red}}^2$: %g)'%chi2FullModelVar)
380 aResVar.plot(meanVecFinal, varVecFinal - varModelFinalQuadratic, color='red', lineStyle='-',
381 label=r'Quadratic fit ($\chi_{\rm{red}}^2$: %g)'%chi2QuadModelVar)
382 aResVar.plot(meanVecFinalNoB, varVecFinalNoB - varVecModelFinalNoB, color='green',
383 lineStyle='-',
384 label=r'Full fit (b=0) ($\chi_{\rm{red}}^2$: %g)'%chi2FullModelNoBVar)
385 aResVar.axhline(color='black')
386 aResVar.set_title(amp, fontsize=titleFontSize)
387 aResVar.set_xlim([minMeanVecFinal - 0.2*deltaXlim, maxMeanVecFinal + 0.2*deltaXlim])
388 aResVar.legend(fontsize=7)
390 a3.set_xlabel(r'Mean signal ($\mu$, ADU)', fontsize=labelFontSize)
391 a3.set_ylabel(r'Cov01 (ADU$^2$)', fontsize=labelFontSize)
392 a3.tick_params(labelsize=11)
393 a3.set_xscale('linear', fontsize=labelFontSize)
394 a3.set_yscale('linear', fontsize=labelFontSize)
395 a3.scatter(meanVecFinalCov01, varVecFinalCov01, c='blue', marker='o', s=markerSize)
396 a3.scatter(meanVecOutliersCov01, varVecOutliersCov01, c='magenta', marker='s', s=markerSize)
397 a3.plot(meanVecFinalCov01, varVecModelFinalCov01, color='red', lineStyle='-')
398 a3.set_title(amp, fontsize=titleFontSize)
399 a3.set_xlim([minMeanVecFinal - 0.2*deltaXlim, maxMeanVecFinal + 0.2*deltaXlim])
401 a4.set_xlabel(r'Mean signal ($\mu$, ADU)', fontsize=labelFontSize)
402 a4.set_ylabel(r'Cov10 (ADU$^2$)', fontsize=labelFontSize)
403 a4.tick_params(labelsize=11)
404 a4.set_xscale('linear', fontsize=labelFontSize)
405 a4.set_yscale('linear', fontsize=labelFontSize)
406 a4.scatter(meanVecFinalCov10, varVecFinalCov10, c='blue', marker='o', s=markerSize)
407 a4.scatter(meanVecOutliersCov10, varVecOutliersCov10, c='magenta', marker='s', s=markerSize)
408 a4.plot(meanVecFinalCov10, varVecModelFinalCov10, color='red', lineStyle='-')
409 a4.set_title(amp, fontsize=titleFontSize)
410 a4.set_xlim([minMeanVecFinal - 0.2*deltaXlim, maxMeanVecFinal + 0.2*deltaXlim])
412 else:
413 a.set_title(f"{amp} (BAD)", fontsize=titleFontSize)
414 a2.set_title(f"{amp} (BAD)", fontsize=titleFontSize)
415 a3.set_title(f"{amp} (BAD)", fontsize=titleFontSize)
416 a4.set_title(f"{amp} (BAD)", fontsize=titleFontSize)
418 f.suptitle("PTC from covariances as in Astier+19 \n Fit: Eq. 20, Astier+19",
419 fontsize=supTitleFontSize)
420 pdfPages.savefig(f)
421 f2.suptitle("PTC from covariances as in Astier+19 (log-log) \n Fit: Eq. 20, Astier+19",
422 fontsize=supTitleFontSize)
423 pdfPages.savefig(f2)
424 fResCov00.suptitle("Residuals (data-model) for Cov00 (Var)", fontsize=supTitleFontSize)
425 pdfPages.savefig(fResCov00)
426 fCov01.suptitle("Cov01 as in Astier+19 (nearest parallel neighbor covariance) \n" +
427 " Fit: Eq. 20, Astier+19", fontsize=supTitleFontSize)
428 pdfPages.savefig(fCov01)
429 fCov10.suptitle("Cov10 as in Astier+19 (nearest serial neighbor covariance) \n" +
430 "Fit: Eq. 20, Astier+19", fontsize=supTitleFontSize)
431 pdfPages.savefig(fCov10)
433 return
435 def plotNormalizedCovariances(self, i, j, inputMu, covs, covsModel, covsWeights, covsNoB, covsModelNoB,
436 covsWeightsNoB, pdfPages, offset=0.004,
437 numberOfBins=10, plotData=True, topPlot=False, log=None):
438 """Plot C_ij/mu vs mu.
440 Figs. 8, 10, and 11 of Astier+19
442 Parameters
443 ----------
444 i : `int`
445 Covariane lag
447 j : `int`
448 Covariance lag
450 inputMu : `dict`, [`str`, `list`]
451 Dictionary keyed by amp name with mean signal values.
453 covs : `dict`, [`str`, `list`]
454 Dictionary keyed by amp names containing a list of measued covariances per mean flux.
456 covsModel : `dict`, [`str`, `list`]
457 Dictionary keyed by amp names containinging covariances model (Eq. 20 of Astier+19) per mean flux.
459 covsWeights : `dict`, [`str`, `list`]
460 Dictionary keyed by amp names containinging sqrt. of covariances weights.
462 covsNoB : `dict`, [`str`, `list`]
463 Dictionary keyed by amp names containing a list of measued covariances per mean flux ('b'=0 in
464 Astier+19).
466 covsModelNoB : `dict`, [`str`, `list`]
467 Dictionary keyed by amp names containing covariances model (with 'b'=0 in Eq. 20 of Astier+19)
468 per mean flux.
470 covsWeightsNoB : `dict`, [`str`, `list`]
471 Dictionary keyed by amp names containing sqrt. of covariances weights ('b' = 0 in Eq. 20 of
472 Astier+19).
474 pdfPages: `matplotlib.backends.backend_pdf.PdfPages`
475 PDF file where the plots will be saved.
477 offset : `float`, optional
478 Constant offset factor to plot covariances in same panel (so they don't overlap).
480 numberOfBins : `int`, optional
481 Number of bins for top and bottom plot.
483 plotData : `bool`, optional
484 Plot the data points?
486 topPlot : `bool`, optional
487 Plot the top plot with the covariances, and the bottom plot with the model residuals?
489 log : `lsst.log.Log`, optional
490 Logger to handle messages.
491 """
492 if (not topPlot):
493 fig = plt.figure(figsize=(8, 10))
494 gs = gridspec.GridSpec(2, 1, height_ratios=[3, 1])
495 gs.update(hspace=0)
496 ax0 = plt.subplot(gs[0])
497 plt.setp(ax0.get_xticklabels(), visible=False)
498 else:
499 fig = plt.figure(figsize=(8, 8))
500 ax0 = plt.subplot(111)
501 ax0.ticklabel_format(style='sci', axis='x', scilimits=(0, 0))
502 ax0.tick_params(axis='both', labelsize='x-large')
503 mue, rese, wce = [], [], []
504 mueNoB, reseNoB, wceNoB = [], [], []
505 for counter, amp in enumerate(covs):
506 muAmp, fullCov, fullCovModel, fullCovWeight = (inputMu[amp], covs[amp], covsModel[amp],
507 covsWeights[amp])
508 if len(fullCov) == 0:
509 continue
510 mu, cov, model, weightCov, _ = getFitDataFromCovariances(i, j, muAmp, fullCov, fullCovModel,
511 fullCovWeight, divideByMu=True,
512 returnMasked=True)
513 mue += list(mu)
514 rese += list(cov - model)
515 wce += list(weightCov)
517 fullCovNoB, fullCovModelNoB, fullCovWeightNoB = (covsNoB[amp], covsModelNoB[amp],
518 covsWeightsNoB[amp])
519 if len(fullCovNoB) == 0:
520 continue
521 (muNoB, covNoB, modelNoB,
522 weightCovNoB, _) = getFitDataFromCovariances(i, j, muAmp, fullCovNoB, fullCovModelNoB,
523 fullCovWeightNoB, divideByMu=True,
524 returnMasked=True)
525 mueNoB += list(muNoB)
526 reseNoB += list(covNoB - modelNoB)
527 wceNoB += list(weightCovNoB)
529 # the corresponding fit
530 fit_curve, = plt.plot(mu, model + counter*offset, '-', linewidth=4.0)
531 # bin plot. len(mu) = no binning
532 gind = self.indexForBins(mu, numberOfBins)
534 xb, yb, wyb, sigyb = self.binData(mu, cov, gind, weightCov)
535 plt.errorbar(xb, yb+counter*offset, yerr=sigyb, marker='o', linestyle='none', markersize=6.5,
536 color=fit_curve.get_color(), label=f"{amp} (N: {len(mu)})")
537 # plot the data
538 if plotData:
539 points, = plt.plot(mu, cov + counter*offset, '.', color=fit_curve.get_color())
540 plt.legend(loc='upper right', fontsize=8)
541 # end loop on amps
542 mue = np.array(mue)
543 rese = np.array(rese)
544 wce = np.array(wce)
545 mueNoB = np.array(mueNoB)
546 reseNoB = np.array(reseNoB)
547 wceNoB = np.array(wceNoB)
549 plt.xlabel(r"$\mu (el)$", fontsize='x-large')
550 plt.ylabel(r"$Cov{%d%d}/\mu + Cst (el)$"%(i, j), fontsize='x-large')
551 if (not topPlot):
552 gind = self.indexForBins(mue, numberOfBins)
553 xb, yb, wyb, sigyb = self.binData(mue, rese, gind, wce)
555 ax1 = plt.subplot(gs[1], sharex=ax0)
556 ax1.errorbar(xb, yb, yerr=sigyb, marker='o', linestyle='none', label='Full fit')
557 gindNoB = self.indexForBins(mueNoB, numberOfBins)
558 xb2, yb2, wyb2, sigyb2 = self.binData(mueNoB, reseNoB, gindNoB, wceNoB)
560 ax1.errorbar(xb2, yb2, yerr=sigyb2, marker='o', linestyle='none', label='b = 0')
561 ax1.tick_params(axis='both', labelsize='x-large')
562 plt.legend(loc='upper left', fontsize='large')
563 # horizontal line at zero
564 plt.plot(xb, [0]*len(xb), '--', color='k')
565 plt.ticklabel_format(style='sci', axis='x', scilimits=(0, 0))
566 plt.ticklabel_format(style='sci', axis='y', scilimits=(0, 0))
567 plt.xlabel(r'$\mu (el)$', fontsize='x-large')
568 plt.ylabel(r'$Cov{%d%d}/\mu$ -model (el)'%(i, j), fontsize='x-large')
569 plt.tight_layout()
570 plt.suptitle(f"Nbins: {numberOfBins}")
571 # overlapping y labels:
572 fig.canvas.draw()
573 labels0 = [item.get_text() for item in ax0.get_yticklabels()]
574 labels0[0] = u''
575 ax0.set_yticklabels(labels0)
576 pdfPages.savefig(fig)
578 return
580 @staticmethod
581 def plot_a_b(aDict, bDict, pdfPages, bRange=3):
582 """Fig. 12 of Astier+19
584 Color display of a and b arrays fits, averaged over channels.
586 Parameters
587 ----------
588 aDict : `dict`, [`numpy.array`]
589 Dictionary keyed by amp names containing the fitted 'a' coefficients from the model
590 in Eq. 20 of Astier+19 (if `ptcFitType` is `FULLCOVARIANCE`).
592 bDict : `dict`, [`numpy.array`]
593 Dictionary keyed by amp names containing the fitted 'b' coefficients from the model
594 in Eq. 20 of Astier+19 (if `ptcFitType` is `FULLCOVARIANCE`).
596 pdfPages: `matplotlib.backends.backend_pdf.PdfPages`
597 PDF file where the plots will be saved.
599 bRange : `int`
600 Maximum lag for b arrays.
601 """
602 a, b = [], []
603 for amp in aDict:
604 if np.isnan(aDict[amp]).all():
605 continue
606 a.append(aDict[amp])
607 b.append(bDict[amp])
608 a = np.array(a).mean(axis=0)
609 b = np.array(b).mean(axis=0)
610 fig = plt.figure(figsize=(7, 11))
611 ax0 = fig.add_subplot(2, 1, 1)
612 im0 = ax0.imshow(np.abs(a.transpose()), origin='lower', norm=mpl.colors.LogNorm())
613 ax0.tick_params(axis='both', labelsize='x-large')
614 ax0.set_title(r'$|a|$', fontsize='x-large')
615 ax0.xaxis.set_ticks_position('bottom')
616 cb0 = plt.colorbar(im0)
617 cb0.ax.tick_params(labelsize='x-large')
619 ax1 = fig.add_subplot(2, 1, 2)
620 ax1.tick_params(axis='both', labelsize='x-large')
621 ax1.yaxis.set_major_locator(MaxNLocator(integer=True))
622 ax1.xaxis.set_major_locator(MaxNLocator(integer=True))
623 im1 = ax1.imshow(1e6*b[:bRange, :bRange].transpose(), origin='lower')
624 cb1 = plt.colorbar(im1)
625 cb1.ax.tick_params(labelsize='x-large')
626 ax1.set_title(r'$b \times 10^6$', fontsize='x-large')
627 ax1.xaxis.set_ticks_position('bottom')
628 plt.tight_layout()
629 pdfPages.savefig(fig)
631 return
633 @staticmethod
634 def ab_vs_dist(aDict, bDict, pdfPages, bRange=4):
635 """Fig. 13 of Astier+19.
637 Values of a and b arrays fits, averaged over amplifiers, as a function of distance.
639 Parameters
640 ----------
641 aDict : `dict`, [`numpy.array`]
642 Dictionary keyed by amp names containing the fitted 'a' coefficients from the model
643 in Eq. 20 of Astier+19 (if `ptcFitType` is `FULLCOVARIANCE`).
645 bDict : `dict`, [`numpy.array`]
646 Dictionary keyed by amp names containing the fitted 'b' coefficients from the model
647 in Eq. 20 of Astier+19 (if `ptcFitType` is `FULLCOVARIANCE`).
649 pdfPages: `matplotlib.backends.backend_pdf.PdfPages`
650 PDF file where the plots will be saved.
652 bRange : `int`
653 Maximum lag for b arrays.
654 """
655 assert (len(aDict) == len(bDict))
656 a = []
657 for amp in aDict:
658 if np.isnan(aDict[amp]).all():
659 continue
660 a.append(aDict[amp])
661 a = np.array(a)
662 y = a.mean(axis=0)
663 sy = a.std(axis=0)/np.sqrt(len(aDict))
664 i, j = np.indices(y.shape)
665 upper = (i >= j).ravel()
666 r = np.sqrt(i**2 + j**2).ravel()
667 y = y.ravel()
668 sy = sy.ravel()
669 fig = plt.figure(figsize=(6, 9))
670 ax = fig.add_subplot(211)
671 ax.set_xlim([0.5, r.max()+1])
672 ax.errorbar(r[upper], y[upper], yerr=sy[upper], marker='o', linestyle='none', color='b',
673 label='$i>=j$')
674 ax.errorbar(r[~upper], y[~upper], yerr=sy[~upper], marker='o', linestyle='none', color='r',
675 label='$i<j$')
676 ax.legend(loc='upper center', fontsize='x-large')
677 ax.set_xlabel(r'$\sqrt{i^2+j^2}$', fontsize='x-large')
678 ax.set_ylabel(r'$a_{ij}$', fontsize='x-large')
679 ax.set_yscale('log')
680 ax.tick_params(axis='both', labelsize='x-large')
682 #
683 axb = fig.add_subplot(212)
684 b = []
685 for amp in bDict:
686 if np.isnan(bDict[amp]).all():
687 continue
688 b.append(bDict[amp])
689 b = np.array(b)
690 yb = b.mean(axis=0)
691 syb = b.std(axis=0)/np.sqrt(len(bDict))
692 ib, jb = np.indices(yb.shape)
693 upper = (ib > jb).ravel()
694 rb = np.sqrt(i**2 + j**2).ravel()
695 yb = yb.ravel()
696 syb = syb.ravel()
697 xmin = -0.2
698 xmax = bRange
699 axb.set_xlim([xmin, xmax+0.2])
700 cutu = (r > xmin) & (r < xmax) & (upper)
701 cutl = (r > xmin) & (r < xmax) & (~upper)
702 axb.errorbar(rb[cutu], yb[cutu], yerr=syb[cutu], marker='o', linestyle='none', color='b',
703 label='$i>=j$')
704 axb.errorbar(rb[cutl], yb[cutl], yerr=syb[cutl], marker='o', linestyle='none', color='r',
705 label='$i<j$')
706 plt.legend(loc='upper center', fontsize='x-large')
707 axb.set_xlabel(r'$\sqrt{i^2+j^2}$', fontsize='x-large')
708 axb.set_ylabel(r'$b_{ij}$', fontsize='x-large')
709 axb.ticklabel_format(style='sci', axis='y', scilimits=(0, 0))
710 axb.tick_params(axis='both', labelsize='x-large')
711 plt.tight_layout()
712 pdfPages.savefig(fig)
714 return
716 @staticmethod
717 def plotAcoeffsSum(aDict, bDict, pdfPages):
718 """Fig. 14. of Astier+19
720 Cumulative sum of a_ij as a function of maximum separation. This plot displays the average over
721 channels.
723 Parameters
724 ----------
725 aDict : `dict`, [`numpy.array`]
726 Dictionary keyed by amp names containing the fitted 'a' coefficients from the model
727 in Eq. 20 of Astier+19 (if `ptcFitType` is `FULLCOVARIANCE`).
729 bDict : `dict`, [`numpy.array`]
730 Dictionary keyed by amp names containing the fitted 'b' coefficients from the model
731 in Eq. 20 of Astier+19 (if `ptcFitType` is `FULLCOVARIANCE`).
733 pdfPages: `matplotlib.backends.backend_pdf.PdfPages`
734 PDF file where the plots will be saved.
735 """
736 assert (len(aDict) == len(bDict))
737 a, b = [], []
738 for amp in aDict:
739 if np.isnan(aDict[amp]).all() or np.isnan(bDict[amp]).all():
740 continue
741 a.append(aDict[amp])
742 b.append(bDict[amp])
743 a = np.array(a).mean(axis=0)
744 b = np.array(b).mean(axis=0)
745 fig = plt.figure(figsize=(7, 6))
746 w = 4*np.ones_like(a)
747 w[0, 1:] = 2
748 w[1:, 0] = 2
749 w[0, 0] = 1
750 wa = w*a
751 indices = range(1, a.shape[0]+1)
752 sums = [wa[0:n, 0:n].sum() for n in indices]
753 ax = plt.subplot(111)
754 ax.plot(indices, sums/sums[0], 'o', color='b')
755 ax.set_yscale('log')
756 ax.set_xlim(indices[0]-0.5, indices[-1]+0.5)
757 ax.set_ylim(None, 1.2)
758 ax.set_ylabel(r'$[\sum_{|i|<n\ &\ |j|<n} a_{ij}] / |a_{00}|$', fontsize='x-large')
759 ax.set_xlabel('n', fontsize='x-large')
760 ax.tick_params(axis='both', labelsize='x-large')
761 plt.tight_layout()
762 pdfPages.savefig(fig)
764 return
766 @staticmethod
767 def plotRelativeBiasACoeffs(aDict, aDictNoB, fullCovsModel, fullCovsModelNoB, signalElectrons,
768 gainDict, pdfPages, maxr=None):
769 """Fig. 15 in Astier+19.
771 Illustrates systematic bias from estimating 'a'
772 coefficients from the slope of correlations as opposed to the
773 full model in Astier+19.
775 Parameters
776 ----------
777 aDict: `dict`
778 Dictionary of 'a' matrices (Eq. 20, Astier+19), with amp names as keys.
780 aDictNoB: `dict`
781 Dictionary of 'a' matrices ('b'= 0 in Eq. 20, Astier+19), with amp names as keys.
783 fullCovsModel : `dict`, [`str`, `list`]
784 Dictionary keyed by amp names containing covariances model per mean flux.
786 fullCovsModelNoB : `dict`, [`str`, `list`]
787 Dictionary keyed by amp names containing covariances model (with 'b'=0 in Eq. 20 of
788 Astier+19) per mean flux.
790 signalElectrons : `float`
791 Signal at which to evaluate the a_ij coefficients.
793 pdfPages: `matplotlib.backends.backend_pdf.PdfPages`
794 PDF file where the plots will be saved.
796 gainDict : `dict`, [`str`, `float`]
797 Dicgionary keyed by amp names with the gains in e-/ADU.
799 maxr : `int`, optional
800 Maximum lag.
801 """
803 fig = plt.figure(figsize=(7, 11))
804 title = [f"'a' relative bias at {signalElectrons} e", "'a' relative bias (b=0)"]
805 data = [(aDict, fullCovsModel), (aDictNoB, fullCovsModelNoB)]
807 for k, pair in enumerate(data):
808 diffs = []
809 amean = []
810 for amp in pair[0]:
811 covModel = pair[1][amp]
812 if np.isnan(covModel).all():
813 continue
814 aOld = computeApproximateAcoeffs(covModel, signalElectrons, gainDict[amp])
815 a = pair[0][amp]
816 amean.append(a)
817 diffs.append((aOld-a))
818 amean = np.array(amean).mean(axis=0)
819 diff = np.array(diffs).mean(axis=0)
820 diff = diff/amean
821 diff = diff[:]
822 # The difference should be close to zero
823 diff[0, 0] = 0
824 if maxr is None:
825 maxr = diff.shape[0]
826 diff = diff[:maxr, :maxr]
827 ax0 = fig.add_subplot(2, 1, k+1)
828 im0 = ax0.imshow(diff.transpose(), origin='lower')
829 ax0.yaxis.set_major_locator(MaxNLocator(integer=True))
830 ax0.xaxis.set_major_locator(MaxNLocator(integer=True))
831 ax0.tick_params(axis='both', labelsize='x-large')
832 plt.colorbar(im0)
833 ax0.set_title(title[k])
835 plt.tight_layout()
836 pdfPages.savefig(fig)
838 return
840 def _plotStandardPtc(self, dataset, ptcFitType, pdfPages):
841 """Plot PTC, var/signal vs signal, linearity, and linearity residual per amplifier.
843 Parameters
844 ----------
845 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset`
846 The dataset containing the means, variances, exposure times, and mask.
848 ptcFitType : `str`
849 Type of the model fit to the PTC. Options: 'FULLCOVARIANCE', EXPAPPROXIMATION, or 'POLYNOMIAL'.
851 pdfPages: `matplotlib.backends.backend_pdf.PdfPages`
852 PDF file where the plots will be saved.
853 """
855 if ptcFitType == 'EXPAPPROXIMATION':
856 ptcFunc = funcAstier
857 stringTitle = (r"Var = $\frac{1}{2g^2a_{00}}(\exp (2a_{00} \mu g) - 1) + \frac{n_{00}}{g^2}$ ")
858 elif ptcFitType == 'POLYNOMIAL':
859 ptcFunc = funcPolynomial
860 for key in dataset.ptcFitPars:
861 deg = len(dataset.ptcFitPars[key]) - 1
862 break
863 stringTitle = r"Polynomial (degree: %g)" % (deg)
864 else:
865 raise RuntimeError(f"The input dataset had an invalid dataset.ptcFitType: {ptcFitType}. \n" +
866 "Options: 'FULLCOVARIANCE', EXPAPPROXIMATION, or 'POLYNOMIAL'.")
868 legendFontSize = 6.5
869 labelFontSize = 8
870 titleFontSize = 9
871 supTitleFontSize = 18
872 markerSize = 25
874 # General determination of the size of the plot grid
875 nAmps = len(dataset.ampNames)
876 if nAmps == 2:
877 nRows, nCols = 2, 1
878 nRows = np.sqrt(nAmps)
879 mantissa, _ = np.modf(nRows)
880 if mantissa > 0:
881 nRows = int(nRows) + 1
882 nCols = nRows
883 else:
884 nRows = int(nRows)
885 nCols = nRows
887 f, ax = plt.subplots(nrows=nRows, ncols=nCols, sharex='col', sharey='row', figsize=(13, 10))
888 f2, ax2 = plt.subplots(nrows=nRows, ncols=nCols, sharex='col', sharey='row', figsize=(13, 10))
889 f3, ax3 = plt.subplots(nrows=nRows, ncols=nCols, sharex='col', sharey='row', figsize=(13, 10))
891 for i, (amp, a, a2, a3) in enumerate(zip(dataset.ampNames, ax.flatten(), ax2.flatten(),
892 ax3.flatten())):
893 meanVecOriginal = np.array(dataset.rawMeans[amp])
894 varVecOriginal = np.array(dataset.rawVars[amp])
895 mask = np.array(dataset.expIdMask[amp])
896 if np.isnan(mask[0]): # All NaNs the whole amp is bad
897 a.set_title(f"{amp} (BAD)", fontsize=titleFontSize)
898 a2.set_title(f"{amp} (BAD)", fontsize=titleFontSize)
899 a3.set_title(f"{amp} (BAD)", fontsize=titleFontSize)
900 continue
901 else:
902 mask = mask.astype(bool)
903 meanVecFinal = meanVecOriginal[mask]
904 varVecFinal = varVecOriginal[mask]
905 meanVecOutliers = meanVecOriginal[np.invert(mask)]
906 varVecOutliers = varVecOriginal[np.invert(mask)]
907 pars, parsErr = np.array(dataset.ptcFitPars[amp]), np.array(dataset.ptcFitParsError[amp])
908 ptcRedChi2 = np.array(dataset.ptcFitChiSq[amp])
909 if ptcFitType == 'EXPAPPROXIMATION':
910 if len(meanVecFinal):
911 ptcA00, ptcA00error = pars[0], parsErr[0]
912 ptcGain, ptcGainError = pars[1], parsErr[1]
913 ptcNoise = np.sqrt((pars[2])) # pars[2] is in (e-)^2
914 ptcNoiseAdu = ptcNoise*(1./ptcGain)
915 ptcNoiseError = 0.5*(parsErr[2]/np.fabs(pars[2]))*np.sqrt(np.fabs(pars[2]))
916 stringLegend = (f"a00: {ptcA00:.2e}+/-{ptcA00error:.2e} 1/e"
917 f"\nGain: {ptcGain:.4}+/-{ptcGainError:.2e} e/ADU"
918 f"\nNoise: {ptcNoise:.4}+/-{ptcNoiseError:.2e} e\n"
919 r"$\chi^2_{\rm{red}}$: " + f"{ptcRedChi2:.4}"
920 f"\nLast in fit: {meanVecFinal[-1]:.7} ADU ")
922 if ptcFitType == 'POLYNOMIAL':
923 if len(meanVecFinal):
924 ptcGain, ptcGainError = 1./pars[1], np.fabs(1./pars[1])*(parsErr[1]/pars[1])
925 ptcNoiseAdu = np.sqrt((pars[0])) # pars[0] is in DU^2
926 ptcNoise = ptcNoiseAdu*ptcGain
927 ptcNoiseError = (0.5*(parsErr[0]/np.fabs(pars[0]))*(np.sqrt(np.fabs(pars[0]))))*ptcGain
928 stringLegend = (f"Gain: {ptcGain:.4}+/-{ptcGainError:.2e} e/ADU\n"
929 f"Noise: {ptcNoise:.4}+/-{ptcNoiseError:.2e} e\n"
930 r"$\chi^2_{\rm{red}}$: " + f"{ptcRedChi2:.4}"
931 f"\nLast in fit: {meanVecFinal[-1]:.7} ADU ")
933 a.set_xlabel(r'Mean signal ($\mu$, ADU)', fontsize=labelFontSize)
934 a.set_ylabel(r'Variance (ADU$^2$)', fontsize=labelFontSize)
935 a.tick_params(labelsize=11)
936 a.set_xscale('linear', fontsize=labelFontSize)
937 a.set_yscale('linear', fontsize=labelFontSize)
939 a2.set_xlabel(r'Mean Signal ($\mu$, ADU)', fontsize=labelFontSize)
940 a2.set_ylabel(r'Variance (ADU$^2$)', fontsize=labelFontSize)
941 a2.tick_params(labelsize=11)
942 a2.set_xscale('log')
943 a2.set_yscale('log')
945 a3.set_xlabel(r'Mean signal ($\mu$, ADU)', fontsize=labelFontSize)
946 a3.set_ylabel(r'Variance/$\mu$ (ADU)', fontsize=labelFontSize)
947 a3.tick_params(labelsize=11)
948 a3.set_xscale('log')
949 a3.set_yscale('linear', fontsize=labelFontSize)
951 minMeanVecFinal = np.nanmin(meanVecFinal)
952 maxMeanVecFinal = np.nanmax(meanVecFinal)
953 meanVecFit = np.linspace(minMeanVecFinal, maxMeanVecFinal, 100*len(meanVecFinal))
954 minMeanVecOriginal = np.nanmin(meanVecOriginal)
955 maxMeanVecOriginal = np.nanmax(meanVecOriginal)
956 deltaXlim = maxMeanVecOriginal - minMeanVecOriginal
957 a.plot(meanVecFit, ptcFunc(pars, meanVecFit), color='red')
958 a.plot(meanVecFinal, ptcNoiseAdu**2 + (1./ptcGain)*meanVecFinal, color='green',
959 linestyle='--')
960 a.scatter(meanVecFinal, varVecFinal, c='blue', marker='o', s=markerSize)
961 a.scatter(meanVecOutliers, varVecOutliers, c='magenta', marker='s', s=markerSize)
962 a.text(0.03, 0.66, stringLegend, transform=a.transAxes, fontsize=legendFontSize)
963 a.set_title(amp, fontsize=titleFontSize)
964 a.set_xlim([minMeanVecOriginal - 0.2*deltaXlim, maxMeanVecOriginal + 0.2*deltaXlim])
966 # Same, but in log-scale
967 a2.plot(meanVecFit, ptcFunc(pars, meanVecFit), color='red')
968 a2.scatter(meanVecFinal, varVecFinal, c='blue', marker='o', s=markerSize)
969 a2.scatter(meanVecOutliers, varVecOutliers, c='magenta', marker='s', s=markerSize)
970 a2.text(0.03, 0.66, stringLegend, transform=a2.transAxes, fontsize=legendFontSize)
971 a2.set_title(amp, fontsize=titleFontSize)
972 a2.set_xlim([minMeanVecOriginal, maxMeanVecOriginal])
974 # Var/mu vs mu
975 a3.plot(meanVecFit, ptcFunc(pars, meanVecFit)/meanVecFit, color='red')
976 a3.scatter(meanVecFinal, varVecFinal/meanVecFinal, c='blue', marker='o', s=markerSize)
977 a3.scatter(meanVecOutliers, varVecOutliers/meanVecOutliers, c='magenta', marker='s',
978 s=markerSize)
979 a3.text(0.05, 0.1, stringLegend, transform=a3.transAxes, fontsize=legendFontSize)
980 a3.set_title(amp, fontsize=titleFontSize)
981 a3.set_xlim([minMeanVecOriginal - 0.2*deltaXlim, maxMeanVecOriginal + 0.2*deltaXlim])
983 f.suptitle("PTC \n Fit: " + stringTitle, fontsize=supTitleFontSize)
984 pdfPages.savefig(f)
985 f2.suptitle("PTC (log-log)", fontsize=supTitleFontSize)
986 pdfPages.savefig(f2)
987 f3.suptitle(r"Var/$\mu$", fontsize=supTitleFontSize)
988 pdfPages.savefig(f3)
990 return
992 def _plotLinearizer(self, dataset, linearizer, pdfPages):
993 """Plot linearity and linearity residual per amplifier
995 Parameters
996 ----------
997 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset`
998 The dataset containing the means, variances, exposure times, and mask.
1000 linearizer : `lsst.ip.isr.Linearizer`
1001 Linearizer object
1002 """
1003 legendFontSize = 7
1004 labelFontSize = 7
1005 titleFontSize = 9
1006 supTitleFontSize = 18
1008 # General determination of the size of the plot grid
1009 nAmps = len(dataset.ampNames)
1010 if nAmps == 2:
1011 nRows, nCols = 2, 1
1012 nRows = np.sqrt(nAmps)
1013 mantissa, _ = np.modf(nRows)
1014 if mantissa > 0:
1015 nRows = int(nRows) + 1
1016 nCols = nRows
1017 else:
1018 nRows = int(nRows)
1019 nCols = nRows
1021 # Plot mean vs time (f1), and fractional residuals (f2)
1022 f, ax = plt.subplots(nrows=nRows, ncols=nCols, sharex='col', sharey='row', figsize=(13, 10))
1023 f2, ax2 = plt.subplots(nrows=nRows, ncols=nCols, sharex='col', sharey='row', figsize=(13, 10))
1024 for i, (amp, a, a2) in enumerate(zip(dataset.ampNames, ax.flatten(), ax2.flatten())):
1025 mask = dataset.expIdMask[amp]
1026 if np.isnan(mask[0]):
1027 a.set_title(f"{amp} (BAD)", fontsize=titleFontSize)
1028 a2.set_title(f"{amp} (BAD)", fontsize=titleFontSize)
1029 continue
1030 else:
1031 mask = mask.astype(bool)
1032 meanVecFinal = np.array(dataset.rawMeans[amp])[mask]
1033 timeVecFinal = np.array(dataset.rawExpTimes[amp])[mask]
1035 a.set_xlabel('Time (sec)', fontsize=labelFontSize)
1036 a.set_ylabel(r'Mean signal ($\mu$, ADU)', fontsize=labelFontSize)
1037 a.tick_params(labelsize=labelFontSize)
1038 a.set_xscale('linear', fontsize=labelFontSize)
1039 a.set_yscale('linear', fontsize=labelFontSize)
1041 a2.axhline(y=0, color='k')
1042 a2.axvline(x=0, color='k', linestyle='-')
1043 a2.set_xlabel(r'Mean signal ($\mu$, ADU)', fontsize=labelFontSize)
1044 a2.set_ylabel('Fractional nonlinearity (%)', fontsize=labelFontSize)
1045 a2.tick_params(labelsize=labelFontSize)
1046 a2.set_xscale('linear', fontsize=labelFontSize)
1047 a2.set_yscale('linear', fontsize=labelFontSize)
1049 pars, parsErr = linearizer.fitParams[amp], linearizer.fitParamsErr[amp]
1050 k0, k0Error = pars[0], parsErr[0]
1051 k1, k1Error = pars[1], parsErr[1]
1052 k2, k2Error = pars[2], parsErr[2]
1053 linRedChi2 = linearizer.fitChiSq[amp]
1054 stringLegend = (f"k0: {k0:.4}+/-{k0Error:.2e} ADU\nk1: {k1:.4}+/-{k1Error:.2e} ADU/t"
1055 f"\nk2: {k2:.2e}+/-{k2Error:.2e} ADU/t^2\n"
1056 r"$\chi^2_{\rm{red}}$: " + f"{linRedChi2:.4}")
1057 a.scatter(timeVecFinal, meanVecFinal)
1058 a.plot(timeVecFinal, funcPolynomial(pars, timeVecFinal), color='red')
1059 a.text(0.03, 0.75, stringLegend, transform=a.transAxes, fontsize=legendFontSize)
1060 a.set_title(f"{amp}", fontsize=titleFontSize)
1062 linearPart = k0 + k1*timeVecFinal
1063 fracLinRes = 100*(linearPart - meanVecFinal)/linearPart
1064 a2.plot(meanVecFinal, fracLinRes, c='g')
1065 a2.set_title(f"{amp}", fontsize=titleFontSize)
1067 f.suptitle("Linearity \n Fit: Polynomial (degree: %g)"
1068 % (len(pars)-1),
1069 fontsize=supTitleFontSize)
1070 f2.suptitle(r"Fractional NL residual" + "\n" +
1071 r"$100\times \frac{(k_0 + k_1*Time-\mu)}{k_0+k_1*Time}$",
1072 fontsize=supTitleFontSize)
1073 pdfPages.savefig(f)
1074 pdfPages.savefig(f2)
1076 @staticmethod
1077 def findGroups(x, maxDiff):
1078 """Group data into bins, with at most maxDiff distance between bins.
1080 Parameters
1081 ----------
1082 x: `list`
1083 Data to bin.
1085 maxDiff: `int`
1086 Maximum distance between bins.
1088 Returns
1089 -------
1090 index: `list`
1091 Bin indices.
1092 """
1093 ix = np.argsort(x)
1094 xsort = np.sort(x)
1095 index = np.zeros_like(x, dtype=np.int32)
1096 xc = xsort[0]
1097 group = 0
1098 ng = 1
1100 for i in range(1, len(ix)):
1101 xval = xsort[i]
1102 if (xval - xc < maxDiff):
1103 xc = (ng*xc + xval)/(ng+1)
1104 ng += 1
1105 index[ix[i]] = group
1106 else:
1107 group += 1
1108 ng = 1
1109 index[ix[i]] = group
1110 xc = xval
1112 return index
1114 @staticmethod
1115 def indexForBins(x, nBins):
1116 """Builds an index with regular binning. The result can be fed into binData.
1118 Parameters
1119 ----------
1120 x: `numpy.array`
1121 Data to bin.
1122 nBins: `int`
1123 Number of bin.
1125 Returns
1126 -------
1127 np.digitize(x, bins): `numpy.array`
1128 Bin indices.
1129 """
1131 bins = np.linspace(x.min(), x.max() + abs(x.max() * 1e-7), nBins + 1)
1132 return np.digitize(x, bins)
1134 @staticmethod
1135 def binData(x, y, binIndex, wy=None):
1136 """Bin data (usually for display purposes).
1138 Patrameters
1139 -----------
1140 x: `numpy.array`
1141 Data to bin.
1143 y: `numpy.array`
1144 Data to bin.
1146 binIdex: `list`
1147 Bin number of each datum.
1149 wy: `numpy.array`
1150 Inverse rms of each datum to use when averaging (the actual weight is wy**2).
1152 Returns:
1153 -------
1155 xbin: `numpy.array`
1156 Binned data in x.
1158 ybin: `numpy.array`
1159 Binned data in y.
1161 wybin: `numpy.array`
1162 Binned weights in y, computed from wy's in each bin.
1164 sybin: `numpy.array`
1165 Uncertainty on the bin average, considering actual scatter, and ignoring weights.
1166 """
1168 if wy is None:
1169 wy = np.ones_like(x)
1170 binIndexSet = set(binIndex)
1171 w2 = wy*wy
1172 xw2 = x*(w2)
1173 xbin = np.array([xw2[binIndex == i].sum()/w2[binIndex == i].sum() for i in binIndexSet])
1175 yw2 = y*w2
1176 ybin = np.array([yw2[binIndex == i].sum()/w2[binIndex == i].sum() for i in binIndexSet])
1178 wybin = np.sqrt(np.array([w2[binIndex == i].sum() for i in binIndexSet]))
1179 sybin = np.array([y[binIndex == i].std()/np.sqrt(np.array([binIndex == i]).sum())
1180 for i in binIndexSet])
1182 return xbin, ybin, wybin, sybin