Coverage for python/lsst/cp/pipe/plotPtc.py : 6%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of cp_pipe.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21#
23__all__ = ['PlotPhotonTransferCurveTask']
25import numpy as np
26import matplotlib.pyplot as plt
27import matplotlib as mpl
28from matplotlib import gridspec
29import os
30from matplotlib.backends.backend_pdf import PdfPages
32import lsst.ip.isr as isr
33import lsst.pex.config as pexConfig
34import lsst.pipe.base as pipeBase
35import pickle
37from .utils import (funcAstier, funcPolynomial, NonexistentDatasetTaskDataIdContainer)
38from matplotlib.ticker import MaxNLocator
40from .astierCovPtcFit import computeApproximateAcoeffs
43class PlotPhotonTransferCurveTaskConfig(pexConfig.Config):
44 """Config class for photon transfer curve measurement task"""
45 datasetFileName = pexConfig.Field(
46 dtype=str,
47 doc="datasetPtc file name (pkl)",
48 default="",
49 )
50 linearizerFileName = pexConfig.Field(
51 dtype=str,
52 doc="linearizer file name (fits)",
53 default="",
54 )
55 ccdKey = pexConfig.Field(
56 dtype=str,
57 doc="The key by which to pull a detector from a dataId, e.g. 'ccd' or 'detector'.",
58 default='detector',
59 )
62class PlotPhotonTransferCurveTask(pipeBase.CmdLineTask):
63 """A class to plot the dataset from MeasurePhotonTransferCurveTask.
65 Parameters
66 ----------
68 *args: `list`
69 Positional arguments passed to the Task constructor. None used at this
70 time.
71 **kwargs: `dict`
72 Keyword arguments passed on to the Task constructor. None used at this
73 time.
75 """
77 ConfigClass = PlotPhotonTransferCurveTaskConfig
78 _DefaultName = "plotPhotonTransferCurve"
80 def __init__(self, *args, **kwargs):
81 pipeBase.CmdLineTask.__init__(self, *args, **kwargs)
82 plt.interactive(False) # stop windows popping up when plotting. When headless, use 'agg' backend too
83 self.config.validate()
84 self.config.freeze()
86 @classmethod
87 def _makeArgumentParser(cls):
88 """Augment argument parser for the MeasurePhotonTransferCurveTask."""
89 parser = pipeBase.ArgumentParser(name=cls._DefaultName)
90 parser.add_id_argument("--id", datasetType="photonTransferCurveDataset",
91 ContainerClass=NonexistentDatasetTaskDataIdContainer,
92 help="The ccds to use, e.g. --id ccd=0..100")
93 return parser
95 @pipeBase.timeMethod
96 def runDataRef(self, dataRef):
97 """Run the Photon Transfer Curve (PTC) plotting measurement task.
99 Parameters
100 ----------
101 dataRef : list of lsst.daf.persistence.ButlerDataRef
102 dataRef for the detector for the visits to be fit.
103 """
105 datasetFile = self.config.datasetFileName
107 with open(datasetFile, "rb") as f:
108 datasetPtc = pickle.load(f)
110 dirname = dataRef.getUri(datasetType='cpPipePlotRoot', write=True)
111 if not os.path.exists(dirname):
112 os.makedirs(dirname)
114 detNum = dataRef.dataId[self.config.ccdKey]
115 filename = f"PTC_det{detNum}.pdf"
116 filenameFull = os.path.join(dirname, filename)
118 if self.config.linearizerFileName:
119 linearizer = isr.linearize.Linearizer.readFits(self.config.linearizerFileName)
120 else:
121 linearizer = None
122 self.run(filenameFull, datasetPtc, linearizer=linearizer, log=self.log)
124 return pipeBase.Struct(exitStatus=0)
126 def run(self, filenameFull, datasetPtc, linearizer=None, log=None):
127 """Make the plots for the PTC task"""
128 ptcFitType = datasetPtc.ptcFitType
129 with PdfPages(filenameFull) as pdfPages:
130 if ptcFitType in ["FULLCOVARIANCE", ]:
131 self.covAstierMakeAllPlots(datasetPtc.covariancesFits, datasetPtc.covariancesFitsWithNoB,
132 pdfPages, log=log)
133 elif ptcFitType in ["EXPAPPROXIMATION", "POLYNOMIAL"]:
134 self._plotStandardPtc(datasetPtc, ptcFitType, pdfPages)
135 else:
136 raise RuntimeError(f"The input dataset had an invalid dataset.ptcFitType: {ptcFitType}. \n" +
137 "Options: 'FULLCOVARIANCE', EXPAPPROXIMATION, or 'POLYNOMIAL'.")
138 if linearizer:
139 self._plotLinearizer(datasetPtc, linearizer, pdfPages)
141 return
143 def covAstierMakeAllPlots(self, covFits, covFitsNoB, pdfPages,
144 log=None, maxMu=1e9):
145 """Make plots for MeasurePhotonTransferCurve task when doCovariancesAstier=True.
147 This function call other functions that mostly reproduce the plots in Astier+19.
148 Most of the code is ported from Pierre Astier's repository https://github.com/PierreAstier/bfptc
150 Parameters
151 ----------
152 covFits: `dict`
153 Dictionary of CovFit objects, with amp names as keys.
155 covFitsNoB: `dict`
156 Dictionary of CovFit objects, with amp names as keys (b=0 in Eq. 20 of Astier+19).
158 pdfPages: `matplotlib.backends.backend_pdf.PdfPages`
159 PDF file where the plots will be saved.
161 log : `lsst.log.Log`, optional
162 Logger to handle messages
164 maxMu: `float`, optional
165 Maximum signal, in ADU.
166 """
167 self.plotCovariances(covFits, pdfPages)
168 self.plotNormalizedCovariances(covFits, covFitsNoB, 0, 0, pdfPages, offset=0.01, topPlot=True,
169 log=log)
170 self.plotNormalizedCovariances(covFits, covFitsNoB, 0, 1, pdfPages, log=log)
171 self.plotNormalizedCovariances(covFits, covFitsNoB, 1, 0, pdfPages, log=log)
172 self.plot_a_b(covFits, pdfPages)
173 self.ab_vs_dist(covFits, pdfPages, bRange=4)
174 self.plotAcoeffsSum(covFits, pdfPages)
175 self.plotRelativeBiasACoeffs(covFits, covFitsNoB, maxMu, pdfPages)
177 return
179 @staticmethod
180 def plotCovariances(covFits, pdfPages):
181 """Plot covariances and models: Cov00, Cov10, Cov01.
183 Figs. 6 and 7 of Astier+19
185 Parameters
186 ----------
187 covFits: `dict`
188 Dictionary of CovFit objects, with amp names as keys.
190 pdfPages: `matplotlib.backends.backend_pdf.PdfPages`
191 PDF file where the plots will be saved.
192 """
194 legendFontSize = 7
195 labelFontSize = 7
196 titleFontSize = 9
197 supTitleFontSize = 18
198 markerSize = 25
200 nAmps = len(covFits)
201 if nAmps == 2:
202 nRows, nCols = 2, 1
203 nRows = np.sqrt(nAmps)
204 mantissa, _ = np.modf(nRows)
205 if mantissa > 0:
206 nRows = int(nRows) + 1
207 nCols = nRows
208 else:
209 nRows = int(nRows)
210 nCols = nRows
212 f, ax = plt.subplots(nrows=nRows, ncols=nCols, sharex='col', sharey='row', figsize=(13, 10))
213 f2, ax2 = plt.subplots(nrows=nRows, ncols=nCols, sharex='col', sharey='row', figsize=(13, 10))
214 fResCov00, axResCov00 = plt.subplots(nrows=nRows, ncols=nCols, sharex='col', sharey='row',
215 figsize=(13, 10))
216 fCov01, axCov01 = plt.subplots(nrows=nRows, ncols=nCols, sharex='col', sharey='row', figsize=(13, 10))
217 fCov10, axCov10 = plt.subplots(nrows=nRows, ncols=nCols, sharex='col', sharey='row', figsize=(13, 10))
219 for i, (fitPair, a, a2, aResVar, a3, a4) in enumerate(zip(covFits.items(), ax.flatten(),
220 ax2.flatten(), axResCov00.flatten(),
221 axCov01.flatten(), axCov10.flatten())):
223 amp = fitPair[0]
224 fit = fitPair[1]
226 meanVecFinal, varVecFinal, varVecModel, wc = fit.getNormalizedFitData(0, 0)
227 meanVecFinalCov01, varVecFinalCov01, varVecModelCov01, wcCov01 = fit.getNormalizedFitData(0, 1)
228 meanVecFinalCov10, varVecFinalCov10, varVecModelCov10, wcCov10 = fit.getNormalizedFitData(1, 0)
230 # cuadratic fit for residuals below
231 par2 = np.polyfit(meanVecFinal, varVecFinal, 2, w=wc)
232 varModelQuadratic = np.polyval(par2, meanVecFinal)
234 # fit with no 'b' coefficient (c = a*b in Eq. 20 of Astier+19)
235 fitNoB = fit.copy()
236 fitNoB.params['c'].fix(val=0)
237 fitNoB.fitFullModel()
238 meanVecFinalNoB, varVecFinalNoB, varVecModelNoB, wcNoB = fitNoB.getNormalizedFitData(0, 0)
240 if len(meanVecFinal): # Empty if the whole amp is bad, for example.
241 stringLegend = (f"Gain: {fit.getGain():.4} e/DN \n Noise: {np.sqrt(fit.getRon()):.4} e \n" +
242 r"$a_{00}$: %.3e 1/e"%fit.getA()[0, 0] +
243 "\n" + r"$b_{00}$: %.3e 1/e"%fit.getB()[0, 0])
244 minMeanVecFinal = np.min(meanVecFinal)
245 maxMeanVecFinal = np.max(meanVecFinal)
246 deltaXlim = maxMeanVecFinal - minMeanVecFinal
248 a.set_xlabel(r'Mean signal ($\mu$, DN)', fontsize=labelFontSize)
249 a.set_ylabel(r'Variance (DN$^2$)', fontsize=labelFontSize)
250 a.tick_params(labelsize=11)
251 a.set_xscale('linear', fontsize=labelFontSize)
252 a.set_yscale('linear', fontsize=labelFontSize)
253 a.scatter(meanVecFinal, varVecFinal, c='blue', marker='o', s=markerSize)
254 a.plot(meanVecFinal, varVecModel, color='red', lineStyle='-')
255 a.text(0.03, 0.7, stringLegend, transform=a.transAxes, fontsize=legendFontSize)
256 a.set_title(amp, fontsize=titleFontSize)
257 a.set_xlim([minMeanVecFinal - 0.2*deltaXlim, maxMeanVecFinal + 0.2*deltaXlim])
259 # Same as above, but in log-scale
260 a2.set_xlabel(r'Mean Signal ($\mu$, DN)', fontsize=labelFontSize)
261 a2.set_ylabel(r'Variance (DN$^2$)', fontsize=labelFontSize)
262 a2.tick_params(labelsize=11)
263 a2.set_xscale('log')
264 a2.set_yscale('log')
265 a2.plot(meanVecFinal, varVecModel, color='red', lineStyle='-')
266 a2.scatter(meanVecFinal, varVecFinal, c='blue', marker='o', s=markerSize)
267 a2.text(0.03, 0.7, stringLegend, transform=a2.transAxes, fontsize=legendFontSize)
268 a2.set_title(amp, fontsize=titleFontSize)
269 a2.set_xlim([minMeanVecFinal, maxMeanVecFinal])
271 # Residuals var - model
272 aResVar.set_xlabel(r'Mean signal ($\mu$, DN)', fontsize=labelFontSize)
273 aResVar.set_ylabel(r'Residuals (DN$^2$)', fontsize=labelFontSize)
274 aResVar.tick_params(labelsize=11)
275 aResVar.set_xscale('linear', fontsize=labelFontSize)
276 aResVar.set_yscale('linear', fontsize=labelFontSize)
277 aResVar.plot(meanVecFinal, varVecFinal - varVecModel, color='blue', lineStyle='-',
278 label='Full fit')
279 aResVar.plot(meanVecFinal, varVecFinal - varModelQuadratic, color='red', lineStyle='-',
280 label='Quadratic fit')
281 aResVar.plot(meanVecFinalNoB, varVecFinalNoB - varVecModelNoB, color='green', lineStyle='-',
282 label='Full fit with b=0')
283 aResVar.axhline(color='black')
284 aResVar.set_title(amp, fontsize=titleFontSize)
285 aResVar.set_xlim([minMeanVecFinal - 0.2*deltaXlim, maxMeanVecFinal + 0.2*deltaXlim])
286 aResVar.legend(fontsize=7)
288 a3.set_xlabel(r'Mean signal ($\mu$, DN)', fontsize=labelFontSize)
289 a3.set_ylabel(r'Cov01 (DN$^2$)', fontsize=labelFontSize)
290 a3.tick_params(labelsize=11)
291 a3.set_xscale('linear', fontsize=labelFontSize)
292 a3.set_yscale('linear', fontsize=labelFontSize)
293 a3.scatter(meanVecFinalCov01, varVecFinalCov01, c='blue', marker='o', s=markerSize)
294 a3.plot(meanVecFinalCov01, varVecModelCov01, color='red', lineStyle='-')
295 a3.set_title(amp, fontsize=titleFontSize)
296 a3.set_xlim([minMeanVecFinal - 0.2*deltaXlim, maxMeanVecFinal + 0.2*deltaXlim])
298 a4.set_xlabel(r'Mean signal ($\mu$, DN)', fontsize=labelFontSize)
299 a4.set_ylabel(r'Cov10 (DN$^2$)', fontsize=labelFontSize)
300 a4.tick_params(labelsize=11)
301 a4.set_xscale('linear', fontsize=labelFontSize)
302 a4.set_yscale('linear', fontsize=labelFontSize)
303 a4.scatter(meanVecFinalCov10, varVecFinalCov10, c='blue', marker='o', s=markerSize)
304 a4.plot(meanVecFinalCov10, varVecModelCov10, color='red', lineStyle='-')
305 a4.set_title(amp, fontsize=titleFontSize)
306 a4.set_xlim([minMeanVecFinal - 0.2*deltaXlim, maxMeanVecFinal + 0.2*deltaXlim])
308 else:
309 a.set_title(f"{amp} (BAD)", fontsize=titleFontSize)
310 a2.set_title(f"{amp} (BAD)", fontsize=titleFontSize)
311 a3.set_title(f"{amp} (BAD)", fontsize=titleFontSize)
312 a4.set_title(f"{amp} (BAD)", fontsize=titleFontSize)
314 f.suptitle("PTC from covariances as in Astier+19 \n Fit: Eq. 20, Astier+19",
315 fontsize=supTitleFontSize)
316 pdfPages.savefig(f)
317 f2.suptitle("PTC from covariances as in Astier+19 (log-log) \n Fit: Eq. 20, Astier+19",
318 fontsize=supTitleFontSize)
319 pdfPages.savefig(f2)
320 fResCov00.suptitle("Residuals (data- model) for Cov00 (Var)", fontsize=supTitleFontSize)
321 pdfPages.savefig(fResCov00)
322 fCov01.suptitle("Cov01 as in Astier+19 (nearest parallel neighbor covariance) \n" +
323 " Fit: Eq. 20, Astier+19", fontsize=supTitleFontSize)
324 pdfPages.savefig(fCov01)
325 fCov10.suptitle("Cov10 as in Astier+19 (nearest serial neighbor covariance) \n" +
326 "Fit: Eq. 20, Astier+19", fontsize=supTitleFontSize)
327 pdfPages.savefig(fCov10)
329 return
331 def plotNormalizedCovariances(self, covFits, covFitsNoB, i, j, pdfPages, offset=0.004,
332 plotData=True, topPlot=False, log=None):
333 """Plot C_ij/mu vs mu.
335 Figs. 8, 10, and 11 of Astier+19
337 Parameters
338 ----------
339 covFits: `dict`
340 Dictionary of CovFit objects, with amp names as keys.
342 covFitsNoB: `dict`
343 Dictionary of CovFit objects, with amp names as keys (b=0 in Eq. 20 of Astier+19).
345 i : `int`
346 Covariane lag
348 j : `int
349 Covariance lag
351 pdfPages: `matplotlib.backends.backend_pdf.PdfPages`
352 PDF file where the plots will be saved.
354 offset : `float`, optional
355 Constant offset factor to plot covariances in same panel (so they don't overlap).
357 plotData : `bool`, optional
358 Plot the data points?
360 topPlot : `bool`, optional
361 Plot the top plot with the covariances, and the bottom plot with the model residuals?
363 log : `lsst.log.Log`, optional
364 Logger to handle messages.
365 """
367 lchi2, la, lb, lcov = [], [], [], []
369 if (not topPlot):
370 fig = plt.figure(figsize=(8, 10))
371 gs = gridspec.GridSpec(2, 1, height_ratios=[3, 1])
372 gs.update(hspace=0)
373 ax0 = plt.subplot(gs[0])
374 plt.setp(ax0.get_xticklabels(), visible=False)
375 else:
376 fig = plt.figure(figsize=(8, 8))
377 ax0 = plt.subplot(111)
378 ax0.ticklabel_format(style='sci', axis='x', scilimits=(0, 0))
379 ax0.tick_params(axis='both', labelsize='x-large')
380 mue, rese, wce = [], [], []
381 mueNoB, reseNoB, wceNoB = [], [], []
382 for counter, (amp, fit) in enumerate(covFits.items()):
383 mu, c, model, wc = fit.getNormalizedFitData(i, j, divideByMu=True)
384 wres = (c-model)*wc
385 chi2 = ((wres*wres).sum())/(len(mu)-3)
386 chi2bin = 0
387 mue += list(mu)
388 rese += list(c - model)
389 wce += list(wc)
391 fitNoB = covFitsNoB[amp]
392 muNoB, cNoB, modelNoB, wcNoB = fitNoB.getNormalizedFitData(i, j, divideByMu=True)
393 mueNoB += list(muNoB)
394 reseNoB += list(cNoB - modelNoB)
395 wceNoB += list(wcNoB)
397 # the corresponding fit
398 fit_curve, = plt.plot(mu, model + counter*offset, '-', linewidth=4.0)
399 # bin plot. len(mu) = no binning
400 gind = self.indexForBins(mu, len(mu))
402 xb, yb, wyb, sigyb = self.binData(mu, c, gind, wc)
403 chi2bin = (sigyb*wyb).mean() # chi2 of enforcing the same value in each bin
404 plt.errorbar(xb, yb+counter*offset, yerr=sigyb, marker='o', linestyle='none', markersize=6.5,
405 color=fit_curve.get_color(), label=f"{amp}")
406 # plot the data
407 if plotData:
408 points, = plt.plot(mu, c + counter*offset, '.', color=fit_curve.get_color())
409 plt.legend(loc='upper right', fontsize=8)
410 aij = fit.getA()[i, j]
411 bij = fit.getB()[i, j]
412 la.append(aij)
413 lb.append(bij)
414 lcov.append(fit.getACov()[i, j, i, j])
415 lchi2.append(chi2)
416 log.info('%s: slope %g b %g chi2 %f chi2bin %f'%(amp, aij, bij, chi2, chi2bin))
417 # end loop on amps
418 la = np.array(la)
419 lb = np.array(lb)
420 lcov = np.array(lcov)
421 lchi2 = np.array(lchi2)
422 mue = np.array(mue)
423 rese = np.array(rese)
424 wce = np.array(wce)
425 mueNoB = np.array(mueNoB)
426 reseNoB = np.array(reseNoB)
427 wceNoB = np.array(wceNoB)
429 plt.xlabel(r"$\mu (el)$", fontsize='x-large')
430 plt.ylabel(r"$C_{%d%d}/\mu + Cst (el)$"%(i, j), fontsize='x-large')
431 if (not topPlot):
432 gind = self.indexForBins(mue, len(mue))
433 xb, yb, wyb, sigyb = self.binData(mue, rese, gind, wce)
435 ax1 = plt.subplot(gs[1], sharex=ax0)
436 ax1.errorbar(xb, yb, yerr=sigyb, marker='o', linestyle='none', label='Full fit')
437 gindNoB = self.indexForBins(mueNoB, len(mueNoB))
438 xb2, yb2, wyb2, sigyb2 = self.binData(mueNoB, reseNoB, gindNoB, wceNoB)
440 ax1.errorbar(xb2, yb2, yerr=sigyb2, marker='o', linestyle='none', label='b = 0')
441 ax1.tick_params(axis='both', labelsize='x-large')
442 plt.legend(loc='upper left', fontsize='large')
443 # horizontal line at zero
444 plt.plot(xb, [0]*len(xb), '--', color='k')
445 plt.ticklabel_format(style='sci', axis='x', scilimits=(0, 0))
446 plt.ticklabel_format(style='sci', axis='y', scilimits=(0, 0))
447 plt.xlabel(r'$\mu (el)$', fontsize='x-large')
448 plt.ylabel(r'$C_{%d%d}/\mu$ -model (el)'%(i, j), fontsize='x-large')
449 plt.tight_layout()
451 # overlapping y labels:
452 fig.canvas.draw()
453 labels0 = [item.get_text() for item in ax0.get_yticklabels()]
454 labels0[0] = u''
455 ax0.set_yticklabels(labels0)
456 pdfPages.savefig(fig)
458 return
460 @staticmethod
461 def plot_a_b(covFits, pdfPages, bRange=3):
462 """Fig. 12 of Astier+19
464 Color display of a and b arrays fits, averaged over channels.
466 Parameters
467 ----------
468 covFits: `dict`
469 Dictionary of CovFit objects, with amp names as keys.
471 pdfPages: `matplotlib.backends.backend_pdf.PdfPages`
472 PDF file where the plots will be saved.
474 bRange : `int`
475 Maximum lag for b arrays.
476 """
477 a, b = [], []
478 for amp, fit in covFits.items():
479 a.append(fit.getA())
480 b.append(fit.getB())
481 a = np.array(a).mean(axis=0)
482 b = np.array(b).mean(axis=0)
483 fig = plt.figure(figsize=(7, 11))
484 ax0 = fig.add_subplot(2, 1, 1)
485 im0 = ax0.imshow(np.abs(a.transpose()), origin='lower', norm=mpl.colors.LogNorm())
486 ax0.tick_params(axis='both', labelsize='x-large')
487 ax0.set_title(r'$|a|$', fontsize='x-large')
488 ax0.xaxis.set_ticks_position('bottom')
489 cb0 = plt.colorbar(im0)
490 cb0.ax.tick_params(labelsize='x-large')
492 ax1 = fig.add_subplot(2, 1, 2)
493 ax1.tick_params(axis='both', labelsize='x-large')
494 ax1.yaxis.set_major_locator(MaxNLocator(integer=True))
495 ax1.xaxis.set_major_locator(MaxNLocator(integer=True))
496 im1 = ax1.imshow(1e6*b[:bRange, :bRange].transpose(), origin='lower')
497 cb1 = plt.colorbar(im1)
498 cb1.ax.tick_params(labelsize='x-large')
499 ax1.set_title(r'$b \times 10^6$', fontsize='x-large')
500 ax1.xaxis.set_ticks_position('bottom')
501 plt.tight_layout()
502 pdfPages.savefig(fig)
504 return
506 @staticmethod
507 def ab_vs_dist(covFits, pdfPages, bRange=4):
508 """Fig. 13 of Astier+19.
510 Values of a and b arrays fits, averaged over amplifiers, as a function of distance.
512 Parameters
513 ----------
514 covFits: `dict`
515 Dictionary of CovFit objects, with amp names as keys.
517 pdfPages: `matplotlib.backends.backend_pdf.PdfPages`
518 PDF file where the plots will be saved.
520 bRange : `int`
521 Maximum lag for b arrays.
522 """
523 a = np.array([f.getA() for f in covFits.values()])
524 y = a.mean(axis=0)
525 sy = a.std(axis=0)/np.sqrt(len(covFits))
526 i, j = np.indices(y.shape)
527 upper = (i >= j).ravel()
528 r = np.sqrt(i**2 + j**2).ravel()
529 y = y.ravel()
530 sy = sy.ravel()
531 fig = plt.figure(figsize=(6, 9))
532 ax = fig.add_subplot(211)
533 ax.set_xlim([0.5, r.max()+1])
534 ax.errorbar(r[upper], y[upper], yerr=sy[upper], marker='o', linestyle='none', color='b',
535 label='$i>=j$')
536 ax.errorbar(r[~upper], y[~upper], yerr=sy[~upper], marker='o', linestyle='none', color='r',
537 label='$i<j$')
538 ax.legend(loc='upper center', fontsize='x-large')
539 ax.set_xlabel(r'$\sqrt{i^2+j^2}$', fontsize='x-large')
540 ax.set_ylabel(r'$a_{ij}$', fontsize='x-large')
541 ax.set_yscale('log')
542 ax.tick_params(axis='both', labelsize='x-large')
544 #
545 axb = fig.add_subplot(212)
546 b = np.array([f.getB() for f in covFits.values()])
547 yb = b.mean(axis=0)
548 syb = b.std(axis=0)/np.sqrt(len(covFits))
549 ib, jb = np.indices(yb.shape)
550 upper = (ib > jb).ravel()
551 rb = np.sqrt(i**2 + j**2).ravel()
552 yb = yb.ravel()
553 syb = syb.ravel()
554 xmin = -0.2
555 xmax = bRange
556 axb.set_xlim([xmin, xmax+0.2])
557 cutu = (r > xmin) & (r < xmax) & (upper)
558 cutl = (r > xmin) & (r < xmax) & (~upper)
559 axb.errorbar(rb[cutu], yb[cutu], yerr=syb[cutu], marker='o', linestyle='none', color='b',
560 label='$i>=j$')
561 axb.errorbar(rb[cutl], yb[cutl], yerr=syb[cutl], marker='o', linestyle='none', color='r',
562 label='$i<j$')
563 plt.legend(loc='upper center', fontsize='x-large')
564 axb.set_xlabel(r'$\sqrt{i^2+j^2}$', fontsize='x-large')
565 axb.set_ylabel(r'$b_{ij}$', fontsize='x-large')
566 axb.ticklabel_format(style='sci', axis='y', scilimits=(0, 0))
567 axb.tick_params(axis='both', labelsize='x-large')
568 plt.tight_layout()
569 pdfPages.savefig(fig)
571 return
573 @staticmethod
574 def plotAcoeffsSum(covFits, pdfPages):
575 """Fig. 14. of Astier+19
577 Cumulative sum of a_ij as a function of maximum separation. This plot displays the average over
578 channels.
580 Parameters
581 ----------
582 covFits: `dict`
583 Dictionary of CovFit objects, with amp names as keys.
585 pdfPages: `matplotlib.backends.backend_pdf.PdfPages`
586 PDF file where the plots will be saved.
587 """
588 a, b = [], []
589 for amp, fit in covFits.items():
590 a.append(fit.getA())
591 b.append(fit.getB())
592 a = np.array(a).mean(axis=0)
593 b = np.array(b).mean(axis=0)
594 fig = plt.figure(figsize=(7, 6))
595 w = 4*np.ones_like(a)
596 w[0, 1:] = 2
597 w[1:, 0] = 2
598 w[0, 0] = 1
599 wa = w*a
600 indices = range(1, a.shape[0]+1)
601 sums = [wa[0:n, 0:n].sum() for n in indices]
602 ax = plt.subplot(111)
603 ax.plot(indices, sums/sums[0], 'o', color='b')
604 ax.set_yscale('log')
605 ax.set_xlim(indices[0]-0.5, indices[-1]+0.5)
606 ax.set_ylim(None, 1.2)
607 ax.set_ylabel(r'$[\sum_{|i|<n\ &\ |j|<n} a_{ij}] / |a_{00}|$', fontsize='x-large')
608 ax.set_xlabel('n', fontsize='x-large')
609 ax.tick_params(axis='both', labelsize='x-large')
610 plt.tight_layout()
611 pdfPages.savefig(fig)
613 return
615 @staticmethod
616 def plotRelativeBiasACoeffs(covFits, covFitsNoB, maxMu, pdfPages, maxr=None):
617 """Fig. 15 in Astier+19.
619 Illustrates systematic bias from estimating 'a'
620 coefficients from the slope of correlations as opposed to the
621 full model in Astier+19.
623 Parameters
624 ----------
625 covFits: `dict`
626 Dictionary of CovFit objects, with amp names as keys.
628 covFitsNoB: `dict`
629 Dictionary of CovFit objects, with amp names as keys (b=0 in Eq. 20 of Astier+19).
631 maxMu: `float`, optional
632 Maximum signal, in ADU.
634 pdfPages: `matplotlib.backends.backend_pdf.PdfPages`
635 PDF file where the plots will be saved.
637 maxr: `int`, optional
638 Maximum lag.
639 """
641 fig = plt.figure(figsize=(7, 11))
642 title = ["'a' relative bias", "'a' relative bias (b=0)"]
643 data = [covFits, covFitsNoB]
645 for k in range(2):
646 diffs = []
647 amean = []
648 for fit in data[k].values():
649 if fit is None:
650 continue
651 maxMuEl = maxMu*fit.getGain()
652 aOld = computeApproximateAcoeffs(fit, maxMuEl)
653 a = fit.getA()
654 amean.append(a)
655 diffs.append((aOld-a))
656 amean = np.array(amean).mean(axis=0)
657 diff = np.array(diffs).mean(axis=0)
658 diff = diff/amean
659 diff[0, 0] = 0
660 if maxr is None:
661 maxr = diff.shape[0]
662 diff = diff[:maxr, :maxr]
663 ax0 = fig.add_subplot(2, 1, k+1)
664 im0 = ax0.imshow(diff.transpose(), origin='lower')
665 ax0.yaxis.set_major_locator(MaxNLocator(integer=True))
666 ax0.xaxis.set_major_locator(MaxNLocator(integer=True))
667 ax0.tick_params(axis='both', labelsize='x-large')
668 plt.colorbar(im0)
669 ax0.set_title(title[k])
671 plt.tight_layout()
672 pdfPages.savefig(fig)
674 return
676 def _plotStandardPtc(self, dataset, ptcFitType, pdfPages):
677 """Plot PTC, linearity, and linearity residual per amplifier
679 Parameters
680 ----------
681 dataset : `lsst.cp.pipe.ptc.PhotonTransferCurveDataset`
682 The dataset containing the means, variances, exposure times, and mask.
684 ptcFitType : `str`
685 Type of the model fit to the PTC. Options: 'FULLCOVARIANCE', EXPAPPROXIMATION, or 'POLYNOMIAL'.
687 pdfPages: `matplotlib.backends.backend_pdf.PdfPages`
688 PDF file where the plots will be saved.
689 """
691 if ptcFitType == 'EXPAPPROXIMATION':
692 ptcFunc = funcAstier
693 stringTitle = (r"Var = $\frac{1}{2g^2a_{00}}(\exp (2a_{00} \mu g) - 1) + \frac{n_{00}}{g^2}$ ")
694 elif ptcFitType == 'POLYNOMIAL':
695 ptcFunc = funcPolynomial
696 for key in dataset.ptcFitPars:
697 deg = len(dataset.ptcFitPars[key]) - 1
698 break
699 stringTitle = r"Polynomial (degree: %g)" % (deg)
700 else:
701 raise RuntimeError(f"The input dataset had an invalid dataset.ptcFitType: {ptcFitType}. \n" +
702 "Options: 'FULLCOVARIANCE', EXPAPPROXIMATION, or 'POLYNOMIAL'.")
704 legendFontSize = 7
705 labelFontSize = 7
706 titleFontSize = 9
707 supTitleFontSize = 18
708 markerSize = 25
710 # General determination of the size of the plot grid
711 nAmps = len(dataset.ampNames)
712 if nAmps == 2:
713 nRows, nCols = 2, 1
714 nRows = np.sqrt(nAmps)
715 mantissa, _ = np.modf(nRows)
716 if mantissa > 0:
717 nRows = int(nRows) + 1
718 nCols = nRows
719 else:
720 nRows = int(nRows)
721 nCols = nRows
723 f, ax = plt.subplots(nrows=nRows, ncols=nCols, sharex='col', sharey='row', figsize=(13, 10))
724 f2, ax2 = plt.subplots(nrows=nRows, ncols=nCols, sharex='col', sharey='row', figsize=(13, 10))
726 for i, (amp, a, a2) in enumerate(zip(dataset.ampNames, ax.flatten(), ax2.flatten())):
727 meanVecOriginal = np.array(dataset.rawMeans[amp])
728 varVecOriginal = np.array(dataset.rawVars[amp])
729 mask = dataset.visitMask[amp]
730 meanVecFinal = meanVecOriginal[mask]
731 varVecFinal = varVecOriginal[mask]
732 meanVecOutliers = meanVecOriginal[np.invert(mask)]
733 varVecOutliers = varVecOriginal[np.invert(mask)]
734 pars, parsErr = dataset.ptcFitPars[amp], dataset.ptcFitParsError[amp]
735 if ptcFitType == 'EXPAPPROXIMATION':
736 if len(meanVecFinal):
737 ptcA00, ptcA00error = pars[0], parsErr[0]
738 ptcGain, ptcGainError = pars[1], parsErr[1]
739 ptcNoise = np.sqrt(np.fabs(pars[2]))
740 ptcNoiseError = 0.5*(parsErr[2]/np.fabs(pars[2]))*np.sqrt(np.fabs(pars[2]))
741 stringLegend = (f"a00: {ptcA00:.2e}+/-{ptcA00error:.2e} 1/e"
742 f"\n Gain: {ptcGain:.4}+/-{ptcGainError:.2e} e/DN"
743 f"\n Noise: {ptcNoise:.4}+/-{ptcNoiseError:.2e} e \n")
745 if ptcFitType == 'POLYNOMIAL':
746 if len(meanVecFinal):
747 ptcGain, ptcGainError = 1./pars[1], np.fabs(1./pars[1])*(parsErr[1]/pars[1])
748 ptcNoise = np.sqrt(np.fabs(pars[0]))*ptcGain
749 ptcNoiseError = (0.5*(parsErr[0]/np.fabs(pars[0]))*(np.sqrt(np.fabs(pars[0]))))*ptcGain
750 stringLegend = (f"Noise: {ptcNoise:.4}+/-{ptcNoiseError:.2e} e \n"
751 f"Gain: {ptcGain:.4}+/-{ptcGainError:.2e} e/DN \n")
753 a.set_xlabel(r'Mean signal ($\mu$, DN)', fontsize=labelFontSize)
754 a.set_ylabel(r'Variance (DN$^2$)', fontsize=labelFontSize)
755 a.tick_params(labelsize=11)
756 a.set_xscale('linear', fontsize=labelFontSize)
757 a.set_yscale('linear', fontsize=labelFontSize)
759 a2.set_xlabel(r'Mean Signal ($\mu$, DN)', fontsize=labelFontSize)
760 a2.set_ylabel(r'Variance (DN$^2$)', fontsize=labelFontSize)
761 a2.tick_params(labelsize=11)
762 a2.set_xscale('log')
763 a2.set_yscale('log')
765 if len(meanVecFinal): # Empty if the whole amp is bad, for example.
766 minMeanVecFinal = np.min(meanVecFinal)
767 maxMeanVecFinal = np.max(meanVecFinal)
768 meanVecFit = np.linspace(minMeanVecFinal, maxMeanVecFinal, 100*len(meanVecFinal))
769 minMeanVecOriginal = np.min(meanVecOriginal)
770 maxMeanVecOriginal = np.max(meanVecOriginal)
771 deltaXlim = maxMeanVecOriginal - minMeanVecOriginal
773 a.plot(meanVecFit, ptcFunc(pars, meanVecFit), color='red')
774 a.plot(meanVecFinal, pars[0] + pars[1]*meanVecFinal, color='green', linestyle='--')
775 a.scatter(meanVecFinal, varVecFinal, c='blue', marker='o', s=markerSize)
776 a.scatter(meanVecOutliers, varVecOutliers, c='magenta', marker='s', s=markerSize)
777 a.text(0.03, 0.7, stringLegend, transform=a.transAxes, fontsize=legendFontSize)
778 a.set_title(amp, fontsize=titleFontSize)
779 a.set_xlim([minMeanVecOriginal - 0.2*deltaXlim, maxMeanVecOriginal + 0.2*deltaXlim])
781 # Same, but in log-scale
782 a2.plot(meanVecFit, ptcFunc(pars, meanVecFit), color='red')
783 a2.scatter(meanVecFinal, varVecFinal, c='blue', marker='o', s=markerSize)
784 a2.scatter(meanVecOutliers, varVecOutliers, c='magenta', marker='s', s=markerSize)
785 a2.text(0.03, 0.7, stringLegend, transform=a2.transAxes, fontsize=legendFontSize)
786 a2.set_title(amp, fontsize=titleFontSize)
787 a2.set_xlim([minMeanVecOriginal, maxMeanVecOriginal])
788 else:
789 a.set_title(f"{amp} (BAD)", fontsize=titleFontSize)
790 a2.set_title(f"{amp} (BAD)", fontsize=titleFontSize)
792 f.suptitle("PTC \n Fit: " + stringTitle, fontsize=supTitleFontSize)
793 pdfPages.savefig(f)
794 f2.suptitle("PTC (log-log)", fontsize=supTitleFontSize)
795 pdfPages.savefig(f2)
797 return
799 def _plotLinearizer(self, dataset, linearizer, pdfPages):
800 """Plot linearity and linearity residual per amplifier
802 Parameters
803 ----------
804 dataset : `lsst.cp.pipe.ptc.PhotonTransferCurveDataset`
805 The dataset containing the means, variances, exposure times, and mask.
807 linearizer : `lsst.ip.isr.Linearizer`
808 Linearizer object
809 """
810 legendFontSize = 7
811 labelFontSize = 7
812 titleFontSize = 9
813 supTitleFontSize = 18
815 # General determination of the size of the plot grid
816 nAmps = len(dataset.ampNames)
817 if nAmps == 2:
818 nRows, nCols = 2, 1
819 nRows = np.sqrt(nAmps)
820 mantissa, _ = np.modf(nRows)
821 if mantissa > 0:
822 nRows = int(nRows) + 1
823 nCols = nRows
824 else:
825 nRows = int(nRows)
826 nCols = nRows
828 # Plot mean vs time (f1), and fractional residuals (f2)
829 f, ax = plt.subplots(nrows=nRows, ncols=nCols, sharex='col', sharey='row', figsize=(13, 10))
830 f2, ax2 = plt.subplots(nrows=nRows, ncols=nCols, sharex='col', sharey='row', figsize=(13, 10))
831 for i, (amp, a, a2) in enumerate(zip(dataset.ampNames, ax.flatten(), ax2.flatten())):
832 meanVecFinal = np.array(dataset.rawMeans[amp])[dataset.visitMask[amp]]
833 timeVecFinal = np.array(dataset.rawExpTimes[amp])[dataset.visitMask[amp]]
835 a.set_xlabel('Time (sec)', fontsize=labelFontSize)
836 a.set_ylabel(r'Mean signal ($\mu$, DN)', fontsize=labelFontSize)
837 a.tick_params(labelsize=labelFontSize)
838 a.set_xscale('linear', fontsize=labelFontSize)
839 a.set_yscale('linear', fontsize=labelFontSize)
841 a2.axhline(y=0, color='k')
842 a2.axvline(x=0, color='k', linestyle='-')
843 a2.set_xlabel(r'Mean signal ($\mu$, DN)', fontsize=labelFontSize)
844 a2.set_ylabel('Fractional nonlinearity (%)', fontsize=labelFontSize)
845 a2.tick_params(labelsize=labelFontSize)
846 a2.set_xscale('linear', fontsize=labelFontSize)
847 a2.set_yscale('linear', fontsize=labelFontSize)
849 if len(meanVecFinal):
850 pars, parsErr = linearizer.fitParams[amp], linearizer.fitParamsErr[amp]
851 k0, k0Error = pars[0], parsErr[0]
852 k1, k1Error = pars[1], parsErr[1]
853 k2, k2Error = pars[2], parsErr[2]
854 stringLegend = (f"k0: {k0:.4}+/-{k0Error:.2e} DN\n k1: {k1:.4}+/-{k1Error:.2e} DN/t"
855 f"\n k2: {k2:.2e}+/-{k2Error:.2e} DN/t^2 \n")
856 a.scatter(timeVecFinal, meanVecFinal)
857 a.plot(timeVecFinal, funcPolynomial(pars, timeVecFinal), color='red')
858 a.text(0.03, 0.75, stringLegend, transform=a.transAxes, fontsize=legendFontSize)
859 a.set_title(f"{amp}", fontsize=titleFontSize)
861 linearPart = k0 + k1*timeVecFinal
862 fracLinRes = 100*(linearPart - meanVecFinal)/linearPart
863 a2.plot(meanVecFinal, fracLinRes, c='g')
864 a2.set_title(f"{amp}", fontsize=titleFontSize)
865 else:
866 a.set_title(f"{amp} (BAD)", fontsize=titleFontSize)
867 a2.set_title(f"{amp} (BAD)", fontsize=titleFontSize)
869 f.suptitle("Linearity \n Fit: Polynomial (degree: %g)"
870 % (len(pars)-1),
871 fontsize=supTitleFontSize)
872 f2.suptitle(r"Fractional NL residual" + "\n" +
873 r"$100\times \frac{(k_0 + k_1*Time-\mu)}{k_0+k_1*Time}$",
874 fontsize=supTitleFontSize)
875 pdfPages.savefig(f)
876 pdfPages.savefig(f2)
878 @staticmethod
879 def findGroups(x, maxDiff):
880 """Group data into bins, with at most maxDiff distance between bins.
882 Parameters
883 ----------
884 x: `list`
885 Data to bin.
887 maxDiff: `int`
888 Maximum distance between bins.
890 Returns
891 -------
892 index: `list`
893 Bin indices.
894 """
895 ix = np.argsort(x)
896 xsort = np.sort(x)
897 index = np.zeros_like(x, dtype=np.int32)
898 xc = xsort[0]
899 group = 0
900 ng = 1
902 for i in range(1, len(ix)):
903 xval = xsort[i]
904 if (xval - xc < maxDiff):
905 xc = (ng*xc + xval)/(ng+1)
906 ng += 1
907 index[ix[i]] = group
908 else:
909 group += 1
910 ng = 1
911 index[ix[i]] = group
912 xc = xval
914 return index
916 @staticmethod
917 def indexForBins(x, nBins):
918 """Builds an index with regular binning. The result can be fed into binData.
920 Parameters
921 ----------
922 x: `numpy.array`
923 Data to bin.
924 nBins: `int`
925 Number of bin.
927 Returns
928 -------
929 np.digitize(x, bins): `numpy.array`
930 Bin indices.
931 """
933 bins = np.linspace(x.min(), x.max() + abs(x.max() * 1e-7), nBins + 1)
935 return np.digitize(x, bins)
937 @staticmethod
938 def binData(x, y, binIndex, wy=None):
939 """Bin data (usually for display purposes).
941 Patrameters
942 -----------
943 x: `numpy.array`
944 Data to bin.
946 y: `numpy.array`
947 Data to bin.
949 binIdex: `list`
950 Bin number of each datum.
952 wy: `numpy.array`
953 Inverse rms of each datum to use when averaging (the actual weight is wy**2).
955 Returns:
956 -------
958 xbin: `numpy.array`
959 Binned data in x.
961 ybin: `numpy.array`
962 Binned data in y.
964 wybin: `numpy.array`
965 Binned weights in y, computed from wy's in each bin.
967 sybin: `numpy.array`
968 Uncertainty on the bin average, considering actual scatter, and ignoring weights.
969 """
971 if wy is None:
972 wy = np.ones_like(x)
973 binIndexSet = set(binIndex)
974 w2 = wy*wy
975 xw2 = x*(w2)
976 xbin = np.array([xw2[binIndex == i].sum()/w2[binIndex == i].sum() for i in binIndexSet])
978 yw2 = y*w2
979 ybin = np.array([yw2[binIndex == i].sum()/w2[binIndex == i].sum() for i in binIndexSet])
981 wybin = np.sqrt(np.array([w2[binIndex == i].sum() for i in binIndexSet]))
982 sybin = np.array([y[binIndex == i].std()/np.sqrt(np.array([binIndex == i]).sum())
983 for i in binIndexSet])
985 return xbin, ybin, wybin, sybin