23 __all__ = [
'PlotPhotonTransferCurveTask']
26 import matplotlib.pyplot
as plt
27 import matplotlib
as mpl
28 from matplotlib
import gridspec
30 from matplotlib.backends.backend_pdf
import PdfPages
35 calculateWeightedReducedChi2)
36 from matplotlib.ticker
import MaxNLocator
47 """A class to plot the dataset from MeasurePhotonTransferCurveTask.
52 datasetFileName : `str`
53 datasetPtc (lsst.ip.isr.PhotonTransferCurveDataset) file
56 linearizerFileName : `str`, optional
57 linearizer (isr.linearize.Linearizer) file
60 outDir : `str`, optional
61 Path to the output directory where the final PDF will
64 detNum : `int`, optional
67 signalElectronsRelativeA : `float`, optional
68 Signal value for relative systematic bias between different
69 methods of estimating a_ij (Fig. 15 of Astier+19).
71 plotNormalizedCovariancesNumberOfBins : `float`, optional
72 Number of bins in `plotNormalizedCovariancesNumber` function
73 (Fig. 8, 10., of Astier+19).
77 The plotting code in this file is almost identical to the code in
78 `plotPtcGen2.py`. If further changes are implemented in this file,
79 `plotPtcGen2.py` needs to be updated accordingly, and vice versa.
80 The file `plotPtcGen2.py` helps with maintaining backwards
81 compatibility with gen2 as we transition to gen3; the code
82 duplication is meant to only last for few month from now
83 (Jan, 2021). At that point only this file, `plotPtc.py`, will
87 def __init__(self, datasetFilename, linearizerFileName=None,
88 outDir='.', detNum=999, signalElectronsRelativeA=75000,
89 plotNormalizedCovariancesNumberOfBins=10):
98 """Run the Photon Transfer Curve (PTC) plotting measurement task.
102 datasetPtc = PhotonTransferCurveDataset.readFits(datasetFile)
104 dirname = self.
outDiroutDir
105 if not os.path.exists(dirname):
108 detNum = self.
detNumdetNum
109 filename = f
"PTC_det{detNum}.pdf"
110 filenameFull = os.path.join(dirname, filename)
113 linearizer = isr.linearize.Linearizer.readFits(self.
linearizerFileNamelinearizerFileName)
116 self.
runrun(filenameFull, datasetPtc, linearizer=linearizer, log=lsstLog)
120 def run(self, filenameFull, datasetPtc, linearizer=None, log=None):
121 """Make the plots for the PTC task"""
122 ptcFitType = datasetPtc.ptcFitType
123 with PdfPages(filenameFull)
as pdfPages:
124 if ptcFitType
in [
"FULLCOVARIANCE", ]:
126 elif ptcFitType
in [
"EXPAPPROXIMATION",
"POLYNOMIAL"]:
129 raise RuntimeError(f
"The input dataset had an invalid dataset.ptcFitType: {ptcFitType}. \n"
130 "Options: 'FULLCOVARIANCE', EXPAPPROXIMATION, or 'POLYNOMIAL'.")
138 """Make plots for MeasurePhotonTransferCurve task when doCovariancesAstier=True.
140 This function call other functions that mostly reproduce the plots in Astier+19.
141 Most of the code is ported from Pierre Astier's repository https://github.com/PierreAstier/bfptc
145 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset`
146 The dataset containing the necessary information to produce the plots.
148 pdfPages: `matplotlib.backends.backend_pdf.PdfPages`
149 PDF file where the plots will be saved.
151 log : `lsst.log.Log`, optional
152 Logger to handle messages
154 mu = dataset.finalMeans
156 fullCovs = dataset.covariances
157 fullCovsModel = dataset.covariancesModel
158 fullCovWeights = dataset.covariancesSqrtWeights
159 aDict = dataset.aMatrix
160 bDict = dataset.bMatrix
161 fullCovsNoB = dataset.covariances
162 fullCovsModelNoB = dataset.covariancesModelNoB
163 fullCovWeightsNoB = dataset.covariancesSqrtWeights
164 aDictNoB = dataset.aMatrixNoB
165 gainDict = dataset.gain
166 noiseDict = dataset.noise
168 self.
plotCovariancesplotCovariances(mu, fullCovs, fullCovsModel, fullCovWeights, fullCovsNoB, fullCovsModelNoB,
169 fullCovWeightsNoB, gainDict, noiseDict, aDict, bDict, pdfPages)
171 fullCovsModelNoB, fullCovWeightsNoB, pdfPages,
172 offset=0.01, topPlot=
True,
176 fullCovsModelNoB, fullCovWeightsNoB, pdfPages,
180 fullCovsModelNoB, fullCovWeightsNoB, pdfPages,
183 self.
plot_a_bplot_a_b(aDict, bDict, pdfPages)
184 self.
ab_vs_distab_vs_dist(aDict, bDict, pdfPages, bRange=4)
192 def plotCovariances(mu, covs, covsModel, covsWeights, covsNoB, covsModelNoB, covsWeightsNoB,
193 gainDict, noiseDict, aDict, bDict, pdfPages):
194 """Plot covariances and models: Cov00, Cov10, Cov01.
196 Figs. 6 and 7 of Astier+19
200 mu : `dict`, [`str`, `list`]
201 Dictionary keyed by amp name with mean signal values.
203 covs : `dict`, [`str`, `list`]
204 Dictionary keyed by amp names containing a list of measued covariances per mean flux.
206 covsModel : `dict`, [`str`, `list`]
207 Dictionary keyed by amp names containinging covariances model (Eq. 20 of Astier+19) per mean flux.
209 covsWeights : `dict`, [`str`, `list`]
210 Dictionary keyed by amp names containinging sqrt. of covariances weights.
212 covsNoB : `dict`, [`str`, `list`]
213 Dictionary keyed by amp names containing a list of measued covariances per mean flux ('b'=0 in
216 covsModelNoB : `dict`, [`str`, `list`]
217 Dictionary keyed by amp names containing covariances model (with 'b'=0 in Eq. 20 of Astier+19)
220 covsWeightsNoB : `dict`, [`str`, `list`]
221 Dictionary keyed by amp names containing sqrt. of covariances weights ('b' = 0 in Eq. 20 of
224 gainDict : `dict`, [`str`, `float`]
225 Dictionary keyed by amp names containing the gains in e-/ADU.
227 noiseDict : `dict`, [`str`, `float`]
228 Dictionary keyed by amp names containing the rms redout noise in e-.
230 aDict : `dict`, [`str`, `numpy.array`]
231 Dictionary keyed by amp names containing 'a' coefficients (Eq. 20 of Astier+19).
233 bDict : `dict`, [`str`, `numpy.array`]
234 Dictionary keyed by amp names containing 'b' coefficients (Eq. 20 of Astier+19).
236 pdfPages: `matplotlib.backends.backend_pdf.PdfPages`
237 PDF file where the plots will be saved.
243 supTitleFontSize = 18
249 nRows = np.sqrt(nAmps)
250 mantissa, _ = np.modf(nRows)
252 nRows = int(nRows) + 1
258 f, ax = plt.subplots(nrows=nRows, ncols=nCols, sharex=
'col', sharey=
'row', figsize=(13, 10))
259 f2, ax2 = plt.subplots(nrows=nRows, ncols=nCols, sharex=
'col', sharey=
'row', figsize=(13, 10))
260 fResCov00, axResCov00 = plt.subplots(nrows=nRows, ncols=nCols, sharex=
'col', sharey=
'row',
262 fCov01, axCov01 = plt.subplots(nrows=nRows, ncols=nCols, sharex=
'col', sharey=
'row', figsize=(13, 10))
263 fCov10, axCov10 = plt.subplots(nrows=nRows, ncols=nCols, sharex=
'col', sharey=
'row', figsize=(13, 10))
265 assert(len(covsModel) == nAmps)
266 assert(len(covsWeights) == nAmps)
268 assert(len(covsNoB) == nAmps)
269 assert(len(covsModelNoB) == nAmps)
270 assert(len(covsWeightsNoB) == nAmps)
272 for i, (amp, a, a2, aResVar, a3, a4)
in enumerate(zip(covs, ax.flatten(),
273 ax2.flatten(), axResCov00.flatten(),
274 axCov01.flatten(), axCov10.flatten())):
276 muAmp, cov, model, weight = mu[amp], covs[amp], covsModel[amp], covsWeights[amp]
277 if not np.isnan(np.array(cov)).all():
278 aCoeffs, bCoeffs = np.array(aDict[amp]), np.array(bDict[amp])
279 gain, noise = gainDict[amp], noiseDict[amp]
280 (meanVecFinal, varVecFinal, varVecModelFinal,
286 varWeightsFinal, len(meanVecFinal), 4)
288 (meanVecFinalCov01, varVecFinalCov01, varVecModelFinalCov01,
291 (meanVecFinalCov10, varVecFinalCov10, varVecModelFinalCov10,
295 par2 = np.polyfit(meanVecFinal, varVecFinal, 2, w=varWeightsFinal)
296 varModelFinalQuadratic = np.polyval(par2, meanVecFinal)
298 varWeightsFinal, len(meanVecFinal), 3)
301 covNoB, modelNoB, weightNoB = covsNoB[amp], covsModelNoB[amp], covsWeightsNoB[amp]
302 (meanVecFinalNoB, varVecFinalNoB, varVecModelFinalNoB,
304 weightNoB, returnMasked=
True)
307 varWeightsFinalNoB, len(meanVecFinalNoB),
309 stringLegend = (f
"Gain: {gain:.4} e/ADU \n"
310 f
"Noise: {noise:.4} e \n"
311 +
r"$a_{00}$: %.3e 1/e"%aCoeffs[0, 0] +
"\n"
312 +
r"$b_{00}$: %.3e 1/e"%bCoeffs[0, 0]
313 + f
"\nLast in fit: {meanVecFinal[-1]:.7} ADU ")
314 minMeanVecFinal = np.nanmin(meanVecFinal)
315 maxMeanVecFinal = np.nanmax(meanVecFinal)
316 deltaXlim = maxMeanVecFinal - minMeanVecFinal
318 a.set_xlabel(
r'Mean signal ($\mu$, ADU)', fontsize=labelFontSize)
319 a.set_ylabel(
r'Variance (ADU$^2$)', fontsize=labelFontSize)
320 a.tick_params(labelsize=11)
321 a.set_xscale(
'linear')
322 a.set_yscale(
'linear')
323 a.scatter(meanVecFinal, varVecFinal, c=
'blue', marker=
'o', s=markerSize)
324 a.plot(meanVecFinal, varVecModelFinal, color=
'red', lineStyle=
'-')
325 a.text(0.03, 0.7, stringLegend, transform=a.transAxes, fontsize=legendFontSize)
326 a.set_title(amp, fontsize=titleFontSize)
327 a.set_xlim([minMeanVecFinal - 0.2*deltaXlim, maxMeanVecFinal + 0.2*deltaXlim])
330 a2.set_xlabel(
r'Mean Signal ($\mu$, ADU)', fontsize=labelFontSize)
331 a2.set_ylabel(
r'Variance (ADU$^2$)', fontsize=labelFontSize)
332 a2.tick_params(labelsize=11)
335 a2.plot(meanVecFinal, varVecModelFinal, color=
'red', lineStyle=
'-')
336 a2.scatter(meanVecFinal, varVecFinal, c=
'blue', marker=
'o', s=markerSize)
337 a2.text(0.03, 0.7, stringLegend, transform=a2.transAxes, fontsize=legendFontSize)
338 a2.set_title(amp, fontsize=titleFontSize)
339 a2.set_xlim([minMeanVecFinal, maxMeanVecFinal])
342 aResVar.set_xlabel(
r'Mean signal ($\mu$, ADU)', fontsize=labelFontSize)
343 aResVar.set_ylabel(
r'Residuals (ADU$^2$)', fontsize=labelFontSize)
344 aResVar.tick_params(labelsize=11)
345 aResVar.set_xscale(
'linear')
346 aResVar.set_yscale(
'linear')
347 aResVar.plot(meanVecFinal, varVecFinal - varVecModelFinal, color=
'blue', lineStyle=
'-',
348 label=
r'Full fit ($\chi_{\rm{red}}^2$: %g)'%chi2FullModelVar)
349 aResVar.plot(meanVecFinal, varVecFinal - varModelFinalQuadratic, color=
'red', lineStyle=
'-',
350 label=
r'Quadratic fit ($\chi_{\rm{red}}^2$: %g)'%chi2QuadModelVar)
351 aResVar.plot(meanVecFinalNoB, varVecFinalNoB - varVecModelFinalNoB, color=
'green',
353 label=
r'Full fit (b=0) ($\chi_{\rm{red}}^2$: %g)'%chi2FullModelNoBVar)
354 aResVar.axhline(color=
'black')
355 aResVar.set_title(amp, fontsize=titleFontSize)
356 aResVar.set_xlim([minMeanVecFinal - 0.2*deltaXlim, maxMeanVecFinal + 0.2*deltaXlim])
357 aResVar.legend(fontsize=7)
359 a3.set_xlabel(
r'Mean signal ($\mu$, ADU)', fontsize=labelFontSize)
360 a3.set_ylabel(
r'Cov01 (ADU$^2$)', fontsize=labelFontSize)
361 a3.tick_params(labelsize=11)
362 a3.set_xscale(
'linear')
363 a3.set_yscale(
'linear')
364 a3.scatter(meanVecFinalCov01, varVecFinalCov01, c=
'blue', marker=
'o', s=markerSize)
365 a3.plot(meanVecFinalCov01, varVecModelFinalCov01, color=
'red', lineStyle=
'-')
366 a3.set_title(amp, fontsize=titleFontSize)
367 a3.set_xlim([minMeanVecFinal - 0.2*deltaXlim, maxMeanVecFinal + 0.2*deltaXlim])
369 a4.set_xlabel(
r'Mean signal ($\mu$, ADU)', fontsize=labelFontSize)
370 a4.set_ylabel(
r'Cov10 (ADU$^2$)', fontsize=labelFontSize)
371 a4.tick_params(labelsize=11)
372 a4.set_xscale(
'linear')
373 a4.set_yscale(
'linear')
374 a4.scatter(meanVecFinalCov10, varVecFinalCov10, c=
'blue', marker=
'o', s=markerSize)
375 a4.plot(meanVecFinalCov10, varVecModelFinalCov10, color=
'red', lineStyle=
'-')
376 a4.set_title(amp, fontsize=titleFontSize)
377 a4.set_xlim([minMeanVecFinal - 0.2*deltaXlim, maxMeanVecFinal + 0.2*deltaXlim])
380 a.set_title(f
"{amp} (BAD)", fontsize=titleFontSize)
381 a2.set_title(f
"{amp} (BAD)", fontsize=titleFontSize)
382 a3.set_title(f
"{amp} (BAD)", fontsize=titleFontSize)
383 a4.set_title(f
"{amp} (BAD)", fontsize=titleFontSize)
385 f.suptitle(
"PTC from covariances as in Astier+19 \n Fit: Eq. 20, Astier+19",
386 fontsize=supTitleFontSize)
388 f2.suptitle(
"PTC from covariances as in Astier+19 (log-log) \n Fit: Eq. 20, Astier+19",
389 fontsize=supTitleFontSize)
391 fResCov00.suptitle(
"Residuals (data-model) for Cov00 (Var)", fontsize=supTitleFontSize)
392 pdfPages.savefig(fResCov00)
393 fCov01.suptitle(
"Cov01 as in Astier+19 (nearest parallel neighbor covariance) \n"
394 " Fit: Eq. 20, Astier+19", fontsize=supTitleFontSize)
395 pdfPages.savefig(fCov01)
396 fCov10.suptitle(
"Cov10 as in Astier+19 (nearest serial neighbor covariance) \n"
397 "Fit: Eq. 20, Astier+19", fontsize=supTitleFontSize)
398 pdfPages.savefig(fCov10)
403 covsWeightsNoB, pdfPages, offset=0.004,
404 numberOfBins=10, plotData=True, topPlot=False, log=None):
405 """Plot C_ij/mu vs mu.
407 Figs. 8, 10, and 11 of Astier+19
417 inputMu : `dict`, [`str`, `list`]
418 Dictionary keyed by amp name with mean signal values.
420 covs : `dict`, [`str`, `list`]
421 Dictionary keyed by amp names containing a list of measued covariances per mean flux.
423 covsModel : `dict`, [`str`, `list`]
424 Dictionary keyed by amp names containinging covariances model (Eq. 20 of Astier+19) per mean flux.
426 covsWeights : `dict`, [`str`, `list`]
427 Dictionary keyed by amp names containinging sqrt. of covariances weights.
429 covsNoB : `dict`, [`str`, `list`]
430 Dictionary keyed by amp names containing a list of measued covariances per mean flux ('b'=0 in
433 covsModelNoB : `dict`, [`str`, `list`]
434 Dictionary keyed by amp names containing covariances model (with 'b'=0 in Eq. 20 of Astier+19)
437 covsWeightsNoB : `dict`, [`str`, `list`]
438 Dictionary keyed by amp names containing sqrt. of covariances weights ('b' = 0 in Eq. 20 of
441 expIdMask : `dict`, [`str`, `list`]
442 Dictionary keyed by amp names containing the masked exposure pairs.
444 pdfPages: `matplotlib.backends.backend_pdf.PdfPages`
445 PDF file where the plots will be saved.
447 offset : `float`, optional
448 Constant offset factor to plot covariances in same panel (so they don't overlap).
450 numberOfBins : `int`, optional
451 Number of bins for top and bottom plot.
453 plotData : `bool`, optional
454 Plot the data points?
456 topPlot : `bool`, optional
457 Plot the top plot with the covariances, and the bottom plot with the model residuals?
459 log : `lsst.log.Log`, optional
460 Logger to handle messages.
463 fig = plt.figure(figsize=(8, 10))
464 gs = gridspec.GridSpec(2, 1, height_ratios=[3, 1])
466 ax0 = plt.subplot(gs[0])
467 plt.setp(ax0.get_xticklabels(), visible=
False)
469 fig = plt.figure(figsize=(8, 8))
470 ax0 = plt.subplot(111)
471 ax0.ticklabel_format(style=
'sci', axis=
'x', scilimits=(0, 0))
472 ax0.tick_params(axis=
'both', labelsize=
'x-large')
473 mue, rese, wce = [], [], []
474 mueNoB, reseNoB, wceNoB = [], [], []
475 for counter, amp
in enumerate(covs):
476 muAmp, fullCov, fullCovModel, fullCovWeight = (inputMu[amp], covs[amp], covsModel[amp],
478 if len(fullCov) == 0:
481 fullCovWeight, divideByMu=
True,
485 rese += list(cov - model)
486 wce += list(weightCov)
488 fullCovNoB, fullCovModelNoB, fullCovWeightNoB = (covsNoB[amp], covsModelNoB[amp],
490 if len(fullCovNoB) == 0:
492 (muNoB, covNoB, modelNoB,
494 fullCovWeightNoB, divideByMu=
True,
497 mueNoB += list(muNoB)
498 reseNoB += list(covNoB - modelNoB)
499 wceNoB += list(weightCovNoB)
502 fit_curve, = plt.plot(mu, model + counter*offset,
'-', linewidth=4.0)
506 xb, yb, wyb, sigyb = self.
binDatabinData(mu, cov, gind, weightCov)
507 plt.errorbar(xb, yb+counter*offset, yerr=sigyb, marker=
'o', linestyle=
'none', markersize=6.5,
508 color=fit_curve.get_color(), label=f
"{amp} (N: {len(mu)})")
511 points, = plt.plot(mu, cov + counter*offset,
'.', color=fit_curve.get_color())
512 plt.legend(loc=
'upper right', fontsize=8)
515 rese = np.array(rese)
517 mueNoB = np.array(mueNoB)
518 reseNoB = np.array(reseNoB)
519 wceNoB = np.array(wceNoB)
521 plt.xlabel(
r"$\mu (el)$", fontsize=
'x-large')
522 plt.ylabel(
r"$Cov{%d%d}/\mu + Cst (el)$"%(i, j), fontsize=
'x-large')
525 xb, yb, wyb, sigyb = self.
binDatabinData(mue, rese, gind, wce)
527 ax1 = plt.subplot(gs[1], sharex=ax0)
528 ax1.errorbar(xb, yb, yerr=sigyb, marker=
'o', linestyle=
'none', label=
'Full fit')
529 gindNoB = self.
indexForBinsindexForBins(mueNoB, numberOfBins)
530 xb2, yb2, wyb2, sigyb2 = self.
binDatabinData(mueNoB, reseNoB, gindNoB, wceNoB)
532 ax1.errorbar(xb2, yb2, yerr=sigyb2, marker=
'o', linestyle=
'none', label=
'b = 0')
533 ax1.tick_params(axis=
'both', labelsize=
'x-large')
534 plt.legend(loc=
'upper left', fontsize=
'large')
536 plt.plot(xb, [0]*len(xb),
'--', color=
'k')
537 plt.ticklabel_format(style=
'sci', axis=
'x', scilimits=(0, 0))
538 plt.ticklabel_format(style=
'sci', axis=
'y', scilimits=(0, 0))
539 plt.xlabel(
r'$\mu (el)$', fontsize=
'x-large')
540 plt.ylabel(
r'$Cov{%d%d}/\mu$ -model (el)'%(i, j), fontsize=
'x-large')
542 plt.suptitle(f
"Nbins: {numberOfBins}")
545 labels0 = [item.get_text()
for item
in ax0.get_yticklabels()]
547 ax0.set_yticklabels(labels0)
548 pdfPages.savefig(fig)
554 """Fig. 12 of Astier+19
556 Color display of a and b arrays fits, averaged over channels.
560 aDict : `dict`, [`numpy.array`]
561 Dictionary keyed by amp names containing the fitted 'a' coefficients from the model
562 in Eq. 20 of Astier+19 (if `ptcFitType` is `FULLCOVARIANCE`).
564 bDict : `dict`, [`numpy.array`]
565 Dictionary keyed by amp names containing the fitted 'b' coefficients from the model
566 in Eq. 20 of Astier+19 (if `ptcFitType` is `FULLCOVARIANCE`).
568 pdfPages: `matplotlib.backends.backend_pdf.PdfPages`
569 PDF file where the plots will be saved.
572 Maximum lag for b arrays.
576 if np.isnan(aDict[amp]).all():
580 a = np.array(a).mean(axis=0)
581 b = np.array(b).mean(axis=0)
582 fig = plt.figure(figsize=(7, 11))
583 ax0 = fig.add_subplot(2, 1, 1)
584 im0 = ax0.imshow(np.abs(a.transpose()), origin=
'lower', norm=mpl.colors.LogNorm())
585 ax0.tick_params(axis=
'both', labelsize=
'x-large')
586 ax0.set_title(
r'$|a|$', fontsize=
'x-large')
587 ax0.xaxis.set_ticks_position(
'bottom')
588 cb0 = plt.colorbar(im0)
589 cb0.ax.tick_params(labelsize=
'x-large')
591 ax1 = fig.add_subplot(2, 1, 2)
592 ax1.tick_params(axis=
'both', labelsize=
'x-large')
593 ax1.yaxis.set_major_locator(MaxNLocator(integer=
True))
594 ax1.xaxis.set_major_locator(MaxNLocator(integer=
True))
595 im1 = ax1.imshow(1e6*b[:bRange, :bRange].transpose(), origin=
'lower')
596 cb1 = plt.colorbar(im1)
597 cb1.ax.tick_params(labelsize=
'x-large')
598 ax1.set_title(
r'$b \times 10^6$', fontsize=
'x-large')
599 ax1.xaxis.set_ticks_position(
'bottom')
601 pdfPages.savefig(fig)
607 """Fig. 13 of Astier+19.
609 Values of a and b arrays fits, averaged over amplifiers, as a function of distance.
613 aDict : `dict`, [`numpy.array`]
614 Dictionary keyed by amp names containing the fitted 'a' coefficients from the model
615 in Eq. 20 of Astier+19 (if `ptcFitType` is `FULLCOVARIANCE`).
617 bDict : `dict`, [`numpy.array`]
618 Dictionary keyed by amp names containing the fitted 'b' coefficients from the model
619 in Eq. 20 of Astier+19 (if `ptcFitType` is `FULLCOVARIANCE`).
621 pdfPages: `matplotlib.backends.backend_pdf.PdfPages`
622 PDF file where the plots will be saved.
625 Maximum lag for b arrays.
627 assert (len(aDict) == len(bDict))
630 if np.isnan(aDict[amp]).all():
635 sy = a.std(axis=0)/np.sqrt(len(aDict))
636 i, j = np.indices(y.shape)
637 upper = (i >= j).ravel()
638 r = np.sqrt(i**2 + j**2).ravel()
641 fig = plt.figure(figsize=(6, 9))
642 ax = fig.add_subplot(211)
643 ax.set_xlim([0.5, r.max()+1])
644 ax.errorbar(r[upper], y[upper], yerr=sy[upper], marker=
'o', linestyle=
'none', color=
'b',
646 ax.errorbar(r[~upper], y[~upper], yerr=sy[~upper], marker=
'o', linestyle=
'none', color=
'r',
648 ax.legend(loc=
'upper center', fontsize=
'x-large')
649 ax.set_xlabel(
r'$\sqrt{i^2+j^2}$', fontsize=
'x-large')
650 ax.set_ylabel(
r'$a_{ij}$', fontsize=
'x-large')
652 ax.tick_params(axis=
'both', labelsize=
'x-large')
655 axb = fig.add_subplot(212)
658 if np.isnan(bDict[amp]).all():
663 syb = b.std(axis=0)/np.sqrt(len(bDict))
664 ib, jb = np.indices(yb.shape)
665 upper = (ib > jb).ravel()
666 rb = np.sqrt(i**2 + j**2).ravel()
671 axb.set_xlim([xmin, xmax+0.2])
672 cutu = (r > xmin) & (r < xmax) & (upper)
673 cutl = (r > xmin) & (r < xmax) & (~upper)
674 axb.errorbar(rb[cutu], yb[cutu], yerr=syb[cutu], marker=
'o', linestyle=
'none', color=
'b',
676 axb.errorbar(rb[cutl], yb[cutl], yerr=syb[cutl], marker=
'o', linestyle=
'none', color=
'r',
678 plt.legend(loc=
'upper center', fontsize=
'x-large')
679 axb.set_xlabel(
r'$\sqrt{i^2+j^2}$', fontsize=
'x-large')
680 axb.set_ylabel(
r'$b_{ij}$', fontsize=
'x-large')
681 axb.ticklabel_format(style=
'sci', axis=
'y', scilimits=(0, 0))
682 axb.tick_params(axis=
'both', labelsize=
'x-large')
684 pdfPages.savefig(fig)
690 """Fig. 14. of Astier+19
692 Cumulative sum of a_ij as a function of maximum separation. This plot displays the average over
697 aDict : `dict`, [`numpy.array`]
698 Dictionary keyed by amp names containing the fitted 'a' coefficients from the model
699 in Eq. 20 of Astier+19 (if `ptcFitType` is `FULLCOVARIANCE`).
701 bDict : `dict`, [`numpy.array`]
702 Dictionary keyed by amp names containing the fitted 'b' coefficients from the model
703 in Eq. 20 of Astier+19 (if `ptcFitType` is `FULLCOVARIANCE`).
705 pdfPages: `matplotlib.backends.backend_pdf.PdfPages`
706 PDF file where the plots will be saved.
708 assert (len(aDict) == len(bDict))
711 if np.isnan(aDict[amp]).all()
or np.isnan(bDict[amp]).all():
715 a = np.array(a).mean(axis=0)
716 b = np.array(b).mean(axis=0)
717 fig = plt.figure(figsize=(7, 6))
718 w = 4*np.ones_like(a)
723 indices = range(1, a.shape[0]+1)
724 sums = [wa[0:n, 0:n].sum()
for n
in indices]
725 ax = plt.subplot(111)
726 ax.plot(indices, sums/sums[0],
'o', color=
'b')
728 ax.set_xlim(indices[0]-0.5, indices[-1]+0.5)
729 ax.set_ylim(
None, 1.2)
730 ax.set_ylabel(
r'$[\sum_{|i|<n\ &\ |j|<n} a_{ij}] / |a_{00}|$', fontsize=
'x-large')
731 ax.set_xlabel(
'n', fontsize=
'x-large')
732 ax.tick_params(axis=
'both', labelsize=
'x-large')
734 pdfPages.savefig(fig)
740 gainDict, pdfPages, maxr=None):
741 """Fig. 15 in Astier+19.
743 Illustrates systematic bias from estimating 'a'
744 coefficients from the slope of correlations as opposed to the
745 full model in Astier+19.
750 Dictionary of 'a' matrices (Eq. 20, Astier+19), with amp names as keys.
753 Dictionary of 'a' matrices ('b'= 0 in Eq. 20, Astier+19), with amp names as keys.
755 fullCovsModel : `dict`, [`str`, `list`]
756 Dictionary keyed by amp names containing covariances model per mean flux.
758 fullCovsModelNoB : `dict`, [`str`, `list`]
759 Dictionary keyed by amp names containing covariances model (with 'b'=0 in Eq. 20 of
760 Astier+19) per mean flux.
762 signalElectrons : `float`
763 Signal at which to evaluate the a_ij coefficients.
765 pdfPages: `matplotlib.backends.backend_pdf.PdfPages`
766 PDF file where the plots will be saved.
768 gainDict : `dict`, [`str`, `float`]
769 Dicgionary keyed by amp names with the gains in e-/ADU.
771 maxr : `int`, optional
775 fig = plt.figure(figsize=(7, 11))
776 title = [f
"'a' relative bias at {signalElectrons} e",
"'a' relative bias (b=0)"]
777 data = [(aDict, fullCovsModel), (aDictNoB, fullCovsModelNoB)]
779 for k, pair
in enumerate(data):
783 covModel = pair[1][amp]
784 if np.isnan(covModel).all():
789 diffs.append((aOld-a))
790 amean = np.array(amean).mean(axis=0)
791 diff = np.array(diffs).mean(axis=0)
798 diff = diff[:maxr, :maxr]
799 ax0 = fig.add_subplot(2, 1, k+1)
800 im0 = ax0.imshow(diff.transpose(), origin=
'lower')
801 ax0.yaxis.set_major_locator(MaxNLocator(integer=
True))
802 ax0.xaxis.set_major_locator(MaxNLocator(integer=
True))
803 ax0.tick_params(axis=
'both', labelsize=
'x-large')
805 ax0.set_title(title[k])
808 pdfPages.savefig(fig)
812 def _plotStandardPtc(self, dataset, ptcFitType, pdfPages):
813 """Plot PTC, var/signal vs signal, linearity, and linearity residual per amplifier.
817 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset`
818 The dataset containing the means, variances, exposure times, and mask.
821 Type of the model fit to the PTC. Options: 'FULLCOVARIANCE', EXPAPPROXIMATION, or 'POLYNOMIAL'.
823 pdfPages: `matplotlib.backends.backend_pdf.PdfPages`
824 PDF file where the plots will be saved.
827 if ptcFitType ==
'EXPAPPROXIMATION':
829 stringTitle = (
r"Var = $\frac{1}{2g^2a_{00}}(\exp (2a_{00} \mu g) - 1) + \frac{n_{00}}{g^2}$ ")
830 elif ptcFitType ==
'POLYNOMIAL':
831 ptcFunc = funcPolynomial
832 for key
in dataset.ptcFitPars:
833 deg = len(dataset.ptcFitPars[key]) - 1
835 stringTitle =
r"Polynomial (degree: %g)" % (deg)
837 raise RuntimeError(f
"The input dataset had an invalid dataset.ptcFitType: {ptcFitType}. \n"
838 "Options: 'FULLCOVARIANCE', EXPAPPROXIMATION, or 'POLYNOMIAL'.")
843 supTitleFontSize = 18
847 nAmps = len(dataset.ampNames)
850 nRows = np.sqrt(nAmps)
851 mantissa, _ = np.modf(nRows)
853 nRows = int(nRows) + 1
859 f, ax = plt.subplots(nrows=nRows, ncols=nCols, sharex=
'col', sharey=
'row', figsize=(13, 10))
860 f2, ax2 = plt.subplots(nrows=nRows, ncols=nCols, sharex=
'col', sharey=
'row', figsize=(13, 10))
861 f3, ax3 = plt.subplots(nrows=nRows, ncols=nCols, sharex=
'col', sharey=
'row', figsize=(13, 10))
863 for i, (amp, a, a2, a3)
in enumerate(zip(dataset.ampNames, ax.flatten(), ax2.flatten(),
865 meanVecOriginal = np.ravel(np.array(dataset.rawMeans[amp]))
866 varVecOriginal = np.ravel(np.array(dataset.rawVars[amp]))
867 mask = np.ravel(np.array(dataset.expIdMask[amp]))
868 if np.isnan(mask[0]):
869 a.set_title(f
"{amp} (BAD)", fontsize=titleFontSize)
870 a2.set_title(f
"{amp} (BAD)", fontsize=titleFontSize)
871 a3.set_title(f
"{amp} (BAD)", fontsize=titleFontSize)
874 mask = mask.astype(bool)
875 meanVecFinal = meanVecOriginal[mask]
876 varVecFinal = varVecOriginal[mask]
877 meanVecOutliers = meanVecOriginal[np.invert(mask)]
878 varVecOutliers = varVecOriginal[np.invert(mask)]
879 pars, parsErr = np.array(dataset.ptcFitPars[amp]), np.array(dataset.ptcFitParsError[amp])
880 ptcRedChi2 = dataset.ptcFitChiSq[amp]
881 if ptcFitType ==
'EXPAPPROXIMATION':
882 if len(meanVecFinal):
883 ptcA00, ptcA00error = pars[0], parsErr[0]
884 ptcGain, ptcGainError = pars[1], parsErr[1]
885 ptcNoise = np.sqrt((pars[2]))
886 ptcNoiseAdu = ptcNoise*(1./ptcGain)
887 ptcNoiseError = 0.5*(parsErr[2]/np.fabs(pars[2]))*np.sqrt(np.fabs(pars[2]))
888 stringLegend = (f
"a00: {ptcA00:.2e}+/-{ptcA00error:.2e} 1/e"
889 f
"\nGain: {ptcGain:.4}+/-{ptcGainError:.2e} e/ADU"
890 f
"\nNoise: {ptcNoise:.4}+/-{ptcNoiseError:.2e} e\n"
891 r"$\chi^2_{\rm{red}}$: " + f
"{ptcRedChi2:.4}"
892 f
"\nLast in fit: {meanVecFinal[-1]:.7} ADU ")
894 if ptcFitType ==
'POLYNOMIAL':
895 if len(meanVecFinal):
896 ptcGain, ptcGainError = 1./pars[1], np.fabs(1./pars[1])*(parsErr[1]/pars[1])
897 ptcNoiseAdu = np.sqrt((pars[0]))
898 ptcNoise = ptcNoiseAdu*ptcGain
899 ptcNoiseError = (0.5*(parsErr[0]/np.fabs(pars[0]))*(np.sqrt(np.fabs(pars[0]))))*ptcGain
900 stringLegend = (f
"Gain: {ptcGain:.4}+/-{ptcGainError:.2e} e/ADU\n"
901 f
"Noise: {ptcNoise:.4}+/-{ptcNoiseError:.2e} e\n"
902 r"$\chi^2_{\rm{red}}$: " + f
"{ptcRedChi2:.4}"
903 f
"\nLast in fit: {meanVecFinal[-1]:.7} ADU ")
905 a.set_xlabel(
r'Mean signal ($\mu$, ADU)', fontsize=labelFontSize)
906 a.set_ylabel(
r'Variance (ADU$^2$)', fontsize=labelFontSize)
907 a.tick_params(labelsize=11)
908 a.set_xscale(
'linear')
909 a.set_yscale(
'linear')
911 a2.set_xlabel(
r'Mean Signal ($\mu$, ADU)', fontsize=labelFontSize)
912 a2.set_ylabel(
r'Variance (ADU$^2$)', fontsize=labelFontSize)
913 a2.tick_params(labelsize=11)
917 a3.set_xlabel(
r'Mean signal ($\mu$, ADU)', fontsize=labelFontSize)
918 a3.set_ylabel(
r'Variance/$\mu$ (ADU)', fontsize=labelFontSize)
919 a3.tick_params(labelsize=11)
921 a3.set_yscale(
'linear')
923 minMeanVecFinal = np.nanmin(meanVecFinal)
924 maxMeanVecFinal = np.nanmax(meanVecFinal)
925 meanVecFit = np.linspace(minMeanVecFinal, maxMeanVecFinal, 100*len(meanVecFinal))
926 minMeanVecOriginal = np.nanmin(meanVecOriginal)
927 maxMeanVecOriginal = np.nanmax(meanVecOriginal)
928 deltaXlim = maxMeanVecOriginal - minMeanVecOriginal
929 a.plot(meanVecFit, ptcFunc(pars, meanVecFit), color=
'red')
930 a.plot(meanVecFinal, ptcNoiseAdu**2 + (1./ptcGain)*meanVecFinal, color=
'green',
932 a.scatter(meanVecFinal, varVecFinal, c=
'blue', marker=
'o', s=markerSize)
933 a.scatter(meanVecOutliers, varVecOutliers, c=
'magenta', marker=
's', s=markerSize)
934 a.text(0.03, 0.66, stringLegend, transform=a.transAxes, fontsize=legendFontSize)
935 a.set_title(amp, fontsize=titleFontSize)
936 a.set_xlim([minMeanVecOriginal - 0.2*deltaXlim, maxMeanVecOriginal + 0.2*deltaXlim])
939 a2.plot(meanVecFit, ptcFunc(pars, meanVecFit), color=
'red')
940 a2.scatter(meanVecFinal, varVecFinal, c=
'blue', marker=
'o', s=markerSize)
941 a2.scatter(meanVecOutliers, varVecOutliers, c=
'magenta', marker=
's', s=markerSize)
942 a2.text(0.03, 0.66, stringLegend, transform=a2.transAxes, fontsize=legendFontSize)
943 a2.set_title(amp, fontsize=titleFontSize)
944 a2.set_xlim([minMeanVecOriginal, maxMeanVecOriginal])
947 a3.plot(meanVecFit, ptcFunc(pars, meanVecFit)/meanVecFit, color=
'red')
948 a3.scatter(meanVecFinal, varVecFinal/meanVecFinal, c=
'blue', marker=
'o', s=markerSize)
949 a3.scatter(meanVecOutliers, varVecOutliers/meanVecOutliers, c=
'magenta', marker=
's',
951 a3.text(0.05, 0.1, stringLegend, transform=a3.transAxes, fontsize=legendFontSize)
952 a3.set_title(amp, fontsize=titleFontSize)
953 a3.set_xlim([minMeanVecOriginal - 0.2*deltaXlim, maxMeanVecOriginal + 0.2*deltaXlim])
955 f.suptitle(
"PTC \n Fit: " + stringTitle, fontsize=supTitleFontSize)
957 f2.suptitle(
"PTC (log-log)", fontsize=supTitleFontSize)
959 f3.suptitle(
r"Var/$\mu$", fontsize=supTitleFontSize)
964 def _plotLinearizer(self, dataset, linearizer, pdfPages):
965 """Plot linearity and linearity residual per amplifier
969 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset`
970 The dataset containing the means, variances, exposure times, and mask.
972 linearizer : `lsst.ip.isr.Linearizer`
978 supTitleFontSize = 18
981 nAmps = len(dataset.ampNames)
984 nRows = np.sqrt(nAmps)
985 mantissa, _ = np.modf(nRows)
987 nRows = int(nRows) + 1
994 f, ax = plt.subplots(nrows=nRows, ncols=nCols, sharex=
'col', sharey=
'row', figsize=(13, 10))
995 f2, ax2 = plt.subplots(nrows=nRows, ncols=nCols, sharex=
'col', sharey=
'row', figsize=(13, 10))
996 for i, (amp, a, a2)
in enumerate(zip(dataset.ampNames, ax.flatten(), ax2.flatten())):
997 mask = dataset.expIdMask[amp]
998 if np.isnan(mask[0]):
999 a.set_title(f
"{amp} (BAD)", fontsize=titleFontSize)
1000 a2.set_title(f
"{amp} (BAD)", fontsize=titleFontSize)
1003 mask = mask.astype(bool)
1004 meanVecFinal = np.array(dataset.rawMeans[amp])[mask]
1005 timeVecFinal = np.array(dataset.rawExpTimes[amp])[mask]
1007 a.set_xlabel(
'Time (sec)', fontsize=labelFontSize)
1008 a.set_ylabel(
r'Mean signal ($\mu$, ADU)', fontsize=labelFontSize)
1009 a.tick_params(labelsize=labelFontSize)
1010 a.set_xscale(
'linear')
1011 a.set_yscale(
'linear')
1013 a2.axhline(y=0, color=
'k')
1014 a2.axvline(x=0, color=
'k', linestyle=
'-')
1015 a2.set_xlabel(
r'Mean signal ($\mu$, ADU)', fontsize=labelFontSize)
1016 a2.set_ylabel(
'Fractional nonlinearity (%)', fontsize=labelFontSize)
1017 a2.tick_params(labelsize=labelFontSize)
1018 a2.set_xscale(
'linear')
1019 a2.set_yscale(
'linear')
1021 pars, parsErr = linearizer.fitParams[amp], linearizer.fitParamsErr[amp]
1022 k0, k0Error = pars[0], parsErr[0]
1023 k1, k1Error = pars[1], parsErr[1]
1024 k2, k2Error = pars[2], parsErr[2]
1025 linRedChi2 = linearizer.fitChiSq[amp]
1026 stringLegend = (f
"k0: {k0:.4}+/-{k0Error:.2e} ADU\nk1: {k1:.4}+/-{k1Error:.2e} ADU/t"
1027 f
"\nk2: {k2:.2e}+/-{k2Error:.2e} ADU/t^2\n"
1028 r"$\chi^2_{\rm{red}}$: " + f
"{linRedChi2:.4}")
1029 a.scatter(timeVecFinal, meanVecFinal)
1030 a.plot(timeVecFinal,
funcPolynomial(pars, timeVecFinal), color=
'red')
1031 a.text(0.03, 0.75, stringLegend, transform=a.transAxes, fontsize=legendFontSize)
1032 a.set_title(f
"{amp}", fontsize=titleFontSize)
1034 linearPart = k0 + k1*timeVecFinal
1035 fracLinRes = 100*(linearPart - meanVecFinal)/linearPart
1036 a2.plot(meanVecFinal, fracLinRes, c=
'g')
1037 a2.set_title(f
"{amp}", fontsize=titleFontSize)
1039 f.suptitle(
"Linearity \n Fit: Polynomial (degree: %g)"
1041 fontsize=supTitleFontSize)
1042 f2.suptitle(
r"Fractional NL residual" "\n"
1043 r"$100\times \frac{(k_0 + k_1*Time-\mu)}{k_0+k_1*Time}$",
1044 fontsize=supTitleFontSize)
1046 pdfPages.savefig(f2)
1050 """Group data into bins, with at most maxDiff distance between bins.
1058 Maximum distance between bins.
1067 index = np.zeros_like(x, dtype=np.int32)
1072 for i
in range(1, len(ix)):
1074 if (xval - xc < maxDiff):
1075 xc = (ng*xc + xval)/(ng+1)
1077 index[ix[i]] = group
1081 index[ix[i]] = group
1088 """Builds an index with regular binning. The result can be fed into binData.
1099 np.digitize(x, bins): `numpy.array`
1103 bins = np.linspace(x.min(), x.max() + abs(x.max() * 1e-7), nBins + 1)
1104 return np.digitize(x, bins)
1108 """Bin data (usually for display purposes).
1119 Bin number of each datum.
1122 Inverse rms of each datum to use when averaging (the actual weight is wy**2).
1133 wybin: `numpy.array`
1134 Binned weights in y, computed from wy's in each bin.
1136 sybin: `numpy.array`
1137 Uncertainty on the bin average, considering actual scatter, and ignoring weights.
1141 wy = np.ones_like(x)
1142 binIndexSet = set(binIndex)
1145 xbin = np.array([xw2[binIndex == i].sum()/w2[binIndex == i].sum()
for i
in binIndexSet])
1148 ybin = np.array([yw2[binIndex == i].sum()/w2[binIndex == i].sum()
for i
in binIndexSet])
1150 wybin = np.sqrt(np.array([w2[binIndex == i].sum()
for i
in binIndexSet]))
1151 sybin = np.array([y[binIndex == i].
std()/np.sqrt(np.array([binIndex == i]).sum())
1152 for i
in binIndexSet])
1154 return xbin, ybin, wybin, sybin
def _plotStandardPtc(self, dataset, ptcFitType, pdfPages)
def plotRelativeBiasACoeffs(aDict, aDictNoB, fullCovsModel, fullCovsModelNoB, signalElectrons, gainDict, pdfPages, maxr=None)
plotNormalizedCovariancesNumberOfBins
def __init__(self, datasetFilename, linearizerFileName=None, outDir='.', detNum=999, signalElectronsRelativeA=75000, plotNormalizedCovariancesNumberOfBins=10)
def indexForBins(x, nBins)
def plotCovariances(mu, covs, covsModel, covsWeights, covsNoB, covsModelNoB, covsWeightsNoB, gainDict, noiseDict, aDict, bDict, pdfPages)
def ab_vs_dist(aDict, bDict, pdfPages, bRange=4)
def plotAcoeffsSum(aDict, bDict, pdfPages)
def plotNormalizedCovariances(self, i, j, inputMu, covs, covsModel, covsWeights, covsNoB, covsModelNoB, covsWeightsNoB, pdfPages, offset=0.004, numberOfBins=10, plotData=True, topPlot=False, log=None)
def binData(x, y, binIndex, wy=None)
def _plotLinearizer(self, dataset, linearizer, pdfPages)
def findGroups(x, maxDiff)
def run(self, filenameFull, datasetPtc, linearizer=None, log=None)
def plot_a_b(aDict, bDict, pdfPages, bRange=3)
def covAstierMakeAllPlots(self, dataset, pdfPages, log=None)
def computeApproximateAcoeffs(covModel, muEl, gain)
def getFitDataFromCovariances(i, j, mu, fullCov, fullCovModel, fullCovSqrtWeights, gain=1.0, divideByMu=False, returnMasked=False)
def calculateWeightedReducedChi2(measured, model, weightsMeasured, nData, nParsModel)
def funcPolynomial(pars, x)