23 __all__ = [
'PlotPhotonTransferCurveTask']
26 import matplotlib.pyplot
as plt
27 import matplotlib
as mpl
28 from matplotlib
import gridspec
30 from matplotlib.backends.backend_pdf
import PdfPages
36 from .utils
import (funcAstier, funcPolynomial, NonexistentDatasetTaskDataIdContainer,
37 calculateWeightedReducedChi2)
38 from matplotlib.ticker
import MaxNLocator
40 from .astierCovPtcFit
import computeApproximateAcoeffs
41 from .astierCovPtcUtils
import getFitDataFromCovariances
47 """Config class for photon transfer curve measurement task"""
48 datasetFileName = pexConfig.Field(
50 doc=
"datasetPtc file name (pkl)",
53 linearizerFileName = pexConfig.Field(
55 doc=
"linearizer file name (fits)",
58 ccdKey = pexConfig.Field(
60 doc=
"The key by which to pull a detector from a dataId, e.g. 'ccd' or 'detector'.",
63 signalElectronsRelativeA = pexConfig.Field(
65 doc=
"Signal value for relative systematic bias between different methods of estimating a_ij "
66 "(Fig. 15 of Astier+19).",
69 plotNormalizedCovariancesNumberOfBins = pexConfig.Field(
71 doc=
"Number of bins in `plotNormalizedCovariancesNumber` function "
72 "(Fig. 8, 10., of Astier+19).",
78 """A class to plot the dataset from MeasurePhotonTransferCurveTask.
84 Positional arguments passed to the Task constructor. None used at this
87 Keyword arguments passed on to the Task constructor. None used at this
92 ConfigClass = PlotPhotonTransferCurveTaskConfig
93 _DefaultName =
"plotPhotonTransferCurve"
96 pipeBase.CmdLineTask.__init__(self, *args, **kwargs)
97 plt.interactive(
False)
98 self.config.validate()
102 def _makeArgumentParser(cls):
103 """Augment argument parser for the MeasurePhotonTransferCurveTask."""
105 parser.add_id_argument(
"--id", datasetType=
"photonTransferCurveDataset",
106 ContainerClass=NonexistentDatasetTaskDataIdContainer,
107 help=
"The ccds to use, e.g. --id ccd=0..100")
112 """Run the Photon Transfer Curve (PTC) plotting measurement task.
116 dataRef : list of lsst.daf.persistence.ButlerDataRef
117 dataRef for the detector for the expIds to be fit.
120 datasetFile = self.config.datasetFileName
121 datasetPtc = PhotonTransferCurveDataset.readFits(datasetFile)
123 dirname = dataRef.getUri(datasetType=
'cpPipePlotRoot', write=
True)
124 if not os.path.exists(dirname):
127 detNum = dataRef.dataId[self.config.ccdKey]
128 filename = f
"PTC_det{detNum}.pdf"
129 filenameFull = os.path.join(dirname, filename)
131 if self.config.linearizerFileName:
132 linearizer = isr.linearize.Linearizer.readFits(self.config.linearizerFileName)
135 self.
run(filenameFull, datasetPtc, linearizer=linearizer, log=self.log)
137 return pipeBase.Struct(exitStatus=0)
139 def run(self, filenameFull, datasetPtc, linearizer=None, log=None):
140 """Make the plots for the PTC task"""
141 ptcFitType = datasetPtc.ptcFitType
142 with PdfPages(filenameFull)
as pdfPages:
143 if ptcFitType
in [
"FULLCOVARIANCE", ]:
145 elif ptcFitType
in [
"EXPAPPROXIMATION",
"POLYNOMIAL"]:
148 raise RuntimeError(f
"The input dataset had an invalid dataset.ptcFitType: {ptcFitType}. \n" +
149 "Options: 'FULLCOVARIANCE', EXPAPPROXIMATION, or 'POLYNOMIAL'.")
157 """Make plots for MeasurePhotonTransferCurve task when doCovariancesAstier=True.
159 This function call other functions that mostly reproduce the plots in Astier+19.
160 Most of the code is ported from Pierre Astier's repository https://github.com/PierreAstier/bfptc
164 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset`
165 The dataset containing the necessary information to produce the plots.
167 pdfPages: `matplotlib.backends.backend_pdf.PdfPages`
168 PDF file where the plots will be saved.
170 log : `lsst.log.Log`, optional
171 Logger to handle messages
173 mu = dataset.rawMeans
175 fullCovs = dataset.covariances
176 fullCovsModel = dataset.covariancesModel
177 fullCovWeights = dataset.covariancesSqrtWeights
178 aDict = dataset.aMatrix
179 bDict = dataset.bMatrix
180 fullCovsNoB = dataset.covariancesNoB
181 fullCovsModelNoB = dataset.covariancesModelNoB
182 fullCovWeightsNoB = dataset.covariancesSqrtWeightsNoB
183 aDictNoB = dataset.aMatrixNoB
184 gainDict = dataset.gain
185 noiseDict = dataset.noise
187 self.
plotCovariances(mu, fullCovs, fullCovsModel, fullCovWeights, fullCovsNoB, fullCovsModelNoB,
188 fullCovWeightsNoB, gainDict, noiseDict, aDict, bDict, pdfPages)
190 fullCovsModelNoB, fullCovWeightsNoB, pdfPages, offset=0.01,
192 numberOfBins=self.config.plotNormalizedCovariancesNumberOfBins,
195 fullCovsModelNoB, fullCovWeightsNoB, pdfPages,
196 numberOfBins=self.config.plotNormalizedCovariancesNumberOfBins,
199 fullCovsModelNoB, fullCovWeightsNoB, pdfPages,
200 numberOfBins=self.config.plotNormalizedCovariancesNumberOfBins,
202 self.
plot_a_b(aDict, bDict, pdfPages)
203 self.
ab_vs_dist(aDict, bDict, pdfPages, bRange=4)
206 self.config.signalElectronsRelativeA, gainDict, pdfPages, maxr=4)
211 def plotCovariances(mu, covs, covsModel, covsWeights, covsNoB, covsModelNoB, covsWeightsNoB,
212 gainDict, noiseDict, aDict, bDict, pdfPages):
213 """Plot covariances and models: Cov00, Cov10, Cov01.
215 Figs. 6 and 7 of Astier+19
219 mu : `dict`, [`str`, `list`]
220 Dictionary keyed by amp name with mean signal values.
222 covs : `dict`, [`str`, `list`]
223 Dictionary keyed by amp names containing a list of measued covariances per mean flux.
225 covsModel : `dict`, [`str`, `list`]
226 Dictionary keyed by amp names containinging covariances model (Eq. 20 of Astier+19) per mean flux.
228 covsWeights : `dict`, [`str`, `list`]
229 Dictionary keyed by amp names containinging sqrt. of covariances weights.
231 covsNoB : `dict`, [`str`, `list`]
232 Dictionary keyed by amp names containing a list of measued covariances per mean flux ('b'=0 in
235 covsModelNoB : `dict`, [`str`, `list`]
236 Dictionary keyed by amp names containing covariances model (with 'b'=0 in Eq. 20 of Astier+19)
239 covsWeightsNoB : `dict`, [`str`, `list`]
240 Dictionary keyed by amp names containing sqrt. of covariances weights ('b' = 0 in Eq. 20 of
243 gainDict : `dict`, [`str`, `float`]
244 Dictionary keyed by amp names containing the gains in e-/ADU.
246 noiseDict : `dict`, [`str`, `float`]
247 Dictionary keyed by amp names containing the rms redout noise in e-.
249 aDict : `dict`, [`str`, `numpy.array`]
250 Dictionary keyed by amp names containing 'a' coefficients (Eq. 20 of Astier+19).
252 bDict : `dict`, [`str`, `numpy.array`]
253 Dictionary keyed by amp names containing 'b' coefficients (Eq. 20 of Astier+19).
255 pdfPages: `matplotlib.backends.backend_pdf.PdfPages`
256 PDF file where the plots will be saved.
262 supTitleFontSize = 18
268 nRows = np.sqrt(nAmps)
269 mantissa, _ = np.modf(nRows)
271 nRows = int(nRows) + 1
277 f, ax = plt.subplots(nrows=nRows, ncols=nCols, sharex=
'col', sharey=
'row', figsize=(13, 10))
278 f2, ax2 = plt.subplots(nrows=nRows, ncols=nCols, sharex=
'col', sharey=
'row', figsize=(13, 10))
279 fResCov00, axResCov00 = plt.subplots(nrows=nRows, ncols=nCols, sharex=
'col', sharey=
'row',
281 fCov01, axCov01 = plt.subplots(nrows=nRows, ncols=nCols, sharex=
'col', sharey=
'row', figsize=(13, 10))
282 fCov10, axCov10 = plt.subplots(nrows=nRows, ncols=nCols, sharex=
'col', sharey=
'row', figsize=(13, 10))
284 assert(len(covsModel) == nAmps)
285 assert(len(covsWeights) == nAmps)
287 assert(len(covsNoB) == nAmps)
288 assert(len(covsModelNoB) == nAmps)
289 assert(len(covsWeightsNoB) == nAmps)
291 for i, (amp, a, a2, aResVar, a3, a4)
in enumerate(zip(covs, ax.flatten(),
292 ax2.flatten(), axResCov00.flatten(),
293 axCov01.flatten(), axCov10.flatten())):
295 muAmp, cov, model, weight = mu[amp], covs[amp], covsModel[amp], covsWeights[amp]
296 if not np.isnan(np.array(cov)).all():
297 aCoeffs, bCoeffs = np.array(aDict[amp]), np.array(bDict[amp])
298 gain, noise = gainDict[amp], noiseDict[amp]
299 (meanVecOriginal, varVecOriginal, varVecModelOriginal,
301 meanVecFinal, varVecFinal = meanVecOriginal[varMask], varVecOriginal[varMask]
302 varVecModelFinal = varVecModelOriginal[varMask]
303 meanVecOutliers = meanVecOriginal[np.invert(varMask)]
304 varVecOutliers = varVecOriginal[np.invert(varMask)]
305 varWeightsFinal = weightsOriginal[varMask]
308 varWeightsFinal, len(meanVecFinal), 4)
310 (meanVecOrigCov01, varVecOrigCov01, varVecModelOrigCov01,
312 meanVecFinalCov01, varVecFinalCov01 = meanVecOrigCov01[maskCov01], varVecOrigCov01[maskCov01]
313 varVecModelFinalCov01 = varVecModelOrigCov01[maskCov01]
314 meanVecOutliersCov01 = meanVecOrigCov01[np.invert(maskCov01)]
315 varVecOutliersCov01 = varVecOrigCov01[np.invert(maskCov01)]
317 (meanVecOrigCov10, varVecOrigCov10, varVecModelOrigCov10,
319 meanVecFinalCov10, varVecFinalCov10 = meanVecOrigCov10[maskCov10], varVecOrigCov10[maskCov10]
320 varVecModelFinalCov10 = varVecModelOrigCov10[maskCov10]
321 meanVecOutliersCov10 = meanVecOrigCov10[np.invert(maskCov10)]
322 varVecOutliersCov10 = varVecOrigCov10[np.invert(maskCov10)]
325 par2 = np.polyfit(meanVecFinal, varVecFinal, 2, w=varWeightsFinal)
326 varModelFinalQuadratic = np.polyval(par2, meanVecFinal)
328 varWeightsFinal, len(meanVecFinal), 3)
331 covNoB, modelNoB, weightNoB = covsNoB[amp], covsModelNoB[amp], covsWeightsNoB[amp]
332 (meanVecFinalNoB, varVecFinalNoB, varVecModelFinalNoB,
334 weightNoB, returnMasked=
True)
336 varWeightsFinalNoB, len(meanVecFinalNoB),
338 stringLegend = (f
"Gain: {gain:.4} e/DN \n" +
339 f
"Noise: {noise:.4} e \n" +
340 r"$a_{00}$: %.3e 1/e"%aCoeffs[0, 0] +
341 "\n" +
r"$b_{00}$: %.3e 1/e"%bCoeffs[0, 0])
342 minMeanVecFinal = np.min(meanVecFinal)
343 maxMeanVecFinal = np.max(meanVecFinal)
344 deltaXlim = maxMeanVecFinal - minMeanVecFinal
346 a.set_xlabel(
r'Mean signal ($\mu$, DN)', fontsize=labelFontSize)
347 a.set_ylabel(
r'Variance (DN$^2$)', fontsize=labelFontSize)
348 a.tick_params(labelsize=11)
349 a.set_xscale(
'linear', fontsize=labelFontSize)
350 a.set_yscale(
'linear', fontsize=labelFontSize)
351 a.scatter(meanVecFinal, varVecFinal, c=
'blue', marker=
'o', s=markerSize)
352 a.scatter(meanVecOutliers, varVecOutliers, c=
'magenta', marker=
's', s=markerSize)
353 a.plot(meanVecFinal, varVecModelFinal, color=
'red', lineStyle=
'-')
354 a.text(0.03, 0.7, stringLegend, transform=a.transAxes, fontsize=legendFontSize)
355 a.set_title(amp, fontsize=titleFontSize)
356 a.set_xlim([minMeanVecFinal - 0.2*deltaXlim, maxMeanVecFinal + 0.2*deltaXlim])
359 a2.set_xlabel(
r'Mean Signal ($\mu$, DN)', fontsize=labelFontSize)
360 a2.set_ylabel(
r'Variance (DN$^2$)', fontsize=labelFontSize)
361 a2.tick_params(labelsize=11)
364 a2.plot(meanVecFinal, varVecModelFinal, color=
'red', lineStyle=
'-')
365 a2.scatter(meanVecFinal, varVecFinal, c=
'blue', marker=
'o', s=markerSize)
366 a2.scatter(meanVecOutliers, varVecOutliers, c=
'magenta', marker=
's', s=markerSize)
367 a2.text(0.03, 0.7, stringLegend, transform=a2.transAxes, fontsize=legendFontSize)
368 a2.set_title(amp, fontsize=titleFontSize)
369 a2.set_xlim([minMeanVecFinal, maxMeanVecFinal])
372 aResVar.set_xlabel(
r'Mean signal ($\mu$, DN)', fontsize=labelFontSize)
373 aResVar.set_ylabel(
r'Residuals (DN$^2$)', fontsize=labelFontSize)
374 aResVar.tick_params(labelsize=11)
375 aResVar.set_xscale(
'linear', fontsize=labelFontSize)
376 aResVar.set_yscale(
'linear', fontsize=labelFontSize)
377 aResVar.plot(meanVecFinal, varVecFinal - varVecModelFinal, color=
'blue', lineStyle=
'-',
378 label=
r'Full fit ($\chi_{\rm{red}}^2$: %g)'%chi2FullModelVar)
379 aResVar.plot(meanVecFinal, varVecFinal - varModelFinalQuadratic, color=
'red', lineStyle=
'-',
380 label=
r'Quadratic fit ($\chi_{\rm{red}}^2$: %g)'%chi2QuadModelVar)
381 aResVar.plot(meanVecFinalNoB, varVecFinalNoB - varVecModelFinalNoB, color=
'green',
383 label=
r'Full fit (b=0) ($\chi_{\rm{red}}^2$: %g)'%chi2FullModelNoBVar)
384 aResVar.axhline(color=
'black')
385 aResVar.set_title(amp, fontsize=titleFontSize)
386 aResVar.set_xlim([minMeanVecFinal - 0.2*deltaXlim, maxMeanVecFinal + 0.2*deltaXlim])
387 aResVar.legend(fontsize=7)
389 a3.set_xlabel(
r'Mean signal ($\mu$, DN)', fontsize=labelFontSize)
390 a3.set_ylabel(
r'Cov01 (DN$^2$)', fontsize=labelFontSize)
391 a3.tick_params(labelsize=11)
392 a3.set_xscale(
'linear', fontsize=labelFontSize)
393 a3.set_yscale(
'linear', fontsize=labelFontSize)
394 a3.scatter(meanVecFinalCov01, varVecFinalCov01, c=
'blue', marker=
'o', s=markerSize)
395 a3.scatter(meanVecOutliersCov01, varVecOutliersCov01, c=
'magenta', marker=
's', s=markerSize)
396 a3.plot(meanVecFinalCov01, varVecModelFinalCov01, color=
'red', lineStyle=
'-')
397 a3.set_title(amp, fontsize=titleFontSize)
398 a3.set_xlim([minMeanVecFinal - 0.2*deltaXlim, maxMeanVecFinal + 0.2*deltaXlim])
400 a4.set_xlabel(
r'Mean signal ($\mu$, DN)', fontsize=labelFontSize)
401 a4.set_ylabel(
r'Cov10 (DN$^2$)', fontsize=labelFontSize)
402 a4.tick_params(labelsize=11)
403 a4.set_xscale(
'linear', fontsize=labelFontSize)
404 a4.set_yscale(
'linear', fontsize=labelFontSize)
405 a4.scatter(meanVecFinalCov10, varVecFinalCov10, c=
'blue', marker=
'o', s=markerSize)
406 a4.scatter(meanVecOutliersCov10, varVecOutliersCov10, c=
'magenta', marker=
's', s=markerSize)
407 a4.plot(meanVecFinalCov10, varVecModelFinalCov10, color=
'red', lineStyle=
'-')
408 a4.set_title(amp, fontsize=titleFontSize)
409 a4.set_xlim([minMeanVecFinal - 0.2*deltaXlim, maxMeanVecFinal + 0.2*deltaXlim])
412 a.set_title(f
"{amp} (BAD)", fontsize=titleFontSize)
413 a2.set_title(f
"{amp} (BAD)", fontsize=titleFontSize)
414 a3.set_title(f
"{amp} (BAD)", fontsize=titleFontSize)
415 a4.set_title(f
"{amp} (BAD)", fontsize=titleFontSize)
417 f.suptitle(
"PTC from covariances as in Astier+19 \n Fit: Eq. 20, Astier+19",
418 fontsize=supTitleFontSize)
420 f2.suptitle(
"PTC from covariances as in Astier+19 (log-log) \n Fit: Eq. 20, Astier+19",
421 fontsize=supTitleFontSize)
423 fResCov00.suptitle(
"Residuals (data-model) for Cov00 (Var)", fontsize=supTitleFontSize)
424 pdfPages.savefig(fResCov00)
425 fCov01.suptitle(
"Cov01 as in Astier+19 (nearest parallel neighbor covariance) \n" +
426 " Fit: Eq. 20, Astier+19", fontsize=supTitleFontSize)
427 pdfPages.savefig(fCov01)
428 fCov10.suptitle(
"Cov10 as in Astier+19 (nearest serial neighbor covariance) \n" +
429 "Fit: Eq. 20, Astier+19", fontsize=supTitleFontSize)
430 pdfPages.savefig(fCov10)
435 covsWeightsNoB, pdfPages, offset=0.004,
436 numberOfBins=10, plotData=True, topPlot=False, log=None):
437 """Plot C_ij/mu vs mu.
439 Figs. 8, 10, and 11 of Astier+19
449 inputMu : `dict`, [`str`, `list`]
450 Dictionary keyed by amp name with mean signal values.
452 covs : `dict`, [`str`, `list`]
453 Dictionary keyed by amp names containing a list of measued covariances per mean flux.
455 covsModel : `dict`, [`str`, `list`]
456 Dictionary keyed by amp names containinging covariances model (Eq. 20 of Astier+19) per mean flux.
458 covsWeights : `dict`, [`str`, `list`]
459 Dictionary keyed by amp names containinging sqrt. of covariances weights.
461 covsNoB : `dict`, [`str`, `list`]
462 Dictionary keyed by amp names containing a list of measued covariances per mean flux ('b'=0 in
465 covsModelNoB : `dict`, [`str`, `list`]
466 Dictionary keyed by amp names containing covariances model (with 'b'=0 in Eq. 20 of Astier+19)
469 covsWeightsNoB : `dict`, [`str`, `list`]
470 Dictionary keyed by amp names containing sqrt. of covariances weights ('b' = 0 in Eq. 20 of
473 pdfPages: `matplotlib.backends.backend_pdf.PdfPages`
474 PDF file where the plots will be saved.
476 offset : `float`, optional
477 Constant offset factor to plot covariances in same panel (so they don't overlap).
479 numberOfBins : `int`, optional
480 Number of bins for top and bottom plot.
482 plotData : `bool`, optional
483 Plot the data points?
485 topPlot : `bool`, optional
486 Plot the top plot with the covariances, and the bottom plot with the model residuals?
488 log : `lsst.log.Log`, optional
489 Logger to handle messages.
492 fig = plt.figure(figsize=(8, 10))
493 gs = gridspec.GridSpec(2, 1, height_ratios=[3, 1])
495 ax0 = plt.subplot(gs[0])
496 plt.setp(ax0.get_xticklabels(), visible=
False)
498 fig = plt.figure(figsize=(8, 8))
499 ax0 = plt.subplot(111)
500 ax0.ticklabel_format(style=
'sci', axis=
'x', scilimits=(0, 0))
501 ax0.tick_params(axis=
'both', labelsize=
'x-large')
502 mue, rese, wce = [], [], []
503 mueNoB, reseNoB, wceNoB = [], [], []
504 for counter, amp
in enumerate(covs):
505 muAmp, fullCov, fullCovModel, fullCovWeight = (inputMu[amp], covs[amp], covsModel[amp],
507 if len(fullCov) == 0:
510 fullCovWeight, divideByMu=
True,
513 rese += list(cov - model)
514 wce += list(weightCov)
516 fullCovNoB, fullCovModelNoB, fullCovWeightNoB = (covsNoB[amp], covsModelNoB[amp],
518 if len(fullCovNoB) == 0:
520 (muNoB, covNoB, modelNoB,
522 fullCovWeightNoB, divideByMu=
True,
524 mueNoB += list(muNoB)
525 reseNoB += list(covNoB - modelNoB)
526 wceNoB += list(weightCovNoB)
529 fit_curve, = plt.plot(mu, model + counter*offset,
'-', linewidth=4.0)
533 xb, yb, wyb, sigyb = self.
binData(mu, cov, gind, weightCov)
534 plt.errorbar(xb, yb+counter*offset, yerr=sigyb, marker=
'o', linestyle=
'none', markersize=6.5,
535 color=fit_curve.get_color(), label=f
"{amp} (N: {len(mu)})")
538 points, = plt.plot(mu, cov + counter*offset,
'.', color=fit_curve.get_color())
539 plt.legend(loc=
'upper right', fontsize=8)
542 rese = np.array(rese)
544 mueNoB = np.array(mueNoB)
545 reseNoB = np.array(reseNoB)
546 wceNoB = np.array(wceNoB)
548 plt.xlabel(
r"$\mu (el)$", fontsize=
'x-large')
549 plt.ylabel(
r"$Cov{%d%d}/\mu + Cst (el)$"%(i, j), fontsize=
'x-large')
552 xb, yb, wyb, sigyb = self.
binData(mue, rese, gind, wce)
554 ax1 = plt.subplot(gs[1], sharex=ax0)
555 ax1.errorbar(xb, yb, yerr=sigyb, marker=
'o', linestyle=
'none', label=
'Full fit')
557 xb2, yb2, wyb2, sigyb2 = self.
binData(mueNoB, reseNoB, gindNoB, wceNoB)
559 ax1.errorbar(xb2, yb2, yerr=sigyb2, marker=
'o', linestyle=
'none', label=
'b = 0')
560 ax1.tick_params(axis=
'both', labelsize=
'x-large')
561 plt.legend(loc=
'upper left', fontsize=
'large')
563 plt.plot(xb, [0]*len(xb),
'--', color=
'k')
564 plt.ticklabel_format(style=
'sci', axis=
'x', scilimits=(0, 0))
565 plt.ticklabel_format(style=
'sci', axis=
'y', scilimits=(0, 0))
566 plt.xlabel(
r'$\mu (el)$', fontsize=
'x-large')
567 plt.ylabel(
r'$Cov{%d%d}/\mu$ -model (el)'%(i, j), fontsize=
'x-large')
569 plt.suptitle(f
"Nbins: {numberOfBins}")
572 labels0 = [item.get_text()
for item
in ax0.get_yticklabels()]
574 ax0.set_yticklabels(labels0)
575 pdfPages.savefig(fig)
581 """Fig. 12 of Astier+19
583 Color display of a and b arrays fits, averaged over channels.
587 aDict : `dict`, [`numpy.array`]
588 Dictionary keyed by amp names containing the fitted 'a' coefficients from the model
589 in Eq. 20 of Astier+19 (if `ptcFitType` is `FULLCOVARIANCE`).
591 bDict : `dict`, [`numpy.array`]
592 Dictionary keyed by amp names containing the fitted 'b' coefficients from the model
593 in Eq. 20 of Astier+19 (if `ptcFitType` is `FULLCOVARIANCE`).
595 pdfPages: `matplotlib.backends.backend_pdf.PdfPages`
596 PDF file where the plots will be saved.
599 Maximum lag for b arrays.
603 if np.isnan(aDict[amp]).all():
607 a = np.array(a).mean(axis=0)
608 b = np.array(b).mean(axis=0)
609 fig = plt.figure(figsize=(7, 11))
610 ax0 = fig.add_subplot(2, 1, 1)
611 im0 = ax0.imshow(np.abs(a.transpose()), origin=
'lower', norm=mpl.colors.LogNorm())
612 ax0.tick_params(axis=
'both', labelsize=
'x-large')
613 ax0.set_title(
r'$|a|$', fontsize=
'x-large')
614 ax0.xaxis.set_ticks_position(
'bottom')
615 cb0 = plt.colorbar(im0)
616 cb0.ax.tick_params(labelsize=
'x-large')
618 ax1 = fig.add_subplot(2, 1, 2)
619 ax1.tick_params(axis=
'both', labelsize=
'x-large')
620 ax1.yaxis.set_major_locator(MaxNLocator(integer=
True))
621 ax1.xaxis.set_major_locator(MaxNLocator(integer=
True))
622 im1 = ax1.imshow(1e6*b[:bRange, :bRange].transpose(), origin=
'lower')
623 cb1 = plt.colorbar(im1)
624 cb1.ax.tick_params(labelsize=
'x-large')
625 ax1.set_title(
r'$b \times 10^6$', fontsize=
'x-large')
626 ax1.xaxis.set_ticks_position(
'bottom')
628 pdfPages.savefig(fig)
634 """Fig. 13 of Astier+19.
636 Values of a and b arrays fits, averaged over amplifiers, as a function of distance.
640 aDict : `dict`, [`numpy.array`]
641 Dictionary keyed by amp names containing the fitted 'a' coefficients from the model
642 in Eq. 20 of Astier+19 (if `ptcFitType` is `FULLCOVARIANCE`).
644 bDict : `dict`, [`numpy.array`]
645 Dictionary keyed by amp names containing the fitted 'b' coefficients from the model
646 in Eq. 20 of Astier+19 (if `ptcFitType` is `FULLCOVARIANCE`).
648 pdfPages: `matplotlib.backends.backend_pdf.PdfPages`
649 PDF file where the plots will be saved.
652 Maximum lag for b arrays.
654 assert (len(aDict) == len(bDict))
657 if np.isnan(aDict[amp]).all():
662 sy = a.std(axis=0)/np.sqrt(len(aDict))
663 i, j = np.indices(y.shape)
664 upper = (i >= j).ravel()
665 r = np.sqrt(i**2 + j**2).ravel()
668 fig = plt.figure(figsize=(6, 9))
669 ax = fig.add_subplot(211)
670 ax.set_xlim([0.5, r.max()+1])
671 ax.errorbar(r[upper], y[upper], yerr=sy[upper], marker=
'o', linestyle=
'none', color=
'b',
673 ax.errorbar(r[~upper], y[~upper], yerr=sy[~upper], marker=
'o', linestyle=
'none', color=
'r',
675 ax.legend(loc=
'upper center', fontsize=
'x-large')
676 ax.set_xlabel(
r'$\sqrt{i^2+j^2}$', fontsize=
'x-large')
677 ax.set_ylabel(
r'$a_{ij}$', fontsize=
'x-large')
679 ax.tick_params(axis=
'both', labelsize=
'x-large')
682 axb = fig.add_subplot(212)
685 if np.isnan(bDict[amp]).all():
690 syb = b.std(axis=0)/np.sqrt(len(bDict))
691 ib, jb = np.indices(yb.shape)
692 upper = (ib > jb).ravel()
693 rb = np.sqrt(i**2 + j**2).ravel()
698 axb.set_xlim([xmin, xmax+0.2])
699 cutu = (r > xmin) & (r < xmax) & (upper)
700 cutl = (r > xmin) & (r < xmax) & (~upper)
701 axb.errorbar(rb[cutu], yb[cutu], yerr=syb[cutu], marker=
'o', linestyle=
'none', color=
'b',
703 axb.errorbar(rb[cutl], yb[cutl], yerr=syb[cutl], marker=
'o', linestyle=
'none', color=
'r',
705 plt.legend(loc=
'upper center', fontsize=
'x-large')
706 axb.set_xlabel(
r'$\sqrt{i^2+j^2}$', fontsize=
'x-large')
707 axb.set_ylabel(
r'$b_{ij}$', fontsize=
'x-large')
708 axb.ticklabel_format(style=
'sci', axis=
'y', scilimits=(0, 0))
709 axb.tick_params(axis=
'both', labelsize=
'x-large')
711 pdfPages.savefig(fig)
717 """Fig. 14. of Astier+19
719 Cumulative sum of a_ij as a function of maximum separation. This plot displays the average over
724 aDict : `dict`, [`numpy.array`]
725 Dictionary keyed by amp names containing the fitted 'a' coefficients from the model
726 in Eq. 20 of Astier+19 (if `ptcFitType` is `FULLCOVARIANCE`).
728 bDict : `dict`, [`numpy.array`]
729 Dictionary keyed by amp names containing the fitted 'b' coefficients from the model
730 in Eq. 20 of Astier+19 (if `ptcFitType` is `FULLCOVARIANCE`).
732 pdfPages: `matplotlib.backends.backend_pdf.PdfPages`
733 PDF file where the plots will be saved.
735 assert (len(aDict) == len(bDict))
738 if np.isnan(aDict[amp]).all()
or np.isnan(bDict[amp]).all():
742 a = np.array(a).mean(axis=0)
743 b = np.array(b).mean(axis=0)
744 fig = plt.figure(figsize=(7, 6))
745 w = 4*np.ones_like(a)
750 indices = range(1, a.shape[0]+1)
751 sums = [wa[0:n, 0:n].sum()
for n
in indices]
752 ax = plt.subplot(111)
753 ax.plot(indices, sums/sums[0],
'o', color=
'b')
755 ax.set_xlim(indices[0]-0.5, indices[-1]+0.5)
756 ax.set_ylim(
None, 1.2)
757 ax.set_ylabel(
r'$[\sum_{|i|<n\ &\ |j|<n} a_{ij}] / |a_{00}|$', fontsize=
'x-large')
758 ax.set_xlabel(
'n', fontsize=
'x-large')
759 ax.tick_params(axis=
'both', labelsize=
'x-large')
761 pdfPages.savefig(fig)
767 gainDict, pdfPages, maxr=None):
768 """Fig. 15 in Astier+19.
770 Illustrates systematic bias from estimating 'a'
771 coefficients from the slope of correlations as opposed to the
772 full model in Astier+19.
777 Dictionary of 'a' matrices (Eq. 20, Astier+19), with amp names as keys.
780 Dictionary of 'a' matrices ('b'= 0 in Eq. 20, Astier+19), with amp names as keys.
782 fullCovsModel : `dict`, [`str`, `list`]
783 Dictionary keyed by amp names containing covariances model per mean flux.
785 fullCovsModelNoB : `dict`, [`str`, `list`]
786 Dictionary keyed by amp names containing covariances model (with 'b'=0 in Eq. 20 of
787 Astier+19) per mean flux.
789 signalElectrons : `float`
790 Signal at which to evaluate the a_ij coefficients.
792 pdfPages: `matplotlib.backends.backend_pdf.PdfPages`
793 PDF file where the plots will be saved.
795 gainDict : `dict`, [`str`, `float`]
796 Dicgionary keyed by amp names with the gains in e-/ADU.
798 maxr : `int`, optional
802 fig = plt.figure(figsize=(7, 11))
803 title = [f
"'a' relative bias at {signalElectrons} e",
"'a' relative bias (b=0)"]
804 data = [(aDict, fullCovsModel), (aDictNoB, fullCovsModelNoB)]
806 for k, pair
in enumerate(data):
810 covModel = pair[1][amp]
811 if np.isnan(covModel).all():
816 diffs.append((aOld-a))
817 amean = np.array(amean).mean(axis=0)
818 diff = np.array(diffs).mean(axis=0)
825 diff = diff[:maxr, :maxr]
826 ax0 = fig.add_subplot(2, 1, k+1)
827 im0 = ax0.imshow(diff.transpose(), origin=
'lower')
828 ax0.yaxis.set_major_locator(MaxNLocator(integer=
True))
829 ax0.xaxis.set_major_locator(MaxNLocator(integer=
True))
830 ax0.tick_params(axis=
'both', labelsize=
'x-large')
832 ax0.set_title(title[k])
835 pdfPages.savefig(fig)
839 def _plotStandardPtc(self, dataset, ptcFitType, pdfPages):
840 """Plot PTC, var/signal vs signal, linearity, and linearity residual per amplifier.
844 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset`
845 The dataset containing the means, variances, exposure times, and mask.
848 Type of the model fit to the PTC. Options: 'FULLCOVARIANCE', EXPAPPROXIMATION, or 'POLYNOMIAL'.
850 pdfPages: `matplotlib.backends.backend_pdf.PdfPages`
851 PDF file where the plots will be saved.
854 if ptcFitType ==
'EXPAPPROXIMATION':
856 stringTitle = (
r"Var = $\frac{1}{2g^2a_{00}}(\exp (2a_{00} \mu g) - 1) + \frac{n_{00}}{g^2}$ ")
857 elif ptcFitType ==
'POLYNOMIAL':
858 ptcFunc = funcPolynomial
859 for key
in dataset.ptcFitPars:
860 deg = len(dataset.ptcFitPars[key]) - 1
862 stringTitle =
r"Polynomial (degree: %g)" % (deg)
864 raise RuntimeError(f
"The input dataset had an invalid dataset.ptcFitType: {ptcFitType}. \n" +
865 "Options: 'FULLCOVARIANCE', EXPAPPROXIMATION, or 'POLYNOMIAL'.")
870 supTitleFontSize = 18
874 nAmps = len(dataset.ampNames)
877 nRows = np.sqrt(nAmps)
878 mantissa, _ = np.modf(nRows)
880 nRows = int(nRows) + 1
886 f, ax = plt.subplots(nrows=nRows, ncols=nCols, sharex=
'col', sharey=
'row', figsize=(13, 10))
887 f2, ax2 = plt.subplots(nrows=nRows, ncols=nCols, sharex=
'col', sharey=
'row', figsize=(13, 10))
888 f3, ax3 = plt.subplots(nrows=nRows, ncols=nCols, sharex=
'col', sharey=
'row', figsize=(13, 10))
890 for i, (amp, a, a2, a3)
in enumerate(zip(dataset.ampNames, ax.flatten(), ax2.flatten(),
892 meanVecOriginal = np.array(dataset.rawMeans[amp])
893 varVecOriginal = np.array(dataset.rawVars[amp])
894 mask = np.array(dataset.expIdMask[amp])
895 if np.isnan(mask[0]):
896 a.set_title(f
"{amp} (BAD)", fontsize=titleFontSize)
897 a2.set_title(f
"{amp} (BAD)", fontsize=titleFontSize)
898 a3.set_title(f
"{amp} (BAD)", fontsize=titleFontSize)
901 mask = mask.astype(bool)
902 meanVecFinal = meanVecOriginal[mask]
903 varVecFinal = varVecOriginal[mask]
904 meanVecOutliers = meanVecOriginal[np.invert(mask)]
905 varVecOutliers = varVecOriginal[np.invert(mask)]
906 pars, parsErr = np.array(dataset.ptcFitPars[amp]), np.array(dataset.ptcFitParsError[amp])
907 ptcRedChi2 = np.array(dataset.ptcFitChiSq[amp])
908 if ptcFitType ==
'EXPAPPROXIMATION':
909 if len(meanVecFinal):
910 ptcA00, ptcA00error = pars[0], parsErr[0]
911 ptcGain, ptcGainError = pars[1], parsErr[1]
912 ptcNoise = np.sqrt((pars[2]))
913 ptcNoiseAdu = ptcNoise*(1./ptcGain)
914 ptcNoiseError = 0.5*(parsErr[2]/np.fabs(pars[2]))*np.sqrt(np.fabs(pars[2]))
915 stringLegend = (f
"a00: {ptcA00:.2e}+/-{ptcA00error:.2e} 1/e"
916 f
"\nGain: {ptcGain:.4}+/-{ptcGainError:.2e} e/DN"
917 f
"\nNoise: {ptcNoise:.4}+/-{ptcNoiseError:.2e} e\n"
918 r"$\chi^2_{\rm{red}}$: " + f
"{ptcRedChi2:.4}")
920 if ptcFitType ==
'POLYNOMIAL':
921 if len(meanVecFinal):
922 ptcGain, ptcGainError = 1./pars[1], np.fabs(1./pars[1])*(parsErr[1]/pars[1])
923 ptcNoiseAdu = np.sqrt((pars[0]))
924 ptcNoise = ptcNoiseAdu*ptcGain
925 ptcNoiseError = (0.5*(parsErr[0]/np.fabs(pars[0]))*(np.sqrt(np.fabs(pars[0]))))*ptcGain
926 stringLegend = (f
"Gain: {ptcGain:.4}+/-{ptcGainError:.2e} e/DN\n"
927 f
"Noise: {ptcNoise:.4}+/-{ptcNoiseError:.2e} e\n"
928 r"$\chi^2_{\rm{red}}$: " + f
"{ptcRedChi2:.4}")
930 a.set_xlabel(
r'Mean signal ($\mu$, DN)', fontsize=labelFontSize)
931 a.set_ylabel(
r'Variance (DN$^2$)', fontsize=labelFontSize)
932 a.tick_params(labelsize=11)
933 a.set_xscale(
'linear', fontsize=labelFontSize)
934 a.set_yscale(
'linear', fontsize=labelFontSize)
936 a2.set_xlabel(
r'Mean Signal ($\mu$, DN)', fontsize=labelFontSize)
937 a2.set_ylabel(
r'Variance (DN$^2$)', fontsize=labelFontSize)
938 a2.tick_params(labelsize=11)
942 a3.set_xlabel(
r'Mean signal ($\mu$, DN)', fontsize=labelFontSize)
943 a3.set_ylabel(
r'Variance/$\mu$ (DN)', fontsize=labelFontSize)
944 a3.tick_params(labelsize=11)
945 a3.set_xscale(
'linear', fontsize=labelFontSize)
946 a3.set_yscale(
'linear', fontsize=labelFontSize)
948 minMeanVecFinal = np.min(meanVecFinal)
949 maxMeanVecFinal = np.max(meanVecFinal)
950 meanVecFit = np.linspace(minMeanVecFinal, maxMeanVecFinal, 100*len(meanVecFinal))
951 minMeanVecOriginal = np.min(meanVecOriginal)
952 maxMeanVecOriginal = np.max(meanVecOriginal)
953 deltaXlim = maxMeanVecOriginal - minMeanVecOriginal
954 a.plot(meanVecFit, ptcFunc(pars, meanVecFit), color=
'red')
955 a.plot(meanVecFinal, ptcNoiseAdu**2 + (1./ptcGain)*meanVecFinal, color=
'green',
957 a.scatter(meanVecFinal, varVecFinal, c=
'blue', marker=
'o', s=markerSize)
958 a.scatter(meanVecOutliers, varVecOutliers, c=
'magenta', marker=
's', s=markerSize)
959 a.text(0.03, 0.7, stringLegend, transform=a.transAxes, fontsize=legendFontSize)
960 a.set_title(amp, fontsize=titleFontSize)
961 a.set_xlim([minMeanVecOriginal - 0.2*deltaXlim, maxMeanVecOriginal + 0.2*deltaXlim])
964 a2.plot(meanVecFit, ptcFunc(pars, meanVecFit), color=
'red')
965 a2.scatter(meanVecFinal, varVecFinal, c=
'blue', marker=
'o', s=markerSize)
966 a2.scatter(meanVecOutliers, varVecOutliers, c=
'magenta', marker=
's', s=markerSize)
967 a2.text(0.03, 0.7, stringLegend, transform=a2.transAxes, fontsize=legendFontSize)
968 a2.set_title(amp, fontsize=titleFontSize)
969 a2.set_xlim([minMeanVecOriginal, maxMeanVecOriginal])
972 a3.plot(meanVecFit, ptcFunc(pars, meanVecFit)/meanVecFit, color=
'red')
973 a3.scatter(meanVecFinal, varVecFinal/meanVecFinal, c=
'blue', marker=
'o', s=markerSize)
974 a3.scatter(meanVecOutliers, varVecOutliers/meanVecOutliers, c=
'magenta', marker=
's',
976 a3.text(0.2, 0.65, stringLegend, transform=a3.transAxes, fontsize=legendFontSize)
977 a3.set_title(amp, fontsize=titleFontSize)
978 a3.set_xlim([minMeanVecOriginal - 0.2*deltaXlim, maxMeanVecOriginal + 0.2*deltaXlim])
980 f.suptitle(
"PTC \n Fit: " + stringTitle, fontsize=supTitleFontSize)
982 f2.suptitle(
"PTC (log-log)", fontsize=supTitleFontSize)
984 f3.suptitle(
r"Var/$\mu$", fontsize=supTitleFontSize)
989 def _plotLinearizer(self, dataset, linearizer, pdfPages):
990 """Plot linearity and linearity residual per amplifier
994 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset`
995 The dataset containing the means, variances, exposure times, and mask.
997 linearizer : `lsst.ip.isr.Linearizer`
1003 supTitleFontSize = 18
1006 nAmps = len(dataset.ampNames)
1009 nRows = np.sqrt(nAmps)
1010 mantissa, _ = np.modf(nRows)
1012 nRows = int(nRows) + 1
1019 f, ax = plt.subplots(nrows=nRows, ncols=nCols, sharex=
'col', sharey=
'row', figsize=(13, 10))
1020 f2, ax2 = plt.subplots(nrows=nRows, ncols=nCols, sharex=
'col', sharey=
'row', figsize=(13, 10))
1021 for i, (amp, a, a2)
in enumerate(zip(dataset.ampNames, ax.flatten(), ax2.flatten())):
1022 mask = dataset.expIdMask[amp]
1023 if np.isnan(mask[0]):
1024 a.set_title(f
"{amp} (BAD)", fontsize=titleFontSize)
1025 a2.set_title(f
"{amp} (BAD)", fontsize=titleFontSize)
1028 mask = mask.astype(bool)
1029 meanVecFinal = np.array(dataset.rawMeans[amp])[mask]
1030 timeVecFinal = np.array(dataset.rawExpTimes[amp])[mask]
1032 a.set_xlabel(
'Time (sec)', fontsize=labelFontSize)
1033 a.set_ylabel(
r'Mean signal ($\mu$, DN)', fontsize=labelFontSize)
1034 a.tick_params(labelsize=labelFontSize)
1035 a.set_xscale(
'linear', fontsize=labelFontSize)
1036 a.set_yscale(
'linear', fontsize=labelFontSize)
1038 a2.axhline(y=0, color=
'k')
1039 a2.axvline(x=0, color=
'k', linestyle=
'-')
1040 a2.set_xlabel(
r'Mean signal ($\mu$, DN)', fontsize=labelFontSize)
1041 a2.set_ylabel(
'Fractional nonlinearity (%)', fontsize=labelFontSize)
1042 a2.tick_params(labelsize=labelFontSize)
1043 a2.set_xscale(
'linear', fontsize=labelFontSize)
1044 a2.set_yscale(
'linear', fontsize=labelFontSize)
1046 pars, parsErr = linearizer.fitParams[amp], linearizer.fitParamsErr[amp]
1047 k0, k0Error = pars[0], parsErr[0]
1048 k1, k1Error = pars[1], parsErr[1]
1049 k2, k2Error = pars[2], parsErr[2]
1050 linRedChi2 = linearizer.fitChiSq[amp]
1051 stringLegend = (f
"k0: {k0:.4}+/-{k0Error:.2e} DN\nk1: {k1:.4}+/-{k1Error:.2e} DN/t"
1052 f
"\nk2: {k2:.2e}+/-{k2Error:.2e} DN/t^2\n"
1053 r"$\chi^2_{\rm{red}}$: " + f
"{linRedChi2:.4}")
1054 a.scatter(timeVecFinal, meanVecFinal)
1055 a.plot(timeVecFinal,
funcPolynomial(pars, timeVecFinal), color=
'red')
1056 a.text(0.03, 0.75, stringLegend, transform=a.transAxes, fontsize=legendFontSize)
1057 a.set_title(f
"{amp}", fontsize=titleFontSize)
1059 linearPart = k0 + k1*timeVecFinal
1060 fracLinRes = 100*(linearPart - meanVecFinal)/linearPart
1061 a2.plot(meanVecFinal, fracLinRes, c=
'g')
1062 a2.set_title(f
"{amp}", fontsize=titleFontSize)
1064 f.suptitle(
"Linearity \n Fit: Polynomial (degree: %g)"
1066 fontsize=supTitleFontSize)
1067 f2.suptitle(
r"Fractional NL residual" +
"\n" +
1068 r"$100\times \frac{(k_0 + k_1*Time-\mu)}{k_0+k_1*Time}$",
1069 fontsize=supTitleFontSize)
1071 pdfPages.savefig(f2)
1075 """Group data into bins, with at most maxDiff distance between bins.
1083 Maximum distance between bins.
1092 index = np.zeros_like(x, dtype=np.int32)
1097 for i
in range(1, len(ix)):
1099 if (xval - xc < maxDiff):
1100 xc = (ng*xc + xval)/(ng+1)
1102 index[ix[i]] = group
1106 index[ix[i]] = group
1113 """Builds an index with regular binning. The result can be fed into binData.
1124 np.digitize(x, bins): `numpy.array`
1128 bins = np.linspace(x.min(), x.max() + abs(x.max() * 1e-7), nBins + 1)
1129 return np.digitize(x, bins)
1133 """Bin data (usually for display purposes).
1144 Bin number of each datum.
1147 Inverse rms of each datum to use when averaging (the actual weight is wy**2).
1158 wybin: `numpy.array`
1159 Binned weights in y, computed from wy's in each bin.
1161 sybin: `numpy.array`
1162 Uncertainty on the bin average, considering actual scatter, and ignoring weights.
1166 wy = np.ones_like(x)
1167 binIndexSet = set(binIndex)
1170 xbin = np.array([xw2[binIndex == i].sum()/w2[binIndex == i].sum()
for i
in binIndexSet])
1173 ybin = np.array([yw2[binIndex == i].sum()/w2[binIndex == i].sum()
for i
in binIndexSet])
1175 wybin = np.sqrt(np.array([w2[binIndex == i].sum()
for i
in binIndexSet]))
1176 sybin = np.array([y[binIndex == i].
std()/np.sqrt(np.array([binIndex == i]).sum())
1177 for i
in binIndexSet])
1179 return xbin, ybin, wybin, sybin