Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of cp_pipe. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21# 

22 

23__all__ = ['PlotPhotonTransferCurveTask'] 

24 

25import numpy as np 

26import matplotlib.pyplot as plt 

27import matplotlib as mpl 

28from matplotlib import gridspec 

29import os 

30from matplotlib.backends.backend_pdf import PdfPages 

31 

32import lsst.ip.isr as isr 

33import lsst.pex.config as pexConfig 

34import lsst.pipe.base as pipeBase 

35import pickle 

36 

37from .utils import (funcAstier, funcPolynomial, NonexistentDatasetTaskDataIdContainer) 

38from matplotlib.ticker import MaxNLocator 

39 

40from .astierCovPtcFit import computeApproximateAcoeffs 

41 

42 

43class PlotPhotonTransferCurveTaskConfig(pexConfig.Config): 

44 """Config class for photon transfer curve measurement task""" 

45 datasetFileName = pexConfig.Field( 

46 dtype=str, 

47 doc="datasetPtc file name (pkl)", 

48 default="", 

49 ) 

50 linearizerFileName = pexConfig.Field( 

51 dtype=str, 

52 doc="linearizer file name (fits)", 

53 default="", 

54 ) 

55 ccdKey = pexConfig.Field( 

56 dtype=str, 

57 doc="The key by which to pull a detector from a dataId, e.g. 'ccd' or 'detector'.", 

58 default='detector', 

59 ) 

60 

61 

62class PlotPhotonTransferCurveTask(pipeBase.CmdLineTask): 

63 """A class to plot the dataset from MeasurePhotonTransferCurveTask. 

64 

65 Parameters 

66 ---------- 

67 

68 *args: `list` 

69 Positional arguments passed to the Task constructor. None used at this 

70 time. 

71 **kwargs: `dict` 

72 Keyword arguments passed on to the Task constructor. None used at this 

73 time. 

74 

75 """ 

76 

77 ConfigClass = PlotPhotonTransferCurveTaskConfig 

78 _DefaultName = "plotPhotonTransferCurve" 

79 

80 def __init__(self, *args, **kwargs): 

81 pipeBase.CmdLineTask.__init__(self, *args, **kwargs) 

82 plt.interactive(False) # stop windows popping up when plotting. When headless, use 'agg' backend too 

83 self.config.validate() 

84 self.config.freeze() 

85 

86 @classmethod 

87 def _makeArgumentParser(cls): 

88 """Augment argument parser for the MeasurePhotonTransferCurveTask.""" 

89 parser = pipeBase.ArgumentParser(name=cls._DefaultName) 

90 parser.add_id_argument("--id", datasetType="photonTransferCurveDataset", 

91 ContainerClass=NonexistentDatasetTaskDataIdContainer, 

92 help="The ccds to use, e.g. --id ccd=0..100") 

93 return parser 

94 

95 @pipeBase.timeMethod 

96 def runDataRef(self, dataRef): 

97 """Run the Photon Transfer Curve (PTC) plotting measurement task. 

98 

99 Parameters 

100 ---------- 

101 dataRef : list of lsst.daf.persistence.ButlerDataRef 

102 dataRef for the detector for the visits to be fit. 

103 """ 

104 

105 datasetFile = self.config.datasetFileName 

106 

107 with open(datasetFile, "rb") as f: 

108 datasetPtc = pickle.load(f) 

109 

110 dirname = dataRef.getUri(datasetType='cpPipePlotRoot', write=True) 

111 if not os.path.exists(dirname): 

112 os.makedirs(dirname) 

113 

114 detNum = dataRef.dataId[self.config.ccdKey] 

115 filename = f"PTC_det{detNum}.pdf" 

116 filenameFull = os.path.join(dirname, filename) 

117 

118 if self.config.linearizerFileName: 

119 linearizer = isr.linearize.Linearizer.readFits(self.config.linearizerFileName) 

120 else: 

121 linearizer = None 

122 self.run(filenameFull, datasetPtc, linearizer=linearizer, log=self.log) 

123 

124 return pipeBase.Struct(exitStatus=0) 

125 

126 def run(self, filenameFull, datasetPtc, linearizer=None, log=None): 

127 """Make the plots for the PTC task""" 

128 ptcFitType = datasetPtc.ptcFitType 

129 with PdfPages(filenameFull) as pdfPages: 

130 if ptcFitType in ["FULLCOVARIANCE", ]: 

131 self.covAstierMakeAllPlots(datasetPtc.covariancesFits, datasetPtc.covariancesFitsWithNoB, 

132 pdfPages, log=log) 

133 elif ptcFitType in ["EXPAPPROXIMATION", "POLYNOMIAL"]: 

134 self._plotStandardPtc(datasetPtc, ptcFitType, pdfPages) 

135 else: 

136 raise RuntimeError(f"The input dataset had an invalid dataset.ptcFitType: {ptcFitType}. \n" + 

137 "Options: 'FULLCOVARIANCE', EXPAPPROXIMATION, or 'POLYNOMIAL'.") 

138 if linearizer: 

139 self._plotLinearizer(datasetPtc, linearizer, pdfPages) 

140 

141 return 

142 

143 def covAstierMakeAllPlots(self, covFits, covFitsNoB, pdfPages, 

144 log=None, maxMu=1e9): 

145 """Make plots for MeasurePhotonTransferCurve task when doCovariancesAstier=True. 

146 

147 This function call other functions that mostly reproduce the plots in Astier+19. 

148 Most of the code is ported from Pierre Astier's repository https://github.com/PierreAstier/bfptc 

149 

150 Parameters 

151 ---------- 

152 covFits: `dict` 

153 Dictionary of CovFit objects, with amp names as keys. 

154 

155 covFitsNoB: `dict` 

156 Dictionary of CovFit objects, with amp names as keys (b=0 in Eq. 20 of Astier+19). 

157 

158 pdfPages: `matplotlib.backends.backend_pdf.PdfPages` 

159 PDF file where the plots will be saved. 

160 

161 log : `lsst.log.Log`, optional 

162 Logger to handle messages 

163 

164 maxMu: `float`, optional 

165 Maximum signal, in ADU. 

166 """ 

167 self.plotCovariances(covFits, pdfPages) 

168 self.plotNormalizedCovariances(covFits, covFitsNoB, 0, 0, pdfPages, offset=0.01, topPlot=True, 

169 log=log) 

170 self.plotNormalizedCovariances(covFits, covFitsNoB, 0, 1, pdfPages, log=log) 

171 self.plotNormalizedCovariances(covFits, covFitsNoB, 1, 0, pdfPages, log=log) 

172 self.plot_a_b(covFits, pdfPages) 

173 self.ab_vs_dist(covFits, pdfPages, bRange=4) 

174 self.plotAcoeffsSum(covFits, pdfPages) 

175 self.plotRelativeBiasACoeffs(covFits, covFitsNoB, maxMu, pdfPages) 

176 

177 return 

178 

179 @staticmethod 

180 def plotCovariances(covFits, pdfPages): 

181 """Plot covariances and models: Cov00, Cov10, Cov01. 

182 

183 Figs. 6 and 7 of Astier+19 

184 

185 Parameters 

186 ---------- 

187 covFits: `dict` 

188 Dictionary of CovFit objects, with amp names as keys. 

189 

190 pdfPages: `matplotlib.backends.backend_pdf.PdfPages` 

191 PDF file where the plots will be saved. 

192 """ 

193 

194 legendFontSize = 7 

195 labelFontSize = 7 

196 titleFontSize = 9 

197 supTitleFontSize = 18 

198 markerSize = 25 

199 

200 nAmps = len(covFits) 

201 if nAmps == 2: 

202 nRows, nCols = 2, 1 

203 nRows = np.sqrt(nAmps) 

204 mantissa, _ = np.modf(nRows) 

205 if mantissa > 0: 

206 nRows = int(nRows) + 1 

207 nCols = nRows 

208 else: 

209 nRows = int(nRows) 

210 nCols = nRows 

211 

212 f, ax = plt.subplots(nrows=nRows, ncols=nCols, sharex='col', sharey='row', figsize=(13, 10)) 

213 f2, ax2 = plt.subplots(nrows=nRows, ncols=nCols, sharex='col', sharey='row', figsize=(13, 10)) 

214 fResCov00, axResCov00 = plt.subplots(nrows=nRows, ncols=nCols, sharex='col', sharey='row', 

215 figsize=(13, 10)) 

216 fCov01, axCov01 = plt.subplots(nrows=nRows, ncols=nCols, sharex='col', sharey='row', figsize=(13, 10)) 

217 fCov10, axCov10 = plt.subplots(nrows=nRows, ncols=nCols, sharex='col', sharey='row', figsize=(13, 10)) 

218 

219 for i, (fitPair, a, a2, aResVar, a3, a4) in enumerate(zip(covFits.items(), ax.flatten(), 

220 ax2.flatten(), axResCov00.flatten(), 

221 axCov01.flatten(), axCov10.flatten())): 

222 

223 amp = fitPair[0] 

224 fit = fitPair[1] 

225 

226 meanVecFinal, varVecFinal, varVecModel, wc = fit.getNormalizedFitData(0, 0) 

227 meanVecFinalCov01, varVecFinalCov01, varVecModelCov01, wcCov01 = fit.getNormalizedFitData(0, 1) 

228 meanVecFinalCov10, varVecFinalCov10, varVecModelCov10, wcCov10 = fit.getNormalizedFitData(1, 0) 

229 

230 # cuadratic fit for residuals below 

231 par2 = np.polyfit(meanVecFinal, varVecFinal, 2, w=wc) 

232 varModelQuadratic = np.polyval(par2, meanVecFinal) 

233 

234 # fit with no 'b' coefficient (c = a*b in Eq. 20 of Astier+19) 

235 fitNoB = fit.copy() 

236 fitNoB.params['c'].fix(val=0) 

237 fitNoB.fitFullModel() 

238 meanVecFinalNoB, varVecFinalNoB, varVecModelNoB, wcNoB = fitNoB.getNormalizedFitData(0, 0) 

239 

240 if len(meanVecFinal): # Empty if the whole amp is bad, for example. 

241 stringLegend = (f"Gain: {fit.getGain():.4} e/DN \n Noise: {np.sqrt(fit.getRon()):.4} e \n" + 

242 r"$a_{00}$: %.3e 1/e"%fit.getA()[0, 0] + 

243 "\n" + r"$b_{00}$: %.3e 1/e"%fit.getB()[0, 0]) 

244 minMeanVecFinal = np.min(meanVecFinal) 

245 maxMeanVecFinal = np.max(meanVecFinal) 

246 deltaXlim = maxMeanVecFinal - minMeanVecFinal 

247 

248 a.set_xlabel(r'Mean signal ($\mu$, DN)', fontsize=labelFontSize) 

249 a.set_ylabel(r'Variance (DN$^2$)', fontsize=labelFontSize) 

250 a.tick_params(labelsize=11) 

251 a.set_xscale('linear', fontsize=labelFontSize) 

252 a.set_yscale('linear', fontsize=labelFontSize) 

253 a.scatter(meanVecFinal, varVecFinal, c='blue', marker='o', s=markerSize) 

254 a.plot(meanVecFinal, varVecModel, color='red', lineStyle='-') 

255 a.text(0.03, 0.7, stringLegend, transform=a.transAxes, fontsize=legendFontSize) 

256 a.set_title(amp, fontsize=titleFontSize) 

257 a.set_xlim([minMeanVecFinal - 0.2*deltaXlim, maxMeanVecFinal + 0.2*deltaXlim]) 

258 

259 # Same as above, but in log-scale 

260 a2.set_xlabel(r'Mean Signal ($\mu$, DN)', fontsize=labelFontSize) 

261 a2.set_ylabel(r'Variance (DN$^2$)', fontsize=labelFontSize) 

262 a2.tick_params(labelsize=11) 

263 a2.set_xscale('log') 

264 a2.set_yscale('log') 

265 a2.plot(meanVecFinal, varVecModel, color='red', lineStyle='-') 

266 a2.scatter(meanVecFinal, varVecFinal, c='blue', marker='o', s=markerSize) 

267 a2.text(0.03, 0.7, stringLegend, transform=a2.transAxes, fontsize=legendFontSize) 

268 a2.set_title(amp, fontsize=titleFontSize) 

269 a2.set_xlim([minMeanVecFinal, maxMeanVecFinal]) 

270 

271 # Residuals var - model 

272 aResVar.set_xlabel(r'Mean signal ($\mu$, DN)', fontsize=labelFontSize) 

273 aResVar.set_ylabel(r'Residuals (DN$^2$)', fontsize=labelFontSize) 

274 aResVar.tick_params(labelsize=11) 

275 aResVar.set_xscale('linear', fontsize=labelFontSize) 

276 aResVar.set_yscale('linear', fontsize=labelFontSize) 

277 aResVar.plot(meanVecFinal, varVecFinal - varVecModel, color='blue', lineStyle='-', 

278 label='Full fit') 

279 aResVar.plot(meanVecFinal, varVecFinal - varModelQuadratic, color='red', lineStyle='-', 

280 label='Quadratic fit') 

281 aResVar.plot(meanVecFinalNoB, varVecFinalNoB - varVecModelNoB, color='green', lineStyle='-', 

282 label='Full fit with b=0') 

283 aResVar.axhline(color='black') 

284 aResVar.set_title(amp, fontsize=titleFontSize) 

285 aResVar.set_xlim([minMeanVecFinal - 0.2*deltaXlim, maxMeanVecFinal + 0.2*deltaXlim]) 

286 aResVar.legend(fontsize=7) 

287 

288 a3.set_xlabel(r'Mean signal ($\mu$, DN)', fontsize=labelFontSize) 

289 a3.set_ylabel(r'Cov01 (DN$^2$)', fontsize=labelFontSize) 

290 a3.tick_params(labelsize=11) 

291 a3.set_xscale('linear', fontsize=labelFontSize) 

292 a3.set_yscale('linear', fontsize=labelFontSize) 

293 a3.scatter(meanVecFinalCov01, varVecFinalCov01, c='blue', marker='o', s=markerSize) 

294 a3.plot(meanVecFinalCov01, varVecModelCov01, color='red', lineStyle='-') 

295 a3.set_title(amp, fontsize=titleFontSize) 

296 a3.set_xlim([minMeanVecFinal - 0.2*deltaXlim, maxMeanVecFinal + 0.2*deltaXlim]) 

297 

298 a4.set_xlabel(r'Mean signal ($\mu$, DN)', fontsize=labelFontSize) 

299 a4.set_ylabel(r'Cov10 (DN$^2$)', fontsize=labelFontSize) 

300 a4.tick_params(labelsize=11) 

301 a4.set_xscale('linear', fontsize=labelFontSize) 

302 a4.set_yscale('linear', fontsize=labelFontSize) 

303 a4.scatter(meanVecFinalCov10, varVecFinalCov10, c='blue', marker='o', s=markerSize) 

304 a4.plot(meanVecFinalCov10, varVecModelCov10, color='red', lineStyle='-') 

305 a4.set_title(amp, fontsize=titleFontSize) 

306 a4.set_xlim([minMeanVecFinal - 0.2*deltaXlim, maxMeanVecFinal + 0.2*deltaXlim]) 

307 

308 else: 

309 a.set_title(f"{amp} (BAD)", fontsize=titleFontSize) 

310 a2.set_title(f"{amp} (BAD)", fontsize=titleFontSize) 

311 a3.set_title(f"{amp} (BAD)", fontsize=titleFontSize) 

312 a4.set_title(f"{amp} (BAD)", fontsize=titleFontSize) 

313 

314 f.suptitle("PTC from covariances as in Astier+19 \n Fit: Eq. 20, Astier+19", 

315 fontsize=supTitleFontSize) 

316 pdfPages.savefig(f) 

317 f2.suptitle("PTC from covariances as in Astier+19 (log-log) \n Fit: Eq. 20, Astier+19", 

318 fontsize=supTitleFontSize) 

319 pdfPages.savefig(f2) 

320 fResCov00.suptitle("Residuals (data- model) for Cov00 (Var)", fontsize=supTitleFontSize) 

321 pdfPages.savefig(fResCov00) 

322 fCov01.suptitle("Cov01 as in Astier+19 (nearest parallel neighbor covariance) \n" + 

323 " Fit: Eq. 20, Astier+19", fontsize=supTitleFontSize) 

324 pdfPages.savefig(fCov01) 

325 fCov10.suptitle("Cov10 as in Astier+19 (nearest serial neighbor covariance) \n" + 

326 "Fit: Eq. 20, Astier+19", fontsize=supTitleFontSize) 

327 pdfPages.savefig(fCov10) 

328 

329 return 

330 

331 def plotNormalizedCovariances(self, covFits, covFitsNoB, i, j, pdfPages, offset=0.004, 

332 plotData=True, topPlot=False, log=None): 

333 """Plot C_ij/mu vs mu. 

334 

335 Figs. 8, 10, and 11 of Astier+19 

336 

337 Parameters 

338 ---------- 

339 covFits: `dict` 

340 Dictionary of CovFit objects, with amp names as keys. 

341 

342 covFitsNoB: `dict` 

343 Dictionary of CovFit objects, with amp names as keys (b=0 in Eq. 20 of Astier+19). 

344 

345 i : `int` 

346 Covariane lag 

347 

348 j : `int 

349 Covariance lag 

350 

351 pdfPages: `matplotlib.backends.backend_pdf.PdfPages` 

352 PDF file where the plots will be saved. 

353 

354 offset : `float`, optional 

355 Constant offset factor to plot covariances in same panel (so they don't overlap). 

356 

357 plotData : `bool`, optional 

358 Plot the data points? 

359 

360 topPlot : `bool`, optional 

361 Plot the top plot with the covariances, and the bottom plot with the model residuals? 

362 

363 log : `lsst.log.Log`, optional 

364 Logger to handle messages. 

365 """ 

366 

367 lchi2, la, lb, lcov = [], [], [], [] 

368 

369 if (not topPlot): 

370 fig = plt.figure(figsize=(8, 10)) 

371 gs = gridspec.GridSpec(2, 1, height_ratios=[3, 1]) 

372 gs.update(hspace=0) 

373 ax0 = plt.subplot(gs[0]) 

374 plt.setp(ax0.get_xticklabels(), visible=False) 

375 else: 

376 fig = plt.figure(figsize=(8, 8)) 

377 ax0 = plt.subplot(111) 

378 ax0.ticklabel_format(style='sci', axis='x', scilimits=(0, 0)) 

379 ax0.tick_params(axis='both', labelsize='x-large') 

380 mue, rese, wce = [], [], [] 

381 mueNoB, reseNoB, wceNoB = [], [], [] 

382 for counter, (amp, fit) in enumerate(covFits.items()): 

383 mu, c, model, wc = fit.getNormalizedFitData(i, j, divideByMu=True) 

384 wres = (c-model)*wc 

385 chi2 = ((wres*wres).sum())/(len(mu)-3) 

386 chi2bin = 0 

387 mue += list(mu) 

388 rese += list(c - model) 

389 wce += list(wc) 

390 

391 fitNoB = covFitsNoB[amp] 

392 muNoB, cNoB, modelNoB, wcNoB = fitNoB.getNormalizedFitData(i, j, divideByMu=True) 

393 mueNoB += list(muNoB) 

394 reseNoB += list(cNoB - modelNoB) 

395 wceNoB += list(wcNoB) 

396 

397 # the corresponding fit 

398 fit_curve, = plt.plot(mu, model + counter*offset, '-', linewidth=4.0) 

399 # bin plot. len(mu) = no binning 

400 gind = self.indexForBins(mu, len(mu)) 

401 

402 xb, yb, wyb, sigyb = self.binData(mu, c, gind, wc) 

403 chi2bin = (sigyb*wyb).mean() # chi2 of enforcing the same value in each bin 

404 plt.errorbar(xb, yb+counter*offset, yerr=sigyb, marker='o', linestyle='none', markersize=6.5, 

405 color=fit_curve.get_color(), label=f"{amp}") 

406 # plot the data 

407 if plotData: 

408 points, = plt.plot(mu, c + counter*offset, '.', color=fit_curve.get_color()) 

409 plt.legend(loc='upper right', fontsize=8) 

410 aij = fit.getA()[i, j] 

411 bij = fit.getB()[i, j] 

412 la.append(aij) 

413 lb.append(bij) 

414 lcov.append(fit.getACov()[i, j, i, j]) 

415 lchi2.append(chi2) 

416 log.info('%s: slope %g b %g chi2 %f chi2bin %f'%(amp, aij, bij, chi2, chi2bin)) 

417 # end loop on amps 

418 la = np.array(la) 

419 lb = np.array(lb) 

420 lcov = np.array(lcov) 

421 lchi2 = np.array(lchi2) 

422 mue = np.array(mue) 

423 rese = np.array(rese) 

424 wce = np.array(wce) 

425 mueNoB = np.array(mueNoB) 

426 reseNoB = np.array(reseNoB) 

427 wceNoB = np.array(wceNoB) 

428 

429 plt.xlabel(r"$\mu (el)$", fontsize='x-large') 

430 plt.ylabel(r"$C_{%d%d}/\mu + Cst (el)$"%(i, j), fontsize='x-large') 

431 if (not topPlot): 

432 gind = self.indexForBins(mue, len(mue)) 

433 xb, yb, wyb, sigyb = self.binData(mue, rese, gind, wce) 

434 

435 ax1 = plt.subplot(gs[1], sharex=ax0) 

436 ax1.errorbar(xb, yb, yerr=sigyb, marker='o', linestyle='none', label='Full fit') 

437 gindNoB = self.indexForBins(mueNoB, len(mueNoB)) 

438 xb2, yb2, wyb2, sigyb2 = self.binData(mueNoB, reseNoB, gindNoB, wceNoB) 

439 

440 ax1.errorbar(xb2, yb2, yerr=sigyb2, marker='o', linestyle='none', label='b = 0') 

441 ax1.tick_params(axis='both', labelsize='x-large') 

442 plt.legend(loc='upper left', fontsize='large') 

443 # horizontal line at zero 

444 plt.plot(xb, [0]*len(xb), '--', color='k') 

445 plt.ticklabel_format(style='sci', axis='x', scilimits=(0, 0)) 

446 plt.ticklabel_format(style='sci', axis='y', scilimits=(0, 0)) 

447 plt.xlabel(r'$\mu (el)$', fontsize='x-large') 

448 plt.ylabel(r'$C_{%d%d}/\mu$ -model (el)'%(i, j), fontsize='x-large') 

449 plt.tight_layout() 

450 

451 # overlapping y labels: 

452 fig.canvas.draw() 

453 labels0 = [item.get_text() for item in ax0.get_yticklabels()] 

454 labels0[0] = u'' 

455 ax0.set_yticklabels(labels0) 

456 pdfPages.savefig(fig) 

457 

458 return 

459 

460 @staticmethod 

461 def plot_a_b(covFits, pdfPages, bRange=3): 

462 """Fig. 12 of Astier+19 

463 

464 Color display of a and b arrays fits, averaged over channels. 

465 

466 Parameters 

467 ---------- 

468 covFits: `dict` 

469 Dictionary of CovFit objects, with amp names as keys. 

470 

471 pdfPages: `matplotlib.backends.backend_pdf.PdfPages` 

472 PDF file where the plots will be saved. 

473 

474 bRange : `int` 

475 Maximum lag for b arrays. 

476 """ 

477 a, b = [], [] 

478 for amp, fit in covFits.items(): 

479 a.append(fit.getA()) 

480 b.append(fit.getB()) 

481 a = np.array(a).mean(axis=0) 

482 b = np.array(b).mean(axis=0) 

483 fig = plt.figure(figsize=(7, 11)) 

484 ax0 = fig.add_subplot(2, 1, 1) 

485 im0 = ax0.imshow(np.abs(a.transpose()), origin='lower', norm=mpl.colors.LogNorm()) 

486 ax0.tick_params(axis='both', labelsize='x-large') 

487 ax0.set_title(r'$|a|$', fontsize='x-large') 

488 ax0.xaxis.set_ticks_position('bottom') 

489 cb0 = plt.colorbar(im0) 

490 cb0.ax.tick_params(labelsize='x-large') 

491 

492 ax1 = fig.add_subplot(2, 1, 2) 

493 ax1.tick_params(axis='both', labelsize='x-large') 

494 ax1.yaxis.set_major_locator(MaxNLocator(integer=True)) 

495 ax1.xaxis.set_major_locator(MaxNLocator(integer=True)) 

496 im1 = ax1.imshow(1e6*b[:bRange, :bRange].transpose(), origin='lower') 

497 cb1 = plt.colorbar(im1) 

498 cb1.ax.tick_params(labelsize='x-large') 

499 ax1.set_title(r'$b \times 10^6$', fontsize='x-large') 

500 ax1.xaxis.set_ticks_position('bottom') 

501 plt.tight_layout() 

502 pdfPages.savefig(fig) 

503 

504 return 

505 

506 @staticmethod 

507 def ab_vs_dist(covFits, pdfPages, bRange=4): 

508 """Fig. 13 of Astier+19. 

509 

510 Values of a and b arrays fits, averaged over amplifiers, as a function of distance. 

511 

512 Parameters 

513 ---------- 

514 covFits: `dict` 

515 Dictionary of CovFit objects, with amp names as keys. 

516 

517 pdfPages: `matplotlib.backends.backend_pdf.PdfPages` 

518 PDF file where the plots will be saved. 

519 

520 bRange : `int` 

521 Maximum lag for b arrays. 

522 """ 

523 a = np.array([f.getA() for f in covFits.values()]) 

524 y = a.mean(axis=0) 

525 sy = a.std(axis=0)/np.sqrt(len(covFits)) 

526 i, j = np.indices(y.shape) 

527 upper = (i >= j).ravel() 

528 r = np.sqrt(i**2 + j**2).ravel() 

529 y = y.ravel() 

530 sy = sy.ravel() 

531 fig = plt.figure(figsize=(6, 9)) 

532 ax = fig.add_subplot(211) 

533 ax.set_xlim([0.5, r.max()+1]) 

534 ax.errorbar(r[upper], y[upper], yerr=sy[upper], marker='o', linestyle='none', color='b', 

535 label='$i>=j$') 

536 ax.errorbar(r[~upper], y[~upper], yerr=sy[~upper], marker='o', linestyle='none', color='r', 

537 label='$i<j$') 

538 ax.legend(loc='upper center', fontsize='x-large') 

539 ax.set_xlabel(r'$\sqrt{i^2+j^2}$', fontsize='x-large') 

540 ax.set_ylabel(r'$a_{ij}$', fontsize='x-large') 

541 ax.set_yscale('log') 

542 ax.tick_params(axis='both', labelsize='x-large') 

543 

544 # 

545 axb = fig.add_subplot(212) 

546 b = np.array([f.getB() for f in covFits.values()]) 

547 yb = b.mean(axis=0) 

548 syb = b.std(axis=0)/np.sqrt(len(covFits)) 

549 ib, jb = np.indices(yb.shape) 

550 upper = (ib > jb).ravel() 

551 rb = np.sqrt(i**2 + j**2).ravel() 

552 yb = yb.ravel() 

553 syb = syb.ravel() 

554 xmin = -0.2 

555 xmax = bRange 

556 axb.set_xlim([xmin, xmax+0.2]) 

557 cutu = (r > xmin) & (r < xmax) & (upper) 

558 cutl = (r > xmin) & (r < xmax) & (~upper) 

559 axb.errorbar(rb[cutu], yb[cutu], yerr=syb[cutu], marker='o', linestyle='none', color='b', 

560 label='$i>=j$') 

561 axb.errorbar(rb[cutl], yb[cutl], yerr=syb[cutl], marker='o', linestyle='none', color='r', 

562 label='$i<j$') 

563 plt.legend(loc='upper center', fontsize='x-large') 

564 axb.set_xlabel(r'$\sqrt{i^2+j^2}$', fontsize='x-large') 

565 axb.set_ylabel(r'$b_{ij}$', fontsize='x-large') 

566 axb.ticklabel_format(style='sci', axis='y', scilimits=(0, 0)) 

567 axb.tick_params(axis='both', labelsize='x-large') 

568 plt.tight_layout() 

569 pdfPages.savefig(fig) 

570 

571 return 

572 

573 @staticmethod 

574 def plotAcoeffsSum(covFits, pdfPages): 

575 """Fig. 14. of Astier+19 

576 

577 Cumulative sum of a_ij as a function of maximum separation. This plot displays the average over 

578 channels. 

579 

580 Parameters 

581 ---------- 

582 covFits: `dict` 

583 Dictionary of CovFit objects, with amp names as keys. 

584 

585 pdfPages: `matplotlib.backends.backend_pdf.PdfPages` 

586 PDF file where the plots will be saved. 

587 """ 

588 a, b = [], [] 

589 for amp, fit in covFits.items(): 

590 a.append(fit.getA()) 

591 b.append(fit.getB()) 

592 a = np.array(a).mean(axis=0) 

593 b = np.array(b).mean(axis=0) 

594 fig = plt.figure(figsize=(7, 6)) 

595 w = 4*np.ones_like(a) 

596 w[0, 1:] = 2 

597 w[1:, 0] = 2 

598 w[0, 0] = 1 

599 wa = w*a 

600 indices = range(1, a.shape[0]+1) 

601 sums = [wa[0:n, 0:n].sum() for n in indices] 

602 ax = plt.subplot(111) 

603 ax.plot(indices, sums/sums[0], 'o', color='b') 

604 ax.set_yscale('log') 

605 ax.set_xlim(indices[0]-0.5, indices[-1]+0.5) 

606 ax.set_ylim(None, 1.2) 

607 ax.set_ylabel(r'$[\sum_{|i|<n\ &\ |j|<n} a_{ij}] / |a_{00}|$', fontsize='x-large') 

608 ax.set_xlabel('n', fontsize='x-large') 

609 ax.tick_params(axis='both', labelsize='x-large') 

610 plt.tight_layout() 

611 pdfPages.savefig(fig) 

612 

613 return 

614 

615 @staticmethod 

616 def plotRelativeBiasACoeffs(covFits, covFitsNoB, maxMu, pdfPages, maxr=None): 

617 """Fig. 15 in Astier+19. 

618 

619 Illustrates systematic bias from estimating 'a' 

620 coefficients from the slope of correlations as opposed to the 

621 full model in Astier+19. 

622 

623 Parameters 

624 ---------- 

625 covFits: `dict` 

626 Dictionary of CovFit objects, with amp names as keys. 

627 

628 covFitsNoB: `dict` 

629 Dictionary of CovFit objects, with amp names as keys (b=0 in Eq. 20 of Astier+19). 

630 

631 maxMu: `float`, optional 

632 Maximum signal, in ADU. 

633 

634 pdfPages: `matplotlib.backends.backend_pdf.PdfPages` 

635 PDF file where the plots will be saved. 

636 

637 maxr: `int`, optional 

638 Maximum lag. 

639 """ 

640 

641 fig = plt.figure(figsize=(7, 11)) 

642 title = ["'a' relative bias", "'a' relative bias (b=0)"] 

643 data = [covFits, covFitsNoB] 

644 

645 for k in range(2): 

646 diffs = [] 

647 amean = [] 

648 for fit in data[k].values(): 

649 if fit is None: 

650 continue 

651 maxMuEl = maxMu*fit.getGain() 

652 aOld = computeApproximateAcoeffs(fit, maxMuEl) 

653 a = fit.getA() 

654 amean.append(a) 

655 diffs.append((aOld-a)) 

656 amean = np.array(amean).mean(axis=0) 

657 diff = np.array(diffs).mean(axis=0) 

658 diff = diff/amean 

659 diff[0, 0] = 0 

660 if maxr is None: 

661 maxr = diff.shape[0] 

662 diff = diff[:maxr, :maxr] 

663 ax0 = fig.add_subplot(2, 1, k+1) 

664 im0 = ax0.imshow(diff.transpose(), origin='lower') 

665 ax0.yaxis.set_major_locator(MaxNLocator(integer=True)) 

666 ax0.xaxis.set_major_locator(MaxNLocator(integer=True)) 

667 ax0.tick_params(axis='both', labelsize='x-large') 

668 plt.colorbar(im0) 

669 ax0.set_title(title[k]) 

670 

671 plt.tight_layout() 

672 pdfPages.savefig(fig) 

673 

674 return 

675 

676 def _plotStandardPtc(self, dataset, ptcFitType, pdfPages): 

677 """Plot PTC, linearity, and linearity residual per amplifier 

678 

679 Parameters 

680 ---------- 

681 dataset : `lsst.cp.pipe.ptc.PhotonTransferCurveDataset` 

682 The dataset containing the means, variances, exposure times, and mask. 

683 

684 ptcFitType : `str` 

685 Type of the model fit to the PTC. Options: 'FULLCOVARIANCE', EXPAPPROXIMATION, or 'POLYNOMIAL'. 

686 

687 pdfPages: `matplotlib.backends.backend_pdf.PdfPages` 

688 PDF file where the plots will be saved. 

689 """ 

690 

691 if ptcFitType == 'EXPAPPROXIMATION': 

692 ptcFunc = funcAstier 

693 stringTitle = (r"Var = $\frac{1}{2g^2a_{00}}(\exp (2a_{00} \mu g) - 1) + \frac{n_{00}}{g^2}$ ") 

694 elif ptcFitType == 'POLYNOMIAL': 

695 ptcFunc = funcPolynomial 

696 for key in dataset.ptcFitPars: 

697 deg = len(dataset.ptcFitPars[key]) - 1 

698 break 

699 stringTitle = r"Polynomial (degree: %g)" % (deg) 

700 else: 

701 raise RuntimeError(f"The input dataset had an invalid dataset.ptcFitType: {ptcFitType}. \n" + 

702 "Options: 'FULLCOVARIANCE', EXPAPPROXIMATION, or 'POLYNOMIAL'.") 

703 

704 legendFontSize = 7 

705 labelFontSize = 7 

706 titleFontSize = 9 

707 supTitleFontSize = 18 

708 markerSize = 25 

709 

710 # General determination of the size of the plot grid 

711 nAmps = len(dataset.ampNames) 

712 if nAmps == 2: 

713 nRows, nCols = 2, 1 

714 nRows = np.sqrt(nAmps) 

715 mantissa, _ = np.modf(nRows) 

716 if mantissa > 0: 

717 nRows = int(nRows) + 1 

718 nCols = nRows 

719 else: 

720 nRows = int(nRows) 

721 nCols = nRows 

722 

723 f, ax = plt.subplots(nrows=nRows, ncols=nCols, sharex='col', sharey='row', figsize=(13, 10)) 

724 f2, ax2 = plt.subplots(nrows=nRows, ncols=nCols, sharex='col', sharey='row', figsize=(13, 10)) 

725 

726 for i, (amp, a, a2) in enumerate(zip(dataset.ampNames, ax.flatten(), ax2.flatten())): 

727 meanVecOriginal = np.array(dataset.rawMeans[amp]) 

728 varVecOriginal = np.array(dataset.rawVars[amp]) 

729 mask = dataset.visitMask[amp] 

730 meanVecFinal = meanVecOriginal[mask] 

731 varVecFinal = varVecOriginal[mask] 

732 meanVecOutliers = meanVecOriginal[np.invert(mask)] 

733 varVecOutliers = varVecOriginal[np.invert(mask)] 

734 pars, parsErr = dataset.ptcFitPars[amp], dataset.ptcFitParsError[amp] 

735 if ptcFitType == 'EXPAPPROXIMATION': 

736 if len(meanVecFinal): 

737 ptcA00, ptcA00error = pars[0], parsErr[0] 

738 ptcGain, ptcGainError = pars[1], parsErr[1] 

739 ptcNoise = np.sqrt(np.fabs(pars[2])) 

740 ptcNoiseError = 0.5*(parsErr[2]/np.fabs(pars[2]))*np.sqrt(np.fabs(pars[2])) 

741 stringLegend = (f"a00: {ptcA00:.2e}+/-{ptcA00error:.2e} 1/e" 

742 f"\n Gain: {ptcGain:.4}+/-{ptcGainError:.2e} e/DN" 

743 f"\n Noise: {ptcNoise:.4}+/-{ptcNoiseError:.2e} e \n") 

744 

745 if ptcFitType == 'POLYNOMIAL': 

746 if len(meanVecFinal): 

747 ptcGain, ptcGainError = 1./pars[1], np.fabs(1./pars[1])*(parsErr[1]/pars[1]) 

748 ptcNoise = np.sqrt(np.fabs(pars[0]))*ptcGain 

749 ptcNoiseError = (0.5*(parsErr[0]/np.fabs(pars[0]))*(np.sqrt(np.fabs(pars[0]))))*ptcGain 

750 stringLegend = (f"Noise: {ptcNoise:.4}+/-{ptcNoiseError:.2e} e \n" 

751 f"Gain: {ptcGain:.4}+/-{ptcGainError:.2e} e/DN \n") 

752 

753 a.set_xlabel(r'Mean signal ($\mu$, DN)', fontsize=labelFontSize) 

754 a.set_ylabel(r'Variance (DN$^2$)', fontsize=labelFontSize) 

755 a.tick_params(labelsize=11) 

756 a.set_xscale('linear', fontsize=labelFontSize) 

757 a.set_yscale('linear', fontsize=labelFontSize) 

758 

759 a2.set_xlabel(r'Mean Signal ($\mu$, DN)', fontsize=labelFontSize) 

760 a2.set_ylabel(r'Variance (DN$^2$)', fontsize=labelFontSize) 

761 a2.tick_params(labelsize=11) 

762 a2.set_xscale('log') 

763 a2.set_yscale('log') 

764 

765 if len(meanVecFinal): # Empty if the whole amp is bad, for example. 

766 minMeanVecFinal = np.min(meanVecFinal) 

767 maxMeanVecFinal = np.max(meanVecFinal) 

768 meanVecFit = np.linspace(minMeanVecFinal, maxMeanVecFinal, 100*len(meanVecFinal)) 

769 minMeanVecOriginal = np.min(meanVecOriginal) 

770 maxMeanVecOriginal = np.max(meanVecOriginal) 

771 deltaXlim = maxMeanVecOriginal - minMeanVecOriginal 

772 

773 a.plot(meanVecFit, ptcFunc(pars, meanVecFit), color='red') 

774 a.plot(meanVecFinal, pars[0] + pars[1]*meanVecFinal, color='green', linestyle='--') 

775 a.scatter(meanVecFinal, varVecFinal, c='blue', marker='o', s=markerSize) 

776 a.scatter(meanVecOutliers, varVecOutliers, c='magenta', marker='s', s=markerSize) 

777 a.text(0.03, 0.7, stringLegend, transform=a.transAxes, fontsize=legendFontSize) 

778 a.set_title(amp, fontsize=titleFontSize) 

779 a.set_xlim([minMeanVecOriginal - 0.2*deltaXlim, maxMeanVecOriginal + 0.2*deltaXlim]) 

780 

781 # Same, but in log-scale 

782 a2.plot(meanVecFit, ptcFunc(pars, meanVecFit), color='red') 

783 a2.scatter(meanVecFinal, varVecFinal, c='blue', marker='o', s=markerSize) 

784 a2.scatter(meanVecOutliers, varVecOutliers, c='magenta', marker='s', s=markerSize) 

785 a2.text(0.03, 0.7, stringLegend, transform=a2.transAxes, fontsize=legendFontSize) 

786 a2.set_title(amp, fontsize=titleFontSize) 

787 a2.set_xlim([minMeanVecOriginal, maxMeanVecOriginal]) 

788 else: 

789 a.set_title(f"{amp} (BAD)", fontsize=titleFontSize) 

790 a2.set_title(f"{amp} (BAD)", fontsize=titleFontSize) 

791 

792 f.suptitle("PTC \n Fit: " + stringTitle, fontsize=supTitleFontSize) 

793 pdfPages.savefig(f) 

794 f2.suptitle("PTC (log-log)", fontsize=supTitleFontSize) 

795 pdfPages.savefig(f2) 

796 

797 return 

798 

799 def _plotLinearizer(self, dataset, linearizer, pdfPages): 

800 """Plot linearity and linearity residual per amplifier 

801 

802 Parameters 

803 ---------- 

804 dataset : `lsst.cp.pipe.ptc.PhotonTransferCurveDataset` 

805 The dataset containing the means, variances, exposure times, and mask. 

806 

807 linearizer : `lsst.ip.isr.Linearizer` 

808 Linearizer object 

809 """ 

810 legendFontSize = 7 

811 labelFontSize = 7 

812 titleFontSize = 9 

813 supTitleFontSize = 18 

814 

815 # General determination of the size of the plot grid 

816 nAmps = len(dataset.ampNames) 

817 if nAmps == 2: 

818 nRows, nCols = 2, 1 

819 nRows = np.sqrt(nAmps) 

820 mantissa, _ = np.modf(nRows) 

821 if mantissa > 0: 

822 nRows = int(nRows) + 1 

823 nCols = nRows 

824 else: 

825 nRows = int(nRows) 

826 nCols = nRows 

827 

828 # Plot mean vs time (f1), and fractional residuals (f2) 

829 f, ax = plt.subplots(nrows=nRows, ncols=nCols, sharex='col', sharey='row', figsize=(13, 10)) 

830 f2, ax2 = plt.subplots(nrows=nRows, ncols=nCols, sharex='col', sharey='row', figsize=(13, 10)) 

831 for i, (amp, a, a2) in enumerate(zip(dataset.ampNames, ax.flatten(), ax2.flatten())): 

832 meanVecFinal = np.array(dataset.rawMeans[amp])[dataset.visitMask[amp]] 

833 timeVecFinal = np.array(dataset.rawExpTimes[amp])[dataset.visitMask[amp]] 

834 

835 a.set_xlabel('Time (sec)', fontsize=labelFontSize) 

836 a.set_ylabel(r'Mean signal ($\mu$, DN)', fontsize=labelFontSize) 

837 a.tick_params(labelsize=labelFontSize) 

838 a.set_xscale('linear', fontsize=labelFontSize) 

839 a.set_yscale('linear', fontsize=labelFontSize) 

840 

841 a2.axhline(y=0, color='k') 

842 a2.axvline(x=0, color='k', linestyle='-') 

843 a2.set_xlabel(r'Mean signal ($\mu$, DN)', fontsize=labelFontSize) 

844 a2.set_ylabel('Fractional nonlinearity (%)', fontsize=labelFontSize) 

845 a2.tick_params(labelsize=labelFontSize) 

846 a2.set_xscale('linear', fontsize=labelFontSize) 

847 a2.set_yscale('linear', fontsize=labelFontSize) 

848 

849 if len(meanVecFinal): 

850 pars, parsErr = linearizer.fitParams[amp], linearizer.fitParamsErr[amp] 

851 k0, k0Error = pars[0], parsErr[0] 

852 k1, k1Error = pars[1], parsErr[1] 

853 k2, k2Error = pars[2], parsErr[2] 

854 stringLegend = (f"k0: {k0:.4}+/-{k0Error:.2e} DN\n k1: {k1:.4}+/-{k1Error:.2e} DN/t" 

855 f"\n k2: {k2:.2e}+/-{k2Error:.2e} DN/t^2 \n") 

856 a.scatter(timeVecFinal, meanVecFinal) 

857 a.plot(timeVecFinal, funcPolynomial(pars, timeVecFinal), color='red') 

858 a.text(0.03, 0.75, stringLegend, transform=a.transAxes, fontsize=legendFontSize) 

859 a.set_title(f"{amp}", fontsize=titleFontSize) 

860 

861 linearPart = k0 + k1*timeVecFinal 

862 fracLinRes = 100*(linearPart - meanVecFinal)/linearPart 

863 a2.plot(meanVecFinal, fracLinRes, c='g') 

864 a2.set_title(f"{amp}", fontsize=titleFontSize) 

865 else: 

866 a.set_title(f"{amp} (BAD)", fontsize=titleFontSize) 

867 a2.set_title(f"{amp} (BAD)", fontsize=titleFontSize) 

868 

869 f.suptitle("Linearity \n Fit: Polynomial (degree: %g)" 

870 % (len(pars)-1), 

871 fontsize=supTitleFontSize) 

872 f2.suptitle(r"Fractional NL residual" + "\n" + 

873 r"$100\times \frac{(k_0 + k_1*Time-\mu)}{k_0+k_1*Time}$", 

874 fontsize=supTitleFontSize) 

875 pdfPages.savefig(f) 

876 pdfPages.savefig(f2) 

877 

878 @staticmethod 

879 def findGroups(x, maxDiff): 

880 """Group data into bins, with at most maxDiff distance between bins. 

881 

882 Parameters 

883 ---------- 

884 x: `list` 

885 Data to bin. 

886 

887 maxDiff: `int` 

888 Maximum distance between bins. 

889 

890 Returns 

891 ------- 

892 index: `list` 

893 Bin indices. 

894 """ 

895 ix = np.argsort(x) 

896 xsort = np.sort(x) 

897 index = np.zeros_like(x, dtype=np.int32) 

898 xc = xsort[0] 

899 group = 0 

900 ng = 1 

901 

902 for i in range(1, len(ix)): 

903 xval = xsort[i] 

904 if (xval - xc < maxDiff): 

905 xc = (ng*xc + xval)/(ng+1) 

906 ng += 1 

907 index[ix[i]] = group 

908 else: 

909 group += 1 

910 ng = 1 

911 index[ix[i]] = group 

912 xc = xval 

913 

914 return index 

915 

916 @staticmethod 

917 def indexForBins(x, nBins): 

918 """Builds an index with regular binning. The result can be fed into binData. 

919 

920 Parameters 

921 ---------- 

922 x: `numpy.array` 

923 Data to bin. 

924 nBins: `int` 

925 Number of bin. 

926 

927 Returns 

928 ------- 

929 np.digitize(x, bins): `numpy.array` 

930 Bin indices. 

931 """ 

932 

933 bins = np.linspace(x.min(), x.max() + abs(x.max() * 1e-7), nBins + 1) 

934 

935 return np.digitize(x, bins) 

936 

937 @staticmethod 

938 def binData(x, y, binIndex, wy=None): 

939 """Bin data (usually for display purposes). 

940 

941 Patrameters 

942 ----------- 

943 x: `numpy.array` 

944 Data to bin. 

945 

946 y: `numpy.array` 

947 Data to bin. 

948 

949 binIdex: `list` 

950 Bin number of each datum. 

951 

952 wy: `numpy.array` 

953 Inverse rms of each datum to use when averaging (the actual weight is wy**2). 

954 

955 Returns: 

956 ------- 

957 

958 xbin: `numpy.array` 

959 Binned data in x. 

960 

961 ybin: `numpy.array` 

962 Binned data in y. 

963 

964 wybin: `numpy.array` 

965 Binned weights in y, computed from wy's in each bin. 

966 

967 sybin: `numpy.array` 

968 Uncertainty on the bin average, considering actual scatter, and ignoring weights. 

969 """ 

970 

971 if wy is None: 

972 wy = np.ones_like(x) 

973 binIndexSet = set(binIndex) 

974 w2 = wy*wy 

975 xw2 = x*(w2) 

976 xbin = np.array([xw2[binIndex == i].sum()/w2[binIndex == i].sum() for i in binIndexSet]) 

977 

978 yw2 = y*w2 

979 ybin = np.array([yw2[binIndex == i].sum()/w2[binIndex == i].sum() for i in binIndexSet]) 

980 

981 wybin = np.sqrt(np.array([w2[binIndex == i].sum() for i in binIndexSet])) 

982 sybin = np.array([y[binIndex == i].std()/np.sqrt(np.array([binIndex == i]).sum()) 

983 for i in binIndexSet]) 

984 

985 return xbin, ybin, wybin, sybin