Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of cp_pipe. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21# 

22 

23__all__ = ['MeasurePhotonTransferCurveTask', 

24 'MeasurePhotonTransferCurveTaskConfig', 

25 'PhotonTransferCurveDataset'] 

26 

27import numpy as np 

28import matplotlib.pyplot as plt 

29import os 

30from matplotlib.backends.backend_pdf import PdfPages 

31from sqlite3 import OperationalError 

32from collections import Counter 

33from dataclasses import dataclass 

34 

35import lsst.afw.math as afwMath 

36import lsst.pex.config as pexConfig 

37import lsst.pipe.base as pipeBase 

38from lsst.ip.isr import IsrTask 

39from .utils import (NonexistentDatasetTaskDataIdContainer, PairedVisitListTaskRunner, 

40 checkExpLengthEqual, validateIsrConfig) 

41from scipy.optimize import leastsq, least_squares 

42import numpy.polynomial.polynomial as poly 

43 

44from lsst.ip.isr.linearize import Linearizer 

45import datetime 

46 

47 

48class MeasurePhotonTransferCurveTaskConfig(pexConfig.Config): 

49 """Config class for photon transfer curve measurement task""" 

50 isr = pexConfig.ConfigurableField( 

51 target=IsrTask, 

52 doc="""Task to perform instrumental signature removal.""", 

53 ) 

54 isrMandatorySteps = pexConfig.ListField( 

55 dtype=str, 

56 doc="isr operations that must be performed for valid results. Raises if any of these are False.", 

57 default=['doAssembleCcd'] 

58 ) 

59 isrForbiddenSteps = pexConfig.ListField( 

60 dtype=str, 

61 doc="isr operations that must NOT be performed for valid results. Raises if any of these are True", 

62 default=['doFlat', 'doFringe', 'doBrighterFatter', 'doUseOpticsTransmission', 

63 'doUseFilterTransmission', 'doUseSensorTransmission', 'doUseAtmosphereTransmission'] 

64 ) 

65 isrDesirableSteps = pexConfig.ListField( 

66 dtype=str, 

67 doc="isr operations that it is advisable to perform, but are not mission-critical." + 

68 " WARNs are logged for any of these found to be False.", 

69 default=['doBias', 'doDark', 'doCrosstalk', 'doDefect'] 

70 ) 

71 isrUndesirableSteps = pexConfig.ListField( 

72 dtype=str, 

73 doc="isr operations that it is *not* advisable to perform in the general case, but are not" + 

74 " forbidden as some use-cases might warrant them." + 

75 " WARNs are logged for any of these found to be True.", 

76 default=['doLinearize'] 

77 ) 

78 ccdKey = pexConfig.Field( 

79 dtype=str, 

80 doc="The key by which to pull a detector from a dataId, e.g. 'ccd' or 'detector'.", 

81 default='ccd', 

82 ) 

83 makePlots = pexConfig.Field( 

84 dtype=bool, 

85 doc="Plot the PTC curves?", 

86 default=False, 

87 ) 

88 ptcFitType = pexConfig.ChoiceField( 

89 dtype=str, 

90 doc="Fit PTC to approximation in Astier+19 (Equation 16) or to a polynomial.", 

91 default="POLYNOMIAL", 

92 allowed={ 

93 "POLYNOMIAL": "n-degree polynomial (use 'polynomialFitDegree' to set 'n').", 

94 "ASTIERAPPROXIMATION": "Approximation in Astier+19 (Eq. 16)." 

95 } 

96 ) 

97 polynomialFitDegree = pexConfig.Field( 

98 dtype=int, 

99 doc="Degree of polynomial to fit the PTC, when 'ptcFitType'=POLYNOMIAL.", 

100 default=2, 

101 ) 

102 polynomialFitDegreeNonLinearity = pexConfig.Field( 

103 dtype=int, 

104 doc="Degree of polynomial to fit the meanSignal vs exposureTime curve to produce" + 

105 " the table for LinearizeLookupTable.", 

106 default=3, 

107 ) 

108 binSize = pexConfig.Field( 

109 dtype=int, 

110 doc="Bin the image by this factor in both dimensions.", 

111 default=1, 

112 ) 

113 minMeanSignal = pexConfig.Field( 

114 dtype=float, 

115 doc="Minimum value (inclusive) of mean signal (in DN) above which to consider.", 

116 default=0, 

117 ) 

118 maxMeanSignal = pexConfig.Field( 

119 dtype=float, 

120 doc="Maximum value (inclusive) of mean signal (in DN) below which to consider.", 

121 default=9e6, 

122 ) 

123 initialNonLinearityExclusionThresholdPositive = pexConfig.RangeField( 

124 dtype=float, 

125 doc="Initially exclude data points with a variance that are more than a factor of this from being" 

126 " linear in the positive direction, from the PTC fit. Note that these points will also be" 

127 " excluded from the non-linearity fit. This is done before the iterative outlier rejection," 

128 " to allow an accurate determination of the sigmas for said iterative fit.", 

129 default=0.12, 

130 min=0.0, 

131 max=1.0, 

132 ) 

133 initialNonLinearityExclusionThresholdNegative = pexConfig.RangeField( 

134 dtype=float, 

135 doc="Initially exclude data points with a variance that are more than a factor of this from being" 

136 " linear in the negative direction, from the PTC fit. Note that these points will also be" 

137 " excluded from the non-linearity fit. This is done before the iterative outlier rejection," 

138 " to allow an accurate determination of the sigmas for said iterative fit.", 

139 default=0.25, 

140 min=0.0, 

141 max=1.0, 

142 ) 

143 sigmaCutPtcOutliers = pexConfig.Field( 

144 dtype=float, 

145 doc="Sigma cut for outlier rejection in PTC.", 

146 default=5.0, 

147 ) 

148 maxIterationsPtcOutliers = pexConfig.Field( 

149 dtype=int, 

150 doc="Maximum number of iterations for outlier rejection in PTC.", 

151 default=2, 

152 ) 

153 doFitBootstrap = pexConfig.Field( 

154 dtype=bool, 

155 doc="Use bootstrap for the PTC fit parameters and errors?.", 

156 default=False, 

157 ) 

158 linResidualTimeIndex = pexConfig.Field( 

159 dtype=int, 

160 doc="Index position in time array for reference time in linearity residual calculation.", 

161 default=2, 

162 ) 

163 maxAduForLookupTableLinearizer = pexConfig.Field( 

164 dtype=int, 

165 doc="Maximum DN value for the LookupTable linearizer.", 

166 default=2**18, 

167 ) 

168 instrumentName = pexConfig.Field( 

169 dtype=str, 

170 doc="Instrument name.", 

171 default='', 

172 ) 

173 

174 

175@dataclass 

176class LinearityResidualsAndLinearizersDataset: 

177 """A simple class to hold the output from the 

178 `calculateLinearityResidualAndLinearizers` function. 

179 """ 

180 # Normalized coefficients for polynomial NL correction 

181 polynomialLinearizerCoefficients: list 

182 # Normalized coefficient for quadratic polynomial NL correction (c0) 

183 quadraticPolynomialLinearizerCoefficient: float 

184 # LUT array row for the amplifier at hand 

185 linearizerTableRow: list 

186 # Linearity residual, Eq. 2.2. of Janesick (2001) 

187 linearityResidual: list 

188 meanSignalVsTimePolyFitPars: list 

189 meanSignalVsTimePolyFitParsErr: list 

190 fractionalNonLinearityResidual: list 

191 meanSignalVsTimePolyFitReducedChiSq: float 

192 

193 

194class PhotonTransferCurveDataset: 

195 """A simple class to hold the output data from the PTC task. 

196 

197 The dataset is made up of a dictionary for each item, keyed by the 

198 amplifiers' names, which much be supplied at construction time. 

199 

200 New items cannot be added to the class to save accidentally saving to the 

201 wrong property, and the class can be frozen if desired. 

202 

203 inputVisitPairs records the visits used to produce the data. 

204 When fitPtcAndNonLinearity() is run, a mask is built up, which is by definition 

205 always the same length as inputVisitPairs, rawExpTimes, rawMeans 

206 and rawVars, and is a list of bools, which are incrementally set to False 

207 as points are discarded from the fits. 

208 

209 PTC fit parameters for polynomials are stored in a list in ascending order 

210 of polynomial term, i.e. par[0]*x^0 + par[1]*x + par[2]*x^2 etc 

211 with the length of the list corresponding to the order of the polynomial 

212 plus one. 

213 """ 

214 def __init__(self, ampNames): 

215 # add items to __dict__ directly because __setattr__ is overridden 

216 

217 # instance variables 

218 self.__dict__["ampNames"] = ampNames 

219 self.__dict__["badAmps"] = [] 

220 

221 # raw data variables 

222 self.__dict__["inputVisitPairs"] = {ampName: [] for ampName in ampNames} 

223 self.__dict__["visitMask"] = {ampName: [] for ampName in ampNames} 

224 self.__dict__["rawExpTimes"] = {ampName: [] for ampName in ampNames} 

225 self.__dict__["rawMeans"] = {ampName: [] for ampName in ampNames} 

226 self.__dict__["rawVars"] = {ampName: [] for ampName in ampNames} 

227 

228 # fit information 

229 self.__dict__["ptcFitType"] = {ampName: "" for ampName in ampNames} 

230 self.__dict__["ptcFitPars"] = {ampName: [] for ampName in ampNames} 

231 self.__dict__["ptcFitParsError"] = {ampName: [] for ampName in ampNames} 

232 self.__dict__["ptcFitReducedChiSquared"] = {ampName: [] for ampName in ampNames} 

233 self.__dict__["nonLinearity"] = {ampName: [] for ampName in ampNames} 

234 self.__dict__["nonLinearityError"] = {ampName: [] for ampName in ampNames} 

235 self.__dict__["nonLinearityResiduals"] = {ampName: [] for ampName in ampNames} 

236 self.__dict__["fractionalNonLinearityResiduals"] = {ampName: [] for ampName in ampNames} 

237 self.__dict__["nonLinearityReducedChiSquared"] = {ampName: [] for ampName in ampNames} 

238 

239 # final results 

240 self.__dict__["gain"] = {ampName: -1. for ampName in ampNames} 

241 self.__dict__["gainErr"] = {ampName: -1. for ampName in ampNames} 

242 self.__dict__["noise"] = {ampName: -1. for ampName in ampNames} 

243 self.__dict__["noiseErr"] = {ampName: -1. for ampName in ampNames} 

244 self.__dict__["coefficientsLinearizePolynomial"] = {ampName: [] for ampName in ampNames} 

245 self.__dict__["coefficientLinearizeSquared"] = {ampName: [] for ampName in ampNames} 

246 

247 def __setattr__(self, attribute, value): 

248 """Protect class attributes""" 

249 if attribute not in self.__dict__: 

250 raise AttributeError(f"{attribute} is not already a member of PhotonTransferCurveDataset, which" 

251 " does not support setting of new attributes.") 

252 else: 

253 self.__dict__[attribute] = value 

254 

255 def getVisitsUsed(self, ampName): 

256 """Get the visits used, i.e. not discarded, for a given amp. 

257 

258 If no mask has been created yet, all visits are returned. 

259 """ 

260 if self.visitMask[ampName] == []: 

261 return self.inputVisitPairs[ampName] 

262 

263 # if the mask exists it had better be the same length as the visitPairs 

264 assert len(self.visitMask[ampName]) == len(self.inputVisitPairs[ampName]) 

265 

266 pairs = self.inputVisitPairs[ampName] 

267 mask = self.visitMask[ampName] 

268 # cast to bool required because numpy 

269 return [(v1, v2) for ((v1, v2), m) in zip(pairs, mask) if bool(m) is True] 

270 

271 def getGoodAmps(self): 

272 return [amp for amp in self.ampNames if amp not in self.badAmps] 

273 

274 

275class MeasurePhotonTransferCurveTask(pipeBase.CmdLineTask): 

276 """A class to calculate, fit, and plot a PTC from a set of flat pairs. 

277 

278 The Photon Transfer Curve (var(signal) vs mean(signal)) is a standard tool 

279 used in astronomical detectors characterization (e.g., Janesick 2001, 

280 Janesick 2007). This task calculates the PTC from a series of pairs of 

281 flat-field images; each pair taken at identical exposure times. The 

282 difference image of each pair is formed to eliminate fixed pattern noise, 

283 and then the variance of the difference image and the mean of the average image 

284 are used to produce the PTC. An n-degree polynomial or the approximation in Equation 

285 16 of Astier+19 ("The Shape of the Photon Transfer Curve of CCD sensors", 

286 arXiv:1905.08677) can be fitted to the PTC curve. These models include 

287 parameters such as the gain (e/DN) and readout noise. 

288 

289 Linearizers to correct for signal-chain non-linearity are also calculated. 

290 The `Linearizer` class, in general, can support per-amp linearizers, but in this 

291 task this is not supported. 

292 Parameters 

293 ---------- 

294 

295 *args: `list` 

296 Positional arguments passed to the Task constructor. None used at this 

297 time. 

298 **kwargs: `dict` 

299 Keyword arguments passed on to the Task constructor. None used at this 

300 time. 

301 

302 """ 

303 

304 RunnerClass = PairedVisitListTaskRunner 

305 ConfigClass = MeasurePhotonTransferCurveTaskConfig 

306 _DefaultName = "measurePhotonTransferCurve" 

307 

308 def __init__(self, *args, **kwargs): 

309 pipeBase.CmdLineTask.__init__(self, *args, **kwargs) 

310 self.makeSubtask("isr") 

311 plt.interactive(False) # stop windows popping up when plotting. When headless, use 'agg' backend too 

312 validateIsrConfig(self.isr, self.config.isrMandatorySteps, 

313 self.config.isrForbiddenSteps, self.config.isrDesirableSteps, checkTrim=False) 

314 self.config.validate() 

315 self.config.freeze() 

316 

317 @classmethod 

318 def _makeArgumentParser(cls): 

319 """Augment argument parser for the MeasurePhotonTransferCurveTask.""" 

320 parser = pipeBase.ArgumentParser(name=cls._DefaultName) 

321 parser.add_argument("--visit-pairs", dest="visitPairs", nargs="*", 

322 help="Visit pairs to use. Each pair must be of the form INT,INT e.g. 123,456") 

323 parser.add_id_argument("--id", datasetType="photonTransferCurveDataset", 

324 ContainerClass=NonexistentDatasetTaskDataIdContainer, 

325 help="The ccds to use, e.g. --id ccd=0..100") 

326 return parser 

327 

328 @pipeBase.timeMethod 

329 def runDataRef(self, dataRef, visitPairs): 

330 """Run the Photon Transfer Curve (PTC) measurement task. 

331 

332 For a dataRef (which is each detector here), 

333 and given a list of visit pairs at different exposure times, 

334 measure the PTC. 

335 

336 Parameters 

337 ---------- 

338 dataRef : list of lsst.daf.persistence.ButlerDataRef 

339 dataRef for the detector for the visits to be fit. 

340 visitPairs : `iterable` of `tuple` of `int` 

341 Pairs of visit numbers to be processed together 

342 """ 

343 

344 # setup necessary objects 

345 detNum = dataRef.dataId[self.config.ccdKey] 

346 detector = dataRef.get('camera')[dataRef.dataId[self.config.ccdKey]] 

347 # expand some missing fields that we need for lsstCam. This is a work-around 

348 # for Gen2 problems that I (RHL) don't feel like solving. The calibs pipelines 

349 # (which inherit from CalibTask) use addMissingKeys() to do basically the same thing 

350 # 

351 # Basically, the butler's trying to look up the fields in `raw_visit` which won't work 

352 for name in dataRef.getButler().getKeys('bias'): 

353 if name not in dataRef.dataId: 

354 try: 

355 dataRef.dataId[name] = \ 

356 dataRef.getButler().queryMetadata('raw', [name], detector=detNum)[0] 

357 except OperationalError: 

358 pass 

359 

360 amps = detector.getAmplifiers() 

361 ampNames = [amp.getName() for amp in amps] 

362 dataset = PhotonTransferCurveDataset(ampNames) 

363 

364 self.log.info('Measuring PTC using %s visits for detector %s' % (visitPairs, detNum)) 

365 

366 for (v1, v2) in visitPairs: 

367 # Perform ISR on each exposure 

368 dataRef.dataId['expId'] = v1 

369 exp1 = self.isr.runDataRef(dataRef).exposure 

370 dataRef.dataId['expId'] = v2 

371 exp2 = self.isr.runDataRef(dataRef).exposure 

372 del dataRef.dataId['expId'] 

373 

374 checkExpLengthEqual(exp1, exp2, v1, v2, raiseWithMessage=True) 

375 expTime = exp1.getInfo().getVisitInfo().getExposureTime() 

376 

377 for amp in detector: 

378 mu, varDiff = self.measureMeanVarPair(exp1, exp2, region=amp.getBBox()) 

379 ampName = amp.getName() 

380 

381 dataset.rawExpTimes[ampName].append(expTime) 

382 dataset.rawMeans[ampName].append(mu) 

383 dataset.rawVars[ampName].append(varDiff) 

384 dataset.inputVisitPairs[ampName].append((v1, v2)) 

385 

386 numberAmps = len(detector.getAmplifiers()) 

387 numberAduValues = self.config.maxAduForLookupTableLinearizer 

388 lookupTableArray = np.zeros((numberAmps, numberAduValues), dtype=np.float32) 

389 

390 # Fit PTC and (non)linearity of signal vs time curve. 

391 # Fill up PhotonTransferCurveDataset object. 

392 # Fill up array for LUT linearizer. 

393 # Produce coefficients for Polynomial ans Squared linearizers. 

394 dataset = self.fitPtcAndNonLinearity(dataset, self.config.ptcFitType, 

395 tableArray=lookupTableArray) 

396 

397 if self.config.makePlots: 

398 self.plot(dataRef, dataset, ptcFitType=self.config.ptcFitType) 

399 

400 # Save data, PTC fit, and NL fit dictionaries 

401 self.log.info(f"Writing PTC and NL data to {dataRef.getUri(write=True)}") 

402 dataRef.put(dataset, datasetType="photonTransferCurveDataset") 

403 

404 butler = dataRef.getButler() 

405 self.log.info(f"Writing linearizers: \n " 

406 "lookup table (linear component of polynomial fit), \n " 

407 "polynomial (coefficients for a polynomial correction), \n " 

408 "and squared linearizer (quadratic coefficient from polynomial)") 

409 

410 detName = detector.getName() 

411 now = datetime.datetime.utcnow() 

412 calibDate = now.strftime("%Y-%m-%d") 

413 

414 for linType, dataType in [("LOOKUPTABLE", 'linearizeLut'), 

415 ("LINEARIZEPOLYNOMIAL", 'linearizePolynomial'), 

416 ("LINEARIZESQUARED", 'linearizeSquared')]: 

417 

418 if linType == "LOOKUPTABLE": 

419 tableArray = lookupTableArray 

420 else: 

421 tableArray = None 

422 

423 linearizer = self.buildLinearizerObject(dataset, detector, calibDate, linType, 

424 instruName=self.config.instrumentName, 

425 tableArray=tableArray, 

426 log=self.log) 

427 butler.put(linearizer, datasetType=dataType, dataId={'detector': detNum, 

428 'detectorName': detName, 'calibDate': calibDate}) 

429 

430 self.log.info('Finished measuring PTC for in detector %s' % detNum) 

431 

432 return pipeBase.Struct(exitStatus=0) 

433 

434 def buildLinearizerObject(self, dataset, detector, calibDate, linearizerType, instruName='', 

435 tableArray=None, log=None): 

436 """Build linearizer object to persist. 

437 

438 Parameters 

439 ---------- 

440 dataset : `lsst.cp.pipe.ptc.PhotonTransferCurveDataset` 

441 The dataset containing the means, variances, and exposure times 

442 detector : `lsst.afw.cameraGeom.Detector` 

443 Detector object 

444 calibDate : `datetime.datetime` 

445 Calibration date 

446 linearizerType : `str` 

447 'LOOKUPTABLE', 'LINEARIZESQUARED', or 'LINEARIZEPOLYNOMIAL' 

448 instruName : `str`, optional 

449 Instrument name 

450 tableArray : `np.array`, optional 

451 Look-up table array with size rows=nAmps and columns=DN values 

452 log : `lsst.log.Log`, optional 

453 Logger to handle messages 

454 

455 Returns 

456 ------- 

457 linearizer : `lsst.ip.isr.Linearizer` 

458 Linearizer object 

459 """ 

460 detName = detector.getName() 

461 detNum = detector.getId() 

462 if linearizerType == "LOOKUPTABLE": 

463 if tableArray is not None: 

464 linearizer = Linearizer(detector=detector, table=tableArray, log=log) 

465 else: 

466 raise RuntimeError("tableArray must be provided when creating a LookupTable linearizer") 

467 elif linearizerType in ("LINEARIZESQUARED", "LINEARIZEPOLYNOMIAL"): 

468 linearizer = Linearizer(log=log) 

469 else: 

470 raise RuntimeError("Invalid linearizerType {linearizerType} to build a Linearizer object. " 

471 "Supported: 'LOOKUPTABLE', 'LINEARIZESQUARED', or 'LINEARIZEPOLYNOMIAL'") 

472 for i, amp in enumerate(detector.getAmplifiers()): 

473 ampName = amp.getName() 

474 if linearizerType == "LOOKUPTABLE": 

475 linearizer.linearityCoeffs[ampName] = [i, 0] 

476 linearizer.linearityType[ampName] = "LookupTable" 

477 elif linearizerType == "LINEARIZESQUARED": 

478 linearizer.linearityCoeffs[ampName] = [dataset.coefficientLinearizeSquared[ampName]] 

479 linearizer.linearityType[ampName] = "Squared" 

480 elif linearizerType == "LINEARIZEPOLYNOMIAL": 

481 linearizer.linearityCoeffs[ampName] = dataset.coefficientsLinearizePolynomial[ampName] 

482 linearizer.linearityType[ampName] = "Polynomial" 

483 linearizer.linearityBBox[ampName] = amp.getBBox() 

484 

485 linearizer.validate() 

486 calibId = f"detectorName={detName} detector={detNum} calibDate={calibDate} ccd={detNum} filter=NONE" 

487 

488 try: 

489 raftName = detName.split("_")[0] 

490 calibId += f" raftName={raftName}" 

491 except Exception: 

492 raftname = "NONE" 

493 calibId += f" raftName={raftname}" 

494 

495 serial = detector.getSerial() 

496 linearizer.updateMetadata(instrumentName=instruName, detectorId=f"{detNum}", 

497 calibId=calibId, serial=serial, detectorName=f"{detName}") 

498 

499 return linearizer 

500 

501 def measureMeanVarPair(self, exposure1, exposure2, region=None): 

502 """Calculate the mean signal of two exposures and the variance of their difference. 

503 

504 Parameters 

505 ---------- 

506 exposure1 : `lsst.afw.image.exposure.exposure.ExposureF` 

507 First exposure of flat field pair. 

508 

509 exposure2 : `lsst.afw.image.exposure.exposure.ExposureF` 

510 Second exposure of flat field pair. 

511 

512 region : `lsst.geom.Box2I` 

513 Region of each exposure where to perform the calculations (e.g, an amplifier). 

514 

515 Return 

516 ------ 

517 

518 mu : `float` 

519 0.5*(mu1 + mu2), where mu1, and mu2 are the clipped means of the regions in 

520 both exposures. 

521 

522 varDiff : `float` 

523 Half of the clipped variance of the difference of the regions inthe two input 

524 exposures. 

525 """ 

526 

527 if region is not None: 

528 im1Area = exposure1.maskedImage[region] 

529 im2Area = exposure2.maskedImage[region] 

530 else: 

531 im1Area = exposure1.maskedImage 

532 im2Area = exposure2.maskedImage 

533 

534 im1Area = afwMath.binImage(im1Area, self.config.binSize) 

535 im2Area = afwMath.binImage(im2Area, self.config.binSize) 

536 

537 # Clipped mean of images; then average of mean. 

538 mu1 = afwMath.makeStatistics(im1Area, afwMath.MEANCLIP).getValue() 

539 mu2 = afwMath.makeStatistics(im2Area, afwMath.MEANCLIP).getValue() 

540 mu = 0.5*(mu1 + mu2) 

541 

542 # Take difference of pairs 

543 # symmetric formula: diff = (mu2*im1-mu1*im2)/(0.5*(mu1+mu2)) 

544 temp = im2Area.clone() 

545 temp *= mu1 

546 diffIm = im1Area.clone() 

547 diffIm *= mu2 

548 diffIm -= temp 

549 diffIm /= mu 

550 

551 varDiff = 0.5*(afwMath.makeStatistics(diffIm, afwMath.VARIANCECLIP).getValue()) 

552 

553 return mu, varDiff 

554 

555 def _fitLeastSq(self, initialParams, dataX, dataY, function): 

556 """Do a fit and estimate the parameter errors using using scipy.optimize.leastq. 

557 

558 optimize.leastsq returns the fractional covariance matrix. To estimate the 

559 standard deviation of the fit parameters, multiply the entries of this matrix 

560 by the unweighted reduced chi squared and take the square root of the diagonal elements. 

561 

562 Parameters 

563 ---------- 

564 initialParams : `list` of `float` 

565 initial values for fit parameters. For ptcFitType=POLYNOMIAL, its length 

566 determines the degree of the polynomial. 

567 

568 dataX : `numpy.array` of `float` 

569 Data in the abscissa axis. 

570 

571 dataY : `numpy.array` of `float` 

572 Data in the ordinate axis. 

573 

574 function : callable object (function) 

575 Function to fit the data with. 

576 

577 Return 

578 ------ 

579 pFitSingleLeastSquares : `list` of `float` 

580 List with fitted parameters. 

581 

582 pErrSingleLeastSquares : `list` of `float` 

583 List with errors for fitted parameters. 

584 

585 reducedChiSqSingleLeastSquares : `float` 

586 Unweighted reduced chi squared 

587 """ 

588 

589 def errFunc(p, x, y): 

590 return function(p, x) - y 

591 

592 pFit, pCov, infoDict, errMessage, success = leastsq(errFunc, initialParams, 

593 args=(dataX, dataY), full_output=1, epsfcn=0.0001) 

594 

595 if (len(dataY) > len(initialParams)) and pCov is not None: 

596 reducedChiSq = (errFunc(pFit, dataX, dataY)**2).sum()/(len(dataY)-len(initialParams)) 

597 pCov *= reducedChiSq 

598 else: 

599 pCov = np.zeros((len(initialParams), len(initialParams))) 

600 pCov[:, :] = np.inf 

601 reducedChiSq = np.inf 

602 

603 errorVec = [] 

604 for i in range(len(pFit)): 

605 errorVec.append(np.fabs(pCov[i][i])**0.5) 

606 

607 pFitSingleLeastSquares = pFit 

608 pErrSingleLeastSquares = np.array(errorVec) 

609 

610 return pFitSingleLeastSquares, pErrSingleLeastSquares, reducedChiSq 

611 

612 def _fitBootstrap(self, initialParams, dataX, dataY, function, confidenceSigma=1.): 

613 """Do a fit using least squares and bootstrap to estimate parameter errors. 

614 

615 The bootstrap error bars are calculated by fitting 100 random data sets. 

616 

617 Parameters 

618 ---------- 

619 initialParams : `list` of `float` 

620 initial values for fit parameters. For ptcFitType=POLYNOMIAL, its length 

621 determines the degree of the polynomial. 

622 

623 dataX : `numpy.array` of `float` 

624 Data in the abscissa axis. 

625 

626 dataY : `numpy.array` of `float` 

627 Data in the ordinate axis. 

628 

629 function : callable object (function) 

630 Function to fit the data with. 

631 

632 confidenceSigma : `float` 

633 Number of sigmas that determine confidence interval for the bootstrap errors. 

634 

635 Return 

636 ------ 

637 pFitBootstrap : `list` of `float` 

638 List with fitted parameters. 

639 

640 pErrBootstrap : `list` of `float` 

641 List with errors for fitted parameters. 

642 

643 reducedChiSqBootstrap : `float` 

644 Reduced chi squared. 

645 """ 

646 

647 def errFunc(p, x, y): 

648 return function(p, x) - y 

649 

650 # Fit first time 

651 pFit, _ = leastsq(errFunc, initialParams, args=(dataX, dataY), full_output=0) 

652 

653 # Get the stdev of the residuals 

654 residuals = errFunc(pFit, dataX, dataY) 

655 sigmaErrTotal = np.std(residuals) 

656 

657 # 100 random data sets are generated and fitted 

658 pars = [] 

659 for i in range(100): 

660 randomDelta = np.random.normal(0., sigmaErrTotal, len(dataY)) 

661 randomDataY = dataY + randomDelta 

662 randomFit, _ = leastsq(errFunc, initialParams, 

663 args=(dataX, randomDataY), full_output=0) 

664 pars.append(randomFit) 

665 pars = np.array(pars) 

666 meanPfit = np.mean(pars, 0) 

667 

668 # confidence interval for parameter estimates 

669 nSigma = confidenceSigma 

670 errPfit = nSigma*np.std(pars, 0) 

671 pFitBootstrap = meanPfit 

672 pErrBootstrap = errPfit 

673 

674 reducedChiSq = (errFunc(pFitBootstrap, dataX, dataY)**2).sum()/(len(dataY)-len(initialParams)) 

675 return pFitBootstrap, pErrBootstrap, reducedChiSq 

676 

677 def funcPolynomial(self, pars, x): 

678 """Polynomial function definition""" 

679 return poly.polyval(x, [*pars]) 

680 

681 def funcAstier(self, pars, x): 

682 """Single brighter-fatter parameter model for PTC; Equation 16 of Astier+19""" 

683 a00, gain, noise = pars 

684 return 0.5/(a00*gain*gain)*(np.exp(2*a00*x*gain)-1) + noise/(gain*gain) 

685 

686 @staticmethod 

687 def _initialParsForPolynomial(order): 

688 assert(order >= 2) 

689 pars = np.zeros(order, dtype=np.float) 

690 pars[0] = 10 

691 pars[1] = 1 

692 pars[2:] = 0.0001 

693 return pars 

694 

695 @staticmethod 

696 def _boundsForPolynomial(initialPars): 

697 lowers = [np.NINF for p in initialPars] 

698 uppers = [np.inf for p in initialPars] 

699 lowers[1] = 0 # no negative gains 

700 return (lowers, uppers) 

701 

702 @staticmethod 

703 def _boundsForAstier(initialPars): 

704 lowers = [np.NINF for p in initialPars] 

705 uppers = [np.inf for p in initialPars] 

706 return (lowers, uppers) 

707 

708 @staticmethod 

709 def _getInitialGoodPoints(means, variances, maxDeviationPositive, maxDeviationNegative): 

710 """Return a boolean array to mask bad points. 

711 

712 A linear function has a constant ratio, so find the median 

713 value of the ratios, and exclude the points that deviate 

714 from that by more than a factor of maxDeviationPositive/negative. 

715 Asymmetric deviations are supported as we expect the PTC to turn 

716 down as the flux increases, but sometimes it anomalously turns 

717 upwards just before turning over, which ruins the fits, so it 

718 is wise to be stricter about restricting positive outliers than 

719 negative ones. 

720 

721 Too high and points that are so bad that fit will fail will be included 

722 Too low and the non-linear points will be excluded, biasing the NL fit.""" 

723 ratios = [b/a for (a, b) in zip(means, variances)] 

724 medianRatio = np.median(ratios) 

725 ratioDeviations = [(r/medianRatio)-1 for r in ratios] 

726 

727 # so that it doesn't matter if the deviation is expressed as positive or negative 

728 maxDeviationPositive = abs(maxDeviationPositive) 

729 maxDeviationNegative = -1. * abs(maxDeviationNegative) 

730 

731 goodPoints = np.array([True if (r < maxDeviationPositive and r > maxDeviationNegative) 

732 else False for r in ratioDeviations]) 

733 return goodPoints 

734 

735 def _makeZeroSafe(self, array, warn=True, substituteValue=1e-9): 

736 """""" 

737 nBad = Counter(array)[0] 

738 if nBad == 0: 

739 return array 

740 

741 if warn: 

742 msg = f"Found {nBad} zeros in array at elements {[x for x in np.where(array==0)[0]]}" 

743 self.log.warn(msg) 

744 

745 array[array == 0] = substituteValue 

746 return array 

747 

748 def calculateLinearityResidualAndLinearizers(self, exposureTimeVector, meanSignalVector): 

749 """Calculate linearity residual and fit an n-order polynomial to the mean vs time curve 

750 to produce corrections (deviation from linear part of polynomial) for a particular amplifier 

751 to populate LinearizeLookupTable. 

752 Use the coefficients of this fit to calculate the correction coefficients for LinearizePolynomial 

753 and LinearizeSquared." 

754 

755 Parameters 

756 --------- 

757 

758 exposureTimeVector: `list` of `float` 

759 List of exposure times for each flat pair 

760 

761 meanSignalVector: `list` of `float` 

762 List of mean signal from diference image of flat pairs 

763 

764 Returns 

765 ------- 

766 dataset : `lsst.cp.pipe.ptc.LinearityResidualsAndLinearizersDataset` 

767 The dataset containing the fit parameters, the NL correction coefficients, and the 

768 LUT row for the amplifier at hand. Explicitly: 

769 

770 dataset.polynomialLinearizerCoefficients : `list` of `float` 

771 Coefficients for LinearizePolynomial, where corrImage = uncorrImage + sum_i c_i uncorrImage^(2 + 

772 i). 

773 c_(j-2) = -k_j/(k_1^j) with units (DN^(1-j)). The units of k_j are DN/t^j, and they are fit from 

774 meanSignalVector = k0 + k1*exposureTimeVector + k2*exposureTimeVector^2 +... 

775 + kn*exposureTimeVector^n, with n = "polynomialFitDegreeNonLinearity". 

776 k_0 and k_1 and degenerate with bias level and gain, and are not used by the non-linearity 

777 correction. Therefore, j = 2...n in the above expression (see `LinearizePolynomial` class in 

778 `linearize.py`.) 

779 

780 dataset.quadraticPolynomialLinearizerCoefficient : `float` 

781 Coefficient for LinearizeSquared, where corrImage = uncorrImage + c0*uncorrImage^2. 

782 c0 = -k2/(k1^2), where k1 and k2 are fit from 

783 meanSignalVector = k0 + k1*exposureTimeVector + k2*exposureTimeVector^2 +... 

784 + kn*exposureTimeVector^n, with n = "polynomialFitDegreeNonLinearity". 

785 

786 dataset.linearizerTableRow : `list` of `float` 

787 One dimensional array with deviation from linear part of n-order polynomial fit 

788 to mean vs time curve. This array will be one row (for the particular amplifier at hand) 

789 of the table array for LinearizeLookupTable. 

790 

791 dataset.linearityResidual : `list` of `float` 

792 Linearity residual from the mean vs time curve, defined as 

793 100*(1 - meanSignalReference/expTimeReference/(meanSignal/expTime). 

794 

795 dataset.meanSignalVsTimePolyFitPars : `list` of `float` 

796 Parameters from n-order polynomial fit to meanSignalVector vs exposureTimeVector. 

797 

798 dataset.meanSignalVsTimePolyFitParsErr : `list` of `float` 

799 Parameters from n-order polynomial fit to meanSignalVector vs exposureTimeVector. 

800 

801 dataset.fractionalNonLinearityResidual : `list` of `float` 

802 Fractional residuals from the meanSignal vs exposureTime curve with respect to linear part of 

803 polynomial fit: 100*(linearPart - meanSignal)/linearPart, where 

804 linearPart = k0 + k1*exposureTimeVector. 

805 

806 dataset.meanSignalVsTimePolyFitReducedChiSq : `float` 

807 Reduced unweighted chi squared from polynomial fit to meanSignalVector vs exposureTimeVector. 

808 """ 

809 

810 # Lookup table linearizer 

811 parsIniNonLinearity = self._initialParsForPolynomial(self.config.polynomialFitDegreeNonLinearity + 1) 

812 if self.config.doFitBootstrap: 

813 parsFit, parsFitErr, reducedChiSquaredNonLinearityFit = self._fitBootstrap(parsIniNonLinearity, 

814 exposureTimeVector, 

815 meanSignalVector, 

816 self.funcPolynomial) 

817 else: 

818 parsFit, parsFitErr, reducedChiSquaredNonLinearityFit = self._fitLeastSq(parsIniNonLinearity, 

819 exposureTimeVector, 

820 meanSignalVector, 

821 self.funcPolynomial) 

822 

823 # LinearizeLookupTable: 

824 # Use linear part to get time at wich signal is maxAduForLookupTableLinearizer DN 

825 tMax = (self.config.maxAduForLookupTableLinearizer - parsFit[0])/parsFit[1] 

826 timeRange = np.linspace(0, tMax, self.config.maxAduForLookupTableLinearizer) 

827 signalIdeal = parsFit[0] + parsFit[1]*timeRange 

828 signalUncorrected = self.funcPolynomial(parsFit, timeRange) 

829 linearizerTableRow = signalIdeal - signalUncorrected # LinearizerLookupTable has corrections 

830 # LinearizePolynomial and LinearizeSquared: 

831 # Check that magnitude of higher order (>= 3) coefficents of the polyFit are small, 

832 # i.e., less than threshold = 1e-10 (typical quadratic and cubic coefficents are ~1e-6 

833 # and ~1e-12). 

834 k1 = parsFit[1] 

835 polynomialLinearizerCoefficients = [] 

836 for i, coefficient in enumerate(parsFit): 

837 c = -coefficient/(k1**i) 

838 polynomialLinearizerCoefficients.append(c) 

839 if np.fabs(c) > 1e-10: 

840 msg = f"Coefficient {c} in polynomial fit larger than threshold 1e-10." 

841 self.log.warn(msg) 

842 # Coefficient for LinearizedSquared. Called "c0" in linearize.py 

843 c0 = polynomialLinearizerCoefficients[2] 

844 

845 # Linearity residual 

846 linResidualTimeIndex = self.config.linResidualTimeIndex 

847 if exposureTimeVector[linResidualTimeIndex] == 0.0: 

848 raise RuntimeError("Reference time for linearity residual can't be 0.0") 

849 linResidual = 100*(1 - ((meanSignalVector[linResidualTimeIndex] / 

850 exposureTimeVector[linResidualTimeIndex]) / 

851 (meanSignalVector/exposureTimeVector))) 

852 

853 # Fractional non-linearity residual, w.r.t linear part of polynomial fit 

854 linearPart = parsFit[0] + k1*exposureTimeVector 

855 fracNonLinearityResidual = 100*(linearPart - meanSignalVector)/linearPart 

856 

857 dataset = LinearityResidualsAndLinearizersDataset([], None, [], [], [], [], [], None) 

858 dataset.polynomialLinearizerCoefficients = polynomialLinearizerCoefficients 

859 dataset.quadraticPolynomialLinearizerCoefficient = c0 

860 dataset.linearizerTableRow = linearizerTableRow 

861 dataset.linearityResidual = linResidual 

862 dataset.meanSignalVsTimePolyFitPars = parsFit 

863 dataset.meanSignalVsTimePolyFitParsErr = parsFitErr 

864 dataset.fractionalNonLinearityResidual = fracNonLinearityResidual 

865 dataset.meanSignalVsTimePolyFitReducedChiSq = reducedChiSquaredNonLinearityFit 

866 

867 return dataset 

868 

869 def fitPtcAndNonLinearity(self, dataset, ptcFitType, tableArray=None): 

870 """Fit the photon transfer curve and calculate linearity and residuals. 

871 

872 Fit the photon transfer curve with either a polynomial of the order 

873 specified in the task config, or using the Astier approximation. 

874 

875 Sigma clipping is performed iteratively for the fit, as well as an 

876 initial clipping of data points that are more than 

877 config.initialNonLinearityExclusionThreshold away from lying on a 

878 straight line. This other step is necessary because the photon transfer 

879 curve turns over catastrophically at very high flux (because saturation 

880 drops the variance to ~0) and these far outliers cause the initial fit 

881 to fail, meaning the sigma cannot be calculated to perform the 

882 sigma-clipping. 

883 

884 Parameters 

885 ---------- 

886 dataset : `lsst.cp.pipe.ptc.PhotonTransferCurveDataset` 

887 The dataset containing the means, variances and exposure times 

888 ptcFitType : `str` 

889 Fit a 'POLYNOMIAL' (degree: 'polynomialFitDegree') or 

890 'ASTIERAPPROXIMATION' to the PTC 

891 tableArray : `np.array` 

892 Optional. Look-up table array with size rows=nAmps and columns=DN values. 

893 It will be modified in-place if supplied. 

894 

895 Returns 

896 ------- 

897 dataset: `lsst.cp.pipe.ptc.PhotonTransferCurveDataset` 

898 This is the same dataset as the input paramter, however, it has been modified 

899 to include information such as the fit vectors and the fit parameters. See 

900 the class `PhotonTransferCurveDatase`. 

901 """ 

902 

903 def errFunc(p, x, y): 

904 return ptcFunc(p, x) - y 

905 

906 sigmaCutPtcOutliers = self.config.sigmaCutPtcOutliers 

907 maxIterationsPtcOutliers = self.config.maxIterationsPtcOutliers 

908 

909 for i, ampName in enumerate(dataset.ampNames): 

910 timeVecOriginal = np.array(dataset.rawExpTimes[ampName]) 

911 meanVecOriginal = np.array(dataset.rawMeans[ampName]) 

912 varVecOriginal = np.array(dataset.rawVars[ampName]) 

913 varVecOriginal = self._makeZeroSafe(varVecOriginal) 

914 

915 mask = ((meanVecOriginal >= self.config.minMeanSignal) & 

916 (meanVecOriginal <= self.config.maxMeanSignal)) 

917 

918 goodPoints = self._getInitialGoodPoints(meanVecOriginal, varVecOriginal, 

919 self.config.initialNonLinearityExclusionThresholdPositive, 

920 self.config.initialNonLinearityExclusionThresholdNegative) 

921 mask = mask & goodPoints 

922 

923 if ptcFitType == 'ASTIERAPPROXIMATION': 

924 ptcFunc = self.funcAstier 

925 parsIniPtc = [-1e-9, 1.0, 10.] # a00, gain, noise 

926 bounds = self._boundsForAstier(parsIniPtc) 

927 if ptcFitType == 'POLYNOMIAL': 

928 ptcFunc = self.funcPolynomial 

929 parsIniPtc = self._initialParsForPolynomial(self.config.polynomialFitDegree + 1) 

930 bounds = self._boundsForPolynomial(parsIniPtc) 

931 

932 # Before bootstrap fit, do an iterative fit to get rid of outliers 

933 count = 1 

934 while count <= maxIterationsPtcOutliers: 

935 # Note that application of the mask actually shrinks the array 

936 # to size rather than setting elements to zero (as we want) so 

937 # always update mask itself and re-apply to the original data 

938 meanTempVec = meanVecOriginal[mask] 

939 varTempVec = varVecOriginal[mask] 

940 res = least_squares(errFunc, parsIniPtc, bounds=bounds, args=(meanTempVec, varTempVec)) 

941 pars = res.x 

942 

943 # change this to the original from the temp because the masks are ANDed 

944 # meaning once a point is masked it's always masked, and the masks must 

945 # always be the same length for broadcasting 

946 sigResids = (varVecOriginal - ptcFunc(pars, meanVecOriginal))/np.sqrt(varVecOriginal) 

947 newMask = np.array([True if np.abs(r) < sigmaCutPtcOutliers else False for r in sigResids]) 

948 mask = mask & newMask 

949 

950 nDroppedTotal = Counter(mask)[False] 

951 self.log.debug(f"Iteration {count}: discarded {nDroppedTotal} points in total for {ampName}") 

952 count += 1 

953 # objects should never shrink 

954 assert (len(mask) == len(timeVecOriginal) == len(meanVecOriginal) == len(varVecOriginal)) 

955 

956 dataset.visitMask[ampName] = mask # store the final mask 

957 

958 parsIniPtc = pars 

959 timeVecFinal = timeVecOriginal[mask] 

960 meanVecFinal = meanVecOriginal[mask] 

961 varVecFinal = varVecOriginal[mask] 

962 

963 if Counter(mask)[False] > 0: 

964 self.log.info((f"Number of points discarded in PTC of amplifier {ampName}:" + 

965 f" {Counter(mask)[False]} out of {len(meanVecOriginal)}")) 

966 

967 if (len(meanVecFinal) < len(parsIniPtc)): 

968 msg = (f"\nSERIOUS: Not enough data points ({len(meanVecFinal)}) compared to the number of" 

969 f"parameters of the PTC model({len(parsIniPtc)}). Setting {ampName} to BAD.") 

970 self.log.warn(msg) 

971 # The first and second parameters of initial fit are discarded (bias and gain) 

972 # for the final NL coefficients 

973 lenNonLinPars = self.config.polynomialFitDegreeNonLinearity - 1 

974 dataset.badAmps.append(ampName) 

975 dataset.gain[ampName] = np.nan 

976 dataset.gainErr[ampName] = np.nan 

977 dataset.noise[ampName] = np.nan 

978 dataset.noiseErr[ampName] = np.nan 

979 dataset.nonLinearity[ampName] = np.nan 

980 dataset.nonLinearityError[ampName] = np.nan 

981 dataset.nonLinearityResiduals[ampName] = np.nan 

982 dataset.fractionalNonLinearityResiduals[ampName] = np.nan 

983 dataset.coefficientLinearizeSquared[ampName] = np.nan 

984 dataset.ptcFitPars[ampName] = np.nan 

985 dataset.ptcFitParsError[ampName] = np.nan 

986 dataset.ptcFitReducedChiSquared[ampName] = np.nan 

987 dataset.coefficientsLinearizePolynomial[ampName] = [np.nan]*lenNonLinPars 

988 tableArray[i, :] = [np.nan]*self.config.maxAduForLookupTableLinearizer 

989 continue 

990 

991 # Fit the PTC 

992 if self.config.doFitBootstrap: 

993 parsFit, parsFitErr, reducedChiSqPtc = self._fitBootstrap(parsIniPtc, meanVecFinal, 

994 varVecFinal, ptcFunc) 

995 else: 

996 parsFit, parsFitErr, reducedChiSqPtc = self._fitLeastSq(parsIniPtc, meanVecFinal, 

997 varVecFinal, ptcFunc) 

998 

999 dataset.ptcFitPars[ampName] = parsFit 

1000 dataset.ptcFitParsError[ampName] = parsFitErr 

1001 dataset.ptcFitReducedChiSquared[ampName] = reducedChiSqPtc 

1002 

1003 if ptcFitType == 'ASTIERAPPROXIMATION': 

1004 ptcGain = parsFit[1] 

1005 ptcGainErr = parsFitErr[1] 

1006 ptcNoise = np.sqrt(np.fabs(parsFit[2])) 

1007 ptcNoiseErr = 0.5*(parsFitErr[2]/np.fabs(parsFit[2]))*np.sqrt(np.fabs(parsFit[2])) 

1008 if ptcFitType == 'POLYNOMIAL': 

1009 ptcGain = 1./parsFit[1] 

1010 ptcGainErr = np.fabs(1./parsFit[1])*(parsFitErr[1]/parsFit[1]) 

1011 ptcNoise = np.sqrt(np.fabs(parsFit[0]))*ptcGain 

1012 ptcNoiseErr = (0.5*(parsFitErr[0]/np.fabs(parsFit[0]))*(np.sqrt(np.fabs(parsFit[0]))))*ptcGain 

1013 

1014 dataset.gain[ampName] = ptcGain 

1015 dataset.gainErr[ampName] = ptcGainErr 

1016 dataset.noise[ampName] = ptcNoise 

1017 dataset.noiseErr[ampName] = ptcNoiseErr 

1018 dataset.ptcFitType[ampName] = ptcFitType 

1019 

1020 # Non-linearity residuals (NL of mean vs time curve): percentage, and fit to a quadratic function 

1021 # In this case, len(parsIniNonLinearity) = 3 indicates that we want a quadratic fit 

1022 

1023 datasetLinRes = self.calculateLinearityResidualAndLinearizers(timeVecFinal, meanVecFinal) 

1024 

1025 # LinearizerLookupTable 

1026 if tableArray is not None: 

1027 tableArray[i, :] = datasetLinRes.linearizerTableRow 

1028 dataset.nonLinearity[ampName] = datasetLinRes.meanSignalVsTimePolyFitPars 

1029 dataset.nonLinearityError[ampName] = datasetLinRes.meanSignalVsTimePolyFitParsErr 

1030 dataset.nonLinearityResiduals[ampName] = datasetLinRes.linearityResidual 

1031 dataset.fractionalNonLinearityResiduals[ampName] = datasetLinRes.fractionalNonLinearityResidual 

1032 dataset.nonLinearityReducedChiSquared[ampName] = datasetLinRes.meanSignalVsTimePolyFitReducedChiSq 

1033 # Slice correction coefficients (starting at 2) for polynomial linearizer. The first 

1034 # and second are reduntant with the bias and gain, respectively, 

1035 # and are not used by LinearizerPolynomial. 

1036 polyLinCoeffs = np.array(datasetLinRes.polynomialLinearizerCoefficients[2:]) 

1037 dataset.coefficientsLinearizePolynomial[ampName] = polyLinCoeffs 

1038 quadPolyLinCoeff = datasetLinRes.quadraticPolynomialLinearizerCoefficient 

1039 dataset.coefficientLinearizeSquared[ampName] = quadPolyLinCoeff 

1040 

1041 return dataset 

1042 

1043 def plot(self, dataRef, dataset, ptcFitType): 

1044 dirname = dataRef.getUri(datasetType='cpPipePlotRoot', write=True) 

1045 if not os.path.exists(dirname): 

1046 os.makedirs(dirname) 

1047 

1048 detNum = dataRef.dataId[self.config.ccdKey] 

1049 filename = f"PTC_det{detNum}.pdf" 

1050 filenameFull = os.path.join(dirname, filename) 

1051 with PdfPages(filenameFull) as pdfPages: 

1052 self._plotPtc(dataset, ptcFitType, pdfPages) 

1053 

1054 def _plotPtc(self, dataset, ptcFitType, pdfPages): 

1055 """Plot PTC, linearity, and linearity residual per amplifier""" 

1056 

1057 reducedChiSqPtc = dataset.ptcFitReducedChiSquared 

1058 if ptcFitType == 'ASTIERAPPROXIMATION': 

1059 ptcFunc = self.funcAstier 

1060 stringTitle = (r"Var = $\frac{1}{2g^2a_{00}}(\exp (2a_{00} \mu g) - 1) + \frac{n_{00}}{g^2}$ " 

1061 r" ($chi^2$/dof = %g)" % (reducedChiSqPtc)) 

1062 if ptcFitType == 'POLYNOMIAL': 

1063 ptcFunc = self.funcPolynomial 

1064 stringTitle = r"Polynomial (degree: %g)" % (self.config.polynomialFitDegree) 

1065 

1066 legendFontSize = 7 

1067 labelFontSize = 7 

1068 titleFontSize = 9 

1069 supTitleFontSize = 18 

1070 markerSize = 25 

1071 

1072 # General determination of the size of the plot grid 

1073 nAmps = len(dataset.ampNames) 

1074 if nAmps == 2: 

1075 nRows, nCols = 2, 1 

1076 nRows = np.sqrt(nAmps) 

1077 mantissa, _ = np.modf(nRows) 

1078 if mantissa > 0: 

1079 nRows = int(nRows) + 1 

1080 nCols = nRows 

1081 else: 

1082 nRows = int(nRows) 

1083 nCols = nRows 

1084 

1085 f, ax = plt.subplots(nrows=nRows, ncols=nCols, sharex='col', sharey='row', figsize=(13, 10)) 

1086 f2, ax2 = plt.subplots(nrows=nRows, ncols=nCols, sharex='col', sharey='row', figsize=(13, 10)) 

1087 

1088 for i, (amp, a, a2) in enumerate(zip(dataset.ampNames, ax.flatten(), ax2.flatten())): 

1089 meanVecOriginal = np.array(dataset.rawMeans[amp]) 

1090 varVecOriginal = np.array(dataset.rawVars[amp]) 

1091 mask = dataset.visitMask[amp] 

1092 meanVecFinal = meanVecOriginal[mask] 

1093 varVecFinal = varVecOriginal[mask] 

1094 meanVecOutliers = meanVecOriginal[np.invert(mask)] 

1095 varVecOutliers = varVecOriginal[np.invert(mask)] 

1096 pars, parsErr = dataset.ptcFitPars[amp], dataset.ptcFitParsError[amp] 

1097 

1098 if ptcFitType == 'ASTIERAPPROXIMATION': 

1099 if len(meanVecFinal): 

1100 ptcA00, ptcA00error = pars[0], parsErr[0] 

1101 ptcGain, ptcGainError = pars[1], parsErr[1] 

1102 ptcNoise = np.sqrt(np.fabs(pars[2])) 

1103 ptcNoiseError = 0.5*(parsErr[2]/np.fabs(pars[2]))*np.sqrt(np.fabs(pars[2])) 

1104 stringLegend = (f"a00: {ptcA00:.2e}+/-{ptcA00error:.2e} 1/e" 

1105 f"\n Gain: {ptcGain:.4}+/-{ptcGainError:.2e} e/DN" 

1106 f"\n Noise: {ptcNoise:.4}+/-{ptcNoiseError:.2e} e \n") 

1107 

1108 if ptcFitType == 'POLYNOMIAL': 

1109 if len(meanVecFinal): 

1110 ptcGain, ptcGainError = 1./pars[1], np.fabs(1./pars[1])*(parsErr[1]/pars[1]) 

1111 ptcNoise = np.sqrt(np.fabs(pars[0]))*ptcGain 

1112 ptcNoiseError = (0.5*(parsErr[0]/np.fabs(pars[0]))*(np.sqrt(np.fabs(pars[0]))))*ptcGain 

1113 stringLegend = (f"Noise: {ptcNoise:.4}+/-{ptcNoiseError:.2e} e \n" 

1114 f"Gain: {ptcGain:.4}+/-{ptcGainError:.2e} e/DN \n") 

1115 

1116 a.set_xlabel(r'Mean signal ($\mu$, DN)', fontsize=labelFontSize) 

1117 a.set_ylabel(r'Variance (DN$^2$)', fontsize=labelFontSize) 

1118 a.tick_params(labelsize=11) 

1119 a.set_xscale('linear', fontsize=labelFontSize) 

1120 a.set_yscale('linear', fontsize=labelFontSize) 

1121 

1122 a2.set_xlabel(r'Mean Signal ($\mu$, DN)', fontsize=labelFontSize) 

1123 a2.set_ylabel(r'Variance (DN$^2$)', fontsize=labelFontSize) 

1124 a2.tick_params(labelsize=11) 

1125 a2.set_xscale('log') 

1126 a2.set_yscale('log') 

1127 

1128 if len(meanVecFinal): # Empty if the whole amp is bad, for example. 

1129 minMeanVecFinal = np.min(meanVecFinal) 

1130 maxMeanVecFinal = np.max(meanVecFinal) 

1131 meanVecFit = np.linspace(minMeanVecFinal, maxMeanVecFinal, 100*len(meanVecFinal)) 

1132 minMeanVecOriginal = np.min(meanVecOriginal) 

1133 maxMeanVecOriginal = np.max(meanVecOriginal) 

1134 deltaXlim = maxMeanVecOriginal - minMeanVecOriginal 

1135 

1136 a.plot(meanVecFit, ptcFunc(pars, meanVecFit), color='red') 

1137 a.plot(meanVecFinal, pars[0] + pars[1]*meanVecFinal, color='green', linestyle='--') 

1138 a.scatter(meanVecFinal, varVecFinal, c='blue', marker='o', s=markerSize) 

1139 a.scatter(meanVecOutliers, varVecOutliers, c='magenta', marker='s', s=markerSize) 

1140 a.text(0.03, 0.8, stringLegend, transform=a.transAxes, fontsize=legendFontSize) 

1141 a.set_title(amp, fontsize=titleFontSize) 

1142 a.set_xlim([minMeanVecOriginal - 0.2*deltaXlim, maxMeanVecOriginal + 0.2*deltaXlim]) 

1143 

1144 # Same, but in log-scale 

1145 a2.plot(meanVecFit, ptcFunc(pars, meanVecFit), color='red') 

1146 a2.scatter(meanVecFinal, varVecFinal, c='blue', marker='o', s=markerSize) 

1147 a2.scatter(meanVecOutliers, varVecOutliers, c='magenta', marker='s', s=markerSize) 

1148 a2.text(0.03, 0.8, stringLegend, transform=a2.transAxes, fontsize=legendFontSize) 

1149 a2.set_title(amp, fontsize=titleFontSize) 

1150 a2.set_xlim([minMeanVecOriginal, maxMeanVecOriginal]) 

1151 else: 

1152 a.set_title(f"{amp} (BAD)", fontsize=titleFontSize) 

1153 a2.set_title(f"{amp} (BAD)", fontsize=titleFontSize) 

1154 

1155 f.suptitle(f"PTC \n Fit: " + stringTitle, fontsize=20) 

1156 pdfPages.savefig(f) 

1157 f2.suptitle(f"PTC (log-log)", fontsize=20) 

1158 pdfPages.savefig(f2) 

1159 

1160 # Plot mean vs time 

1161 f, ax = plt.subplots(nrows=4, ncols=4, sharex='col', sharey='row', figsize=(13, 10)) 

1162 for i, (amp, a) in enumerate(zip(dataset.ampNames, ax.flatten())): 

1163 meanVecFinal = np.array(dataset.rawMeans[amp])[dataset.visitMask[amp]] 

1164 timeVecFinal = np.array(dataset.rawExpTimes[amp])[dataset.visitMask[amp]] 

1165 

1166 a.set_xlabel('Time (sec)', fontsize=labelFontSize) 

1167 a.set_ylabel(r'Mean signal ($\mu$, DN)', fontsize=labelFontSize) 

1168 a.tick_params(labelsize=labelFontSize) 

1169 a.set_xscale('linear', fontsize=labelFontSize) 

1170 a.set_yscale('linear', fontsize=labelFontSize) 

1171 

1172 if len(meanVecFinal): 

1173 pars, parsErr = dataset.nonLinearity[amp], dataset.nonLinearityError[amp] 

1174 k0, k0Error = pars[0], parsErr[0] 

1175 k1, k1Error = pars[1], parsErr[1] 

1176 k2, k2Error = pars[2], parsErr[2] 

1177 stringLegend = (f"k0: {k0:.4}+/-{k0Error:.2e} DN\n k1: {k1:.4}+/-{k1Error:.2e} DN/t" 

1178 f"\n k2: {k2:.2e}+/-{k2Error:.2e} DN/t^2 \n") 

1179 a.scatter(timeVecFinal, meanVecFinal) 

1180 a.plot(timeVecFinal, self.funcPolynomial(pars, timeVecFinal), color='red') 

1181 a.text(0.03, 0.75, stringLegend, transform=a.transAxes, fontsize=legendFontSize) 

1182 a.set_title(f"{amp}", fontsize=titleFontSize) 

1183 else: 

1184 a.set_title(f"{amp} (BAD)", fontsize=titleFontSize) 

1185 

1186 f.suptitle("Linearity \n Fit: Polynomial (degree: %g)" 

1187 % (self.config.polynomialFitDegreeNonLinearity), 

1188 fontsize=supTitleFontSize) 

1189 pdfPages.savefig(f) 

1190 

1191 # Plot linearity residual 

1192 f, ax = plt.subplots(nrows=4, ncols=4, sharex='col', sharey='row', figsize=(13, 10)) 

1193 for i, (amp, a) in enumerate(zip(dataset.ampNames, ax.flatten())): 

1194 meanVecFinal = np.array(dataset.rawMeans[amp])[dataset.visitMask[amp]] 

1195 linRes = np.array(dataset.nonLinearityResiduals[amp]) 

1196 

1197 a.set_xlabel(r'Mean signal ($\mu$, DN)', fontsize=labelFontSize) 

1198 a.set_ylabel('LR (%)', fontsize=labelFontSize) 

1199 a.set_xscale('linear', fontsize=labelFontSize) 

1200 a.set_yscale('linear', fontsize=labelFontSize) 

1201 if len(meanVecFinal): 

1202 a.axvline(x=timeVecFinal[self.config.linResidualTimeIndex], color='g', linestyle='--') 

1203 a.scatter(meanVecFinal, linRes) 

1204 a.set_title(f"{amp}", fontsize=titleFontSize) 

1205 a.axhline(y=0, color='k') 

1206 a.tick_params(labelsize=labelFontSize) 

1207 else: 

1208 a.set_title(f"{amp} (BAD)", fontsize=titleFontSize) 

1209 

1210 f.suptitle(r"Linearity Residual: $100\times(1 - \mu_{\rm{ref}}/t_{\rm{ref}})/(\mu / t))$" + "\n" + 

1211 r"$t_{\rm{ref}}$: " + f"{timeVecFinal[2]} s", fontsize=supTitleFontSize) 

1212 pdfPages.savefig(f) 

1213 

1214 # Plot fractional non-linearity residual (w.r.t linear part of polynomial fit) 

1215 f, ax = plt.subplots(nrows=4, ncols=4, sharex='col', sharey='row', figsize=(13, 10)) 

1216 for i, (amp, a) in enumerate(zip(dataset.ampNames, ax.flatten())): 

1217 meanVecFinal = np.array(dataset.rawMeans[amp])[dataset.visitMask[amp]] 

1218 fracLinRes = np.array(dataset.fractionalNonLinearityResiduals[amp]) 

1219 

1220 a.axhline(y=0, color='k') 

1221 a.axvline(x=0, color='k', linestyle='-') 

1222 a.set_xlabel(r'Mean signal ($\mu$, DN)', fontsize=labelFontSize) 

1223 a.set_ylabel('Fractional nonlinearity (%)', fontsize=labelFontSize) 

1224 a.tick_params(labelsize=labelFontSize) 

1225 a.set_xscale('linear', fontsize=labelFontSize) 

1226 a.set_yscale('linear', fontsize=labelFontSize) 

1227 

1228 if len(meanVecFinal): 

1229 a.scatter(meanVecFinal, fracLinRes, c='g') 

1230 a.set_title(amp, fontsize=titleFontSize) 

1231 else: 

1232 a.set_title(f"{amp} (BAD)", fontsize=titleFontSize) 

1233 

1234 f.suptitle(r"Fractional NL residual" + "\n" + 

1235 r"$100\times \frac{(k_0 + k_1*Time-\mu)}{k_0+k_1*Time}$", 

1236 fontsize=supTitleFontSize) 

1237 pdfPages.savefig(f) 

1238 

1239 return