Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of cp_pipe. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21# 

22 

23__all__ = ['MeasurePhotonTransferCurveTask', 

24 'MeasurePhotonTransferCurveTaskConfig', 

25 'PhotonTransferCurveDataset'] 

26 

27import numpy as np 

28import matplotlib.pyplot as plt 

29import os 

30from matplotlib.backends.backend_pdf import PdfPages 

31from sqlite3 import OperationalError 

32from collections import Counter 

33 

34import lsst.afw.math as afwMath 

35import lsst.pex.config as pexConfig 

36import lsst.pipe.base as pipeBase 

37from lsst.ip.isr import IsrTask 

38from .utils import (NonexistentDatasetTaskDataIdContainer, PairedVisitListTaskRunner, 

39 checkExpLengthEqual, validateIsrConfig) 

40from scipy.optimize import leastsq, least_squares 

41import numpy.polynomial.polynomial as poly 

42 

43 

44class MeasurePhotonTransferCurveTaskConfig(pexConfig.Config): 

45 """Config class for photon transfer curve measurement task""" 

46 isr = pexConfig.ConfigurableField( 

47 target=IsrTask, 

48 doc="""Task to perform instrumental signature removal.""", 

49 ) 

50 isrMandatorySteps = pexConfig.ListField( 

51 dtype=str, 

52 doc="isr operations that must be performed for valid results. Raises if any of these are False.", 

53 default=['doAssembleCcd'] 

54 ) 

55 isrForbiddenSteps = pexConfig.ListField( 

56 dtype=str, 

57 doc="isr operations that must NOT be performed for valid results. Raises if any of these are True", 

58 default=['doFlat', 'doFringe', 'doAddDistortionModel', 'doBrighterFatter', 'doUseOpticsTransmission', 

59 'doUseFilterTransmission', 'doUseSensorTransmission', 'doUseAtmosphereTransmission'] 

60 ) 

61 isrDesirableSteps = pexConfig.ListField( 

62 dtype=str, 

63 doc="isr operations that it is advisable to perform, but are not mission-critical." + 

64 " WARNs are logged for any of these found to be False.", 

65 default=['doBias', 'doDark', 'doCrosstalk', 'doDefect'] 

66 ) 

67 isrUndesirableSteps = pexConfig.ListField( 

68 dtype=str, 

69 doc="isr operations that it is *not* advisable to perform in the general case, but are not" + 

70 " forbidden as some use-cases might warrant them." + 

71 " WARNs are logged for any of these found to be True.", 

72 default=['doLinearize'] 

73 ) 

74 ccdKey = pexConfig.Field( 

75 dtype=str, 

76 doc="The key by which to pull a detector from a dataId, e.g. 'ccd' or 'detector'.", 

77 default='ccd', 

78 ) 

79 makePlots = pexConfig.Field( 

80 dtype=bool, 

81 doc="Plot the PTC curves?", 

82 default=False, 

83 ) 

84 ptcFitType = pexConfig.ChoiceField( 

85 dtype=str, 

86 doc="Fit PTC to approximation in Astier+19 (Equation 16) or to a polynomial.", 

87 default="POLYNOMIAL", 

88 allowed={ 

89 "POLYNOMIAL": "n-degree polynomial (use 'polynomialFitDegree' to set 'n').", 

90 "ASTIERAPPROXIMATION": "Approximation in Astier+19 (Eq. 16)." 

91 } 

92 ) 

93 polynomialFitDegree = pexConfig.Field( 

94 dtype=int, 

95 doc="Degree of polynomial to fit the PTC, when 'ptcFitType'=POLYNOMIAL.", 

96 default=2, 

97 ) 

98 polynomialFitDegreeNonLinearity = pexConfig.Field( 

99 dtype=int, 

100 doc="Degree of polynomial to fit the meanSignal vs exposureTime curve to produce" + 

101 " the table for LinearizerLookupTable.", 

102 default=3, 

103 ) 

104 binSize = pexConfig.Field( 

105 dtype=int, 

106 doc="Bin the image by this factor in both dimensions.", 

107 default=1, 

108 ) 

109 minMeanSignal = pexConfig.Field( 

110 dtype=float, 

111 doc="Minimum value (inclusive) of mean signal (in ADU) above which to consider.", 

112 default=0, 

113 ) 

114 maxMeanSignal = pexConfig.Field( 

115 dtype=float, 

116 doc="Maximum value (inclusive) of mean signal (in ADU) below which to consider.", 

117 default=9e6, 

118 ) 

119 initialNonLinearityExclusionThresholdPositive = pexConfig.RangeField( 

120 dtype=float, 

121 doc="Initially exclude data points with a variance that are more than a factor of this from being" 

122 " linear in the positive direction, from the PTC fit. Note that these points will also be" 

123 " excluded from the non-linearity fit. This is done before the iterative outlier rejection," 

124 " to allow an accurate determination of the sigmas for said iterative fit.", 

125 default=0.12, 

126 min=0.0, 

127 max=1.0, 

128 ) 

129 initialNonLinearityExclusionThresholdNegative = pexConfig.RangeField( 

130 dtype=float, 

131 doc="Initially exclude data points with a variance that are more than a factor of this from being" 

132 " linear in the negative direction, from the PTC fit. Note that these points will also be" 

133 " excluded from the non-linearity fit. This is done before the iterative outlier rejection," 

134 " to allow an accurate determination of the sigmas for said iterative fit.", 

135 default=0.25, 

136 min=0.0, 

137 max=1.0, 

138 ) 

139 sigmaCutPtcOutliers = pexConfig.Field( 

140 dtype=float, 

141 doc="Sigma cut for outlier rejection in PTC.", 

142 default=5.0, 

143 ) 

144 maxIterationsPtcOutliers = pexConfig.Field( 

145 dtype=int, 

146 doc="Maximum number of iterations for outlier rejection in PTC.", 

147 default=2, 

148 ) 

149 doFitBootstrap = pexConfig.Field( 

150 dtype=bool, 

151 doc="Use bootstrap for the PTC fit parameters and errors?.", 

152 default=False, 

153 ) 

154 linResidualTimeIndex = pexConfig.Field( 

155 dtype=int, 

156 doc="Index position in time array for reference time in linearity residual calculation.", 

157 default=2, 

158 ) 

159 maxAduForLookupTableLinearizer = pexConfig.Field( 

160 dtype=int, 

161 doc="Maximum ADU value for the LookupTable linearizer.", 

162 default=2**18, 

163 ) 

164 

165 

166class PhotonTransferCurveDataset: 

167 """A simple class to hold the output data from the PTC task. 

168 

169 The dataset is made up of a dictionary for each item, keyed by the 

170 amplifiers' names, which much be supplied at construction time. 

171 

172 New items cannot be added to the class to save accidentally saving to the 

173 wrong property, and the class can be frozen if desired. 

174 

175 inputVisitPairs records the visits used to produce the data. 

176 When fitPtcAndNonLinearity() is run, a mask is built up, which is by definition 

177 always the same length as inputVisitPairs, rawExpTimes, rawMeans 

178 and rawVars, and is a list of bools, which are incrementally set to False 

179 as points are discarded from the fits. 

180 

181 PTC fit parameters for polynomials are stored in a list in ascending order 

182 of polynomial term, i.e. par[0]*x^0 + par[1]*x + par[2]*x^2 etc 

183 with the length of the list corresponding to the order of the polynomial 

184 plus one. 

185 """ 

186 def __init__(self, ampNames): 

187 # add items to __dict__ directly because __setattr__ is overridden 

188 

189 # instance variables 

190 self.__dict__["ampNames"] = ampNames 

191 self.__dict__["badAmps"] = [] 

192 

193 # raw data variables 

194 self.__dict__["inputVisitPairs"] = {ampName: [] for ampName in ampNames} 

195 self.__dict__["visitMask"] = {ampName: [] for ampName in ampNames} 

196 self.__dict__["rawExpTimes"] = {ampName: [] for ampName in ampNames} 

197 self.__dict__["rawMeans"] = {ampName: [] for ampName in ampNames} 

198 self.__dict__["rawVars"] = {ampName: [] for ampName in ampNames} 

199 

200 # fit information 

201 self.__dict__["ptcFitType"] = {ampName: "" for ampName in ampNames} 

202 self.__dict__["ptcFitPars"] = {ampName: [] for ampName in ampNames} 

203 self.__dict__["ptcFitParsError"] = {ampName: [] for ampName in ampNames} 

204 self.__dict__["nonLinearity"] = {ampName: [] for ampName in ampNames} 

205 self.__dict__["nonLinearityError"] = {ampName: [] for ampName in ampNames} 

206 self.__dict__["nonLinearityResiduals"] = {ampName: [] for ampName in ampNames} 

207 self.__dict__["coefficientLinearizeSquared"] = {ampName: [] for ampName in ampNames} 

208 

209 # final results 

210 self.__dict__["gain"] = {ampName: -1. for ampName in ampNames} 

211 self.__dict__["gainErr"] = {ampName: -1. for ampName in ampNames} 

212 self.__dict__["noise"] = {ampName: -1. for ampName in ampNames} 

213 self.__dict__["noiseErr"] = {ampName: -1. for ampName in ampNames} 

214 self.__dict__["coefficientLinearizeSquared"] = {ampName: [] for ampName in ampNames} 

215 

216 def __setattr__(self, attribute, value): 

217 """Protect class attributes""" 

218 if attribute not in self.__dict__: 

219 raise AttributeError(f"{attribute} is not already a member of PhotonTransferCurveDataset, which" 

220 " does not support setting of new attributes.") 

221 else: 

222 self.__dict__[attribute] = value 

223 

224 def getVisitsUsed(self, ampName): 

225 """Get the visits used, i.e. not discarded, for a given amp. 

226 

227 If no mask has been created yet, all visits are returned. 

228 """ 

229 if self.visitMask[ampName] == []: 

230 return self.inputVisitPairs[ampName] 

231 

232 # if the mask exists it had better be the same length as the visitPairs 

233 assert len(self.visitMask[ampName]) == len(self.inputVisitPairs[ampName]) 

234 

235 pairs = self.inputVisitPairs[ampName] 

236 mask = self.visitMask[ampName] 

237 # cast to bool required because numpy 

238 return [(v1, v2) for ((v1, v2), m) in zip(pairs, mask) if bool(m) is True] 

239 

240 def getGoodAmps(self): 

241 return [amp for amp in self.ampNames if amp not in self.badAmps] 

242 

243 

244class MeasurePhotonTransferCurveTask(pipeBase.CmdLineTask): 

245 """A class to calculate, fit, and plot a PTC from a set of flat pairs. 

246 

247 The Photon Transfer Curve (var(signal) vs mean(signal)) is a standard tool 

248 used in astronomical detectors characterization (e.g., Janesick 2001, 

249 Janesick 2007). This task calculates the PTC from a series of pairs of 

250 flat-field images; each pair taken at identical exposure times. The 

251 difference image of each pair is formed to eliminate fixed pattern noise, 

252 and then the variance of the difference image and the mean of the average image 

253 are used to produce the PTC. An n-degree polynomial or the approximation in Equation 

254 16 of Astier+19 ("The Shape of the Photon Transfer Curve of CCD sensors", 

255 arXiv:1905.08677) can be fitted to the PTC curve. These models include 

256 parameters such as the gain (e/ADU) and readout noise. 

257 

258 Parameters 

259 ---------- 

260 

261 *args: `list` 

262 Positional arguments passed to the Task constructor. None used at this 

263 time. 

264 **kwargs: `dict` 

265 Keyword arguments passed on to the Task constructor. None used at this 

266 time. 

267 

268 """ 

269 

270 RunnerClass = PairedVisitListTaskRunner 

271 ConfigClass = MeasurePhotonTransferCurveTaskConfig 

272 _DefaultName = "measurePhotonTransferCurve" 

273 

274 def __init__(self, *args, **kwargs): 

275 pipeBase.CmdLineTask.__init__(self, *args, **kwargs) 

276 self.makeSubtask("isr") 

277 plt.interactive(False) # stop windows popping up when plotting. When headless, use 'agg' backend too 

278 validateIsrConfig(self.isr, self.config.isrMandatorySteps, 

279 self.config.isrForbiddenSteps, self.config.isrDesirableSteps, checkTrim=False) 

280 self.config.validate() 

281 self.config.freeze() 

282 

283 @classmethod 

284 def _makeArgumentParser(cls): 

285 """Augment argument parser for the MeasurePhotonTransferCurveTask.""" 

286 parser = pipeBase.ArgumentParser(name=cls._DefaultName) 

287 parser.add_argument("--visit-pairs", dest="visitPairs", nargs="*", 

288 help="Visit pairs to use. Each pair must be of the form INT,INT e.g. 123,456") 

289 parser.add_id_argument("--id", datasetType="photonTransferCurveDataset", 

290 ContainerClass=NonexistentDatasetTaskDataIdContainer, 

291 help="The ccds to use, e.g. --id ccd=0..100") 

292 return parser 

293 

294 @pipeBase.timeMethod 

295 def runDataRef(self, dataRef, visitPairs): 

296 """Run the Photon Transfer Curve (PTC) measurement task. 

297 

298 For a dataRef (which is each detector here), 

299 and given a list of visit pairs at different exposure times, 

300 measure the PTC. 

301 

302 Parameters 

303 ---------- 

304 dataRef : list of lsst.daf.persistence.ButlerDataRef 

305 dataRef for the detector for the visits to be fit. 

306 visitPairs : `iterable` of `tuple` of `int` 

307 Pairs of visit numbers to be processed together 

308 """ 

309 

310 # setup necessary objects 

311 detNum = dataRef.dataId[self.config.ccdKey] 

312 detector = dataRef.get('camera')[dataRef.dataId[self.config.ccdKey]] 

313 # expand some missing fields that we need for lsstCam. This is a work-around 

314 # for Gen2 problems that I (RHL) don't feel like solving. The calibs pipelines 

315 # (which inherit from CalibTask) use addMissingKeys() to do basically the same thing 

316 # 

317 # Basically, the butler's trying to look up the fields in `raw_visit` which won't work 

318 for name in dataRef.getButler().getKeys('bias'): 

319 if name not in dataRef.dataId: 

320 try: 

321 dataRef.dataId[name] = \ 

322 dataRef.getButler().queryMetadata('raw', [name], detector=detNum)[0] 

323 except OperationalError: 

324 pass 

325 

326 amps = detector.getAmplifiers() 

327 ampNames = [amp.getName() for amp in amps] 

328 dataset = PhotonTransferCurveDataset(ampNames) 

329 

330 self.log.info('Measuring PTC using %s visits for detector %s' % (visitPairs, detNum)) 

331 

332 for (v1, v2) in visitPairs: 

333 # Perform ISR on each exposure 

334 dataRef.dataId['expId'] = v1 

335 exp1 = self.isr.runDataRef(dataRef).exposure 

336 dataRef.dataId['expId'] = v2 

337 exp2 = self.isr.runDataRef(dataRef).exposure 

338 del dataRef.dataId['expId'] 

339 

340 checkExpLengthEqual(exp1, exp2, v1, v2, raiseWithMessage=True) 

341 expTime = exp1.getInfo().getVisitInfo().getExposureTime() 

342 

343 for amp in detector: 

344 mu, varDiff = self.measureMeanVarPair(exp1, exp2, region=amp.getBBox()) 

345 ampName = amp.getName() 

346 

347 dataset.rawExpTimes[ampName].append(expTime) 

348 dataset.rawMeans[ampName].append(mu) 

349 dataset.rawVars[ampName].append(varDiff) 

350 dataset.inputVisitPairs[ampName].append((v1, v2)) 

351 

352 numberAmps = len(detector.getAmplifiers()) 

353 numberAduValues = self.config.maxAduForLookupTableLinearizer 

354 lookupTableArray = np.zeros((numberAmps, numberAduValues), dtype=np.float32) 

355 

356 # Fit PTC and (non)linearity of signal vs time curve. 

357 # Fill up PhotonTransferCurveDataset object. 

358 # Fill up array for LUT linearizer. 

359 dataset = self.fitPtcAndNonLinearity(dataset, lookupTableArray, ptcFitType=self.config.ptcFitType) 

360 

361 if self.config.makePlots: 

362 self.plot(dataRef, dataset, ptcFitType=self.config.ptcFitType) 

363 

364 # Save data, PTC fit, and NL fit dictionaries 

365 self.log.info(f"Writing PTC and NL data to {dataRef.getUri(write=True)}") 

366 dataRef.put(dataset, datasetType="photonTransferCurveDataset") 

367 

368 self.log.info('Finished measuring PTC for in detector %s' % detNum) 

369 

370 return pipeBase.Struct(exitStatus=0) 

371 

372 def measureMeanVarPair(self, exposure1, exposure2, region=None): 

373 """Calculate the mean signal of two exposures and the variance of their difference. 

374 

375 Parameters 

376 ---------- 

377 exposure1 : `lsst.afw.image.exposure.exposure.ExposureF` 

378 First exposure of flat field pair. 

379 

380 exposure2 : `lsst.afw.image.exposure.exposure.ExposureF` 

381 Second exposure of flat field pair. 

382 

383 region : `lsst.geom.Box2I` 

384 Region of each exposure where to perform the calculations (e.g, an amplifier). 

385 

386 Return 

387 ------ 

388 

389 mu : `np.float` 

390 0.5*(mu1 + mu2), where mu1, and mu2 are the clipped means of the regions in 

391 both exposures. 

392 

393 varDiff : `np.float` 

394 Half of the clipped variance of the difference of the regions inthe two input 

395 exposures. 

396 """ 

397 

398 if region is not None: 

399 im1Area = exposure1.maskedImage[region] 

400 im2Area = exposure2.maskedImage[region] 

401 else: 

402 im1Area = exposure1.maskedImage 

403 im2Area = exposure2.maskedImage 

404 

405 im1Area = afwMath.binImage(im1Area, self.config.binSize) 

406 im2Area = afwMath.binImage(im2Area, self.config.binSize) 

407 

408 # Clipped mean of images; then average of mean. 

409 mu1 = afwMath.makeStatistics(im1Area, afwMath.MEANCLIP).getValue() 

410 mu2 = afwMath.makeStatistics(im2Area, afwMath.MEANCLIP).getValue() 

411 mu = 0.5*(mu1 + mu2) 

412 

413 # Take difference of pairs 

414 # symmetric formula: diff = (mu2*im1-mu1*im2)/(0.5*(mu1+mu2)) 

415 temp = im2Area.clone() 

416 temp *= mu1 

417 diffIm = im1Area.clone() 

418 diffIm *= mu2 

419 diffIm -= temp 

420 diffIm /= mu 

421 

422 varDiff = 0.5*(afwMath.makeStatistics(diffIm, afwMath.VARIANCECLIP).getValue()) 

423 

424 return mu, varDiff 

425 

426 def _fitLeastSq(self, initialParams, dataX, dataY, function): 

427 """Do a fit and estimate the parameter errors using using scipy.optimize.leastq. 

428 

429 optimize.leastsq returns the fractional covariance matrix. To estimate the 

430 standard deviation of the fit parameters, multiply the entries of this matrix 

431 by the reduced chi squared and take the square root of the diagonal elements. 

432 

433 Parameters 

434 ---------- 

435 initialParams : list of np.float 

436 initial values for fit parameters. For ptcFitType=POLYNOMIAL, its length 

437 determines the degree of the polynomial. 

438 

439 dataX : np.array of np.float 

440 Data in the abscissa axis. 

441 

442 dataY : np.array of np.float 

443 Data in the ordinate axis. 

444 

445 function : callable object (function) 

446 Function to fit the data with. 

447 

448 Return 

449 ------ 

450 pFitSingleLeastSquares : list of np.float 

451 List with fitted parameters. 

452 

453 pErrSingleLeastSquares : list of np.float 

454 List with errors for fitted parameters. 

455 """ 

456 

457 def errFunc(p, x, y): 

458 return function(p, x) - y 

459 

460 pFit, pCov, infoDict, errMessage, success = leastsq(errFunc, initialParams, 

461 args=(dataX, dataY), full_output=1, epsfcn=0.0001) 

462 

463 if (len(dataY) > len(initialParams)) and pCov is not None: 

464 reducedChiSq = (errFunc(pFit, dataX, dataY)**2).sum()/(len(dataY)-len(initialParams)) 

465 pCov *= reducedChiSq 

466 else: 

467 pCov[:, :] = np.inf 

468 

469 errorVec = [] 

470 for i in range(len(pFit)): 

471 errorVec.append(np.fabs(pCov[i][i])**0.5) 

472 

473 pFitSingleLeastSquares = pFit 

474 pErrSingleLeastSquares = np.array(errorVec) 

475 

476 return pFitSingleLeastSquares, pErrSingleLeastSquares 

477 

478 def _fitBootstrap(self, initialParams, dataX, dataY, function, confidenceSigma=1.): 

479 """Do a fit using least squares and bootstrap to estimate parameter errors. 

480 

481 The bootstrap error bars are calculated by fitting 100 random data sets. 

482 

483 Parameters 

484 ---------- 

485 initialParams : list of np.float 

486 initial values for fit parameters. For ptcFitType=POLYNOMIAL, its length 

487 determines the degree of the polynomial. 

488 

489 dataX : np.array of np.float 

490 Data in the abscissa axis. 

491 

492 dataY : np.array of np.float 

493 Data in the ordinate axis. 

494 

495 function : callable object (function) 

496 Function to fit the data with. 

497 

498 confidenceSigma : np.float 

499 Number of sigmas that determine confidence interval for the bootstrap errors. 

500 

501 Return 

502 ------ 

503 pFitBootstrap : list of np.float 

504 List with fitted parameters. 

505 

506 pErrBootstrap : list of np.float 

507 List with errors for fitted parameters. 

508 """ 

509 

510 def errFunc(p, x, y): 

511 return function(p, x) - y 

512 

513 # Fit first time 

514 pFit, _ = leastsq(errFunc, initialParams, args=(dataX, dataY), full_output=0) 

515 

516 # Get the stdev of the residuals 

517 residuals = errFunc(pFit, dataX, dataY) 

518 sigmaErrTotal = np.std(residuals) 

519 

520 # 100 random data sets are generated and fitted 

521 pars = [] 

522 for i in range(100): 

523 randomDelta = np.random.normal(0., sigmaErrTotal, len(dataY)) 

524 randomDataY = dataY + randomDelta 

525 randomFit, _ = leastsq(errFunc, initialParams, 

526 args=(dataX, randomDataY), full_output=0) 

527 pars.append(randomFit) 

528 pars = np.array(pars) 

529 meanPfit = np.mean(pars, 0) 

530 

531 # confidence interval for parameter estimates 

532 nSigma = confidenceSigma 

533 errPfit = nSigma*np.std(pars, 0) 

534 pFitBootstrap = meanPfit 

535 pErrBootstrap = errPfit 

536 return pFitBootstrap, pErrBootstrap 

537 

538 def funcPolynomial(self, pars, x): 

539 """Polynomial function definition""" 

540 return poly.polyval(x, [*pars]) 

541 

542 def funcAstier(self, pars, x): 

543 """Single brighter-fatter parameter model for PTC; Equation 16 of Astier+19""" 

544 a00, gain, noise = pars 

545 return 0.5/(a00*gain*gain)*(np.exp(2*a00*x*gain)-1) + noise/(gain*gain) 

546 

547 @staticmethod 

548 def _initialParsForPolynomial(order): 

549 assert(order >= 2) 

550 pars = np.zeros(order, dtype=np.float) 

551 pars[0] = 10 

552 pars[1] = 1 

553 pars[2:] = 0.0001 

554 return pars 

555 

556 @staticmethod 

557 def _boundsForPolynomial(initialPars): 

558 lowers = [np.NINF for p in initialPars] 

559 uppers = [np.inf for p in initialPars] 

560 lowers[1] = 0 # no negative gains 

561 return (lowers, uppers) 

562 

563 @staticmethod 

564 def _boundsForAstier(initialPars): 

565 lowers = [np.NINF for p in initialPars] 

566 uppers = [np.inf for p in initialPars] 

567 return (lowers, uppers) 

568 

569 @staticmethod 

570 def _getInitialGoodPoints(means, variances, maxDeviationPositive, maxDeviationNegative): 

571 """Return a boolean array to mask bad points. 

572 

573 A linear function has a constant ratio, so find the median 

574 value of the ratios, and exclude the points that deviate 

575 from that by more than a factor of maxDeviationPositive/negative. 

576 Asymmetric deviations are supported as we expect the PTC to turn 

577 down as the flux increases, but sometimes it anomalously turns 

578 upwards just before turning over, which ruins the fits, so it 

579 is wise to be stricter about restricting positive outliers than 

580 negative ones. 

581 

582 Too high and points that are so bad that fit will fail will be included 

583 Too low and the non-linear points will be excluded, biasing the NL fit.""" 

584 ratios = [b/a for (a, b) in zip(means, variances)] 

585 medianRatio = np.median(ratios) 

586 ratioDeviations = [(r/medianRatio)-1 for r in ratios] 

587 

588 # so that it doesn't matter if the deviation is expressed as positive or negative 

589 maxDeviationPositive = abs(maxDeviationPositive) 

590 maxDeviationNegative = -1. * abs(maxDeviationNegative) 

591 

592 goodPoints = np.array([True if (r < maxDeviationPositive and r > maxDeviationNegative) 

593 else False for r in ratioDeviations]) 

594 return goodPoints 

595 

596 def _makeZeroSafe(self, array, warn=True, substituteValue=1e-9): 

597 """""" 

598 nBad = Counter(array)[0] 

599 if nBad == 0: 

600 return array 

601 

602 if warn: 

603 msg = f"Found {nBad} zeros in array at elements {[x for x in np.where(array==0)[0]]}" 

604 self.log.warn(msg) 

605 

606 array[array == 0] = substituteValue 

607 return array 

608 

609 def calculateLinearityResidualAndLinearizers(self, exposureTimeVector, meanSignalVector): 

610 """Calculate linearity residual and fit an n-order polynomial to the mean vs time curve 

611 to produce corrections (deviation from linear part of polynomial) for a particular amplifier 

612 to populate LinearizeLookupTable. Use quadratic and linear parts of this polynomial to approximate 

613 c0 for LinearizeSquared." 

614 

615 Parameters 

616 --------- 

617 

618 exposureTimeVector: `list` of `np.float` 

619 List of exposure times for each flat pair 

620 

621 meanSignalVector: `list` of `np.float` 

622 List of mean signal from diference image of flat pairs 

623 

624 Returns 

625 ------- 

626 c0: `np.float` 

627 Coefficient for LinearizeSquared, where corrImage = uncorrImage + c0*uncorrImage^2. 

628 c0 ~ -k2/(k1^2), where k1 and k2 are fit from 

629 meanSignalVector = k0 + k1*exposureTimeVector + k2*exposureTimeVector^2 +... 

630 + kn*exposureTimeVector^n, with n = "polynomialFitDegreeNonLinearity". 

631 

632 linearizerTableRow: list of `np.float` 

633 One dimensional array with deviation from linear part of n-order polynomial fit 

634 to mean vs time curve. This array will be one row (for the particular amplifier at hand) 

635 of the table array for LinearizeLookupTable. 

636 

637 linResidual: list of `np.float` 

638 Linearity residual from the mean vs time curve, defined as 

639 100*(1 - meanSignalReference/expTimeReference/(meanSignal/expTime). 

640 

641 parsFit: list of `np.float` 

642 Parameters from n-order polynomial fit to mean vs time curve. 

643 

644 parsFitErr: list of `np.float` 

645 Parameters from n-order polynomial fit to mean vs time curve. 

646 

647 """ 

648 

649 # Lookup table linearizer 

650 parsIniNonLinearity = self._initialParsForPolynomial(self.config.polynomialFitDegreeNonLinearity + 1) 

651 if self.config.doFitBootstrap: 

652 parsFit, parsFitErr = self._fitBootstrap(parsIniNonLinearity, exposureTimeVector, 

653 meanSignalVector, self.funcPolynomial) 

654 else: 

655 parsFit, parsFitErr = self._fitLeastSq(parsIniNonLinearity, exposureTimeVector, meanSignalVector, 

656 self.funcPolynomial) 

657 

658 # Use linear part to get time at wich signal is maxAduForLookupTableLinearizer ADU 

659 tMax = (self.config.maxAduForLookupTableLinearizer - parsFit[0])/parsFit[1] 

660 timeRange = np.linspace(0, tMax, self.config.maxAduForLookupTableLinearizer) 

661 signalIdeal = parsFit[0] + parsFit[1]*timeRange 

662 signalUncorrected = self.funcPolynomial(parsFit, timeRange) 

663 linearizerTableRow = signalIdeal - signalUncorrected # LinearizerLookupTable has corrections 

664 

665 # Use quadratic and linear part of fit to produce c0 for LinearizeSquared 

666 # Check that magnitude of higher order (>= 3) coefficents of the polyFit are small, 

667 # i.e., less than threshold = 1e-10 (typical quadratic and cubic coefficents are ~1e-6 

668 # and ~1e-12). 

669 k1, k2 = parsFit[1], parsFit[2] 

670 c0 = -k2/(k1**2) # c0 coefficient for LinearizeSquared 

671 for coefficient in parsFit[3:]: 

672 if np.fabs(coefficient) > 1e-10: 

673 msg = f"Coefficient {coefficient} in polynomial fit larger than threshold 1e-10." 

674 self.log.warn(msg) 

675 

676 # Linearity residual 

677 linResidualTimeIndex = self.config.linResidualTimeIndex 

678 if exposureTimeVector[linResidualTimeIndex] == 0.0: 

679 raise RuntimeError("Reference time for linearity residual can't be 0.0") 

680 linResidual = 100*(1 - ((meanSignalVector[linResidualTimeIndex] / 

681 exposureTimeVector[linResidualTimeIndex]) / 

682 (meanSignalVector/exposureTimeVector))) 

683 

684 return c0, linearizerTableRow, linResidual, parsFit, parsFitErr 

685 

686 def fitPtcAndNonLinearity(self, dataset, tableArray, ptcFitType): 

687 """Fit the photon transfer curve and calculate linearity and residuals. 

688 

689 Fit the photon transfer curve with either a polynomial of the order 

690 specified in the task config, or using the Astier approximation. 

691 

692 Sigma clipping is performed iteratively for the fit, as well as an 

693 initial clipping of data points that are more than 

694 config.initialNonLinearityExclusionThreshold away from lying on a 

695 straight line. This other step is necessary because the photon transfer 

696 curve turns over catastrophically at very high flux (because saturation 

697 drops the variance to ~0) and these far outliers cause the initial fit 

698 to fail, meaning the sigma cannot be calculated to perform the 

699 sigma-clipping. 

700 

701 Parameters 

702 ---------- 

703 dataset : `lsst.cp.pipe.ptc.PhotonTransferCurveDataset` 

704 The dataset containing the means, variances and exposure times 

705 ptcFitType : `str` 

706 Fit a 'POLYNOMIAL' (degree: 'polynomialFitDegree') or 

707 'ASTIERAPPROXIMATION' to the PTC 

708 tableArray : `np.array` 

709 Look-up table array with size rows=nAmps and columns=ADU values 

710 

711 Returns 

712 ------- 

713 dataset: `lsst.cp.pipe.ptc.PhotonTransferCurveDataset` 

714 This is the same dataset as the input paramter, however, it has been modified 

715 to include information such as the fit vectors and the fit parameters. See 

716 the class `PhotonTransferCurveDatase`. 

717 """ 

718 

719 def errFunc(p, x, y): 

720 return ptcFunc(p, x) - y 

721 

722 sigmaCutPtcOutliers = self.config.sigmaCutPtcOutliers 

723 maxIterationsPtcOutliers = self.config.maxIterationsPtcOutliers 

724 

725 for i, ampName in enumerate(dataset.ampNames): 

726 timeVecOriginal = np.array(dataset.rawExpTimes[ampName]) 

727 meanVecOriginal = np.array(dataset.rawMeans[ampName]) 

728 varVecOriginal = np.array(dataset.rawVars[ampName]) 

729 varVecOriginal = self._makeZeroSafe(varVecOriginal) 

730 

731 mask = ((meanVecOriginal >= self.config.minMeanSignal) & 

732 (meanVecOriginal <= self.config.maxMeanSignal)) 

733 

734 goodPoints = self._getInitialGoodPoints(meanVecOriginal, varVecOriginal, 

735 self.config.initialNonLinearityExclusionThresholdPositive, 

736 self.config.initialNonLinearityExclusionThresholdNegative) 

737 mask = mask & goodPoints 

738 

739 if ptcFitType == 'ASTIERAPPROXIMATION': 

740 ptcFunc = self.funcAstier 

741 parsIniPtc = [-1e-9, 1.0, 10.] # a00, gain, noise 

742 bounds = self._boundsForAstier(parsIniPtc) 

743 if ptcFitType == 'POLYNOMIAL': 

744 ptcFunc = self.funcPolynomial 

745 parsIniPtc = self._initialParsForPolynomial(self.config.polynomialFitDegree + 1) 

746 bounds = self._boundsForPolynomial(parsIniPtc) 

747 

748 # Before bootstrap fit, do an iterative fit to get rid of outliers 

749 count = 1 

750 while count <= maxIterationsPtcOutliers: 

751 # Note that application of the mask actually shrinks the array 

752 # to size rather than setting elements to zero (as we want) so 

753 # always update mask itself and re-apply to the original data 

754 meanTempVec = meanVecOriginal[mask] 

755 varTempVec = varVecOriginal[mask] 

756 res = least_squares(errFunc, parsIniPtc, bounds=bounds, args=(meanTempVec, varTempVec)) 

757 pars = res.x 

758 

759 # change this to the original from the temp because the masks are ANDed 

760 # meaning once a point is masked it's always masked, and the masks must 

761 # always be the same length for broadcasting 

762 sigResids = (varVecOriginal - ptcFunc(pars, meanVecOriginal))/np.sqrt(varVecOriginal) 

763 newMask = np.array([True if np.abs(r) < sigmaCutPtcOutliers else False for r in sigResids]) 

764 mask = mask & newMask 

765 

766 nDroppedTotal = Counter(mask)[False] 

767 self.log.debug(f"Iteration {count}: discarded {nDroppedTotal} points in total for {ampName}") 

768 count += 1 

769 # objects should never shrink 

770 assert (len(mask) == len(timeVecOriginal) == len(meanVecOriginal) == len(varVecOriginal)) 

771 

772 dataset.visitMask[ampName] = mask # store the final mask 

773 

774 parsIniPtc = pars 

775 timeVecFinal = timeVecOriginal[mask] 

776 meanVecFinal = meanVecOriginal[mask] 

777 varVecFinal = varVecOriginal[mask] 

778 

779 if Counter(mask)[False] > 0: 

780 self.log.info((f"Number of points discarded in PTC of amplifier {ampName}:" + 

781 f" {Counter(mask)[False]} out of {len(meanVecOriginal)}")) 

782 

783 if (len(meanVecFinal) < len(parsIniPtc)): 

784 msg = (f"\nSERIOUS: Not enough data points ({len(meanVecFinal)}) compared to the number of" 

785 f"parameters of the PTC model({len(parsIniPtc)}). Setting {ampName} to BAD.") 

786 self.log.warn(msg) 

787 dataset.badAmps.append(ampName) 

788 dataset.gain[ampName] = np.nan 

789 dataset.gainErr[ampName] = np.nan 

790 dataset.noise[ampName] = np.nan 

791 dataset.noiseErr[ampName] = np.nan 

792 dataset.nonLinearity[ampName] = np.nan 

793 dataset.nonLinearityError[ampName] = np.nan 

794 dataset.nonLinearityResiduals[ampName] = np.nan 

795 continue 

796 

797 # Fit the PTC 

798 if self.config.doFitBootstrap: 

799 parsFit, parsFitErr = self._fitBootstrap(parsIniPtc, meanVecFinal, varVecFinal, ptcFunc) 

800 else: 

801 parsFit, parsFitErr = self._fitLeastSq(parsIniPtc, meanVecFinal, varVecFinal, ptcFunc) 

802 

803 dataset.ptcFitPars[ampName] = parsFit 

804 dataset.ptcFitParsError[ampName] = parsFitErr 

805 

806 if ptcFitType == 'ASTIERAPPROXIMATION': 

807 ptcGain = parsFit[1] 

808 ptcGainErr = parsFitErr[1] 

809 ptcNoise = np.sqrt(np.fabs(parsFit[2])) 

810 ptcNoiseErr = 0.5*(parsFitErr[2]/np.fabs(parsFit[2]))*np.sqrt(np.fabs(parsFit[2])) 

811 if ptcFitType == 'POLYNOMIAL': 

812 ptcGain = 1./parsFit[1] 

813 ptcGainErr = np.fabs(1./parsFit[1])*(parsFitErr[1]/parsFit[1]) 

814 ptcNoise = np.sqrt(np.fabs(parsFit[0]))*ptcGain 

815 ptcNoiseErr = (0.5*(parsFitErr[0]/np.fabs(parsFit[0]))*(np.sqrt(np.fabs(parsFit[0]))))*ptcGain 

816 

817 dataset.gain[ampName] = ptcGain 

818 dataset.gainErr[ampName] = ptcGainErr 

819 dataset.noise[ampName] = ptcNoise 

820 dataset.noiseErr[ampName] = ptcNoiseErr 

821 dataset.ptcFitType[ampName] = ptcFitType 

822 

823 # Non-linearity residuals (NL of mean vs time curve): percentage, and fit to a quadratic function 

824 # In this case, len(parsIniNonLinearity) = 3 indicates that we want a quadratic fit 

825 

826 (c0, linearizerTableRow, linResidualNonLinearity, parsFitNonLinearity, 

827 parsFitErrNonLinearity) = self.calculateLinearityResidualAndLinearizers(timeVecFinal, 

828 meanVecFinal) 

829 # LinearizerLookupTable 

830 tableArray[i, :] = linearizerTableRow 

831 

832 dataset.nonLinearity[ampName] = parsFitNonLinearity 

833 dataset.nonLinearityError[ampName] = parsFitErrNonLinearity 

834 dataset.nonLinearityResiduals[ampName] = linResidualNonLinearity 

835 dataset.coefficientLinearizeSquared[ampName] = c0 

836 

837 return dataset 

838 

839 def plot(self, dataRef, dataset, ptcFitType): 

840 dirname = dataRef.getUri(datasetType='cpPipePlotRoot', write=True) 

841 if not os.path.exists(dirname): 

842 os.makedirs(dirname) 

843 

844 detNum = dataRef.dataId[self.config.ccdKey] 

845 filename = f"PTC_det{detNum}.pdf" 

846 filenameFull = os.path.join(dirname, filename) 

847 with PdfPages(filenameFull) as pdfPages: 

848 self._plotPtc(dataset, ptcFitType, pdfPages) 

849 

850 def _plotPtc(self, dataset, ptcFitType, pdfPages): 

851 """Plot PTC, linearity, and linearity residual per amplifier""" 

852 

853 if ptcFitType == 'ASTIERAPPROXIMATION': 

854 ptcFunc = self.funcAstier 

855 stringTitle = r"Var = $\frac{1}{2g^2a_{00}}(\exp (2a_{00} \mu g) - 1) + \frac{n_{00}}{g^2}$" 

856 

857 if ptcFitType == 'POLYNOMIAL': 

858 ptcFunc = self.funcPolynomial 

859 stringTitle = f"Polynomial (degree: {self.config.polynomialFitDegree})" 

860 

861 legendFontSize = 7.5 

862 labelFontSize = 8 

863 titleFontSize = 10 

864 supTitleFontSize = 18 

865 markerSize = 25 

866 

867 # General determination of the size of the plot grid 

868 nAmps = len(dataset.ampNames) 

869 if nAmps == 2: 

870 nRows, nCols = 2, 1 

871 nRows = np.sqrt(nAmps) 

872 mantissa, _ = np.modf(nRows) 

873 if mantissa > 0: 

874 nRows = int(nRows) + 1 

875 nCols = nRows 

876 else: 

877 nRows = int(nRows) 

878 nCols = nRows 

879 

880 f, ax = plt.subplots(nrows=nRows, ncols=nCols, sharex='col', sharey='row', figsize=(13, 10)) 

881 f2, ax2 = plt.subplots(nrows=nRows, ncols=nCols, sharex='col', sharey='row', figsize=(13, 10)) 

882 

883 for i, (amp, a, a2) in enumerate(zip(dataset.ampNames, ax.flatten(), ax2.flatten())): 

884 meanVecOriginal = np.array(dataset.rawMeans[amp]) 

885 varVecOriginal = np.array(dataset.rawVars[amp]) 

886 mask = dataset.visitMask[amp] 

887 meanVecFinal = meanVecOriginal[mask] 

888 varVecFinal = varVecOriginal[mask] 

889 meanVecOutliers = meanVecOriginal[np.invert(mask)] 

890 varVecOutliers = varVecOriginal[np.invert(mask)] 

891 pars, parsErr = dataset.ptcFitPars[amp], dataset.ptcFitParsError[amp] 

892 

893 if ptcFitType == 'ASTIERAPPROXIMATION': 

894 ptcA00, ptcA00error = pars[0], parsErr[0] 

895 ptcGain, ptcGainError = pars[1], parsErr[1] 

896 ptcNoise = np.sqrt(np.fabs(pars[2])) 

897 ptcNoiseError = 0.5*(parsErr[2]/np.fabs(pars[2]))*np.sqrt(np.fabs(pars[2])) 

898 stringLegend = (f"a00: {ptcA00:.2e}+/-{ptcA00error:.2e}" 

899 f"\n Gain: {ptcGain:.4}+/-{ptcGainError:.2e}" 

900 f"\n Noise: {ptcNoise:.4}+/-{ptcNoiseError:.2e}") 

901 

902 if ptcFitType == 'POLYNOMIAL': 

903 ptcGain, ptcGainError = 1./pars[1], np.fabs(1./pars[1])*(parsErr[1]/pars[1]) 

904 ptcNoise = np.sqrt(np.fabs(pars[0]))*ptcGain 

905 ptcNoiseError = (0.5*(parsErr[0]/np.fabs(pars[0]))*(np.sqrt(np.fabs(pars[0]))))*ptcGain 

906 stringLegend = (f"Noise: {ptcNoise:.4}+/-{ptcNoiseError:.2e} \n" 

907 f"Gain: {ptcGain:.4}+/-{ptcGainError:.2e}") 

908 

909 minMeanVecFinal = np.min(meanVecFinal) 

910 maxMeanVecFinal = np.max(meanVecFinal) 

911 meanVecFit = np.linspace(minMeanVecFinal, maxMeanVecFinal, 100*len(meanVecFinal)) 

912 minMeanVecOriginal = np.min(meanVecOriginal) 

913 maxMeanVecOriginal = np.max(meanVecOriginal) 

914 deltaXlim = maxMeanVecOriginal - minMeanVecOriginal 

915 

916 a.plot(meanVecFit, ptcFunc(pars, meanVecFit), color='red') 

917 a.plot(meanVecFinal, pars[0] + pars[1]*meanVecFinal, color='green', linestyle='--') 

918 a.scatter(meanVecFinal, varVecFinal, c='blue', marker='o', s=markerSize) 

919 a.scatter(meanVecOutliers, varVecOutliers, c='magenta', marker='s', s=markerSize) 

920 a.set_xlabel(r'Mean signal ($\mu$, ADU)', fontsize=labelFontSize) 

921 a.set_xticks(meanVecOriginal) 

922 a.set_ylabel(r'Variance (ADU$^2$)', fontsize=labelFontSize) 

923 a.tick_params(labelsize=11) 

924 a.text(0.03, 0.8, stringLegend, transform=a.transAxes, fontsize=legendFontSize) 

925 a.set_xscale('linear', fontsize=labelFontSize) 

926 a.set_yscale('linear', fontsize=labelFontSize) 

927 a.set_title(amp, fontsize=titleFontSize) 

928 a.set_xlim([minMeanVecOriginal - 0.2*deltaXlim, maxMeanVecOriginal + 0.2*deltaXlim]) 

929 

930 # Same, but in log-scale 

931 a2.plot(meanVecFit, ptcFunc(pars, meanVecFit), color='red') 

932 a2.scatter(meanVecFinal, varVecFinal, c='blue', marker='o', s=markerSize) 

933 a2.scatter(meanVecOutliers, varVecOutliers, c='magenta', marker='s', s=markerSize) 

934 a2.set_xlabel(r'Mean Signal ($\mu$, ADU)', fontsize=labelFontSize) 

935 a2.set_ylabel(r'Variance (ADU$^2$)', fontsize=labelFontSize) 

936 a2.tick_params(labelsize=11) 

937 a2.text(0.03, 0.8, stringLegend, transform=a2.transAxes, fontsize=legendFontSize) 

938 a2.set_xscale('log') 

939 a2.set_yscale('log') 

940 a2.set_title(amp, fontsize=titleFontSize) 

941 a2.set_xlim([minMeanVecOriginal, maxMeanVecOriginal]) 

942 

943 f.suptitle(f"PTC \n Fit: " + stringTitle, fontsize=20) 

944 pdfPages.savefig(f) 

945 f2.suptitle(f"PTC (log-log)", fontsize=20) 

946 pdfPages.savefig(f2) 

947 

948 # Plot mean vs time 

949 f, ax = plt.subplots(nrows=4, ncols=4, sharex='col', sharey='row', figsize=(13, 10)) 

950 for i, (amp, a) in enumerate(zip(dataset.ampNames, ax.flatten())): 

951 meanVecFinal = np.array(dataset.rawMeans[amp])[dataset.visitMask[amp]] 

952 timeVecFinal = np.array(dataset.rawExpTimes[amp])[dataset.visitMask[amp]] 

953 

954 pars, parsErr = dataset.nonLinearity[amp], dataset.nonLinearityError[amp] 

955 c0, c0Error = pars[0], parsErr[0] 

956 c1, c1Error = pars[1], parsErr[1] 

957 c2, c2Error = pars[2], parsErr[2] 

958 stringLegend = f"c0: {c0:.4}+/-{c0Error:.2e}\n c1: {c1:.4}+/-{c1Error:.2e}" \ 

959 + f"\n c2(NL): {c2:.2e}+/-{c2Error:.2e}" 

960 a.scatter(timeVecFinal, meanVecFinal) 

961 a.plot(timeVecFinal, self.funcPolynomial(pars, timeVecFinal), color='red') 

962 a.set_xlabel('Time (sec)', fontsize=labelFontSize) 

963 a.set_xticks(timeVecFinal) 

964 a.set_ylabel(r'Mean signal ($\mu$, ADU)', fontsize=labelFontSize) 

965 a.tick_params(labelsize=labelFontSize) 

966 a.text(0.03, 0.75, stringLegend, transform=a.transAxes, fontsize=legendFontSize) 

967 a.set_xscale('linear', fontsize=labelFontSize) 

968 a.set_yscale('linear', fontsize=labelFontSize) 

969 a.set_title(amp, fontsize=titleFontSize) 

970 

971 f.suptitle("Linearity \n Fit: " + r"$\mu = c_0 + c_1 t + c_2 t^2$", fontsize=supTitleFontSize) 

972 pdfPages.savefig() 

973 

974 # Plot linearity residual 

975 f, ax = plt.subplots(nrows=4, ncols=4, sharex='col', sharey='row', figsize=(13, 10)) 

976 for i, (amp, a) in enumerate(zip(dataset.ampNames, ax.flatten())): 

977 meanVecFinal = np.array(dataset.rawMeans[amp])[dataset.visitMask[amp]] 

978 linRes = np.array(dataset.nonLinearityResiduals[amp]) 

979 

980 a.scatter(meanVecFinal, linRes) 

981 a.axhline(y=0, color='k') 

982 a.axvline(x=timeVecFinal[self.config.linResidualTimeIndex], color='g', linestyle='--') 

983 a.set_xlabel(r'Mean signal ($\mu$, ADU)', fontsize=labelFontSize) 

984 a.set_xticks(meanVecFinal) 

985 a.set_ylabel('LR (%)', fontsize=labelFontSize) 

986 a.tick_params(labelsize=labelFontSize) 

987 a.set_xscale('linear', fontsize=labelFontSize) 

988 a.set_yscale('linear', fontsize=labelFontSize) 

989 a.set_title(amp, fontsize=titleFontSize) 

990 

991 f.suptitle(r"Linearity Residual: $100(1 - \mu_{\rm{ref}}/t_{\rm{ref}})/(\mu / t))$" + "\n" + 

992 r"$t_{\rm{ref}}$: " + f"{timeVecFinal[2]} s", fontsize=supTitleFontSize) 

993 pdfPages.savefig() 

994 

995 return