Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of cp_pipe. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21# 

22 

23__all__ = ['MeasurePhotonTransferCurveTask', 

24 'MeasurePhotonTransferCurveTaskConfig', 

25 'PhotonTransferCurveDataset'] 

26 

27import numpy as np 

28import matplotlib.pyplot as plt 

29import os 

30from matplotlib.backends.backend_pdf import PdfPages 

31from sqlite3 import OperationalError 

32from collections import Counter 

33 

34import lsst.afw.math as afwMath 

35import lsst.pex.config as pexConfig 

36import lsst.pipe.base as pipeBase 

37from lsst.ip.isr import IsrTask 

38from .utils import (NonexistentDatasetTaskDataIdContainer, PairedVisitListTaskRunner, 

39 checkExpLengthEqual, validateIsrConfig) 

40from scipy.optimize import leastsq, least_squares 

41import numpy.polynomial.polynomial as poly 

42 

43 

44class MeasurePhotonTransferCurveTaskConfig(pexConfig.Config): 

45 """Config class for photon transfer curve measurement task""" 

46 isr = pexConfig.ConfigurableField( 

47 target=IsrTask, 

48 doc="""Task to perform instrumental signature removal.""", 

49 ) 

50 isrMandatorySteps = pexConfig.ListField( 

51 dtype=str, 

52 doc="isr operations that must be performed for valid results. Raises if any of these are False.", 

53 default=['doAssembleCcd'] 

54 ) 

55 isrForbiddenSteps = pexConfig.ListField( 

56 dtype=str, 

57 doc="isr operations that must NOT be performed for valid results. Raises if any of these are True", 

58 default=['doFlat', 'doFringe', 'doAddDistortionModel', 'doBrighterFatter', 'doUseOpticsTransmission', 

59 'doUseFilterTransmission', 'doUseSensorTransmission', 'doUseAtmosphereTransmission'] 

60 ) 

61 isrDesirableSteps = pexConfig.ListField( 

62 dtype=str, 

63 doc="isr operations that it is advisable to perform, but are not mission-critical." + 

64 " WARNs are logged for any of these found to be False.", 

65 default=['doBias', 'doDark', 'doCrosstalk', 'doDefect'] 

66 ) 

67 isrUndesirableSteps = pexConfig.ListField( 

68 dtype=str, 

69 doc="isr operations that it is *not* advisable to perform in the general case, but are not" + 

70 " forbidden as some use-cases might warrant them." + 

71 " WARNs are logged for any of these found to be True.", 

72 default=['doLinearize'] 

73 ) 

74 ccdKey = pexConfig.Field( 

75 dtype=str, 

76 doc="The key by which to pull a detector from a dataId, e.g. 'ccd' or 'detector'.", 

77 default='ccd', 

78 ) 

79 makePlots = pexConfig.Field( 

80 dtype=bool, 

81 doc="Plot the PTC curves?", 

82 default=False, 

83 ) 

84 ptcFitType = pexConfig.ChoiceField( 

85 dtype=str, 

86 doc="Fit PTC to approximation in Astier+19 (Equation 16) or to a polynomial.", 

87 default="POLYNOMIAL", 

88 allowed={ 

89 "POLYNOMIAL": "n-degree polynomial (use 'polynomialFitDegree' to set 'n').", 

90 "ASTIERAPPROXIMATION": "Approximation in Astier+19 (Eq. 16)." 

91 } 

92 ) 

93 polynomialFitDegree = pexConfig.Field( 

94 dtype=int, 

95 doc="Degree of polynomial to fit the PTC, when 'ptcFitType'=POLYNOMIAL.", 

96 default=2, 

97 ) 

98 polynomialFitDegreeNonLinearity = pexConfig.Field( 

99 dtype=int, 

100 doc="Degree of polynomial to fit the meanSignal vs exposureTime curve to produce" + 

101 " the table for LinearizerLookupTable.", 

102 default=3, 

103 ) 

104 binSize = pexConfig.Field( 

105 dtype=int, 

106 doc="Bin the image by this factor in both dimensions.", 

107 default=1, 

108 ) 

109 minMeanSignal = pexConfig.Field( 

110 dtype=float, 

111 doc="Minimum value (inclusive) of mean signal (in ADU) above which to consider.", 

112 default=0, 

113 ) 

114 maxMeanSignal = pexConfig.Field( 

115 dtype=float, 

116 doc="Maximum value (inclusive) of mean signal (in ADU) below which to consider.", 

117 default=9e6, 

118 ) 

119 initialNonLinearityExclusionThresholdPositive = pexConfig.RangeField( 

120 dtype=float, 

121 doc="Initially exclude data points with a variance that are more than a factor of this from being" 

122 " linear in the positive direction, from the PTC fit. Note that these points will also be" 

123 " excluded from the non-linearity fit. This is done before the iterative outlier rejection," 

124 " to allow an accurate determination of the sigmas for said iterative fit.", 

125 default=0.12, 

126 min=0.0, 

127 max=1.0, 

128 ) 

129 initialNonLinearityExclusionThresholdNegative = pexConfig.RangeField( 

130 dtype=float, 

131 doc="Initially exclude data points with a variance that are more than a factor of this from being" 

132 " linear in the negative direction, from the PTC fit. Note that these points will also be" 

133 " excluded from the non-linearity fit. This is done before the iterative outlier rejection," 

134 " to allow an accurate determination of the sigmas for said iterative fit.", 

135 default=0.25, 

136 min=0.0, 

137 max=1.0, 

138 ) 

139 sigmaCutPtcOutliers = pexConfig.Field( 

140 dtype=float, 

141 doc="Sigma cut for outlier rejection in PTC.", 

142 default=5.0, 

143 ) 

144 maxIterationsPtcOutliers = pexConfig.Field( 

145 dtype=int, 

146 doc="Maximum number of iterations for outlier rejection in PTC.", 

147 default=2, 

148 ) 

149 doFitBootstrap = pexConfig.Field( 

150 dtype=bool, 

151 doc="Use bootstrap for the PTC fit parameters and errors?.", 

152 default=False, 

153 ) 

154 linResidualTimeIndex = pexConfig.Field( 

155 dtype=int, 

156 doc="Index position in time array for reference time in linearity residual calculation.", 

157 default=2, 

158 ) 

159 maxAduForLookupTableLinearizer = pexConfig.Field( 

160 dtype=int, 

161 doc="Maximum ADU value for the LookupTable linearizer.", 

162 default=2**18, 

163 ) 

164 

165 

166class PhotonTransferCurveDataset: 

167 """A simple class to hold the output data from the PTC task. 

168 

169 The dataset is made up of a dictionary for each item, keyed by the 

170 amplifiers' names, which much be supplied at construction time. 

171 

172 New items cannot be added to the class to save accidentally saving to the 

173 wrong property, and the class can be frozen if desired. 

174 

175 inputVisitPairs records the visits used to produce the data. 

176 When fitPtcAndNonLinearity() is run, a mask is built up, which is by definition 

177 always the same length as inputVisitPairs, rawExpTimes, rawMeans 

178 and rawVars, and is a list of bools, which are incrementally set to False 

179 as points are discarded from the fits. 

180 

181 PTC fit parameters for polynomials are stored in a list in ascending order 

182 of polynomial term, i.e. par[0]*x^0 + par[1]*x + par[2]*x^2 etc 

183 with the length of the list corresponding to the order of the polynomial 

184 plus one. 

185 """ 

186 def __init__(self, ampNames): 

187 # add items to __dict__ directly because __setattr__ is overridden 

188 

189 # instance variables 

190 self.__dict__["ampNames"] = ampNames 

191 self.__dict__["badAmps"] = [] 

192 

193 # raw data variables 

194 self.__dict__["inputVisitPairs"] = {ampName: [] for ampName in ampNames} 

195 self.__dict__["visitMask"] = {ampName: [] for ampName in ampNames} 

196 self.__dict__["rawExpTimes"] = {ampName: [] for ampName in ampNames} 

197 self.__dict__["rawMeans"] = {ampName: [] for ampName in ampNames} 

198 self.__dict__["rawVars"] = {ampName: [] for ampName in ampNames} 

199 

200 # fit information 

201 self.__dict__["ptcFitType"] = {ampName: "" for ampName in ampNames} 

202 self.__dict__["ptcFitPars"] = {ampName: [] for ampName in ampNames} 

203 self.__dict__["ptcFitParsError"] = {ampName: [] for ampName in ampNames} 

204 self.__dict__["nonLinearity"] = {ampName: [] for ampName in ampNames} 

205 self.__dict__["nonLinearityError"] = {ampName: [] for ampName in ampNames} 

206 self.__dict__["nonLinearityResiduals"] = {ampName: [] for ampName in ampNames} 

207 self.__dict__["coefficientLinearizeSquared"] = {ampName: [] for ampName in ampNames} 

208 

209 # final results 

210 self.__dict__["gain"] = {ampName: -1. for ampName in ampNames} 

211 self.__dict__["gainErr"] = {ampName: -1. for ampName in ampNames} 

212 self.__dict__["noise"] = {ampName: -1. for ampName in ampNames} 

213 self.__dict__["noiseErr"] = {ampName: -1. for ampName in ampNames} 

214 self.__dict__["coefficientLinearizeSquared"] = {ampName: [] for ampName in ampNames} 

215 

216 def __setattr__(self, attribute, value): 

217 """Protect class attributes""" 

218 if attribute not in self.__dict__: 

219 raise AttributeError(f"{attribute} is not already a member of PhotonTransferCurveDataset, which" 

220 " does not support setting of new attributes.") 

221 else: 

222 self.__dict__[attribute] = value 

223 

224 def getVisitsUsed(self, ampName): 

225 """Get the visits used, i.e. not discarded, for a given amp. 

226 

227 If no mask has been created yet, all visits are returned. 

228 """ 

229 if self.visitMask[ampName] == []: 

230 return self.inputVisitPairs[ampName] 

231 

232 # if the mask exists it had better be the same length as the visitPairs 

233 assert len(self.visitMask[ampName]) == len(self.inputVisitPairs[ampName]) 

234 

235 pairs = self.inputVisitPairs[ampName] 

236 mask = self.visitMask[ampName] 

237 # cast to bool required because numpy 

238 return [(v1, v2) for ((v1, v2), m) in zip(pairs, mask) if bool(m) is True] 

239 

240 def getGoodAmps(self): 

241 return [amp for amp in self.ampNames if amp not in self.badAmps] 

242 

243 

244class MeasurePhotonTransferCurveTask(pipeBase.CmdLineTask): 

245 """A class to calculate, fit, and plot a PTC from a set of flat pairs. 

246 

247 The Photon Transfer Curve (var(signal) vs mean(signal)) is a standard tool 

248 used in astronomical detectors characterization (e.g., Janesick 2001, 

249 Janesick 2007). This task calculates the PTC from a series of pairs of 

250 flat-field images; each pair taken at identical exposure times. The 

251 difference image of each pair is formed to eliminate fixed pattern noise, 

252 and then the variance of the difference image and the mean of the average image 

253 are used to produce the PTC. An n-degree polynomial or the approximation in Equation 

254 16 of Astier+19 ("The Shape of the Photon Transfer Curve of CCD sensors", 

255 arXiv:1905.08677) can be fitted to the PTC curve. These models include 

256 parameters such as the gain (e/ADU) and readout noise. 

257 

258 Parameters 

259 ---------- 

260 

261 *args: `list` 

262 Positional arguments passed to the Task constructor. None used at this 

263 time. 

264 **kwargs: `dict` 

265 Keyword arguments passed on to the Task constructor. None used at this 

266 time. 

267 

268 """ 

269 

270 RunnerClass = PairedVisitListTaskRunner 

271 ConfigClass = MeasurePhotonTransferCurveTaskConfig 

272 _DefaultName = "measurePhotonTransferCurve" 

273 

274 def __init__(self, *args, **kwargs): 

275 pipeBase.CmdLineTask.__init__(self, *args, **kwargs) 

276 self.makeSubtask("isr") 

277 plt.interactive(False) # stop windows popping up when plotting. When headless, use 'agg' backend too 

278 validateIsrConfig(self.isr, self.config.isrMandatorySteps, 

279 self.config.isrForbiddenSteps, self.config.isrDesirableSteps, checkTrim=False) 

280 self.config.validate() 

281 self.config.freeze() 

282 

283 @classmethod 

284 def _makeArgumentParser(cls): 

285 """Augment argument parser for the MeasurePhotonTransferCurveTask.""" 

286 parser = pipeBase.ArgumentParser(name=cls._DefaultName) 

287 parser.add_argument("--visit-pairs", dest="visitPairs", nargs="*", 

288 help="Visit pairs to use. Each pair must be of the form INT,INT e.g. 123,456") 

289 parser.add_id_argument("--id", datasetType="photonTransferCurveDataset", 

290 ContainerClass=NonexistentDatasetTaskDataIdContainer, 

291 help="The ccds to use, e.g. --id ccd=0..100") 

292 return parser 

293 

294 @pipeBase.timeMethod 

295 def runDataRef(self, dataRef, visitPairs): 

296 """Run the Photon Transfer Curve (PTC) measurement task. 

297 

298 For a dataRef (which is each detector here), 

299 and given a list of visit pairs at different exposure times, 

300 measure the PTC. 

301 

302 Parameters 

303 ---------- 

304 dataRef : list of lsst.daf.persistence.ButlerDataRef 

305 dataRef for the detector for the visits to be fit. 

306 visitPairs : `iterable` of `tuple` of `int` 

307 Pairs of visit numbers to be processed together 

308 """ 

309 

310 # setup necessary objects 

311 detNum = dataRef.dataId[self.config.ccdKey] 

312 detector = dataRef.get('camera')[dataRef.dataId[self.config.ccdKey]] 

313 # expand some missing fields that we need for lsstCam. This is a work-around 

314 # for Gen2 problems that I (RHL) don't feel like solving. The calibs pipelines 

315 # (which inherit from CalibTask) use addMissingKeys() to do basically the same thing 

316 # 

317 # Basically, the butler's trying to look up the fields in `raw_visit` which won't work 

318 for name in dataRef.getButler().getKeys('bias'): 

319 if name not in dataRef.dataId: 

320 try: 

321 dataRef.dataId[name] = \ 

322 dataRef.getButler().queryMetadata('raw', [name], detector=detNum)[0] 

323 except OperationalError: 

324 pass 

325 

326 amps = detector.getAmplifiers() 

327 ampNames = [amp.getName() for amp in amps] 

328 dataset = PhotonTransferCurveDataset(ampNames) 

329 

330 self.log.info('Measuring PTC using %s visits for detector %s' % (visitPairs, detNum)) 

331 

332 for (v1, v2) in visitPairs: 

333 # Perform ISR on each exposure 

334 dataRef.dataId['visit'] = v1 

335 exp1 = self.isr.runDataRef(dataRef).exposure 

336 dataRef.dataId['visit'] = v2 

337 exp2 = self.isr.runDataRef(dataRef).exposure 

338 del dataRef.dataId['visit'] 

339 

340 checkExpLengthEqual(exp1, exp2, v1, v2, raiseWithMessage=True) 

341 expTime = exp1.getInfo().getVisitInfo().getExposureTime() 

342 

343 for amp in detector: 

344 mu, varDiff = self.measureMeanVarPair(exp1, exp2, region=amp.getBBox()) 

345 ampName = amp.getName() 

346 

347 dataset.rawExpTimes[ampName].append(expTime) 

348 dataset.rawMeans[ampName].append(mu) 

349 dataset.rawVars[ampName].append(varDiff) 

350 dataset.inputVisitPairs[ampName].append((v1, v2)) 

351 

352 numberAmps = len(detector.getAmplifiers()) 

353 numberAduValues = self.config.maxAduForLookupTableLinearizer 

354 lookupTableArray = np.zeros((numberAmps, numberAduValues), dtype=np.float32) 

355 

356 # Fit PTC and (non)linearity of signal vs time curve, produce linearizer 

357 self.fitPtcAndNonLinearity(dataset, lookupTableArray, ptcFitType=self.config.ptcFitType) 

358 

359 if self.config.makePlots: 

360 self.plot(dataRef, dataset, ptcFitType=self.config.ptcFitType) 

361 

362 # Save data, PTC fit, and NL fit dictionaries 

363 self.log.info(f"Writing PTC and NL data to {dataRef.getUri(write=True)}") 

364 dataRef.put(dataset, datasetType="photonTransferCurveDataset") 

365 

366 self.log.info('Finished measuring PTC for in detector %s' % detNum) 

367 

368 return pipeBase.Struct(exitStatus=0) 

369 

370 def measureMeanVarPair(self, exposure1, exposure2, region=None): 

371 """Calculate the mean signal of two exposures and the variance of their difference. 

372 

373 Parameters 

374 ---------- 

375 exposure1 : `lsst.afw.image.exposure.exposure.ExposureF` 

376 First exposure of flat field pair. 

377 

378 exposure2 : `lsst.afw.image.exposure.exposure.ExposureF` 

379 Second exposure of flat field pair. 

380 

381 region : `lsst.geom.Box2I` 

382 Region of each exposure where to perform the calculations (e.g, an amplifier). 

383 

384 Return 

385 ------ 

386 

387 mu : `np.float` 

388 0.5*(mu1 + mu2), where mu1, and mu2 are the clipped means of the regions in 

389 both exposures. 

390 

391 varDiff : `np.float` 

392 Half of the clipped variance of the difference of the regions inthe two input 

393 exposures. 

394 """ 

395 

396 if region is not None: 

397 im1Area = exposure1.maskedImage[region] 

398 im2Area = exposure2.maskedImage[region] 

399 else: 

400 im1Area = exposure1.maskedImage 

401 im2Area = exposure2.maskedImage 

402 

403 im1Area = afwMath.binImage(im1Area, self.config.binSize) 

404 im2Area = afwMath.binImage(im2Area, self.config.binSize) 

405 

406 # Clipped mean of images; then average of mean. 

407 mu1 = afwMath.makeStatistics(im1Area, afwMath.MEANCLIP).getValue() 

408 mu2 = afwMath.makeStatistics(im2Area, afwMath.MEANCLIP).getValue() 

409 mu = 0.5*(mu1 + mu2) 

410 

411 # Take difference of pairs 

412 # symmetric formula: diff = (mu2*im1-mu1*im2)/(0.5*(mu1+mu2)) 

413 temp = im2Area.clone() 

414 temp *= mu1 

415 diffIm = im1Area.clone() 

416 diffIm *= mu2 

417 diffIm -= temp 

418 diffIm /= mu 

419 

420 varDiff = 0.5*(afwMath.makeStatistics(diffIm, afwMath.VARIANCECLIP).getValue()) 

421 

422 return mu, varDiff 

423 

424 def _fitLeastSq(self, initialParams, dataX, dataY, function): 

425 """Do a fit and estimate the parameter errors using using scipy.optimize.leastq. 

426 

427 optimize.leastsq returns the fractional covariance matrix. To estimate the 

428 standard deviation of the fit parameters, multiply the entries of this matrix 

429 by the reduced chi squared and take the square root of the diagonal elements. 

430 

431 Parameters 

432 ---------- 

433 initialParams : list of np.float 

434 initial values for fit parameters. For ptcFitType=POLYNOMIAL, its length 

435 determines the degree of the polynomial. 

436 

437 dataX : np.array of np.float 

438 Data in the abscissa axis. 

439 

440 dataY : np.array of np.float 

441 Data in the ordinate axis. 

442 

443 function : callable object (function) 

444 Function to fit the data with. 

445 

446 Return 

447 ------ 

448 pFitSingleLeastSquares : list of np.float 

449 List with fitted parameters. 

450 

451 pErrSingleLeastSquares : list of np.float 

452 List with errors for fitted parameters. 

453 """ 

454 

455 def errFunc(p, x, y): 

456 return function(p, x) - y 

457 

458 pFit, pCov, infoDict, errMessage, success = leastsq(errFunc, initialParams, 

459 args=(dataX, dataY), full_output=1, epsfcn=0.0001) 

460 

461 if (len(dataY) > len(initialParams)) and pCov is not None: 

462 reducedChiSq = (errFunc(pFit, dataX, dataY)**2).sum()/(len(dataY)-len(initialParams)) 

463 pCov *= reducedChiSq 

464 else: 

465 pCov[:, :] = np.inf 

466 

467 errorVec = [] 

468 for i in range(len(pFit)): 

469 errorVec.append(np.fabs(pCov[i][i])**0.5) 

470 

471 pFitSingleLeastSquares = pFit 

472 pErrSingleLeastSquares = np.array(errorVec) 

473 

474 return pFitSingleLeastSquares, pErrSingleLeastSquares 

475 

476 def _fitBootstrap(self, initialParams, dataX, dataY, function, confidenceSigma=1.): 

477 """Do a fit using least squares and bootstrap to estimate parameter errors. 

478 

479 The bootstrap error bars are calculated by fitting 100 random data sets. 

480 

481 Parameters 

482 ---------- 

483 initialParams : list of np.float 

484 initial values for fit parameters. For ptcFitType=POLYNOMIAL, its length 

485 determines the degree of the polynomial. 

486 

487 dataX : np.array of np.float 

488 Data in the abscissa axis. 

489 

490 dataY : np.array of np.float 

491 Data in the ordinate axis. 

492 

493 function : callable object (function) 

494 Function to fit the data with. 

495 

496 confidenceSigma : np.float 

497 Number of sigmas that determine confidence interval for the bootstrap errors. 

498 

499 Return 

500 ------ 

501 pFitBootstrap : list of np.float 

502 List with fitted parameters. 

503 

504 pErrBootstrap : list of np.float 

505 List with errors for fitted parameters. 

506 """ 

507 

508 def errFunc(p, x, y): 

509 return function(p, x) - y 

510 

511 # Fit first time 

512 pFit, _ = leastsq(errFunc, initialParams, args=(dataX, dataY), full_output=0) 

513 

514 # Get the stdev of the residuals 

515 residuals = errFunc(pFit, dataX, dataY) 

516 sigmaErrTotal = np.std(residuals) 

517 

518 # 100 random data sets are generated and fitted 

519 pars = [] 

520 for i in range(100): 

521 randomDelta = np.random.normal(0., sigmaErrTotal, len(dataY)) 

522 randomDataY = dataY + randomDelta 

523 randomFit, _ = leastsq(errFunc, initialParams, 

524 args=(dataX, randomDataY), full_output=0) 

525 pars.append(randomFit) 

526 pars = np.array(pars) 

527 meanPfit = np.mean(pars, 0) 

528 

529 # confidence interval for parameter estimates 

530 nSigma = confidenceSigma 

531 errPfit = nSigma*np.std(pars, 0) 

532 pFitBootstrap = meanPfit 

533 pErrBootstrap = errPfit 

534 return pFitBootstrap, pErrBootstrap 

535 

536 def funcPolynomial(self, pars, x): 

537 """Polynomial function definition""" 

538 return poly.polyval(x, [*pars]) 

539 

540 def funcAstier(self, pars, x): 

541 """Single brighter-fatter parameter model for PTC; Equation 16 of Astier+19""" 

542 a00, gain, noise = pars 

543 return 0.5/(a00*gain*gain)*(np.exp(2*a00*x*gain)-1) + noise/(gain*gain) 

544 

545 @staticmethod 

546 def _initialParsForPolynomial(order): 

547 assert(order >= 2) 

548 pars = np.zeros(order, dtype=np.float) 

549 pars[0] = 10 

550 pars[1] = 1 

551 pars[2:] = 0.0001 

552 return pars 

553 

554 @staticmethod 

555 def _boundsForPolynomial(initialPars): 

556 lowers = [np.NINF for p in initialPars] 

557 uppers = [np.inf for p in initialPars] 

558 lowers[1] = 0 # no negative gains 

559 return (lowers, uppers) 

560 

561 @staticmethod 

562 def _boundsForAstier(initialPars): 

563 lowers = [np.NINF for p in initialPars] 

564 uppers = [np.inf for p in initialPars] 

565 return (lowers, uppers) 

566 

567 @staticmethod 

568 def _getInitialGoodPoints(means, variances, maxDeviationPositive, maxDeviationNegative): 

569 """Return a boolean array to mask bad points. 

570 

571 A linear function has a constant ratio, so find the median 

572 value of the ratios, and exclude the points that deviate 

573 from that by more than a factor of maxDeviationPositive/negative. 

574 Asymmetric deviations are supported as we expect the PTC to turn 

575 down as the flux increases, but sometimes it anomalously turns 

576 upwards just before turning over, which ruins the fits, so it 

577 is wise to be stricter about restricting positive outliers than 

578 negative ones. 

579 

580 Too high and points that are so bad that fit will fail will be included 

581 Too low and the non-linear points will be excluded, biasing the NL fit.""" 

582 ratios = [b/a for (a, b) in zip(means, variances)] 

583 medianRatio = np.median(ratios) 

584 ratioDeviations = [(r/medianRatio)-1 for r in ratios] 

585 

586 # so that it doesn't matter if the deviation is expressed as positive or negative 

587 maxDeviationPositive = abs(maxDeviationPositive) 

588 maxDeviationNegative = -1. * abs(maxDeviationNegative) 

589 

590 goodPoints = np.array([True if (r < maxDeviationPositive and r > maxDeviationNegative) 

591 else False for r in ratioDeviations]) 

592 return goodPoints 

593 

594 def _makeZeroSafe(self, array, warn=True, substituteValue=1e-9): 

595 """""" 

596 nBad = Counter(array)[0] 

597 if nBad == 0: 

598 return array 

599 

600 if warn: 

601 msg = f"Found {nBad} zeros in array at elements {[x for x in np.where(array==0)[0]]}" 

602 self.log.warn(msg) 

603 

604 array[array == 0] = substituteValue 

605 return array 

606 

607 def calculateLinearityResidualAndLinearizers(self, exposureTimeVector, meanSignalVector): 

608 """Calculate linearity residual and fit an n-order polynomial to the mean vs time curve 

609 to produce corrections (deviation from linear part of polynomial) for a particular amplifier 

610 to populate LinearizeLookupTable. Use quadratic and linear parts of this polynomial to approximate 

611 c0 for LinearizeSquared." 

612 

613 Parameters 

614 --------- 

615 

616 exposureTimeVector: `list` of `np.float` 

617 List of exposure times for each flat pair 

618 

619 meanSignalVector: `list` of `np.float` 

620 List of mean signal from diference image of flat pairs 

621 

622 Returns 

623 ------- 

624 c0: `np.float` 

625 Coefficient for LinearizeSquared, where corrImage = uncorrImage + c0*uncorrImage^2. 

626 c0 ~ -k2/(k1^2), where k1 and k2 are fit from 

627 meanSignalVector = k0 + k1*exposureTimeVector + k2*exposureTimeVector^2 +... 

628 + kn*exposureTimeVector^n, with n = "polynomialFitDegreeNonLinearity". 

629 

630 linearizerTableRow: list of `np.float` 

631 One dimensional array with deviation from linear part of n-order polynomial fit 

632 to mean vs time curve. This array will be one row (for the particular amplifier at hand) 

633 of the table array for LinearizeLookupTable. 

634 

635 linResidual: list of `np.float` 

636 Linearity residual from the mean vs time curve, defined as 

637 100*(1 - meanSignalReference/expTimeReference/(meanSignal/expTime). 

638 

639 parsFit: list of `np.float` 

640 Parameters from n-order polynomial fit to mean vs time curve. 

641 

642 parsFitErr: list of `np.float` 

643 Parameters from n-order polynomial fit to mean vs time curve. 

644 

645 """ 

646 

647 # Lookup table linearizer 

648 parsIniNonLinearity = self._initialParsForPolynomial(self.config.polynomialFitDegreeNonLinearity + 1) 

649 if self.config.doFitBootstrap: 

650 parsFit, parsFitErr = self._fitBootstrap(parsIniNonLinearity, exposureTimeVector, 

651 meanSignalVector, self.funcPolynomial) 

652 else: 

653 parsFit, parsFitErr = self._fitLeastSq(parsIniNonLinearity, exposureTimeVector, meanSignalVector, 

654 self.funcPolynomial) 

655 

656 # Use linear part to get time at wich signal is maxAduForLookupTableLinearizer ADU 

657 tMax = (self.config.maxAduForLookupTableLinearizer - parsFit[0])/parsFit[1] 

658 timeRange = np.linspace(0, tMax, self.config.maxAduForLookupTableLinearizer) 

659 signalIdeal = parsFit[0] + parsFit[1]*timeRange 

660 signalUncorrected = self.funcPolynomial(parsFit, timeRange) 

661 linearizerTableRow = signalIdeal - signalUncorrected # LinearizerLookupTable has corrections 

662 

663 # Use quadratic and linear part of fit to produce c0 for LinearizeSquared 

664 # Check that magnitude of higher order (>= 3) coefficents of the polyFit are small, 

665 # i.e., less than threshold = 1e-10 (typical quadratic and cubic coefficents are ~1e-6 

666 # and ~1e-12). 

667 k1, k2 = parsFit[1], parsFit[2] 

668 c0 = -k2/(k1**2) # c0 coefficient for LinearizeSquared 

669 for coefficient in parsFit[3:]: 

670 if np.fabs(coefficient) > 1e-10: 

671 msg = f"Coefficient {coefficient} in polynomial fit larger than threshold 1e-10." 

672 self.log.warn(msg) 

673 

674 # Linearity residual 

675 linResidualTimeIndex = self.config.linResidualTimeIndex 

676 if exposureTimeVector[linResidualTimeIndex] == 0.0: 

677 raise RuntimeError("Reference time for linearity residual can't be 0.0") 

678 linResidual = 100*(1 - ((meanSignalVector[linResidualTimeIndex] / 

679 exposureTimeVector[linResidualTimeIndex]) / 

680 (meanSignalVector/exposureTimeVector))) 

681 

682 return c0, linearizerTableRow, linResidual, parsFit, parsFitErr 

683 

684 def fitPtcAndNonLinearity(self, dataset, tableArray, ptcFitType): 

685 """Fit the photon transfer curve and calculate linearity and residuals. 

686 

687 Fit the photon transfer curve with either a polynomial of the order 

688 specified in the task config, or using the Astier approximation. 

689 

690 Sigma clipping is performed iteratively for the fit, as well as an 

691 initial clipping of data points that are more than 

692 config.initialNonLinearityExclusionThreshold away from lying on a 

693 straight line. This other step is necessary because the photon transfer 

694 curve turns over catastrophically at very high flux (because saturation 

695 drops the variance to ~0) and these far outliers cause the initial fit 

696 to fail, meaning the sigma cannot be calculated to perform the 

697 sigma-clipping. 

698 

699 Parameters 

700 ---------- 

701 dataset : `lsst.cp.pipe.ptc.PhotonTransferCurveDataset` 

702 The dataset containing the means, variances and exposure times 

703 ptcFitType : `str` 

704 Fit a 'POLYNOMIAL' (degree: 'polynomialFitDegree') or 

705 'ASTIERAPPROXIMATION' to the PTC 

706 tableArray : `np.array` 

707 Look-up table array with size rows=nAmps and columns=ADU values 

708 """ 

709 

710 def errFunc(p, x, y): 

711 return ptcFunc(p, x) - y 

712 

713 sigmaCutPtcOutliers = self.config.sigmaCutPtcOutliers 

714 maxIterationsPtcOutliers = self.config.maxIterationsPtcOutliers 

715 

716 for i, ampName in enumerate(dataset.ampNames): 

717 timeVecOriginal = np.array(dataset.rawExpTimes[ampName]) 

718 meanVecOriginal = np.array(dataset.rawMeans[ampName]) 

719 varVecOriginal = np.array(dataset.rawVars[ampName]) 

720 varVecOriginal = self._makeZeroSafe(varVecOriginal) 

721 

722 mask = ((meanVecOriginal >= self.config.minMeanSignal) & 

723 (meanVecOriginal <= self.config.maxMeanSignal)) 

724 

725 goodPoints = self._getInitialGoodPoints(meanVecOriginal, varVecOriginal, 

726 self.config.initialNonLinearityExclusionThresholdPositive, 

727 self.config.initialNonLinearityExclusionThresholdNegative) 

728 mask = mask & goodPoints 

729 

730 if ptcFitType == 'ASTIERAPPROXIMATION': 

731 ptcFunc = self.funcAstier 

732 parsIniPtc = [-1e-9, 1.0, 10.] # a00, gain, noise 

733 bounds = self._boundsForAstier(parsIniPtc) 

734 if ptcFitType == 'POLYNOMIAL': 

735 ptcFunc = self.funcPolynomial 

736 parsIniPtc = self._initialParsForPolynomial(self.config.polynomialFitDegree + 1) 

737 bounds = self._boundsForPolynomial(parsIniPtc) 

738 

739 # Before bootstrap fit, do an iterative fit to get rid of outliers 

740 count = 1 

741 while count <= maxIterationsPtcOutliers: 

742 # Note that application of the mask actually shrinks the array 

743 # to size rather than setting elements to zero (as we want) so 

744 # always update mask itself and re-apply to the original data 

745 meanTempVec = meanVecOriginal[mask] 

746 varTempVec = varVecOriginal[mask] 

747 res = least_squares(errFunc, parsIniPtc, bounds=bounds, args=(meanTempVec, varTempVec)) 

748 pars = res.x 

749 

750 # change this to the original from the temp because the masks are ANDed 

751 # meaning once a point is masked it's always masked, and the masks must 

752 # always be the same length for broadcasting 

753 sigResids = (varVecOriginal - ptcFunc(pars, meanVecOriginal))/np.sqrt(varVecOriginal) 

754 newMask = np.array([True if np.abs(r) < sigmaCutPtcOutliers else False for r in sigResids]) 

755 mask = mask & newMask 

756 

757 nDroppedTotal = Counter(mask)[False] 

758 self.log.debug(f"Iteration {count}: discarded {nDroppedTotal} points in total for {ampName}") 

759 count += 1 

760 # objects should never shrink 

761 assert (len(mask) == len(timeVecOriginal) == len(meanVecOriginal) == len(varVecOriginal)) 

762 

763 dataset.visitMask[ampName] = mask # store the final mask 

764 

765 parsIniPtc = pars 

766 timeVecFinal = timeVecOriginal[mask] 

767 meanVecFinal = meanVecOriginal[mask] 

768 varVecFinal = varVecOriginal[mask] 

769 

770 if Counter(mask)[False] > 0: 

771 self.log.info((f"Number of points discarded in PTC of amplifier {ampName}:" + 

772 f" {Counter(mask)[False]} out of {len(meanVecOriginal)}")) 

773 

774 if (len(meanVecFinal) < len(parsIniPtc)): 

775 msg = (f"\nSERIOUS: Not enough data points ({len(meanVecFinal)}) compared to the number of" 

776 f"parameters of the PTC model({len(parsIniPtc)}). Setting {ampName} to BAD.") 

777 self.log.warn(msg) 

778 dataset.badAmps.append(ampName) 

779 dataset.gain[ampName] = np.nan 

780 dataset.gainErr[ampName] = np.nan 

781 dataset.noise[ampName] = np.nan 

782 dataset.noiseErr[ampName] = np.nan 

783 dataset.nonLinearity[ampName] = np.nan 

784 dataset.nonLinearityError[ampName] = np.nan 

785 dataset.nonLinearityResiduals[ampName] = np.nan 

786 continue 

787 

788 # Fit the PTC 

789 if self.config.doFitBootstrap: 

790 parsFit, parsFitErr = self._fitBootstrap(parsIniPtc, meanVecFinal, varVecFinal, ptcFunc) 

791 else: 

792 parsFit, parsFitErr = self._fitLeastSq(parsIniPtc, meanVecFinal, varVecFinal, ptcFunc) 

793 

794 dataset.ptcFitPars[ampName] = parsFit 

795 dataset.ptcFitParsError[ampName] = parsFitErr 

796 

797 if ptcFitType == 'ASTIERAPPROXIMATION': 

798 ptcGain = parsFit[1] 

799 ptcGainErr = parsFitErr[1] 

800 ptcNoise = np.sqrt(np.fabs(parsFit[2])) 

801 ptcNoiseErr = 0.5*(parsFitErr[2]/np.fabs(parsFit[2]))*np.sqrt(np.fabs(parsFit[2])) 

802 if ptcFitType == 'POLYNOMIAL': 

803 ptcGain = 1./parsFit[1] 

804 ptcGainErr = np.fabs(1./parsFit[1])*(parsFitErr[1]/parsFit[1]) 

805 ptcNoise = np.sqrt(np.fabs(parsFit[0]))*ptcGain 

806 ptcNoiseErr = (0.5*(parsFitErr[0]/np.fabs(parsFit[0]))*(np.sqrt(np.fabs(parsFit[0]))))*ptcGain 

807 

808 dataset.gain[ampName] = ptcGain 

809 dataset.gainErr[ampName] = ptcGainErr 

810 dataset.noise[ampName] = ptcNoise 

811 dataset.noiseErr[ampName] = ptcNoiseErr 

812 dataset.ptcFitType[ampName] = ptcFitType 

813 

814 # Non-linearity residuals (NL of mean vs time curve): percentage, and fit to a quadratic function 

815 # In this case, len(parsIniNonLinearity) = 3 indicates that we want a quadratic fit 

816 

817 (c0, linearizerTableRow, linResidualNonLinearity, parsFitNonLinearity, 

818 parsFitErrNonLinearity) = self.calculateLinearityResidualAndLinearizers(timeVecFinal, 

819 meanVecFinal) 

820 # LinearizerLookupTable 

821 tableArray[i, :] = linearizerTableRow 

822 

823 dataset.nonLinearity[ampName] = parsFitNonLinearity 

824 dataset.nonLinearityError[ampName] = parsFitErrNonLinearity 

825 dataset.nonLinearityResiduals[ampName] = linResidualNonLinearity 

826 dataset.coefficientLinearizeSquared[ampName] = c0 

827 

828 return 

829 

830 def plot(self, dataRef, dataset, ptcFitType): 

831 dirname = dataRef.getUri(datasetType='cpPipePlotRoot', write=True) 

832 if not os.path.exists(dirname): 

833 os.makedirs(dirname) 

834 

835 detNum = dataRef.dataId[self.config.ccdKey] 

836 filename = f"PTC_det{detNum}.pdf" 

837 filenameFull = os.path.join(dirname, filename) 

838 with PdfPages(filenameFull) as pdfPages: 

839 self._plotPtc(dataset, ptcFitType, pdfPages) 

840 

841 def _plotPtc(self, dataset, ptcFitType, pdfPages): 

842 """Plot PTC, linearity, and linearity residual per amplifier""" 

843 

844 if ptcFitType == 'ASTIERAPPROXIMATION': 

845 ptcFunc = self.funcAstier 

846 stringTitle = r"Var = $\frac{1}{2g^2a_{00}}(\exp (2a_{00} \mu g) - 1) + \frac{n_{00}}{g^2}$" 

847 

848 if ptcFitType == 'POLYNOMIAL': 

849 ptcFunc = self.funcPolynomial 

850 stringTitle = f"Polynomial (degree: {self.config.polynomialFitDegree})" 

851 

852 legendFontSize = 7.5 

853 labelFontSize = 8 

854 titleFontSize = 10 

855 supTitleFontSize = 18 

856 markerSize = 25 

857 

858 # General determination of the size of the plot grid 

859 nAmps = len(dataset.ampNames) 

860 if nAmps == 2: 

861 nRows, nCols = 2, 1 

862 nRows = np.sqrt(nAmps) 

863 mantissa, _ = np.modf(nRows) 

864 if mantissa > 0: 

865 nRows = int(nRows) + 1 

866 nCols = nRows 

867 else: 

868 nRows = int(nRows) 

869 nCols = nRows 

870 

871 f, ax = plt.subplots(nrows=nRows, ncols=nCols, sharex='col', sharey='row', figsize=(13, 10)) 

872 f2, ax2 = plt.subplots(nrows=nRows, ncols=nCols, sharex='col', sharey='row', figsize=(13, 10)) 

873 

874 for i, (amp, a, a2) in enumerate(zip(dataset.ampNames, ax.flatten(), ax2.flatten())): 

875 meanVecOriginal = np.array(dataset.rawMeans[amp]) 

876 varVecOriginal = np.array(dataset.rawVars[amp]) 

877 mask = dataset.visitMask[amp] 

878 meanVecFinal = meanVecOriginal[mask] 

879 varVecFinal = varVecOriginal[mask] 

880 meanVecOutliers = meanVecOriginal[np.invert(mask)] 

881 varVecOutliers = varVecOriginal[np.invert(mask)] 

882 pars, parsErr = dataset.ptcFitPars[amp], dataset.ptcFitParsError[amp] 

883 

884 if ptcFitType == 'ASTIERAPPROXIMATION': 

885 ptcA00, ptcA00error = pars[0], parsErr[0] 

886 ptcGain, ptcGainError = pars[1], parsErr[1] 

887 ptcNoise = np.sqrt(np.fabs(pars[2])) 

888 ptcNoiseError = 0.5*(parsErr[2]/np.fabs(pars[2]))*np.sqrt(np.fabs(pars[2])) 

889 stringLegend = (f"a00: {ptcA00:.2e}+/-{ptcA00error:.2e}" 

890 f"\n Gain: {ptcGain:.4}+/-{ptcGainError:.2e}" 

891 f"\n Noise: {ptcNoise:.4}+/-{ptcNoiseError:.2e}") 

892 

893 if ptcFitType == 'POLYNOMIAL': 

894 ptcGain, ptcGainError = 1./pars[1], np.fabs(1./pars[1])*(parsErr[1]/pars[1]) 

895 ptcNoise = np.sqrt(np.fabs(pars[0]))*ptcGain 

896 ptcNoiseError = (0.5*(parsErr[0]/np.fabs(pars[0]))*(np.sqrt(np.fabs(pars[0]))))*ptcGain 

897 stringLegend = (f"Noise: {ptcNoise:.4}+/-{ptcNoiseError:.2e} \n" 

898 f"Gain: {ptcGain:.4}+/-{ptcGainError:.2e}") 

899 

900 minMeanVecFinal = np.min(meanVecFinal) 

901 maxMeanVecFinal = np.max(meanVecFinal) 

902 meanVecFit = np.linspace(minMeanVecFinal, maxMeanVecFinal, 100*len(meanVecFinal)) 

903 minMeanVecOriginal = np.min(meanVecOriginal) 

904 maxMeanVecOriginal = np.max(meanVecOriginal) 

905 deltaXlim = maxMeanVecOriginal - minMeanVecOriginal 

906 

907 a.plot(meanVecFit, ptcFunc(pars, meanVecFit), color='red') 

908 a.plot(meanVecFinal, pars[0] + pars[1]*meanVecFinal, color='green', linestyle='--') 

909 a.scatter(meanVecFinal, varVecFinal, c='blue', marker='o', s=markerSize) 

910 a.scatter(meanVecOutliers, varVecOutliers, c='magenta', marker='s', s=markerSize) 

911 a.set_xlabel(r'Mean signal ($\mu$, ADU)', fontsize=labelFontSize) 

912 a.set_xticks(meanVecOriginal) 

913 a.set_ylabel(r'Variance (ADU$^2$)', fontsize=labelFontSize) 

914 a.tick_params(labelsize=11) 

915 a.text(0.03, 0.8, stringLegend, transform=a.transAxes, fontsize=legendFontSize) 

916 a.set_xscale('linear', fontsize=labelFontSize) 

917 a.set_yscale('linear', fontsize=labelFontSize) 

918 a.set_title(amp, fontsize=titleFontSize) 

919 a.set_xlim([minMeanVecOriginal - 0.2*deltaXlim, maxMeanVecOriginal + 0.2*deltaXlim]) 

920 

921 # Same, but in log-scale 

922 a2.plot(meanVecFit, ptcFunc(pars, meanVecFit), color='red') 

923 a2.scatter(meanVecFinal, varVecFinal, c='blue', marker='o', s=markerSize) 

924 a2.scatter(meanVecOutliers, varVecOutliers, c='magenta', marker='s', s=markerSize) 

925 a2.set_xlabel(r'Mean Signal ($\mu$, ADU)', fontsize=labelFontSize) 

926 a2.set_ylabel(r'Variance (ADU$^2$)', fontsize=labelFontSize) 

927 a2.tick_params(labelsize=11) 

928 a2.text(0.03, 0.8, stringLegend, transform=a2.transAxes, fontsize=legendFontSize) 

929 a2.set_xscale('log') 

930 a2.set_yscale('log') 

931 a2.set_title(amp, fontsize=titleFontSize) 

932 a2.set_xlim([minMeanVecOriginal, maxMeanVecOriginal]) 

933 

934 f.suptitle(f"PTC \n Fit: " + stringTitle, fontsize=20) 

935 pdfPages.savefig(f) 

936 f2.suptitle(f"PTC (log-log)", fontsize=20) 

937 pdfPages.savefig(f2) 

938 

939 # Plot mean vs time 

940 f, ax = plt.subplots(nrows=4, ncols=4, sharex='col', sharey='row', figsize=(13, 10)) 

941 for i, (amp, a) in enumerate(zip(dataset.ampNames, ax.flatten())): 

942 meanVecFinal = np.array(dataset.rawMeans[amp])[dataset.visitMask[amp]] 

943 timeVecFinal = np.array(dataset.rawExpTimes[amp])[dataset.visitMask[amp]] 

944 

945 pars, parsErr = dataset.nonLinearity[amp], dataset.nonLinearityError[amp] 

946 c0, c0Error = pars[0], parsErr[0] 

947 c1, c1Error = pars[1], parsErr[1] 

948 c2, c2Error = pars[2], parsErr[2] 

949 stringLegend = f"c0: {c0:.4}+/-{c0Error:.2e}\n c1: {c1:.4}+/-{c1Error:.2e}" \ 

950 + f"\n c2(NL): {c2:.2e}+/-{c2Error:.2e}" 

951 a.scatter(timeVecFinal, meanVecFinal) 

952 a.plot(timeVecFinal, self.funcPolynomial(pars, timeVecFinal), color='red') 

953 a.set_xlabel('Time (sec)', fontsize=labelFontSize) 

954 a.set_xticks(timeVecFinal) 

955 a.set_ylabel(r'Mean signal ($\mu$, ADU)', fontsize=labelFontSize) 

956 a.tick_params(labelsize=labelFontSize) 

957 a.text(0.03, 0.75, stringLegend, transform=a.transAxes, fontsize=legendFontSize) 

958 a.set_xscale('linear', fontsize=labelFontSize) 

959 a.set_yscale('linear', fontsize=labelFontSize) 

960 a.set_title(amp, fontsize=titleFontSize) 

961 

962 f.suptitle("Linearity \n Fit: " + r"$\mu = c_0 + c_1 t + c_2 t^2$", fontsize=supTitleFontSize) 

963 pdfPages.savefig() 

964 

965 # Plot linearity residual 

966 f, ax = plt.subplots(nrows=4, ncols=4, sharex='col', sharey='row', figsize=(13, 10)) 

967 for i, (amp, a) in enumerate(zip(dataset.ampNames, ax.flatten())): 

968 meanVecFinal = np.array(dataset.rawMeans[amp])[dataset.visitMask[amp]] 

969 linRes = np.array(dataset.nonLinearityResiduals[amp]) 

970 

971 a.scatter(meanVecFinal, linRes) 

972 a.axhline(y=0, color='k') 

973 a.axvline(x=timeVecFinal[self.config.linResidualTimeIndex], color='g', linestyle='--') 

974 a.set_xlabel(r'Mean signal ($\mu$, ADU)', fontsize=labelFontSize) 

975 a.set_xticks(meanVecFinal) 

976 a.set_ylabel('LR (%)', fontsize=labelFontSize) 

977 a.tick_params(labelsize=labelFontSize) 

978 a.set_xscale('linear', fontsize=labelFontSize) 

979 a.set_yscale('linear', fontsize=labelFontSize) 

980 a.set_title(amp, fontsize=titleFontSize) 

981 

982 f.suptitle(r"Linearity Residual: $100(1 - \mu_{\rm{ref}}/t_{\rm{ref}})/(\mu / t))$" + "\n" + 

983 r"$t_{\rm{ref}}$: " + f"{timeVecFinal[2]} s", fontsize=supTitleFontSize) 

984 pdfPages.savefig() 

985 

986 return