Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of cp_pipe. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21# 

22import numpy as np 

23import matplotlib.pyplot as plt 

24from collections import Counter 

25 

26import lsst.afw.math as afwMath 

27import lsst.pex.config as pexConfig 

28import lsst.pipe.base as pipeBase 

29from .utils import (fitLeastSq, fitBootstrap, funcPolynomial, funcAstier) 

30from scipy.optimize import least_squares 

31 

32import datetime 

33 

34from .astierCovPtcUtils import (fftSize, CovFft, computeCovDirect, fitData) 

35from .linearity import LinearitySolveTask 

36from .photodiode import getBOTphotodiodeData 

37 

38from lsst.pipe.tasks.getRepositoryData import DataRefListRunner 

39from lsst.ip.isr import PhotonTransferCurveDataset 

40 

41__all__ = ['MeasurePhotonTransferCurveTask', 

42 'MeasurePhotonTransferCurveTaskConfig'] 

43 

44 

45class MeasurePhotonTransferCurveTaskConfig(pexConfig.Config): 

46 """Config class for photon transfer curve measurement task""" 

47 ccdKey = pexConfig.Field( 

48 dtype=str, 

49 doc="The key by which to pull a detector from a dataId, e.g. 'ccd' or 'detector'.", 

50 default='ccd', 

51 ) 

52 ptcFitType = pexConfig.ChoiceField( 

53 dtype=str, 

54 doc="Fit PTC to Eq. 16, Eq. 20 in Astier+19, or to a polynomial.", 

55 default="POLYNOMIAL", 

56 allowed={ 

57 "POLYNOMIAL": "n-degree polynomial (use 'polynomialFitDegree' to set 'n').", 

58 "EXPAPPROXIMATION": "Approximation in Astier+19 (Eq. 16).", 

59 "FULLCOVARIANCE": "Full covariances model in Astier+19 (Eq. 20)" 

60 } 

61 ) 

62 sigmaClipFullFitCovariancesAstier = pexConfig.Field( 

63 dtype=float, 

64 doc="sigma clip for full model fit for FULLCOVARIANCE ptcFitType ", 

65 default=5.0, 

66 ) 

67 maxIterFullFitCovariancesAstier = pexConfig.Field( 

68 dtype=int, 

69 doc="Maximum number of iterations in full model fit for FULLCOVARIANCE ptcFitType", 

70 default=3, 

71 ) 

72 maximumRangeCovariancesAstier = pexConfig.Field( 

73 dtype=int, 

74 doc="Maximum range of covariances as in Astier+19", 

75 default=8, 

76 ) 

77 covAstierRealSpace = pexConfig.Field( 

78 dtype=bool, 

79 doc="Calculate covariances in real space or via FFT? (see appendix A of Astier+19).", 

80 default=False, 

81 ) 

82 polynomialFitDegree = pexConfig.Field( 

83 dtype=int, 

84 doc="Degree of polynomial to fit the PTC, when 'ptcFitType'=POLYNOMIAL.", 

85 default=3, 

86 ) 

87 linearity = pexConfig.ConfigurableField( 

88 target=LinearitySolveTask, 

89 doc="Task to solve the linearity." 

90 ) 

91 

92 doCreateLinearizer = pexConfig.Field( 

93 dtype=bool, 

94 doc="Calculate non-linearity and persist linearizer?", 

95 default=False, 

96 ) 

97 

98 binSize = pexConfig.Field( 

99 dtype=int, 

100 doc="Bin the image by this factor in both dimensions.", 

101 default=1, 

102 ) 

103 minMeanSignal = pexConfig.Field( 

104 dtype=float, 

105 doc="Minimum value (inclusive) of mean signal (in DN) above which to consider.", 

106 default=0, 

107 ) 

108 maxMeanSignal = pexConfig.Field( 

109 dtype=float, 

110 doc="Maximum value (inclusive) of mean signal (in DN) below which to consider.", 

111 default=9e6, 

112 ) 

113 initialNonLinearityExclusionThresholdPositive = pexConfig.RangeField( 

114 dtype=float, 

115 doc="Initially exclude data points with a variance that are more than a factor of this from being" 

116 " linear in the positive direction, from the PTC fit. Note that these points will also be" 

117 " excluded from the non-linearity fit. This is done before the iterative outlier rejection," 

118 " to allow an accurate determination of the sigmas for said iterative fit.", 

119 default=0.12, 

120 min=0.0, 

121 max=1.0, 

122 ) 

123 initialNonLinearityExclusionThresholdNegative = pexConfig.RangeField( 

124 dtype=float, 

125 doc="Initially exclude data points with a variance that are more than a factor of this from being" 

126 " linear in the negative direction, from the PTC fit. Note that these points will also be" 

127 " excluded from the non-linearity fit. This is done before the iterative outlier rejection," 

128 " to allow an accurate determination of the sigmas for said iterative fit.", 

129 default=0.25, 

130 min=0.0, 

131 max=1.0, 

132 ) 

133 sigmaCutPtcOutliers = pexConfig.Field( 

134 dtype=float, 

135 doc="Sigma cut for outlier rejection in PTC.", 

136 default=5.0, 

137 ) 

138 maskNameList = pexConfig.ListField( 

139 dtype=str, 

140 doc="Mask list to exclude from statistics calculations.", 

141 default=['SUSPECT', 'BAD', 'NO_DATA'], 

142 ) 

143 nSigmaClipPtc = pexConfig.Field( 

144 dtype=float, 

145 doc="Sigma cut for afwMath.StatisticsControl()", 

146 default=5.5, 

147 ) 

148 nIterSigmaClipPtc = pexConfig.Field( 

149 dtype=int, 

150 doc="Number of sigma-clipping iterations for afwMath.StatisticsControl()", 

151 default=1, 

152 ) 

153 maxIterationsPtcOutliers = pexConfig.Field( 

154 dtype=int, 

155 doc="Maximum number of iterations for outlier rejection in PTC.", 

156 default=2, 

157 ) 

158 doFitBootstrap = pexConfig.Field( 

159 dtype=bool, 

160 doc="Use bootstrap for the PTC fit parameters and errors?.", 

161 default=False, 

162 ) 

163 doPhotodiode = pexConfig.Field( 

164 dtype=bool, 

165 doc="Apply a correction based on the photodiode readings if available?", 

166 default=True, 

167 ) 

168 photodiodeDataPath = pexConfig.Field( 

169 dtype=str, 

170 doc="Gen2 only: path to locate the data photodiode data files.", 

171 default="" 

172 ) 

173 instrumentName = pexConfig.Field( 

174 dtype=str, 

175 doc="Instrument name.", 

176 default='', 

177 ) 

178 

179 

180class MeasurePhotonTransferCurveTask(pipeBase.CmdLineTask): 

181 """A class to calculate, fit, and plot a PTC from a set of flat pairs. 

182 

183 The Photon Transfer Curve (var(signal) vs mean(signal)) is a standard tool 

184 used in astronomical detectors characterization (e.g., Janesick 2001, 

185 Janesick 2007). If ptcFitType is "EXPAPPROXIMATION" or "POLYNOMIAL", this task calculates the 

186 PTC from a series of pairs of flat-field images; each pair taken at identical exposure 

187 times. The difference image of each pair is formed to eliminate fixed pattern noise, 

188 and then the variance of the difference image and the mean of the average image 

189 are used to produce the PTC. An n-degree polynomial or the approximation in Equation 

190 16 of Astier+19 ("The Shape of the Photon Transfer Curve of CCD sensors", 

191 arXiv:1905.08677) can be fitted to the PTC curve. These models include 

192 parameters such as the gain (e/DN) and readout noise. 

193 

194 Linearizers to correct for signal-chain non-linearity are also calculated. 

195 The `Linearizer` class, in general, can support per-amp linearizers, but in this 

196 task this is not supported. 

197 

198 If ptcFitType is "FULLCOVARIANCE", the covariances of the difference images are calculated via the 

199 DFT methods described in Astier+19 and the variances for the PTC are given by the cov[0,0] elements 

200 at each signal level. The full model in Equation 20 of Astier+19 is fit to the PTC to get the gain 

201 and the noise. 

202 

203 Parameters 

204 ---------- 

205 

206 *args: `list` 

207 Positional arguments passed to the Task constructor. None used at this 

208 time. 

209 **kwargs: `dict` 

210 Keyword arguments passed on to the Task constructor. None used at this 

211 time. 

212 

213 """ 

214 

215 RunnerClass = DataRefListRunner 

216 ConfigClass = MeasurePhotonTransferCurveTaskConfig 

217 _DefaultName = "measurePhotonTransferCurve" 

218 

219 def __init__(self, *args, **kwargs): 

220 pipeBase.CmdLineTask.__init__(self, *args, **kwargs) 

221 self.makeSubtask("linearity") 

222 plt.interactive(False) # stop windows popping up when plotting. When headless, use 'agg' backend too 

223 self.config.validate() 

224 self.config.freeze() 

225 

226 @pipeBase.timeMethod 

227 def runDataRef(self, dataRefList): 

228 """Run the Photon Transfer Curve (PTC) measurement task. 

229 

230 For a dataRef (which is each detector here), 

231 and given a list of exposure pairs (postISR) at different exposure times, 

232 measure the PTC. 

233 

234 Parameters 

235 ---------- 

236 dataRefList : `list` [`lsst.daf.peristence.ButlerDataRef`] 

237 Data references for exposures for detectors to process. 

238 """ 

239 if len(dataRefList) < 2: 

240 raise RuntimeError("Insufficient inputs to combine.") 

241 

242 # setup necessary objects 

243 dataRef = dataRefList[0] 

244 

245 detNum = dataRef.dataId[self.config.ccdKey] 

246 camera = dataRef.get('camera') 

247 detector = camera[dataRef.dataId[self.config.ccdKey]] 

248 

249 amps = detector.getAmplifiers() 

250 ampNames = [amp.getName() for amp in amps] 

251 datasetPtc = PhotonTransferCurveDataset(ampNames, self.config.ptcFitType) 

252 

253 # Get the pairs of flat indexed by expTime 

254 expPairs = self.makePairs(dataRefList) 

255 expIds = [] 

256 for (exp1, exp2) in expPairs.values(): 

257 id1 = exp1.getInfo().getVisitInfo().getExposureId() 

258 id2 = exp2.getInfo().getVisitInfo().getExposureId() 

259 expIds.append((id1, id2)) 

260 self.log.info(f"Measuring PTC using {expIds} exposures for detector {detector.getId()}") 

261 

262 # get photodiode data early so that logic can be put in to only use the 

263 # data if all files are found, as partial corrections are not possible 

264 # or at least require significant logic to deal with 

265 if self.config.doPhotodiode: 

266 for (expId1, expId2) in expIds: 

267 charges = [-1, -1] # necessary to have a not-found value to keep lists in step 

268 for i, expId in enumerate([expId1, expId2]): 

269 # //1000 is a Gen2 only hack, working around the fact an 

270 # exposure's ID is not the same as the expId in the 

271 # registry. Currently expId is concatenated with the 

272 # zero-padded detector ID. This will all go away in Gen3. 

273 dataRef.dataId['expId'] = expId//1000 

274 if self.config.photodiodeDataPath: 

275 photodiodeData = getBOTphotodiodeData(dataRef, self.config.photodiodeDataPath) 

276 else: 

277 photodiodeData = getBOTphotodiodeData(dataRef) 

278 if photodiodeData: # default path stored in function def to keep task clean 

279 charges[i] = photodiodeData.getCharge() 

280 else: 

281 # full expId (not //1000) here, as that encodes the 

282 # the detector number as so is fully qualifying 

283 self.log.warn(f"No photodiode data found for {expId}") 

284 

285 for ampName in ampNames: 

286 datasetPtc.photoCharge[ampName].append((charges[0], charges[1])) 

287 else: 

288 # Can't be an empty list, as initialized, because astropy.Table won't allow it 

289 # when saving as fits 

290 for ampName in ampNames: 

291 datasetPtc.photoCharge[ampName] = np.repeat(np.nan, len(expIds)) 

292 

293 for ampName in ampNames: 

294 datasetPtc.inputExpIdPairs[ampName] = expIds 

295 

296 tupleRecords = [] 

297 allTags = [] 

298 for expTime, (exp1, exp2) in expPairs.items(): 

299 expId1 = exp1.getInfo().getVisitInfo().getExposureId() 

300 expId2 = exp2.getInfo().getVisitInfo().getExposureId() 

301 tupleRows = [] 

302 nAmpsNan = 0 

303 for ampNumber, amp in enumerate(detector): 

304 ampName = amp.getName() 

305 # covAstier: (i, j, var (cov[0,0]), cov, npix) 

306 doRealSpace = self.config.covAstierRealSpace 

307 muDiff, varDiff, covAstier = self.measureMeanVarCov(exp1, exp2, region=amp.getBBox(), 

308 covAstierRealSpace=doRealSpace) 

309 datasetPtc.rawExpTimes[ampName].append(expTime) 

310 datasetPtc.rawMeans[ampName].append(muDiff) 

311 datasetPtc.rawVars[ampName].append(varDiff) 

312 

313 if np.isnan(muDiff) or np.isnan(varDiff) or (covAstier is None): 

314 msg = (f"NaN mean or var, or None cov in amp {ampName} in exposure pair {expId1}," 

315 f" {expId2} of detector {detNum}.") 

316 self.log.warn(msg) 

317 nAmpsNan += 1 

318 continue 

319 tags = ['mu', 'i', 'j', 'var', 'cov', 'npix', 'ext', 'expTime', 'ampName'] 

320 if (muDiff <= self.config.minMeanSignal) or (muDiff >= self.config.maxMeanSignal): 

321 continue 

322 

323 tupleRows += [(muDiff, ) + covRow + (ampNumber, expTime, ampName) for covRow in covAstier] 

324 if nAmpsNan == len(ampNames): 

325 msg = f"NaN mean in all amps of exposure pair {expId1}, {expId2} of detector {detNum}." 

326 self.log.warn(msg) 

327 continue 

328 allTags += tags 

329 tupleRecords += tupleRows 

330 covariancesWithTags = np.core.records.fromrecords(tupleRecords, names=allTags) 

331 

332 if self.config.ptcFitType in ["FULLCOVARIANCE", ]: 

333 # Calculate covariances and fit them, including the PTC, to Astier+19 full model (Eq. 20) 

334 datasetPtc = self.fitCovariancesAstier(datasetPtc, covariancesWithTags) 

335 elif self.config.ptcFitType in ["EXPAPPROXIMATION", "POLYNOMIAL"]: 

336 # Fit the PTC to a polynomial or to Astier+19 exponential approximation (Eq. 16) 

337 # Fill up PhotonTransferCurveDataset object. 

338 datasetPtc = self.fitPtc(datasetPtc, self.config.ptcFitType) 

339 

340 detName = detector.getName() 

341 now = datetime.datetime.utcnow() 

342 calibDate = now.strftime("%Y-%m-%d") 

343 butler = dataRef.getButler() 

344 

345 datasetPtc.updateMetadata(setDate=True, camera=camera, detector=detector) 

346 

347 # Fit a poynomial to calculate non-linearity and persist linearizer. 

348 if self.config.doCreateLinearizer: 

349 # Fit (non)linearity of signal vs time curve. 

350 # Fill up PhotonTransferCurveDataset object. 

351 # Fill up array for LUT linearizer (tableArray). 

352 # Produce coefficients for Polynomial and Squared linearizers. 

353 # Build linearizer objects. 

354 dimensions = {'camera': camera.getName(), 'detector': detector.getId()} 

355 linearityResults = self.linearity.run(datasetPtc, camera, dimensions) 

356 linearizer = linearityResults.outputLinearizer 

357 

358 self.log.info("Writing linearizer:") 

359 

360 detName = detector.getName() 

361 now = datetime.datetime.utcnow() 

362 calibDate = now.strftime("%Y-%m-%d") 

363 

364 butler.put(linearizer, datasetType='linearizer', 

365 dataId={'detector': detNum, 'detectorName': detName, 'calibDate': calibDate}) 

366 

367 self.log.info(f"Writing PTC data.") 

368 butler.put(datasetPtc, datasetType='photonTransferCurveDataset', dataId={'detector': detNum, 

369 'detectorName': detName, 'calibDate': calibDate}) 

370 

371 return pipeBase.Struct(exitStatus=0) 

372 

373 def makePairs(self, dataRefList): 

374 """Produce a list of flat pairs indexed by exposure time. 

375 

376 Parameters 

377 ---------- 

378 dataRefList : `list` [`lsst.daf.peristence.ButlerDataRef`] 

379 Data references for exposures for detectors to process. 

380 

381 Return 

382 ------ 

383 flatPairs : `dict` [`float`, `lsst.afw.image.exposure.exposure.ExposureF`] 

384 Dictionary that groups flat-field exposures that have the same exposure time (seconds). 

385 

386 Notes 

387 ----- 

388 We use the difference of one pair of flat-field images taken at the same exposure time when 

389 calculating the PTC to reduce Fixed Pattern Noise. If there are > 2 flat-field images with the 

390 same exposure time, the first two are kept and the rest discarded. 

391 """ 

392 

393 # Organize exposures by observation date. 

394 expDict = {} 

395 for dataRef in dataRefList: 

396 try: 

397 tempFlat = dataRef.get("postISRCCD") 

398 except RuntimeError: 

399 self.log.warn("postISR exposure could not be retrieved. Ignoring flat.") 

400 continue 

401 expDate = tempFlat.getInfo().getVisitInfo().getDate().get() 

402 expDict.setdefault(expDate, tempFlat) 

403 sortedExps = {k: expDict[k] for k in sorted(expDict)} 

404 

405 flatPairs = {} 

406 for exp in sortedExps: 

407 tempFlat = sortedExps[exp] 

408 expTime = tempFlat.getInfo().getVisitInfo().getExposureTime() 

409 listAtExpTime = flatPairs.setdefault(expTime, []) 

410 if len(listAtExpTime) >= 2: 

411 self.log.warn(f"Already found 2 exposures at expTime {expTime}. " 

412 f"Ignoring exposure {tempFlat.getInfo().getVisitInfo().getExposureId()}") 

413 else: 

414 listAtExpTime.append(tempFlat) 

415 

416 keysToDrop = [] 

417 for (key, value) in flatPairs.items(): 

418 if len(value) < 2: 

419 keysToDrop.append(key) 

420 

421 if len(keysToDrop): 

422 for key in keysToDrop: 

423 self.log.warn(f"Only one exposure found at expTime {key}. Dropping exposure " 

424 f"{flatPairs[key][0].getInfo().getVisitInfo().getExposureId()}.") 

425 flatPairs.pop(key) 

426 return flatPairs 

427 

428 def fitCovariancesAstier(self, dataset, covariancesWithTagsArray): 

429 """Fit measured flat covariances to full model in Astier+19. 

430 

431 Parameters 

432 ---------- 

433 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset` 

434 The dataset containing information such as the means, variances and exposure times. 

435 

436 covariancesWithTagsArray : `numpy.recarray` 

437 Tuple with at least (mu, cov, var, i, j, npix), where: 

438 mu : 0.5*(m1 + m2), where: 

439 mu1: mean value of flat1 

440 mu2: mean value of flat2 

441 cov: covariance value at lag(i, j) 

442 var: variance(covariance value at lag(0, 0)) 

443 i: lag dimension 

444 j: lag dimension 

445 npix: number of pixels used for covariance calculation. 

446 

447 Returns 

448 ------- 

449 dataset: `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset` 

450 This is the same dataset as the input paramter, however, it has been modified 

451 to include information such as the fit vectors and the fit parameters. See 

452 the class `PhotonTransferCurveDatase`. 

453 """ 

454 

455 covFits, covFitsNoB = fitData(covariancesWithTagsArray, maxMu=self.config.maxMeanSignal, 

456 r=self.config.maximumRangeCovariancesAstier, 

457 nSigmaFullFit=self.config.sigmaClipFullFitCovariancesAstier, 

458 maxIterFullFit=self.config.maxIterFullFitCovariancesAstier) 

459 

460 dataset = self.getOutputPtcDataCovAstier(dataset, covFits, covFitsNoB) 

461 

462 return dataset 

463 

464 def getOutputPtcDataCovAstier(self, dataset, covFits, covFitsNoB): 

465 """Get output data for PhotonTransferCurveCovAstierDataset from CovFit objects. 

466 

467 Parameters 

468 ---------- 

469 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset` 

470 The dataset containing information such as the means, variances and exposure times. 

471 

472 covFits: `dict` 

473 Dictionary of CovFit objects, with amp names as keys. 

474 

475 covFitsNoB : `dict` 

476 Dictionary of CovFit objects, with amp names as keys, and 'b=0' in Eq. 20 of Astier+19. 

477 

478 Returns 

479 ------- 

480 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset` 

481 This is the same dataset as the input paramter, however, it has been modified 

482 to include extra information such as the mask 1D array, gains, reoudout noise, measured signal, 

483 measured variance, modeled variance, a, and b coefficient matrices (see Astier+19) per amplifier. 

484 See the class `PhotonTransferCurveDatase`. 

485 """ 

486 assert(len(covFits) == len(covFitsNoB)) 

487 

488 for i, amp in enumerate(dataset.ampNames): 

489 lenInputTimes = len(dataset.rawExpTimes[amp]) 

490 # Not used when ptcFitType is 'FULLCOVARIANCE' 

491 dataset.ptcFitPars[amp] = np.nan 

492 dataset.ptcFitParsError[amp] = np.nan 

493 dataset.ptcFitChiSq[amp] = np.nan 

494 if amp in covFits: 

495 fit = covFits[amp] 

496 fitNoB = covFitsNoB[amp] 

497 # Save full covariances, covariances models, and their weights 

498 dataset.covariances[amp] = fit.cov 

499 dataset.covariancesModel[amp] = fit.evalCovModel() 

500 dataset.covariancesSqrtWeights[amp] = fit.sqrtW 

501 dataset.aMatrix[amp] = fit.getA() 

502 dataset.bMatrix[amp] = fit.getB() 

503 dataset.covariancesNoB[amp] = fitNoB.cov 

504 dataset.covariancesModelNoB[amp] = fitNoB.evalCovModel() 

505 dataset.covariancesSqrtWeightsNoB[amp] = fitNoB.sqrtW 

506 dataset.aMatrixNoB[amp] = fitNoB.getA() 

507 

508 (meanVecFinal, varVecFinal, varVecModel, 

509 wc, varMask) = fit.getFitData(0, 0, divideByMu=False, returnMasked=True) 

510 gain = fit.getGain() 

511 dataset.expIdMask[amp] = varMask 

512 dataset.gain[amp] = gain 

513 dataset.gainErr[amp] = fit.getGainErr() 

514 dataset.noise[amp] = np.sqrt(fit.getRon()) 

515 dataset.noiseErr[amp] = fit.getRonErr() 

516 

517 padLength = lenInputTimes - len(varVecFinal) 

518 dataset.finalVars[amp] = np.pad(varVecFinal/(gain**2), (0, padLength), 'constant', 

519 constant_values=np.nan) 

520 dataset.finalModelVars[amp] = np.pad(varVecModel/(gain**2), (0, padLength), 'constant', 

521 constant_values=np.nan) 

522 dataset.finalMeans[amp] = np.pad(meanVecFinal/gain, (0, padLength), 'constant', 

523 constant_values=np.nan) 

524 else: 

525 # Bad amp 

526 # Entries need to have proper dimensions so read/write with astropy.Table works. 

527 matrixSide = self.config.maximumRangeCovariancesAstier 

528 nanMatrix = np.full((matrixSide, matrixSide), np.nan) 

529 listNanMatrix = np.full((lenInputTimes, matrixSide, matrixSide), np.nan) 

530 

531 dataset.covariances[amp] = listNanMatrix 

532 dataset.covariancesModel[amp] = listNanMatrix 

533 dataset.covariancesSqrtWeights[amp] = listNanMatrix 

534 dataset.aMatrix[amp] = nanMatrix 

535 dataset.bMatrix[amp] = nanMatrix 

536 dataset.covariancesNoB[amp] = listNanMatrix 

537 dataset.covariancesModelNoB[amp] = listNanMatrix 

538 dataset.covariancesSqrtWeightsNoB[amp] = listNanMatrix 

539 dataset.aMatrixNoB[amp] = nanMatrix 

540 

541 dataset.expIdMask[amp] = np.repeat(np.nan, lenInputTimes) 

542 dataset.gain[amp] = np.nan 

543 dataset.gainErr[amp] = np.nan 

544 dataset.noise[amp] = np.nan 

545 dataset.noiseErr[amp] = np.nan 

546 dataset.finalVars[amp] = np.repeat(np.nan, lenInputTimes) 

547 dataset.finalModelVars[amp] = np.repeat(np.nan, lenInputTimes) 

548 dataset.finalMeans[amp] = np.repeat(np.nan, lenInputTimes) 

549 

550 return dataset 

551 

552 def measureMeanVarCov(self, exposure1, exposure2, region=None, covAstierRealSpace=False): 

553 """Calculate the mean of each of two exposures and the variance and covariance of their difference. 

554 

555 The variance is calculated via afwMath, and the covariance via the methods in Astier+19 (appendix A). 

556 In theory, var = covariance[0,0]. This should be validated, and in the future, we may decide to just 

557 keep one (covariance). 

558 

559 Parameters 

560 ---------- 

561 exposure1 : `lsst.afw.image.exposure.exposure.ExposureF` 

562 First exposure of flat field pair. 

563 

564 exposure2 : `lsst.afw.image.exposure.exposure.ExposureF` 

565 Second exposure of flat field pair. 

566 

567 region : `lsst.geom.Box2I`, optional 

568 Region of each exposure where to perform the calculations (e.g, an amplifier). 

569 

570 covAstierRealSpace : `bool`, optional 

571 Should the covariannces in Astier+19 be calculated in real space or via FFT? 

572 See Appendix A of Astier+19. 

573 

574 Returns 

575 ------- 

576 mu : `float` or `NaN` 

577 0.5*(mu1 + mu2), where mu1, and mu2 are the clipped means of the regions in 

578 both exposures. If either mu1 or m2 are NaN's, the returned value is NaN. 

579 

580 varDiff : `float` or `NaN` 

581 Half of the clipped variance of the difference of the regions inthe two input 

582 exposures. If either mu1 or m2 are NaN's, the returned value is NaN. 

583 

584 covDiffAstier : `list` or `NaN` 

585 List with tuples of the form (dx, dy, var, cov, npix), where: 

586 dx : `int` 

587 Lag in x 

588 dy : `int` 

589 Lag in y 

590 var : `float` 

591 Variance at (dx, dy). 

592 cov : `float` 

593 Covariance at (dx, dy). 

594 nPix : `int` 

595 Number of pixel pairs used to evaluate var and cov. 

596 If either mu1 or m2 are NaN's, the returned value is NaN. 

597 """ 

598 

599 if region is not None: 

600 im1Area = exposure1.maskedImage[region] 

601 im2Area = exposure2.maskedImage[region] 

602 else: 

603 im1Area = exposure1.maskedImage 

604 im2Area = exposure2.maskedImage 

605 

606 if self.config.binSize > 1: 

607 im1Area = afwMath.binImage(im1Area, self.config.binSize) 

608 im2Area = afwMath.binImage(im2Area, self.config.binSize) 

609 

610 im1MaskVal = exposure1.getMask().getPlaneBitMask(self.config.maskNameList) 

611 im1StatsCtrl = afwMath.StatisticsControl(self.config.nSigmaClipPtc, 

612 self.config.nIterSigmaClipPtc, 

613 im1MaskVal) 

614 im1StatsCtrl.setNanSafe(True) 

615 im1StatsCtrl.setAndMask(im1MaskVal) 

616 

617 im2MaskVal = exposure2.getMask().getPlaneBitMask(self.config.maskNameList) 

618 im2StatsCtrl = afwMath.StatisticsControl(self.config.nSigmaClipPtc, 

619 self.config.nIterSigmaClipPtc, 

620 im2MaskVal) 

621 im2StatsCtrl.setNanSafe(True) 

622 im2StatsCtrl.setAndMask(im2MaskVal) 

623 

624 # Clipped mean of images; then average of mean. 

625 mu1 = afwMath.makeStatistics(im1Area, afwMath.MEANCLIP, im1StatsCtrl).getValue() 

626 mu2 = afwMath.makeStatistics(im2Area, afwMath.MEANCLIP, im2StatsCtrl).getValue() 

627 if np.isnan(mu1) or np.isnan(mu2): 

628 return np.nan, np.nan, None 

629 mu = 0.5*(mu1 + mu2) 

630 

631 # Take difference of pairs 

632 # symmetric formula: diff = (mu2*im1-mu1*im2)/(0.5*(mu1+mu2)) 

633 temp = im2Area.clone() 

634 temp *= mu1 

635 diffIm = im1Area.clone() 

636 diffIm *= mu2 

637 diffIm -= temp 

638 diffIm /= mu 

639 

640 diffImMaskVal = diffIm.getMask().getPlaneBitMask(self.config.maskNameList) 

641 diffImStatsCtrl = afwMath.StatisticsControl(self.config.nSigmaClipPtc, 

642 self.config.nIterSigmaClipPtc, 

643 diffImMaskVal) 

644 diffImStatsCtrl.setNanSafe(True) 

645 diffImStatsCtrl.setAndMask(diffImMaskVal) 

646 

647 varDiff = 0.5*(afwMath.makeStatistics(diffIm, afwMath.VARIANCECLIP, diffImStatsCtrl).getValue()) 

648 

649 # Get the mask and identify good pixels as '1', and the rest as '0'. 

650 w1 = np.where(im1Area.getMask().getArray() == 0, 1, 0) 

651 w2 = np.where(im2Area.getMask().getArray() == 0, 1, 0) 

652 

653 w12 = w1*w2 

654 wDiff = np.where(diffIm.getMask().getArray() == 0, 1, 0) 

655 w = w12*wDiff 

656 

657 maxRangeCov = self.config.maximumRangeCovariancesAstier 

658 if covAstierRealSpace: 

659 covDiffAstier = computeCovDirect(diffIm.getImage().getArray(), w, maxRangeCov) 

660 else: 

661 shapeDiff = diffIm.getImage().getArray().shape 

662 fftShape = (fftSize(shapeDiff[0] + maxRangeCov), fftSize(shapeDiff[1]+maxRangeCov)) 

663 c = CovFft(diffIm.getImage().getArray(), w, fftShape, maxRangeCov) 

664 covDiffAstier = c.reportCovFft(maxRangeCov) 

665 

666 return mu, varDiff, covDiffAstier 

667 

668 def computeCovDirect(self, diffImage, weightImage, maxRange): 

669 """Compute covariances of diffImage in real space. 

670 

671 For lags larger than ~25, it is slower than the FFT way. 

672 Taken from https://github.com/PierreAstier/bfptc/ 

673 

674 Parameters 

675 ---------- 

676 diffImage : `numpy.array` 

677 Image to compute the covariance of. 

678 

679 weightImage : `numpy.array` 

680 Weight image of diffImage (1's and 0's for good and bad pixels, respectively). 

681 

682 maxRange : `int` 

683 Last index of the covariance to be computed. 

684 

685 Returns 

686 ------- 

687 outList : `list` 

688 List with tuples of the form (dx, dy, var, cov, npix), where: 

689 dx : `int` 

690 Lag in x 

691 dy : `int` 

692 Lag in y 

693 var : `float` 

694 Variance at (dx, dy). 

695 cov : `float` 

696 Covariance at (dx, dy). 

697 nPix : `int` 

698 Number of pixel pairs used to evaluate var and cov. 

699 """ 

700 outList = [] 

701 var = 0 

702 # (dy,dx) = (0,0) has to be first 

703 for dy in range(maxRange + 1): 

704 for dx in range(0, maxRange + 1): 

705 if (dx*dy > 0): 

706 cov1, nPix1 = self.covDirectValue(diffImage, weightImage, dx, dy) 

707 cov2, nPix2 = self.covDirectValue(diffImage, weightImage, dx, -dy) 

708 cov = 0.5*(cov1 + cov2) 

709 nPix = nPix1 + nPix2 

710 else: 

711 cov, nPix = self.covDirectValue(diffImage, weightImage, dx, dy) 

712 if (dx == 0 and dy == 0): 

713 var = cov 

714 outList.append((dx, dy, var, cov, nPix)) 

715 

716 return outList 

717 

718 def covDirectValue(self, diffImage, weightImage, dx, dy): 

719 """Compute covariances of diffImage in real space at lag (dx, dy). 

720 

721 Taken from https://github.com/PierreAstier/bfptc/ (c.f., appendix of Astier+19). 

722 

723 Parameters 

724 ---------- 

725 diffImage : `numpy.array` 

726 Image to compute the covariance of. 

727 

728 weightImage : `numpy.array` 

729 Weight image of diffImage (1's and 0's for good and bad pixels, respectively). 

730 

731 dx : `int` 

732 Lag in x. 

733 

734 dy : `int` 

735 Lag in y. 

736 

737 Returns 

738 ------- 

739 cov : `float` 

740 Covariance at (dx, dy) 

741 

742 nPix : `int` 

743 Number of pixel pairs used to evaluate var and cov. 

744 """ 

745 (nCols, nRows) = diffImage.shape 

746 # switching both signs does not change anything: 

747 # it just swaps im1 and im2 below 

748 if (dx < 0): 

749 (dx, dy) = (-dx, -dy) 

750 # now, we have dx >0. We have to distinguish two cases 

751 # depending on the sign of dy 

752 if dy >= 0: 

753 im1 = diffImage[dy:, dx:] 

754 w1 = weightImage[dy:, dx:] 

755 im2 = diffImage[:nCols - dy, :nRows - dx] 

756 w2 = weightImage[:nCols - dy, :nRows - dx] 

757 else: 

758 im1 = diffImage[:nCols + dy, dx:] 

759 w1 = weightImage[:nCols + dy, dx:] 

760 im2 = diffImage[-dy:, :nRows - dx] 

761 w2 = weightImage[-dy:, :nRows - dx] 

762 # use the same mask for all 3 calculations 

763 wAll = w1*w2 

764 # do not use mean() because weightImage=0 pixels would then count 

765 nPix = wAll.sum() 

766 im1TimesW = im1*wAll 

767 s1 = im1TimesW.sum()/nPix 

768 s2 = (im2*wAll).sum()/nPix 

769 p = (im1TimesW*im2).sum()/nPix 

770 cov = p - s1*s2 

771 

772 return cov, nPix 

773 

774 @staticmethod 

775 def _initialParsForPolynomial(order): 

776 assert(order >= 2) 

777 pars = np.zeros(order, dtype=np.float) 

778 pars[0] = 10 

779 pars[1] = 1 

780 pars[2:] = 0.0001 

781 return pars 

782 

783 @staticmethod 

784 def _boundsForPolynomial(initialPars): 

785 lowers = [np.NINF for p in initialPars] 

786 uppers = [np.inf for p in initialPars] 

787 lowers[1] = 0 # no negative gains 

788 return (lowers, uppers) 

789 

790 @staticmethod 

791 def _boundsForAstier(initialPars): 

792 lowers = [np.NINF for p in initialPars] 

793 uppers = [np.inf for p in initialPars] 

794 return (lowers, uppers) 

795 

796 @staticmethod 

797 def _getInitialGoodPoints(means, variances, maxDeviationPositive, maxDeviationNegative): 

798 """Return a boolean array to mask bad points. 

799 

800 Parameters 

801 ---------- 

802 means : `numpy.array` 

803 Input array with mean signal values. 

804 

805 variances : `numpy.array` 

806 Input array with variances at each mean value. 

807 

808 maxDeviationPositive : `float` 

809 Maximum deviation from being constant for the variance/mean 

810 ratio, in the positive direction. 

811 

812 maxDeviationNegative : `float` 

813 Maximum deviation from being constant for the variance/mean 

814 ratio, in the negative direction. 

815 

816 Return 

817 ------ 

818 goodPoints : `numpy.array` [`bool`] 

819 Boolean array to select good (`True`) and bad (`False`) 

820 points. 

821 

822 Notes 

823 ----- 

824 A linear function has a constant ratio, so find the median 

825 value of the ratios, and exclude the points that deviate 

826 from that by more than a factor of maxDeviationPositive/negative. 

827 Asymmetric deviations are supported as we expect the PTC to turn 

828 down as the flux increases, but sometimes it anomalously turns 

829 upwards just before turning over, which ruins the fits, so it 

830 is wise to be stricter about restricting positive outliers than 

831 negative ones. 

832 

833 Too high and points that are so bad that fit will fail will be included 

834 Too low and the non-linear points will be excluded, biasing the NL fit.""" 

835 

836 assert(len(means) == len(variances)) 

837 ratios = [b/a for (a, b) in zip(means, variances)] 

838 medianRatio = np.nanmedian(ratios) 

839 ratioDeviations = [(r/medianRatio)-1 for r in ratios] 

840 

841 # so that it doesn't matter if the deviation is expressed as positive or negative 

842 maxDeviationPositive = abs(maxDeviationPositive) 

843 maxDeviationNegative = -1. * abs(maxDeviationNegative) 

844 

845 goodPoints = np.array([True if (r < maxDeviationPositive and r > maxDeviationNegative) 

846 else False for r in ratioDeviations]) 

847 return goodPoints 

848 

849 def _makeZeroSafe(self, array, warn=True, substituteValue=1e-9): 

850 """""" 

851 nBad = Counter(array)[0] 

852 if nBad == 0: 

853 return array 

854 

855 if warn: 

856 msg = f"Found {nBad} zeros in array at elements {[x for x in np.where(array==0)[0]]}" 

857 self.log.warn(msg) 

858 

859 array[array == 0] = substituteValue 

860 return array 

861 

862 def fitPtc(self, dataset, ptcFitType): 

863 """Fit the photon transfer curve to a polynimial or to Astier+19 approximation. 

864 

865 Fit the photon transfer curve with either a polynomial of the order 

866 specified in the task config, or using the Astier approximation. 

867 

868 Sigma clipping is performed iteratively for the fit, as well as an 

869 initial clipping of data points that are more than 

870 config.initialNonLinearityExclusionThreshold away from lying on a 

871 straight line. This other step is necessary because the photon transfer 

872 curve turns over catastrophically at very high flux (because saturation 

873 drops the variance to ~0) and these far outliers cause the initial fit 

874 to fail, meaning the sigma cannot be calculated to perform the 

875 sigma-clipping. 

876 

877 Parameters 

878 ---------- 

879 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset` 

880 The dataset containing the means, variances and exposure times 

881 

882 ptcFitType : `str` 

883 Fit a 'POLYNOMIAL' (degree: 'polynomialFitDegree') or 

884 'EXPAPPROXIMATION' (Eq. 16 of Astier+19) to the PTC 

885 

886 Returns 

887 ------- 

888 dataset: `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset` 

889 This is the same dataset as the input paramter, however, it has been modified 

890 to include information such as the fit vectors and the fit parameters. See 

891 the class `PhotonTransferCurveDatase`. 

892 """ 

893 

894 matrixSide = self.config.maximumRangeCovariancesAstier 

895 nanMatrix = np.empty((matrixSide, matrixSide)) 

896 nanMatrix[:] = np.nan 

897 

898 for amp in dataset.ampNames: 

899 lenInputTimes = len(dataset.rawExpTimes[amp]) 

900 listNanMatrix = np.empty((lenInputTimes, matrixSide, matrixSide)) 

901 listNanMatrix[:] = np.nan 

902 

903 dataset.covariances[amp] = listNanMatrix 

904 dataset.covariancesModel[amp] = listNanMatrix 

905 dataset.covariancesSqrtWeights[amp] = listNanMatrix 

906 dataset.aMatrix[amp] = nanMatrix 

907 dataset.bMatrix[amp] = nanMatrix 

908 dataset.covariancesNoB[amp] = listNanMatrix 

909 dataset.covariancesModelNoB[amp] = listNanMatrix 

910 dataset.covariancesSqrtWeightsNoB[amp] = listNanMatrix 

911 dataset.aMatrixNoB[amp] = nanMatrix 

912 

913 def errFunc(p, x, y): 

914 return ptcFunc(p, x) - y 

915 

916 sigmaCutPtcOutliers = self.config.sigmaCutPtcOutliers 

917 maxIterationsPtcOutliers = self.config.maxIterationsPtcOutliers 

918 

919 for i, ampName in enumerate(dataset.ampNames): 

920 timeVecOriginal = np.array(dataset.rawExpTimes[ampName]) 

921 meanVecOriginal = np.array(dataset.rawMeans[ampName]) 

922 varVecOriginal = np.array(dataset.rawVars[ampName]) 

923 varVecOriginal = self._makeZeroSafe(varVecOriginal) 

924 

925 mask = ((meanVecOriginal >= self.config.minMeanSignal) & 

926 (meanVecOriginal <= self.config.maxMeanSignal)) 

927 

928 goodPoints = self._getInitialGoodPoints(meanVecOriginal, varVecOriginal, 

929 self.config.initialNonLinearityExclusionThresholdPositive, 

930 self.config.initialNonLinearityExclusionThresholdNegative) 

931 if not (mask.any() and goodPoints.any()): 

932 msg = (f"\nSERIOUS: All points in either mask: {mask} or goodPoints: {goodPoints} are bad." 

933 f"Setting {ampName} to BAD.") 

934 self.log.warn(msg) 

935 # The first and second parameters of initial fit are discarded (bias and gain) 

936 # for the final NL coefficients 

937 dataset.badAmps.append(ampName) 

938 dataset.expIdMask[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName])) 

939 dataset.gain[ampName] = np.nan 

940 dataset.gainErr[ampName] = np.nan 

941 dataset.noise[ampName] = np.nan 

942 dataset.noiseErr[ampName] = np.nan 

943 dataset.ptcFitPars[ampName] = (np.repeat(np.nan, self.config.polynomialFitDegree + 1) if 

944 ptcFitType in ["POLYNOMIAL", ] else np.repeat(np.nan, 3)) 

945 dataset.ptcFitParsError[ampName] = (np.repeat(np.nan, self.config.polynomialFitDegree + 1) if 

946 ptcFitType in ["POLYNOMIAL", ] else np.repeat(np.nan, 3)) 

947 dataset.ptcFitChiSq[ampName] = np.nan 

948 dataset.finalVars[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName])) 

949 dataset.finalModelVars[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName])) 

950 dataset.finalMeans[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName])) 

951 continue 

952 

953 mask = mask & goodPoints 

954 

955 if ptcFitType == 'EXPAPPROXIMATION': 

956 ptcFunc = funcAstier 

957 parsIniPtc = [-1e-9, 1.0, 10.] # a00, gain, noise 

958 bounds = self._boundsForAstier(parsIniPtc) 

959 if ptcFitType == 'POLYNOMIAL': 

960 ptcFunc = funcPolynomial 

961 parsIniPtc = self._initialParsForPolynomial(self.config.polynomialFitDegree + 1) 

962 bounds = self._boundsForPolynomial(parsIniPtc) 

963 

964 # Before bootstrap fit, do an iterative fit to get rid of outliers 

965 count = 1 

966 while count <= maxIterationsPtcOutliers: 

967 # Note that application of the mask actually shrinks the array 

968 # to size rather than setting elements to zero (as we want) so 

969 # always update mask itself and re-apply to the original data 

970 meanTempVec = meanVecOriginal[mask] 

971 varTempVec = varVecOriginal[mask] 

972 res = least_squares(errFunc, parsIniPtc, bounds=bounds, args=(meanTempVec, varTempVec)) 

973 pars = res.x 

974 

975 # change this to the original from the temp because the masks are ANDed 

976 # meaning once a point is masked it's always masked, and the masks must 

977 # always be the same length for broadcasting 

978 sigResids = (varVecOriginal - ptcFunc(pars, meanVecOriginal))/np.sqrt(varVecOriginal) 

979 newMask = np.array([True if np.abs(r) < sigmaCutPtcOutliers else False for r in sigResids]) 

980 mask = mask & newMask 

981 if not (mask.any() and newMask.any()): 

982 msg = (f"\nSERIOUS: All points in either mask: {mask} or newMask: {newMask} are bad. " 

983 f"Setting {ampName} to BAD.") 

984 self.log.warn(msg) 

985 # The first and second parameters of initial fit are discarded (bias and gain) 

986 # for the final NL coefficients 

987 dataset.badAmps.append(ampName) 

988 dataset.expIdMask[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName])) 

989 dataset.gain[ampName] = np.nan 

990 dataset.gainErr[ampName] = np.nan 

991 dataset.noise[ampName] = np.nan 

992 dataset.noiseErr[ampName] = np.nan 

993 dataset.ptcFitPars[ampName] = (np.repeat(np.nan, self.config.polynomialFitDegree + 1) 

994 if ptcFitType in ["POLYNOMIAL", ] else 

995 np.repeat(np.nan, 3)) 

996 dataset.ptcFitParsError[ampName] = (np.repeat(np.nan, self.config.polynomialFitDegree + 1) 

997 if ptcFitType in ["POLYNOMIAL", ] else 

998 np.repeat(np.nan, 3)) 

999 dataset.ptcFitChiSq[ampName] = np.nan 

1000 dataset.finalVars[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName])) 

1001 dataset.finalModelVars[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName])) 

1002 dataset.finalMeans[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName])) 

1003 break 

1004 nDroppedTotal = Counter(mask)[False] 

1005 self.log.debug(f"Iteration {count}: discarded {nDroppedTotal} points in total for {ampName}") 

1006 count += 1 

1007 # objects should never shrink 

1008 assert (len(mask) == len(timeVecOriginal) == len(meanVecOriginal) == len(varVecOriginal)) 

1009 

1010 if not (mask.any() and newMask.any()): 

1011 continue 

1012 dataset.expIdMask[ampName] = mask # store the final mask 

1013 parsIniPtc = pars 

1014 meanVecFinal = meanVecOriginal[mask] 

1015 varVecFinal = varVecOriginal[mask] 

1016 

1017 if Counter(mask)[False] > 0: 

1018 self.log.info((f"Number of points discarded in PTC of amplifier {ampName}:" + 

1019 f" {Counter(mask)[False]} out of {len(meanVecOriginal)}")) 

1020 

1021 if (len(meanVecFinal) < len(parsIniPtc)): 

1022 msg = (f"\nSERIOUS: Not enough data points ({len(meanVecFinal)}) compared to the number of" 

1023 f"parameters of the PTC model({len(parsIniPtc)}). Setting {ampName} to BAD.") 

1024 self.log.warn(msg) 

1025 # The first and second parameters of initial fit are discarded (bias and gain) 

1026 # for the final NL coefficients 

1027 dataset.badAmps.append(ampName) 

1028 dataset.expIdMask[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName])) 

1029 dataset.gain[ampName] = np.nan 

1030 dataset.gainErr[ampName] = np.nan 

1031 dataset.noise[ampName] = np.nan 

1032 dataset.noiseErr[ampName] = np.nan 

1033 dataset.ptcFitPars[ampName] = (np.repeat(np.nan, self.config.polynomialFitDegree + 1) if 

1034 ptcFitType in ["POLYNOMIAL", ] else np.repeat(np.nan, 3)) 

1035 dataset.ptcFitParsError[ampName] = (np.repeat(np.nan, self.config.polynomialFitDegree + 1) if 

1036 ptcFitType in ["POLYNOMIAL", ] else np.repeat(np.nan, 3)) 

1037 dataset.ptcFitChiSq[ampName] = np.nan 

1038 dataset.finalVars[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName])) 

1039 dataset.finalModelVars[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName])) 

1040 dataset.finalMeans[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName])) 

1041 continue 

1042 

1043 # Fit the PTC 

1044 if self.config.doFitBootstrap: 

1045 parsFit, parsFitErr, reducedChiSqPtc = fitBootstrap(parsIniPtc, meanVecFinal, 

1046 varVecFinal, ptcFunc, 

1047 weightsY=1./np.sqrt(varVecFinal)) 

1048 else: 

1049 parsFit, parsFitErr, reducedChiSqPtc = fitLeastSq(parsIniPtc, meanVecFinal, 

1050 varVecFinal, ptcFunc, 

1051 weightsY=1./np.sqrt(varVecFinal)) 

1052 dataset.ptcFitPars[ampName] = parsFit 

1053 dataset.ptcFitParsError[ampName] = parsFitErr 

1054 dataset.ptcFitChiSq[ampName] = reducedChiSqPtc 

1055 # Masked variances (measured and modeled) and means. Need to pad the array so astropy.Table does 

1056 # not crash (the mask may vary per amp). 

1057 padLength = len(dataset.rawExpTimes[ampName]) - len(varVecFinal) 

1058 dataset.finalVars[ampName] = np.pad(varVecFinal, (0, padLength), 'constant', 

1059 constant_values=np.nan) 

1060 dataset.finalModelVars[ampName] = np.pad(ptcFunc(parsFit, meanVecFinal), (0, padLength), 

1061 'constant', constant_values=np.nan) 

1062 dataset.finalMeans[ampName] = np.pad(meanVecFinal, (0, padLength), 'constant', 

1063 constant_values=np.nan) 

1064 

1065 if ptcFitType == 'EXPAPPROXIMATION': 

1066 ptcGain = parsFit[1] 

1067 ptcGainErr = parsFitErr[1] 

1068 ptcNoise = np.sqrt(np.fabs(parsFit[2])) 

1069 ptcNoiseErr = 0.5*(parsFitErr[2]/np.fabs(parsFit[2]))*np.sqrt(np.fabs(parsFit[2])) 

1070 if ptcFitType == 'POLYNOMIAL': 

1071 ptcGain = 1./parsFit[1] 

1072 ptcGainErr = np.fabs(1./parsFit[1])*(parsFitErr[1]/parsFit[1]) 

1073 ptcNoise = np.sqrt(np.fabs(parsFit[0]))*ptcGain 

1074 ptcNoiseErr = (0.5*(parsFitErr[0]/np.fabs(parsFit[0]))*(np.sqrt(np.fabs(parsFit[0]))))*ptcGain 

1075 dataset.gain[ampName] = ptcGain 

1076 dataset.gainErr[ampName] = ptcGainErr 

1077 dataset.noise[ampName] = ptcNoise 

1078 dataset.noiseErr[ampName] = ptcNoiseErr 

1079 if not len(dataset.ptcFitType) == 0: 

1080 dataset.ptcFitType = ptcFitType 

1081 if len(dataset.badAmps) == 0: 

1082 dataset.badAmps = np.repeat(np.nan, len(list(dataset.rawExpTimes.values())[0])) 

1083 

1084 return dataset