Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of cp_pipe. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21# 

22import numpy as np 

23import matplotlib.pyplot as plt 

24from collections import Counter 

25 

26import lsst.afw.math as afwMath 

27import lsst.pex.config as pexConfig 

28import lsst.pipe.base as pipeBase 

29from .utils import (fitLeastSq, fitBootstrap, funcPolynomial, funcAstier) 

30from scipy.optimize import least_squares 

31 

32import datetime 

33 

34from .astierCovPtcUtils import (fftSize, CovFft, computeCovDirect, fitData) 

35from .linearity import LinearitySolveTask 

36from .photodiode import getBOTphotodiodeData 

37 

38from lsst.pipe.tasks.getRepositoryData import DataRefListRunner 

39from lsst.ip.isr import PhotonTransferCurveDataset 

40 

41__all__ = ['MeasurePhotonTransferCurveTask', 

42 'MeasurePhotonTransferCurveTaskConfig'] 

43 

44 

45class MeasurePhotonTransferCurveTaskConfig(pexConfig.Config): 

46 """Config class for photon transfer curve measurement task""" 

47 ccdKey = pexConfig.Field( 

48 dtype=str, 

49 doc="The key by which to pull a detector from a dataId, e.g. 'ccd' or 'detector'.", 

50 default='ccd', 

51 ) 

52 ptcFitType = pexConfig.ChoiceField( 

53 dtype=str, 

54 doc="Fit PTC to Eq. 16, Eq. 20 in Astier+19, or to a polynomial.", 

55 default="POLYNOMIAL", 

56 allowed={ 

57 "POLYNOMIAL": "n-degree polynomial (use 'polynomialFitDegree' to set 'n').", 

58 "EXPAPPROXIMATION": "Approximation in Astier+19 (Eq. 16).", 

59 "FULLCOVARIANCE": "Full covariances model in Astier+19 (Eq. 20)" 

60 } 

61 ) 

62 sigmaClipFullFitCovariancesAstier = pexConfig.Field( 

63 dtype=float, 

64 doc="Sigma clip for full model fit for FULLCOVARIANCE ptcFitType ", 

65 default=5.0, 

66 ) 

67 maxIterFullFitCovariancesAstier = pexConfig.Field( 

68 dtype=int, 

69 doc="Maximum number of iterations in full model fit for FULLCOVARIANCE ptcFitType", 

70 default=3, 

71 ) 

72 maximumRangeCovariancesAstier = pexConfig.Field( 

73 dtype=int, 

74 doc="Maximum range of covariances as in Astier+19", 

75 default=8, 

76 ) 

77 covAstierRealSpace = pexConfig.Field( 

78 dtype=bool, 

79 doc="Calculate covariances in real space or via FFT? (see appendix A of Astier+19).", 

80 default=False, 

81 ) 

82 polynomialFitDegree = pexConfig.Field( 

83 dtype=int, 

84 doc="Degree of polynomial to fit the PTC, when 'ptcFitType'=POLYNOMIAL.", 

85 default=3, 

86 ) 

87 linearity = pexConfig.ConfigurableField( 

88 target=LinearitySolveTask, 

89 doc="Task to solve the linearity." 

90 ) 

91 

92 doCreateLinearizer = pexConfig.Field( 

93 dtype=bool, 

94 doc="Calculate non-linearity and persist linearizer?", 

95 default=False, 

96 ) 

97 

98 binSize = pexConfig.Field( 

99 dtype=int, 

100 doc="Bin the image by this factor in both dimensions.", 

101 default=1, 

102 ) 

103 minMeanSignal = pexConfig.DictField( 

104 keytype=str, 

105 itemtype=float, 

106 doc="Minimum values (inclusive) of mean signal (in ADU) above which to consider, per amp." 

107 " The same cut is applied to all amps if this dictionary is of the form" 

108 " {'ALL_AMPS': value}", 

109 default={'ALL_AMPS': 0.0}, 

110 ) 

111 maxMeanSignal = pexConfig.DictField( 

112 keytype=str, 

113 itemtype=float, 

114 doc="Maximum values (inclusive) of mean signal (in ADU) below which to consider, per amp." 

115 " The same cut is applied to all amps if this dictionary is of the form" 

116 " {'ALL_AMPS': value}", 

117 default={'ALL_AMPS': 1e6}, 

118 ) 

119 initialNonLinearityExclusionThresholdPositive = pexConfig.RangeField( 

120 dtype=float, 

121 doc="Initially exclude data points with a variance that are more than a factor of this from being" 

122 " linear in the positive direction, from the PTC fit. Note that these points will also be" 

123 " excluded from the non-linearity fit. This is done before the iterative outlier rejection," 

124 " to allow an accurate determination of the sigmas for said iterative fit.", 

125 default=0.12, 

126 min=0.0, 

127 max=1.0, 

128 ) 

129 initialNonLinearityExclusionThresholdNegative = pexConfig.RangeField( 

130 dtype=float, 

131 doc="Initially exclude data points with a variance that are more than a factor of this from being" 

132 " linear in the negative direction, from the PTC fit. Note that these points will also be" 

133 " excluded from the non-linearity fit. This is done before the iterative outlier rejection," 

134 " to allow an accurate determination of the sigmas for said iterative fit.", 

135 default=0.25, 

136 min=0.0, 

137 max=1.0, 

138 ) 

139 sigmaCutPtcOutliers = pexConfig.Field( 

140 dtype=float, 

141 doc="Sigma cut for outlier rejection in PTC.", 

142 default=5.0, 

143 ) 

144 maskNameList = pexConfig.ListField( 

145 dtype=str, 

146 doc="Mask list to exclude from statistics calculations.", 

147 default=['SUSPECT', 'BAD', 'NO_DATA'], 

148 ) 

149 nSigmaClipPtc = pexConfig.Field( 

150 dtype=float, 

151 doc="Sigma cut for afwMath.StatisticsControl()", 

152 default=5.5, 

153 ) 

154 nIterSigmaClipPtc = pexConfig.Field( 

155 dtype=int, 

156 doc="Number of sigma-clipping iterations for afwMath.StatisticsControl()", 

157 default=1, 

158 ) 

159 maxIterationsPtcOutliers = pexConfig.Field( 

160 dtype=int, 

161 doc="Maximum number of iterations for outlier rejection in PTC.", 

162 default=2, 

163 ) 

164 doFitBootstrap = pexConfig.Field( 

165 dtype=bool, 

166 doc="Use bootstrap for the PTC fit parameters and errors?.", 

167 default=False, 

168 ) 

169 doPhotodiode = pexConfig.Field( 

170 dtype=bool, 

171 doc="Apply a correction based on the photodiode readings if available?", 

172 default=True, 

173 ) 

174 photodiodeDataPath = pexConfig.Field( 

175 dtype=str, 

176 doc="Gen2 only: path to locate the data photodiode data files.", 

177 default="" 

178 ) 

179 instrumentName = pexConfig.Field( 

180 dtype=str, 

181 doc="Instrument name.", 

182 default='', 

183 ) 

184 

185 

186class MeasurePhotonTransferCurveTask(pipeBase.CmdLineTask): 

187 """A class to calculate, fit, and plot a PTC from a set of flat pairs. 

188 

189 The Photon Transfer Curve (var(signal) vs mean(signal)) is a standard tool 

190 used in astronomical detectors characterization (e.g., Janesick 2001, 

191 Janesick 2007). If ptcFitType is "EXPAPPROXIMATION" or "POLYNOMIAL", this task calculates the 

192 PTC from a series of pairs of flat-field images; each pair taken at identical exposure 

193 times. The difference image of each pair is formed to eliminate fixed pattern noise, 

194 and then the variance of the difference image and the mean of the average image 

195 are used to produce the PTC. An n-degree polynomial or the approximation in Equation 

196 16 of Astier+19 ("The Shape of the Photon Transfer Curve of CCD sensors", 

197 arXiv:1905.08677) can be fitted to the PTC curve. These models include 

198 parameters such as the gain (e/DN) and readout noise. 

199 

200 Linearizers to correct for signal-chain non-linearity are also calculated. 

201 The `Linearizer` class, in general, can support per-amp linearizers, but in this 

202 task this is not supported. 

203 

204 If ptcFitType is "FULLCOVARIANCE", the covariances of the difference images are calculated via the 

205 DFT methods described in Astier+19 and the variances for the PTC are given by the cov[0,0] elements 

206 at each signal level. The full model in Equation 20 of Astier+19 is fit to the PTC to get the gain 

207 and the noise. 

208 

209 Parameters 

210 ---------- 

211 

212 *args: `list` 

213 Positional arguments passed to the Task constructor. None used at this 

214 time. 

215 **kwargs: `dict` 

216 Keyword arguments passed on to the Task constructor. None used at this 

217 time. 

218 

219 """ 

220 

221 RunnerClass = DataRefListRunner 

222 ConfigClass = MeasurePhotonTransferCurveTaskConfig 

223 _DefaultName = "measurePhotonTransferCurve" 

224 

225 def __init__(self, *args, **kwargs): 

226 pipeBase.CmdLineTask.__init__(self, *args, **kwargs) 

227 self.makeSubtask("linearity") 

228 plt.interactive(False) # stop windows popping up when plotting. When headless, use 'agg' backend too 

229 self.config.validate() 

230 self.config.freeze() 

231 

232 @pipeBase.timeMethod 

233 def runDataRef(self, dataRefList): 

234 """Run the Photon Transfer Curve (PTC) measurement task. 

235 

236 For a dataRef (which is each detector here), 

237 and given a list of exposure pairs (postISR) at different exposure times, 

238 measure the PTC. 

239 

240 Parameters 

241 ---------- 

242 dataRefList : `list` [`lsst.daf.peristence.ButlerDataRef`] 

243 Data references for exposures for detectors to process. 

244 """ 

245 if len(dataRefList) < 2: 

246 raise RuntimeError("Insufficient inputs to combine.") 

247 

248 # setup necessary objects 

249 dataRef = dataRefList[0] 

250 

251 detNum = dataRef.dataId[self.config.ccdKey] 

252 camera = dataRef.get('camera') 

253 detector = camera[dataRef.dataId[self.config.ccdKey]] 

254 

255 amps = detector.getAmplifiers() 

256 ampNames = [amp.getName() for amp in amps] 

257 datasetPtc = PhotonTransferCurveDataset(ampNames, self.config.ptcFitType) 

258 

259 # Get the pairs of flat indexed by expTime 

260 expPairs = self.makePairs(dataRefList) 

261 expIds = [] 

262 for (exp1, exp2) in expPairs.values(): 

263 id1 = exp1.getInfo().getVisitInfo().getExposureId() 

264 id2 = exp2.getInfo().getVisitInfo().getExposureId() 

265 expIds.append((id1, id2)) 

266 self.log.info(f"Measuring PTC using {expIds} exposures for detector {detector.getId()}") 

267 

268 # get photodiode data early so that logic can be put in to only use the 

269 # data if all files are found, as partial corrections are not possible 

270 # or at least require significant logic to deal with 

271 if self.config.doPhotodiode: 

272 for (expId1, expId2) in expIds: 

273 charges = [-1, -1] # necessary to have a not-found value to keep lists in step 

274 for i, expId in enumerate([expId1, expId2]): 

275 # //1000 is a Gen2 only hack, working around the fact an 

276 # exposure's ID is not the same as the expId in the 

277 # registry. Currently expId is concatenated with the 

278 # zero-padded detector ID. This will all go away in Gen3. 

279 dataRef.dataId['expId'] = expId//1000 

280 if self.config.photodiodeDataPath: 

281 photodiodeData = getBOTphotodiodeData(dataRef, self.config.photodiodeDataPath) 

282 else: 

283 photodiodeData = getBOTphotodiodeData(dataRef) 

284 if photodiodeData: # default path stored in function def to keep task clean 

285 charges[i] = photodiodeData.getCharge() 

286 else: 

287 # full expId (not //1000) here, as that encodes the 

288 # the detector number as so is fully qualifying 

289 self.log.warn(f"No photodiode data found for {expId}") 

290 

291 for ampName in ampNames: 

292 datasetPtc.photoCharge[ampName].append((charges[0], charges[1])) 

293 else: 

294 # Can't be an empty list, as initialized, because astropy.Table won't allow it 

295 # when saving as fits 

296 for ampName in ampNames: 

297 datasetPtc.photoCharge[ampName] = np.repeat(np.nan, len(expIds)) 

298 

299 for ampName in ampNames: 

300 datasetPtc.inputExpIdPairs[ampName] = expIds 

301 

302 maxMeanSignalDict = {ampName: 1e6 for ampName in ampNames} 

303 minMeanSignalDict = {ampName: 0.0 for ampName in ampNames} 

304 for ampName in ampNames: 

305 if 'ALL_AMPS' in self.config.maxMeanSignal: 

306 maxMeanSignalDict[ampName] = self.config.maxMeanSignal['ALL_AMPS'] 

307 elif ampName in self.config.maxMeanSignal: 

308 maxMeanSignalDict[ampName] = self.config.maxMeanSignal[ampName] 

309 

310 if 'ALL_AMPS' in self.config.minMeanSignal: 

311 minMeanSignalDict[ampName] = self.config.minMeanSignal['ALL_AMPS'] 

312 elif ampName in self.config.minMeanSignal: 

313 minMeanSignalDict[ampName] = self.config.minMeanSignal[ampName] 

314 

315 tupleRecords = [] 

316 allTags = [] 

317 for expTime, (exp1, exp2) in expPairs.items(): 

318 expId1 = exp1.getInfo().getVisitInfo().getExposureId() 

319 expId2 = exp2.getInfo().getVisitInfo().getExposureId() 

320 tupleRows = [] 

321 nAmpsNan = 0 

322 tags = ['mu', 'i', 'j', 'var', 'cov', 'npix', 'ext', 'expTime', 'ampName'] 

323 for ampNumber, amp in enumerate(detector): 

324 ampName = amp.getName() 

325 # covAstier: (i, j, var (cov[0,0]), cov, npix) 

326 doRealSpace = self.config.covAstierRealSpace 

327 muDiff, varDiff, covAstier = self.measureMeanVarCov(exp1, exp2, region=amp.getBBox(), 

328 covAstierRealSpace=doRealSpace) 

329 

330 if np.isnan(muDiff) or np.isnan(varDiff) or (covAstier is None): 

331 msg = (f"NaN mean or var, or None cov in amp {ampName} in exposure pair {expId1}," 

332 f" {expId2} of detector {detNum}.") 

333 self.log.warn(msg) 

334 nAmpsNan += 1 

335 continue 

336 if (muDiff <= minMeanSignalDict[ampName]) or (muDiff >= maxMeanSignalDict[ampName]): 

337 continue 

338 

339 datasetPtc.rawExpTimes[ampName].append(expTime) 

340 datasetPtc.rawMeans[ampName].append(muDiff) 

341 datasetPtc.rawVars[ampName].append(varDiff) 

342 

343 tupleRows += [(muDiff, ) + covRow + (ampNumber, expTime, ampName) for covRow in covAstier] 

344 if nAmpsNan == len(ampNames): 

345 msg = f"NaN mean in all amps of exposure pair {expId1}, {expId2} of detector {detNum}." 

346 self.log.warn(msg) 

347 continue 

348 allTags += tags 

349 tupleRecords += tupleRows 

350 covariancesWithTags = np.core.records.fromrecords(tupleRecords, names=allTags) 

351 # Sort raw vectors by rawMeans index 

352 for ampName in datasetPtc.ampNames: 

353 index = np.argsort(datasetPtc.rawMeans[ampName]) 

354 datasetPtc.rawExpTimes[ampName] = np.array(datasetPtc.rawExpTimes[ampName])[index] 

355 datasetPtc.rawMeans[ampName] = np.array(datasetPtc.rawMeans[ampName])[index] 

356 datasetPtc.rawVars[ampName] = np.array(datasetPtc.rawVars[ampName])[index] 

357 

358 if self.config.ptcFitType in ["FULLCOVARIANCE", ]: 

359 # Calculate covariances and fit them, including the PTC, to Astier+19 full model (Eq. 20) 

360 datasetPtc = self.fitCovariancesAstier(datasetPtc, covariancesWithTags) 

361 elif self.config.ptcFitType in ["EXPAPPROXIMATION", "POLYNOMIAL"]: 

362 # Fit the PTC to a polynomial or to Astier+19 exponential approximation (Eq. 16) 

363 # Fill up PhotonTransferCurveDataset object. 

364 datasetPtc = self.fitPtc(datasetPtc, self.config.ptcFitType) 

365 

366 detName = detector.getName() 

367 now = datetime.datetime.utcnow() 

368 calibDate = now.strftime("%Y-%m-%d") 

369 butler = dataRef.getButler() 

370 

371 datasetPtc.updateMetadata(setDate=True, camera=camera, detector=detector) 

372 

373 # Fit a poynomial to calculate non-linearity and persist linearizer. 

374 if self.config.doCreateLinearizer: 

375 # Fit (non)linearity of signal vs time curve. 

376 # Fill up PhotonTransferCurveDataset object. 

377 # Fill up array for LUT linearizer (tableArray). 

378 # Produce coefficients for Polynomial and Squared linearizers. 

379 # Build linearizer objects. 

380 dimensions = {'camera': camera.getName(), 'detector': detector.getId()} 

381 linearityResults = self.linearity.run(datasetPtc, camera, dimensions) 

382 linearizer = linearityResults.outputLinearizer 

383 

384 self.log.info("Writing linearizer:") 

385 

386 detName = detector.getName() 

387 now = datetime.datetime.utcnow() 

388 calibDate = now.strftime("%Y-%m-%d") 

389 

390 butler.put(linearizer, datasetType='linearizer', 

391 dataId={'detector': detNum, 'detectorName': detName, 'calibDate': calibDate}) 

392 

393 self.log.info(f"Writing PTC data.") 

394 butler.put(datasetPtc, datasetType='photonTransferCurveDataset', dataId={'detector': detNum, 

395 'detectorName': detName, 'calibDate': calibDate}) 

396 

397 return pipeBase.Struct(exitStatus=0) 

398 

399 def makePairs(self, dataRefList): 

400 """Produce a list of flat pairs indexed by exposure time. 

401 

402 Parameters 

403 ---------- 

404 dataRefList : `list` [`lsst.daf.peristence.ButlerDataRef`] 

405 Data references for exposures for detectors to process. 

406 

407 Return 

408 ------ 

409 flatPairs : `dict` [`float`, `lsst.afw.image.exposure.exposure.ExposureF`] 

410 Dictionary that groups flat-field exposures that have the same exposure time (seconds). 

411 

412 Notes 

413 ----- 

414 We use the difference of one pair of flat-field images taken at the same exposure time when 

415 calculating the PTC to reduce Fixed Pattern Noise. If there are > 2 flat-field images with the 

416 same exposure time, the first two are kept and the rest discarded. 

417 """ 

418 

419 # Organize exposures by observation date. 

420 expDict = {} 

421 for dataRef in dataRefList: 

422 try: 

423 tempFlat = dataRef.get("postISRCCD") 

424 except RuntimeError: 

425 self.log.warn("postISR exposure could not be retrieved. Ignoring flat.") 

426 continue 

427 expDate = tempFlat.getInfo().getVisitInfo().getDate().get() 

428 expDict.setdefault(expDate, tempFlat) 

429 sortedExps = {k: expDict[k] for k in sorted(expDict)} 

430 

431 flatPairs = {} 

432 for exp in sortedExps: 

433 tempFlat = sortedExps[exp] 

434 expTime = tempFlat.getInfo().getVisitInfo().getExposureTime() 

435 listAtExpTime = flatPairs.setdefault(expTime, []) 

436 if len(listAtExpTime) >= 2: 

437 self.log.warn(f"Already found 2 exposures at expTime {expTime}. " 

438 f"Ignoring exposure {tempFlat.getInfo().getVisitInfo().getExposureId()}") 

439 else: 

440 listAtExpTime.append(tempFlat) 

441 

442 keysToDrop = [] 

443 for (key, value) in flatPairs.items(): 

444 if len(value) < 2: 

445 keysToDrop.append(key) 

446 

447 if len(keysToDrop): 

448 for key in keysToDrop: 

449 self.log.warn(f"Only one exposure found at expTime {key}. Dropping exposure " 

450 f"{flatPairs[key][0].getInfo().getVisitInfo().getExposureId()}.") 

451 flatPairs.pop(key) 

452 sortedFlatPairs = {k: flatPairs[k] for k in sorted(flatPairs)} 

453 return sortedFlatPairs 

454 

455 def fitCovariancesAstier(self, dataset, covariancesWithTagsArray): 

456 """Fit measured flat covariances to full model in Astier+19. 

457 

458 Parameters 

459 ---------- 

460 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset` 

461 The dataset containing information such as the means, variances and exposure times. 

462 

463 covariancesWithTagsArray : `numpy.recarray` 

464 Tuple with at least (mu, cov, var, i, j, npix), where: 

465 mu : 0.5*(m1 + m2), where: 

466 mu1: mean value of flat1 

467 mu2: mean value of flat2 

468 cov: covariance value at lag(i, j) 

469 var: variance(covariance value at lag(0, 0)) 

470 i: lag dimension 

471 j: lag dimension 

472 npix: number of pixels used for covariance calculation. 

473 

474 Returns 

475 ------- 

476 dataset: `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset` 

477 This is the same dataset as the input paramter, however, it has been modified 

478 to include information such as the fit vectors and the fit parameters. See 

479 the class `PhotonTransferCurveDatase`. 

480 """ 

481 

482 covFits, covFitsNoB = fitData(covariancesWithTagsArray, 

483 r=self.config.maximumRangeCovariancesAstier, 

484 nSigmaFullFit=self.config.sigmaClipFullFitCovariancesAstier, 

485 maxIterFullFit=self.config.maxIterFullFitCovariancesAstier) 

486 

487 dataset = self.getOutputPtcDataCovAstier(dataset, covFits, covFitsNoB) 

488 

489 return dataset 

490 

491 def getOutputPtcDataCovAstier(self, dataset, covFits, covFitsNoB): 

492 """Get output data for PhotonTransferCurveCovAstierDataset from CovFit objects. 

493 

494 Parameters 

495 ---------- 

496 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset` 

497 The dataset containing information such as the means, variances and exposure times. 

498 

499 covFits: `dict` 

500 Dictionary of CovFit objects, with amp names as keys. 

501 

502 covFitsNoB : `dict` 

503 Dictionary of CovFit objects, with amp names as keys, and 'b=0' in Eq. 20 of Astier+19. 

504 

505 Returns 

506 ------- 

507 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset` 

508 This is the same dataset as the input paramter, however, it has been modified 

509 to include extra information such as the mask 1D array, gains, reoudout noise, measured signal, 

510 measured variance, modeled variance, a, and b coefficient matrices (see Astier+19) per amplifier. 

511 See the class `PhotonTransferCurveDatase`. 

512 """ 

513 assert(len(covFits) == len(covFitsNoB)) 

514 

515 for i, amp in enumerate(dataset.ampNames): 

516 lenInputTimes = len(dataset.rawExpTimes[amp]) 

517 # Not used when ptcFitType is 'FULLCOVARIANCE' 

518 dataset.ptcFitPars[amp] = np.nan 

519 dataset.ptcFitParsError[amp] = np.nan 

520 dataset.ptcFitChiSq[amp] = np.nan 

521 if (amp in covFits and (covFits[amp].covParams is not None) and 

522 (covFitsNoB[amp].covParams is not None)): 

523 fit = covFits[amp] 

524 fitNoB = covFitsNoB[amp] 

525 # Save full covariances, covariances models, and their weights 

526 dataset.covariances[amp] = fit.cov 

527 dataset.covariancesModel[amp] = fit.evalCovModel() 

528 dataset.covariancesSqrtWeights[amp] = fit.sqrtW 

529 dataset.aMatrix[amp] = fit.getA() 

530 dataset.bMatrix[amp] = fit.getB() 

531 dataset.covariancesNoB[amp] = fitNoB.cov 

532 dataset.covariancesModelNoB[amp] = fitNoB.evalCovModel() 

533 dataset.covariancesSqrtWeightsNoB[amp] = fitNoB.sqrtW 

534 dataset.aMatrixNoB[amp] = fitNoB.getA() 

535 

536 (meanVecFinal, varVecFinal, varVecModel, 

537 wc, varMask) = fit.getFitData(0, 0, divideByMu=False, returnMasked=True) 

538 gain = fit.getGain() 

539 # adjust mask to original size of rawExpTimes 

540 # so far, only the min/max signal cut is in dataset.expIdMask 

541 dataset.expIdMask[amp] = varMask 

542 dataset.gain[amp] = gain 

543 dataset.gainErr[amp] = fit.getGainErr() 

544 dataset.noise[amp] = np.sqrt(fit.getRon()) 

545 dataset.noiseErr[amp] = fit.getRonErr() 

546 

547 padLength = lenInputTimes - len(varVecFinal) 

548 dataset.finalVars[amp] = np.pad(varVecFinal/(gain**2), (0, padLength), 'constant', 

549 constant_values=np.nan) 

550 dataset.finalModelVars[amp] = np.pad(varVecModel/(gain**2), (0, padLength), 'constant', 

551 constant_values=np.nan) 

552 dataset.finalMeans[amp] = np.pad(meanVecFinal/gain, (0, padLength), 'constant', 

553 constant_values=np.nan) 

554 else: 

555 # Bad amp 

556 # Entries need to have proper dimensions so read/write with astropy.Table works. 

557 matrixSide = self.config.maximumRangeCovariancesAstier 

558 nanMatrix = np.full((matrixSide, matrixSide), np.nan) 

559 listNanMatrix = np.full((lenInputTimes, matrixSide, matrixSide), np.nan) 

560 

561 dataset.covariances[amp] = listNanMatrix 

562 dataset.covariancesModel[amp] = listNanMatrix 

563 dataset.covariancesSqrtWeights[amp] = listNanMatrix 

564 dataset.aMatrix[amp] = nanMatrix 

565 dataset.bMatrix[amp] = nanMatrix 

566 dataset.covariancesNoB[amp] = listNanMatrix 

567 dataset.covariancesModelNoB[amp] = listNanMatrix 

568 dataset.covariancesSqrtWeightsNoB[amp] = listNanMatrix 

569 dataset.aMatrixNoB[amp] = nanMatrix 

570 

571 dataset.expIdMask[amp] = np.repeat(np.nan, lenInputTimes) 

572 dataset.gain[amp] = np.nan 

573 dataset.gainErr[amp] = np.nan 

574 dataset.noise[amp] = np.nan 

575 dataset.noiseErr[amp] = np.nan 

576 dataset.finalVars[amp] = np.repeat(np.nan, lenInputTimes) 

577 dataset.finalModelVars[amp] = np.repeat(np.nan, lenInputTimes) 

578 dataset.finalMeans[amp] = np.repeat(np.nan, lenInputTimes) 

579 

580 return dataset 

581 

582 def measureMeanVarCov(self, exposure1, exposure2, region=None, covAstierRealSpace=False): 

583 """Calculate the mean of each of two exposures and the variance and covariance of their difference. 

584 

585 The variance is calculated via afwMath, and the covariance via the methods in Astier+19 (appendix A). 

586 In theory, var = covariance[0,0]. This should be validated, and in the future, we may decide to just 

587 keep one (covariance). 

588 

589 Parameters 

590 ---------- 

591 exposure1 : `lsst.afw.image.exposure.exposure.ExposureF` 

592 First exposure of flat field pair. 

593 

594 exposure2 : `lsst.afw.image.exposure.exposure.ExposureF` 

595 Second exposure of flat field pair. 

596 

597 region : `lsst.geom.Box2I`, optional 

598 Region of each exposure where to perform the calculations (e.g, an amplifier). 

599 

600 covAstierRealSpace : `bool`, optional 

601 Should the covariannces in Astier+19 be calculated in real space or via FFT? 

602 See Appendix A of Astier+19. 

603 

604 Returns 

605 ------- 

606 mu : `float` or `NaN` 

607 0.5*(mu1 + mu2), where mu1, and mu2 are the clipped means of the regions in 

608 both exposures. If either mu1 or m2 are NaN's, the returned value is NaN. 

609 

610 varDiff : `float` or `NaN` 

611 Half of the clipped variance of the difference of the regions inthe two input 

612 exposures. If either mu1 or m2 are NaN's, the returned value is NaN. 

613 

614 covDiffAstier : `list` or `NaN` 

615 List with tuples of the form (dx, dy, var, cov, npix), where: 

616 dx : `int` 

617 Lag in x 

618 dy : `int` 

619 Lag in y 

620 var : `float` 

621 Variance at (dx, dy). 

622 cov : `float` 

623 Covariance at (dx, dy). 

624 nPix : `int` 

625 Number of pixel pairs used to evaluate var and cov. 

626 If either mu1 or m2 are NaN's, the returned value is NaN. 

627 """ 

628 

629 if region is not None: 

630 im1Area = exposure1.maskedImage[region] 

631 im2Area = exposure2.maskedImage[region] 

632 else: 

633 im1Area = exposure1.maskedImage 

634 im2Area = exposure2.maskedImage 

635 

636 if self.config.binSize > 1: 

637 im1Area = afwMath.binImage(im1Area, self.config.binSize) 

638 im2Area = afwMath.binImage(im2Area, self.config.binSize) 

639 

640 im1MaskVal = exposure1.getMask().getPlaneBitMask(self.config.maskNameList) 

641 im1StatsCtrl = afwMath.StatisticsControl(self.config.nSigmaClipPtc, 

642 self.config.nIterSigmaClipPtc, 

643 im1MaskVal) 

644 im1StatsCtrl.setNanSafe(True) 

645 im1StatsCtrl.setAndMask(im1MaskVal) 

646 

647 im2MaskVal = exposure2.getMask().getPlaneBitMask(self.config.maskNameList) 

648 im2StatsCtrl = afwMath.StatisticsControl(self.config.nSigmaClipPtc, 

649 self.config.nIterSigmaClipPtc, 

650 im2MaskVal) 

651 im2StatsCtrl.setNanSafe(True) 

652 im2StatsCtrl.setAndMask(im2MaskVal) 

653 

654 # Clipped mean of images; then average of mean. 

655 mu1 = afwMath.makeStatistics(im1Area, afwMath.MEANCLIP, im1StatsCtrl).getValue() 

656 mu2 = afwMath.makeStatistics(im2Area, afwMath.MEANCLIP, im2StatsCtrl).getValue() 

657 if np.isnan(mu1) or np.isnan(mu2): 

658 return np.nan, np.nan, None 

659 mu = 0.5*(mu1 + mu2) 

660 

661 # Take difference of pairs 

662 # symmetric formula: diff = (mu2*im1-mu1*im2)/(0.5*(mu1+mu2)) 

663 temp = im2Area.clone() 

664 temp *= mu1 

665 diffIm = im1Area.clone() 

666 diffIm *= mu2 

667 diffIm -= temp 

668 diffIm /= mu 

669 

670 diffImMaskVal = diffIm.getMask().getPlaneBitMask(self.config.maskNameList) 

671 diffImStatsCtrl = afwMath.StatisticsControl(self.config.nSigmaClipPtc, 

672 self.config.nIterSigmaClipPtc, 

673 diffImMaskVal) 

674 diffImStatsCtrl.setNanSafe(True) 

675 diffImStatsCtrl.setAndMask(diffImMaskVal) 

676 

677 varDiff = 0.5*(afwMath.makeStatistics(diffIm, afwMath.VARIANCECLIP, diffImStatsCtrl).getValue()) 

678 

679 # Get the mask and identify good pixels as '1', and the rest as '0'. 

680 w1 = np.where(im1Area.getMask().getArray() == 0, 1, 0) 

681 w2 = np.where(im2Area.getMask().getArray() == 0, 1, 0) 

682 

683 w12 = w1*w2 

684 wDiff = np.where(diffIm.getMask().getArray() == 0, 1, 0) 

685 w = w12*wDiff 

686 

687 maxRangeCov = self.config.maximumRangeCovariancesAstier 

688 if covAstierRealSpace: 

689 covDiffAstier = computeCovDirect(diffIm.getImage().getArray(), w, maxRangeCov) 

690 else: 

691 shapeDiff = diffIm.getImage().getArray().shape 

692 fftShape = (fftSize(shapeDiff[0] + maxRangeCov), fftSize(shapeDiff[1]+maxRangeCov)) 

693 c = CovFft(diffIm.getImage().getArray(), w, fftShape, maxRangeCov) 

694 covDiffAstier = c.reportCovFft(maxRangeCov) 

695 

696 return mu, varDiff, covDiffAstier 

697 

698 def computeCovDirect(self, diffImage, weightImage, maxRange): 

699 """Compute covariances of diffImage in real space. 

700 

701 For lags larger than ~25, it is slower than the FFT way. 

702 Taken from https://github.com/PierreAstier/bfptc/ 

703 

704 Parameters 

705 ---------- 

706 diffImage : `numpy.array` 

707 Image to compute the covariance of. 

708 

709 weightImage : `numpy.array` 

710 Weight image of diffImage (1's and 0's for good and bad pixels, respectively). 

711 

712 maxRange : `int` 

713 Last index of the covariance to be computed. 

714 

715 Returns 

716 ------- 

717 outList : `list` 

718 List with tuples of the form (dx, dy, var, cov, npix), where: 

719 dx : `int` 

720 Lag in x 

721 dy : `int` 

722 Lag in y 

723 var : `float` 

724 Variance at (dx, dy). 

725 cov : `float` 

726 Covariance at (dx, dy). 

727 nPix : `int` 

728 Number of pixel pairs used to evaluate var and cov. 

729 """ 

730 outList = [] 

731 var = 0 

732 # (dy,dx) = (0,0) has to be first 

733 for dy in range(maxRange + 1): 

734 for dx in range(0, maxRange + 1): 

735 if (dx*dy > 0): 

736 cov1, nPix1 = self.covDirectValue(diffImage, weightImage, dx, dy) 

737 cov2, nPix2 = self.covDirectValue(diffImage, weightImage, dx, -dy) 

738 cov = 0.5*(cov1 + cov2) 

739 nPix = nPix1 + nPix2 

740 else: 

741 cov, nPix = self.covDirectValue(diffImage, weightImage, dx, dy) 

742 if (dx == 0 and dy == 0): 

743 var = cov 

744 outList.append((dx, dy, var, cov, nPix)) 

745 

746 return outList 

747 

748 def covDirectValue(self, diffImage, weightImage, dx, dy): 

749 """Compute covariances of diffImage in real space at lag (dx, dy). 

750 

751 Taken from https://github.com/PierreAstier/bfptc/ (c.f., appendix of Astier+19). 

752 

753 Parameters 

754 ---------- 

755 diffImage : `numpy.array` 

756 Image to compute the covariance of. 

757 

758 weightImage : `numpy.array` 

759 Weight image of diffImage (1's and 0's for good and bad pixels, respectively). 

760 

761 dx : `int` 

762 Lag in x. 

763 

764 dy : `int` 

765 Lag in y. 

766 

767 Returns 

768 ------- 

769 cov : `float` 

770 Covariance at (dx, dy) 

771 

772 nPix : `int` 

773 Number of pixel pairs used to evaluate var and cov. 

774 """ 

775 (nCols, nRows) = diffImage.shape 

776 # switching both signs does not change anything: 

777 # it just swaps im1 and im2 below 

778 if (dx < 0): 

779 (dx, dy) = (-dx, -dy) 

780 # now, we have dx >0. We have to distinguish two cases 

781 # depending on the sign of dy 

782 if dy >= 0: 

783 im1 = diffImage[dy:, dx:] 

784 w1 = weightImage[dy:, dx:] 

785 im2 = diffImage[:nCols - dy, :nRows - dx] 

786 w2 = weightImage[:nCols - dy, :nRows - dx] 

787 else: 

788 im1 = diffImage[:nCols + dy, dx:] 

789 w1 = weightImage[:nCols + dy, dx:] 

790 im2 = diffImage[-dy:, :nRows - dx] 

791 w2 = weightImage[-dy:, :nRows - dx] 

792 # use the same mask for all 3 calculations 

793 wAll = w1*w2 

794 # do not use mean() because weightImage=0 pixels would then count 

795 nPix = wAll.sum() 

796 im1TimesW = im1*wAll 

797 s1 = im1TimesW.sum()/nPix 

798 s2 = (im2*wAll).sum()/nPix 

799 p = (im1TimesW*im2).sum()/nPix 

800 cov = p - s1*s2 

801 

802 return cov, nPix 

803 

804 @staticmethod 

805 def _initialParsForPolynomial(order): 

806 assert(order >= 2) 

807 pars = np.zeros(order, dtype=np.float) 

808 pars[0] = 10 

809 pars[1] = 1 

810 pars[2:] = 0.0001 

811 return pars 

812 

813 @staticmethod 

814 def _boundsForPolynomial(initialPars, lowers=[], uppers=[]): 

815 if not len(lowers): 

816 lowers = [np.NINF for p in initialPars] 

817 if not len(uppers): 

818 uppers = [np.inf for p in initialPars] 

819 lowers[1] = 0 # no negative gains 

820 return (lowers, uppers) 

821 

822 @staticmethod 

823 def _boundsForAstier(initialPars, lowers=[], uppers=[]): 

824 if not len(lowers): 

825 lowers = [np.NINF for p in initialPars] 

826 if not len(uppers): 

827 uppers = [np.inf for p in initialPars] 

828 return (lowers, uppers) 

829 

830 @staticmethod 

831 def _getInitialGoodPoints(means, variances, maxDeviationPositive, maxDeviationNegative): 

832 """Return a boolean array to mask bad points. 

833 

834 Parameters 

835 ---------- 

836 means : `numpy.array` 

837 Input array with mean signal values. 

838 

839 variances : `numpy.array` 

840 Input array with variances at each mean value. 

841 

842 maxDeviationPositive : `float` 

843 Maximum deviation from being constant for the variance/mean 

844 ratio, in the positive direction. 

845 

846 maxDeviationNegative : `float` 

847 Maximum deviation from being constant for the variance/mean 

848 ratio, in the negative direction. 

849 

850 Return 

851 ------ 

852 goodPoints : `numpy.array` [`bool`] 

853 Boolean array to select good (`True`) and bad (`False`) 

854 points. 

855 

856 Notes 

857 ----- 

858 A linear function has a constant ratio, so find the median 

859 value of the ratios, and exclude the points that deviate 

860 from that by more than a factor of maxDeviationPositive/negative. 

861 Asymmetric deviations are supported as we expect the PTC to turn 

862 down as the flux increases, but sometimes it anomalously turns 

863 upwards just before turning over, which ruins the fits, so it 

864 is wise to be stricter about restricting positive outliers than 

865 negative ones. 

866 

867 Too high and points that are so bad that fit will fail will be included 

868 Too low and the non-linear points will be excluded, biasing the NL fit.""" 

869 

870 assert(len(means) == len(variances)) 

871 ratios = [b/a for (a, b) in zip(means, variances)] 

872 medianRatio = np.nanmedian(ratios) 

873 ratioDeviations = [(r/medianRatio)-1 for r in ratios] 

874 

875 # so that it doesn't matter if the deviation is expressed as positive or negative 

876 maxDeviationPositive = abs(maxDeviationPositive) 

877 maxDeviationNegative = -1. * abs(maxDeviationNegative) 

878 

879 goodPoints = np.array([True if (r < maxDeviationPositive and r > maxDeviationNegative) 

880 else False for r in ratioDeviations]) 

881 return goodPoints 

882 

883 def _makeZeroSafe(self, array, warn=True, substituteValue=1e-9): 

884 """""" 

885 nBad = Counter(array)[0] 

886 if nBad == 0: 

887 return array 

888 

889 if warn: 

890 msg = f"Found {nBad} zeros in array at elements {[x for x in np.where(array==0)[0]]}" 

891 self.log.warn(msg) 

892 

893 array[array == 0] = substituteValue 

894 return array 

895 

896 def fitPtc(self, dataset, ptcFitType): 

897 """Fit the photon transfer curve to a polynimial or to Astier+19 approximation. 

898 

899 Fit the photon transfer curve with either a polynomial of the order 

900 specified in the task config, or using the Astier approximation. 

901 

902 Sigma clipping is performed iteratively for the fit, as well as an 

903 initial clipping of data points that are more than 

904 config.initialNonLinearityExclusionThreshold away from lying on a 

905 straight line. This other step is necessary because the photon transfer 

906 curve turns over catastrophically at very high flux (because saturation 

907 drops the variance to ~0) and these far outliers cause the initial fit 

908 to fail, meaning the sigma cannot be calculated to perform the 

909 sigma-clipping. 

910 

911 Parameters 

912 ---------- 

913 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset` 

914 The dataset containing the means, variances and exposure times 

915 

916 ptcFitType : `str` 

917 Fit a 'POLYNOMIAL' (degree: 'polynomialFitDegree') or 

918 'EXPAPPROXIMATION' (Eq. 16 of Astier+19) to the PTC 

919 

920 Returns 

921 ------- 

922 dataset: `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset` 

923 This is the same dataset as the input paramter, however, it has been modified 

924 to include information such as the fit vectors and the fit parameters. See 

925 the class `PhotonTransferCurveDatase`. 

926 """ 

927 

928 matrixSide = self.config.maximumRangeCovariancesAstier 

929 nanMatrix = np.empty((matrixSide, matrixSide)) 

930 nanMatrix[:] = np.nan 

931 

932 for amp in dataset.ampNames: 

933 lenInputTimes = len(dataset.rawExpTimes[amp]) 

934 listNanMatrix = np.empty((lenInputTimes, matrixSide, matrixSide)) 

935 listNanMatrix[:] = np.nan 

936 

937 dataset.covariances[amp] = listNanMatrix 

938 dataset.covariancesModel[amp] = listNanMatrix 

939 dataset.covariancesSqrtWeights[amp] = listNanMatrix 

940 dataset.aMatrix[amp] = nanMatrix 

941 dataset.bMatrix[amp] = nanMatrix 

942 dataset.covariancesNoB[amp] = listNanMatrix 

943 dataset.covariancesModelNoB[amp] = listNanMatrix 

944 dataset.covariancesSqrtWeightsNoB[amp] = listNanMatrix 

945 dataset.aMatrixNoB[amp] = nanMatrix 

946 

947 def errFunc(p, x, y): 

948 return ptcFunc(p, x) - y 

949 

950 sigmaCutPtcOutliers = self.config.sigmaCutPtcOutliers 

951 maxIterationsPtcOutliers = self.config.maxIterationsPtcOutliers 

952 

953 for i, ampName in enumerate(dataset.ampNames): 

954 timeVecOriginal = np.array(dataset.rawExpTimes[ampName]) 

955 meanVecOriginal = np.array(dataset.rawMeans[ampName]) 

956 varVecOriginal = np.array(dataset.rawVars[ampName]) 

957 varVecOriginal = self._makeZeroSafe(varVecOriginal) 

958 

959 goodPoints = self._getInitialGoodPoints(meanVecOriginal, varVecOriginal, 

960 self.config.initialNonLinearityExclusionThresholdPositive, 

961 self.config.initialNonLinearityExclusionThresholdNegative) 

962 if not (goodPoints.any()): 

963 msg = (f"\nSERIOUS: All points in goodPoints: {goodPoints} are bad." 

964 f"Setting {ampName} to BAD.") 

965 self.log.warn(msg) 

966 # The first and second parameters of initial fit are discarded (bias and gain) 

967 # for the final NL coefficients 

968 dataset.badAmps.append(ampName) 

969 dataset.expIdMask[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName])) 

970 dataset.gain[ampName] = np.nan 

971 dataset.gainErr[ampName] = np.nan 

972 dataset.noise[ampName] = np.nan 

973 dataset.noiseErr[ampName] = np.nan 

974 dataset.ptcFitPars[ampName] = (np.repeat(np.nan, self.config.polynomialFitDegree + 1) if 

975 ptcFitType in ["POLYNOMIAL", ] else np.repeat(np.nan, 3)) 

976 dataset.ptcFitParsError[ampName] = (np.repeat(np.nan, self.config.polynomialFitDegree + 1) if 

977 ptcFitType in ["POLYNOMIAL", ] else np.repeat(np.nan, 3)) 

978 dataset.ptcFitChiSq[ampName] = np.nan 

979 dataset.finalVars[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName])) 

980 dataset.finalModelVars[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName])) 

981 dataset.finalMeans[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName])) 

982 continue 

983 

984 mask = goodPoints 

985 

986 if ptcFitType == 'EXPAPPROXIMATION': 

987 ptcFunc = funcAstier 

988 parsIniPtc = [-1e-9, 1.0, 10.] # a00, gain, noise 

989 # lowers and uppers obtained from studies by C. Lage (UC Davis, 11/2020). 

990 bounds = self._boundsForAstier(parsIniPtc, lowers=[-1e-4, 0.5, -100], 

991 uppers=[1e-4, 2.5, 100]) 

992 if ptcFitType == 'POLYNOMIAL': 

993 ptcFunc = funcPolynomial 

994 parsIniPtc = self._initialParsForPolynomial(self.config.polynomialFitDegree + 1) 

995 bounds = self._boundsForPolynomial(parsIniPtc) 

996 

997 # Before bootstrap fit, do an iterative fit to get rid of outliers 

998 count = 1 

999 while count <= maxIterationsPtcOutliers: 

1000 # Note that application of the mask actually shrinks the array 

1001 # to size rather than setting elements to zero (as we want) so 

1002 # always update mask itself and re-apply to the original data 

1003 meanTempVec = meanVecOriginal[mask] 

1004 varTempVec = varVecOriginal[mask] 

1005 res = least_squares(errFunc, parsIniPtc, bounds=bounds, args=(meanTempVec, varTempVec)) 

1006 pars = res.x 

1007 

1008 # change this to the original from the temp because the masks are ANDed 

1009 # meaning once a point is masked it's always masked, and the masks must 

1010 # always be the same length for broadcasting 

1011 sigResids = (varVecOriginal - ptcFunc(pars, meanVecOriginal))/np.sqrt(varVecOriginal) 

1012 newMask = np.array([True if np.abs(r) < sigmaCutPtcOutliers else False for r in sigResids]) 

1013 mask = mask & newMask 

1014 if not (mask.any() and newMask.any()): 

1015 msg = (f"\nSERIOUS: All points in either mask: {mask} or newMask: {newMask} are bad. " 

1016 f"Setting {ampName} to BAD.") 

1017 self.log.warn(msg) 

1018 # The first and second parameters of initial fit are discarded (bias and gain) 

1019 # for the final NL coefficients 

1020 dataset.badAmps.append(ampName) 

1021 dataset.expIdMask[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName])) 

1022 dataset.gain[ampName] = np.nan 

1023 dataset.gainErr[ampName] = np.nan 

1024 dataset.noise[ampName] = np.nan 

1025 dataset.noiseErr[ampName] = np.nan 

1026 dataset.ptcFitPars[ampName] = (np.repeat(np.nan, self.config.polynomialFitDegree + 1) 

1027 if ptcFitType in ["POLYNOMIAL", ] else 

1028 np.repeat(np.nan, 3)) 

1029 dataset.ptcFitParsError[ampName] = (np.repeat(np.nan, self.config.polynomialFitDegree + 1) 

1030 if ptcFitType in ["POLYNOMIAL", ] else 

1031 np.repeat(np.nan, 3)) 

1032 dataset.ptcFitChiSq[ampName] = np.nan 

1033 dataset.finalVars[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName])) 

1034 dataset.finalModelVars[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName])) 

1035 dataset.finalMeans[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName])) 

1036 break 

1037 nDroppedTotal = Counter(mask)[False] 

1038 self.log.debug(f"Iteration {count}: discarded {nDroppedTotal} points in total for {ampName}") 

1039 count += 1 

1040 # objects should never shrink 

1041 assert (len(mask) == len(timeVecOriginal) == len(meanVecOriginal) == len(varVecOriginal)) 

1042 

1043 if not (mask.any() and newMask.any()): 

1044 continue 

1045 dataset.expIdMask[ampName] = mask # store the final mask 

1046 parsIniPtc = pars 

1047 meanVecFinal = meanVecOriginal[mask] 

1048 varVecFinal = varVecOriginal[mask] 

1049 

1050 if Counter(mask)[False] > 0: 

1051 self.log.info((f"Number of points discarded in PTC of amplifier {ampName}:" + 

1052 f" {Counter(mask)[False]} out of {len(meanVecOriginal)}")) 

1053 

1054 if (len(meanVecFinal) < len(parsIniPtc)): 

1055 msg = (f"\nSERIOUS: Not enough data points ({len(meanVecFinal)}) compared to the number of" 

1056 f"parameters of the PTC model({len(parsIniPtc)}). Setting {ampName} to BAD.") 

1057 self.log.warn(msg) 

1058 # The first and second parameters of initial fit are discarded (bias and gain) 

1059 # for the final NL coefficients 

1060 dataset.badAmps.append(ampName) 

1061 dataset.expIdMask[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName])) 

1062 dataset.gain[ampName] = np.nan 

1063 dataset.gainErr[ampName] = np.nan 

1064 dataset.noise[ampName] = np.nan 

1065 dataset.noiseErr[ampName] = np.nan 

1066 dataset.ptcFitPars[ampName] = (np.repeat(np.nan, self.config.polynomialFitDegree + 1) if 

1067 ptcFitType in ["POLYNOMIAL", ] else np.repeat(np.nan, 3)) 

1068 dataset.ptcFitParsError[ampName] = (np.repeat(np.nan, self.config.polynomialFitDegree + 1) if 

1069 ptcFitType in ["POLYNOMIAL", ] else np.repeat(np.nan, 3)) 

1070 dataset.ptcFitChiSq[ampName] = np.nan 

1071 dataset.finalVars[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName])) 

1072 dataset.finalModelVars[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName])) 

1073 dataset.finalMeans[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName])) 

1074 continue 

1075 

1076 # Fit the PTC 

1077 if self.config.doFitBootstrap: 

1078 parsFit, parsFitErr, reducedChiSqPtc = fitBootstrap(parsIniPtc, meanVecFinal, 

1079 varVecFinal, ptcFunc, 

1080 weightsY=1./np.sqrt(varVecFinal)) 

1081 else: 

1082 parsFit, parsFitErr, reducedChiSqPtc = fitLeastSq(parsIniPtc, meanVecFinal, 

1083 varVecFinal, ptcFunc, 

1084 weightsY=1./np.sqrt(varVecFinal)) 

1085 dataset.ptcFitPars[ampName] = parsFit 

1086 dataset.ptcFitParsError[ampName] = parsFitErr 

1087 dataset.ptcFitChiSq[ampName] = reducedChiSqPtc 

1088 # Masked variances (measured and modeled) and means. Need to pad the array so astropy.Table does 

1089 # not crash (the mask may vary per amp). 

1090 padLength = len(dataset.rawExpTimes[ampName]) - len(varVecFinal) 

1091 dataset.finalVars[ampName] = np.pad(varVecFinal, (0, padLength), 'constant', 

1092 constant_values=np.nan) 

1093 dataset.finalModelVars[ampName] = np.pad(ptcFunc(parsFit, meanVecFinal), (0, padLength), 

1094 'constant', constant_values=np.nan) 

1095 dataset.finalMeans[ampName] = np.pad(meanVecFinal, (0, padLength), 'constant', 

1096 constant_values=np.nan) 

1097 

1098 if ptcFitType == 'EXPAPPROXIMATION': 

1099 ptcGain = parsFit[1] 

1100 ptcGainErr = parsFitErr[1] 

1101 ptcNoise = np.sqrt(np.fabs(parsFit[2])) 

1102 ptcNoiseErr = 0.5*(parsFitErr[2]/np.fabs(parsFit[2]))*np.sqrt(np.fabs(parsFit[2])) 

1103 if ptcFitType == 'POLYNOMIAL': 

1104 ptcGain = 1./parsFit[1] 

1105 ptcGainErr = np.fabs(1./parsFit[1])*(parsFitErr[1]/parsFit[1]) 

1106 ptcNoise = np.sqrt(np.fabs(parsFit[0]))*ptcGain 

1107 ptcNoiseErr = (0.5*(parsFitErr[0]/np.fabs(parsFit[0]))*(np.sqrt(np.fabs(parsFit[0]))))*ptcGain 

1108 dataset.gain[ampName] = ptcGain 

1109 dataset.gainErr[ampName] = ptcGainErr 

1110 dataset.noise[ampName] = ptcNoise 

1111 dataset.noiseErr[ampName] = ptcNoiseErr 

1112 if not len(dataset.ptcFitType) == 0: 

1113 dataset.ptcFitType = ptcFitType 

1114 if len(dataset.badAmps) == 0: 

1115 dataset.badAmps = np.repeat(np.nan, len(list(dataset.rawExpTimes.values())[0])) 

1116 

1117 return dataset