Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of cp_pipe. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21# 

22import numpy as np 

23import matplotlib.pyplot as plt 

24from collections import Counter 

25 

26import lsst.afw.math as afwMath 

27import lsst.pex.config as pexConfig 

28import lsst.pipe.base as pipeBase 

29from .utils import (fitLeastSq, fitBootstrap, funcPolynomial, funcAstier) 

30from scipy.optimize import least_squares 

31 

32import datetime 

33 

34from .astierCovPtcUtils import (fftSize, CovFft, computeCovDirect, fitData) 

35from .linearity import LinearitySolveTask 

36from .photodiode import getBOTphotodiodeData 

37 

38from lsst.pipe.tasks.getRepositoryData import DataRefListRunner 

39from lsst.ip.isr import PhotonTransferCurveDataset 

40 

41import copy 

42 

43__all__ = ['MeasurePhotonTransferCurveTask', 

44 'MeasurePhotonTransferCurveTaskConfig'] 

45 

46 

47class MeasurePhotonTransferCurveTaskConfig(pexConfig.Config): 

48 """Config class for photon transfer curve measurement task""" 

49 ccdKey = pexConfig.Field( 

50 dtype=str, 

51 doc="The key by which to pull a detector from a dataId, e.g. 'ccd' or 'detector'.", 

52 default='ccd', 

53 ) 

54 ptcFitType = pexConfig.ChoiceField( 

55 dtype=str, 

56 doc="Fit PTC to Eq. 16, Eq. 20 in Astier+19, or to a polynomial.", 

57 default="POLYNOMIAL", 

58 allowed={ 

59 "POLYNOMIAL": "n-degree polynomial (use 'polynomialFitDegree' to set 'n').", 

60 "EXPAPPROXIMATION": "Approximation in Astier+19 (Eq. 16).", 

61 "FULLCOVARIANCE": "Full covariances model in Astier+19 (Eq. 20)" 

62 } 

63 ) 

64 maximumRangeCovariancesAstier = pexConfig.Field( 

65 dtype=int, 

66 doc="Maximum range of covariances as in Astier+19", 

67 default=8, 

68 ) 

69 covAstierRealSpace = pexConfig.Field( 

70 dtype=bool, 

71 doc="Calculate covariances in real space or via FFT? (see appendix A of Astier+19).", 

72 default=False, 

73 ) 

74 polynomialFitDegree = pexConfig.Field( 

75 dtype=int, 

76 doc="Degree of polynomial to fit the PTC, when 'ptcFitType'=POLYNOMIAL.", 

77 default=3, 

78 ) 

79 linearity = pexConfig.ConfigurableField( 

80 target=LinearitySolveTask, 

81 doc="Task to solve the linearity." 

82 ) 

83 

84 doCreateLinearizer = pexConfig.Field( 

85 dtype=bool, 

86 doc="Calculate non-linearity and persist linearizer?", 

87 default=False, 

88 ) 

89 

90 binSize = pexConfig.Field( 

91 dtype=int, 

92 doc="Bin the image by this factor in both dimensions.", 

93 default=1, 

94 ) 

95 minMeanSignal = pexConfig.DictField( 

96 keytype=str, 

97 itemtype=float, 

98 doc="Minimum values (inclusive) of mean signal (in ADU) above which to consider, per amp." 

99 " The same cut is applied to all amps if this dictionary is of the form" 

100 " {'ALL_AMPS': value}", 

101 default={'ALL_AMPS': 0.0}, 

102 ) 

103 maxMeanSignal = pexConfig.DictField( 

104 keytype=str, 

105 itemtype=float, 

106 doc="Maximum values (inclusive) of mean signal (in ADU) below which to consider, per amp." 

107 " The same cut is applied to all amps if this dictionary is of the form" 

108 " {'ALL_AMPS': value}", 

109 default={'ALL_AMPS': 1e6}, 

110 ) 

111 initialNonLinearityExclusionThresholdPositive = pexConfig.RangeField( 

112 dtype=float, 

113 doc="Initially exclude data points with a variance that are more than a factor of this from being" 

114 " linear in the positive direction, from the PTC fit. Note that these points will also be" 

115 " excluded from the non-linearity fit. This is done before the iterative outlier rejection," 

116 " to allow an accurate determination of the sigmas for said iterative fit.", 

117 default=0.12, 

118 min=0.0, 

119 max=1.0, 

120 ) 

121 initialNonLinearityExclusionThresholdNegative = pexConfig.RangeField( 

122 dtype=float, 

123 doc="Initially exclude data points with a variance that are more than a factor of this from being" 

124 " linear in the negative direction, from the PTC fit. Note that these points will also be" 

125 " excluded from the non-linearity fit. This is done before the iterative outlier rejection," 

126 " to allow an accurate determination of the sigmas for said iterative fit.", 

127 default=0.25, 

128 min=0.0, 

129 max=1.0, 

130 ) 

131 sigmaCutPtcOutliers = pexConfig.Field( 

132 dtype=float, 

133 doc="Sigma cut for outlier rejection in PTC.", 

134 default=5.0, 

135 ) 

136 maskNameList = pexConfig.ListField( 

137 dtype=str, 

138 doc="Mask list to exclude from statistics calculations.", 

139 default=['SUSPECT', 'BAD', 'NO_DATA'], 

140 ) 

141 nSigmaClipPtc = pexConfig.Field( 

142 dtype=float, 

143 doc="Sigma cut for afwMath.StatisticsControl()", 

144 default=5.5, 

145 ) 

146 nIterSigmaClipPtc = pexConfig.Field( 

147 dtype=int, 

148 doc="Number of sigma-clipping iterations for afwMath.StatisticsControl()", 

149 default=1, 

150 ) 

151 minNumberGoodPixelsForFft = pexConfig.Field( 

152 dtype=int, 

153 doc="Minimum number of acceptable good pixels per amp to calculate the covariances via FFT.", 

154 default=10000, 

155 ) 

156 maxIterationsPtcOutliers = pexConfig.Field( 

157 dtype=int, 

158 doc="Maximum number of iterations for outlier rejection in PTC.", 

159 default=2, 

160 ) 

161 doFitBootstrap = pexConfig.Field( 

162 dtype=bool, 

163 doc="Use bootstrap for the PTC fit parameters and errors?.", 

164 default=False, 

165 ) 

166 doPhotodiode = pexConfig.Field( 

167 dtype=bool, 

168 doc="Apply a correction based on the photodiode readings if available?", 

169 default=True, 

170 ) 

171 photodiodeDataPath = pexConfig.Field( 

172 dtype=str, 

173 doc="Gen2 only: path to locate the data photodiode data files.", 

174 default="" 

175 ) 

176 instrumentName = pexConfig.Field( 

177 dtype=str, 

178 doc="Instrument name.", 

179 default='', 

180 ) 

181 

182 

183class MeasurePhotonTransferCurveTask(pipeBase.CmdLineTask): 

184 """A class to calculate, fit, and plot a PTC from a set of flat pairs. 

185 

186 The Photon Transfer Curve (var(signal) vs mean(signal)) is a standard tool 

187 used in astronomical detectors characterization (e.g., Janesick 2001, 

188 Janesick 2007). If ptcFitType is "EXPAPPROXIMATION" or "POLYNOMIAL", this task calculates the 

189 PTC from a series of pairs of flat-field images; each pair taken at identical exposure 

190 times. The difference image of each pair is formed to eliminate fixed pattern noise, 

191 and then the variance of the difference image and the mean of the average image 

192 are used to produce the PTC. An n-degree polynomial or the approximation in Equation 

193 16 of Astier+19 ("The Shape of the Photon Transfer Curve of CCD sensors", 

194 arXiv:1905.08677) can be fitted to the PTC curve. These models include 

195 parameters such as the gain (e/DN) and readout noise. 

196 

197 Linearizers to correct for signal-chain non-linearity are also calculated. 

198 The `Linearizer` class, in general, can support per-amp linearizers, but in this 

199 task this is not supported. 

200 

201 If ptcFitType is "FULLCOVARIANCE", the covariances of the difference images are calculated via the 

202 DFT methods described in Astier+19 and the variances for the PTC are given by the cov[0,0] elements 

203 at each signal level. The full model in Equation 20 of Astier+19 is fit to the PTC to get the gain 

204 and the noise. 

205 

206 Parameters 

207 ---------- 

208 

209 *args: `list` 

210 Positional arguments passed to the Task constructor. None used at this 

211 time. 

212 **kwargs: `dict` 

213 Keyword arguments passed on to the Task constructor. None used at this 

214 time. 

215 

216 """ 

217 

218 RunnerClass = DataRefListRunner 

219 ConfigClass = MeasurePhotonTransferCurveTaskConfig 

220 _DefaultName = "measurePhotonTransferCurve" 

221 

222 def __init__(self, *args, **kwargs): 

223 pipeBase.CmdLineTask.__init__(self, *args, **kwargs) 

224 self.makeSubtask("linearity") 

225 plt.interactive(False) # stop windows popping up when plotting. When headless, use 'agg' backend too 

226 self.config.validate() 

227 self.config.freeze() 

228 

229 @pipeBase.timeMethod 

230 def runDataRef(self, dataRefList): 

231 """Run the Photon Transfer Curve (PTC) measurement task. 

232 

233 For a dataRef (which is each detector here), 

234 and given a list of exposure pairs (postISR) at different exposure times, 

235 measure the PTC. 

236 

237 Parameters 

238 ---------- 

239 dataRefList : `list` [`lsst.daf.peristence.ButlerDataRef`] 

240 Data references for exposures for detectors to process. 

241 """ 

242 if len(dataRefList) < 2: 

243 raise RuntimeError("Insufficient inputs to combine.") 

244 

245 # setup necessary objects 

246 dataRef = dataRefList[0] 

247 

248 detNum = dataRef.dataId[self.config.ccdKey] 

249 camera = dataRef.get('camera') 

250 detector = camera[dataRef.dataId[self.config.ccdKey]] 

251 

252 amps = detector.getAmplifiers() 

253 ampNames = [amp.getName() for amp in amps] 

254 datasetPtc = PhotonTransferCurveDataset(ampNames, self.config.ptcFitType) 

255 

256 # Get the pairs of flat indexed by expTime 

257 expPairs = self.makePairs(dataRefList) 

258 expIds = [] 

259 for (exp1, exp2) in expPairs.values(): 

260 id1 = exp1.getInfo().getVisitInfo().getExposureId() 

261 id2 = exp2.getInfo().getVisitInfo().getExposureId() 

262 expIds.append((id1, id2)) 

263 self.log.info(f"Measuring PTC using {expIds} exposures for detector {detector.getId()}") 

264 

265 # get photodiode data early so that logic can be put in to only use the 

266 # data if all files are found, as partial corrections are not possible 

267 # or at least require significant logic to deal with 

268 if self.config.doPhotodiode: 

269 for (expId1, expId2) in expIds: 

270 charges = [-1, -1] # necessary to have a not-found value to keep lists in step 

271 for i, expId in enumerate([expId1, expId2]): 

272 # //1000 is a Gen2 only hack, working around the fact an 

273 # exposure's ID is not the same as the expId in the 

274 # registry. Currently expId is concatenated with the 

275 # zero-padded detector ID. This will all go away in Gen3. 

276 dataRef.dataId['expId'] = expId//1000 

277 if self.config.photodiodeDataPath: 

278 photodiodeData = getBOTphotodiodeData(dataRef, self.config.photodiodeDataPath) 

279 else: 

280 photodiodeData = getBOTphotodiodeData(dataRef) 

281 if photodiodeData: # default path stored in function def to keep task clean 

282 charges[i] = photodiodeData.getCharge() 

283 else: 

284 # full expId (not //1000) here, as that encodes the 

285 # the detector number as so is fully qualifying 

286 self.log.warn(f"No photodiode data found for {expId}") 

287 

288 for ampName in ampNames: 

289 datasetPtc.photoCharge[ampName].append((charges[0], charges[1])) 

290 else: 

291 # Can't be an empty list, as initialized, because astropy.Table won't allow it 

292 # when saving as fits 

293 for ampName in ampNames: 

294 datasetPtc.photoCharge[ampName] = np.repeat(np.nan, len(expIds)) 

295 

296 for ampName in ampNames: 

297 datasetPtc.inputExpIdPairs[ampName] = expIds 

298 

299 maxMeanSignalDict = {ampName: 1e6 for ampName in ampNames} 

300 minMeanSignalDict = {ampName: 0.0 for ampName in ampNames} 

301 for ampName in ampNames: 

302 if 'ALL_AMPS' in self.config.maxMeanSignal: 

303 maxMeanSignalDict[ampName] = self.config.maxMeanSignal['ALL_AMPS'] 

304 elif ampName in self.config.maxMeanSignal: 

305 maxMeanSignalDict[ampName] = self.config.maxMeanSignal[ampName] 

306 

307 if 'ALL_AMPS' in self.config.minMeanSignal: 

308 minMeanSignalDict[ampName] = self.config.minMeanSignal['ALL_AMPS'] 

309 elif ampName in self.config.minMeanSignal: 

310 minMeanSignalDict[ampName] = self.config.minMeanSignal[ampName] 

311 

312 tupleRecords = [] 

313 allTags = [] 

314 for expTime, (exp1, exp2) in expPairs.items(): 

315 expId1 = exp1.getInfo().getVisitInfo().getExposureId() 

316 expId2 = exp2.getInfo().getVisitInfo().getExposureId() 

317 tupleRows = [] 

318 nAmpsNan = 0 

319 tags = ['mu', 'i', 'j', 'var', 'cov', 'npix', 'ext', 'expTime', 'ampName'] 

320 for ampNumber, amp in enumerate(detector): 

321 ampName = amp.getName() 

322 # covAstier: (i, j, var (cov[0,0]), cov, npix) 

323 doRealSpace = self.config.covAstierRealSpace 

324 muDiff, varDiff, covAstier = self.measureMeanVarCov(exp1, exp2, region=amp.getBBox(), 

325 covAstierRealSpace=doRealSpace) 

326 

327 if np.isnan(muDiff) or np.isnan(varDiff) or (covAstier is None): 

328 msg = (f"NaN mean or var, or None cov in amp {ampName} in exposure pair {expId1}," 

329 f" {expId2} of detector {detNum}.") 

330 self.log.warn(msg) 

331 nAmpsNan += 1 

332 continue 

333 if (muDiff <= minMeanSignalDict[ampName]) or (muDiff >= maxMeanSignalDict[ampName]): 

334 continue 

335 

336 datasetPtc.rawExpTimes[ampName].append(expTime) 

337 datasetPtc.rawMeans[ampName].append(muDiff) 

338 datasetPtc.rawVars[ampName].append(varDiff) 

339 

340 tupleRows += [(muDiff, ) + covRow + (ampNumber, expTime, ampName) for covRow in covAstier] 

341 if nAmpsNan == len(ampNames): 

342 msg = f"NaN mean in all amps of exposure pair {expId1}, {expId2} of detector {detNum}." 

343 self.log.warn(msg) 

344 continue 

345 allTags += tags 

346 tupleRecords += tupleRows 

347 covariancesWithTags = np.core.records.fromrecords(tupleRecords, names=allTags) 

348 

349 for ampName in datasetPtc.ampNames: 

350 # Sort raw vectors by rawMeans index 

351 index = np.argsort(datasetPtc.rawMeans[ampName]) 

352 datasetPtc.rawExpTimes[ampName] = np.array(datasetPtc.rawExpTimes[ampName])[index] 

353 datasetPtc.rawMeans[ampName] = np.array(datasetPtc.rawMeans[ampName])[index] 

354 datasetPtc.rawVars[ampName] = np.array(datasetPtc.rawVars[ampName])[index] 

355 

356 if self.config.ptcFitType in ["FULLCOVARIANCE", ]: 

357 # Calculate covariances and fit them, including the PTC, to Astier+19 full model (Eq. 20) 

358 # First, fit get the flat pairs that are masked, according to the regular PTC (C_00 vs mu) 

359 # The points at these fluxes will also be masked when calculating the other covariances, C_ij) 

360 newDatasetPtc = copy.copy(datasetPtc) 

361 newDatasetPtc = self.fitPtc(newDatasetPtc, 'EXPAPPROXIMATION') 

362 for ampName in datasetPtc.ampNames: 

363 datasetPtc.expIdMask[ampName] = newDatasetPtc.expIdMask[ampName] 

364 

365 datasetPtc.fitType = "FULLCOVARIANCE" 

366 datasetPtc = self.fitCovariancesAstier(datasetPtc, covariancesWithTags) 

367 elif self.config.ptcFitType in ["EXPAPPROXIMATION", "POLYNOMIAL"]: 

368 # Fit the PTC to a polynomial or to Astier+19 exponential approximation (Eq. 16) 

369 # Fill up PhotonTransferCurveDataset object. 

370 datasetPtc = self.fitPtc(datasetPtc, self.config.ptcFitType) 

371 

372 detName = detector.getName() 

373 now = datetime.datetime.utcnow() 

374 calibDate = now.strftime("%Y-%m-%d") 

375 butler = dataRef.getButler() 

376 

377 datasetPtc.updateMetadata(setDate=True, camera=camera, detector=detector) 

378 

379 # Fit a poynomial to calculate non-linearity and persist linearizer. 

380 if self.config.doCreateLinearizer: 

381 # Fit (non)linearity of signal vs time curve. 

382 # Fill up PhotonTransferCurveDataset object. 

383 # Fill up array for LUT linearizer (tableArray). 

384 # Produce coefficients for Polynomial and Squared linearizers. 

385 # Build linearizer objects. 

386 dimensions = {'camera': camera.getName(), 'detector': detector.getId()} 

387 linearityResults = self.linearity.run(datasetPtc, camera, dimensions) 

388 linearizer = linearityResults.outputLinearizer 

389 

390 self.log.info("Writing linearizer:") 

391 

392 detName = detector.getName() 

393 now = datetime.datetime.utcnow() 

394 calibDate = now.strftime("%Y-%m-%d") 

395 

396 butler.put(linearizer, datasetType='linearizer', 

397 dataId={'detector': detNum, 'detectorName': detName, 'calibDate': calibDate}) 

398 

399 self.log.info(f"Writing PTC data.") 

400 butler.put(datasetPtc, datasetType='photonTransferCurveDataset', dataId={'detector': detNum, 

401 'detectorName': detName, 'calibDate': calibDate}) 

402 

403 return pipeBase.Struct(exitStatus=0) 

404 

405 def makePairs(self, dataRefList): 

406 """Produce a list of flat pairs indexed by exposure time. 

407 

408 Parameters 

409 ---------- 

410 dataRefList : `list` [`lsst.daf.peristence.ButlerDataRef`] 

411 Data references for exposures for detectors to process. 

412 

413 Return 

414 ------ 

415 flatPairs : `dict` [`float`, `lsst.afw.image.exposure.exposure.ExposureF`] 

416 Dictionary that groups flat-field exposures that have the same exposure time (seconds). 

417 

418 Notes 

419 ----- 

420 We use the difference of one pair of flat-field images taken at the same exposure time when 

421 calculating the PTC to reduce Fixed Pattern Noise. If there are > 2 flat-field images with the 

422 same exposure time, the first two are kept and the rest discarded. 

423 """ 

424 

425 # Organize exposures by observation date. 

426 expDict = {} 

427 for dataRef in dataRefList: 

428 try: 

429 tempFlat = dataRef.get("postISRCCD") 

430 except RuntimeError: 

431 self.log.warn("postISR exposure could not be retrieved. Ignoring flat.") 

432 continue 

433 expDate = tempFlat.getInfo().getVisitInfo().getDate().get() 

434 expDict.setdefault(expDate, tempFlat) 

435 sortedExps = {k: expDict[k] for k in sorted(expDict)} 

436 

437 flatPairs = {} 

438 for exp in sortedExps: 

439 tempFlat = sortedExps[exp] 

440 expTime = tempFlat.getInfo().getVisitInfo().getExposureTime() 

441 listAtExpTime = flatPairs.setdefault(expTime, []) 

442 if len(listAtExpTime) >= 2: 

443 self.log.warn(f"Already found 2 exposures at expTime {expTime}. " 

444 f"Ignoring exposure {tempFlat.getInfo().getVisitInfo().getExposureId()}") 

445 else: 

446 listAtExpTime.append(tempFlat) 

447 

448 keysToDrop = [] 

449 for (key, value) in flatPairs.items(): 

450 if len(value) < 2: 

451 keysToDrop.append(key) 

452 

453 if len(keysToDrop): 

454 for key in keysToDrop: 

455 self.log.warn(f"Only one exposure found at expTime {key}. Dropping exposure " 

456 f"{flatPairs[key][0].getInfo().getVisitInfo().getExposureId()}.") 

457 flatPairs.pop(key) 

458 sortedFlatPairs = {k: flatPairs[k] for k in sorted(flatPairs)} 

459 return sortedFlatPairs 

460 

461 def fitCovariancesAstier(self, dataset, covariancesWithTagsArray): 

462 """Fit measured flat covariances to full model in Astier+19. 

463 

464 Parameters 

465 ---------- 

466 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset` 

467 The dataset containing information such as the means, variances and exposure times. 

468 

469 covariancesWithTagsArray : `numpy.recarray` 

470 Tuple with at least (mu, cov, var, i, j, npix), where: 

471 mu : 0.5*(m1 + m2), where: 

472 mu1: mean value of flat1 

473 mu2: mean value of flat2 

474 cov: covariance value at lag(i, j) 

475 var: variance(covariance value at lag(0, 0)) 

476 i: lag dimension 

477 j: lag dimension 

478 npix: number of pixels used for covariance calculation. 

479 

480 Returns 

481 ------- 

482 dataset: `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset` 

483 This is the same dataset as the input paramter, however, it has been modified 

484 to include information such as the fit vectors and the fit parameters. See 

485 the class `PhotonTransferCurveDatase`. 

486 """ 

487 

488 covFits, covFitsNoB = fitData(covariancesWithTagsArray, 

489 r=self.config.maximumRangeCovariancesAstier, 

490 expIdMask=dataset.expIdMask) 

491 dataset = self.getOutputPtcDataCovAstier(dataset, covFits, covFitsNoB) 

492 return dataset 

493 

494 def getOutputPtcDataCovAstier(self, dataset, covFits, covFitsNoB): 

495 """Get output data for PhotonTransferCurveCovAstierDataset from CovFit objects. 

496 

497 Parameters 

498 ---------- 

499 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset` 

500 The dataset containing information such as the means, variances and exposure times. 

501 

502 covFits: `dict` 

503 Dictionary of CovFit objects, with amp names as keys. 

504 

505 covFitsNoB : `dict` 

506 Dictionary of CovFit objects, with amp names as keys, and 'b=0' in Eq. 20 of Astier+19. 

507 

508 Returns 

509 ------- 

510 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset` 

511 This is the same dataset as the input paramter, however, it has been modified 

512 to include extra information such as the mask 1D array, gains, reoudout noise, measured signal, 

513 measured variance, modeled variance, a, and b coefficient matrices (see Astier+19) per amplifier. 

514 See the class `PhotonTransferCurveDatase`. 

515 """ 

516 assert(len(covFits) == len(covFitsNoB)) 

517 

518 for i, amp in enumerate(dataset.ampNames): 

519 lenInputTimes = len(dataset.rawExpTimes[amp]) 

520 # Not used when ptcFitType is 'FULLCOVARIANCE' 

521 dataset.ptcFitPars[amp] = np.nan 

522 dataset.ptcFitParsError[amp] = np.nan 

523 dataset.ptcFitChiSq[amp] = np.nan 

524 if (amp in covFits and (covFits[amp].covParams is not None) and 

525 (covFitsNoB[amp].covParams is not None)): 

526 fit = covFits[amp] 

527 fitNoB = covFitsNoB[amp] 

528 # Save full covariances, covariances models, and their weights 

529 dataset.covariances[amp] = fit.cov 

530 dataset.covariancesModel[amp] = fit.evalCovModel() 

531 dataset.covariancesSqrtWeights[amp] = fit.sqrtW 

532 dataset.aMatrix[amp] = fit.getA() 

533 dataset.bMatrix[amp] = fit.getB() 

534 dataset.covariancesNoB[amp] = fitNoB.cov 

535 dataset.covariancesModelNoB[amp] = fitNoB.evalCovModel() 

536 dataset.covariancesSqrtWeightsNoB[amp] = fitNoB.sqrtW 

537 dataset.aMatrixNoB[amp] = fitNoB.getA() 

538 

539 (meanVecFinal, varVecFinal, varVecModel, 

540 wc, varMask) = fit.getFitData(0, 0, divideByMu=False) 

541 gain = fit.getGain() 

542 

543 dataset.gain[amp] = gain 

544 dataset.gainErr[amp] = fit.getGainErr() 

545 dataset.noise[amp] = np.sqrt(fit.getRon()) 

546 dataset.noiseErr[amp] = fit.getRonErr() 

547 

548 padLength = lenInputTimes - len(varVecFinal) 

549 dataset.finalVars[amp] = np.pad(varVecFinal/(gain**2), (0, padLength), 'constant', 

550 constant_values=np.nan) 

551 dataset.finalModelVars[amp] = np.pad(varVecModel/(gain**2), (0, padLength), 'constant', 

552 constant_values=np.nan) 

553 dataset.finalMeans[amp] = np.pad(meanVecFinal/gain, (0, padLength), 'constant', 

554 constant_values=np.nan) 

555 else: 

556 # Bad amp 

557 # Entries need to have proper dimensions so read/write with astropy.Table works. 

558 matrixSide = self.config.maximumRangeCovariancesAstier 

559 nanMatrix = np.full((matrixSide, matrixSide), np.nan) 

560 listNanMatrix = np.full((lenInputTimes, matrixSide, matrixSide), np.nan) 

561 

562 dataset.covariances[amp] = listNanMatrix 

563 dataset.covariancesModel[amp] = listNanMatrix 

564 dataset.covariancesSqrtWeights[amp] = listNanMatrix 

565 dataset.aMatrix[amp] = nanMatrix 

566 dataset.bMatrix[amp] = nanMatrix 

567 dataset.covariancesNoB[amp] = listNanMatrix 

568 dataset.covariancesModelNoB[amp] = listNanMatrix 

569 dataset.covariancesSqrtWeightsNoB[amp] = listNanMatrix 

570 dataset.aMatrixNoB[amp] = nanMatrix 

571 

572 dataset.expIdMask[amp] = np.repeat(np.nan, lenInputTimes) 

573 dataset.gain[amp] = np.nan 

574 dataset.gainErr[amp] = np.nan 

575 dataset.noise[amp] = np.nan 

576 dataset.noiseErr[amp] = np.nan 

577 dataset.finalVars[amp] = np.repeat(np.nan, lenInputTimes) 

578 dataset.finalModelVars[amp] = np.repeat(np.nan, lenInputTimes) 

579 dataset.finalMeans[amp] = np.repeat(np.nan, lenInputTimes) 

580 

581 return dataset 

582 

583 def measureMeanVarCov(self, exposure1, exposure2, region=None, covAstierRealSpace=False): 

584 """Calculate the mean of each of two exposures and the variance and covariance of their difference. 

585 

586 The variance is calculated via afwMath, and the covariance via the methods in Astier+19 (appendix A). 

587 In theory, var = covariance[0,0]. This should be validated, and in the future, we may decide to just 

588 keep one (covariance). 

589 

590 Parameters 

591 ---------- 

592 exposure1 : `lsst.afw.image.exposure.exposure.ExposureF` 

593 First exposure of flat field pair. 

594 

595 exposure2 : `lsst.afw.image.exposure.exposure.ExposureF` 

596 Second exposure of flat field pair. 

597 

598 region : `lsst.geom.Box2I`, optional 

599 Region of each exposure where to perform the calculations (e.g, an amplifier). 

600 

601 covAstierRealSpace : `bool`, optional 

602 Should the covariannces in Astier+19 be calculated in real space or via FFT? 

603 See Appendix A of Astier+19. 

604 

605 Returns 

606 ------- 

607 mu : `float` or `NaN` 

608 0.5*(mu1 + mu2), where mu1, and mu2 are the clipped means of the regions in 

609 both exposures. If either mu1 or m2 are NaN's, the returned value is NaN. 

610 

611 varDiff : `float` or `NaN` 

612 Half of the clipped variance of the difference of the regions inthe two input 

613 exposures. If either mu1 or m2 are NaN's, the returned value is NaN. 

614 

615 covDiffAstier : `list` or `None` 

616 List with tuples of the form (dx, dy, var, cov, npix), where: 

617 dx : `int` 

618 Lag in x 

619 dy : `int` 

620 Lag in y 

621 var : `float` 

622 Variance at (dx, dy). 

623 cov : `float` 

624 Covariance at (dx, dy). 

625 nPix : `int` 

626 Number of pixel pairs used to evaluate var and cov. 

627 If either mu1 or m2 are NaN's, the returned value is None. 

628 """ 

629 

630 if region is not None: 

631 im1Area = exposure1.maskedImage[region] 

632 im2Area = exposure2.maskedImage[region] 

633 else: 

634 im1Area = exposure1.maskedImage 

635 im2Area = exposure2.maskedImage 

636 

637 if self.config.binSize > 1: 

638 im1Area = afwMath.binImage(im1Area, self.config.binSize) 

639 im2Area = afwMath.binImage(im2Area, self.config.binSize) 

640 

641 im1MaskVal = exposure1.getMask().getPlaneBitMask(self.config.maskNameList) 

642 im1StatsCtrl = afwMath.StatisticsControl(self.config.nSigmaClipPtc, 

643 self.config.nIterSigmaClipPtc, 

644 im1MaskVal) 

645 im1StatsCtrl.setNanSafe(True) 

646 im1StatsCtrl.setAndMask(im1MaskVal) 

647 

648 im2MaskVal = exposure2.getMask().getPlaneBitMask(self.config.maskNameList) 

649 im2StatsCtrl = afwMath.StatisticsControl(self.config.nSigmaClipPtc, 

650 self.config.nIterSigmaClipPtc, 

651 im2MaskVal) 

652 im2StatsCtrl.setNanSafe(True) 

653 im2StatsCtrl.setAndMask(im2MaskVal) 

654 

655 # Clipped mean of images; then average of mean. 

656 mu1 = afwMath.makeStatistics(im1Area, afwMath.MEANCLIP, im1StatsCtrl).getValue() 

657 mu2 = afwMath.makeStatistics(im2Area, afwMath.MEANCLIP, im2StatsCtrl).getValue() 

658 if np.isnan(mu1) or np.isnan(mu2): 

659 self.log.warn(f"Mean of amp in image 1 or 2 is NaN: {mu1}, {mu2}.") 

660 return np.nan, np.nan, None 

661 mu = 0.5*(mu1 + mu2) 

662 

663 # Take difference of pairs 

664 # symmetric formula: diff = (mu2*im1-mu1*im2)/(0.5*(mu1+mu2)) 

665 temp = im2Area.clone() 

666 temp *= mu1 

667 diffIm = im1Area.clone() 

668 diffIm *= mu2 

669 diffIm -= temp 

670 diffIm /= mu 

671 

672 diffImMaskVal = diffIm.getMask().getPlaneBitMask(self.config.maskNameList) 

673 diffImStatsCtrl = afwMath.StatisticsControl(self.config.nSigmaClipPtc, 

674 self.config.nIterSigmaClipPtc, 

675 diffImMaskVal) 

676 diffImStatsCtrl.setNanSafe(True) 

677 diffImStatsCtrl.setAndMask(diffImMaskVal) 

678 

679 varDiff = 0.5*(afwMath.makeStatistics(diffIm, afwMath.VARIANCECLIP, diffImStatsCtrl).getValue()) 

680 

681 # Get the mask and identify good pixels as '1', and the rest as '0'. 

682 w1 = np.where(im1Area.getMask().getArray() == 0, 1, 0) 

683 w2 = np.where(im2Area.getMask().getArray() == 0, 1, 0) 

684 

685 w12 = w1*w2 

686 wDiff = np.where(diffIm.getMask().getArray() == 0, 1, 0) 

687 w = w12*wDiff 

688 

689 if np.sum(w) < self.config.minNumberGoodPixelsForFft: 

690 self.log.warn(f"Number of good points for FFT ({np.sum(w)}) is less than threshold " 

691 f"({self.config.minNumberGoodPixelsForFft})") 

692 return np.nan, np.nan, None 

693 

694 maxRangeCov = self.config.maximumRangeCovariancesAstier 

695 if covAstierRealSpace: 

696 covDiffAstier = computeCovDirect(diffIm.getImage().getArray(), w, maxRangeCov) 

697 else: 

698 shapeDiff = diffIm.getImage().getArray().shape 

699 fftShape = (fftSize(shapeDiff[0] + maxRangeCov), fftSize(shapeDiff[1]+maxRangeCov)) 

700 c = CovFft(diffIm.getImage().getArray(), w, fftShape, maxRangeCov) 

701 covDiffAstier = c.reportCovFft(maxRangeCov) 

702 

703 return mu, varDiff, covDiffAstier 

704 

705 def computeCovDirect(self, diffImage, weightImage, maxRange): 

706 """Compute covariances of diffImage in real space. 

707 

708 For lags larger than ~25, it is slower than the FFT way. 

709 Taken from https://github.com/PierreAstier/bfptc/ 

710 

711 Parameters 

712 ---------- 

713 diffImage : `numpy.array` 

714 Image to compute the covariance of. 

715 

716 weightImage : `numpy.array` 

717 Weight image of diffImage (1's and 0's for good and bad pixels, respectively). 

718 

719 maxRange : `int` 

720 Last index of the covariance to be computed. 

721 

722 Returns 

723 ------- 

724 outList : `list` 

725 List with tuples of the form (dx, dy, var, cov, npix), where: 

726 dx : `int` 

727 Lag in x 

728 dy : `int` 

729 Lag in y 

730 var : `float` 

731 Variance at (dx, dy). 

732 cov : `float` 

733 Covariance at (dx, dy). 

734 nPix : `int` 

735 Number of pixel pairs used to evaluate var and cov. 

736 """ 

737 outList = [] 

738 var = 0 

739 # (dy,dx) = (0,0) has to be first 

740 for dy in range(maxRange + 1): 

741 for dx in range(0, maxRange + 1): 

742 if (dx*dy > 0): 

743 cov1, nPix1 = self.covDirectValue(diffImage, weightImage, dx, dy) 

744 cov2, nPix2 = self.covDirectValue(diffImage, weightImage, dx, -dy) 

745 cov = 0.5*(cov1 + cov2) 

746 nPix = nPix1 + nPix2 

747 else: 

748 cov, nPix = self.covDirectValue(diffImage, weightImage, dx, dy) 

749 if (dx == 0 and dy == 0): 

750 var = cov 

751 outList.append((dx, dy, var, cov, nPix)) 

752 

753 return outList 

754 

755 def covDirectValue(self, diffImage, weightImage, dx, dy): 

756 """Compute covariances of diffImage in real space at lag (dx, dy). 

757 

758 Taken from https://github.com/PierreAstier/bfptc/ (c.f., appendix of Astier+19). 

759 

760 Parameters 

761 ---------- 

762 diffImage : `numpy.array` 

763 Image to compute the covariance of. 

764 

765 weightImage : `numpy.array` 

766 Weight image of diffImage (1's and 0's for good and bad pixels, respectively). 

767 

768 dx : `int` 

769 Lag in x. 

770 

771 dy : `int` 

772 Lag in y. 

773 

774 Returns 

775 ------- 

776 cov : `float` 

777 Covariance at (dx, dy) 

778 

779 nPix : `int` 

780 Number of pixel pairs used to evaluate var and cov. 

781 """ 

782 (nCols, nRows) = diffImage.shape 

783 # switching both signs does not change anything: 

784 # it just swaps im1 and im2 below 

785 if (dx < 0): 

786 (dx, dy) = (-dx, -dy) 

787 # now, we have dx >0. We have to distinguish two cases 

788 # depending on the sign of dy 

789 if dy >= 0: 

790 im1 = diffImage[dy:, dx:] 

791 w1 = weightImage[dy:, dx:] 

792 im2 = diffImage[:nCols - dy, :nRows - dx] 

793 w2 = weightImage[:nCols - dy, :nRows - dx] 

794 else: 

795 im1 = diffImage[:nCols + dy, dx:] 

796 w1 = weightImage[:nCols + dy, dx:] 

797 im2 = diffImage[-dy:, :nRows - dx] 

798 w2 = weightImage[-dy:, :nRows - dx] 

799 # use the same mask for all 3 calculations 

800 wAll = w1*w2 

801 # do not use mean() because weightImage=0 pixels would then count 

802 nPix = wAll.sum() 

803 im1TimesW = im1*wAll 

804 s1 = im1TimesW.sum()/nPix 

805 s2 = (im2*wAll).sum()/nPix 

806 p = (im1TimesW*im2).sum()/nPix 

807 cov = p - s1*s2 

808 

809 return cov, nPix 

810 

811 @staticmethod 

812 def _initialParsForPolynomial(order): 

813 assert(order >= 2) 

814 pars = np.zeros(order, dtype=np.float) 

815 pars[0] = 10 

816 pars[1] = 1 

817 pars[2:] = 0.0001 

818 return pars 

819 

820 @staticmethod 

821 def _boundsForPolynomial(initialPars, lowers=[], uppers=[]): 

822 if not len(lowers): 

823 lowers = [np.NINF for p in initialPars] 

824 if not len(uppers): 

825 uppers = [np.inf for p in initialPars] 

826 lowers[1] = 0 # no negative gains 

827 return (lowers, uppers) 

828 

829 @staticmethod 

830 def _boundsForAstier(initialPars, lowers=[], uppers=[]): 

831 if not len(lowers): 

832 lowers = [np.NINF for p in initialPars] 

833 if not len(uppers): 

834 uppers = [np.inf for p in initialPars] 

835 return (lowers, uppers) 

836 

837 @staticmethod 

838 def _getInitialGoodPoints(means, variances, maxDeviationPositive, maxDeviationNegative): 

839 """Return a boolean array to mask bad points. 

840 

841 Parameters 

842 ---------- 

843 means : `numpy.array` 

844 Input array with mean signal values. 

845 

846 variances : `numpy.array` 

847 Input array with variances at each mean value. 

848 

849 maxDeviationPositive : `float` 

850 Maximum deviation from being constant for the variance/mean 

851 ratio, in the positive direction. 

852 

853 maxDeviationNegative : `float` 

854 Maximum deviation from being constant for the variance/mean 

855 ratio, in the negative direction. 

856 

857 Return 

858 ------ 

859 goodPoints : `numpy.array` [`bool`] 

860 Boolean array to select good (`True`) and bad (`False`) 

861 points. 

862 

863 Notes 

864 ----- 

865 A linear function has a constant ratio, so find the median 

866 value of the ratios, and exclude the points that deviate 

867 from that by more than a factor of maxDeviationPositive/negative. 

868 Asymmetric deviations are supported as we expect the PTC to turn 

869 down as the flux increases, but sometimes it anomalously turns 

870 upwards just before turning over, which ruins the fits, so it 

871 is wise to be stricter about restricting positive outliers than 

872 negative ones. 

873 

874 Too high and points that are so bad that fit will fail will be included 

875 Too low and the non-linear points will be excluded, biasing the NL fit. 

876 

877 This function also masks points after the variance starts decreasing. 

878 """ 

879 

880 assert(len(means) == len(variances)) 

881 ratios = [b/a for (a, b) in zip(means, variances)] 

882 medianRatio = np.nanmedian(ratios) 

883 ratioDeviations = [(r/medianRatio)-1 for r in ratios] 

884 

885 # so that it doesn't matter if the deviation is expressed as positive or negative 

886 maxDeviationPositive = abs(maxDeviationPositive) 

887 maxDeviationNegative = -1. * abs(maxDeviationNegative) 

888 

889 goodPoints = np.array([True if (r < maxDeviationPositive and r > maxDeviationNegative) 

890 else False for r in ratioDeviations]) 

891 

892 # Discard points when variance starts decreasing 

893 pivot = np.where(np.array(np.diff(variances)) < 0)[0] 

894 if len(pivot) == 0: 

895 return goodPoints 

896 else: 

897 pivot = np.min(pivot) 

898 goodPoints[pivot:len(goodPoints)] = False 

899 return goodPoints 

900 

901 def _makeZeroSafe(self, array, warn=True, substituteValue=1e-9): 

902 """""" 

903 nBad = Counter(array)[0] 

904 if nBad == 0: 

905 return array 

906 

907 if warn: 

908 msg = f"Found {nBad} zeros in array at elements {[x for x in np.where(array==0)[0]]}" 

909 self.log.warn(msg) 

910 

911 array[array == 0] = substituteValue 

912 return array 

913 

914 def fitPtc(self, dataset, ptcFitType): 

915 """Fit the photon transfer curve to a polynimial or to Astier+19 approximation. 

916 

917 Fit the photon transfer curve with either a polynomial of the order 

918 specified in the task config, or using the Astier approximation. 

919 

920 Sigma clipping is performed iteratively for the fit, as well as an 

921 initial clipping of data points that are more than 

922 config.initialNonLinearityExclusionThreshold away from lying on a 

923 straight line. This other step is necessary because the photon transfer 

924 curve turns over catastrophically at very high flux (because saturation 

925 drops the variance to ~0) and these far outliers cause the initial fit 

926 to fail, meaning the sigma cannot be calculated to perform the 

927 sigma-clipping. 

928 

929 Parameters 

930 ---------- 

931 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset` 

932 The dataset containing the means, variances and exposure times 

933 

934 ptcFitType : `str` 

935 Fit a 'POLYNOMIAL' (degree: 'polynomialFitDegree') or 

936 'EXPAPPROXIMATION' (Eq. 16 of Astier+19) to the PTC 

937 

938 Returns 

939 ------- 

940 dataset: `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset` 

941 This is the same dataset as the input paramter, however, it has been modified 

942 to include information such as the fit vectors and the fit parameters. See 

943 the class `PhotonTransferCurveDatase`. 

944 """ 

945 

946 matrixSide = self.config.maximumRangeCovariancesAstier 

947 nanMatrix = np.empty((matrixSide, matrixSide)) 

948 nanMatrix[:] = np.nan 

949 

950 for amp in dataset.ampNames: 

951 lenInputTimes = len(dataset.rawExpTimes[amp]) 

952 listNanMatrix = np.empty((lenInputTimes, matrixSide, matrixSide)) 

953 listNanMatrix[:] = np.nan 

954 

955 dataset.covariances[amp] = listNanMatrix 

956 dataset.covariancesModel[amp] = listNanMatrix 

957 dataset.covariancesSqrtWeights[amp] = listNanMatrix 

958 dataset.aMatrix[amp] = nanMatrix 

959 dataset.bMatrix[amp] = nanMatrix 

960 dataset.covariancesNoB[amp] = listNanMatrix 

961 dataset.covariancesModelNoB[amp] = listNanMatrix 

962 dataset.covariancesSqrtWeightsNoB[amp] = listNanMatrix 

963 dataset.aMatrixNoB[amp] = nanMatrix 

964 

965 def errFunc(p, x, y): 

966 return ptcFunc(p, x) - y 

967 

968 sigmaCutPtcOutliers = self.config.sigmaCutPtcOutliers 

969 maxIterationsPtcOutliers = self.config.maxIterationsPtcOutliers 

970 

971 for i, ampName in enumerate(dataset.ampNames): 

972 timeVecOriginal = np.array(dataset.rawExpTimes[ampName]) 

973 meanVecOriginal = np.array(dataset.rawMeans[ampName]) 

974 varVecOriginal = np.array(dataset.rawVars[ampName]) 

975 varVecOriginal = self._makeZeroSafe(varVecOriginal) 

976 

977 goodPoints = self._getInitialGoodPoints(meanVecOriginal, varVecOriginal, 

978 self.config.initialNonLinearityExclusionThresholdPositive, 

979 self.config.initialNonLinearityExclusionThresholdNegative) 

980 if not (goodPoints.any()): 

981 msg = (f"\nSERIOUS: All points in goodPoints: {goodPoints} are bad." 

982 f"Setting {ampName} to BAD.") 

983 self.log.warn(msg) 

984 # The first and second parameters of initial fit are discarded (bias and gain) 

985 # for the final NL coefficients 

986 dataset.badAmps.append(ampName) 

987 dataset.expIdMask[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName])) 

988 dataset.gain[ampName] = np.nan 

989 dataset.gainErr[ampName] = np.nan 

990 dataset.noise[ampName] = np.nan 

991 dataset.noiseErr[ampName] = np.nan 

992 dataset.ptcFitPars[ampName] = (np.repeat(np.nan, self.config.polynomialFitDegree + 1) if 

993 ptcFitType in ["POLYNOMIAL", ] else np.repeat(np.nan, 3)) 

994 dataset.ptcFitParsError[ampName] = (np.repeat(np.nan, self.config.polynomialFitDegree + 1) if 

995 ptcFitType in ["POLYNOMIAL", ] else np.repeat(np.nan, 3)) 

996 dataset.ptcFitChiSq[ampName] = np.nan 

997 dataset.finalVars[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName])) 

998 dataset.finalModelVars[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName])) 

999 dataset.finalMeans[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName])) 

1000 continue 

1001 

1002 mask = goodPoints 

1003 

1004 if ptcFitType == 'EXPAPPROXIMATION': 

1005 ptcFunc = funcAstier 

1006 parsIniPtc = [-1e-9, 1.0, 10.] # a00, gain, noise 

1007 # lowers and uppers obtained from studies by C. Lage (UC Davis, 11/2020). 

1008 bounds = self._boundsForAstier(parsIniPtc, lowers=[-1e-4, 0.5, -2000], 

1009 uppers=[1e-4, 2.5, 2000]) 

1010 if ptcFitType == 'POLYNOMIAL': 

1011 ptcFunc = funcPolynomial 

1012 parsIniPtc = self._initialParsForPolynomial(self.config.polynomialFitDegree + 1) 

1013 bounds = self._boundsForPolynomial(parsIniPtc) 

1014 

1015 # Before bootstrap fit, do an iterative fit to get rid of outliers 

1016 count = 1 

1017 while count <= maxIterationsPtcOutliers: 

1018 # Note that application of the mask actually shrinks the array 

1019 # to size rather than setting elements to zero (as we want) so 

1020 # always update mask itself and re-apply to the original data 

1021 meanTempVec = meanVecOriginal[mask] 

1022 varTempVec = varVecOriginal[mask] 

1023 res = least_squares(errFunc, parsIniPtc, bounds=bounds, args=(meanTempVec, varTempVec)) 

1024 pars = res.x 

1025 

1026 # change this to the original from the temp because the masks are ANDed 

1027 # meaning once a point is masked it's always masked, and the masks must 

1028 # always be the same length for broadcasting 

1029 sigResids = (varVecOriginal - ptcFunc(pars, meanVecOriginal))/np.sqrt(varVecOriginal) 

1030 newMask = np.array([True if np.abs(r) < sigmaCutPtcOutliers else False for r in sigResids]) 

1031 mask = mask & newMask 

1032 if not (mask.any() and newMask.any()): 

1033 msg = (f"\nSERIOUS: All points in either mask: {mask} or newMask: {newMask} are bad. " 

1034 f"Setting {ampName} to BAD.") 

1035 self.log.warn(msg) 

1036 # The first and second parameters of initial fit are discarded (bias and gain) 

1037 # for the final NL coefficients 

1038 dataset.badAmps.append(ampName) 

1039 dataset.expIdMask[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName])) 

1040 dataset.gain[ampName] = np.nan 

1041 dataset.gainErr[ampName] = np.nan 

1042 dataset.noise[ampName] = np.nan 

1043 dataset.noiseErr[ampName] = np.nan 

1044 dataset.ptcFitPars[ampName] = (np.repeat(np.nan, self.config.polynomialFitDegree + 1) 

1045 if ptcFitType in ["POLYNOMIAL", ] else 

1046 np.repeat(np.nan, 3)) 

1047 dataset.ptcFitParsError[ampName] = (np.repeat(np.nan, self.config.polynomialFitDegree + 1) 

1048 if ptcFitType in ["POLYNOMIAL", ] else 

1049 np.repeat(np.nan, 3)) 

1050 dataset.ptcFitChiSq[ampName] = np.nan 

1051 dataset.finalVars[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName])) 

1052 dataset.finalModelVars[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName])) 

1053 dataset.finalMeans[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName])) 

1054 break 

1055 nDroppedTotal = Counter(mask)[False] 

1056 self.log.debug(f"Iteration {count}: discarded {nDroppedTotal} points in total for {ampName}") 

1057 count += 1 

1058 # objects should never shrink 

1059 assert (len(mask) == len(timeVecOriginal) == len(meanVecOriginal) == len(varVecOriginal)) 

1060 

1061 if not (mask.any() and newMask.any()): 

1062 continue 

1063 dataset.expIdMask[ampName] = mask # store the final mask 

1064 parsIniPtc = pars 

1065 meanVecFinal = meanVecOriginal[mask] 

1066 varVecFinal = varVecOriginal[mask] 

1067 

1068 if Counter(mask)[False] > 0: 

1069 self.log.info((f"Number of points discarded in PTC of amplifier {ampName}:" + 

1070 f" {Counter(mask)[False]} out of {len(meanVecOriginal)}")) 

1071 

1072 if (len(meanVecFinal) < len(parsIniPtc)): 

1073 msg = (f"\nSERIOUS: Not enough data points ({len(meanVecFinal)}) compared to the number of" 

1074 f"parameters of the PTC model({len(parsIniPtc)}). Setting {ampName} to BAD.") 

1075 self.log.warn(msg) 

1076 # The first and second parameters of initial fit are discarded (bias and gain) 

1077 # for the final NL coefficients 

1078 dataset.badAmps.append(ampName) 

1079 dataset.expIdMask[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName])) 

1080 dataset.gain[ampName] = np.nan 

1081 dataset.gainErr[ampName] = np.nan 

1082 dataset.noise[ampName] = np.nan 

1083 dataset.noiseErr[ampName] = np.nan 

1084 dataset.ptcFitPars[ampName] = (np.repeat(np.nan, self.config.polynomialFitDegree + 1) if 

1085 ptcFitType in ["POLYNOMIAL", ] else np.repeat(np.nan, 3)) 

1086 dataset.ptcFitParsError[ampName] = (np.repeat(np.nan, self.config.polynomialFitDegree + 1) if 

1087 ptcFitType in ["POLYNOMIAL", ] else np.repeat(np.nan, 3)) 

1088 dataset.ptcFitChiSq[ampName] = np.nan 

1089 dataset.finalVars[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName])) 

1090 dataset.finalModelVars[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName])) 

1091 dataset.finalMeans[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName])) 

1092 continue 

1093 

1094 # Fit the PTC 

1095 if self.config.doFitBootstrap: 

1096 parsFit, parsFitErr, reducedChiSqPtc = fitBootstrap(parsIniPtc, meanVecFinal, 

1097 varVecFinal, ptcFunc, 

1098 weightsY=1./np.sqrt(varVecFinal)) 

1099 else: 

1100 parsFit, parsFitErr, reducedChiSqPtc = fitLeastSq(parsIniPtc, meanVecFinal, 

1101 varVecFinal, ptcFunc, 

1102 weightsY=1./np.sqrt(varVecFinal)) 

1103 dataset.ptcFitPars[ampName] = parsFit 

1104 dataset.ptcFitParsError[ampName] = parsFitErr 

1105 dataset.ptcFitChiSq[ampName] = reducedChiSqPtc 

1106 # Masked variances (measured and modeled) and means. Need to pad the array so astropy.Table does 

1107 # not crash (the mask may vary per amp). 

1108 padLength = len(dataset.rawExpTimes[ampName]) - len(varVecFinal) 

1109 dataset.finalVars[ampName] = np.pad(varVecFinal, (0, padLength), 'constant', 

1110 constant_values=np.nan) 

1111 dataset.finalModelVars[ampName] = np.pad(ptcFunc(parsFit, meanVecFinal), (0, padLength), 

1112 'constant', constant_values=np.nan) 

1113 dataset.finalMeans[ampName] = np.pad(meanVecFinal, (0, padLength), 'constant', 

1114 constant_values=np.nan) 

1115 

1116 if ptcFitType == 'EXPAPPROXIMATION': 

1117 ptcGain = parsFit[1] 

1118 ptcGainErr = parsFitErr[1] 

1119 ptcNoise = np.sqrt(np.fabs(parsFit[2])) 

1120 ptcNoiseErr = 0.5*(parsFitErr[2]/np.fabs(parsFit[2]))*np.sqrt(np.fabs(parsFit[2])) 

1121 if ptcFitType == 'POLYNOMIAL': 

1122 ptcGain = 1./parsFit[1] 

1123 ptcGainErr = np.fabs(1./parsFit[1])*(parsFitErr[1]/parsFit[1]) 

1124 ptcNoise = np.sqrt(np.fabs(parsFit[0]))*ptcGain 

1125 ptcNoiseErr = (0.5*(parsFitErr[0]/np.fabs(parsFit[0]))*(np.sqrt(np.fabs(parsFit[0]))))*ptcGain 

1126 dataset.gain[ampName] = ptcGain 

1127 dataset.gainErr[ampName] = ptcGainErr 

1128 dataset.noise[ampName] = ptcNoise 

1129 dataset.noiseErr[ampName] = ptcNoiseErr 

1130 if not len(dataset.ptcFitType) == 0: 

1131 dataset.ptcFitType = ptcFitType 

1132 if len(dataset.badAmps) == 0: 

1133 dataset.badAmps = np.repeat(np.nan, len(list(dataset.rawExpTimes.values())[0])) 

1134 

1135 return dataset