Coverage for python/lsst/cp/pipe/ptc/cpSolvePtcTask.py: 10%

400 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2023-02-15 03:29 -0800

1# This file is part of cp_pipe. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21# 

22import numpy as np 

23from collections import Counter 

24 

25import lsst.pex.config as pexConfig 

26import lsst.pipe.base as pipeBase 

27from lsst.cp.pipe.utils import (fitLeastSq, fitBootstrap, funcPolynomial, funcAstier, symmetrize) 

28 

29from scipy.signal import fftconvolve 

30from scipy.optimize import least_squares 

31from itertools import groupby 

32from operator import itemgetter 

33 

34import lsst.pipe.base.connectionTypes as cT 

35 

36from lsst.ip.isr import PhotonTransferCurveDataset 

37 

38from lsst.cp.pipe._lookupStaticCalibration import lookupStaticCalibration 

39 

40import copy 

41 

42 

43__all__ = ['PhotonTransferCurveSolveConfig', 'PhotonTransferCurveSolveTask'] 

44 

45 

46class PhotonTransferCurveSolveConnections(pipeBase.PipelineTaskConnections, 

47 dimensions=("instrument", "detector")): 

48 inputCovariances = cT.Input( 

49 name="ptcCovariances", 

50 doc="Tuple with measured covariances from flats.", 

51 storageClass="PhotonTransferCurveDataset", 

52 dimensions=("instrument", "exposure", "detector"), 

53 isCalibration=True, 

54 multiple=True, 

55 ) 

56 camera = cT.PrerequisiteInput( 

57 name="camera", 

58 doc="Camera the input data comes from.", 

59 storageClass="Camera", 

60 dimensions=("instrument",), 

61 isCalibration=True, 

62 lookupFunction=lookupStaticCalibration, 

63 ) 

64 outputPtcDataset = cT.Output( 

65 name="ptcDatsetProposal", 

66 doc="Output proposed ptc dataset.", 

67 storageClass="PhotonTransferCurveDataset", 

68 dimensions=("instrument", "detector"), 

69 multiple=False, 

70 isCalibration=True, 

71 ) 

72 

73 

74class PhotonTransferCurveSolveConfig(pipeBase.PipelineTaskConfig, 

75 pipelineConnections=PhotonTransferCurveSolveConnections): 

76 """Configuration for fitting measured covariances. 

77 """ 

78 

79 ptcFitType = pexConfig.ChoiceField( 

80 dtype=str, 

81 doc="Fit PTC to Eq. 16, Eq. 20 in Astier+19, or to a polynomial.", 

82 default="POLYNOMIAL", 

83 allowed={ 

84 "POLYNOMIAL": "n-degree polynomial (use 'polynomialFitDegree' to set 'n').", 

85 "EXPAPPROXIMATION": "Approximation in Astier+19 (Eq. 16).", 

86 "FULLCOVARIANCE": "Full covariances model in Astier+19 (Eq. 20)" 

87 } 

88 ) 

89 maximumRangeCovariancesAstier = pexConfig.Field( 

90 dtype=int, 

91 doc="Maximum range of covariances as in Astier+19", 

92 default=8, 

93 ) 

94 sigmaClipFullFitCovariancesAstier = pexConfig.Field( 

95 dtype=float, 

96 doc="sigma clip for full model fit for FULLCOVARIANCE ptcFitType ", 

97 default=5.0, 

98 ) 

99 maxIterFullFitCovariancesAstier = pexConfig.Field( 

100 dtype=int, 

101 doc="Maximum number of iterations in full model fit for FULLCOVARIANCE ptcFitType", 

102 default=3, 

103 ) 

104 polynomialFitDegree = pexConfig.Field( 

105 dtype=int, 

106 doc="Degree of polynomial to fit the PTC, when 'ptcFitType'=POLYNOMIAL.", 

107 default=3, 

108 ) 

109 sigmaCutPtcOutliers = pexConfig.Field( 

110 dtype=float, 

111 doc="Sigma cut for outlier rejection in PTC.", 

112 default=5.0, 

113 ) 

114 maxIterationsPtcOutliers = pexConfig.RangeField( 

115 dtype=int, 

116 doc="Maximum number of iterations for outlier rejection in PTC.", 

117 default=2, 

118 min=0 

119 ) 

120 minVarPivotSearch = pexConfig.Field( 

121 dtype=float, 

122 doc="The code looks for a pivot signal point after which the variance starts decreasing at high-flux" 

123 " to exclude then from the PTC model fit. However, sometimes at low fluxes, the variance" 

124 " decreases slightly. Set this variable for the variance value, in ADU^2, after which the pivot " 

125 " should be sought.", 

126 default=10000, 

127 ) 

128 consecutivePointsVarDecreases = pexConfig.RangeField( 

129 dtype=int, 

130 doc="Required number of consecutive points/fluxes in the PTC where the variance " 

131 "decreases in order to find a first estimate of the PTC turn-off. ", 

132 default=2, 

133 min=2 

134 ) 

135 doFitBootstrap = pexConfig.Field( 

136 dtype=bool, 

137 doc="Use bootstrap for the PTC fit parameters and errors?.", 

138 default=False, 

139 ) 

140 binSize = pexConfig.Field( 

141 dtype=int, 

142 doc="Bin the image by this factor in both dimensions.", 

143 default=1, 

144 ) 

145 

146 

147class PhotonTransferCurveSolveTask(pipeBase.PipelineTask): 

148 """Task to fit the PTC from flat covariances. 

149 

150 The first task of the PTC measurement pipeline, 

151 ``PhotonTransferCurveMeasureTask`` (and assumed to have been run 

152 before this task), produced a list of 

153 `~lsst.ip.isr.PhotonTransferCurveDataset` objects. Each dataset 

154 contains the mean signal and covariances of the 

155 difference image of the flat-field images taken at 

156 the same exposure time. The list also contains dummy 

157 datasets (with no measurements), whose purpose is to have 

158 the input and output dimensions of ``PhotonTransferCurveMeasureTask`` 

159 match. 

160 

161 This task, ``PhotonTransferCurveSolveTask``, assembles the list 

162 of individual PTC datasets produced 

163 by ``PhotonTransferCurveMeasureTask`` into one single final PTC 

164 dataset, discarding the dummy datset as appropiate. 

165 The task fits the measured (co)variances to one of three models: 

166 a polynomial model of a given order, or the models described 

167 in equations 16 and 20 of Astier+19. These options are referred 

168 to as ``POLYNOMIAL``, ``EXPAPPROXIMATION``, and ``FULLCOVARIANCE`` 

169 in the configuration options of the task, respectively). 

170 Parameters of interest such as the gain and noise are derived 

171 from the fits. The ``FULLCOVARIANCE`` model is fitted to the 

172 full covariance data (as oppossed to the other two models, which 

173 are fit to the variance vs mean measurements only). 

174 

175 Astier+19: "The Shape of the Photon Transfer Curve 

176 of CCD sensors", arXiv:1905.08677 

177 """ 

178 

179 ConfigClass = PhotonTransferCurveSolveConfig 

180 _DefaultName = 'cpPhotonTransferCurveSolve' 

181 

182 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

183 """Ensure that the input and output dimensions are passed along. 

184 

185 Parameters 

186 ---------- 

187 butlerQC : `~lsst.daf.butler.butlerQuantumContext.ButlerQuantumContext` 

188 Butler to operate on. 

189 inputRefs : `~lsst.pipe.base.connections.InputQuantizedConnection` 

190 Input data refs to load. 

191 ouptutRefs : `~lsst.pipe.base.connections.OutputQuantizedConnection` 

192 Output data refs to persist. 

193 """ 

194 inputs = butlerQC.get(inputRefs) 

195 detId = inputRefs.inputCovariances[0].dataId['detector'] 

196 outputs = self.run(inputCovariances=inputs['inputCovariances'], camera=inputs['camera'], detId=detId) 

197 butlerQC.put(outputs, outputRefs) 

198 

199 def run(self, inputCovariances, camera=None, detId=0): 

200 """Fit measured covariances to different models. 

201 

202 Parameters 

203 ---------- 

204 inputCovariances : `list` [`lsst.ip.isr.PhotonTransferCurveDataset`] 

205 List of lsst.ip.isr.PhotonTransferCurveDataset datasets. 

206 camera : `lsst.afw.cameraGeom.Camera`, optional 

207 Input camera. 

208 detId : `int` 

209 Detector ID to locate the detector in the camera and 

210 populate the `lsst.ip.isr.PhotonTransferCurveDataset` 

211 metadata. 

212 Returns 

213 ------- 

214 results : `lsst.pipe.base.Struct` 

215 The resultins structure contains: 

216 

217 ``outputPtcDatset`` 

218 Final PTC dataset, containing information such as the 

219 means, variances, and exposure times 

220 (`lsst.ip.isr.PhotonTransferCurveDataset`). 

221 """ 

222 # Assemble individual PTC datasets into a single PTC dataset. 

223 ampNames = np.unique(inputCovariances[0].ampNames) 

224 datasetPtc = PhotonTransferCurveDataset(ampNames=ampNames, 

225 ptcFitType=self.config.ptcFitType, 

226 covMatrixSide=self.config.maximumRangeCovariancesAstier) 

227 for partialPtcDataset in inputCovariances: 

228 # Ignore dummy datasets 

229 if partialPtcDataset.ptcFitType == 'DUMMY': 

230 continue 

231 for ampName in ampNames: 

232 datasetPtc.inputExpIdPairs[ampName].append(partialPtcDataset.inputExpIdPairs[ampName]) 

233 if type(partialPtcDataset.rawExpTimes[ampName]) is list: 

234 datasetPtc.rawExpTimes[ampName].append(partialPtcDataset.rawExpTimes[ampName][0]) 

235 else: 

236 datasetPtc.rawExpTimes[ampName].append(partialPtcDataset.rawExpTimes[ampName]) 

237 if type(partialPtcDataset.rawMeans[ampName]) is list: 

238 datasetPtc.rawMeans[ampName].append(partialPtcDataset.rawMeans[ampName][0]) 

239 else: 

240 datasetPtc.rawMeans[ampName].append(partialPtcDataset.rawMeans[ampName]) 

241 if type(partialPtcDataset.rawVars[ampName]) is list: 

242 datasetPtc.rawVars[ampName].append(partialPtcDataset.rawVars[ampName][0]) 

243 else: 

244 datasetPtc.rawVars[ampName].append(partialPtcDataset.rawVars[ampName]) 

245 if type(partialPtcDataset.expIdMask[ampName]) is list: 

246 datasetPtc.expIdMask[ampName].append(partialPtcDataset.expIdMask[ampName][0]) 

247 else: 

248 datasetPtc.expIdMask[ampName].append(partialPtcDataset.expIdMask[ampName]) 

249 datasetPtc.covariances[ampName].append(np.array(partialPtcDataset.covariances[ampName][0])) 

250 datasetPtc.covariancesSqrtWeights[ampName].append( 

251 np.array(partialPtcDataset.covariancesSqrtWeights[ampName][0])) 

252 # Sort arrays that are filled so far in the final dataset by 

253 # rawMeans index 

254 for ampName in ampNames: 

255 index = np.argsort(np.ravel(np.array(datasetPtc.rawMeans[ampName]))) 

256 datasetPtc.inputExpIdPairs[ampName] = np.array(datasetPtc.inputExpIdPairs[ampName])[index] 

257 datasetPtc.rawExpTimes[ampName] = np.array(datasetPtc.rawExpTimes[ampName])[index] 

258 datasetPtc.rawMeans[ampName] = np.array(datasetPtc.rawMeans[ampName])[index] 

259 datasetPtc.rawVars[ampName] = np.array(datasetPtc.rawVars[ampName])[index] 

260 datasetPtc.expIdMask[ampName] = np.array(datasetPtc.expIdMask[ampName])[index] 

261 datasetPtc.covariances[ampName] = np.array(datasetPtc.covariances[ampName])[index] 

262 datasetPtc.covariancesSqrtWeights[ampName] = np.array( 

263 datasetPtc.covariancesSqrtWeights[ampName])[index] 

264 if self.config.ptcFitType == "FULLCOVARIANCE": 

265 # Fit the measured covariances vs mean signal to 

266 # the Astier+19 full model (Eq. 20). Before that 

267 # do a preliminary fit to the variance (C_00) vs mean 

268 # signal (mu) curve using the EXPAPPROXIMATION model 

269 # (Eq. 16 in Astier+19) in order to 

270 # get the flat pairs that are masked. The 

271 # points at these fluxes will also be masked when 

272 # calculating the other elements of the covariance 

273 # matrix, C_ij, i!=j). 

274 

275 # Preliminary fit, usign a temp dataset to get the mask 

276 tempDatasetPtc = copy.copy(datasetPtc) 

277 tempDatasetPtc.ptcFitType = "EXPAPPROXIMATION" 

278 tempDatasetPtc = self.fitMeasurementsToModel(tempDatasetPtc) 

279 

280 # "FULLCOVARIANCE", using the mask obtained from the 

281 # previous fit. 

282 for ampName in datasetPtc.ampNames: 

283 datasetPtc.expIdMask[ampName] = tempDatasetPtc.expIdMask[ampName] 

284 datasetPtc.fitType = "FULLCOVARIANCE" 

285 datasetPtc = self.fitMeasurementsToModel(datasetPtc) 

286 # The other options are: self.config.ptcFitType in 

287 # ("EXPAPPROXIMATION", "POLYNOMIAL") 

288 else: 

289 # Fit the PTC to a polynomial or to Astier+19 exponential 

290 # approximation (Eq. 16). Fill up 

291 # PhotonTransferCurveDataset object. 

292 datasetPtc = self.fitMeasurementsToModel(datasetPtc) 

293 

294 if camera: 

295 detector = camera[detId] 

296 else: 

297 detector = None 

298 datasetPtc.updateMetadata(setDate=True, camera=camera, detector=detector) 

299 

300 return pipeBase.Struct( 

301 outputPtcDataset=datasetPtc, 

302 ) 

303 

304 def fitMeasurementsToModel(self, dataset): 

305 """Fit the measured covariances vs mean signal to a 

306 polynomial or one of the models in Astier+19 

307 (Eq. 16 or Eq.20). 

308 

309 Parameters 

310 ---------- 

311 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset` 

312 The dataset containing information such as the means, 

313 (co)variances, and exposure times. 

314 

315 Returns 

316 ------- 

317 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset` 

318 This is the same dataset as the input parameter, however, 

319 it has been modified to include information such as the 

320 fit vectors and the fit parameters. See the class 

321 `PhotonTransferCurveDatase`. 

322 """ 

323 fitType = dataset.ptcFitType 

324 if fitType in ["FULLCOVARIANCE", ]: 

325 # This model uses the full covariance matrix in the fit. 

326 # The PTC is technically defined as variance vs signal, 

327 # with variance = Cov_00 

328 dataset = self.fitDataFullCovariance(dataset) 

329 elif fitType in ["POLYNOMIAL", "EXPAPPROXIMATION"]: 

330 # The PTC is technically defined as variance vs signal 

331 dataset = self.fitPtc(dataset) 

332 else: 

333 raise RuntimeError( 

334 f"Fitting option {fitType} not one of " 

335 "'POLYNOMIAL', 'EXPAPPROXIMATION', or 'FULLCOVARIANCE'" 

336 ) 

337 

338 return dataset 

339 

340 def fitDataFullCovariance(self, dataset): 

341 """Fit measured flat covariances to the full model in 

342 Astier+19 (Eq. 20). 

343 

344 Parameters 

345 ---------- 

346 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset` 

347 The dataset containing information such as the means, 

348 (co)variances, and exposure times. 

349 

350 Returns 

351 ------- 

352 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset` 

353 This is the same dataset as the input parameter, however, 

354 it has been modified to include information such as the 

355 fit vectors and the fit parameters. See the class 

356 `PhotonTransferCurveDatase`. 

357 

358 Notes 

359 ----- 

360 The parameters of the full model for C_ij(mu) ("C_ij" and "mu" 

361 in ADU^2 and ADU, respectively) in Astier+19 (Eq. 20) are: 

362 

363 - "a" coefficients (r by r matrix), units: 1/e 

364 - "b" coefficients (r by r matrix), units: 1/e 

365 - noise matrix (r by r matrix), units: e^2 

366 - gain, units: e/ADU 

367 

368 "b" appears in Eq. 20 only through the "ab" combination, which 

369 is defined in this code as "c=ab". 

370 

371 Total number of parameters: #entries(a) + #entries(c) + #entries(noise) 

372 + 1. This is equivalent to r^2 + r^2 + r^2 + 1, where "r" is the 

373 maximum lag considered for the covariances calculation, and the 

374 extra "1" is the gain. If "b" is 0, then "c" is 0, and len(pInit) will 

375 have r^2 fewer entries. 

376 """ 

377 matrixSide = self.config.maximumRangeCovariancesAstier 

378 lenParams = matrixSide*matrixSide 

379 

380 for ampName in dataset.ampNames: 

381 lenInputTimes = len(dataset.rawExpTimes[ampName]) 

382 # Not used when ptcFitType is 'FULLCOVARIANCE' 

383 dataset.ptcFitPars[ampName] = [np.nan] 

384 dataset.ptcFitParsError[ampName] = [np.nan] 

385 dataset.ptcFitChiSq[ampName] = np.nan 

386 

387 if ampName in dataset.badAmps: 

388 # Bad amp 

389 # Entries need to have proper dimensions so read/write 

390 # with astropy.Table works. 

391 nanMatrix = np.full((matrixSide, matrixSide), np.nan) 

392 listNanMatrix = np.full((lenInputTimes, matrixSide, matrixSide), np.nan) 

393 dataset.covariancesModel[ampName] = listNanMatrix 

394 dataset.covariancesSqrtWeights[ampName] = listNanMatrix 

395 dataset.aMatrix[ampName] = nanMatrix 

396 dataset.bMatrix[ampName] = nanMatrix 

397 dataset.covariancesModelNoB[ampName] = listNanMatrix 

398 dataset.aMatrixNoB[ampName] = nanMatrix 

399 

400 dataset.expIdMask[ampName] = np.repeat(False, lenInputTimes) 

401 dataset.gain[ampName] = np.nan 

402 dataset.gainErr[ampName] = np.nan 

403 dataset.noise[ampName] = np.nan 

404 dataset.noiseErr[ampName] = np.nan 

405 dataset.finalVars[ampName] = np.repeat(np.nan, lenInputTimes) 

406 dataset.finalModelVars[ampName] = np.repeat(np.nan, lenInputTimes) 

407 dataset.finalMeans[ampName] = np.repeat(np.nan, lenInputTimes) 

408 continue 

409 

410 muAtAmp = dataset.rawMeans[ampName] 

411 maskAtAmp = dataset.expIdMask[ampName] 

412 if len(maskAtAmp) == 0: 

413 maskAtAmp = np.repeat(True, len(muAtAmp)) 

414 

415 muAtAmp = muAtAmp[maskAtAmp] 

416 covAtAmp = np.nan_to_num(dataset.covariances[ampName])[maskAtAmp] 

417 covSqrtWeightsAtAmp = np.nan_to_num(dataset.covariancesSqrtWeights[ampName])[maskAtAmp] 

418 

419 # Initial fit, to approximate parameters, with c=0 

420 a0, c0, noise0, gain0 = self.initialFitFullCovariance(muAtAmp, covAtAmp, covSqrtWeightsAtAmp) 

421 

422 # Fit full model (Eq. 20 of Astier+19) and same model with 

423 # b=0 (c=0 in this code) 

424 pInit = np.concatenate((a0.flatten(), c0.flatten(), noise0.flatten(), np.array(gain0)), axis=None) 

425 functionsDict = {'fullModel': self.funcFullCovarianceModel, 

426 'fullModelNoB': self.funcFullCovarianceModelNoB} 

427 fitResults = {'fullModel': {'a': [], 'c': [], 'noise': [], 'gain': [], 'paramsErr': []}, 

428 'fullModelNoB': {'a': [], 'c': [], 'noise': [], 'gain': [], 'paramsErr': []}} 

429 for key in functionsDict: 

430 params, paramsErr, _ = fitLeastSq(pInit, muAtAmp, 

431 covAtAmp.flatten(), functionsDict[key], 

432 weightsY=covSqrtWeightsAtAmp.flatten()) 

433 a = params[:lenParams].reshape((matrixSide, matrixSide)) 

434 c = params[lenParams:2*lenParams].reshape((matrixSide, matrixSide)) 

435 noise = params[2*lenParams:3*lenParams].reshape((matrixSide, matrixSide)) 

436 gain = params[-1] 

437 

438 fitResults[key]['a'] = a 

439 fitResults[key]['c'] = c 

440 fitResults[key]['noise'] = noise 

441 fitResults[key]['gain'] = gain 

442 fitResults[key]['paramsErr'] = paramsErr 

443 

444 # Put the information in the PTC dataset 

445 

446 # Not used when ptcFitType is 'FULLCOVARIANCE' 

447 dataset.ptcFitPars[ampName] = [np.nan] 

448 dataset.ptcFitParsError[ampName] = [np.nan] 

449 dataset.ptcFitChiSq[ampName] = np.nan 

450 

451 # Save full covariances, covariances models, and their weights. 

452 # dataset.expIdMask is already full, but needs to be 

453 # converted to bool. 

454 dataset.expIdMask[ampName] = np.array(dataset.expIdMask[ampName], dtype=bool) 

455 dataset.covariances[ampName] = covAtAmp 

456 dataset.covariancesModel[ampName] = self.evalCovModel(muAtAmp, 

457 fitResults['fullModel']['a'], 

458 fitResults['fullModel']['c'], 

459 fitResults['fullModel']['noise'], 

460 fitResults['fullModel']['gain']) 

461 dataset.covariancesSqrtWeights[ampName] = covSqrtWeightsAtAmp 

462 dataset.aMatrix[ampName] = fitResults['fullModel']['a'] 

463 dataset.bMatrix[ampName] = fitResults['fullModel']['c']/fitResults['fullModel']['a'] 

464 dataset.covariancesModelNoB[ampName] = self.evalCovModel(muAtAmp, 

465 fitResults['fullModelNoB']['a'], 

466 fitResults['fullModelNoB']['c'], 

467 fitResults['fullModelNoB']['noise'], 

468 fitResults['fullModelNoB']['gain'], 

469 setBtoZero=True) 

470 dataset.aMatrixNoB[ampName] = fitResults['fullModelNoB']['a'] 

471 dataset.gain[ampName] = fitResults['fullModel']['gain'] 

472 dataset.gainErr[ampName] = fitResults['fullModel']['paramsErr'][-1] 

473 readoutNoise = fitResults['fullModel']['noise'][0][0] 

474 readoutNoiseSqrt = np.sqrt(np.fabs(readoutNoise)) 

475 dataset.noise[ampName] = readoutNoise 

476 readoutNoiseSigma = fitResults['fullModel']['paramsErr'][2*lenParams] 

477 dataset.noiseErr[ampName] = 0.5*(readoutNoiseSigma/np.fabs(readoutNoise))*readoutNoiseSqrt 

478 dataset.finalVars[ampName] = covAtAmp[:, 0, 0] 

479 dataset.finalModelVars[ampName] = dataset.covariancesModel[ampName][:, 0, 0] 

480 dataset.finalMeans[ampName] = muAtAmp 

481 

482 return dataset 

483 

484 def initialFitFullCovariance(self, mu, cov, sqrtW): 

485 """ Performs a crude parabolic fit of the data in order to start 

486 the full fit close to the solution, setting b=0 (c=0) in Eq. 20 

487 of Astier+19. 

488 

489 Parameters 

490 ---------- 

491 mu : `numpy.array`, (N,) 

492 Signal `mu` (ADU) 

493 cov : `numpy.array`, (N, M, M) 

494 Covariance arrays of size `(M, M)` (with 

495 `M = config.maximumRangeCovariancesAstier`), 

496 indexed by mean signal `mu`. 

497 sqrtW : `numpy.array`, (N,) 

498 Covariance weights, defined as 1./sqrt(Variances) 

499 

500 Returns 

501 ------- 

502 a : `numpy.array`, (M, M) 

503 "a" parameter per flux in Eq. 20 of Astier+19. 

504 c : `numpy.array`, (M, M) 

505 "c"="ab" parameter per flux in Eq. 20 of Astier+19. 

506 noise : `numpy.array`, (M, M) 

507 "noise" parameter per flux in Eq. 20 of Astier+19. 

508 gain : `float` 

509 Amplifier gain (e/ADU) 

510 """ 

511 matrixSide = self.config.maximumRangeCovariancesAstier 

512 

513 # Initialize fit parameters 

514 a = np.zeros((matrixSide, matrixSide)) 

515 c = np.zeros((matrixSide, matrixSide)) 

516 noise = np.zeros((matrixSide, matrixSide)) 

517 gain = 1. 

518 

519 # iterate the fit to account for higher orders 

520 # the chi2 does not necessarily go down, so one could 

521 # stop when it increases 

522 oldChi2 = 1e30 

523 for _ in range(5): 

524 model = np.nan_to_num(self.evalCovModel(mu, a, c, noise, gain, setBtoZero=True)) 

525 # loop on lags 

526 for i in range(matrixSide): 

527 for j in range(matrixSide): 

528 # fit a parabola for a given lag 

529 parsFit = np.polyfit(mu, cov[:, i, j] - model[:, i, j], 

530 2, w=sqrtW[:, i, j]) 

531 # model equation (Eq. 20) in Astier+19, with c=a*b=0: 

532 a[i, j] += parsFit[0] 

533 noise[i, j] += parsFit[2] 

534 if(i + j == 0): 

535 gain = 1./(1/gain+parsFit[1]) 

536 weightedRes = (model - cov)*sqrtW 

537 chi2 = (weightedRes.flatten()**2).sum() 

538 if chi2 > oldChi2: 

539 break 

540 oldChi2 = chi2 

541 

542 return a, c, noise, gain 

543 

544 def funcFullCovarianceModel(self, params, x): 

545 """Model to fit covariances from flat fields; Equation 20 of 

546 Astier+19. 

547 

548 Parameters 

549 ---------- 

550 params : `list` 

551 Parameters of the model: aMatrix, CMatrix, noiseMatrix, 

552 gain (e/ADU). 

553 x : `numpy.array`, (N,) 

554 Signal `mu` (ADU) 

555 

556 Returns 

557 ------- 

558 y : `numpy.array`, (N,) 

559 Covariance matrix. 

560 """ 

561 matrixSide = self.config.maximumRangeCovariancesAstier 

562 lenParams = matrixSide*matrixSide 

563 aMatrix = params[:lenParams].reshape((matrixSide, matrixSide)) 

564 cMatrix = params[lenParams:2*lenParams].reshape((matrixSide, matrixSide)) 

565 noiseMatrix = params[2*lenParams:3*lenParams].reshape((matrixSide, matrixSide)) 

566 gain = params[-1] 

567 

568 return self.evalCovModel(x, aMatrix, cMatrix, noiseMatrix, gain).flatten() 

569 

570 def funcFullCovarianceModelNoB(self, params, x): 

571 """Model to fit covariances from flat fields; Equation 20 of 

572 Astier+19, with b=0 (equivalent to c=a*b=0 in this code). 

573 

574 Parameters 

575 ---------- 

576 params : `list` 

577 Parameters of the model: aMatrix, CMatrix, noiseMatrix, 

578 gain (e/ADU). 

579 x : `numpy.array`, (N,) 

580 Signal mu (ADU) 

581 

582 Returns 

583 ------- 

584 y : `numpy.array`, (N,) 

585 Covariance matrix. 

586 """ 

587 matrixSide = self.config.maximumRangeCovariancesAstier 

588 lenParams = matrixSide*matrixSide 

589 aMatrix = params[:lenParams].reshape((matrixSide, matrixSide)) 

590 cMatrix = params[lenParams:2*lenParams].reshape((matrixSide, matrixSide)) 

591 noiseMatrix = params[2*lenParams:3*lenParams].reshape((matrixSide, matrixSide)) 

592 gain = params[-1] 

593 

594 return self.evalCovModel(x, aMatrix, cMatrix, noiseMatrix, gain, setBtoZero=True).flatten() 

595 

596 def evalCovModel(self, mu, aMatrix, cMatrix, noiseMatrix, gain, setBtoZero=False): 

597 """Computes full covariances model (Eq. 20 of Astier+19). 

598 

599 Parameters 

600 ---------- 

601 mu : `numpy.array`, (N,) 

602 List of mean signals. 

603 aMatrix : `numpy.array`, (M, M) 

604 "a" parameter per flux in Eq. 20 of Astier+19. 

605 cMatrix : `numpy.array`, (M, M) 

606 "c"="ab" parameter per flux in Eq. 20 of Astier+19. 

607 noiseMatrix : `numpy.array`, (M, M) 

608 "noise" parameter per flux in Eq. 20 of Astier+19. 

609 gain : `float` 

610 Amplifier gain (e/ADU) 

611 setBtoZero=False : `bool`, optional 

612 Set "b" parameter in full model (see Astier+19) to zero. 

613 

614 Returns 

615 ------- 

616 covModel : `numpy.array`, (N, M, M) 

617 Covariances model. 

618 

619 Notes 

620 ----- 

621 By default, computes the covModel for the mu's stored(self.mu). 

622 Returns cov[Nmu, M, M]. The variance for the PTC is 

623 cov[:, 0, 0]. mu and cov are in ADUs and ADUs squared. To use 

624 electrons for both, the gain should be set to 1. This routine 

625 implements the model in Astier+19 (1905.08677). 

626 The parameters of the full model for C_ij(mu) ("C_ij" and "mu" 

627 in ADU^2 and ADU, respectively) in Astier+19 (Eq. 20) are: 

628 

629 - "a" coefficients (M by M matrix), units: 1/e 

630 - "b" coefficients (M by M matrix), units: 1/e 

631 - noise matrix (M by M matrix), units: e^2 

632 - gain, units: e/ADU 

633 

634 "b" appears in Eq. 20 only through the "ab" combination, which 

635 is defined in this code as "c=ab". 

636 """ 

637 matrixSide = self.config.maximumRangeCovariancesAstier 

638 sa = (matrixSide, matrixSide) 

639 # pad a with zeros and symmetrize 

640 aEnlarged = np.zeros((int(sa[0]*1.5)+1, int(sa[1]*1.5)+1)) 

641 aEnlarged[0:sa[0], 0:sa[1]] = aMatrix 

642 aSym = symmetrize(aEnlarged) 

643 # pad c with zeros and symmetrize 

644 cEnlarged = np.zeros((int(sa[0]*1.5)+1, int(sa[1]*1.5)+1)) 

645 cEnlarged[0:sa[0], 0:sa[1]] = cMatrix 

646 cSym = symmetrize(cEnlarged) 

647 a2 = fftconvolve(aSym, aSym, mode='same') 

648 a3 = fftconvolve(a2, aSym, mode='same') 

649 ac = fftconvolve(aSym, cSym, mode='same') 

650 (xc, yc) = np.unravel_index(np.abs(aSym).argmax(), a2.shape) 

651 

652 a1 = aMatrix[np.newaxis, :, :] 

653 a2 = a2[np.newaxis, xc:xc + matrixSide, yc:yc + matrixSide] 

654 a3 = a3[np.newaxis, xc:xc + matrixSide, yc:yc + matrixSide] 

655 ac = ac[np.newaxis, xc:xc + matrixSide, yc:yc + matrixSide] 

656 c1 = cMatrix[np.newaxis, ::] 

657 

658 # assumes that mu is 1d 

659 bigMu = mu[:, np.newaxis, np.newaxis]*gain 

660 # c(=a*b in Astier+19) also has a contribution to the last 

661 # term, that is absent for now. 

662 if setBtoZero: 

663 c1 = np.zeros_like(c1) 

664 ac = np.zeros_like(ac) 

665 covModel = (bigMu/(gain*gain)*(a1*bigMu+2./3.*(bigMu*bigMu)*(a2 + c1) 

666 + (1./3.*a3 + 5./6.*ac)*(bigMu*bigMu*bigMu)) + noiseMatrix[np.newaxis, :, :]/gain**2) 

667 # add the Poisson term, and the read out noise (variance) 

668 covModel[:, 0, 0] += mu/gain 

669 

670 return covModel 

671 

672 # EXPAPPROXIMATION and POLYNOMIAL fit methods 

673 @staticmethod 

674 def _initialParsForPolynomial(order): 

675 assert(order >= 2) 

676 pars = np.zeros(order, dtype=float) 

677 pars[0] = 10 

678 pars[1] = 1 

679 pars[2:] = 0.0001 

680 return pars 

681 

682 @staticmethod 

683 def _boundsForPolynomial(initialPars, lowers=[], uppers=[]): 

684 if not len(lowers): 

685 lowers = [np.NINF for p in initialPars] 

686 if not len(uppers): 

687 uppers = [np.inf for p in initialPars] 

688 lowers[1] = 0 # no negative gains 

689 return (lowers, uppers) 

690 

691 @staticmethod 

692 def _boundsForAstier(initialPars, lowers=[], uppers=[]): 

693 if not len(lowers): 

694 lowers = [np.NINF for p in initialPars] 

695 if not len(uppers): 

696 uppers = [np.inf for p in initialPars] 

697 return (lowers, uppers) 

698 

699 @staticmethod 

700 def _getInitialGoodPoints(means, variances, minVarPivotSearch, consecutivePointsVarDecreases): 

701 """Return a boolean array to mask bad points. 

702 

703 Parameters 

704 ---------- 

705 means : `numpy.array` 

706 Input array with mean signal values. 

707 variances : `numpy.array` 

708 Input array with variances at each mean value. 

709 minVarPivotSearch : `float` 

710 The variance (in ADU^2), above which, the point 

711 of decreasing variance should be sought. 

712 consecutivePointsVarDecreases : `int` 

713 Required number of consecutive points/fluxes 

714 in the PTC where the variance 

715 decreases in order to find a first 

716 estimate of the PTC turn-off. 

717 

718 Returns 

719 ------ 

720 goodPoints : `numpy.array` [`bool`] 

721 Boolean array to select good (`True`) and bad (`False`) 

722 points. 

723 

724 Notes 

725 ----- 

726 Eliminate points beyond which the variance decreases. 

727 """ 

728 goodPoints = np.ones_like(means, dtype=bool) 

729 # Variances are sorted and should monotonically increase 

730 pivotList = np.where(np.array(np.diff(variances)) < 0)[0] 

731 if len(pivotList) > 0: 

732 # For small values, sometimes the variance decreases slightly 

733 # Only look when var > self.config.minVarPivotSearch 

734 pivotList = [p for p in pivotList if variances[p] > minVarPivotSearch] 

735 # Require that the varince decreases during 

736 # consecutivePointsVarDecreases 

737 # consecutive points. This will give a first 

738 # estimate of the PTC turn-off, which 

739 # may be updated (reduced) further in the code. 

740 if len(pivotList) > 1: 

741 # enumerate(pivotList) creates tuples (index, value), for 

742 # each value in pivotList. The lambda function subtracts 

743 # each value from the index. 

744 # groupby groups elements by equal key value. 

745 for k, g in groupby(enumerate(pivotList), lambda x: x[0]-x[1]): 

746 group = (map(itemgetter(1), g)) 

747 # Form groups of consecute values from pivotList 

748 group = list(map(int, group)) 

749 # values in pivotList are indices where np.diff(variances) 

750 # is negative, i.e., where the variance starts decreasing. 

751 # Find the first group of consecutive numbers when 

752 # variance decreases. 

753 if len(group) >= consecutivePointsVarDecreases: 

754 pivotIndex = np.min(group) 

755 goodPoints[pivotIndex+1:] = False 

756 break 

757 

758 return goodPoints 

759 

760 def _makeZeroSafe(self, array, substituteValue=1e-9): 

761 """""" 

762 array = np.array(array) 

763 nBad = Counter(np.ravel(array))[0] 

764 if nBad == 0: 

765 return array 

766 

767 index, = np.where(array == 0) 

768 if len(index): 

769 msg = f"Found {nBad} zeros in array at elements {index}" 

770 self.log.warning(msg) 

771 

772 array[index] = substituteValue 

773 

774 return array 

775 

776 def fitPtc(self, dataset): 

777 """Fit the photon transfer curve to a polynomial or to the 

778 Astier+19 approximation (Eq. 16). 

779 

780 Fit the photon transfer curve with either a polynomial of 

781 the order specified in the task config, or using the 

782 exponential approximation in Astier+19 (Eq. 16). 

783 

784 Sigma clipping is performed iteratively for the fit, as 

785 well as an initial clipping of data points that are more 

786 than `config.initialNonLinearityExclusionThreshold` away 

787 from lying on a straight line. This other step is necessary 

788 because the photon transfer curve turns over catastrophically 

789 at very high flux (because saturation 

790 drops the variance to ~0) and these far outliers cause the 

791 initial fit to fail, meaning the sigma cannot be calculated 

792 to perform the sigma-clipping. 

793 

794 Parameters 

795 ---------- 

796 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset` 

797 The dataset containing the means, variances and 

798 exposure times. 

799 

800 Returns 

801 ------- 

802 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset` 

803 This is the same dataset as the input parameter, however, 

804 it has been modified to include information such as the 

805 fit vectors and the fit parameters. See the class 

806 `PhotonTransferCurveDatase`. 

807 

808 Raises 

809 ------ 

810 RuntimeError 

811 Raised if dataset.ptcFitType is None or empty. 

812 """ 

813 if dataset.ptcFitType: 

814 ptcFitType = dataset.ptcFitType 

815 else: 

816 raise RuntimeError("ptcFitType is None of empty in PTC dataset.") 

817 matrixSide = self.config.maximumRangeCovariancesAstier 

818 nanMatrix = np.empty((matrixSide, matrixSide)) 

819 nanMatrix[:] = np.nan 

820 

821 for amp in dataset.ampNames: 

822 lenInputTimes = len(dataset.rawExpTimes[amp]) 

823 listNanMatrix = np.empty((lenInputTimes, matrixSide, matrixSide)) 

824 listNanMatrix[:] = np.nan 

825 

826 dataset.covariancesModel[amp] = listNanMatrix 

827 dataset.aMatrix[amp] = nanMatrix 

828 dataset.bMatrix[amp] = nanMatrix 

829 dataset.covariancesModelNoB[amp] = listNanMatrix 

830 dataset.aMatrixNoB[amp] = nanMatrix 

831 

832 def errFunc(p, x, y): 

833 return ptcFunc(p, x) - y 

834 

835 sigmaCutPtcOutliers = self.config.sigmaCutPtcOutliers 

836 maxIterationsPtcOutliers = self.config.maxIterationsPtcOutliers 

837 

838 for i, ampName in enumerate(dataset.ampNames): 

839 timeVecOriginal = np.ravel(np.array(dataset.rawExpTimes[ampName])) 

840 meanVecOriginal = np.ravel(np.array(dataset.rawMeans[ampName])) 

841 varVecOriginal = np.ravel(np.array(dataset.rawVars[ampName])) 

842 varVecOriginal = self._makeZeroSafe(varVecOriginal) 

843 

844 # Discard points when the variance starts to decrease after two 

845 # consecutive signal levels 

846 goodPoints = self._getInitialGoodPoints(meanVecOriginal, varVecOriginal, 

847 self.config.minVarPivotSearch, 

848 self.config.consecutivePointsVarDecreases) 

849 

850 # Check if all points are bad from the 'cpExtractPtcTask' 

851 initialExpIdMask = np.ravel(np.array(dataset.expIdMask[ampName])) 

852 

853 if not (goodPoints.any() and initialExpIdMask.any()): 

854 msg = (f"SERIOUS: All points in goodPoints: {goodPoints} or " 

855 f"in initialExpIdMask: {initialExpIdMask} are bad." 

856 f"Setting {ampName} to BAD.") 

857 self.log.warning(msg) 

858 # Fill entries with NaNs 

859 self.fillBadAmp(dataset, ptcFitType, ampName) 

860 continue 

861 

862 # Save the point where the variance starts decreasing as the 

863 # PTC turnoff point 

864 ptcTurnoff = meanVecOriginal[goodPoints][-1] 

865 dataset.ptcTurnoff[ampName] = ptcTurnoff 

866 

867 mask = goodPoints 

868 

869 if ptcFitType == 'EXPAPPROXIMATION': 

870 ptcFunc = funcAstier 

871 parsIniPtc = [-1e-9, 1.0, 10.] # a00, gain, noise^2 

872 # lowers and uppers obtained from BOT data studies by 

873 # C. Lage (UC Davis, 11/2020). 

874 if self.config.binSize > 1: 

875 bounds = self._boundsForAstier(parsIniPtc) 

876 else: 

877 bounds = self._boundsForAstier(parsIniPtc, lowers=[-1e-4, 0.5, -2000], 

878 uppers=[1e-4, 2.5, 2000]) 

879 if ptcFitType == 'POLYNOMIAL': 

880 ptcFunc = funcPolynomial 

881 parsIniPtc = self._initialParsForPolynomial(self.config.polynomialFitDegree + 1) 

882 bounds = self._boundsForPolynomial(parsIniPtc) 

883 

884 # Before bootstrap fit, do an iterative fit to get rid of outliers. 

885 # This further process of outlier rejection be skipped 

886 # if self.config.maxIterationsPtcOutliers = 0. 

887 # We already did some initial outlier rejection above in 

888 # self._getInitialGoodPoints. 

889 count = 1 

890 newMask = np.ones_like(meanVecOriginal, dtype=bool) 

891 pars = parsIniPtc 

892 while count <= maxIterationsPtcOutliers: 

893 # Note that application of the mask actually shrinks the array 

894 # to size rather than setting elements to zero (as we want) so 

895 # always update mask itself and re-apply to the original data 

896 meanTempVec = meanVecOriginal[mask] 

897 varTempVec = varVecOriginal[mask] 

898 res = least_squares(errFunc, parsIniPtc, bounds=bounds, args=(meanTempVec, varTempVec)) 

899 pars = res.x 

900 

901 # change this to the original from the temp because 

902 # the masks are ANDed meaning once a point is masked 

903 # it's always masked, and the masks must always be the 

904 # same length for broadcasting 

905 sigResids = (varVecOriginal - ptcFunc(pars, meanVecOriginal))/np.sqrt(varVecOriginal) 

906 newMask = np.array([True if np.abs(r) < sigmaCutPtcOutliers else False for r in sigResids]) 

907 mask = mask & newMask 

908 if not (mask.any() and newMask.any()): 

909 msg = (f"SERIOUS: All points in either mask: {mask} or newMask: {newMask} are bad. " 

910 f"Setting {ampName} to BAD.") 

911 self.log.warning(msg) 

912 # Fill entries with NaNs 

913 self.fillBadAmp(dataset, ptcFitType, ampName) 

914 break 

915 nDroppedTotal = Counter(mask)[False] 

916 self.log.debug("Iteration %d: discarded %d points in total for %s", 

917 count, nDroppedTotal, ampName) 

918 count += 1 

919 # objects should never shrink 

920 assert (len(mask) == len(timeVecOriginal) == len(meanVecOriginal) == len(varVecOriginal)) 

921 if not (mask.any() and newMask.any()): 

922 continue 

923 dataset.expIdMask[ampName] = np.array(dataset.expIdMask[ampName]) 

924 # store the final mask 

925 if len(dataset.expIdMask[ampName]): 

926 dataset.expIdMask[ampName] &= mask # bitwise_and if there is already a mask 

927 else: 

928 dataset.expIdMask[ampName] = mask 

929 # In case there was a previous mask stored 

930 mask = dataset.expIdMask[ampName] 

931 parsIniPtc = pars 

932 meanVecFinal = meanVecOriginal[mask] 

933 varVecFinal = varVecOriginal[mask] 

934 

935 if Counter(mask)[False] > 0: 

936 self.log.info("Number of points discarded in PTC of amplifier %s:" 

937 " %d out of %d", ampName, Counter(mask)[False], len(meanVecOriginal)) 

938 

939 if (len(meanVecFinal) < len(parsIniPtc)): 

940 msg = (f"SERIOUS: Not enough data points ({len(meanVecFinal)}) compared to the number of " 

941 f"parameters of the PTC model({len(parsIniPtc)}). Setting {ampName} to BAD.") 

942 self.log.warning(msg) 

943 # Fill entries with NaNs 

944 self.fillBadAmp(dataset, ptcFitType, ampName) 

945 continue 

946 # Fit the PTC. 

947 # The variance of the variance is Var(v)=2*v^2/Npix. This is 

948 # already calculated in `makeCovArray` of CpPtcExtract. 

949 # dataset.covariancesSqrtWeights[ampName][:,0,0] 

950 # has 1/sqrt(Var(v)). 

951 weightsY = dataset.covariancesSqrtWeights[ampName][:, 0, 0][mask] 

952 if self.config.doFitBootstrap: 

953 parsFit, parsFitErr, reducedChiSqPtc = fitBootstrap(parsIniPtc, meanVecFinal, 

954 varVecFinal, ptcFunc, 

955 weightsY=weightsY) 

956 else: 

957 parsFit, parsFitErr, reducedChiSqPtc = fitLeastSq(parsIniPtc, meanVecFinal, 

958 varVecFinal, ptcFunc, 

959 weightsY=weightsY) 

960 dataset.ptcFitPars[ampName] = parsFit 

961 dataset.ptcFitParsError[ampName] = parsFitErr 

962 dataset.ptcFitChiSq[ampName] = reducedChiSqPtc 

963 # Masked variances (measured and modeled) and means. Need 

964 # to pad the array so astropy.Table does not crash (the 

965 # mask may vary per amp). 

966 padLength = len(dataset.rawExpTimes[ampName]) - len(varVecFinal) 

967 dataset.finalVars[ampName] = np.pad(varVecFinal, (0, padLength), 'constant', 

968 constant_values=np.nan) 

969 dataset.finalModelVars[ampName] = np.pad(ptcFunc(parsFit, meanVecFinal), (0, padLength), 

970 'constant', constant_values=np.nan) 

971 dataset.finalMeans[ampName] = np.pad(meanVecFinal, (0, padLength), 'constant', 

972 constant_values=np.nan) 

973 if ptcFitType == 'EXPAPPROXIMATION': 

974 ptcGain = parsFit[1] 

975 ptcGainErr = parsFitErr[1] 

976 ptcNoise = np.sqrt(np.fabs(parsFit[2])) 

977 ptcNoiseErr = 0.5*(parsFitErr[2]/np.fabs(parsFit[2]))*np.sqrt(np.fabs(parsFit[2])) 

978 if ptcFitType == 'POLYNOMIAL': 

979 ptcGain = 1./parsFit[1] 

980 ptcGainErr = np.fabs(1./parsFit[1])*(parsFitErr[1]/parsFit[1]) 

981 ptcNoise = np.sqrt(np.fabs(parsFit[0]))*ptcGain 

982 ptcNoiseErr = (0.5*(parsFitErr[0]/np.fabs(parsFit[0]))*(np.sqrt(np.fabs(parsFit[0]))))*ptcGain 

983 dataset.gain[ampName] = ptcGain 

984 dataset.gainErr[ampName] = ptcGainErr 

985 dataset.noise[ampName] = ptcNoise 

986 dataset.noiseErr[ampName] = ptcNoiseErr 

987 

988 if not len(dataset.ptcFitType) == 0: 

989 dataset.ptcFitType = ptcFitType 

990 if len(dataset.badAmps) == 0: 

991 dataset.badAmps = np.repeat(np.nan, len(list(dataset.rawExpTimes.values())[0])) 

992 

993 return dataset 

994 

995 def fillBadAmp(self, dataset, ptcFitType, ampName): 

996 """Fill the dataset with NaNs if there are not enough 

997 good points. 

998 

999 Parameters 

1000 ---------- 

1001 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset` 

1002 The dataset containing the means, variances and 

1003 exposure times. 

1004 ptcFitType : {'POLYNOMIAL', 'EXPAPPROXIMATION'} 

1005 Fit a 'POLYNOMIAL' (degree: 'polynomialFitDegree') or 

1006 'EXPAPPROXIMATION' (Eq. 16 of Astier+19) to the PTC. 

1007 ampName : `str` 

1008 Amplifier name. 

1009 """ 

1010 dataset.badAmps.append(ampName) 

1011 dataset.expIdMask[ampName] = np.repeat(False, len(dataset.rawExpTimes[ampName])) 

1012 dataset.gain[ampName] = np.nan 

1013 dataset.gainErr[ampName] = np.nan 

1014 dataset.noise[ampName] = np.nan 

1015 dataset.noiseErr[ampName] = np.nan 

1016 dataset.ptcFitPars[ampName] = (np.repeat(np.nan, self.config.polynomialFitDegree + 1) if 

1017 ptcFitType in ["POLYNOMIAL", ] else np.repeat(np.nan, 3)) 

1018 dataset.ptcFitParsError[ampName] = (np.repeat(np.nan, self.config.polynomialFitDegree + 1) if 

1019 ptcFitType in ["POLYNOMIAL", ] else np.repeat(np.nan, 3)) 

1020 dataset.ptcFitChiSq[ampName] = np.nan 

1021 dataset.ptcTurnoff[ampName] = np.nan 

1022 dataset.finalVars[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName])) 

1023 dataset.finalModelVars[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName])) 

1024 dataset.finalMeans[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName])) 

1025 

1026 return