Coverage for python/lsst/cp/pipe/ptc/cpSolvePtcTask.py: 11%

397 statements  

« prev     ^ index     » next       coverage.py v6.4.4, created at 2022-08-25 01:44 -0700

1# This file is part of cp_pipe. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21# 

22import numpy as np 

23from collections import Counter 

24 

25import lsst.pex.config as pexConfig 

26import lsst.pipe.base as pipeBase 

27from lsst.cp.pipe.utils import (fitLeastSq, fitBootstrap, funcPolynomial, funcAstier, symmetrize) 

28 

29from scipy.signal import fftconvolve 

30from scipy.optimize import least_squares 

31from itertools import groupby 

32from operator import itemgetter 

33 

34import lsst.pipe.base.connectionTypes as cT 

35 

36from lsst.ip.isr import PhotonTransferCurveDataset 

37 

38from lsst.cp.pipe._lookupStaticCalibration import lookupStaticCalibration 

39 

40import copy 

41 

42 

43__all__ = ['PhotonTransferCurveSolveConfig', 'PhotonTransferCurveSolveTask'] 

44 

45 

46class PhotonTransferCurveSolveConnections(pipeBase.PipelineTaskConnections, 

47 dimensions=("instrument", "detector")): 

48 inputCovariances = cT.Input( 

49 name="ptcCovariances", 

50 doc="Tuple with measured covariances from flats.", 

51 storageClass="PhotonTransferCurveDataset", 

52 dimensions=("instrument", "exposure", "detector"), 

53 isCalibration=True, 

54 multiple=True, 

55 ) 

56 camera = cT.PrerequisiteInput( 

57 name="camera", 

58 doc="Camera the input data comes from.", 

59 storageClass="Camera", 

60 dimensions=("instrument",), 

61 isCalibration=True, 

62 lookupFunction=lookupStaticCalibration, 

63 ) 

64 outputPtcDataset = cT.Output( 

65 name="ptcDatsetProposal", 

66 doc="Output proposed ptc dataset.", 

67 storageClass="PhotonTransferCurveDataset", 

68 dimensions=("instrument", "detector"), 

69 multiple=False, 

70 isCalibration=True, 

71 ) 

72 

73 

74class PhotonTransferCurveSolveConfig(pipeBase.PipelineTaskConfig, 

75 pipelineConnections=PhotonTransferCurveSolveConnections): 

76 """Configuration for fitting measured covariances. 

77 """ 

78 

79 ptcFitType = pexConfig.ChoiceField( 

80 dtype=str, 

81 doc="Fit PTC to Eq. 16, Eq. 20 in Astier+19, or to a polynomial.", 

82 default="POLYNOMIAL", 

83 allowed={ 

84 "POLYNOMIAL": "n-degree polynomial (use 'polynomialFitDegree' to set 'n').", 

85 "EXPAPPROXIMATION": "Approximation in Astier+19 (Eq. 16).", 

86 "FULLCOVARIANCE": "Full covariances model in Astier+19 (Eq. 20)" 

87 } 

88 ) 

89 maximumRangeCovariancesAstier = pexConfig.Field( 

90 dtype=int, 

91 doc="Maximum range of covariances as in Astier+19", 

92 default=8, 

93 ) 

94 sigmaClipFullFitCovariancesAstier = pexConfig.Field( 

95 dtype=float, 

96 doc="sigma clip for full model fit for FULLCOVARIANCE ptcFitType ", 

97 default=5.0, 

98 ) 

99 maxIterFullFitCovariancesAstier = pexConfig.Field( 

100 dtype=int, 

101 doc="Maximum number of iterations in full model fit for FULLCOVARIANCE ptcFitType", 

102 default=3, 

103 ) 

104 polynomialFitDegree = pexConfig.Field( 

105 dtype=int, 

106 doc="Degree of polynomial to fit the PTC, when 'ptcFitType'=POLYNOMIAL.", 

107 default=3, 

108 ) 

109 sigmaCutPtcOutliers = pexConfig.Field( 

110 dtype=float, 

111 doc="Sigma cut for outlier rejection in PTC.", 

112 default=5.0, 

113 ) 

114 maxIterationsPtcOutliers = pexConfig.RangeField( 

115 dtype=int, 

116 doc="Maximum number of iterations for outlier rejection in PTC.", 

117 default=2, 

118 min=0 

119 ) 

120 minVarPivotSearch = pexConfig.Field( 

121 dtype=float, 

122 doc="The code looks for a pivot signal point after which the variance starts decreasing at high-flux" 

123 " to exclude then from the PTC model fit. However, sometimes at low fluxes, the variance" 

124 " decreases slightly. Set this variable for the variance value, in ADU^2, after which the pivot " 

125 " should be sought.", 

126 default=10000, 

127 ) 

128 consecutivePointsVarDecreases = pexConfig.RangeField( 

129 dtype=int, 

130 doc="Required number of consecutive points/fluxes in the PTC where the variance " 

131 "decreases in order to find a first estimate of the PTC turn-off. ", 

132 default=2, 

133 min=2 

134 ) 

135 doFitBootstrap = pexConfig.Field( 

136 dtype=bool, 

137 doc="Use bootstrap for the PTC fit parameters and errors?.", 

138 default=False, 

139 ) 

140 

141 

142class PhotonTransferCurveSolveTask(pipeBase.PipelineTask): 

143 """Task to fit the PTC from flat covariances. 

144 

145 The first task of the PTC measurement pipeline, 

146 ``PhotonTransferCurveMeasureTask`` (and assumed to have been run 

147 before this task), produced a list of 

148 `~lsst.ip.isr.PhotonTransferCurveDataset` objects. Each dataset 

149 contains the mean signal and covariances of the 

150 difference image of the flat-field images taken at 

151 the same exposure time. The list also contains dummy 

152 datasets (with no measurements), whose purpose is to have 

153 the input and output dimensions of ``PhotonTransferCurveMeasureTask`` 

154 match. 

155 

156 This task, ``PhotonTransferCurveSolveTask``, assembles the list 

157 of individual PTC datasets produced 

158 by ``PhotonTransferCurveMeasureTask`` into one single final PTC 

159 dataset, discarding the dummy datset as appropiate. 

160 The task fits the measured (co)variances to one of three models: 

161 a polynomial model of a given order, or the models described 

162 in equations 16 and 20 of Astier+19. These options are referred 

163 to as ``POLYNOMIAL``, ``EXPAPPROXIMATION``, and ``FULLCOVARIANCE`` 

164 in the configuration options of the task, respectively). 

165 Parameters of interest such as the gain and noise are derived 

166 from the fits. The ``FULLCOVARIANCE`` model is fitted to the 

167 full covariance data (as oppossed to the other two models, which 

168 are fit to the variance vs mean measurements only). 

169 

170 Astier+19: "The Shape of the Photon Transfer Curve 

171 of CCD sensors", arXiv:1905.08677 

172 """ 

173 

174 ConfigClass = PhotonTransferCurveSolveConfig 

175 _DefaultName = 'cpPhotonTransferCurveSolve' 

176 

177 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

178 """Ensure that the input and output dimensions are passed along. 

179 

180 Parameters 

181 ---------- 

182 butlerQC : `~lsst.daf.butler.butlerQuantumContext.ButlerQuantumContext` 

183 Butler to operate on. 

184 inputRefs : `~lsst.pipe.base.connections.InputQuantizedConnection` 

185 Input data refs to load. 

186 ouptutRefs : `~lsst.pipe.base.connections.OutputQuantizedConnection` 

187 Output data refs to persist. 

188 """ 

189 inputs = butlerQC.get(inputRefs) 

190 detId = inputRefs.inputCovariances[0].dataId['detector'] 

191 outputs = self.run(inputCovariances=inputs['inputCovariances'], camera=inputs['camera'], detId=detId) 

192 butlerQC.put(outputs, outputRefs) 

193 

194 def run(self, inputCovariances, camera=None, detId=0): 

195 """Fit measured covariances to different models. 

196 

197 Parameters 

198 ---------- 

199 inputCovariances : `list` [`lsst.ip.isr.PhotonTransferCurveDataset`] 

200 List of lsst.ip.isr.PhotonTransferCurveDataset datasets. 

201 camera : `lsst.afw.cameraGeom.Camera`, optional 

202 Input camera. 

203 detId : `int` 

204 Detector ID to locate the detector in the camera and 

205 populate the `lsst.ip.isr.PhotonTransferCurveDataset` 

206 metadata. 

207 Returns 

208 ------- 

209 results : `lsst.pipe.base.Struct` 

210 The resultins structure contains: 

211 ``outputPtcDatset`` 

212 Final PTC dataset, containing information such as the 

213 means, variances, and exposure times 

214 (`lsst.ip.isr.PhotonTransferCurveDataset`). 

215 """ 

216 # Assemble individual PTC datasets into a single PTC dataset. 

217 ampNames = np.unique(inputCovariances[0].ampNames) 

218 datasetPtc = PhotonTransferCurveDataset(ampNames=ampNames, 

219 ptcFitType=self.config.ptcFitType, 

220 covMatrixSide=self.config.maximumRangeCovariancesAstier) 

221 for partialPtcDataset in inputCovariances: 

222 # Ignore dummy datasets 

223 if partialPtcDataset.ptcFitType == 'DUMMY': 

224 continue 

225 for ampName in ampNames: 

226 datasetPtc.inputExpIdPairs[ampName].append(partialPtcDataset.inputExpIdPairs[ampName]) 

227 if type(partialPtcDataset.rawExpTimes[ampName]) is list: 

228 datasetPtc.rawExpTimes[ampName].append(partialPtcDataset.rawExpTimes[ampName][0]) 

229 else: 

230 datasetPtc.rawExpTimes[ampName].append(partialPtcDataset.rawExpTimes[ampName]) 

231 if type(partialPtcDataset.rawMeans[ampName]) is list: 

232 datasetPtc.rawMeans[ampName].append(partialPtcDataset.rawMeans[ampName][0]) 

233 else: 

234 datasetPtc.rawMeans[ampName].append(partialPtcDataset.rawMeans[ampName]) 

235 if type(partialPtcDataset.rawVars[ampName]) is list: 

236 datasetPtc.rawVars[ampName].append(partialPtcDataset.rawVars[ampName][0]) 

237 else: 

238 datasetPtc.rawVars[ampName].append(partialPtcDataset.rawVars[ampName]) 

239 if type(partialPtcDataset.expIdMask[ampName]) is list: 

240 datasetPtc.expIdMask[ampName].append(partialPtcDataset.expIdMask[ampName][0]) 

241 else: 

242 datasetPtc.expIdMask[ampName].append(partialPtcDataset.expIdMask[ampName]) 

243 datasetPtc.covariances[ampName].append(np.array(partialPtcDataset.covariances[ampName][0])) 

244 datasetPtc.covariancesSqrtWeights[ampName].append( 

245 np.array(partialPtcDataset.covariancesSqrtWeights[ampName][0])) 

246 # Sort arrays that are filled so far in the final dataset by 

247 # rawMeans index 

248 for ampName in ampNames: 

249 index = np.argsort(np.ravel(np.array(datasetPtc.rawMeans[ampName]))) 

250 datasetPtc.inputExpIdPairs[ampName] = np.array(datasetPtc.inputExpIdPairs[ampName])[index] 

251 datasetPtc.rawExpTimes[ampName] = np.array(datasetPtc.rawExpTimes[ampName])[index] 

252 datasetPtc.rawMeans[ampName] = np.array(datasetPtc.rawMeans[ampName])[index] 

253 datasetPtc.rawVars[ampName] = np.array(datasetPtc.rawVars[ampName])[index] 

254 datasetPtc.expIdMask[ampName] = np.array(datasetPtc.expIdMask[ampName])[index] 

255 datasetPtc.covariances[ampName] = np.array(datasetPtc.covariances[ampName])[index] 

256 datasetPtc.covariancesSqrtWeights[ampName] = np.array( 

257 datasetPtc.covariancesSqrtWeights[ampName])[index] 

258 if self.config.ptcFitType == "FULLCOVARIANCE": 

259 # Fit the measured covariances vs mean signal to 

260 # the Astier+19 full model (Eq. 20). Before that 

261 # do a preliminary fit to the variance (C_00) vs mean 

262 # signal (mu) curve using the EXPAPPROXIMATION model 

263 # (Eq. 16 in Astier+19) in order to 

264 # get the flat pairs that are masked. The 

265 # points at these fluxes will also be masked when 

266 # calculating the other elements of the covariance 

267 # matrix, C_ij, i!=j). 

268 

269 # Preliminary fit, usign a temp dataset to get the mask 

270 tempDatasetPtc = copy.copy(datasetPtc) 

271 tempDatasetPtc.ptcFitType = "EXPAPPROXIMATION" 

272 tempDatasetPtc = self.fitMeasurementsToModel(tempDatasetPtc) 

273 

274 # "FULLCOVARIANCE", using the mask obtained from the 

275 # previous fit. 

276 for ampName in datasetPtc.ampNames: 

277 datasetPtc.expIdMask[ampName] = tempDatasetPtc.expIdMask[ampName] 

278 datasetPtc.fitType = "FULLCOVARIANCE" 

279 datasetPtc = self.fitMeasurementsToModel(datasetPtc) 

280 # The other options are: self.config.ptcFitType in 

281 # ("EXPAPPROXIMATION", "POLYNOMIAL") 

282 else: 

283 # Fit the PTC to a polynomial or to Astier+19 exponential 

284 # approximation (Eq. 16). Fill up 

285 # PhotonTransferCurveDataset object. 

286 datasetPtc = self.fitMeasurementsToModel(datasetPtc) 

287 

288 if camera: 

289 detector = camera[detId] 

290 else: 

291 detector = None 

292 datasetPtc.updateMetadata(setDate=True, camera=camera, detector=detector) 

293 

294 return pipeBase.Struct( 

295 outputPtcDataset=datasetPtc, 

296 ) 

297 

298 def fitMeasurementsToModel(self, dataset): 

299 """Fit the measured covariances vs mean signal to a 

300 polynomial or one of the models in Astier+19 

301 (Eq. 16 or Eq.20). 

302 

303 Parameters 

304 ---------- 

305 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset` 

306 The dataset containing information such as the means, 

307 (co)variances, and exposure times. 

308 

309 Returns 

310 ------- 

311 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset` 

312 This is the same dataset as the input parameter, however, 

313 it has been modified to include information such as the 

314 fit vectors and the fit parameters. See the class 

315 `PhotonTransferCurveDatase`. 

316 """ 

317 fitType = dataset.ptcFitType 

318 if fitType in ["FULLCOVARIANCE", ]: 

319 # This model uses the full covariance matrix in the fit. 

320 # The PTC is technically defined as variance vs signal, 

321 # with variance = Cov_00 

322 dataset = self.fitDataFullCovariance(dataset) 

323 elif fitType in ["POLYNOMIAL", "EXPAPPROXIMATION"]: 

324 # The PTC is technically defined as variance vs signal 

325 dataset = self.fitPtc(dataset) 

326 else: 

327 raise RuntimeError( 

328 f"Fitting option {fitType} not one of " 

329 "'POLYNOMIAL', 'EXPAPPROXIMATION', or 'FULLCOVARIANCE'" 

330 ) 

331 

332 return dataset 

333 

334 def fitDataFullCovariance(self, dataset): 

335 """Fit measured flat covariances to the full model in 

336 Astier+19 (Eq. 20). 

337 

338 Parameters 

339 ---------- 

340 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset` 

341 The dataset containing information such as the means, 

342 (co)variances, and exposure times. 

343 

344 Returns 

345 ------- 

346 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset` 

347 This is the same dataset as the input parameter, however, 

348 it has been modified to include information such as the 

349 fit vectors and the fit parameters. See the class 

350 `PhotonTransferCurveDatase`. 

351 

352 Notes 

353 ----- 

354 The parameters of the full model for C_ij(mu) ("C_ij" and "mu" 

355 in ADU^2 and ADU, respectively) in Astier+19 (Eq. 20) are: 

356 "a" coefficients (r by r matrix), units: 1/e 

357 "b" coefficients (r by r matrix), units: 1/e 

358 noise matrix (r by r matrix), units: e^2 

359 gain, units: e/ADU 

360 "b" appears in Eq. 20 only through the "ab" combination, which 

361 is defined in this code as "c=ab". 

362 

363 Total number of parameters: #entries(a) + #entries(c) + #entries(noise) 

364 + 1. This is equivalent to r^2 + r^2 + r^2 + 1, where "r" is the 

365 maximum lag considered for the covariances calculation, and the 

366 extra "1" is the gain. If "b" is 0, then "c" is 0, and len(pInit) will 

367 have r^2 fewer entries. 

368 """ 

369 matrixSide = self.config.maximumRangeCovariancesAstier 

370 lenParams = matrixSide*matrixSide 

371 

372 for ampName in dataset.ampNames: 

373 lenInputTimes = len(dataset.rawExpTimes[ampName]) 

374 # Not used when ptcFitType is 'FULLCOVARIANCE' 

375 dataset.ptcFitPars[ampName] = [np.nan] 

376 dataset.ptcFitParsError[ampName] = [np.nan] 

377 dataset.ptcFitChiSq[ampName] = np.nan 

378 

379 if ampName in dataset.badAmps: 

380 # Bad amp 

381 # Entries need to have proper dimensions so read/write 

382 # with astropy.Table works. 

383 nanMatrix = np.full((matrixSide, matrixSide), np.nan) 

384 listNanMatrix = np.full((lenInputTimes, matrixSide, matrixSide), np.nan) 

385 dataset.covariancesModel[ampName] = listNanMatrix 

386 dataset.covariancesSqrtWeights[ampName] = listNanMatrix 

387 dataset.aMatrix[ampName] = nanMatrix 

388 dataset.bMatrix[ampName] = nanMatrix 

389 dataset.covariancesModelNoB[ampName] = listNanMatrix 

390 dataset.aMatrixNoB[ampName] = nanMatrix 

391 

392 dataset.expIdMask[ampName] = np.repeat(False, lenInputTimes) 

393 dataset.gain[ampName] = np.nan 

394 dataset.gainErr[ampName] = np.nan 

395 dataset.noise[ampName] = np.nan 

396 dataset.noiseErr[ampName] = np.nan 

397 dataset.finalVars[ampName] = np.repeat(np.nan, lenInputTimes) 

398 dataset.finalModelVars[ampName] = np.repeat(np.nan, lenInputTimes) 

399 dataset.finalMeans[ampName] = np.repeat(np.nan, lenInputTimes) 

400 continue 

401 

402 muAtAmp = dataset.rawMeans[ampName] 

403 maskAtAmp = dataset.expIdMask[ampName] 

404 if len(maskAtAmp) == 0: 

405 maskAtAmp = np.repeat(True, len(muAtAmp)) 

406 

407 muAtAmp = muAtAmp[maskAtAmp] 

408 covAtAmp = np.nan_to_num(dataset.covariances[ampName])[maskAtAmp] 

409 covSqrtWeightsAtAmp = np.nan_to_num(dataset.covariancesSqrtWeights[ampName])[maskAtAmp] 

410 

411 # Initial fit, to approximate parameters, with c=0 

412 a0, c0, noise0, gain0 = self.initialFitFullCovariance(muAtAmp, covAtAmp, covSqrtWeightsAtAmp) 

413 

414 # Fit full model (Eq. 20 of Astier+19) and same model with 

415 # b=0 (c=0 in this code) 

416 pInit = np.concatenate((a0.flatten(), c0.flatten(), noise0.flatten(), np.array(gain0)), axis=None) 

417 functionsDict = {'fullModel': self.funcFullCovarianceModel, 

418 'fullModelNoB': self.funcFullCovarianceModelNoB} 

419 fitResults = {'fullModel': {'a': [], 'c': [], 'noise': [], 'gain': [], 'paramsErr': []}, 

420 'fullModelNoB': {'a': [], 'c': [], 'noise': [], 'gain': [], 'paramsErr': []}} 

421 for key in functionsDict: 

422 params, paramsErr, _ = fitLeastSq(pInit, muAtAmp, 

423 covAtAmp.flatten(), functionsDict[key], 

424 weightsY=covSqrtWeightsAtAmp.flatten()) 

425 a = params[:lenParams].reshape((matrixSide, matrixSide)) 

426 c = params[lenParams:2*lenParams].reshape((matrixSide, matrixSide)) 

427 noise = params[2*lenParams:3*lenParams].reshape((matrixSide, matrixSide)) 

428 gain = params[-1] 

429 

430 fitResults[key]['a'] = a 

431 fitResults[key]['c'] = c 

432 fitResults[key]['noise'] = noise 

433 fitResults[key]['gain'] = gain 

434 fitResults[key]['paramsErr'] = paramsErr 

435 

436 # Put the information in the PTC dataset 

437 

438 # Not used when ptcFitType is 'FULLCOVARIANCE' 

439 dataset.ptcFitPars[ampName] = [np.nan] 

440 dataset.ptcFitParsError[ampName] = [np.nan] 

441 dataset.ptcFitChiSq[ampName] = np.nan 

442 

443 # Save full covariances, covariances models, and their weights. 

444 # dataset.expIdMask is already full, but needs to be 

445 # converted to bool. 

446 dataset.expIdMask[ampName] = np.array(dataset.expIdMask[ampName], dtype=bool) 

447 dataset.covariances[ampName] = covAtAmp 

448 dataset.covariancesModel[ampName] = self.evalCovModel(muAtAmp, 

449 fitResults['fullModel']['a'], 

450 fitResults['fullModel']['c'], 

451 fitResults['fullModel']['noise'], 

452 fitResults['fullModel']['gain']) 

453 dataset.covariancesSqrtWeights[ampName] = covSqrtWeightsAtAmp 

454 dataset.aMatrix[ampName] = fitResults['fullModel']['a'] 

455 dataset.bMatrix[ampName] = fitResults['fullModel']['c']/fitResults['fullModel']['a'] 

456 dataset.covariancesModelNoB[ampName] = self.evalCovModel(muAtAmp, 

457 fitResults['fullModelNoB']['a'], 

458 fitResults['fullModelNoB']['c'], 

459 fitResults['fullModelNoB']['noise'], 

460 fitResults['fullModelNoB']['gain'], 

461 setBtoZero=True) 

462 dataset.aMatrixNoB[ampName] = fitResults['fullModelNoB']['a'] 

463 dataset.gain[ampName] = fitResults['fullModel']['gain'] 

464 dataset.gainErr[ampName] = fitResults['fullModel']['paramsErr'][-1] 

465 readoutNoise = fitResults['fullModel']['noise'][0][0] 

466 readoutNoiseSqrt = np.sqrt(np.fabs(readoutNoise)) 

467 dataset.noise[ampName] = readoutNoise 

468 readoutNoiseSigma = fitResults['fullModel']['paramsErr'][2*lenParams] 

469 dataset.noiseErr[ampName] = 0.5*(readoutNoiseSigma/np.fabs(readoutNoise))*readoutNoiseSqrt 

470 dataset.finalVars[ampName] = covAtAmp[:, 0, 0] 

471 dataset.finalModelVars[ampName] = dataset.covariancesModel[ampName][:, 0, 0] 

472 dataset.finalMeans[ampName] = muAtAmp 

473 

474 return dataset 

475 

476 def initialFitFullCovariance(self, mu, cov, sqrtW): 

477 """ Performs a crude parabolic fit of the data in order to start 

478 the full fit close to the solution, setting b=0 (c=0) in Eq. 20 

479 of Astier+19. 

480 

481 Parameters 

482 ---------- 

483 mu : `numpy.array`, (N,) 

484 Signal `mu` (ADU) 

485 cov : `numpy.array`, (N, M, M) 

486 Covariance arrays of size `(M, M)` (with 

487 `M = config.maximumRangeCovariancesAstier`), 

488 indexed by mean signal `mu`. 

489 sqrtW : `numpy.array`, (N,) 

490 Covariance weights, defined as 1./sqrt(Variances) 

491 

492 Returns 

493 ------- 

494 a : `numpy.array`, (M, M) 

495 "a" parameter per flux in Eq. 20 of Astier+19. 

496 c : `numpy.array`, (M, M) 

497 "c"="ab" parameter per flux in Eq. 20 of Astier+19. 

498 noise : `numpy.array`, (M, M) 

499 "noise" parameter per flux in Eq. 20 of Astier+19. 

500 gain : `float` 

501 Amplifier gain (e/ADU) 

502 """ 

503 matrixSide = self.config.maximumRangeCovariancesAstier 

504 

505 # Initialize fit parameters 

506 a = np.zeros((matrixSide, matrixSide)) 

507 c = np.zeros((matrixSide, matrixSide)) 

508 noise = np.zeros((matrixSide, matrixSide)) 

509 gain = 1. 

510 

511 # iterate the fit to account for higher orders 

512 # the chi2 does not necessarily go down, so one could 

513 # stop when it increases 

514 oldChi2 = 1e30 

515 for _ in range(5): 

516 model = np.nan_to_num(self.evalCovModel(mu, a, c, noise, gain, setBtoZero=True)) 

517 # loop on lags 

518 for i in range(matrixSide): 

519 for j in range(matrixSide): 

520 # fit a parabola for a given lag 

521 parsFit = np.polyfit(mu, cov[:, i, j] - model[:, i, j], 

522 2, w=sqrtW[:, i, j]) 

523 # model equation (Eq. 20) in Astier+19, with c=a*b=0: 

524 a[i, j] += parsFit[0] 

525 noise[i, j] += parsFit[2] 

526 if(i + j == 0): 

527 gain = 1./(1/gain+parsFit[1]) 

528 weightedRes = (model - cov)*sqrtW 

529 chi2 = (weightedRes.flatten()**2).sum() 

530 if chi2 > oldChi2: 

531 break 

532 oldChi2 = chi2 

533 

534 return a, c, noise, gain 

535 

536 def funcFullCovarianceModel(self, params, x): 

537 """Model to fit covariances from flat fields; Equation 20 of 

538 Astier+19. 

539 

540 Parameters 

541 ---------- 

542 params : `list` 

543 Parameters of the model: aMatrix, CMatrix, noiseMatrix, 

544 gain (e/ADU). 

545 x : `numpy.array`, (N,) 

546 Signal `mu` (ADU) 

547 

548 Returns 

549 ------- 

550 y : `numpy.array`, (N,) 

551 Covariance matrix. 

552 """ 

553 matrixSide = self.config.maximumRangeCovariancesAstier 

554 lenParams = matrixSide*matrixSide 

555 aMatrix = params[:lenParams].reshape((matrixSide, matrixSide)) 

556 cMatrix = params[lenParams:2*lenParams].reshape((matrixSide, matrixSide)) 

557 noiseMatrix = params[2*lenParams:3*lenParams].reshape((matrixSide, matrixSide)) 

558 gain = params[-1] 

559 

560 return self.evalCovModel(x, aMatrix, cMatrix, noiseMatrix, gain).flatten() 

561 

562 def funcFullCovarianceModelNoB(self, params, x): 

563 """Model to fit covariances from flat fields; Equation 20 of 

564 Astier+19, with b=0 (equivalent to c=a*b=0 in this code). 

565 

566 Parameters 

567 ---------- 

568 params : `list` 

569 Parameters of the model: aMatrix, CMatrix, noiseMatrix, 

570 gain (e/ADU). 

571 x : `numpy.array`, (N,) 

572 Signal mu (ADU) 

573 

574 Returns 

575 ------- 

576 y : `numpy.array`, (N,) 

577 Covariance matrix. 

578 """ 

579 matrixSide = self.config.maximumRangeCovariancesAstier 

580 lenParams = matrixSide*matrixSide 

581 aMatrix = params[:lenParams].reshape((matrixSide, matrixSide)) 

582 cMatrix = params[lenParams:2*lenParams].reshape((matrixSide, matrixSide)) 

583 noiseMatrix = params[2*lenParams:3*lenParams].reshape((matrixSide, matrixSide)) 

584 gain = params[-1] 

585 

586 return self.evalCovModel(x, aMatrix, cMatrix, noiseMatrix, gain, setBtoZero=True).flatten() 

587 

588 def evalCovModel(self, mu, aMatrix, cMatrix, noiseMatrix, gain, setBtoZero=False): 

589 """Computes full covariances model (Eq. 20 of Astier+19). 

590 

591 Parameters 

592 ---------- 

593 mu : `numpy.array`, (N,) 

594 List of mean signals. 

595 aMatrix : `numpy.array`, (M, M) 

596 "a" parameter per flux in Eq. 20 of Astier+19. 

597 cMatrix : `numpy.array`, (M, M) 

598 "c"="ab" parameter per flux in Eq. 20 of Astier+19. 

599 noiseMatrix : `numpy.array`, (M, M) 

600 "noise" parameter per flux in Eq. 20 of Astier+19. 

601 gain : `float` 

602 Amplifier gain (e/ADU) 

603 setBtoZero=False : `bool`, optional 

604 Set "b" parameter in full model (see Astier+19) to zero. 

605 

606 Returns 

607 ------- 

608 covModel : `numpy.array`, (N, M, M) 

609 Covariances model. 

610 

611 Notes 

612 ----- 

613 By default, computes the covModel for the mu's stored(self.mu). 

614 Returns cov[Nmu, M, M]. The variance for the PTC is 

615 cov[:, 0, 0]. mu and cov are in ADUs and ADUs squared. To use 

616 electrons for both, the gain should be set to 1. This routine 

617 implements the model in Astier+19 (1905.08677). 

618 The parameters of the full model for C_ij(mu) ("C_ij" and "mu" 

619 in ADU^2 and ADU, respectively) in Astier+19 (Eq. 20) are: 

620 "a" coefficients (M by M matrix), units: 1/e 

621 "b" coefficients (M by M matrix), units: 1/e 

622 noise matrix (M by M matrix), units: e^2 

623 gain, units: e/ADU 

624 "b" appears in Eq. 20 only through the "ab" combination, which 

625 is defined in this code as "c=ab". 

626 """ 

627 matrixSide = self.config.maximumRangeCovariancesAstier 

628 sa = (matrixSide, matrixSide) 

629 # pad a with zeros and symmetrize 

630 aEnlarged = np.zeros((int(sa[0]*1.5)+1, int(sa[1]*1.5)+1)) 

631 aEnlarged[0:sa[0], 0:sa[1]] = aMatrix 

632 aSym = symmetrize(aEnlarged) 

633 # pad c with zeros and symmetrize 

634 cEnlarged = np.zeros((int(sa[0]*1.5)+1, int(sa[1]*1.5)+1)) 

635 cEnlarged[0:sa[0], 0:sa[1]] = cMatrix 

636 cSym = symmetrize(cEnlarged) 

637 a2 = fftconvolve(aSym, aSym, mode='same') 

638 a3 = fftconvolve(a2, aSym, mode='same') 

639 ac = fftconvolve(aSym, cSym, mode='same') 

640 (xc, yc) = np.unravel_index(np.abs(aSym).argmax(), a2.shape) 

641 

642 a1 = aMatrix[np.newaxis, :, :] 

643 a2 = a2[np.newaxis, xc:xc + matrixSide, yc:yc + matrixSide] 

644 a3 = a3[np.newaxis, xc:xc + matrixSide, yc:yc + matrixSide] 

645 ac = ac[np.newaxis, xc:xc + matrixSide, yc:yc + matrixSide] 

646 c1 = cMatrix[np.newaxis, ::] 

647 

648 # assumes that mu is 1d 

649 bigMu = mu[:, np.newaxis, np.newaxis]*gain 

650 # c(=a*b in Astier+19) also has a contribution to the last 

651 # term, that is absent for now. 

652 if setBtoZero: 

653 c1 = np.zeros_like(c1) 

654 ac = np.zeros_like(ac) 

655 covModel = (bigMu/(gain*gain)*(a1*bigMu+2./3.*(bigMu*bigMu)*(a2 + c1) 

656 + (1./3.*a3 + 5./6.*ac)*(bigMu*bigMu*bigMu)) + noiseMatrix[np.newaxis, :, :]/gain**2) 

657 # add the Poisson term, and the read out noise (variance) 

658 covModel[:, 0, 0] += mu/gain 

659 

660 return covModel 

661 

662 # EXPAPPROXIMATION and POLYNOMIAL fit methods 

663 @staticmethod 

664 def _initialParsForPolynomial(order): 

665 assert(order >= 2) 

666 pars = np.zeros(order, dtype=float) 

667 pars[0] = 10 

668 pars[1] = 1 

669 pars[2:] = 0.0001 

670 return pars 

671 

672 @staticmethod 

673 def _boundsForPolynomial(initialPars, lowers=[], uppers=[]): 

674 if not len(lowers): 

675 lowers = [np.NINF for p in initialPars] 

676 if not len(uppers): 

677 uppers = [np.inf for p in initialPars] 

678 lowers[1] = 0 # no negative gains 

679 return (lowers, uppers) 

680 

681 @staticmethod 

682 def _boundsForAstier(initialPars, lowers=[], uppers=[]): 

683 if not len(lowers): 

684 lowers = [np.NINF for p in initialPars] 

685 if not len(uppers): 

686 uppers = [np.inf for p in initialPars] 

687 return (lowers, uppers) 

688 

689 @staticmethod 

690 def _getInitialGoodPoints(means, variances, minVarPivotSearch, consecutivePointsVarDecreases): 

691 """Return a boolean array to mask bad points. 

692 

693 Parameters 

694 ---------- 

695 means : `numpy.array` 

696 Input array with mean signal values. 

697 variances : `numpy.array` 

698 Input array with variances at each mean value. 

699 minVarPivotSearch : `float` 

700 The variance (in ADU^2), above which, the point 

701 of decreasing variance should be sought. 

702 consecutivePointsVarDecreases : `int` 

703 Required number of consecutive points/fluxes 

704 in the PTC where the variance 

705 decreases in order to find a first 

706 estimate of the PTC turn-off. 

707 

708 Returns 

709 ------ 

710 goodPoints : `numpy.array` [`bool`] 

711 Boolean array to select good (`True`) and bad (`False`) 

712 points. 

713 

714 Notes 

715 ----- 

716 Eliminate points beyond which the variance decreases. 

717 """ 

718 goodPoints = np.ones_like(means, dtype=bool) 

719 # Variances are sorted and should monotonically increase 

720 pivotList = np.where(np.array(np.diff(variances)) < 0)[0] 

721 if len(pivotList) > 0: 

722 # For small values, sometimes the variance decreases slightly 

723 # Only look when var > self.config.minVarPivotSearch 

724 pivotList = [p for p in pivotList if variances[p] > minVarPivotSearch] 

725 # Require that the varince decreases during 

726 # consecutivePointsVarDecreases 

727 # consecutive points. This will give a first 

728 # estimate of the PTC turn-off, which 

729 # may be updated (reduced) further in the code. 

730 if len(pivotList) > 1: 

731 # enumerate(pivotList) creates tuples (index, value), for 

732 # each value in pivotList. The lambda function subtracts 

733 # each value from the index. 

734 # groupby groups elements by equal key value. 

735 for k, g in groupby(enumerate(pivotList), lambda x: x[0]-x[1]): 

736 group = (map(itemgetter(1), g)) 

737 # Form groups of consecute values from pivotList 

738 group = list(map(int, group)) 

739 # values in pivotList are indices where np.diff(variances) 

740 # is negative, i.e., where the variance starts decreasing. 

741 # Find the first group of consecutive numbers when 

742 # variance decreases. 

743 if len(group) >= consecutivePointsVarDecreases: 

744 pivotIndex = np.min(group) 

745 goodPoints[pivotIndex+1:] = False 

746 break 

747 

748 return goodPoints 

749 

750 def _makeZeroSafe(self, array, substituteValue=1e-9): 

751 """""" 

752 array = np.array(array) 

753 nBad = Counter(np.ravel(array))[0] 

754 if nBad == 0: 

755 return array 

756 

757 index, = np.where(array == 0) 

758 if len(index): 

759 msg = f"Found {nBad} zeros in array at elements {index}" 

760 self.log.warning(msg) 

761 

762 array[index] = substituteValue 

763 

764 return array 

765 

766 def fitPtc(self, dataset): 

767 """Fit the photon transfer curve to a polynomial or to the 

768 Astier+19 approximation (Eq. 16). 

769 

770 Fit the photon transfer curve with either a polynomial of 

771 the order specified in the task config, or using the 

772 exponential approximation in Astier+19 (Eq. 16). 

773 

774 Sigma clipping is performed iteratively for the fit, as 

775 well as an initial clipping of data points that are more 

776 than `config.initialNonLinearityExclusionThreshold` away 

777 from lying on a straight line. This other step is necessary 

778 because the photon transfer curve turns over catastrophically 

779 at very high flux (because saturation 

780 drops the variance to ~0) and these far outliers cause the 

781 initial fit to fail, meaning the sigma cannot be calculated 

782 to perform the sigma-clipping. 

783 

784 Parameters 

785 ---------- 

786 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset` 

787 The dataset containing the means, variances and 

788 exposure times. 

789 

790 Returns 

791 ------- 

792 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset` 

793 This is the same dataset as the input parameter, however, 

794 it has been modified to include information such as the 

795 fit vectors and the fit parameters. See the class 

796 `PhotonTransferCurveDatase`. 

797 

798 Raises 

799 ------ 

800 RuntimeError: 

801 Raises if dataset.ptcFitType is None or empty. 

802 """ 

803 if dataset.ptcFitType: 

804 ptcFitType = dataset.ptcFitType 

805 else: 

806 raise RuntimeError("ptcFitType is None of empty in PTC dataset.") 

807 matrixSide = self.config.maximumRangeCovariancesAstier 

808 nanMatrix = np.empty((matrixSide, matrixSide)) 

809 nanMatrix[:] = np.nan 

810 

811 for amp in dataset.ampNames: 

812 lenInputTimes = len(dataset.rawExpTimes[amp]) 

813 listNanMatrix = np.empty((lenInputTimes, matrixSide, matrixSide)) 

814 listNanMatrix[:] = np.nan 

815 

816 dataset.covariancesModel[amp] = listNanMatrix 

817 dataset.aMatrix[amp] = nanMatrix 

818 dataset.bMatrix[amp] = nanMatrix 

819 dataset.covariancesModelNoB[amp] = listNanMatrix 

820 dataset.aMatrixNoB[amp] = nanMatrix 

821 

822 def errFunc(p, x, y): 

823 return ptcFunc(p, x) - y 

824 

825 sigmaCutPtcOutliers = self.config.sigmaCutPtcOutliers 

826 maxIterationsPtcOutliers = self.config.maxIterationsPtcOutliers 

827 

828 for i, ampName in enumerate(dataset.ampNames): 

829 timeVecOriginal = np.ravel(np.array(dataset.rawExpTimes[ampName])) 

830 meanVecOriginal = np.ravel(np.array(dataset.rawMeans[ampName])) 

831 varVecOriginal = np.ravel(np.array(dataset.rawVars[ampName])) 

832 varVecOriginal = self._makeZeroSafe(varVecOriginal) 

833 

834 # Discard points when the variance starts to decrease after two 

835 # consecutive signal levels 

836 goodPoints = self._getInitialGoodPoints(meanVecOriginal, varVecOriginal, 

837 self.config.minVarPivotSearch, 

838 self.config.consecutivePointsVarDecreases) 

839 

840 # Check if all points are bad from the 'cpExtractPtcTask' 

841 initialExpIdMask = np.ravel(np.array(dataset.expIdMask[ampName])) 

842 

843 if not (goodPoints.any() and initialExpIdMask.any()): 

844 msg = (f"SERIOUS: All points in goodPoints: {goodPoints} or " 

845 f"in initialExpIdMask: {initialExpIdMask} are bad." 

846 f"Setting {ampName} to BAD.") 

847 self.log.warning(msg) 

848 # Fill entries with NaNs 

849 self.fillBadAmp(dataset, ptcFitType, ampName) 

850 continue 

851 

852 # Save the point where the variance starts decreasing as the 

853 # PTC turnoff point 

854 ptcTurnoff = meanVecOriginal[goodPoints][-1] 

855 dataset.ptcTurnoff[ampName] = ptcTurnoff 

856 

857 mask = goodPoints 

858 

859 if ptcFitType == 'EXPAPPROXIMATION': 

860 ptcFunc = funcAstier 

861 parsIniPtc = [-1e-9, 1.0, 10.] # a00, gain, noise^2 

862 # lowers and uppers obtained from BOT data studies by 

863 # C. Lage (UC Davis, 11/2020). 

864 bounds = self._boundsForAstier(parsIniPtc, lowers=[-1e-4, 0.5, -2000], 

865 uppers=[1e-4, 2.5, 2000]) 

866 if ptcFitType == 'POLYNOMIAL': 

867 ptcFunc = funcPolynomial 

868 parsIniPtc = self._initialParsForPolynomial(self.config.polynomialFitDegree + 1) 

869 bounds = self._boundsForPolynomial(parsIniPtc) 

870 

871 # Before bootstrap fit, do an iterative fit to get rid of outliers. 

872 # This further process of outlier rejection be skipped 

873 # if self.config.maxIterationsPtcOutliers = 0. 

874 # We already did some initial outlier rejection above in 

875 # self._getInitialGoodPoints. 

876 count = 1 

877 newMask = np.ones_like(meanVecOriginal, dtype=bool) 

878 pars = parsIniPtc 

879 while count <= maxIterationsPtcOutliers: 

880 # Note that application of the mask actually shrinks the array 

881 # to size rather than setting elements to zero (as we want) so 

882 # always update mask itself and re-apply to the original data 

883 meanTempVec = meanVecOriginal[mask] 

884 varTempVec = varVecOriginal[mask] 

885 res = least_squares(errFunc, parsIniPtc, bounds=bounds, args=(meanTempVec, varTempVec)) 

886 pars = res.x 

887 

888 # change this to the original from the temp because 

889 # the masks are ANDed meaning once a point is masked 

890 # it's always masked, and the masks must always be the 

891 # same length for broadcasting 

892 sigResids = (varVecOriginal - ptcFunc(pars, meanVecOriginal))/np.sqrt(varVecOriginal) 

893 newMask = np.array([True if np.abs(r) < sigmaCutPtcOutliers else False for r in sigResids]) 

894 mask = mask & newMask 

895 if not (mask.any() and newMask.any()): 

896 msg = (f"SERIOUS: All points in either mask: {mask} or newMask: {newMask} are bad. " 

897 f"Setting {ampName} to BAD.") 

898 self.log.warning(msg) 

899 # Fill entries with NaNs 

900 self.fillBadAmp(dataset, ptcFitType, ampName) 

901 break 

902 nDroppedTotal = Counter(mask)[False] 

903 self.log.debug("Iteration %d: discarded %d points in total for %s", 

904 count, nDroppedTotal, ampName) 

905 count += 1 

906 # objects should never shrink 

907 assert (len(mask) == len(timeVecOriginal) == len(meanVecOriginal) == len(varVecOriginal)) 

908 if not (mask.any() and newMask.any()): 

909 continue 

910 dataset.expIdMask[ampName] = np.array(dataset.expIdMask[ampName]) 

911 # store the final mask 

912 if len(dataset.expIdMask[ampName]): 

913 dataset.expIdMask[ampName] &= mask # bitwise_and if there is already a mask 

914 else: 

915 dataset.expIdMask[ampName] = mask 

916 # In case there was a previous mask stored 

917 mask = dataset.expIdMask[ampName] 

918 parsIniPtc = pars 

919 meanVecFinal = meanVecOriginal[mask] 

920 varVecFinal = varVecOriginal[mask] 

921 

922 if Counter(mask)[False] > 0: 

923 self.log.info("Number of points discarded in PTC of amplifier %s:" 

924 " %d out of %d", ampName, Counter(mask)[False], len(meanVecOriginal)) 

925 

926 if (len(meanVecFinal) < len(parsIniPtc)): 

927 msg = (f"SERIOUS: Not enough data points ({len(meanVecFinal)}) compared to the number of " 

928 f"parameters of the PTC model({len(parsIniPtc)}). Setting {ampName} to BAD.") 

929 self.log.warning(msg) 

930 # Fill entries with NaNs 

931 self.fillBadAmp(dataset, ptcFitType, ampName) 

932 continue 

933 # Fit the PTC. 

934 # The variance of the variance is Var(v)=2*v^2/Npix. This is 

935 # already calculated in `makeCovArray` of CpPtcExtract. 

936 # dataset.covariancesSqrtWeights[ampName][:,0,0] 

937 # has 1/sqrt(Var(v)). 

938 weightsY = dataset.covariancesSqrtWeights[ampName][:, 0, 0][mask] 

939 if self.config.doFitBootstrap: 

940 parsFit, parsFitErr, reducedChiSqPtc = fitBootstrap(parsIniPtc, meanVecFinal, 

941 varVecFinal, ptcFunc, 

942 weightsY=weightsY) 

943 else: 

944 parsFit, parsFitErr, reducedChiSqPtc = fitLeastSq(parsIniPtc, meanVecFinal, 

945 varVecFinal, ptcFunc, 

946 weightsY=weightsY) 

947 dataset.ptcFitPars[ampName] = parsFit 

948 dataset.ptcFitParsError[ampName] = parsFitErr 

949 dataset.ptcFitChiSq[ampName] = reducedChiSqPtc 

950 # Masked variances (measured and modeled) and means. Need 

951 # to pad the array so astropy.Table does not crash (the 

952 # mask may vary per amp). 

953 padLength = len(dataset.rawExpTimes[ampName]) - len(varVecFinal) 

954 dataset.finalVars[ampName] = np.pad(varVecFinal, (0, padLength), 'constant', 

955 constant_values=np.nan) 

956 dataset.finalModelVars[ampName] = np.pad(ptcFunc(parsFit, meanVecFinal), (0, padLength), 

957 'constant', constant_values=np.nan) 

958 dataset.finalMeans[ampName] = np.pad(meanVecFinal, (0, padLength), 'constant', 

959 constant_values=np.nan) 

960 if ptcFitType == 'EXPAPPROXIMATION': 

961 ptcGain = parsFit[1] 

962 ptcGainErr = parsFitErr[1] 

963 ptcNoise = np.sqrt(np.fabs(parsFit[2])) 

964 ptcNoiseErr = 0.5*(parsFitErr[2]/np.fabs(parsFit[2]))*np.sqrt(np.fabs(parsFit[2])) 

965 if ptcFitType == 'POLYNOMIAL': 

966 ptcGain = 1./parsFit[1] 

967 ptcGainErr = np.fabs(1./parsFit[1])*(parsFitErr[1]/parsFit[1]) 

968 ptcNoise = np.sqrt(np.fabs(parsFit[0]))*ptcGain 

969 ptcNoiseErr = (0.5*(parsFitErr[0]/np.fabs(parsFit[0]))*(np.sqrt(np.fabs(parsFit[0]))))*ptcGain 

970 dataset.gain[ampName] = ptcGain 

971 dataset.gainErr[ampName] = ptcGainErr 

972 dataset.noise[ampName] = ptcNoise 

973 dataset.noiseErr[ampName] = ptcNoiseErr 

974 

975 if not len(dataset.ptcFitType) == 0: 

976 dataset.ptcFitType = ptcFitType 

977 if len(dataset.badAmps) == 0: 

978 dataset.badAmps = np.repeat(np.nan, len(list(dataset.rawExpTimes.values())[0])) 

979 

980 return dataset 

981 

982 def fillBadAmp(self, dataset, ptcFitType, ampName): 

983 """Fill the dataset with NaNs if there are not enough 

984 good points. 

985 

986 Parameters 

987 ---------- 

988 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset` 

989 The dataset containing the means, variances and 

990 exposure times. 

991 ptcFitType : {'POLYNOMIAL', 'EXPAPPROXIMATION'} 

992 Fit a 'POLYNOMIAL' (degree: 'polynomialFitDegree') or 

993 'EXPAPPROXIMATION' (Eq. 16 of Astier+19) to the PTC. 

994 ampName : `str` 

995 Amplifier name. 

996 """ 

997 dataset.badAmps.append(ampName) 

998 dataset.expIdMask[ampName] = np.repeat(False, len(dataset.rawExpTimes[ampName])) 

999 dataset.gain[ampName] = np.nan 

1000 dataset.gainErr[ampName] = np.nan 

1001 dataset.noise[ampName] = np.nan 

1002 dataset.noiseErr[ampName] = np.nan 

1003 dataset.ptcFitPars[ampName] = (np.repeat(np.nan, self.config.polynomialFitDegree + 1) if 

1004 ptcFitType in ["POLYNOMIAL", ] else np.repeat(np.nan, 3)) 

1005 dataset.ptcFitParsError[ampName] = (np.repeat(np.nan, self.config.polynomialFitDegree + 1) if 

1006 ptcFitType in ["POLYNOMIAL", ] else np.repeat(np.nan, 3)) 

1007 dataset.ptcFitChiSq[ampName] = np.nan 

1008 dataset.ptcTurnoff[ampName] = np.nan 

1009 dataset.finalVars[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName])) 

1010 dataset.finalModelVars[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName])) 

1011 dataset.finalMeans[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName])) 

1012 

1013 return