Coverage for python/lsst/cp/pipe/ptc/cpSolvePtcTask.py: 11%

394 statements  

« prev     ^ index     » next       coverage.py v6.4.2, created at 2022-08-03 12:39 +0000

1# This file is part of cp_pipe. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21# 

22import numpy as np 

23from collections import Counter 

24 

25import lsst.pex.config as pexConfig 

26import lsst.pipe.base as pipeBase 

27from lsst.cp.pipe.utils import (fitLeastSq, fitBootstrap, funcPolynomial, funcAstier, symmetrize) 

28 

29from scipy.signal import fftconvolve 

30from scipy.optimize import least_squares 

31from itertools import groupby 

32from operator import itemgetter 

33 

34import lsst.pipe.base.connectionTypes as cT 

35 

36from lsst.ip.isr import PhotonTransferCurveDataset 

37 

38from lsst.cp.pipe._lookupStaticCalibration import lookupStaticCalibration 

39 

40import copy 

41 

42 

43__all__ = ['PhotonTransferCurveSolveConfig', 'PhotonTransferCurveSolveTask'] 

44 

45 

46class PhotonTransferCurveSolveConnections(pipeBase.PipelineTaskConnections, 

47 dimensions=("instrument", "detector")): 

48 inputCovariances = cT.Input( 

49 name="ptcCovariances", 

50 doc="Tuple with measured covariances from flats.", 

51 storageClass="PhotonTransferCurveDataset", 

52 dimensions=("instrument", "exposure", "detector"), 

53 multiple=True, 

54 ) 

55 camera = cT.PrerequisiteInput( 

56 name="camera", 

57 doc="Camera the input data comes from.", 

58 storageClass="Camera", 

59 dimensions=("instrument",), 

60 isCalibration=True, 

61 lookupFunction=lookupStaticCalibration, 

62 ) 

63 outputPtcDataset = cT.Output( 

64 name="ptcDatsetProposal", 

65 doc="Output proposed ptc dataset.", 

66 storageClass="PhotonTransferCurveDataset", 

67 dimensions=("instrument", "detector"), 

68 multiple=False, 

69 isCalibration=True, 

70 ) 

71 

72 

73class PhotonTransferCurveSolveConfig(pipeBase.PipelineTaskConfig, 

74 pipelineConnections=PhotonTransferCurveSolveConnections): 

75 """Configuration for fitting measured covariances. 

76 """ 

77 

78 ptcFitType = pexConfig.ChoiceField( 

79 dtype=str, 

80 doc="Fit PTC to Eq. 16, Eq. 20 in Astier+19, or to a polynomial.", 

81 default="POLYNOMIAL", 

82 allowed={ 

83 "POLYNOMIAL": "n-degree polynomial (use 'polynomialFitDegree' to set 'n').", 

84 "EXPAPPROXIMATION": "Approximation in Astier+19 (Eq. 16).", 

85 "FULLCOVARIANCE": "Full covariances model in Astier+19 (Eq. 20)" 

86 } 

87 ) 

88 maximumRangeCovariancesAstier = pexConfig.Field( 

89 dtype=int, 

90 doc="Maximum range of covariances as in Astier+19", 

91 default=8, 

92 ) 

93 sigmaClipFullFitCovariancesAstier = pexConfig.Field( 

94 dtype=float, 

95 doc="sigma clip for full model fit for FULLCOVARIANCE ptcFitType ", 

96 default=5.0, 

97 ) 

98 maxIterFullFitCovariancesAstier = pexConfig.Field( 

99 dtype=int, 

100 doc="Maximum number of iterations in full model fit for FULLCOVARIANCE ptcFitType", 

101 default=3, 

102 ) 

103 polynomialFitDegree = pexConfig.Field( 

104 dtype=int, 

105 doc="Degree of polynomial to fit the PTC, when 'ptcFitType'=POLYNOMIAL.", 

106 default=3, 

107 ) 

108 sigmaCutPtcOutliers = pexConfig.Field( 

109 dtype=float, 

110 doc="Sigma cut for outlier rejection in PTC.", 

111 default=5.0, 

112 ) 

113 maxIterationsPtcOutliers = pexConfig.RangeField( 

114 dtype=int, 

115 doc="Maximum number of iterations for outlier rejection in PTC.", 

116 default=2, 

117 min=0 

118 ) 

119 minVarPivotSearch = pexConfig.Field( 

120 dtype=float, 

121 doc="The code looks for a pivot signal point after which the variance starts decreasing at high-flux" 

122 " to exclude then from the PTC model fit. However, sometimes at low fluxes, the variance" 

123 " decreases slightly. Set this variable for the variance value, in ADU^2, after which the pivot " 

124 " should be sought.", 

125 default=10000, 

126 ) 

127 consecutivePointsVarDecreases = pexConfig.RangeField( 

128 dtype=int, 

129 doc="Required number of consecutive points/fluxes in the PTC where the variance " 

130 "decreases in order to find a first estimate of the PTC turn-off. ", 

131 default=2, 

132 min=2 

133 ) 

134 doFitBootstrap = pexConfig.Field( 

135 dtype=bool, 

136 doc="Use bootstrap for the PTC fit parameters and errors?.", 

137 default=False, 

138 ) 

139 

140 

141class PhotonTransferCurveSolveTask(pipeBase.PipelineTask, 

142 pipeBase.CmdLineTask): 

143 """Task to fit the PTC from flat covariances. 

144 

145 The first task of the PTC measurement pipeline, 

146 ``PhotonTransferCurveMeasureTask`` (and assumed to have been run 

147 before this task), produced a list of 

148 `~lsst.ip.isr.PhotonTransferCurveDataset` objects. Each dataset 

149 contains the mean signal and covariances of the 

150 difference image of the flat-field images taken at 

151 the same exposure time. The list also contains dummy 

152 datasets (with no measurements), whose purpose is to have 

153 the input and output dimensions of ``PhotonTransferCurveMeasureTask`` 

154 match. 

155 

156 This task, ``PhotonTransferCurveSolveTask``, assembles the list 

157 of individual PTC datasets produced 

158 by ``PhotonTransferCurveMeasureTask`` into one single final PTC 

159 dataset, discarding the dummy datset as appropiate. 

160 The task fits the measured (co)variances to one of three models: 

161 a polynomial model of a given order, or the models described 

162 in equations 16 and 20 of Astier+19. These options are referred 

163 to as ``POLYNOMIAL``, ``EXPAPPROXIMATION``, and ``FULLCOVARIANCE`` 

164 in the configuration options of the task, respectively). 

165 Parameters of interest such as the gain and noise are derived 

166 from the fits. The ``FULLCOVARIANCE`` model is fitted to the 

167 full covariance data (as oppossed to the other two models, which 

168 are fit to the variance vs mean measurements only). 

169 

170 Astier+19: "The Shape of the Photon Transfer Curve 

171 of CCD sensors", arXiv:1905.08677 

172 """ 

173 

174 ConfigClass = PhotonTransferCurveSolveConfig 

175 _DefaultName = 'cpPhotonTransferCurveSolve' 

176 

177 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

178 """Ensure that the input and output dimensions are passed along. 

179 

180 Parameters 

181 ---------- 

182 butlerQC : `~lsst.daf.butler.butlerQuantumContext.ButlerQuantumContext` 

183 Butler to operate on. 

184 inputRefs : `~lsst.pipe.base.connections.InputQuantizedConnection` 

185 Input data refs to load. 

186 ouptutRefs : `~lsst.pipe.base.connections.OutputQuantizedConnection` 

187 Output data refs to persist. 

188 """ 

189 inputs = butlerQC.get(inputRefs) 

190 detId = inputRefs.inputCovariances[0].dataId['detector'] 

191 outputs = self.run(inputCovariances=inputs['inputCovariances'], camera=inputs['camera'], detId=detId) 

192 butlerQC.put(outputs, outputRefs) 

193 

194 def run(self, inputCovariances, camera=None, detId=0): 

195 """Fit measured covariances to different models. 

196 

197 Parameters 

198 ---------- 

199 inputCovariances : `list` [`lsst.ip.isr.PhotonTransferCurveDataset`] 

200 List of lsst.ip.isr.PhotonTransferCurveDataset datasets. 

201 camera : `lsst.afw.cameraGeom.Camera`, optional 

202 Input camera. 

203 detId : `int` 

204 Detector ID to locate the detector in the camera and 

205 populate the `lsst.ip.isr.PhotonTransferCurveDataset` 

206 metadata. 

207 Returns 

208 ------- 

209 results : `lsst.pipe.base.Struct` 

210 The resultins structure contains: 

211 ``outputPtcDatset`` 

212 Final PTC dataset, containing information such as the 

213 means, variances, and exposure times 

214 (`lsst.ip.isr.PhotonTransferCurveDataset`). 

215 """ 

216 # Assemble individual PTC datasets into a single PTC dataset. 

217 ampNames = np.unique(inputCovariances[0].ampNames) 

218 datasetPtc = PhotonTransferCurveDataset(ampNames=ampNames, 

219 ptcFitType=self.config.ptcFitType, 

220 covMatrixSide=self.config.maximumRangeCovariancesAstier) 

221 for partialPtcDataset in inputCovariances: 

222 # Ignore dummy datasets 

223 if partialPtcDataset.ptcFitType == 'DUMMY': 

224 continue 

225 for ampName in ampNames: 

226 datasetPtc.inputExpIdPairs[ampName].append(partialPtcDataset.inputExpIdPairs[ampName]) 

227 if type(partialPtcDataset.rawExpTimes[ampName]) is list: 

228 datasetPtc.rawExpTimes[ampName].append(partialPtcDataset.rawExpTimes[ampName][0]) 

229 else: 

230 datasetPtc.rawExpTimes[ampName].append(partialPtcDataset.rawExpTimes[ampName]) 

231 if type(partialPtcDataset.rawMeans[ampName]) is list: 

232 datasetPtc.rawMeans[ampName].append(partialPtcDataset.rawMeans[ampName][0]) 

233 else: 

234 datasetPtc.rawMeans[ampName].append(partialPtcDataset.rawMeans[ampName]) 

235 if type(partialPtcDataset.rawVars[ampName]) is list: 

236 datasetPtc.rawVars[ampName].append(partialPtcDataset.rawVars[ampName][0]) 

237 else: 

238 datasetPtc.rawVars[ampName].append(partialPtcDataset.rawVars[ampName]) 

239 if type(partialPtcDataset.expIdMask[ampName]) is list: 

240 datasetPtc.expIdMask[ampName].append(partialPtcDataset.expIdMask[ampName][0]) 

241 else: 

242 datasetPtc.expIdMask[ampName].append(partialPtcDataset.expIdMask[ampName]) 

243 datasetPtc.covariances[ampName].append(np.array(partialPtcDataset.covariances[ampName][0])) 

244 datasetPtc.covariancesSqrtWeights[ampName].append( 

245 np.array(partialPtcDataset.covariancesSqrtWeights[ampName][0])) 

246 # Sort arrays that are filled so far in the final dataset by 

247 # rawMeans index 

248 for ampName in ampNames: 

249 index = np.argsort(np.ravel(np.array(datasetPtc.rawMeans[ampName]))) 

250 datasetPtc.inputExpIdPairs[ampName] = np.array(datasetPtc.inputExpIdPairs[ampName])[index] 

251 datasetPtc.rawExpTimes[ampName] = np.array(datasetPtc.rawExpTimes[ampName])[index] 

252 datasetPtc.rawMeans[ampName] = np.array(datasetPtc.rawMeans[ampName])[index] 

253 datasetPtc.rawVars[ampName] = np.array(datasetPtc.rawVars[ampName])[index] 

254 datasetPtc.expIdMask[ampName] = np.array(datasetPtc.expIdMask[ampName])[index] 

255 datasetPtc.covariances[ampName] = np.array(datasetPtc.covariances[ampName])[index] 

256 datasetPtc.covariancesSqrtWeights[ampName] = np.array( 

257 datasetPtc.covariancesSqrtWeights[ampName])[index] 

258 if self.config.ptcFitType == "FULLCOVARIANCE": 

259 # Fit the measured covariances vs mean signal to 

260 # the Astier+19 full model (Eq. 20). Before that 

261 # do a preliminary fit to the variance (C_00) vs mean 

262 # signal (mu) curve using the EXPAPPROXIMATION model 

263 # (Eq. 16 in Astier+19) in order to 

264 # get the flat pairs that are masked. The 

265 # points at these fluxes will also be masked when 

266 # calculating the other elements of the covariance 

267 # matrix, C_ij, i!=j). 

268 

269 # Preliminary fit, usign a temp dataset to get the mask 

270 tempDatasetPtc = copy.copy(datasetPtc) 

271 tempDatasetPtc.ptcFitType = "EXPAPPROXIMATION" 

272 tempDatasetPtc = self.fitMeasurementsToModel(tempDatasetPtc) 

273 

274 # "FULLCOVARIANCE", using the mask obtained from the 

275 # previous fit. 

276 for ampName in datasetPtc.ampNames: 

277 datasetPtc.expIdMask[ampName] = tempDatasetPtc.expIdMask[ampName] 

278 datasetPtc.fitType = "FULLCOVARIANCE" 

279 datasetPtc = self.fitMeasurementsToModel(datasetPtc) 

280 # The other options are: self.config.ptcFitType in 

281 # ("EXPAPPROXIMATION", "POLYNOMIAL") 

282 else: 

283 # Fit the PTC to a polynomial or to Astier+19 exponential 

284 # approximation (Eq. 16). Fill up 

285 # PhotonTransferCurveDataset object. 

286 datasetPtc = self.fitMeasurementsToModel(datasetPtc) 

287 

288 if camera: 

289 detector = camera[detId] 

290 else: 

291 detector = None 

292 datasetPtc.updateMetadata(setDate=True, camera=camera, detector=detector) 

293 

294 return pipeBase.Struct( 

295 outputPtcDataset=datasetPtc, 

296 ) 

297 

298 def fitMeasurementsToModel(self, dataset): 

299 """Fit the measured covariances vs mean signal to a 

300 polynomial or one of the models in Astier+19 

301 (Eq. 16 or Eq.20). 

302 

303 Parameters 

304 ---------- 

305 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset` 

306 The dataset containing information such as the means, 

307 (co)variances, and exposure times. 

308 

309 Returns 

310 ------- 

311 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset` 

312 This is the same dataset as the input parameter, however, 

313 it has been modified to include information such as the 

314 fit vectors and the fit parameters. See the class 

315 `PhotonTransferCurveDatase`. 

316 """ 

317 fitType = dataset.ptcFitType 

318 if fitType in ["FULLCOVARIANCE", ]: 

319 # This model uses the full covariance matrix in the fit. 

320 # The PTC is technically defined as variance vs signal, 

321 # with variance = Cov_00 

322 dataset = self.fitDataFullCovariance(dataset) 

323 elif fitType in ["POLYNOMIAL", "EXPAPPROXIMATION"]: 

324 # The PTC is technically defined as variance vs signal 

325 dataset = self.fitPtc(dataset) 

326 else: 

327 raise RuntimeError( 

328 f"Fitting option {fitType} not one of " 

329 "'POLYNOMIAL', 'EXPAPPROXIMATION', or 'FULLCOVARIANCE'" 

330 ) 

331 

332 return dataset 

333 

334 def fitDataFullCovariance(self, dataset): 

335 """Fit measured flat covariances to the full model in 

336 Astier+19 (Eq. 20). 

337 

338 Parameters 

339 ---------- 

340 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset` 

341 The dataset containing information such as the means, 

342 (co)variances, and exposure times. 

343 

344 Returns 

345 ------- 

346 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset` 

347 This is the same dataset as the input parameter, however, 

348 it has been modified to include information such as the 

349 fit vectors and the fit parameters. See the class 

350 `PhotonTransferCurveDatase`. 

351 

352 Notes 

353 ----- 

354 The parameters of the full model for C_ij(mu) ("C_ij" and "mu" 

355 in ADU^2 and ADU, respectively) in Astier+19 (Eq. 20) are: 

356 "a" coefficients (r by r matrix), units: 1/e 

357 "b" coefficients (r by r matrix), units: 1/e 

358 noise matrix (r by r matrix), units: e^2 

359 gain, units: e/ADU 

360 "b" appears in Eq. 20 only through the "ab" combination, which 

361 is defined in this code as "c=ab". 

362 

363 Total number of parameters: #entries(a) + #entries(c) + #entries(noise) 

364 + 1. This is equivalent to r^2 + r^2 + r^2 + 1, where "r" is the 

365 maximum lag considered for the covariances calculation, and the 

366 extra "1" is the gain. If "b" is 0, then "c" is 0, and len(pInit) will 

367 have r^2 fewer entries. 

368 """ 

369 matrixSide = self.config.maximumRangeCovariancesAstier 

370 lenParams = matrixSide*matrixSide 

371 

372 for ampName in dataset.ampNames: 

373 lenInputTimes = len(dataset.rawExpTimes[ampName]) 

374 # Not used when ptcFitType is 'FULLCOVARIANCE' 

375 dataset.ptcFitPars[ampName] = [np.nan] 

376 dataset.ptcFitParsError[ampName] = [np.nan] 

377 dataset.ptcFitChiSq[ampName] = np.nan 

378 

379 if ampName in dataset.badAmps: 

380 # Bad amp 

381 # Entries need to have proper dimensions so read/write 

382 # with astropy.Table works. 

383 nanMatrix = np.full((matrixSide, matrixSide), np.nan) 

384 listNanMatrix = np.full((lenInputTimes, matrixSide, matrixSide), np.nan) 

385 dataset.covariancesModel[ampName] = listNanMatrix 

386 dataset.covariancesSqrtWeights[ampName] = listNanMatrix 

387 dataset.aMatrix[ampName] = nanMatrix 

388 dataset.bMatrix[ampName] = nanMatrix 

389 dataset.covariancesModelNoB[ampName] = listNanMatrix 

390 dataset.aMatrixNoB[ampName] = nanMatrix 

391 

392 dataset.expIdMask[ampName] = np.repeat(np.nan, lenInputTimes) 

393 dataset.gain[ampName] = np.nan 

394 dataset.gainErr[ampName] = np.nan 

395 dataset.noise[ampName] = np.nan 

396 dataset.noiseErr[ampName] = np.nan 

397 dataset.finalVars[ampName] = np.repeat(np.nan, lenInputTimes) 

398 dataset.finalModelVars[ampName] = np.repeat(np.nan, lenInputTimes) 

399 dataset.finalMeans[ampName] = np.repeat(np.nan, lenInputTimes) 

400 continue 

401 

402 muAtAmp = dataset.rawMeans[ampName] 

403 maskAtAmp = dataset.expIdMask[ampName] 

404 if len(maskAtAmp) == 0: 

405 maskAtAmp = np.repeat(True, len(muAtAmp)) 

406 

407 muAtAmp = muAtAmp[maskAtAmp] 

408 covAtAmp = np.nan_to_num(dataset.covariances[ampName])[maskAtAmp] 

409 covSqrtWeightsAtAmp = np.nan_to_num(dataset.covariancesSqrtWeights[ampName])[maskAtAmp] 

410 

411 # Initial fit, to approximate parameters, with c=0 

412 a0, c0, noise0, gain0 = self.initialFitFullCovariance(muAtAmp, covAtAmp, covSqrtWeightsAtAmp) 

413 

414 # Fit full model (Eq. 20 of Astier+19) and same model with 

415 # b=0 (c=0 in this code) 

416 pInit = np.concatenate((a0.flatten(), c0.flatten(), noise0.flatten(), np.array(gain0)), axis=None) 

417 functionsDict = {'fullModel': self.funcFullCovarianceModel, 

418 'fullModelNoB': self.funcFullCovarianceModelNoB} 

419 fitResults = {'fullModel': {'a': [], 'c': [], 'noise': [], 'gain': [], 'paramsErr': []}, 

420 'fullModelNoB': {'a': [], 'c': [], 'noise': [], 'gain': [], 'paramsErr': []}} 

421 for key in functionsDict: 

422 params, paramsErr, _ = fitLeastSq(pInit, muAtAmp, 

423 covAtAmp.flatten(), functionsDict[key], 

424 weightsY=covSqrtWeightsAtAmp.flatten()) 

425 a = params[:lenParams].reshape((matrixSide, matrixSide)) 

426 c = params[lenParams:2*lenParams].reshape((matrixSide, matrixSide)) 

427 noise = params[2*lenParams:3*lenParams].reshape((matrixSide, matrixSide)) 

428 gain = params[-1] 

429 

430 fitResults[key]['a'] = a 

431 fitResults[key]['c'] = c 

432 fitResults[key]['noise'] = noise 

433 fitResults[key]['gain'] = gain 

434 fitResults[key]['paramsErr'] = paramsErr 

435 

436 # Put the information in the PTC dataset 

437 

438 # Not used when ptcFitType is 'FULLCOVARIANCE' 

439 dataset.ptcFitPars[ampName] = [np.nan] 

440 dataset.ptcFitParsError[ampName] = [np.nan] 

441 dataset.ptcFitChiSq[ampName] = np.nan 

442 

443 # Save full covariances, covariances models, and their weights 

444 # dataset.expIdMask is already full 

445 dataset.covariances[ampName] = covAtAmp 

446 dataset.covariancesModel[ampName] = self.evalCovModel(muAtAmp, 

447 fitResults['fullModel']['a'], 

448 fitResults['fullModel']['c'], 

449 fitResults['fullModel']['noise'], 

450 fitResults['fullModel']['gain']) 

451 dataset.covariancesSqrtWeights[ampName] = covSqrtWeightsAtAmp 

452 dataset.aMatrix[ampName] = fitResults['fullModel']['a'] 

453 dataset.bMatrix[ampName] = fitResults['fullModel']['c']/fitResults['fullModel']['a'] 

454 dataset.covariancesModelNoB[ampName] = self.evalCovModel(muAtAmp, 

455 fitResults['fullModelNoB']['a'], 

456 fitResults['fullModelNoB']['c'], 

457 fitResults['fullModelNoB']['noise'], 

458 fitResults['fullModelNoB']['gain'], 

459 setBtoZero=True) 

460 dataset.aMatrixNoB[ampName] = fitResults['fullModelNoB']['a'] 

461 dataset.gain[ampName] = fitResults['fullModel']['gain'] 

462 dataset.gainErr[ampName] = fitResults['fullModel']['paramsErr'][-1] 

463 readoutNoise = fitResults['fullModel']['noise'][0][0] 

464 readoutNoiseSqrt = np.sqrt(np.fabs(readoutNoise)) 

465 dataset.noise[ampName] = readoutNoise 

466 readoutNoiseSigma = fitResults['fullModel']['paramsErr'][2*lenParams] 

467 dataset.noiseErr[ampName] = 0.5*(readoutNoiseSigma/np.fabs(readoutNoise))*readoutNoiseSqrt 

468 dataset.finalVars[ampName] = covAtAmp[:, 0, 0] 

469 dataset.finalModelVars[ampName] = dataset.covariancesModel[ampName][:, 0, 0] 

470 dataset.finalMeans[ampName] = muAtAmp 

471 

472 return dataset 

473 

474 def initialFitFullCovariance(self, mu, cov, sqrtW): 

475 """ Performs a crude parabolic fit of the data in order to start 

476 the full fit close to the solution, setting b=0 (c=0) in Eq. 20 

477 of Astier+19. 

478 

479 Parameters 

480 ---------- 

481 mu : `numpy.array`, (N,) 

482 Signal `mu` (ADU) 

483 cov : `numpy.array`, (N, M, M) 

484 Covariance arrays of size `(M, M)` (with 

485 `M = config.maximumRangeCovariancesAstier`), 

486 indexed by mean signal `mu`. 

487 sqrtW : `numpy.array`, (N,) 

488 Covariance weights, defined as 1./sqrt(Variances) 

489 

490 Returns 

491 ------- 

492 a : `numpy.array`, (M, M) 

493 "a" parameter per flux in Eq. 20 of Astier+19. 

494 c : `numpy.array`, (M, M) 

495 "c"="ab" parameter per flux in Eq. 20 of Astier+19. 

496 noise : `numpy.array`, (M, M) 

497 "noise" parameter per flux in Eq. 20 of Astier+19. 

498 gain : `float` 

499 Amplifier gain (e/ADU) 

500 """ 

501 matrixSide = self.config.maximumRangeCovariancesAstier 

502 

503 # Initialize fit parameters 

504 a = np.zeros((matrixSide, matrixSide)) 

505 c = np.zeros((matrixSide, matrixSide)) 

506 noise = np.zeros((matrixSide, matrixSide)) 

507 gain = 1. 

508 

509 # iterate the fit to account for higher orders 

510 # the chi2 does not necessarily go down, so one could 

511 # stop when it increases 

512 oldChi2 = 1e30 

513 for _ in range(5): 

514 model = np.nan_to_num(self.evalCovModel(mu, a, c, noise, gain, setBtoZero=True)) 

515 # loop on lags 

516 for i in range(matrixSide): 

517 for j in range(matrixSide): 

518 # fit a parabola for a given lag 

519 parsFit = np.polyfit(mu, cov[:, i, j] - model[:, i, j], 

520 2, w=sqrtW[:, i, j]) 

521 # model equation (Eq. 20) in Astier+19, with c=a*b=0: 

522 a[i, j] += parsFit[0] 

523 noise[i, j] += parsFit[2] 

524 if(i + j == 0): 

525 gain = 1./(1/gain+parsFit[1]) 

526 weightedRes = (model - cov)*sqrtW 

527 chi2 = (weightedRes.flatten()**2).sum() 

528 if chi2 > oldChi2: 

529 break 

530 oldChi2 = chi2 

531 

532 return a, c, noise, gain 

533 

534 def funcFullCovarianceModel(self, params, x): 

535 """Model to fit covariances from flat fields; Equation 20 of 

536 Astier+19. 

537 

538 Parameters 

539 ---------- 

540 params : `list` 

541 Parameters of the model: aMatrix, CMatrix, noiseMatrix, 

542 gain (e/ADU). 

543 x : `numpy.array`, (N,) 

544 Signal `mu` (ADU) 

545 

546 Returns 

547 ------- 

548 y : `numpy.array`, (N,) 

549 Covariance matrix. 

550 """ 

551 matrixSide = self.config.maximumRangeCovariancesAstier 

552 lenParams = matrixSide*matrixSide 

553 aMatrix = params[:lenParams].reshape((matrixSide, matrixSide)) 

554 cMatrix = params[lenParams:2*lenParams].reshape((matrixSide, matrixSide)) 

555 noiseMatrix = params[2*lenParams:3*lenParams].reshape((matrixSide, matrixSide)) 

556 gain = params[-1] 

557 

558 return self.evalCovModel(x, aMatrix, cMatrix, noiseMatrix, gain).flatten() 

559 

560 def funcFullCovarianceModelNoB(self, params, x): 

561 """Model to fit covariances from flat fields; Equation 20 of 

562 Astier+19, with b=0 (equivalent to c=a*b=0 in this code). 

563 

564 Parameters 

565 ---------- 

566 params : `list` 

567 Parameters of the model: aMatrix, CMatrix, noiseMatrix, 

568 gain (e/ADU). 

569 x : `numpy.array`, (N,) 

570 Signal mu (ADU) 

571 

572 Returns 

573 ------- 

574 y : `numpy.array`, (N,) 

575 Covariance matrix. 

576 """ 

577 matrixSide = self.config.maximumRangeCovariancesAstier 

578 lenParams = matrixSide*matrixSide 

579 aMatrix = params[:lenParams].reshape((matrixSide, matrixSide)) 

580 cMatrix = params[lenParams:2*lenParams].reshape((matrixSide, matrixSide)) 

581 noiseMatrix = params[2*lenParams:3*lenParams].reshape((matrixSide, matrixSide)) 

582 gain = params[-1] 

583 

584 return self.evalCovModel(x, aMatrix, cMatrix, noiseMatrix, gain, setBtoZero=True).flatten() 

585 

586 def evalCovModel(self, mu, aMatrix, cMatrix, noiseMatrix, gain, setBtoZero=False): 

587 """Computes full covariances model (Eq. 20 of Astier+19). 

588 

589 Parameters 

590 ---------- 

591 mu : `numpy.array`, (N,) 

592 List of mean signals. 

593 aMatrix : `numpy.array`, (M, M) 

594 "a" parameter per flux in Eq. 20 of Astier+19. 

595 cMatrix : `numpy.array`, (M, M) 

596 "c"="ab" parameter per flux in Eq. 20 of Astier+19. 

597 noiseMatrix : `numpy.array`, (M, M) 

598 "noise" parameter per flux in Eq. 20 of Astier+19. 

599 gain : `float` 

600 Amplifier gain (e/ADU) 

601 setBtoZero=False : `bool`, optional 

602 Set "b" parameter in full model (see Astier+19) to zero. 

603 

604 Returns 

605 ------- 

606 covModel : `numpy.array`, (N, M, M) 

607 Covariances model. 

608 

609 Notes 

610 ----- 

611 By default, computes the covModel for the mu's stored(self.mu). 

612 Returns cov[Nmu, M, M]. The variance for the PTC is 

613 cov[:, 0, 0]. mu and cov are in ADUs and ADUs squared. To use 

614 electrons for both, the gain should be set to 1. This routine 

615 implements the model in Astier+19 (1905.08677). 

616 The parameters of the full model for C_ij(mu) ("C_ij" and "mu" 

617 in ADU^2 and ADU, respectively) in Astier+19 (Eq. 20) are: 

618 "a" coefficients (M by M matrix), units: 1/e 

619 "b" coefficients (M by M matrix), units: 1/e 

620 noise matrix (M by M matrix), units: e^2 

621 gain, units: e/ADU 

622 "b" appears in Eq. 20 only through the "ab" combination, which 

623 is defined in this code as "c=ab". 

624 """ 

625 matrixSide = self.config.maximumRangeCovariancesAstier 

626 sa = (matrixSide, matrixSide) 

627 # pad a with zeros and symmetrize 

628 aEnlarged = np.zeros((int(sa[0]*1.5)+1, int(sa[1]*1.5)+1)) 

629 aEnlarged[0:sa[0], 0:sa[1]] = aMatrix 

630 aSym = symmetrize(aEnlarged) 

631 # pad c with zeros and symmetrize 

632 cEnlarged = np.zeros((int(sa[0]*1.5)+1, int(sa[1]*1.5)+1)) 

633 cEnlarged[0:sa[0], 0:sa[1]] = cMatrix 

634 cSym = symmetrize(cEnlarged) 

635 a2 = fftconvolve(aSym, aSym, mode='same') 

636 a3 = fftconvolve(a2, aSym, mode='same') 

637 ac = fftconvolve(aSym, cSym, mode='same') 

638 (xc, yc) = np.unravel_index(np.abs(aSym).argmax(), a2.shape) 

639 

640 a1 = aMatrix[np.newaxis, :, :] 

641 a2 = a2[np.newaxis, xc:xc + matrixSide, yc:yc + matrixSide] 

642 a3 = a3[np.newaxis, xc:xc + matrixSide, yc:yc + matrixSide] 

643 ac = ac[np.newaxis, xc:xc + matrixSide, yc:yc + matrixSide] 

644 c1 = cMatrix[np.newaxis, ::] 

645 

646 # assumes that mu is 1d 

647 bigMu = mu[:, np.newaxis, np.newaxis]*gain 

648 # c(=a*b in Astier+19) also has a contribution to the last 

649 # term, that is absent for now. 

650 if setBtoZero: 

651 c1 = np.zeros_like(c1) 

652 ac = np.zeros_like(ac) 

653 covModel = (bigMu/(gain*gain)*(a1*bigMu+2./3.*(bigMu*bigMu)*(a2 + c1) 

654 + (1./3.*a3 + 5./6.*ac)*(bigMu*bigMu*bigMu)) + noiseMatrix[np.newaxis, :, :]/gain**2) 

655 # add the Poisson term, and the read out noise (variance) 

656 covModel[:, 0, 0] += mu/gain 

657 

658 return covModel 

659 

660 # EXPAPPROXIMATION and POLYNOMIAL fit methods 

661 @staticmethod 

662 def _initialParsForPolynomial(order): 

663 assert(order >= 2) 

664 pars = np.zeros(order, dtype=float) 

665 pars[0] = 10 

666 pars[1] = 1 

667 pars[2:] = 0.0001 

668 return pars 

669 

670 @staticmethod 

671 def _boundsForPolynomial(initialPars, lowers=[], uppers=[]): 

672 if not len(lowers): 

673 lowers = [np.NINF for p in initialPars] 

674 if not len(uppers): 

675 uppers = [np.inf for p in initialPars] 

676 lowers[1] = 0 # no negative gains 

677 return (lowers, uppers) 

678 

679 @staticmethod 

680 def _boundsForAstier(initialPars, lowers=[], uppers=[]): 

681 if not len(lowers): 

682 lowers = [np.NINF for p in initialPars] 

683 if not len(uppers): 

684 uppers = [np.inf for p in initialPars] 

685 return (lowers, uppers) 

686 

687 @staticmethod 

688 def _getInitialGoodPoints(means, variances, minVarPivotSearch, consecutivePointsVarDecreases): 

689 """Return a boolean array to mask bad points. 

690 

691 Parameters 

692 ---------- 

693 means : `numpy.array` 

694 Input array with mean signal values. 

695 variances : `numpy.array` 

696 Input array with variances at each mean value. 

697 minVarPivotSearch : `float` 

698 The variance (in ADU^2), above which, the point 

699 of decreasing variance should be sought. 

700 consecutivePointsVarDecreases : `int` 

701 Required number of consecutive points/fluxes 

702 in the PTC where the variance 

703 decreases in order to find a first 

704 estimate of the PTC turn-off. 

705 

706 Returns 

707 ------ 

708 goodPoints : `numpy.array` [`bool`] 

709 Boolean array to select good (`True`) and bad (`False`) 

710 points. 

711 

712 Notes 

713 ----- 

714 Eliminate points beyond which the variance decreases. 

715 """ 

716 goodPoints = np.ones_like(means, dtype=bool) 

717 # Variances are sorted and should monotonically increase 

718 pivotList = np.where(np.array(np.diff(variances)) < 0)[0] 

719 if len(pivotList) > 0: 

720 # For small values, sometimes the variance decreases slightly 

721 # Only look when var > self.config.minVarPivotSearch 

722 pivotList = [p for p in pivotList if variances[p] > minVarPivotSearch] 

723 # Require that the varince decreases during 

724 # consecutivePointsVarDecreases 

725 # consecutive points. This will give a first 

726 # estimate of the PTC turn-off, which 

727 # may be updated (reduced) further in the code. 

728 if len(pivotList) > 1: 

729 # enumerate(pivotList) creates tuples (index, value), for 

730 # each value in pivotList. The lambda function subtracts 

731 # each value from the index. 

732 # groupby groups elements by equal key value. 

733 for k, g in groupby(enumerate(pivotList), lambda x: x[0]-x[1]): 

734 group = (map(itemgetter(1), g)) 

735 # Form groups of consecute values from pivotList 

736 group = list(map(int, group)) 

737 # values in pivotList are indices where np.diff(variances) 

738 # is negative, i.e., where the variance starts decreasing. 

739 # Find the first group of consecutive numbers when 

740 # variance decreases. 

741 if len(group) >= consecutivePointsVarDecreases: 

742 pivotIndex = np.min(group) 

743 goodPoints[pivotIndex+1:] = False 

744 break 

745 

746 return goodPoints 

747 

748 def _makeZeroSafe(self, array, substituteValue=1e-9): 

749 """""" 

750 array = np.array(array) 

751 nBad = Counter(np.ravel(array))[0] 

752 if nBad == 0: 

753 return array 

754 

755 index, = np.where(array == 0) 

756 if len(index): 

757 msg = f"Found {nBad} zeros in array at elements {index}" 

758 self.log.warning(msg) 

759 

760 array[index] = substituteValue 

761 

762 return array 

763 

764 def fitPtc(self, dataset): 

765 """Fit the photon transfer curve to a polynomial or to the 

766 Astier+19 approximation (Eq. 16). 

767 

768 Fit the photon transfer curve with either a polynomial of 

769 the order specified in the task config, or using the 

770 exponential approximation in Astier+19 (Eq. 16). 

771 

772 Sigma clipping is performed iteratively for the fit, as 

773 well as an initial clipping of data points that are more 

774 than `config.initialNonLinearityExclusionThreshold` away 

775 from lying on a straight line. This other step is necessary 

776 because the photon transfer curve turns over catastrophically 

777 at very high flux (because saturation 

778 drops the variance to ~0) and these far outliers cause the 

779 initial fit to fail, meaning the sigma cannot be calculated 

780 to perform the sigma-clipping. 

781 

782 Parameters 

783 ---------- 

784 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset` 

785 The dataset containing the means, variances and 

786 exposure times. 

787 

788 Returns 

789 ------- 

790 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset` 

791 This is the same dataset as the input parameter, however, 

792 it has been modified to include information such as the 

793 fit vectors and the fit parameters. See the class 

794 `PhotonTransferCurveDatase`. 

795 

796 Raises 

797 ------ 

798 RuntimeError: 

799 Raises if dataset.ptcFitType is None or empty. 

800 """ 

801 if dataset.ptcFitType: 

802 ptcFitType = dataset.ptcFitType 

803 else: 

804 raise RuntimeError("ptcFitType is None of empty in PTC dataset.") 

805 matrixSide = self.config.maximumRangeCovariancesAstier 

806 nanMatrix = np.empty((matrixSide, matrixSide)) 

807 nanMatrix[:] = np.nan 

808 

809 for amp in dataset.ampNames: 

810 lenInputTimes = len(dataset.rawExpTimes[amp]) 

811 listNanMatrix = np.empty((lenInputTimes, matrixSide, matrixSide)) 

812 listNanMatrix[:] = np.nan 

813 

814 dataset.covariancesModel[amp] = listNanMatrix 

815 dataset.aMatrix[amp] = nanMatrix 

816 dataset.bMatrix[amp] = nanMatrix 

817 dataset.covariancesModelNoB[amp] = listNanMatrix 

818 dataset.aMatrixNoB[amp] = nanMatrix 

819 

820 def errFunc(p, x, y): 

821 return ptcFunc(p, x) - y 

822 

823 sigmaCutPtcOutliers = self.config.sigmaCutPtcOutliers 

824 maxIterationsPtcOutliers = self.config.maxIterationsPtcOutliers 

825 

826 for i, ampName in enumerate(dataset.ampNames): 

827 timeVecOriginal = np.ravel(np.array(dataset.rawExpTimes[ampName])) 

828 meanVecOriginal = np.ravel(np.array(dataset.rawMeans[ampName])) 

829 varVecOriginal = np.ravel(np.array(dataset.rawVars[ampName])) 

830 varVecOriginal = self._makeZeroSafe(varVecOriginal) 

831 

832 # Discard points when the variance starts to decrease after two 

833 # consecutive signal levels 

834 goodPoints = self._getInitialGoodPoints(meanVecOriginal, varVecOriginal, 

835 self.config.minVarPivotSearch, 

836 self.config.consecutivePointsVarDecreases) 

837 

838 # Check if all points are bad from the 'cpExtractPtcTask' 

839 initialExpIdMask = np.ravel(np.array(dataset.expIdMask[ampName])) 

840 

841 if not (goodPoints.any() and initialExpIdMask.any()): 

842 msg = (f"SERIOUS: All points in goodPoints: {goodPoints} or " 

843 f"in initialExpIdMask: {initialExpIdMask} are bad." 

844 f"Setting {ampName} to BAD.") 

845 self.log.warning(msg) 

846 # Fill entries with NaNs 

847 self.fillBadAmp(dataset, ptcFitType, ampName) 

848 continue 

849 

850 # Save the point where the variance starts decreasing as the 

851 # PTC turnoff point 

852 ptcTurnoff = meanVecOriginal[goodPoints][-1] 

853 dataset.ptcTurnoff[ampName] = ptcTurnoff 

854 

855 mask = goodPoints 

856 

857 if ptcFitType == 'EXPAPPROXIMATION': 

858 ptcFunc = funcAstier 

859 parsIniPtc = [-1e-9, 1.0, 10.] # a00, gain, noise^2 

860 # lowers and uppers obtained from BOT data studies by 

861 # C. Lage (UC Davis, 11/2020). 

862 bounds = self._boundsForAstier(parsIniPtc, lowers=[-1e-4, 0.5, -2000], 

863 uppers=[1e-4, 2.5, 2000]) 

864 if ptcFitType == 'POLYNOMIAL': 

865 ptcFunc = funcPolynomial 

866 parsIniPtc = self._initialParsForPolynomial(self.config.polynomialFitDegree + 1) 

867 bounds = self._boundsForPolynomial(parsIniPtc) 

868 

869 # Before bootstrap fit, do an iterative fit to get rid of outliers. 

870 # This further process of outlier rejection be skipped 

871 # if self.config.maxIterationsPtcOutliers = 0. 

872 # We already did some initial outlier rejection above in 

873 # self._getInitialGoodPoints. 

874 count = 1 

875 newMask = np.ones_like(meanVecOriginal, dtype=bool) 

876 pars = parsIniPtc 

877 while count <= maxIterationsPtcOutliers: 

878 # Note that application of the mask actually shrinks the array 

879 # to size rather than setting elements to zero (as we want) so 

880 # always update mask itself and re-apply to the original data 

881 meanTempVec = meanVecOriginal[mask] 

882 varTempVec = varVecOriginal[mask] 

883 res = least_squares(errFunc, parsIniPtc, bounds=bounds, args=(meanTempVec, varTempVec)) 

884 pars = res.x 

885 

886 # change this to the original from the temp because 

887 # the masks are ANDed meaning once a point is masked 

888 # it's always masked, and the masks must always be the 

889 # same length for broadcasting 

890 sigResids = (varVecOriginal - ptcFunc(pars, meanVecOriginal))/np.sqrt(varVecOriginal) 

891 newMask = np.array([True if np.abs(r) < sigmaCutPtcOutliers else False for r in sigResids]) 

892 mask = mask & newMask 

893 if not (mask.any() and newMask.any()): 

894 msg = (f"SERIOUS: All points in either mask: {mask} or newMask: {newMask} are bad. " 

895 f"Setting {ampName} to BAD.") 

896 self.log.warning(msg) 

897 # Fill entries with NaNs 

898 self.fillBadAmp(dataset, ptcFitType, ampName) 

899 break 

900 nDroppedTotal = Counter(mask)[False] 

901 self.log.debug("Iteration %d: discarded %d points in total for %s", 

902 count, nDroppedTotal, ampName) 

903 count += 1 

904 # objects should never shrink 

905 assert (len(mask) == len(timeVecOriginal) == len(meanVecOriginal) == len(varVecOriginal)) 

906 if not (mask.any() and newMask.any()): 

907 continue 

908 dataset.expIdMask[ampName] = np.array(dataset.expIdMask[ampName]) 

909 # store the final mask 

910 if len(dataset.expIdMask[ampName]): 

911 dataset.expIdMask[ampName] &= mask # bitwise_and if there is already a mask 

912 else: 

913 dataset.expIdMask[ampName] = mask 

914 parsIniPtc = pars 

915 meanVecFinal = meanVecOriginal[mask] 

916 varVecFinal = varVecOriginal[mask] 

917 

918 if Counter(mask)[False] > 0: 

919 self.log.info("Number of points discarded in PTC of amplifier %s:" 

920 " %d out of %d", ampName, Counter(mask)[False], len(meanVecOriginal)) 

921 

922 if (len(meanVecFinal) < len(parsIniPtc)): 

923 msg = (f"SERIOUS: Not enough data points ({len(meanVecFinal)}) compared to the number of " 

924 f"parameters of the PTC model({len(parsIniPtc)}). Setting {ampName} to BAD.") 

925 self.log.warning(msg) 

926 # Fill entries with NaNs 

927 self.fillBadAmp(dataset, ptcFitType, ampName) 

928 continue 

929 # Fit the PTC 

930 if self.config.doFitBootstrap: 

931 parsFit, parsFitErr, reducedChiSqPtc = fitBootstrap(parsIniPtc, meanVecFinal, 

932 varVecFinal, ptcFunc, 

933 weightsY=1./np.sqrt(varVecFinal)) 

934 else: 

935 parsFit, parsFitErr, reducedChiSqPtc = fitLeastSq(parsIniPtc, meanVecFinal, 

936 varVecFinal, ptcFunc, 

937 weightsY=1./np.sqrt(varVecFinal)) 

938 dataset.ptcFitPars[ampName] = parsFit 

939 dataset.ptcFitParsError[ampName] = parsFitErr 

940 dataset.ptcFitChiSq[ampName] = reducedChiSqPtc 

941 # Masked variances (measured and modeled) and means. Need 

942 # to pad the array so astropy.Table does not crash (the 

943 # mask may vary per amp). 

944 padLength = len(dataset.rawExpTimes[ampName]) - len(varVecFinal) 

945 dataset.finalVars[ampName] = np.pad(varVecFinal, (0, padLength), 'constant', 

946 constant_values=np.nan) 

947 dataset.finalModelVars[ampName] = np.pad(ptcFunc(parsFit, meanVecFinal), (0, padLength), 

948 'constant', constant_values=np.nan) 

949 dataset.finalMeans[ampName] = np.pad(meanVecFinal, (0, padLength), 'constant', 

950 constant_values=np.nan) 

951 if ptcFitType == 'EXPAPPROXIMATION': 

952 ptcGain = parsFit[1] 

953 ptcGainErr = parsFitErr[1] 

954 ptcNoise = np.sqrt(np.fabs(parsFit[2])) 

955 ptcNoiseErr = 0.5*(parsFitErr[2]/np.fabs(parsFit[2]))*np.sqrt(np.fabs(parsFit[2])) 

956 if ptcFitType == 'POLYNOMIAL': 

957 ptcGain = 1./parsFit[1] 

958 ptcGainErr = np.fabs(1./parsFit[1])*(parsFitErr[1]/parsFit[1]) 

959 ptcNoise = np.sqrt(np.fabs(parsFit[0]))*ptcGain 

960 ptcNoiseErr = (0.5*(parsFitErr[0]/np.fabs(parsFit[0]))*(np.sqrt(np.fabs(parsFit[0]))))*ptcGain 

961 dataset.gain[ampName] = ptcGain 

962 dataset.gainErr[ampName] = ptcGainErr 

963 dataset.noise[ampName] = ptcNoise 

964 dataset.noiseErr[ampName] = ptcNoiseErr 

965 

966 if not len(dataset.ptcFitType) == 0: 

967 dataset.ptcFitType = ptcFitType 

968 if len(dataset.badAmps) == 0: 

969 dataset.badAmps = np.repeat(np.nan, len(list(dataset.rawExpTimes.values())[0])) 

970 

971 return dataset 

972 

973 def fillBadAmp(self, dataset, ptcFitType, ampName): 

974 """Fill the dataset with NaNs if there are not enough 

975 good points. 

976 

977 Parameters 

978 ---------- 

979 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset` 

980 The dataset containing the means, variances and 

981 exposure times. 

982 ptcFitType : {'POLYNOMIAL', 'EXPAPPROXIMATION'} 

983 Fit a 'POLYNOMIAL' (degree: 'polynomialFitDegree') or 

984 'EXPAPPROXIMATION' (Eq. 16 of Astier+19) to the PTC. 

985 ampName : `str` 

986 Amplifier name. 

987 """ 

988 dataset.badAmps.append(ampName) 

989 dataset.expIdMask[ampName] = np.repeat(False, len(dataset.rawExpTimes[ampName])) 

990 dataset.gain[ampName] = np.nan 

991 dataset.gainErr[ampName] = np.nan 

992 dataset.noise[ampName] = np.nan 

993 dataset.noiseErr[ampName] = np.nan 

994 dataset.ptcFitPars[ampName] = (np.repeat(np.nan, self.config.polynomialFitDegree + 1) if 

995 ptcFitType in ["POLYNOMIAL", ] else np.repeat(np.nan, 3)) 

996 dataset.ptcFitParsError[ampName] = (np.repeat(np.nan, self.config.polynomialFitDegree + 1) if 

997 ptcFitType in ["POLYNOMIAL", ] else np.repeat(np.nan, 3)) 

998 dataset.ptcFitChiSq[ampName] = np.nan 

999 dataset.ptcTurnoff[ampName] = np.nan 

1000 dataset.finalVars[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName])) 

1001 dataset.finalModelVars[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName])) 

1002 dataset.finalMeans[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName])) 

1003 

1004 return