Coverage for python/lsst/cp/pipe/ptc/cpSolvePtcTask.py: 10%

451 statements  

« prev     ^ index     » next       coverage.py v7.2.7, created at 2023-07-12 12:06 -0700

1# This file is part of cp_pipe. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21# 

22import numpy as np 

23from collections import Counter 

24 

25import lsst.pex.config as pexConfig 

26import lsst.pipe.base as pipeBase 

27from lsst.cp.pipe.utils import (fitLeastSq, fitBootstrap, funcPolynomial, funcAstier, symmetrize) 

28 

29from scipy.signal import fftconvolve 

30from scipy.optimize import least_squares 

31from itertools import groupby 

32from operator import itemgetter 

33 

34import lsst.pipe.base.connectionTypes as cT 

35 

36from lsst.ip.isr import PhotonTransferCurveDataset 

37 

38from lsst.cp.pipe._lookupStaticCalibration import lookupStaticCalibration 

39 

40import copy 

41 

42 

43__all__ = ['PhotonTransferCurveSolveConfig', 'PhotonTransferCurveSolveTask'] 

44 

45 

46class PhotonTransferCurveSolveConnections(pipeBase.PipelineTaskConnections, 

47 dimensions=("instrument", "detector")): 

48 inputCovariances = cT.Input( 

49 name="ptcCovariances", 

50 doc="Tuple with measured covariances from flats.", 

51 storageClass="PhotonTransferCurveDataset", 

52 dimensions=("instrument", "exposure", "detector"), 

53 isCalibration=True, 

54 multiple=True, 

55 ) 

56 camera = cT.PrerequisiteInput( 

57 name="camera", 

58 doc="Camera the input data comes from.", 

59 storageClass="Camera", 

60 dimensions=("instrument",), 

61 isCalibration=True, 

62 lookupFunction=lookupStaticCalibration, 

63 ) 

64 outputPtcDataset = cT.Output( 

65 name="ptcDatsetProposal", 

66 doc="Output proposed ptc dataset.", 

67 storageClass="PhotonTransferCurveDataset", 

68 dimensions=("instrument", "detector"), 

69 multiple=False, 

70 isCalibration=True, 

71 ) 

72 

73 

74class PhotonTransferCurveSolveConfig(pipeBase.PipelineTaskConfig, 

75 pipelineConnections=PhotonTransferCurveSolveConnections): 

76 """Configuration for fitting measured covariances. 

77 """ 

78 

79 ptcFitType = pexConfig.ChoiceField( 

80 dtype=str, 

81 doc="Fit PTC to Eq. 16, Eq. 20 in Astier+19, or to a polynomial.", 

82 default="POLYNOMIAL", 

83 allowed={ 

84 "POLYNOMIAL": "n-degree polynomial (use 'polynomialFitDegree' to set 'n').", 

85 "EXPAPPROXIMATION": "Approximation in Astier+19 (Eq. 16).", 

86 "FULLCOVARIANCE": "Full covariances model in Astier+19 (Eq. 20)" 

87 } 

88 ) 

89 minMeanSignal = pexConfig.DictField( 

90 keytype=str, 

91 itemtype=float, 

92 doc="Minimum values (inclusive) of mean signal (in ADU) per amp to use." 

93 " The same cut is applied to all amps if this parameter [`dict`] is passed as " 

94 " {'ALL_AMPS': value}", 

95 default={'ALL_AMPS': 0.0}, 

96 ) 

97 maxMeanSignal = pexConfig.DictField( 

98 keytype=str, 

99 itemtype=float, 

100 doc="Maximum values (inclusive) of mean signal (in ADU) below which to consider, per amp." 

101 " The same cut is applied to all amps if this dictionary is of the form" 

102 " {'ALL_AMPS': value}", 

103 default={'ALL_AMPS': 1e6}, 

104 ) 

105 maximumRangeCovariancesAstier = pexConfig.Field( 

106 dtype=int, 

107 doc="Maximum range of covariances as in Astier+19", 

108 default=8, 

109 ) 

110 sigmaClipFullFitCovariancesAstier = pexConfig.Field( 

111 dtype=float, 

112 doc="sigma clip for full model fit for FULLCOVARIANCE ptcFitType ", 

113 default=5.0, 

114 ) 

115 maxIterFullFitCovariancesAstier = pexConfig.Field( 

116 dtype=int, 

117 doc="Maximum number of iterations in full model fit for FULLCOVARIANCE ptcFitType", 

118 default=3, 

119 ) 

120 polynomialFitDegree = pexConfig.Field( 

121 dtype=int, 

122 doc="Degree of polynomial to fit the PTC, when 'ptcFitType'=POLYNOMIAL.", 

123 default=3, 

124 ) 

125 doLegacyTurnoffSelection = pexConfig.Field( 

126 dtype=bool, 

127 doc="Use 'legacy' computation for PTC turnoff selection. If set " 

128 "to False, then the KS test p-value selection will be used instead.", 

129 default=False, 

130 ) 

131 sigmaCutPtcOutliers = pexConfig.Field( 

132 dtype=float, 

133 doc="Sigma cut for outlier rejection in PTC.", 

134 default=5.0, 

135 ) 

136 maxIterationsPtcOutliers = pexConfig.RangeField( 

137 dtype=int, 

138 doc="Maximum number of iterations for outlier rejection in PTC.", 

139 default=2, 

140 min=0 

141 ) 

142 maxSignalInitialPtcOutlierFit = pexConfig.Field( 

143 dtype=float, 

144 doc="Maximum signal considered for intial outlier fit. This should be below " 

145 "the PTC turnoff to ensure accurate outlier rejection.", 

146 default=30_000., 

147 ) 

148 minVarPivotSearch = pexConfig.Field( 

149 dtype=float, 

150 doc="The code looks for a pivot signal point after which the variance starts decreasing at high-flux" 

151 " to exclude then from the PTC model fit. However, sometimes at low fluxes, the variance" 

152 " decreases slightly. Set this variable for the variance value, in ADU^2, after which the pivot " 

153 " should be sought. Only used if doLegacyTurnoffSelection is True.", 

154 default=10000, 

155 ) 

156 consecutivePointsVarDecreases = pexConfig.RangeField( 

157 dtype=int, 

158 doc="Required number of consecutive points/fluxes in the PTC where the variance " 

159 "decreases in order to find a first estimate of the PTC turn-off. " 

160 "Only used if doLegacyTurnoffSelection is True.", 

161 default=2, 

162 min=2 

163 ) 

164 ksTestMinPvalue = pexConfig.Field( 

165 dtype=float, 

166 doc="Minimum value of the Gaussian histogram KS test p-value to be used in PTC fit. " 

167 "Only used if doLegacyTurnoffSelection is False.", 

168 default=0.01, 

169 ) 

170 doFitBootstrap = pexConfig.Field( 

171 dtype=bool, 

172 doc="Use bootstrap for the PTC fit parameters and errors?.", 

173 default=False, 

174 ) 

175 binSize = pexConfig.Field( 

176 dtype=int, 

177 doc="Bin the image by this factor in both dimensions.", 

178 default=1, 

179 ) 

180 

181 

182class PhotonTransferCurveSolveTask(pipeBase.PipelineTask): 

183 """Task to fit the PTC from flat covariances. 

184 

185 The first task of the PTC measurement pipeline, 

186 ``PhotonTransferCurveMeasureTask`` (and assumed to have been run 

187 before this task), produced a list of 

188 `~lsst.ip.isr.PhotonTransferCurveDataset` objects. Each dataset 

189 contains the mean signal and covariances of the 

190 difference image of the flat-field images taken at 

191 the same exposure time. The list also contains dummy 

192 datasets (with no measurements), whose purpose is to have 

193 the input and output dimensions of ``PhotonTransferCurveMeasureTask`` 

194 match. 

195 

196 This task, ``PhotonTransferCurveSolveTask``, assembles the list 

197 of individual PTC datasets produced 

198 by ``PhotonTransferCurveMeasureTask`` into one single final PTC 

199 dataset, discarding the dummy datset as appropiate. 

200 The task fits the measured (co)variances to one of three models: 

201 a polynomial model of a given order, or the models described 

202 in equations 16 and 20 of Astier+19. These options are referred 

203 to as ``POLYNOMIAL``, ``EXPAPPROXIMATION``, and ``FULLCOVARIANCE`` 

204 in the configuration options of the task, respectively). 

205 Parameters of interest such as the gain and noise are derived 

206 from the fits. The ``FULLCOVARIANCE`` model is fitted to the 

207 full covariance data (as oppossed to the other two models, which 

208 are fit to the variance vs mean measurements only). 

209 

210 Astier+19: "The Shape of the Photon Transfer Curve 

211 of CCD sensors", arXiv:1905.08677 

212 """ 

213 

214 ConfigClass = PhotonTransferCurveSolveConfig 

215 _DefaultName = 'cpPhotonTransferCurveSolve' 

216 

217 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

218 """Ensure that the input and output dimensions are passed along. 

219 

220 Parameters 

221 ---------- 

222 butlerQC : `~lsst.daf.butler.butlerQuantumContext.ButlerQuantumContext` 

223 Butler to operate on. 

224 inputRefs : `~lsst.pipe.base.connections.InputQuantizedConnection` 

225 Input data refs to load. 

226 ouptutRefs : `~lsst.pipe.base.connections.OutputQuantizedConnection` 

227 Output data refs to persist. 

228 """ 

229 inputs = butlerQC.get(inputRefs) 

230 detId = inputRefs.inputCovariances[0].dataId['detector'] 

231 outputs = self.run(inputCovariances=inputs['inputCovariances'], camera=inputs['camera'], detId=detId) 

232 butlerQC.put(outputs, outputRefs) 

233 

234 def run(self, inputCovariances, camera=None, detId=0): 

235 """Fit measured covariances to different models. 

236 

237 Parameters 

238 ---------- 

239 inputCovariances : `list` [`lsst.ip.isr.PhotonTransferCurveDataset`] 

240 List of lsst.ip.isr.PhotonTransferCurveDataset datasets. 

241 camera : `lsst.afw.cameraGeom.Camera`, optional 

242 Input camera. 

243 detId : `int` 

244 Detector ID to locate the detector in the camera and 

245 populate the `lsst.ip.isr.PhotonTransferCurveDataset` 

246 metadata. 

247 Returns 

248 ------- 

249 results : `lsst.pipe.base.Struct` 

250 The resultins structure contains: 

251 

252 ``outputPtcDatset`` 

253 Final PTC dataset, containing information such as the 

254 means, variances, and exposure times 

255 (`lsst.ip.isr.PhotonTransferCurveDataset`). 

256 """ 

257 # Find the ampNames from a non-dummy ptc. 

258 ampNames = [] 

259 for partialPtcDataset in inputCovariances: 

260 if partialPtcDataset.ptcFitType != 'DUMMY': 

261 ampNames = partialPtcDataset.ampNames 

262 break 

263 

264 # Each amp may have a different min and max ADU signal 

265 # specified in the config. 

266 maxMeanSignalDict = {ampName: 1e6 for ampName in ampNames} 

267 minMeanSignalDict = {ampName: 0.0 for ampName in ampNames} 

268 for ampName in ampNames: 

269 if 'ALL_AMPS' in self.config.maxMeanSignal: 

270 maxMeanSignalDict[ampName] = self.config.maxMeanSignal['ALL_AMPS'] 

271 elif ampName in self.config.maxMeanSignal: 

272 maxMeanSignalDict[ampName] = self.config.maxMeanSignal[ampName] 

273 

274 if 'ALL_AMPS' in self.config.minMeanSignal: 

275 minMeanSignalDict[ampName] = self.config.minMeanSignal['ALL_AMPS'] 

276 elif ampName in self.config.minMeanSignal: 

277 minMeanSignalDict[ampName] = self.config.minMeanSignal[ampName] 

278 

279 # Assemble individual PTC datasets into a single PTC dataset. 

280 datasetPtc = PhotonTransferCurveDataset(ampNames=ampNames, 

281 ptcFitType=self.config.ptcFitType, 

282 covMatrixSide=self.config.maximumRangeCovariancesAstier) 

283 for partialPtcDataset in inputCovariances: 

284 # Ignore dummy datasets 

285 if partialPtcDataset.ptcFitType == 'DUMMY': 

286 continue 

287 for ampName in ampNames: 

288 # The partial dataset consists of lists of values for each 

289 # quantity. In the case of the input exposure pairs, this is a 

290 # list of tuples. In all cases we only want the first 

291 # (and only) element of the list. 

292 datasetPtc.inputExpIdPairs[ampName].append(partialPtcDataset.inputExpIdPairs[ampName][0]) 

293 datasetPtc.rawExpTimes[ampName] = np.append(datasetPtc.rawExpTimes[ampName], 

294 partialPtcDataset.rawExpTimes[ampName][0]) 

295 datasetPtc.rawMeans[ampName] = np.append(datasetPtc.rawMeans[ampName], 

296 partialPtcDataset.rawMeans[ampName][0]) 

297 datasetPtc.rawVars[ampName] = np.append(datasetPtc.rawVars[ampName], 

298 partialPtcDataset.rawVars[ampName][0]) 

299 datasetPtc.histVars[ampName] = np.append(datasetPtc.histVars[ampName], 

300 partialPtcDataset.histVars[ampName][0]) 

301 datasetPtc.histChi2Dofs[ampName] = np.append(datasetPtc.histChi2Dofs[ampName], 

302 partialPtcDataset.histChi2Dofs[ampName][0]) 

303 datasetPtc.kspValues[ampName] = np.append(datasetPtc.kspValues[ampName], 

304 partialPtcDataset.kspValues[ampName][0]) 

305 datasetPtc.covariances[ampName] = np.append( 

306 datasetPtc.covariances[ampName].ravel(), 

307 partialPtcDataset.covariances[ampName].ravel() 

308 ).reshape( 

309 ( 

310 len(datasetPtc.rawExpTimes[ampName]), 

311 datasetPtc.covMatrixSide, 

312 datasetPtc.covMatrixSide, 

313 ) 

314 ) 

315 datasetPtc.covariancesSqrtWeights[ampName] = np.append( 

316 datasetPtc.covariancesSqrtWeights[ampName].ravel(), 

317 partialPtcDataset.covariancesSqrtWeights[ampName].ravel() 

318 ).reshape( 

319 ( 

320 len(datasetPtc.rawExpTimes[ampName]), 

321 datasetPtc.covMatrixSide, 

322 datasetPtc.covMatrixSide, 

323 ) 

324 ) 

325 

326 # Apply min/max masking. 

327 rawMean = partialPtcDataset.rawMeans[ampName][0] 

328 rawVar = partialPtcDataset.rawVars[ampName][0] 

329 expIdMask = partialPtcDataset.expIdMask[ampName][0] 

330 if (rawMean <= minMeanSignalDict[ampName]) or (rawMean >= maxMeanSignalDict[ampName]) \ 

331 or not np.isfinite(rawMean) or not np.isfinite(rawVar): 

332 expIdMask = False 

333 

334 kspValue = partialPtcDataset.kspValues[ampName][0] 

335 if not self.config.doLegacyTurnoffSelection and \ 

336 kspValue < self.config.ksTestMinPvalue: 

337 expIdMask = False 

338 

339 datasetPtc.expIdMask[ampName] = np.append(datasetPtc.expIdMask[ampName], expIdMask) 

340 

341 for key, value in partialPtcDataset.auxValues.items(): 

342 if key in datasetPtc.auxValues: 

343 datasetPtc.auxValues[key] = np.append(datasetPtc.auxValues[key], value) 

344 else: 

345 datasetPtc.auxValues[key] = value 

346 

347 # Sort arrays that are filled so far in the final dataset by 

348 # rawMeans index. 

349 # First compute the mean across all the amps to make sure that they are 

350 # all sorted the same way. 

351 detectorMeans = np.zeros(len(datasetPtc.inputExpIdPairs[ampNames[0]])) 

352 

353 for i in range(len(detectorMeans)): 

354 arr = np.array([datasetPtc.rawMeans[ampName][i] for ampName in ampNames]) 

355 good, = (np.isfinite(arr)).nonzero() 

356 if good.size == 0: 

357 detectorMeans[i] = np.nan 

358 else: 

359 detectorMeans[i] = np.mean(arr[good]) 

360 

361 index = np.argsort(detectorMeans) 

362 

363 for ampName in ampNames: 

364 datasetPtc.inputExpIdPairs[ampName] = np.array( 

365 datasetPtc.inputExpIdPairs[ampName] 

366 )[index].tolist() 

367 datasetPtc.rawExpTimes[ampName] = datasetPtc.rawExpTimes[ampName][index] 

368 datasetPtc.rawMeans[ampName] = datasetPtc.rawMeans[ampName][index] 

369 datasetPtc.rawVars[ampName] = datasetPtc.rawVars[ampName][index] 

370 datasetPtc.histVars[ampName] = datasetPtc.histVars[ampName][index] 

371 datasetPtc.histChi2Dofs[ampName] = datasetPtc.histChi2Dofs[ampName][index] 

372 datasetPtc.kspValues[ampName] = datasetPtc.kspValues[ampName][index] 

373 datasetPtc.expIdMask[ampName] = datasetPtc.expIdMask[ampName][index] 

374 datasetPtc.covariances[ampName] = datasetPtc.covariances[ampName][index] 

375 datasetPtc.covariancesSqrtWeights[ampName] = datasetPtc.covariancesSqrtWeights[ampName][index] 

376 for key, value in datasetPtc.auxValues.items(): 

377 datasetPtc.auxValues[key] = value[index] 

378 

379 if self.config.ptcFitType == "FULLCOVARIANCE": 

380 # Fit the measured covariances vs mean signal to 

381 # the Astier+19 full model (Eq. 20). Before that 

382 # do a preliminary fit to the variance (C_00) vs mean 

383 # signal (mu) curve using the EXPAPPROXIMATION model 

384 # (Eq. 16 in Astier+19) in order to 

385 # get the flat pairs that are masked. The 

386 # points at these fluxes will also be masked when 

387 # calculating the other elements of the covariance 

388 # matrix, C_ij, i!=j). 

389 

390 # Preliminary fit, usign a temp dataset to get the mask 

391 tempDatasetPtc = copy.copy(datasetPtc) 

392 tempDatasetPtc.ptcFitType = "EXPAPPROXIMATION" 

393 tempDatasetPtc = self.fitMeasurementsToModel(tempDatasetPtc) 

394 

395 # "FULLCOVARIANCE", using the mask obtained from the 

396 # previous fit. 

397 for ampName in datasetPtc.ampNames: 

398 datasetPtc.expIdMask[ampName] = tempDatasetPtc.expIdMask[ampName] 

399 datasetPtc.fitType = "FULLCOVARIANCE" 

400 datasetPtc = self.fitMeasurementsToModel(datasetPtc) 

401 # The other options are: self.config.ptcFitType in 

402 # ("EXPAPPROXIMATION", "POLYNOMIAL") 

403 else: 

404 # Fit the PTC to a polynomial or to Astier+19 exponential 

405 # approximation (Eq. 16). Fill up 

406 # PhotonTransferCurveDataset object. 

407 datasetPtc = self.fitMeasurementsToModel(datasetPtc) 

408 

409 if camera: 

410 detector = camera[detId] 

411 else: 

412 detector = None 

413 datasetPtc.updateMetadataFromExposures(inputCovariances) 

414 datasetPtc.updateMetadata(setDate=True, camera=camera, detector=detector) 

415 

416 return pipeBase.Struct( 

417 outputPtcDataset=datasetPtc, 

418 ) 

419 

420 def fitMeasurementsToModel(self, dataset): 

421 """Fit the measured covariances vs mean signal to a 

422 polynomial or one of the models in Astier+19 

423 (Eq. 16 or Eq.20). 

424 

425 Parameters 

426 ---------- 

427 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset` 

428 The dataset containing information such as the means, 

429 (co)variances, and exposure times. 

430 

431 Returns 

432 ------- 

433 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset` 

434 This is the same dataset as the input parameter, however, 

435 it has been modified to include information such as the 

436 fit vectors and the fit parameters. See the class 

437 `PhotonTransferCurveDatase`. 

438 """ 

439 fitType = dataset.ptcFitType 

440 if fitType in ["FULLCOVARIANCE", ]: 

441 # This model uses the full covariance matrix in the fit. 

442 # The PTC is technically defined as variance vs signal, 

443 # with variance = Cov_00 

444 dataset = self.fitDataFullCovariance(dataset) 

445 elif fitType in ["POLYNOMIAL", "EXPAPPROXIMATION"]: 

446 # The PTC is technically defined as variance vs signal 

447 dataset = self.fitPtc(dataset) 

448 else: 

449 raise RuntimeError( 

450 f"Fitting option {fitType} not one of " 

451 "'POLYNOMIAL', 'EXPAPPROXIMATION', or 'FULLCOVARIANCE'" 

452 ) 

453 

454 return dataset 

455 

456 def fitDataFullCovariance(self, dataset): 

457 """Fit measured flat covariances to the full model in 

458 Astier+19 (Eq. 20). 

459 

460 Parameters 

461 ---------- 

462 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset` 

463 The dataset containing information such as the means, 

464 (co)variances, and exposure times. 

465 

466 Returns 

467 ------- 

468 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset` 

469 This is the same dataset as the input parameter, however, 

470 it has been modified to include information such as the 

471 fit vectors and the fit parameters. See the class 

472 `PhotonTransferCurveDatase`. 

473 

474 Notes 

475 ----- 

476 The parameters of the full model for C_ij(mu) ("C_ij" and "mu" 

477 in ADU^2 and ADU, respectively) in Astier+19 (Eq. 20) are: 

478 

479 - "a" coefficients (r by r matrix), units: 1/e 

480 - "b" coefficients (r by r matrix), units: 1/e 

481 - noise matrix (r by r matrix), units: e^2 

482 - gain, units: e/ADU 

483 

484 "b" appears in Eq. 20 only through the "ab" combination, which 

485 is defined in this code as "c=ab". 

486 

487 Total number of parameters: #entries(a) + #entries(c) + #entries(noise) 

488 + 1. This is equivalent to r^2 + r^2 + r^2 + 1, where "r" is the 

489 maximum lag considered for the covariances calculation, and the 

490 extra "1" is the gain. If "b" is 0, then "c" is 0, and len(pInit) will 

491 have r^2 fewer entries. 

492 """ 

493 matrixSide = self.config.maximumRangeCovariancesAstier 

494 lenParams = matrixSide*matrixSide 

495 

496 for ampName in dataset.ampNames: 

497 lenInputTimes = len(dataset.rawExpTimes[ampName]) 

498 # Not used when ptcFitType is 'FULLCOVARIANCE' 

499 dataset.ptcFitPars[ampName] = np.array([np.nan]) 

500 dataset.ptcFitParsError[ampName] = np.array([np.nan]) 

501 dataset.ptcFitChiSq[ampName] = np.nan 

502 

503 if ampName in dataset.badAmps: 

504 # Bad amp 

505 # Entries need to have proper dimensions so read/write 

506 # with astropy.Table works. 

507 nanMatrix = np.full((matrixSide, matrixSide), np.nan) 

508 listNanMatrix = np.full((lenInputTimes, matrixSide, matrixSide), np.nan) 

509 dataset.covariancesModel[ampName] = listNanMatrix 

510 dataset.covariancesSqrtWeights[ampName] = listNanMatrix 

511 dataset.aMatrix[ampName] = nanMatrix 

512 dataset.bMatrix[ampName] = nanMatrix 

513 dataset.covariancesModelNoB[ampName] = listNanMatrix 

514 dataset.aMatrixNoB[ampName] = nanMatrix 

515 dataset.noiseMatrix[ampName] = nanMatrix 

516 dataset.noiseMatrixNoB[ampName] = nanMatrix 

517 

518 dataset.expIdMask[ampName] = np.repeat(False, lenInputTimes) 

519 dataset.gain[ampName] = np.nan 

520 dataset.gainErr[ampName] = np.nan 

521 dataset.noise[ampName] = np.nan 

522 dataset.noiseErr[ampName] = np.nan 

523 dataset.finalVars[ampName] = np.repeat(np.nan, lenInputTimes) 

524 dataset.finalModelVars[ampName] = np.repeat(np.nan, lenInputTimes) 

525 dataset.finalMeans[ampName] = np.repeat(np.nan, lenInputTimes) 

526 continue 

527 

528 muAtAmp = dataset.rawMeans[ampName] 

529 maskAtAmp = dataset.expIdMask[ampName] 

530 if len(maskAtAmp) == 0: 

531 maskAtAmp = np.repeat(True, len(muAtAmp)) 

532 

533 muAtAmpMasked = muAtAmp[maskAtAmp] 

534 covAtAmp = dataset.covariances[ampName] 

535 covAtAmpMasked = np.nan_to_num(covAtAmp)[maskAtAmp] 

536 covSqrtWeightsAtAmp = dataset.covariancesSqrtWeights[ampName] 

537 covSqrtWeightsAtAmpMasked = np.nan_to_num(covSqrtWeightsAtAmp)[maskAtAmp] 

538 

539 # Initial fit, to approximate parameters, with c=0 

540 a0, c0, noise0, gain0 = self.initialFitFullCovariance( 

541 muAtAmpMasked, 

542 covAtAmpMasked, 

543 covSqrtWeightsAtAmpMasked 

544 ) 

545 

546 # Fit full model (Eq. 20 of Astier+19) and same model with 

547 # b=0 (c=0 in this code) 

548 pInit = np.concatenate((a0.ravel(), c0.ravel(), noise0.ravel(), np.array(gain0)), axis=None) 

549 functionsDict = {'fullModel': self.funcFullCovarianceModel, 

550 'fullModelNoB': self.funcFullCovarianceModelNoB} 

551 fitResults = {'fullModel': {'a': [], 'c': [], 'noise': [], 'gain': [], 'paramsErr': []}, 

552 'fullModelNoB': {'a': [], 'c': [], 'noise': [], 'gain': [], 'paramsErr': []}} 

553 for key in functionsDict: 

554 params, paramsErr, _ = fitLeastSq(pInit, muAtAmpMasked, 

555 covAtAmpMasked.ravel(), functionsDict[key], 

556 weightsY=covSqrtWeightsAtAmpMasked.ravel()) 

557 a = params[:lenParams].reshape((matrixSide, matrixSide)) 

558 c = params[lenParams:2*lenParams].reshape((matrixSide, matrixSide)) 

559 noise = params[2*lenParams:3*lenParams].reshape((matrixSide, matrixSide)) 

560 gain = params[-1] 

561 

562 fitResults[key]['a'] = a 

563 fitResults[key]['c'] = c 

564 fitResults[key]['noise'] = noise 

565 fitResults[key]['gain'] = gain 

566 fitResults[key]['paramsErr'] = paramsErr 

567 

568 # Put the information in the PTC dataset 

569 

570 # Not used when ptcFitType is 'FULLCOVARIANCE' 

571 dataset.ptcFitPars[ampName] = np.array([np.nan]) 

572 dataset.ptcFitParsError[ampName] = np.array([np.nan]) 

573 dataset.ptcFitChiSq[ampName] = np.nan 

574 

575 # Save full covariances, covariances models, and their weights. 

576 # dataset.expIdMask is already full, but needs to be 

577 # converted to bool. 

578 dataset.expIdMask[ampName] = np.array(dataset.expIdMask[ampName], dtype=bool) 

579 dataset.covariances[ampName] = covAtAmp 

580 # We evaluate the covariance model everywhere, even the 

581 # masked amps. 

582 dataset.covariancesModel[ampName] = self.evalCovModel(muAtAmp, 

583 fitResults['fullModel']['a'], 

584 fitResults['fullModel']['c'], 

585 fitResults['fullModel']['noise'], 

586 fitResults['fullModel']['gain']) 

587 dataset.covariancesSqrtWeights[ampName] = covSqrtWeightsAtAmp 

588 dataset.aMatrix[ampName] = fitResults['fullModel']['a'] 

589 dataset.bMatrix[ampName] = fitResults['fullModel']['c']/fitResults['fullModel']['a'] 

590 dataset.covariancesModelNoB[ampName] = self.evalCovModel(muAtAmp, 

591 fitResults['fullModelNoB']['a'], 

592 fitResults['fullModelNoB']['c'], 

593 fitResults['fullModelNoB']['noise'], 

594 fitResults['fullModelNoB']['gain'], 

595 setBtoZero=True) 

596 dataset.aMatrixNoB[ampName] = fitResults['fullModelNoB']['a'] 

597 dataset.gain[ampName] = fitResults['fullModel']['gain'] 

598 dataset.gainErr[ampName] = fitResults['fullModel']['paramsErr'][-1] 

599 readoutNoise = fitResults['fullModel']['noise'][0][0] 

600 readoutNoiseSqrt = np.sqrt(np.fabs(readoutNoise)) 

601 dataset.noise[ampName] = readoutNoise 

602 readoutNoiseSigma = fitResults['fullModel']['paramsErr'][2*lenParams] 

603 dataset.noiseErr[ampName] = 0.5*(readoutNoiseSigma/np.fabs(readoutNoise))*readoutNoiseSqrt 

604 dataset.noiseMatrix[ampName] = fitResults['fullModel']['noise'] 

605 dataset.noiseMatrixNoB[ampName] = fitResults['fullModelNoB']['noise'] 

606 

607 dataset.finalVars[ampName] = covAtAmp[:, 0, 0] 

608 dataset.finalModelVars[ampName] = dataset.covariancesModel[ampName][:, 0, 0] 

609 dataset.finalMeans[ampName] = muAtAmp 

610 

611 return dataset 

612 

613 def initialFitFullCovariance(self, mu, cov, sqrtW): 

614 """ Performs a crude parabolic fit of the data in order to start 

615 the full fit close to the solution, setting b=0 (c=0) in Eq. 20 

616 of Astier+19. 

617 

618 Parameters 

619 ---------- 

620 mu : `numpy.array`, (N,) 

621 Signal `mu` (ADU) 

622 cov : `numpy.array`, (N, M, M) 

623 Covariance arrays of size `(M, M)` (with 

624 `M = config.maximumRangeCovariancesAstier`), 

625 indexed by mean signal `mu`. 

626 sqrtW : `numpy.array`, (N,) 

627 Covariance weights, defined as 1./sqrt(Variances) 

628 

629 Returns 

630 ------- 

631 a : `numpy.array`, (M, M) 

632 "a" parameter per flux in Eq. 20 of Astier+19. 

633 c : `numpy.array`, (M, M) 

634 "c"="ab" parameter per flux in Eq. 20 of Astier+19. 

635 noise : `numpy.array`, (M, M) 

636 "noise" parameter per flux in Eq. 20 of Astier+19. 

637 gain : `float` 

638 Amplifier gain (e/ADU) 

639 """ 

640 matrixSide = self.config.maximumRangeCovariancesAstier 

641 

642 # Initialize fit parameters 

643 a = np.zeros((matrixSide, matrixSide)) 

644 c = np.zeros((matrixSide, matrixSide)) 

645 noise = np.zeros((matrixSide, matrixSide)) 

646 gain = 1. 

647 

648 # iterate the fit to account for higher orders 

649 # the chi2 does not necessarily go down, so one could 

650 # stop when it increases 

651 oldChi2 = 1e30 

652 for _ in range(5): 

653 model = np.nan_to_num(self.evalCovModel(mu, a, c, noise, gain, setBtoZero=True)) 

654 # loop on lags 

655 for i in range(matrixSide): 

656 for j in range(matrixSide): 

657 # fit a parabola for a given lag 

658 parsFit = np.polyfit(mu, cov[:, i, j] - model[:, i, j], 

659 2, w=sqrtW[:, i, j]) 

660 # model equation (Eq. 20) in Astier+19, with c=a*b=0: 

661 a[i, j] += parsFit[0] 

662 noise[i, j] += parsFit[2] 

663 if(i + j == 0): 

664 gain = 1./(1/gain+parsFit[1]) 

665 weightedRes = (model - cov)*sqrtW 

666 chi2 = (weightedRes.flatten()**2).sum() 

667 if chi2 > oldChi2: 

668 break 

669 oldChi2 = chi2 

670 

671 return a, c, noise, gain 

672 

673 def funcFullCovarianceModel(self, params, x): 

674 """Model to fit covariances from flat fields; Equation 20 of 

675 Astier+19. 

676 

677 Parameters 

678 ---------- 

679 params : `list` 

680 Parameters of the model: aMatrix, CMatrix, noiseMatrix, 

681 gain (e/ADU). 

682 x : `numpy.array`, (N,) 

683 Signal `mu` (ADU) 

684 

685 Returns 

686 ------- 

687 y : `numpy.array`, (N,) 

688 Covariance matrix. 

689 """ 

690 matrixSide = self.config.maximumRangeCovariancesAstier 

691 lenParams = matrixSide*matrixSide 

692 aMatrix = params[:lenParams].reshape((matrixSide, matrixSide)) 

693 cMatrix = params[lenParams:2*lenParams].reshape((matrixSide, matrixSide)) 

694 noiseMatrix = params[2*lenParams:3*lenParams].reshape((matrixSide, matrixSide)) 

695 gain = params[-1] 

696 

697 return self.evalCovModel(x, aMatrix, cMatrix, noiseMatrix, gain).flatten() 

698 

699 def funcFullCovarianceModelNoB(self, params, x): 

700 """Model to fit covariances from flat fields; Equation 20 of 

701 Astier+19, with b=0 (equivalent to c=a*b=0 in this code). 

702 

703 Parameters 

704 ---------- 

705 params : `list` 

706 Parameters of the model: aMatrix, CMatrix, noiseMatrix, 

707 gain (e/ADU). 

708 x : `numpy.array`, (N,) 

709 Signal mu (ADU) 

710 

711 Returns 

712 ------- 

713 y : `numpy.array`, (N,) 

714 Covariance matrix. 

715 """ 

716 matrixSide = self.config.maximumRangeCovariancesAstier 

717 lenParams = matrixSide*matrixSide 

718 aMatrix = params[:lenParams].reshape((matrixSide, matrixSide)) 

719 cMatrix = params[lenParams:2*lenParams].reshape((matrixSide, matrixSide)) 

720 noiseMatrix = params[2*lenParams:3*lenParams].reshape((matrixSide, matrixSide)) 

721 gain = params[-1] 

722 

723 return self.evalCovModel(x, aMatrix, cMatrix, noiseMatrix, gain, setBtoZero=True).flatten() 

724 

725 def evalCovModel(self, mu, aMatrix, cMatrix, noiseMatrix, gain, setBtoZero=False): 

726 """Computes full covariances model (Eq. 20 of Astier+19). 

727 

728 Parameters 

729 ---------- 

730 mu : `numpy.array`, (N,) 

731 List of mean signals. 

732 aMatrix : `numpy.array`, (M, M) 

733 "a" parameter per flux in Eq. 20 of Astier+19. 

734 cMatrix : `numpy.array`, (M, M) 

735 "c"="ab" parameter per flux in Eq. 20 of Astier+19. 

736 noiseMatrix : `numpy.array`, (M, M) 

737 "noise" parameter per flux in Eq. 20 of Astier+19. 

738 gain : `float` 

739 Amplifier gain (e/ADU) 

740 setBtoZero=False : `bool`, optional 

741 Set "b" parameter in full model (see Astier+19) to zero. 

742 

743 Returns 

744 ------- 

745 covModel : `numpy.array`, (N, M, M) 

746 Covariances model. 

747 

748 Notes 

749 ----- 

750 By default, computes the covModel for the mu's stored(self.mu). 

751 Returns cov[Nmu, M, M]. The variance for the PTC is 

752 cov[:, 0, 0]. mu and cov are in ADUs and ADUs squared. To use 

753 electrons for both, the gain should be set to 1. This routine 

754 implements the model in Astier+19 (1905.08677). 

755 The parameters of the full model for C_ij(mu) ("C_ij" and "mu" 

756 in ADU^2 and ADU, respectively) in Astier+19 (Eq. 20) are: 

757 

758 - "a" coefficients (M by M matrix), units: 1/e 

759 - "b" coefficients (M by M matrix), units: 1/e 

760 - noise matrix (M by M matrix), units: e^2 

761 - gain, units: e/ADU 

762 

763 "b" appears in Eq. 20 only through the "ab" combination, which 

764 is defined in this code as "c=ab". 

765 """ 

766 matrixSide = self.config.maximumRangeCovariancesAstier 

767 sa = (matrixSide, matrixSide) 

768 # pad a with zeros and symmetrize 

769 aEnlarged = np.zeros((int(sa[0]*1.5)+1, int(sa[1]*1.5)+1)) 

770 aEnlarged[0:sa[0], 0:sa[1]] = aMatrix 

771 aSym = symmetrize(aEnlarged) 

772 # pad c with zeros and symmetrize 

773 cEnlarged = np.zeros((int(sa[0]*1.5)+1, int(sa[1]*1.5)+1)) 

774 cEnlarged[0:sa[0], 0:sa[1]] = cMatrix 

775 cSym = symmetrize(cEnlarged) 

776 a2 = fftconvolve(aSym, aSym, mode='same') 

777 a3 = fftconvolve(a2, aSym, mode='same') 

778 ac = fftconvolve(aSym, cSym, mode='same') 

779 (xc, yc) = np.unravel_index(np.abs(aSym).argmax(), a2.shape) 

780 

781 a1 = aMatrix[np.newaxis, :, :] 

782 a2 = a2[np.newaxis, xc:xc + matrixSide, yc:yc + matrixSide] 

783 a3 = a3[np.newaxis, xc:xc + matrixSide, yc:yc + matrixSide] 

784 ac = ac[np.newaxis, xc:xc + matrixSide, yc:yc + matrixSide] 

785 c1 = cMatrix[np.newaxis, ::] 

786 

787 # assumes that mu is 1d 

788 bigMu = mu[:, np.newaxis, np.newaxis]*gain 

789 # c(=a*b in Astier+19) also has a contribution to the last 

790 # term, that is absent for now. 

791 if setBtoZero: 

792 c1 = np.zeros_like(c1) 

793 ac = np.zeros_like(ac) 

794 covModel = (bigMu/(gain*gain)*(a1*bigMu+2./3.*(bigMu*bigMu)*(a2 + c1) 

795 + (1./3.*a3 + 5./6.*ac)*(bigMu*bigMu*bigMu)) + noiseMatrix[np.newaxis, :, :]/gain**2) 

796 # add the Poisson term, and the read out noise (variance) 

797 covModel[:, 0, 0] += mu/gain 

798 

799 return covModel 

800 

801 # EXPAPPROXIMATION and POLYNOMIAL fit methods 

802 @staticmethod 

803 def _initialParsForPolynomial(order): 

804 assert(order >= 2) 

805 pars = np.zeros(order, dtype=float) 

806 pars[0] = 10 

807 pars[1] = 1 

808 pars[2:] = 0.0001 

809 return pars 

810 

811 @staticmethod 

812 def _boundsForPolynomial(initialPars, lowers=[], uppers=[]): 

813 if not len(lowers): 

814 lowers = [np.NINF for p in initialPars] 

815 if not len(uppers): 

816 uppers = [np.inf for p in initialPars] 

817 lowers[1] = 0 # no negative gains 

818 return (lowers, uppers) 

819 

820 @staticmethod 

821 def _boundsForAstier(initialPars, lowers=[], uppers=[]): 

822 if not len(lowers): 

823 lowers = [np.NINF for p in initialPars] 

824 if not len(uppers): 

825 uppers = [np.inf for p in initialPars] 

826 return (lowers, uppers) 

827 

828 @staticmethod 

829 def _getInitialGoodPoints(means, variances, minVarPivotSearch, consecutivePointsVarDecreases): 

830 """Return a boolean array to mask bad points. 

831 

832 Parameters 

833 ---------- 

834 means : `numpy.array` 

835 Input array with mean signal values. 

836 variances : `numpy.array` 

837 Input array with variances at each mean value. 

838 minVarPivotSearch : `float` 

839 The variance (in ADU^2), above which, the point 

840 of decreasing variance should be sought. 

841 consecutivePointsVarDecreases : `int` 

842 Required number of consecutive points/fluxes 

843 in the PTC where the variance 

844 decreases in order to find a first 

845 estimate of the PTC turn-off. 

846 

847 Returns 

848 ------ 

849 goodPoints : `numpy.array` [`bool`] 

850 Boolean array to select good (`True`) and bad (`False`) 

851 points. 

852 

853 Notes 

854 ----- 

855 Eliminate points beyond which the variance decreases. 

856 """ 

857 goodPoints = np.ones_like(means, dtype=bool) 

858 # Variances are sorted and should monotonically increase 

859 pivotList = np.where(np.array(np.diff(variances)) < 0)[0] 

860 if len(pivotList) > 0: 

861 # For small values, sometimes the variance decreases slightly 

862 # Only look when var > self.config.minVarPivotSearch 

863 pivotList = [p for p in pivotList if variances[p] > minVarPivotSearch] 

864 # Require that the varince decreases during 

865 # consecutivePointsVarDecreases 

866 # consecutive points. This will give a first 

867 # estimate of the PTC turn-off, which 

868 # may be updated (reduced) further in the code. 

869 if len(pivotList) > 1: 

870 # enumerate(pivotList) creates tuples (index, value), for 

871 # each value in pivotList. The lambda function subtracts 

872 # each value from the index. 

873 # groupby groups elements by equal key value. 

874 for k, g in groupby(enumerate(pivotList), lambda x: x[0]-x[1]): 

875 group = (map(itemgetter(1), g)) 

876 # Form groups of consecute values from pivotList 

877 group = list(map(int, group)) 

878 # values in pivotList are indices where np.diff(variances) 

879 # is negative, i.e., where the variance starts decreasing. 

880 # Find the first group of consecutive numbers when 

881 # variance decreases. 

882 if len(group) >= consecutivePointsVarDecreases: 

883 pivotIndex = np.min(group) 

884 goodPoints[pivotIndex+1:] = False 

885 break 

886 

887 # Finally, we filter out any infinities or NaNs. 

888 goodPoints[(~np.isfinite(means)) | (~np.isfinite(variances))] = False 

889 

890 return goodPoints 

891 

892 def _makeZeroSafe(self, array, substituteValue=1e-9): 

893 """""" 

894 array = np.array(array) 

895 nBad = Counter(np.ravel(array))[0] 

896 if nBad == 0: 

897 return array 

898 

899 index, = np.where(array == 0) 

900 if len(index): 

901 msg = f"Found {nBad} zeros in array at elements {index}" 

902 self.log.warning(msg) 

903 

904 array[index] = substituteValue 

905 

906 return array 

907 

908 def fitPtc(self, dataset): 

909 """Fit the photon transfer curve to a polynomial or to the 

910 Astier+19 approximation (Eq. 16). 

911 

912 Fit the photon transfer curve with either a polynomial of 

913 the order specified in the task config, or using the 

914 exponential approximation in Astier+19 (Eq. 16). 

915 

916 Sigma clipping is performed iteratively for the fit, as 

917 well as an initial clipping of data points that are more 

918 than `config.initialNonLinearityExclusionThreshold` away 

919 from lying on a straight line. This other step is necessary 

920 because the photon transfer curve turns over catastrophically 

921 at very high flux (because saturation 

922 drops the variance to ~0) and these far outliers cause the 

923 initial fit to fail, meaning the sigma cannot be calculated 

924 to perform the sigma-clipping. 

925 

926 Parameters 

927 ---------- 

928 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset` 

929 The dataset containing the means, variances and 

930 exposure times. 

931 

932 Returns 

933 ------- 

934 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset` 

935 This is the same dataset as the input parameter, however, 

936 it has been modified to include information such as the 

937 fit vectors and the fit parameters. See the class 

938 `PhotonTransferCurveDatase`. 

939 

940 Raises 

941 ------ 

942 RuntimeError 

943 Raised if dataset.ptcFitType is None or empty. 

944 """ 

945 if dataset.ptcFitType: 

946 ptcFitType = dataset.ptcFitType 

947 else: 

948 raise RuntimeError("ptcFitType is None of empty in PTC dataset.") 

949 matrixSide = self.config.maximumRangeCovariancesAstier 

950 nanMatrix = np.empty((matrixSide, matrixSide)) 

951 nanMatrix[:] = np.nan 

952 

953 for amp in dataset.ampNames: 

954 lenInputTimes = len(dataset.rawExpTimes[amp]) 

955 listNanMatrix = np.empty((lenInputTimes, matrixSide, matrixSide)) 

956 listNanMatrix[:] = np.nan 

957 

958 dataset.covariancesModel[amp] = listNanMatrix 

959 dataset.aMatrix[amp] = nanMatrix 

960 dataset.bMatrix[amp] = nanMatrix 

961 dataset.covariancesModelNoB[amp] = listNanMatrix 

962 dataset.aMatrixNoB[amp] = nanMatrix 

963 dataset.noiseMatrix[amp] = nanMatrix 

964 dataset.noiseMatrixNoB[amp] = nanMatrix 

965 

966 def errFunc(p, x, y): 

967 return ptcFunc(p, x) - y 

968 

969 sigmaCutPtcOutliers = self.config.sigmaCutPtcOutliers 

970 maxIterationsPtcOutliers = self.config.maxIterationsPtcOutliers 

971 

972 for i, ampName in enumerate(dataset.ampNames): 

973 meanVecOriginal = dataset.rawMeans[ampName].copy() 

974 varVecOriginal = dataset.rawVars[ampName].copy() 

975 varVecOriginal = self._makeZeroSafe(varVecOriginal) 

976 

977 if self.config.doLegacyTurnoffSelection: 

978 # Discard points when the variance starts to decrease after two 

979 # consecutive signal levels 

980 goodPoints = self._getInitialGoodPoints(meanVecOriginal, varVecOriginal, 

981 self.config.minVarPivotSearch, 

982 self.config.consecutivePointsVarDecreases) 

983 else: 

984 goodPoints = dataset.expIdMask[ampName] 

985 

986 # Check if all points are bad from the 'cpExtractPtcTask' 

987 initialExpIdMask = dataset.expIdMask[ampName] 

988 

989 if not (goodPoints.any() and initialExpIdMask.any()): 

990 msg = (f"SERIOUS: All points in goodPoints: {goodPoints} or " 

991 f"in initialExpIdMask: {initialExpIdMask} are bad." 

992 f"Setting {ampName} to BAD.") 

993 self.log.warning(msg) 

994 # Fill entries with NaNs 

995 self.fillBadAmp(dataset, ptcFitType, ampName) 

996 continue 

997 

998 mask = goodPoints 

999 

1000 if ptcFitType == 'EXPAPPROXIMATION': 

1001 ptcFunc = funcAstier 

1002 parsIniPtc = [-1e-9, 1.0, 10.] # a00, gain, noise^2 

1003 # lowers and uppers obtained from BOT data studies by 

1004 # C. Lage (UC Davis, 11/2020). 

1005 if self.config.binSize > 1: 

1006 bounds = self._boundsForAstier(parsIniPtc) 

1007 else: 

1008 bounds = self._boundsForAstier(parsIniPtc, lowers=[-1e-4, 0.5, -2000], 

1009 uppers=[1e-4, 2.5, 2000]) 

1010 if ptcFitType == 'POLYNOMIAL': 

1011 ptcFunc = funcPolynomial 

1012 parsIniPtc = self._initialParsForPolynomial(self.config.polynomialFitDegree + 1) 

1013 bounds = self._boundsForPolynomial(parsIniPtc) 

1014 

1015 # We perform an initial (unweighted) fit of variance vs signal 

1016 # (after initial KS test or post-drop selection) to look for 

1017 # outliers, particularly at the high-flux end. The initial fit 

1018 # is performed only for points that are guaranteed to be below 

1019 # the PTC turnoff and then extrapolated to ensure that high 

1020 # flux points that have abnormal variance values can be properly 

1021 # rejected in this phase without biasing the initial fit. 

1022 # This algorithm was initially developed by Seth Digel for 

1023 # the EO Testing pipeline. 

1024 

1025 if maxIterationsPtcOutliers == 0: 

1026 # We are not doing any outlier rejection here, but we do want 

1027 # an initial fit. 

1028 res = least_squares( 

1029 errFunc, 

1030 parsIniPtc, 

1031 bounds=bounds, 

1032 args=(meanVecOriginal[mask], varVecOriginal[mask]), 

1033 ) 

1034 pars = res.x 

1035 newMask = mask.copy() 

1036 else: 

1037 newMask = (mask & (meanVecOriginal <= self.config.maxSignalInitialPtcOutlierFit)) 

1038 

1039 count = 0 

1040 lastMask = mask.copy() 

1041 while count < maxIterationsPtcOutliers: 

1042 res = least_squares( 

1043 errFunc, 

1044 parsIniPtc, 

1045 bounds=bounds, 

1046 args=(meanVecOriginal[newMask], varVecOriginal[newMask]), 

1047 ) 

1048 pars = res.x 

1049 

1050 sigResids = (varVecOriginal - ptcFunc(pars, meanVecOriginal))/np.sqrt(varVecOriginal) 

1051 # The new mask includes points where the residuals are 

1052 # finite, are less than the cut, and include the original 

1053 # mask of known points that should not be used. 

1054 newMask = ( 

1055 np.isfinite(sigResids) 

1056 & (np.abs(np.nan_to_num(sigResids)) < sigmaCutPtcOutliers) 

1057 & mask 

1058 ) 

1059 if np.count_nonzero(newMask) == 0: 

1060 msg = (f"SERIOUS: All points after outlier rejection are bad. " 

1061 f"Setting {ampName} to BAD.") 

1062 self.log.warning(msg) 

1063 # Fill entries with NaNs 

1064 self.fillBadAmp(dataset, ptcFitType, ampName) 

1065 break 

1066 

1067 self.log.debug( 

1068 "Iteration %d: Removed %d points in total for %s.", 

1069 count, 

1070 np.count_nonzero(mask) - np.count_nonzero(newMask), 

1071 ampName, 

1072 ) 

1073 

1074 # If the mask hasn't changed then break out. 

1075 if np.all(newMask == lastMask): 

1076 self.log.debug("Convergence at iteration %d; breaking loop for %s.", count, ampName) 

1077 break 

1078 

1079 lastMask = newMask.copy() 

1080 

1081 count += 1 

1082 

1083 # Set the mask to the new mask 

1084 mask = newMask.copy() 

1085 

1086 if not mask.any(): 

1087 # We hae already filled the bad amp above, so continue. 

1088 continue 

1089 

1090 dataset.expIdMask[ampName] = mask 

1091 

1092 parsIniPtc = pars 

1093 meanVecFinal = meanVecOriginal[mask] 

1094 varVecFinal = varVecOriginal[mask] 

1095 

1096 # Save the maximum point after outlier detection as the 

1097 # PTC turnoff point. 

1098 dataset.ptcTurnoff[ampName] = meanVecFinal[-1] 

1099 

1100 if Counter(mask)[False] > 0: 

1101 self.log.info("Number of points discarded in PTC of amplifier %s:" 

1102 " %d out of %d", ampName, Counter(mask)[False], len(meanVecOriginal)) 

1103 

1104 if (len(meanVecFinal) < len(parsIniPtc)): 

1105 msg = (f"SERIOUS: Not enough data points ({len(meanVecFinal)}) compared to the number of " 

1106 f"parameters of the PTC model({len(parsIniPtc)}). Setting {ampName} to BAD.") 

1107 self.log.warning(msg) 

1108 # Fill entries with NaNs 

1109 self.fillBadAmp(dataset, ptcFitType, ampName) 

1110 continue 

1111 # Fit the PTC. 

1112 # The variance of the variance is Var(v)=2*v^2/Npix. This is 

1113 # already calculated in `makeCovArray` of CpPtcExtract. 

1114 # dataset.covariancesSqrtWeights[ampName][:,0,0] 

1115 # has 1/sqrt(Var(v)). 

1116 weightsY = dataset.covariancesSqrtWeights[ampName][:, 0, 0][mask] 

1117 if self.config.doFitBootstrap: 

1118 parsFit, parsFitErr, reducedChiSqPtc = fitBootstrap(parsIniPtc, meanVecFinal, 

1119 varVecFinal, ptcFunc, 

1120 weightsY=weightsY) 

1121 else: 

1122 parsFit, parsFitErr, reducedChiSqPtc = fitLeastSq(parsIniPtc, meanVecFinal, 

1123 varVecFinal, ptcFunc, 

1124 weightsY=weightsY) 

1125 dataset.ptcFitPars[ampName] = parsFit 

1126 dataset.ptcFitParsError[ampName] = parsFitErr 

1127 dataset.ptcFitChiSq[ampName] = reducedChiSqPtc 

1128 

1129 dataset.finalVars[ampName] = varVecOriginal 

1130 dataset.finalVars[ampName][~mask] = np.nan 

1131 dataset.finalModelVars[ampName] = ptcFunc(parsFit, meanVecOriginal) 

1132 dataset.finalModelVars[ampName][~mask] = np.nan 

1133 dataset.finalMeans[ampName] = meanVecOriginal 

1134 dataset.finalMeans[ampName][~mask] = np.nan 

1135 

1136 if ptcFitType == 'EXPAPPROXIMATION': 

1137 ptcGain = parsFit[1] 

1138 ptcGainErr = parsFitErr[1] 

1139 ptcNoise = np.sqrt(np.fabs(parsFit[2])) 

1140 ptcNoiseErr = 0.5*(parsFitErr[2]/np.fabs(parsFit[2]))*np.sqrt(np.fabs(parsFit[2])) 

1141 if ptcFitType == 'POLYNOMIAL': 

1142 ptcGain = 1./parsFit[1] 

1143 ptcGainErr = np.fabs(1./parsFit[1])*(parsFitErr[1]/parsFit[1]) 

1144 ptcNoise = np.sqrt(np.fabs(parsFit[0]))*ptcGain 

1145 ptcNoiseErr = (0.5*(parsFitErr[0]/np.fabs(parsFit[0]))*(np.sqrt(np.fabs(parsFit[0]))))*ptcGain 

1146 dataset.gain[ampName] = ptcGain 

1147 dataset.gainErr[ampName] = ptcGainErr 

1148 dataset.noise[ampName] = ptcNoise 

1149 dataset.noiseErr[ampName] = ptcNoiseErr 

1150 

1151 if not len(dataset.ptcFitType) == 0: 

1152 dataset.ptcFitType = ptcFitType 

1153 if len(dataset.badAmps) == 0: 

1154 dataset.badAmps = [] 

1155 

1156 return dataset 

1157 

1158 def fillBadAmp(self, dataset, ptcFitType, ampName): 

1159 """Fill the dataset with NaNs if there are not enough 

1160 good points. 

1161 

1162 Parameters 

1163 ---------- 

1164 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset` 

1165 The dataset containing the means, variances and 

1166 exposure times. 

1167 ptcFitType : {'POLYNOMIAL', 'EXPAPPROXIMATION'} 

1168 Fit a 'POLYNOMIAL' (degree: 'polynomialFitDegree') or 

1169 'EXPAPPROXIMATION' (Eq. 16 of Astier+19) to the PTC. 

1170 ampName : `str` 

1171 Amplifier name. 

1172 """ 

1173 dataset.badAmps.append(ampName) 

1174 dataset.expIdMask[ampName] = np.repeat(False, len(dataset.rawExpTimes[ampName])) 

1175 dataset.gain[ampName] = np.nan 

1176 dataset.gainErr[ampName] = np.nan 

1177 dataset.noise[ampName] = np.nan 

1178 dataset.noiseErr[ampName] = np.nan 

1179 dataset.ptcFitPars[ampName] = (np.repeat(np.nan, self.config.polynomialFitDegree + 1) if 

1180 ptcFitType in ["POLYNOMIAL", ] else np.repeat(np.nan, 3)) 

1181 dataset.ptcFitParsError[ampName] = (np.repeat(np.nan, self.config.polynomialFitDegree + 1) if 

1182 ptcFitType in ["POLYNOMIAL", ] else np.repeat(np.nan, 3)) 

1183 dataset.ptcFitChiSq[ampName] = np.nan 

1184 dataset.ptcTurnoff[ampName] = np.nan 

1185 dataset.finalVars[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName])) 

1186 dataset.finalModelVars[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName])) 

1187 dataset.finalMeans[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName])) 

1188 

1189 return