Coverage for python/lsst/cp/pipe/ptc/cpSolvePtcTask.py: 10%

529 statements  

« prev     ^ index     » next       coverage.py v7.5.0, created at 2024-05-03 03:48 -0700

1# This file is part of cp_pipe. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21# 

22import numpy as np 

23from collections import Counter 

24 

25import lsst.pex.config as pexConfig 

26import lsst.pipe.base as pipeBase 

27from lsst.cp.pipe.utils import (fitLeastSq, fitBootstrap, funcPolynomial, 

28 funcAstier, symmetrize, Pol2D) 

29 

30from scipy.signal import fftconvolve 

31from scipy.optimize import least_squares 

32from itertools import groupby 

33from operator import itemgetter 

34 

35import lsst.pipe.base.connectionTypes as cT 

36 

37from lsst.ip.isr import PhotonTransferCurveDataset 

38 

39import copy 

40 

41 

42__all__ = ['PhotonTransferCurveSolveConfig', 'PhotonTransferCurveSolveTask'] 

43 

44 

45class PhotonTransferCurveSolveConnections(pipeBase.PipelineTaskConnections, 

46 dimensions=("instrument", "detector")): 

47 inputCovariances = cT.Input( 

48 name="ptcCovariances", 

49 doc="Tuple with measured covariances from flats.", 

50 storageClass="PhotonTransferCurveDataset", 

51 dimensions=("instrument", "exposure", "detector"), 

52 isCalibration=True, 

53 multiple=True, 

54 ) 

55 camera = cT.PrerequisiteInput( 

56 name="camera", 

57 doc="Camera the input data comes from.", 

58 storageClass="Camera", 

59 dimensions=("instrument",), 

60 isCalibration=True, 

61 ) 

62 outputPtcDataset = cT.Output( 

63 name="ptcDatsetProposal", 

64 doc="Output proposed ptc dataset.", 

65 storageClass="PhotonTransferCurveDataset", 

66 dimensions=("instrument", "detector"), 

67 multiple=False, 

68 isCalibration=True, 

69 ) 

70 

71 

72class PhotonTransferCurveSolveConfig(pipeBase.PipelineTaskConfig, 

73 pipelineConnections=PhotonTransferCurveSolveConnections): 

74 """Configuration for fitting measured covariances. 

75 """ 

76 

77 ptcFitType = pexConfig.ChoiceField( 

78 dtype=str, 

79 doc="Fit PTC to Eq. 16, Eq. 20 in Astier+19, or to a polynomial.", 

80 default="POLYNOMIAL", 

81 allowed={ 

82 "POLYNOMIAL": "n-degree polynomial (use 'polynomialFitDegree' to set 'n').", 

83 "EXPAPPROXIMATION": "Approximation in Astier+19 (Eq. 16).", 

84 "FULLCOVARIANCE": "Full covariances model in Astier+19 (Eq. 20)" 

85 } 

86 ) 

87 minMeanSignal = pexConfig.DictField( 

88 keytype=str, 

89 itemtype=float, 

90 doc="Minimum values (inclusive) of mean signal (in ADU) per amp to use." 

91 " The same cut is applied to all amps if this parameter [`dict`] is passed as " 

92 " {'ALL_AMPS': value}", 

93 default={'ALL_AMPS': 0.0}, 

94 ) 

95 maxMeanSignal = pexConfig.DictField( 

96 keytype=str, 

97 itemtype=float, 

98 doc="Maximum values (inclusive) of mean signal (in ADU) below which to consider, per amp." 

99 " The same cut is applied to all amps if this dictionary is of the form" 

100 " {'ALL_AMPS': value}", 

101 default={'ALL_AMPS': 1e6}, 

102 ) 

103 maximumRangeCovariancesAstier = pexConfig.Field( 

104 dtype=int, 

105 doc="Maximum range of measured covariances as in Astier+19", 

106 default=8, 

107 ) 

108 maximumRangeCovariancesAstierFullCovFit = pexConfig.Field( 

109 dtype=int, 

110 doc="Maximum range up to where to fit covariances as in Astier+19, " 

111 "for the FULLCOVARIANCE model." 

112 "This is different from maximumRangeCovariancesAstier." 

113 "It should be less or equal than maximumRangeCovariancesAstier." 

114 "The number of parameters for this model is " 

115 "3*maximumRangeCovariancesAstierFullCovFit^2 + 1, so increase with care " 

116 "so that the fit is not too slow.", 

117 default=8, 

118 ) 

119 doSubtractLongRangeCovariances = pexConfig.Field( 

120 dtype=bool, 

121 doc="Subtract long-range covariances before FULLCOVARIANCE fit, " 

122 "beyond startLongRangeCovariances?", 

123 default=False, 

124 ) 

125 startLongRangeCovariances = pexConfig.Field( 

126 dtype=int, 

127 doc="If doSubtractLongRangeCovariances is True, subtract covariances " 

128 "beyond this range. It should be less than maximumRangeCovariancesAstier. ", 

129 default=4, 

130 ) 

131 polyDegLongRangeCovariances = pexConfig.Field( 

132 dtype=int, 

133 doc="If doSubtractLongRangeCovariances is True, polynomial " 

134 "degree to fit data beyond startLongRangeCovariances.", 

135 default=1, 

136 ) 

137 sigmaClipFullFitCovariancesAstier = pexConfig.Field( 

138 dtype=float, 

139 doc="sigma clip for full model fit for FULLCOVARIANCE ptcFitType ", 

140 default=5.0, 

141 ) 

142 maxIterFullFitCovariancesAstier = pexConfig.Field( 

143 dtype=int, 

144 doc="Maximum number of iterations in full model fit for FULLCOVARIANCE ptcFitType", 

145 default=3, 

146 ) 

147 polynomialFitDegree = pexConfig.Field( 

148 dtype=int, 

149 doc="Degree of polynomial to fit the PTC, when 'ptcFitType'=POLYNOMIAL.", 

150 default=3, 

151 ) 

152 doLegacyTurnoffSelection = pexConfig.Field( 

153 dtype=bool, 

154 doc="Use 'legacy' computation for PTC turnoff selection. If set " 

155 "to False, then the KS test p-value selection will be used instead.", 

156 default=False, 

157 ) 

158 sigmaCutPtcOutliers = pexConfig.Field( 

159 dtype=float, 

160 doc="Sigma cut for outlier rejection in PTC.", 

161 default=5.0, 

162 ) 

163 maxIterationsPtcOutliers = pexConfig.RangeField( 

164 dtype=int, 

165 doc="Maximum number of iterations for outlier rejection in PTC.", 

166 default=2, 

167 min=0 

168 ) 

169 maxSignalInitialPtcOutlierFit = pexConfig.Field( 

170 dtype=float, 

171 doc="Maximum signal considered for intial outlier fit. This should be below " 

172 "the PTC turnoff to ensure accurate outlier rejection. If " 

173 "scaleMaxSignalInitialPtcOutlierFit=True then the units are electrons; " 

174 "otherwise ADU.", 

175 default=50_000., 

176 ) 

177 maxDeltaInitialPtcOutlierFit = pexConfig.Field( 

178 dtype=float, 

179 doc="If there are any outliers in the initial fit that have mean greater than " 

180 "maxSignalInitialPtcOutlierFit, then no points that have this delta " 

181 "mean from the previous ``good`` point are allowed. If " 

182 "scaleMaxSignalInitialPtcOutlierFit=True then the units are electrons; " 

183 "otherwise ADU.", 

184 default=9_000., 

185 ) 

186 scaleMaxSignalInitialPtcOutlierFit = pexConfig.Field( 

187 dtype=bool, 

188 doc="Scale maxSignalInitialPtcOutlierFit and maxDeltaInitialPtcOutlierFit by " 

189 "approximate gain? If yes then " 

190 "maxSignalInitialPtcOutlierFit and maxDeltaInitialPtcOutlierFit are assumed " 

191 "to have units of electrons, otherwise ADU.", 

192 default=True, 

193 ) 

194 minVarPivotSearch = pexConfig.Field( 

195 dtype=float, 

196 doc="The code looks for a pivot signal point after which the variance starts decreasing at high-flux" 

197 " to exclude then from the PTC model fit. However, sometimes at low fluxes, the variance" 

198 " decreases slightly. Set this variable for the variance value, in ADU^2, after which the pivot " 

199 " should be sought. Only used if doLegacyTurnoffSelection is True.", 

200 default=10000, 

201 ) 

202 consecutivePointsVarDecreases = pexConfig.RangeField( 

203 dtype=int, 

204 doc="Required number of consecutive points/fluxes in the PTC where the variance " 

205 "decreases in order to find a first estimate of the PTC turn-off. " 

206 "Only used if doLegacyTurnoffSelection is True.", 

207 default=2, 

208 min=2 

209 ) 

210 ksTestMinPvalue = pexConfig.Field( 

211 dtype=float, 

212 doc="Minimum value of the Gaussian histogram KS test p-value to be used in PTC fit. " 

213 "Only used if doLegacyTurnoffSelection is False.", 

214 default=0.01, 

215 ) 

216 doFitBootstrap = pexConfig.Field( 

217 dtype=bool, 

218 doc="Use bootstrap for the PTC fit parameters and errors?.", 

219 default=False, 

220 ) 

221 binSize = pexConfig.Field( 

222 dtype=int, 

223 doc="Bin the image by this factor in both dimensions.", 

224 default=1, 

225 ) 

226 

227 def validate(self): 

228 super().validate() 

229 fitMatrixSide = self.maximumRangeCovariancesAstierFullCovFit 

230 measureMatrixSide = self.maximumRangeCovariancesAstier 

231 if self.ptcFitType == "FULLCOVARIANCE": 

232 if fitMatrixSide > measureMatrixSide: 

233 raise RuntimeError("Covariance fit size %s is larger than" 

234 "measurement size %s.", 

235 fitMatrixSide, measureMatrixSide) 

236 if self.doSubtractLongRangeCovariances: 

237 startLag = self.startLongRangeCovariances 

238 if measureMatrixSide < startLag: 

239 raise RuntimeError("Covariance measure size %s is smaller than long" 

240 "-range covariance starting point %s.", 

241 measureMatrixSide, startLag) 

242 

243 

244class PhotonTransferCurveSolveTask(pipeBase.PipelineTask): 

245 """Task to fit the PTC from flat covariances. 

246 

247 The first task of the PTC measurement pipeline, 

248 ``PhotonTransferCurveMeasureTask`` (and assumed to have been run 

249 before this task), produced a list of 

250 `~lsst.ip.isr.PhotonTransferCurveDataset` objects. Each dataset 

251 contains the mean signal and covariances of the 

252 difference image of the flat-field images taken at 

253 the same exposure time. The list also contains dummy 

254 datasets (with no measurements), whose purpose is to have 

255 the input and output dimensions of ``PhotonTransferCurveMeasureTask`` 

256 match. 

257 

258 This task, ``PhotonTransferCurveSolveTask``, assembles the list 

259 of individual PTC datasets produced 

260 by ``PhotonTransferCurveMeasureTask`` into one single final PTC 

261 dataset, discarding the dummy datset as appropiate. 

262 The task fits the measured (co)variances to one of three models: 

263 a polynomial model of a given order, or the models described 

264 in equations 16 and 20 of Astier+19. These options are referred 

265 to as ``POLYNOMIAL``, ``EXPAPPROXIMATION``, and ``FULLCOVARIANCE`` 

266 in the configuration options of the task, respectively). 

267 Parameters of interest such as the gain and noise are derived 

268 from the fits. The ``FULLCOVARIANCE`` model is fitted to the 

269 full covariance data (as oppossed to the other two models, which 

270 are fit to the variance vs mean measurements only). 

271 

272 Astier+19: "The Shape of the Photon Transfer Curve 

273 of CCD sensors", arXiv:1905.08677 

274 """ 

275 

276 ConfigClass = PhotonTransferCurveSolveConfig 

277 _DefaultName = 'cpPhotonTransferCurveSolve' 

278 

279 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

280 """Ensure that the input and output dimensions are passed along. 

281 

282 Parameters 

283 ---------- 

284 butlerQC : `~lsst.daf.butler.QuantumContext` 

285 Butler to operate on. 

286 inputRefs : `~lsst.pipe.base.InputQuantizedConnection` 

287 Input data refs to load. 

288 ouptutRefs : `~lsst.pipe.base.OutputQuantizedConnection` 

289 Output data refs to persist. 

290 """ 

291 inputs = butlerQC.get(inputRefs) 

292 detId = inputRefs.inputCovariances[0].dataId['detector'] 

293 outputs = self.run(inputCovariances=inputs['inputCovariances'], camera=inputs['camera'], detId=detId) 

294 butlerQC.put(outputs, outputRefs) 

295 

296 def run(self, inputCovariances, camera=None, detId=0): 

297 """Fit measured covariances to different models. 

298 

299 Parameters 

300 ---------- 

301 inputCovariances : `list` [`lsst.ip.isr.PhotonTransferCurveDataset`] 

302 List of lsst.ip.isr.PhotonTransferCurveDataset datasets. 

303 camera : `lsst.afw.cameraGeom.Camera`, optional 

304 Input camera. 

305 detId : `int` 

306 Detector ID to locate the detector in the camera and 

307 populate the `lsst.ip.isr.PhotonTransferCurveDataset` 

308 metadata. 

309 Returns 

310 ------- 

311 results : `lsst.pipe.base.Struct` 

312 The resultins structure contains: 

313 

314 ``outputPtcDatset`` 

315 Final PTC dataset, containing information such as the 

316 means, variances, and exposure times 

317 (`lsst.ip.isr.PhotonTransferCurveDataset`). 

318 """ 

319 # Find the ampNames from a non-dummy ptc. 

320 ampNames = [] 

321 for partialPtcDataset in inputCovariances: 

322 if partialPtcDataset.ptcFitType != 'DUMMY': 

323 ampNames = partialPtcDataset.ampNames 

324 break 

325 

326 # Each amp may have a different min and max ADU signal 

327 # specified in the config. 

328 maxMeanSignalDict = {ampName: 1e6 for ampName in ampNames} 

329 minMeanSignalDict = {ampName: 0.0 for ampName in ampNames} 

330 for ampName in ampNames: 

331 if 'ALL_AMPS' in self.config.maxMeanSignal: 

332 maxMeanSignalDict[ampName] = self.config.maxMeanSignal['ALL_AMPS'] 

333 elif ampName in self.config.maxMeanSignal: 

334 maxMeanSignalDict[ampName] = self.config.maxMeanSignal[ampName] 

335 

336 if 'ALL_AMPS' in self.config.minMeanSignal: 

337 minMeanSignalDict[ampName] = self.config.minMeanSignal['ALL_AMPS'] 

338 elif ampName in self.config.minMeanSignal: 

339 minMeanSignalDict[ampName] = self.config.minMeanSignal[ampName] 

340 

341 # Assemble individual PTC datasets into a single PTC dataset. 

342 datasetPtc = PhotonTransferCurveDataset( 

343 ampNames=ampNames, 

344 ptcFitType=self.config.ptcFitType, 

345 covMatrixSide=self.config.maximumRangeCovariancesAstier, 

346 covMatrixSideFullCovFit=self.config.maximumRangeCovariancesAstierFullCovFit) 

347 

348 for partialPtcDataset in inputCovariances: 

349 # Ignore dummy datasets 

350 if partialPtcDataset.ptcFitType == 'DUMMY': 

351 continue 

352 for ampName in ampNames: 

353 # The partial dataset consists of lists of values for each 

354 # quantity. In the case of the input exposure pairs, this is a 

355 # list of tuples. In all cases we only want the first 

356 # (and only) element of the list. 

357 datasetPtc.inputExpIdPairs[ampName].append(partialPtcDataset.inputExpIdPairs[ampName][0]) 

358 datasetPtc.rawExpTimes[ampName] = np.append(datasetPtc.rawExpTimes[ampName], 

359 partialPtcDataset.rawExpTimes[ampName][0]) 

360 datasetPtc.rawMeans[ampName] = np.append(datasetPtc.rawMeans[ampName], 

361 partialPtcDataset.rawMeans[ampName][0]) 

362 datasetPtc.rawVars[ampName] = np.append(datasetPtc.rawVars[ampName], 

363 partialPtcDataset.rawVars[ampName][0]) 

364 datasetPtc.rowMeanVariance[ampName] = np.append(datasetPtc.rowMeanVariance[ampName], 

365 partialPtcDataset.rowMeanVariance[ampName][0]) 

366 datasetPtc.photoCharges[ampName] = np.append(datasetPtc.photoCharges[ampName], 

367 partialPtcDataset.photoCharges[ampName][0]) 

368 datasetPtc.histVars[ampName] = np.append(datasetPtc.histVars[ampName], 

369 partialPtcDataset.histVars[ampName][0]) 

370 datasetPtc.histChi2Dofs[ampName] = np.append(datasetPtc.histChi2Dofs[ampName], 

371 partialPtcDataset.histChi2Dofs[ampName][0]) 

372 datasetPtc.kspValues[ampName] = np.append(datasetPtc.kspValues[ampName], 

373 partialPtcDataset.kspValues[ampName][0]) 

374 datasetPtc.noiseList[ampName] = np.append(datasetPtc.noiseList[ampName], 

375 partialPtcDataset.noise[ampName]) 

376 datasetPtc.covariances[ampName] = np.append( 

377 datasetPtc.covariances[ampName].ravel(), 

378 partialPtcDataset.covariances[ampName].ravel() 

379 ).reshape( 

380 ( 

381 len(datasetPtc.rawExpTimes[ampName]), 

382 datasetPtc.covMatrixSide, 

383 datasetPtc.covMatrixSide, 

384 ) 

385 ) 

386 datasetPtc.covariancesSqrtWeights[ampName] = np.append( 

387 datasetPtc.covariancesSqrtWeights[ampName].ravel(), 

388 partialPtcDataset.covariancesSqrtWeights[ampName].ravel() 

389 ).reshape( 

390 ( 

391 len(datasetPtc.rawExpTimes[ampName]), 

392 datasetPtc.covMatrixSide, 

393 datasetPtc.covMatrixSide, 

394 ) 

395 ) 

396 

397 # Apply min/max masking. 

398 rawMean = partialPtcDataset.rawMeans[ampName][0] 

399 rawVar = partialPtcDataset.rawVars[ampName][0] 

400 expIdMask = partialPtcDataset.expIdMask[ampName][0] 

401 if (rawMean <= minMeanSignalDict[ampName]) or (rawMean >= maxMeanSignalDict[ampName]) \ 

402 or not np.isfinite(rawMean) or not np.isfinite(rawVar): 

403 expIdMask = False 

404 

405 kspValue = partialPtcDataset.kspValues[ampName][0] 

406 if not self.config.doLegacyTurnoffSelection and \ 

407 kspValue < self.config.ksTestMinPvalue: 

408 expIdMask = False 

409 

410 datasetPtc.expIdMask[ampName] = np.append(datasetPtc.expIdMask[ampName], expIdMask) 

411 

412 for key, value in partialPtcDataset.auxValues.items(): 

413 if key in datasetPtc.auxValues: 

414 datasetPtc.auxValues[key] = np.append(datasetPtc.auxValues[key], value) 

415 else: 

416 datasetPtc.auxValues[key] = value 

417 

418 # Sort arrays that are filled so far in the final dataset by 

419 # rawMeans index. 

420 # First compute the mean across all the amps to make sure that they are 

421 # all sorted the same way. 

422 detectorMeans = np.zeros(len(datasetPtc.inputExpIdPairs[ampNames[0]])) 

423 

424 for i in range(len(detectorMeans)): 

425 arr = np.array([datasetPtc.rawMeans[ampName][i] for ampName in ampNames]) 

426 good, = (np.isfinite(arr)).nonzero() 

427 if good.size == 0: 

428 detectorMeans[i] = np.nan 

429 else: 

430 detectorMeans[i] = np.mean(arr[good]) 

431 

432 index = np.argsort(detectorMeans) 

433 

434 for ampName in ampNames: 

435 datasetPtc.inputExpIdPairs[ampName] = np.array( 

436 datasetPtc.inputExpIdPairs[ampName] 

437 )[index].tolist() 

438 datasetPtc.rawExpTimes[ampName] = datasetPtc.rawExpTimes[ampName][index] 

439 datasetPtc.rawMeans[ampName] = datasetPtc.rawMeans[ampName][index] 

440 datasetPtc.rawVars[ampName] = datasetPtc.rawVars[ampName][index] 

441 datasetPtc.rowMeanVariance[ampName] = datasetPtc.rowMeanVariance[ampName][index] 

442 datasetPtc.photoCharges[ampName] = datasetPtc.photoCharges[ampName][index] 

443 datasetPtc.histVars[ampName] = datasetPtc.histVars[ampName][index] 

444 datasetPtc.histChi2Dofs[ampName] = datasetPtc.histChi2Dofs[ampName][index] 

445 datasetPtc.kspValues[ampName] = datasetPtc.kspValues[ampName][index] 

446 datasetPtc.expIdMask[ampName] = datasetPtc.expIdMask[ampName][index] 

447 datasetPtc.covariances[ampName] = datasetPtc.covariances[ampName][index] 

448 datasetPtc.covariancesSqrtWeights[ampName] = datasetPtc.covariancesSqrtWeights[ampName][index] 

449 for key, value in datasetPtc.auxValues.items(): 

450 datasetPtc.auxValues[key] = value[index] 

451 

452 if self.config.ptcFitType == "FULLCOVARIANCE": 

453 # Fit the measured covariances vs mean signal to 

454 # the Astier+19 full model (Eq. 20). Before that 

455 # do a preliminary fit to the variance (C_00) vs mean 

456 # signal (mu) curve using the EXPAPPROXIMATION model 

457 # (Eq. 16 in Astier+19) in order to 

458 # get the flat pairs that are masked. The 

459 # points at these fluxes will also be masked when 

460 # calculating the other elements of the covariance 

461 # matrix, C_ij, i!=j). 

462 

463 # Preliminary fit, usign a temp dataset to get the mask 

464 tempDatasetPtc = copy.copy(datasetPtc) 

465 tempDatasetPtc.ptcFitType = "EXPAPPROXIMATION" 

466 tempDatasetPtc = self.fitMeasurementsToModel(tempDatasetPtc) 

467 

468 # "FULLCOVARIANCE", using the mask obtained from the 

469 # previous fit. 

470 for ampName in datasetPtc.ampNames: 

471 datasetPtc.expIdMask[ampName] = tempDatasetPtc.expIdMask[ampName] 

472 datasetPtc.fitType = "FULLCOVARIANCE" 

473 datasetPtc = self.fitMeasurementsToModel(datasetPtc) 

474 # The other options are: self.config.ptcFitType in 

475 # ("EXPAPPROXIMATION", "POLYNOMIAL") 

476 else: 

477 # Fit the PTC to a polynomial or to Astier+19 exponential 

478 # approximation (Eq. 16). Fill up 

479 # PhotonTransferCurveDataset object. 

480 datasetPtc = self.fitMeasurementsToModel(datasetPtc) 

481 

482 # Initial validation of PTC fit. 

483 for ampName in ampNames: 

484 noise = np.nanmedian(datasetPtc.noiseList[ampName]) 

485 noiseFitted = np.sqrt(datasetPtc.noise[ampName]) 

486 

487 # Check if noise is close to noiseFitted 

488 if not np.isclose(noiseFitted, noise, rtol=0.05, atol=0.0): 

489 self.log.warning(f"Read noise from PTC fit ({noiseFitted}) is not consistent " 

490 f"with read noise measured from overscan ({noise}) for " 

491 f"amplifier {ampName}. Try adjusting the fit range.") 

492 

493 if camera: 

494 detector = camera[detId] 

495 else: 

496 detector = None 

497 datasetPtc.updateMetadataFromExposures(inputCovariances) 

498 datasetPtc.updateMetadata(setDate=True, camera=camera, detector=detector) 

499 

500 return pipeBase.Struct( 

501 outputPtcDataset=datasetPtc, 

502 ) 

503 

504 def fitMeasurementsToModel(self, dataset): 

505 """Fit the measured covariances vs mean signal to a 

506 polynomial or one of the models in Astier+19 

507 (Eq. 16 or Eq.20). 

508 

509 Parameters 

510 ---------- 

511 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset` 

512 The dataset containing information such as the means, 

513 (co)variances, and exposure times. 

514 

515 Returns 

516 ------- 

517 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset` 

518 This is the same dataset as the input parameter, however, 

519 it has been modified to include information such as the 

520 fit vectors and the fit parameters. See the class 

521 `PhotonTransferCurveDatase`. 

522 """ 

523 fitType = dataset.ptcFitType 

524 if fitType in ["FULLCOVARIANCE", ]: 

525 # This model uses the full covariance matrix in the fit. 

526 # The PTC is technically defined as variance vs signal, 

527 # with variance = Cov_00 

528 dataset = self.fitDataFullCovariance(dataset) 

529 elif fitType in ["POLYNOMIAL", "EXPAPPROXIMATION"]: 

530 # The PTC is technically defined as variance vs signal 

531 dataset = self.fitPtc(dataset) 

532 else: 

533 raise RuntimeError( 

534 f"Fitting option {fitType} not one of " 

535 "'POLYNOMIAL', 'EXPAPPROXIMATION', or 'FULLCOVARIANCE'" 

536 ) 

537 

538 return dataset 

539 

540 def fitDataFullCovariance(self, dataset): 

541 """Fit measured flat covariances to the full model in 

542 Astier+19 (Eq. 20). 

543 

544 Parameters 

545 ---------- 

546 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset` 

547 The dataset containing information such as the means, 

548 (co)variances, and exposure times. 

549 

550 Returns 

551 ------- 

552 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset` 

553 This is the same dataset as the input parameter, however, 

554 it has been modified to include information such as the 

555 fit vectors and the fit parameters. See the class 

556 `PhotonTransferCurveDatase`. 

557 

558 Notes 

559 ----- 

560 The parameters of the full model for C_ij(mu) ("C_ij" and "mu" 

561 in ADU^2 and ADU, respectively) in Astier+19 (Eq. 20) are: 

562 

563 - "a" coefficients (r by r matrix), units: 1/e 

564 - "b" coefficients (r by r matrix), units: 1/e 

565 - noise matrix (r by r matrix), units: e^2 

566 - gain, units: e/ADU 

567 

568 "b" appears in Eq. 20 only through the "ab" combination, which 

569 is defined in this code as "c=ab". 

570 

571 Total number of parameters: #entries(a) + #entries(c) + #entries(noise) 

572 + 1. This is equivalent to r^2 + r^2 + r^2 + 1, where "r" is the 

573 maximum lag considered for the covariances calculation, and the 

574 extra "1" is the gain. If "b" is 0, then "c" is 0, and len(pInit) will 

575 have r^2 fewer entries. 

576 """ 

577 matrixSide = dataset.covMatrixSide 

578 matrixSideFit = dataset.covMatrixSideFullCovFit 

579 lenParams = matrixSideFit*matrixSideFit 

580 

581 for ampName in dataset.ampNames: 

582 lenInputTimes = len(dataset.rawExpTimes[ampName]) 

583 # Not used when ptcFitType is 'FULLCOVARIANCE' 

584 dataset.ptcFitPars[ampName] = np.array([np.nan]) 

585 dataset.ptcFitParsError[ampName] = np.array([np.nan]) 

586 dataset.ptcFitChiSq[ampName] = np.nan 

587 

588 if ampName in dataset.badAmps: 

589 # Bad amp 

590 # Entries need to have proper dimensions so read/write 

591 # with astropy.Table works. 

592 nanMatrixFit = np.full((matrixSideFit, matrixSideFit), np.nan) 

593 listNanMatrix = np.full((lenInputTimes, matrixSide, matrixSide), np.nan) 

594 listNanMatrixFit = np.full((lenInputTimes, matrixSideFit, matrixSideFit), np.nan) 

595 dataset.covariancesModel[ampName] = listNanMatrixFit 

596 dataset.covariancesSqrtWeights[ampName] = listNanMatrix 

597 dataset.aMatrix[ampName] = nanMatrixFit 

598 dataset.bMatrix[ampName] = nanMatrixFit 

599 dataset.covariancesModelNoB[ampName] = listNanMatrixFit 

600 dataset.aMatrixNoB[ampName] = nanMatrixFit 

601 dataset.noiseMatrix[ampName] = nanMatrixFit 

602 dataset.noiseMatrixNoB[ampName] = nanMatrixFit 

603 

604 dataset.expIdMask[ampName] = np.repeat(False, lenInputTimes) 

605 dataset.gain[ampName] = np.nan 

606 dataset.gainErr[ampName] = np.nan 

607 dataset.noise[ampName] = np.nan 

608 dataset.noiseErr[ampName] = np.nan 

609 dataset.finalVars[ampName] = np.repeat(np.nan, lenInputTimes) 

610 dataset.finalModelVars[ampName] = np.repeat(np.nan, lenInputTimes) 

611 dataset.finalMeans[ampName] = np.repeat(np.nan, lenInputTimes) 

612 continue 

613 

614 muAtAmp = dataset.rawMeans[ampName] 

615 maskAtAmp = dataset.expIdMask[ampName] 

616 if len(maskAtAmp) == 0: 

617 maskAtAmp = np.repeat(True, len(muAtAmp)) 

618 

619 muAtAmpMasked = muAtAmp[maskAtAmp] 

620 covAtAmp = dataset.covariances[ampName] 

621 covAtAmpMasked = np.nan_to_num(covAtAmp)[maskAtAmp] 

622 covSqrtWeightsAtAmp = dataset.covariancesSqrtWeights[ampName] 

623 covSqrtWeightsAtAmpMasked = np.nan_to_num(covSqrtWeightsAtAmp)[maskAtAmp] 

624 

625 # Subtract long-range covariances 

626 if self.config.doSubtractLongRangeCovariances: 

627 startLag = self.config.startLongRangeCovariances 

628 covAtAmpMasked, covSqrtWeightsAtAmpMasked = self.subtractDistantOffset( 

629 muAtAmpMasked, covAtAmpMasked, 

630 covSqrtWeightsAtAmpMasked, 

631 start=startLag, 

632 degree=self.config.polyDegLongRangeCovariances) 

633 

634 # In principle, we could fit to a lag smaller than the measured 

635 # covariances. 

636 r = self.config.maximumRangeCovariancesAstierFullCovFit 

637 covAtAmpForFitMasked = covAtAmpMasked[:, :r, :r] 

638 covSqrtWeightsAtAmpForFitMasked = covSqrtWeightsAtAmpMasked[:, :r, :r] 

639 

640 # Initial fit, to approximate parameters, with c=0 

641 a0, c0, noise0, gain0 = self.initialFitFullCovariance( 

642 muAtAmpMasked, 

643 covAtAmpForFitMasked, 

644 covSqrtWeightsAtAmpForFitMasked 

645 ) 

646 

647 # Fit full model (Eq. 20 of Astier+19) and same model with 

648 # b=0 (c=0 in this code) 

649 pInit = np.concatenate((a0.ravel(), c0.ravel(), noise0.ravel(), np.array(gain0)), axis=None) 

650 functionsDict = {'fullModel': self.funcFullCovarianceModel, 

651 'fullModelNoB': self.funcFullCovarianceModelNoB} 

652 fitResults = {'fullModel': {'a': [], 'c': [], 'noise': [], 'gain': [], 'paramsErr': []}, 

653 'fullModelNoB': {'a': [], 'c': [], 'noise': [], 'gain': [], 'paramsErr': []}} 

654 for key in functionsDict: 

655 params, paramsErr, _ = fitLeastSq(pInit, muAtAmpMasked, 

656 covAtAmpForFitMasked.ravel(), functionsDict[key], 

657 weightsY=covSqrtWeightsAtAmpForFitMasked.ravel()) 

658 a = params[:lenParams].reshape((matrixSideFit, matrixSideFit)) 

659 c = params[lenParams:2*lenParams].reshape((matrixSideFit, matrixSideFit)) 

660 noise = params[2*lenParams:3*lenParams].reshape((matrixSideFit, matrixSideFit)) 

661 gain = params[-1] 

662 

663 fitResults[key]['a'] = a 

664 fitResults[key]['c'] = c 

665 fitResults[key]['noise'] = noise 

666 fitResults[key]['gain'] = gain 

667 fitResults[key]['paramsErr'] = paramsErr 

668 

669 # Put the information in the PTC dataset 

670 

671 # Not used when ptcFitType is 'FULLCOVARIANCE' 

672 dataset.ptcFitPars[ampName] = np.array([np.nan]) 

673 dataset.ptcFitParsError[ampName] = np.array([np.nan]) 

674 dataset.ptcFitChiSq[ampName] = np.nan 

675 

676 # Save full covariances, covariances models, and their weights. 

677 # dataset.expIdMask is already full, but needs to be 

678 # converted to bool. 

679 dataset.expIdMask[ampName] = np.array(dataset.expIdMask[ampName], dtype=bool) 

680 dataset.covariances[ampName] = covAtAmp 

681 # We evaluate the covariance model everywhere, even the 

682 # masked amps. 

683 dataset.covariancesModel[ampName] = self.evalCovModel(muAtAmp, 

684 fitResults['fullModel']['a'], 

685 fitResults['fullModel']['c'], 

686 fitResults['fullModel']['noise'], 

687 fitResults['fullModel']['gain']) 

688 dataset.covariancesSqrtWeights[ampName] = covSqrtWeightsAtAmp 

689 dataset.aMatrix[ampName] = fitResults['fullModel']['a'] 

690 dataset.bMatrix[ampName] = fitResults['fullModel']['c']/fitResults['fullModel']['a'] 

691 dataset.covariancesModelNoB[ampName] = self.evalCovModel(muAtAmp, 

692 fitResults['fullModelNoB']['a'], 

693 fitResults['fullModelNoB']['c'], 

694 fitResults['fullModelNoB']['noise'], 

695 fitResults['fullModelNoB']['gain'], 

696 setBtoZero=True) 

697 dataset.aMatrixNoB[ampName] = fitResults['fullModelNoB']['a'] 

698 dataset.gain[ampName] = fitResults['fullModel']['gain'] 

699 dataset.gainErr[ampName] = fitResults['fullModel']['paramsErr'][-1] 

700 readoutNoise = fitResults['fullModel']['noise'][0][0] 

701 readoutNoiseSqrt = np.sqrt(np.fabs(readoutNoise)) 

702 dataset.noise[ampName] = readoutNoise 

703 readoutNoiseSigma = fitResults['fullModel']['paramsErr'][2*lenParams] 

704 dataset.noiseErr[ampName] = 0.5*(readoutNoiseSigma/np.fabs(readoutNoise))*readoutNoiseSqrt 

705 dataset.noiseMatrix[ampName] = fitResults['fullModel']['noise'] 

706 dataset.noiseMatrixNoB[ampName] = fitResults['fullModelNoB']['noise'] 

707 

708 dataset.finalVars[ampName] = covAtAmp[:, 0, 0] 

709 dataset.finalModelVars[ampName] = dataset.covariancesModel[ampName][:, 0, 0] 

710 dataset.finalMeans[ampName] = muAtAmp 

711 

712 return dataset 

713 

714 def initialFitFullCovariance(self, mu, cov, sqrtW): 

715 """ Performs a crude parabolic fit of the data in order to start 

716 the full fit close to the solution, setting b=0 (c=0) in Eq. 20 

717 of Astier+19. 

718 

719 Parameters 

720 ---------- 

721 mu : `numpy.array`, (N,) 

722 Signal `mu` (ADU) 

723 cov : `numpy.array`, (N, M, M) 

724 Covariance arrays of size `(M, M)` (with 

725 `M = config.maximumRangeCovariancesAstier`), 

726 indexed by mean signal `mu`. 

727 sqrtW : `numpy.array`, (N,) 

728 Covariance weights, defined as 1./sqrt(Variances) 

729 

730 Returns 

731 ------- 

732 a : `numpy.array`, (M, M) 

733 "a" parameter per flux in Eq. 20 of Astier+19. 

734 c : `numpy.array`, (M, M) 

735 "c"="ab" parameter per flux in Eq. 20 of Astier+19. 

736 noise : `numpy.array`, (M, M) 

737 "noise" parameter per flux in Eq. 20 of Astier+19. 

738 gain : `float` 

739 Amplifier gain (e/ADU) 

740 """ 

741 matrixSideFit = self.config.maximumRangeCovariancesAstierFullCovFit 

742 

743 # Initialize fit parameters 

744 a = np.zeros((matrixSideFit, matrixSideFit)) 

745 c = np.zeros((matrixSideFit, matrixSideFit)) 

746 noise = np.zeros((matrixSideFit, matrixSideFit)) 

747 gain = 1. 

748 

749 # iterate the fit to account for higher orders 

750 # the chi2 does not necessarily go down, so one could 

751 # stop when it increases 

752 oldChi2 = 1e30 

753 for _ in range(5): 

754 model = np.nan_to_num(self.evalCovModel(mu, a, c, noise, gain, setBtoZero=True)) 

755 # loop on lags 

756 for i in range(matrixSideFit): 

757 for j in range(matrixSideFit): 

758 # fit a parabola for a given lag 

759 parsFit = np.polyfit(mu, cov[:, i, j] - model[:, i, j], 

760 2, w=sqrtW[:, i, j]) 

761 # model equation (Eq. 20) in Astier+19, with c=a*b=0: 

762 a[i, j] += parsFit[0] 

763 noise[i, j] += parsFit[2] 

764 if i + j == 0: 

765 gain = 1./(1/gain+parsFit[1]) 

766 weightedRes = (model - cov)*sqrtW 

767 chi2 = (weightedRes.flatten()**2).sum() 

768 if chi2 > oldChi2: 

769 break 

770 oldChi2 = chi2 

771 

772 return a, c, noise, gain 

773 

774 def funcFullCovarianceModel(self, params, x): 

775 """Model to fit covariances from flat fields; Equation 20 of 

776 Astier+19. 

777 

778 Parameters 

779 ---------- 

780 params : `list` 

781 Parameters of the model: aMatrix, CMatrix, noiseMatrix, 

782 gain (e/ADU). 

783 x : `numpy.array`, (N,) 

784 Signal `mu` (ADU) 

785 

786 Returns 

787 ------- 

788 y : `numpy.array`, (N,) 

789 Covariance matrix. 

790 """ 

791 matrixSideFit = self.config.maximumRangeCovariancesAstierFullCovFit 

792 lenParams = matrixSideFit*matrixSideFit 

793 aMatrix = params[:lenParams].reshape((matrixSideFit, matrixSideFit)) 

794 cMatrix = params[lenParams:2*lenParams].reshape((matrixSideFit, matrixSideFit)) 

795 noiseMatrix = params[2*lenParams:3*lenParams].reshape((matrixSideFit, matrixSideFit)) 

796 gain = params[-1] 

797 

798 return self.evalCovModel(x, aMatrix, cMatrix, noiseMatrix, gain).flatten() 

799 

800 def funcFullCovarianceModelNoB(self, params, x): 

801 """Model to fit covariances from flat fields; Equation 20 of 

802 Astier+19, with b=0 (equivalent to c=a*b=0 in this code). 

803 

804 Parameters 

805 ---------- 

806 params : `list` 

807 Parameters of the model: aMatrix, CMatrix, noiseMatrix, 

808 gain (e/ADU). 

809 x : `numpy.array`, (N,) 

810 Signal mu (ADU) 

811 

812 Returns 

813 ------- 

814 y : `numpy.array`, (N,) 

815 Covariance matrix. 

816 """ 

817 matrixSideFit = self.config.maximumRangeCovariancesAstierFullCovFit 

818 lenParams = matrixSideFit*matrixSideFit 

819 aMatrix = params[:lenParams].reshape((matrixSideFit, matrixSideFit)) 

820 cMatrix = params[lenParams:2*lenParams].reshape((matrixSideFit, matrixSideFit)) 

821 noiseMatrix = params[2*lenParams:3*lenParams].reshape((matrixSideFit, matrixSideFit)) 

822 gain = params[-1] 

823 

824 return self.evalCovModel(x, aMatrix, cMatrix, noiseMatrix, gain, setBtoZero=True).flatten() 

825 

826 def evalCovModel(self, mu, aMatrix, cMatrix, noiseMatrix, gain, setBtoZero=False): 

827 """Computes full covariances model (Eq. 20 of Astier+19). 

828 

829 Parameters 

830 ---------- 

831 mu : `numpy.array`, (N,) 

832 List of mean signals. 

833 aMatrix : `numpy.array`, (M, M) 

834 "a" parameter per flux in Eq. 20 of Astier+19. 

835 cMatrix : `numpy.array`, (M, M) 

836 "c"="ab" parameter per flux in Eq. 20 of Astier+19. 

837 noiseMatrix : `numpy.array`, (M, M) 

838 "noise" parameter per flux in Eq. 20 of Astier+19. 

839 gain : `float` 

840 Amplifier gain (e/ADU) 

841 setBtoZero=False : `bool`, optional 

842 Set "b" parameter in full model (see Astier+19) to zero. 

843 

844 Returns 

845 ------- 

846 covModel : `numpy.array`, (N, M, M) 

847 Covariances model. 

848 

849 Notes 

850 ----- 

851 By default, computes the covModel for the mu's stored(self.mu). 

852 Returns cov[Nmu, M, M]. The variance for the PTC is 

853 cov[:, 0, 0]. mu and cov are in ADUs and ADUs squared. To use 

854 electrons for both, the gain should be set to 1. This routine 

855 implements the model in Astier+19 (1905.08677). 

856 The parameters of the full model for C_ij(mu) ("C_ij" and "mu" 

857 in ADU^2 and ADU, respectively) in Astier+19 (Eq. 20) are: 

858 

859 - "a" coefficients (M by M matrix), units: 1/e 

860 - "b" coefficients (M by M matrix), units: 1/e 

861 - noise matrix (M by M matrix), units: e^2 

862 - gain, units: e/ADU 

863 

864 "b" appears in Eq. 20 only through the "ab" combination, which 

865 is defined in this code as "c=ab". 

866 """ 

867 matrixSideFit = self.config.maximumRangeCovariancesAstierFullCovFit 

868 sa = (matrixSideFit, matrixSideFit) 

869 # pad a with zeros and symmetrize 

870 aEnlarged = np.zeros((int(sa[0]*1.5)+1, int(sa[1]*1.5)+1)) 

871 aEnlarged[0:sa[0], 0:sa[1]] = aMatrix 

872 aSym = symmetrize(aEnlarged) 

873 # pad c with zeros and symmetrize 

874 cEnlarged = np.zeros((int(sa[0]*1.5)+1, int(sa[1]*1.5)+1)) 

875 cEnlarged[0:sa[0], 0:sa[1]] = cMatrix 

876 cSym = symmetrize(cEnlarged) 

877 a2 = fftconvolve(aSym, aSym, mode='same') 

878 a3 = fftconvolve(a2, aSym, mode='same') 

879 ac = fftconvolve(aSym, cSym, mode='same') 

880 (xc, yc) = np.unravel_index(np.abs(aSym).argmax(), a2.shape) 

881 

882 a1 = aMatrix[np.newaxis, :, :] 

883 a2 = a2[np.newaxis, xc:xc + matrixSideFit, yc:yc + matrixSideFit] 

884 a3 = a3[np.newaxis, xc:xc + matrixSideFit, yc:yc + matrixSideFit] 

885 ac = ac[np.newaxis, xc:xc + matrixSideFit, yc:yc + matrixSideFit] 

886 c1 = cMatrix[np.newaxis, ::] 

887 

888 # assumes that mu is 1d 

889 bigMu = mu[:, np.newaxis, np.newaxis]*gain 

890 # c(=a*b in Astier+19) also has a contribution to the last 

891 # term, that is absent for now. 

892 if setBtoZero: 

893 c1 = np.zeros_like(c1) 

894 ac = np.zeros_like(ac) 

895 covModel = (bigMu/(gain*gain)*(a1*bigMu+2./3.*(bigMu*bigMu)*(a2 + c1) 

896 + (1./3.*a3 + 5./6.*ac)*(bigMu*bigMu*bigMu)) + noiseMatrix[np.newaxis, :, :]/gain**2) 

897 # add the Poisson term, and the read out noise (variance) 

898 covModel[:, 0, 0] += mu/gain 

899 

900 return covModel 

901 

902 def subtractDistantOffset(self, muAtAmpMasked, covAtAmpMasked, covSqrtWeightsAtAmpMasked, 

903 start, degree=1): 

904 """Subtract distant offset from the covariance matrices. 

905 

906 Parameters 

907 ---------- 

908 muAtAmpMasked : `numpy.array` 

909 Masked mean flux array for a particular amplifier. 

910 covAtAmpMasked : `numpy.array` 

911 Masked measured covariances for a particular amplifier. 

912 covSqrtWeightsAtAmpMasked : `numpy.array` 

913 Masked inverse covariance weights for a particular amplifier. 

914 start : int, optional 

915 The starting index to eliminate the core for the fit. 

916 degree : int, optional 

917 Degree of the polynomial fit. 

918 

919 Returns 

920 ------- 

921 covAtAmpMasked : `numpy.array` 

922 Subtracted measured covariances for a particular amplifier. 

923 covSqrtWeightsAtAmpMasked : `numpy.array` 

924 Masked inverse covariance weights for a particular amplifier. 

925 

926 Notes 

927 ----- 

928 Ported from https://gitlab.in2p3.fr/astier/bfptc by P. Astier. 

929 

930 This function subtracts a distant offset from the 

931 covariance matrices using polynomial fitting. The core 

932 of the matrices is eliminated for the fit. 

933 

934 The function modifies the internal state of the object, updating the 

935 covariance matrices and related attributes. 

936 """ 

937 for k in range(len(muAtAmpMasked)): 

938 # Make a copy because it will be altered 

939 w = np.copy(covSqrtWeightsAtAmpMasked[k, ...]) 

940 wShape = w.shape 

941 i, j = np.meshgrid(range(wShape[0]), range(wShape[1]), indexing='ij') 

942 

943 # Eliminate the core for the fit 

944 w[:start, :start] = 0 

945 

946 poly = Pol2D(i, j, covAtAmpMasked[k, ...], degree, w=w) 

947 back = poly.eval(i, j) 

948 

949 covAtAmpMasked[k, ...] -= back 

950 

951 return covAtAmpMasked, covSqrtWeightsAtAmpMasked 

952 

953 # EXPAPPROXIMATION and POLYNOMIAL fit methods 

954 @staticmethod 

955 def _initialParsForPolynomial(order): 

956 assert order >= 2 

957 pars = np.zeros(order, dtype=float) 

958 pars[0] = 10 

959 pars[1] = 1 

960 pars[2:] = 0.0001 

961 return pars 

962 

963 @staticmethod 

964 def _boundsForPolynomial(initialPars, lowers=[], uppers=[]): 

965 if not len(lowers): 

966 lowers = [np.NINF for p in initialPars] 

967 if not len(uppers): 

968 uppers = [np.inf for p in initialPars] 

969 lowers[1] = 0 # no negative gains 

970 return (lowers, uppers) 

971 

972 @staticmethod 

973 def _boundsForAstier(initialPars, lowers=[], uppers=[]): 

974 if not len(lowers): 

975 lowers = [np.NINF for p in initialPars] 

976 if not len(uppers): 

977 uppers = [np.inf for p in initialPars] 

978 return (lowers, uppers) 

979 

980 @staticmethod 

981 def _getInitialGoodPoints(means, variances, minVarPivotSearch, consecutivePointsVarDecreases): 

982 """Return a boolean array to mask bad points. 

983 

984 Parameters 

985 ---------- 

986 means : `numpy.array` 

987 Input array with mean signal values. 

988 variances : `numpy.array` 

989 Input array with variances at each mean value. 

990 minVarPivotSearch : `float` 

991 The variance (in ADU^2), above which, the point 

992 of decreasing variance should be sought. 

993 consecutivePointsVarDecreases : `int` 

994 Required number of consecutive points/fluxes 

995 in the PTC where the variance 

996 decreases in order to find a first 

997 estimate of the PTC turn-off. 

998 

999 Returns 

1000 ------ 

1001 goodPoints : `numpy.array` [`bool`] 

1002 Boolean array to select good (`True`) and bad (`False`) 

1003 points. 

1004 

1005 Notes 

1006 ----- 

1007 Eliminate points beyond which the variance decreases. 

1008 """ 

1009 goodPoints = np.ones_like(means, dtype=bool) 

1010 # Variances are sorted and should monotonically increase 

1011 pivotList = np.where(np.array(np.diff(variances)) < 0)[0] 

1012 if len(pivotList) > 0: 

1013 # For small values, sometimes the variance decreases slightly 

1014 # Only look when var > self.config.minVarPivotSearch 

1015 pivotList = [p for p in pivotList if variances[p] > minVarPivotSearch] 

1016 # Require that the varince decreases during 

1017 # consecutivePointsVarDecreases 

1018 # consecutive points. This will give a first 

1019 # estimate of the PTC turn-off, which 

1020 # may be updated (reduced) further in the code. 

1021 if len(pivotList) > 1: 

1022 # enumerate(pivotList) creates tuples (index, value), for 

1023 # each value in pivotList. The lambda function subtracts 

1024 # each value from the index. 

1025 # groupby groups elements by equal key value. 

1026 for k, g in groupby(enumerate(pivotList), lambda x: x[0]-x[1]): 

1027 group = (map(itemgetter(1), g)) 

1028 # Form groups of consecute values from pivotList 

1029 group = list(map(int, group)) 

1030 # values in pivotList are indices where np.diff(variances) 

1031 # is negative, i.e., where the variance starts decreasing. 

1032 # Find the first group of consecutive numbers when 

1033 # variance decreases. 

1034 if len(group) >= consecutivePointsVarDecreases: 

1035 pivotIndex = np.min(group) 

1036 goodPoints[pivotIndex+1:] = False 

1037 break 

1038 

1039 # Finally, we filter out any infinities or NaNs. 

1040 goodPoints[(~np.isfinite(means)) | (~np.isfinite(variances))] = False 

1041 

1042 return goodPoints 

1043 

1044 def _makeZeroSafe(self, array, substituteValue=1e-9): 

1045 """""" 

1046 array = np.array(array) 

1047 nBad = Counter(np.ravel(array))[0] 

1048 if nBad == 0: 

1049 return array 

1050 

1051 index, = np.where(array == 0) 

1052 if len(index): 

1053 msg = f"Found {nBad} zeros in array at elements {index}" 

1054 self.log.warning(msg) 

1055 

1056 array[index] = substituteValue 

1057 

1058 return array 

1059 

1060 def fitPtc(self, dataset): 

1061 """Fit the photon transfer curve to a polynomial or to the 

1062 Astier+19 approximation (Eq. 16). 

1063 

1064 Fit the photon transfer curve with either a polynomial of 

1065 the order specified in the task config (POLNOMIAL), 

1066 or using the exponential approximation in Astier+19 

1067 (Eq. 16, FULLCOVARIANCE). 

1068 

1069 Sigma clipping is performed iteratively for the fit, as 

1070 well as an initial clipping of data points that are more 

1071 than `config.initialNonLinearityExclusionThreshold` away 

1072 from lying on a straight line. This other step is necessary 

1073 because the photon transfer curve turns over catastrophically 

1074 at very high flux (because saturation 

1075 drops the variance to ~0) and these far outliers cause the 

1076 initial fit to fail, meaning the sigma cannot be calculated 

1077 to perform the sigma-clipping. 

1078 

1079 Parameters 

1080 ---------- 

1081 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset` 

1082 The dataset containing the means, variances and 

1083 exposure times. 

1084 

1085 Returns 

1086 ------- 

1087 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset` 

1088 This is the same dataset as the input parameter, however, 

1089 it has been modified to include information such as the 

1090 fit vectors and the fit parameters. See the class 

1091 `PhotonTransferCurveDatase`. 

1092 

1093 Raises 

1094 ------ 

1095 RuntimeError 

1096 Raised if dataset.ptcFitType is None or empty. 

1097 """ 

1098 if dataset.ptcFitType: 

1099 ptcFitType = dataset.ptcFitType 

1100 else: 

1101 raise RuntimeError("ptcFitType is None of empty in PTC dataset.") 

1102 # For FULLCOVARIANCE model fit 

1103 matrixSideFit = dataset.covMatrixSideFullCovFit 

1104 nanMatrixFit = np.empty((matrixSideFit, matrixSideFit)) 

1105 nanMatrixFit[:] = np.nan 

1106 

1107 for amp in dataset.ampNames: 

1108 lenInputTimes = len(dataset.rawExpTimes[amp]) 

1109 listNanMatrixFit = np.empty((lenInputTimes, matrixSideFit, matrixSideFit)) 

1110 listNanMatrixFit[:] = np.nan 

1111 

1112 dataset.covariancesModel[amp] = listNanMatrixFit 

1113 dataset.aMatrix[amp] = nanMatrixFit 

1114 dataset.bMatrix[amp] = nanMatrixFit 

1115 dataset.covariancesModelNoB[amp] = listNanMatrixFit 

1116 dataset.aMatrixNoB[amp] = nanMatrixFit 

1117 dataset.noiseMatrix[amp] = nanMatrixFit 

1118 dataset.noiseMatrixNoB[amp] = nanMatrixFit 

1119 

1120 def errFunc(p, x, y): 

1121 return ptcFunc(p, x) - y 

1122 

1123 sigmaCutPtcOutliers = self.config.sigmaCutPtcOutliers 

1124 maxIterationsPtcOutliers = self.config.maxIterationsPtcOutliers 

1125 

1126 for i, ampName in enumerate(dataset.ampNames): 

1127 meanVecOriginal = dataset.rawMeans[ampName].copy() 

1128 varVecOriginal = dataset.rawVars[ampName].copy() 

1129 varVecOriginal = self._makeZeroSafe(varVecOriginal) 

1130 

1131 # These must be sorted for the given amplifier. 

1132 meanVecSort = np.argsort(meanVecOriginal) 

1133 meanVecSorted = meanVecOriginal[meanVecSort] 

1134 varVecSorted = varVecOriginal[meanVecSort] 

1135 

1136 if self.config.doLegacyTurnoffSelection: 

1137 # Discard points when the variance starts to decrease after two 

1138 # consecutive signal levels 

1139 goodPoints = self._getInitialGoodPoints(meanVecSorted, varVecSorted, 

1140 self.config.minVarPivotSearch, 

1141 self.config.consecutivePointsVarDecreases) 

1142 else: 

1143 # Make sure we have this properly sorted. 

1144 goodPoints = dataset.expIdMask[ampName].copy() 

1145 goodPoints = goodPoints[meanVecSort] 

1146 

1147 # Check if all points are bad from the 'cpExtractPtcTask' 

1148 initialExpIdMask = dataset.expIdMask[ampName].copy() 

1149 

1150 if not (goodPoints.any() and initialExpIdMask.any()): 

1151 msg = (f"SERIOUS: All points in goodPoints: {goodPoints} or " 

1152 f"in initialExpIdMask: {initialExpIdMask} are bad." 

1153 f"Setting {ampName} to BAD.") 

1154 self.log.warning(msg) 

1155 # Fill entries with NaNs 

1156 self.fillBadAmp(dataset, ptcFitType, ampName) 

1157 continue 

1158 

1159 mask = goodPoints.copy() 

1160 

1161 if ptcFitType == 'EXPAPPROXIMATION': 

1162 ptcFunc = funcAstier 

1163 parsIniPtc = [-1e-9, 1.0, 10.] # a00, gain, noise^2 

1164 # lowers and uppers obtained from BOT data studies by 

1165 # C. Lage (UC Davis, 11/2020). 

1166 if self.config.binSize > 1: 

1167 bounds = self._boundsForAstier(parsIniPtc) 

1168 else: 

1169 bounds = self._boundsForAstier(parsIniPtc, lowers=[-1e-4, 0.1, -2000], 

1170 uppers=[1e-4, 10.0, 2000]) 

1171 if ptcFitType == 'POLYNOMIAL': 

1172 ptcFunc = funcPolynomial 

1173 parsIniPtc = self._initialParsForPolynomial(self.config.polynomialFitDegree + 1) 

1174 bounds = self._boundsForPolynomial(parsIniPtc) 

1175 

1176 # We perform an initial (unweighted) fit of variance vs signal 

1177 # (after initial KS test or post-drop selection) to look for 

1178 # outliers, particularly at the high-flux end. The initial fit 

1179 # is performed only for points that are guaranteed to be below 

1180 # the PTC turnoff and then extrapolated to ensure that high 

1181 # flux points that have abnormal variance values can be properly 

1182 # rejected in this phase without biasing the initial fit. 

1183 # This algorithm was initially developed by Seth Digel for 

1184 # the EO Testing pipeline. 

1185 

1186 if self.config.scaleMaxSignalInitialPtcOutlierFit: 

1187 approxGain = np.nanmedian(meanVecSorted/varVecSorted) 

1188 maxADUInitialPtcOutlierFit = self.config.maxSignalInitialPtcOutlierFit/approxGain 

1189 maxDeltaADUInitialPtcOutlierFit = self.config.maxDeltaInitialPtcOutlierFit/approxGain 

1190 self.log.info( 

1191 "Using approximate gain %.3f and ADU signal cutoff of %.1f and delta %.1f " 

1192 "for amplifier %s", 

1193 approxGain, 

1194 maxADUInitialPtcOutlierFit, 

1195 maxDeltaADUInitialPtcOutlierFit, 

1196 ampName, 

1197 ) 

1198 else: 

1199 maxADUInitialPtcOutlierFit = self.config.maxSignalInitialPtcOutlierFit 

1200 maxDeltaADUInitialPtcOutlierFit = self.config.maxDeltaInitialPtcOutlierFit 

1201 

1202 if maxIterationsPtcOutliers == 0: 

1203 # We are not doing any outlier rejection here, but we do want 

1204 # an initial fit. 

1205 res = least_squares( 

1206 errFunc, 

1207 parsIniPtc, 

1208 bounds=bounds, 

1209 args=(meanVecSorted[mask], varVecSorted[mask]), 

1210 ) 

1211 pars = res.x 

1212 newMask = mask.copy() 

1213 else: 

1214 newMask = (mask & (meanVecSorted <= maxADUInitialPtcOutlierFit)) 

1215 

1216 converged = False 

1217 count = 0 

1218 lastMask = mask.copy() 

1219 while count < maxIterationsPtcOutliers: 

1220 res = least_squares( 

1221 errFunc, 

1222 parsIniPtc, 

1223 bounds=bounds, 

1224 args=(meanVecSorted[newMask], varVecSorted[newMask]), 

1225 ) 

1226 pars = res.x 

1227 

1228 sigResids = (varVecSorted - ptcFunc(pars, meanVecSorted))/np.sqrt(varVecSorted) 

1229 # The new mask includes points where the residuals are 

1230 # finite, are less than the cut, and include the original 

1231 # mask of known points that should not be used. 

1232 newMask = ( 

1233 np.isfinite(sigResids) 

1234 & (np.abs(np.nan_to_num(sigResids)) < sigmaCutPtcOutliers) 

1235 & mask 

1236 ) 

1237 # Demand at least 2 points to continue. 

1238 if np.count_nonzero(newMask) < 2: 

1239 msg = (f"SERIOUS: All points after outlier rejection are bad. " 

1240 f"Setting {ampName} to BAD.") 

1241 self.log.warning(msg) 

1242 # Fill entries with NaNs 

1243 self.fillBadAmp(dataset, ptcFitType, ampName) 

1244 break 

1245 

1246 self.log.debug( 

1247 "Iteration %d: Removed %d points in total for %s.", 

1248 count, 

1249 np.count_nonzero(mask) - np.count_nonzero(newMask), 

1250 ampName, 

1251 ) 

1252 

1253 # Loop over all used (True) points. If one of them follows 

1254 # a False point, then it must be within 

1255 # maxDeltaADUInitialPtcOutlierFit of a True point. If it 

1256 # is a large gap, everything above is marked False. 

1257 useMask, = np.where(newMask) 

1258 for useIndex, usePoint in enumerate(useMask): 

1259 if useIndex == 0 or newMask[usePoint - 1]: 

1260 # The previous point was good; continue. 

1261 continue 

1262 deltaADU = meanVecSorted[usePoint] - meanVecSorted[useMask[useIndex - 1]] 

1263 if deltaADU < maxDeltaADUInitialPtcOutlierFit: 

1264 # This jump is fine; continue. 

1265 continue 

1266 

1267 # Mark all further points bad. 

1268 newMask[usePoint:] = False 

1269 break 

1270 

1271 # If the mask hasn't changed then break out. 

1272 if np.all(newMask == lastMask): 

1273 self.log.debug("Convergence at iteration %d; breaking loop for %s.", count, ampName) 

1274 converged = True 

1275 break 

1276 

1277 lastMask = newMask.copy() 

1278 

1279 count += 1 

1280 

1281 if not converged: 

1282 self.log.warning( 

1283 "Outlier detection was not converged prior to %d iteration for %s", 

1284 count, 

1285 ampName 

1286 ) 

1287 

1288 # Set the mask to the new mask, and reset the sorting. 

1289 mask = np.zeros(len(meanVecSort), dtype=np.bool_) 

1290 mask[meanVecSort[newMask]] = True 

1291 maskSorted = newMask.copy() 

1292 

1293 if not mask.any(): 

1294 # We hae already filled the bad amp above, so continue. 

1295 continue 

1296 

1297 dataset.expIdMask[ampName] = mask 

1298 

1299 parsIniPtc = pars 

1300 meanVecFinal = meanVecOriginal[mask] 

1301 varVecFinal = varVecOriginal[mask] 

1302 

1303 # Save the maximum point after outlier detection as the 

1304 # PTC turnoff point. We need to pay attention to the sorting 

1305 # here. 

1306 dataset.ptcTurnoff[ampName] = np.max(meanVecFinal) 

1307 # And compute the ptcTurnoffSamplingError as one half the 

1308 # difference between the previous and next point. 

1309 lastGoodIndex = np.where(maskSorted)[0][-1] 

1310 ptcTurnoffLow = meanVecSorted[lastGoodIndex - 1] 

1311 if lastGoodIndex == (len(meanVecSorted) - 1): 

1312 # If it's the last index, just use the interval. 

1313 ptcTurnoffSamplingError = dataset.ptcTurnoff[ampName] - ptcTurnoffLow 

1314 elif not np.isfinite(meanVecSorted[lastGoodIndex + 1]): 

1315 # If the next index is not finite, just use the interval. 

1316 ptcTurnoffSamplingError = dataset.ptcTurnoff[ampName] - ptcTurnoffLow 

1317 else: 

1318 ptcTurnoffSamplingError = (meanVecSorted[lastGoodIndex + 1] - ptcTurnoffLow)/2. 

1319 dataset.ptcTurnoffSamplingError[ampName] = ptcTurnoffSamplingError 

1320 

1321 if Counter(mask)[False] > 0: 

1322 self.log.info("Number of points discarded in PTC of amplifier %s:" 

1323 " %d out of %d", ampName, Counter(mask)[False], len(meanVecOriginal)) 

1324 

1325 if (len(meanVecFinal) < len(parsIniPtc)): 

1326 msg = (f"SERIOUS: Not enough data points ({len(meanVecFinal)}) compared to the number of " 

1327 f"parameters of the PTC model({len(parsIniPtc)}). Setting {ampName} to BAD.") 

1328 self.log.warning(msg) 

1329 # Fill entries with NaNs 

1330 self.fillBadAmp(dataset, ptcFitType, ampName) 

1331 continue 

1332 # Fit the PTC. 

1333 # The variance of the variance is Var(v)=2*v^2/Npix. This is 

1334 # already calculated in `makeCovArray` of CpPtcExtract. 

1335 # dataset.covariancesSqrtWeights[ampName][:,0,0] 

1336 # has 1/sqrt(Var(v)). 

1337 weightsY = dataset.covariancesSqrtWeights[ampName][:, 0, 0][mask] 

1338 if self.config.doFitBootstrap: 

1339 parsFit, parsFitErr, reducedChiSqPtc = fitBootstrap(parsIniPtc, meanVecFinal, 

1340 varVecFinal, ptcFunc, 

1341 weightsY=weightsY) 

1342 else: 

1343 parsFit, parsFitErr, reducedChiSqPtc = fitLeastSq(parsIniPtc, meanVecFinal, 

1344 varVecFinal, ptcFunc, 

1345 weightsY=weightsY) 

1346 dataset.ptcFitPars[ampName] = parsFit 

1347 dataset.ptcFitParsError[ampName] = parsFitErr 

1348 dataset.ptcFitChiSq[ampName] = reducedChiSqPtc 

1349 

1350 dataset.finalVars[ampName] = varVecOriginal 

1351 dataset.finalVars[ampName][~mask] = np.nan 

1352 dataset.finalModelVars[ampName] = ptcFunc(parsFit, meanVecOriginal) 

1353 dataset.finalModelVars[ampName][~mask] = np.nan 

1354 dataset.finalMeans[ampName] = meanVecOriginal 

1355 dataset.finalMeans[ampName][~mask] = np.nan 

1356 

1357 if ptcFitType == 'EXPAPPROXIMATION': 

1358 ptcGain = parsFit[1] 

1359 ptcGainErr = parsFitErr[1] 

1360 ptcNoise = np.sqrt(np.fabs(parsFit[2])) 

1361 ptcNoiseErr = 0.5*(parsFitErr[2]/np.fabs(parsFit[2]))*np.sqrt(np.fabs(parsFit[2])) 

1362 if ptcFitType == 'POLYNOMIAL': 

1363 ptcGain = 1./parsFit[1] 

1364 ptcGainErr = np.fabs(1./parsFit[1])*(parsFitErr[1]/parsFit[1]) 

1365 ptcNoise = np.sqrt(np.fabs(parsFit[0]))*ptcGain 

1366 ptcNoiseErr = (0.5*(parsFitErr[0]/np.fabs(parsFit[0]))*(np.sqrt(np.fabs(parsFit[0]))))*ptcGain 

1367 dataset.gain[ampName] = ptcGain 

1368 dataset.gainErr[ampName] = ptcGainErr 

1369 dataset.noise[ampName] = ptcNoise 

1370 dataset.noiseErr[ampName] = ptcNoiseErr 

1371 

1372 if not len(dataset.ptcFitType) == 0: 

1373 dataset.ptcFitType = ptcFitType 

1374 if len(dataset.badAmps) == 0: 

1375 dataset.badAmps = [] 

1376 

1377 return dataset 

1378 

1379 def fillBadAmp(self, dataset, ptcFitType, ampName): 

1380 """Fill the dataset with NaNs if there are not enough 

1381 good points. 

1382 

1383 Parameters 

1384 ---------- 

1385 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset` 

1386 The dataset containing the means, variances and 

1387 exposure times. 

1388 ptcFitType : {'POLYNOMIAL', 'EXPAPPROXIMATION'} 

1389 Fit a 'POLYNOMIAL' (degree: 'polynomialFitDegree') or 

1390 'EXPAPPROXIMATION' (Eq. 16 of Astier+19) to the PTC. 

1391 ampName : `str` 

1392 Amplifier name. 

1393 """ 

1394 dataset.badAmps.append(ampName) 

1395 dataset.expIdMask[ampName] = np.repeat(False, len(dataset.rawExpTimes[ampName])) 

1396 dataset.gain[ampName] = np.nan 

1397 dataset.gainErr[ampName] = np.nan 

1398 dataset.noise[ampName] = np.nan 

1399 dataset.noiseErr[ampName] = np.nan 

1400 dataset.ptcFitPars[ampName] = (np.repeat(np.nan, self.config.polynomialFitDegree + 1) if 

1401 ptcFitType in ["POLYNOMIAL", ] else np.repeat(np.nan, 3)) 

1402 dataset.ptcFitParsError[ampName] = (np.repeat(np.nan, self.config.polynomialFitDegree + 1) if 

1403 ptcFitType in ["POLYNOMIAL", ] else np.repeat(np.nan, 3)) 

1404 dataset.ptcFitChiSq[ampName] = np.nan 

1405 dataset.ptcTurnoff[ampName] = np.nan 

1406 dataset.finalVars[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName])) 

1407 dataset.finalModelVars[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName])) 

1408 dataset.finalMeans[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName])) 

1409 

1410 return