Coverage for python/lsst/cp/pipe/ptc/cpSolvePtcTask.py: 10%

532 statements  

« prev     ^ index     » next       coverage.py v7.5.1, created at 2024-05-15 02:24 -0700

1# This file is part of cp_pipe. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21# 

22import numpy as np 

23from collections import Counter 

24import warnings 

25 

26import lsst.pex.config as pexConfig 

27import lsst.pipe.base as pipeBase 

28from lsst.cp.pipe.utils import (fitLeastSq, fitBootstrap, funcPolynomial, 

29 funcAstier, symmetrize, Pol2D) 

30 

31from scipy.signal import fftconvolve 

32from scipy.optimize import least_squares 

33from itertools import groupby 

34from operator import itemgetter 

35 

36import lsst.pipe.base.connectionTypes as cT 

37 

38from lsst.ip.isr import PhotonTransferCurveDataset 

39 

40import copy 

41 

42 

43__all__ = ['PhotonTransferCurveSolveConfig', 'PhotonTransferCurveSolveTask'] 

44 

45 

46class PhotonTransferCurveSolveConnections(pipeBase.PipelineTaskConnections, 

47 dimensions=("instrument", "detector")): 

48 inputCovariances = cT.Input( 

49 name="ptcCovariances", 

50 doc="Tuple with measured covariances from flats.", 

51 storageClass="PhotonTransferCurveDataset", 

52 dimensions=("instrument", "exposure", "detector"), 

53 isCalibration=True, 

54 multiple=True, 

55 ) 

56 camera = cT.PrerequisiteInput( 

57 name="camera", 

58 doc="Camera the input data comes from.", 

59 storageClass="Camera", 

60 dimensions=("instrument",), 

61 isCalibration=True, 

62 ) 

63 outputPtcDataset = cT.Output( 

64 name="ptcDatsetProposal", 

65 doc="Output proposed ptc dataset.", 

66 storageClass="PhotonTransferCurveDataset", 

67 dimensions=("instrument", "detector"), 

68 multiple=False, 

69 isCalibration=True, 

70 ) 

71 

72 

73class PhotonTransferCurveSolveConfig(pipeBase.PipelineTaskConfig, 

74 pipelineConnections=PhotonTransferCurveSolveConnections): 

75 """Configuration for fitting measured covariances. 

76 """ 

77 

78 ptcFitType = pexConfig.ChoiceField( 

79 dtype=str, 

80 doc="Fit PTC to Eq. 16, Eq. 20 in Astier+19, or to a polynomial.", 

81 default="POLYNOMIAL", 

82 allowed={ 

83 "POLYNOMIAL": "n-degree polynomial (use 'polynomialFitDegree' to set 'n').", 

84 "EXPAPPROXIMATION": "Approximation in Astier+19 (Eq. 16).", 

85 "FULLCOVARIANCE": "Full covariances model in Astier+19 (Eq. 20)" 

86 } 

87 ) 

88 minMeanSignal = pexConfig.DictField( 

89 keytype=str, 

90 itemtype=float, 

91 doc="Minimum values (inclusive) of mean signal (in ADU) per amp to use." 

92 " The same cut is applied to all amps if this parameter [`dict`] is passed as " 

93 " {'ALL_AMPS': value}", 

94 default={'ALL_AMPS': 0.0}, 

95 ) 

96 maxMeanSignal = pexConfig.DictField( 

97 keytype=str, 

98 itemtype=float, 

99 doc="Maximum values (inclusive) of mean signal (in ADU) below which to consider, per amp." 

100 " The same cut is applied to all amps if this dictionary is of the form" 

101 " {'ALL_AMPS': value}", 

102 default={'ALL_AMPS': 1e6}, 

103 ) 

104 maximumRangeCovariancesAstier = pexConfig.Field( 

105 dtype=int, 

106 doc="Maximum range of measured covariances as in Astier+19", 

107 default=8, 

108 ) 

109 maximumRangeCovariancesAstierFullCovFit = pexConfig.Field( 

110 dtype=int, 

111 doc="Maximum range up to where to fit covariances as in Astier+19, " 

112 "for the FULLCOVARIANCE model." 

113 "This is different from maximumRangeCovariancesAstier." 

114 "It should be less or equal than maximumRangeCovariancesAstier." 

115 "The number of parameters for this model is " 

116 "3*maximumRangeCovariancesAstierFullCovFit^2 + 1, so increase with care " 

117 "so that the fit is not too slow.", 

118 default=8, 

119 ) 

120 doSubtractLongRangeCovariances = pexConfig.Field( 

121 dtype=bool, 

122 doc="Subtract long-range covariances before FULLCOVARIANCE fit, " 

123 "beyond startLongRangeCovariances?", 

124 default=False, 

125 ) 

126 startLongRangeCovariances = pexConfig.Field( 

127 dtype=int, 

128 doc="If doSubtractLongRangeCovariances is True, subtract covariances " 

129 "beyond this range. It should be less than maximumRangeCovariancesAstier. ", 

130 default=4, 

131 ) 

132 polyDegLongRangeCovariances = pexConfig.Field( 

133 dtype=int, 

134 doc="If doSubtractLongRangeCovariances is True, polynomial " 

135 "degree to fit data beyond startLongRangeCovariances.", 

136 default=1, 

137 ) 

138 sigmaClipFullFitCovariancesAstier = pexConfig.Field( 

139 dtype=float, 

140 doc="sigma clip for full model fit for FULLCOVARIANCE ptcFitType ", 

141 default=5.0, 

142 ) 

143 maxIterFullFitCovariancesAstier = pexConfig.Field( 

144 dtype=int, 

145 doc="Maximum number of iterations in full model fit for FULLCOVARIANCE ptcFitType", 

146 default=3, 

147 ) 

148 polynomialFitDegree = pexConfig.Field( 

149 dtype=int, 

150 doc="Degree of polynomial to fit the PTC, when 'ptcFitType'=POLYNOMIAL.", 

151 default=3, 

152 ) 

153 doLegacyTurnoffSelection = pexConfig.Field( 

154 dtype=bool, 

155 doc="Use 'legacy' computation for PTC turnoff selection. If set " 

156 "to False, then the KS test p-value selection will be used instead.", 

157 default=False, 

158 ) 

159 sigmaCutPtcOutliers = pexConfig.Field( 

160 dtype=float, 

161 doc="Sigma cut for outlier rejection in PTC.", 

162 default=5.0, 

163 ) 

164 maxIterationsPtcOutliers = pexConfig.RangeField( 

165 dtype=int, 

166 doc="Maximum number of iterations for outlier rejection in PTC.", 

167 default=2, 

168 min=0 

169 ) 

170 maxSignalInitialPtcOutlierFit = pexConfig.Field( 

171 dtype=float, 

172 doc="Maximum signal considered for intial outlier fit. This should be below " 

173 "the PTC turnoff to ensure accurate outlier rejection. If " 

174 "scaleMaxSignalInitialPtcOutlierFit=True then the units are electrons; " 

175 "otherwise ADU.", 

176 default=50_000., 

177 ) 

178 maxDeltaInitialPtcOutlierFit = pexConfig.Field( 

179 dtype=float, 

180 doc="If there are any outliers in the initial fit that have mean greater than " 

181 "maxSignalInitialPtcOutlierFit, then no points that have this delta " 

182 "mean from the previous ``good`` point are allowed. If " 

183 "scaleMaxSignalInitialPtcOutlierFit=True then the units are electrons; " 

184 "otherwise ADU.", 

185 default=9_000., 

186 ) 

187 scaleMaxSignalInitialPtcOutlierFit = pexConfig.Field( 

188 dtype=bool, 

189 doc="Scale maxSignalInitialPtcOutlierFit and maxDeltaInitialPtcOutlierFit by " 

190 "approximate gain? If yes then " 

191 "maxSignalInitialPtcOutlierFit and maxDeltaInitialPtcOutlierFit are assumed " 

192 "to have units of electrons, otherwise ADU.", 

193 default=True, 

194 ) 

195 minVarPivotSearch = pexConfig.Field( 

196 dtype=float, 

197 doc="The code looks for a pivot signal point after which the variance starts decreasing at high-flux" 

198 " to exclude then from the PTC model fit. However, sometimes at low fluxes, the variance" 

199 " decreases slightly. Set this variable for the variance value, in ADU^2, after which the pivot " 

200 " should be sought. Only used if doLegacyTurnoffSelection is True.", 

201 default=10000, 

202 ) 

203 consecutivePointsVarDecreases = pexConfig.RangeField( 

204 dtype=int, 

205 doc="Required number of consecutive points/fluxes in the PTC where the variance " 

206 "decreases in order to find a first estimate of the PTC turn-off. " 

207 "Only used if doLegacyTurnoffSelection is True.", 

208 default=2, 

209 min=2 

210 ) 

211 ksTestMinPvalue = pexConfig.Field( 

212 dtype=float, 

213 doc="Minimum value of the Gaussian histogram KS test p-value to be used in PTC fit. " 

214 "Only used if doLegacyTurnoffSelection is False.", 

215 default=0.01, 

216 ) 

217 doFitBootstrap = pexConfig.Field( 

218 dtype=bool, 

219 doc="Use bootstrap for the PTC fit parameters and errors?.", 

220 default=False, 

221 ) 

222 binSize = pexConfig.Field( 

223 dtype=int, 

224 doc="Bin the image by this factor in both dimensions.", 

225 default=1, 

226 ) 

227 

228 def validate(self): 

229 super().validate() 

230 fitMatrixSide = self.maximumRangeCovariancesAstierFullCovFit 

231 measureMatrixSide = self.maximumRangeCovariancesAstier 

232 if self.ptcFitType == "FULLCOVARIANCE": 

233 if fitMatrixSide > measureMatrixSide: 

234 raise RuntimeError("Covariance fit size %s is larger than" 

235 "measurement size %s.", 

236 fitMatrixSide, measureMatrixSide) 

237 if self.doSubtractLongRangeCovariances: 

238 startLag = self.startLongRangeCovariances 

239 if measureMatrixSide < startLag: 

240 raise RuntimeError("Covariance measure size %s is smaller than long" 

241 "-range covariance starting point %s.", 

242 measureMatrixSide, startLag) 

243 

244 

245class PhotonTransferCurveSolveTask(pipeBase.PipelineTask): 

246 """Task to fit the PTC from flat covariances. 

247 

248 The first task of the PTC measurement pipeline, 

249 ``PhotonTransferCurveMeasureTask`` (and assumed to have been run 

250 before this task), produced a list of 

251 `~lsst.ip.isr.PhotonTransferCurveDataset` objects. Each dataset 

252 contains the mean signal and covariances of the 

253 difference image of the flat-field images taken at 

254 the same exposure time. The list also contains dummy 

255 datasets (with no measurements), whose purpose is to have 

256 the input and output dimensions of ``PhotonTransferCurveMeasureTask`` 

257 match. 

258 

259 This task, ``PhotonTransferCurveSolveTask``, assembles the list 

260 of individual PTC datasets produced 

261 by ``PhotonTransferCurveMeasureTask`` into one single final PTC 

262 dataset, discarding the dummy datset as appropiate. 

263 The task fits the measured (co)variances to one of three models: 

264 a polynomial model of a given order, or the models described 

265 in equations 16 and 20 of Astier+19. These options are referred 

266 to as ``POLYNOMIAL``, ``EXPAPPROXIMATION``, and ``FULLCOVARIANCE`` 

267 in the configuration options of the task, respectively). 

268 Parameters of interest such as the gain and noise are derived 

269 from the fits. The ``FULLCOVARIANCE`` model is fitted to the 

270 full covariance data (as oppossed to the other two models, which 

271 are fit to the variance vs mean measurements only). 

272 

273 Astier+19: "The Shape of the Photon Transfer Curve 

274 of CCD sensors", arXiv:1905.08677 

275 """ 

276 

277 ConfigClass = PhotonTransferCurveSolveConfig 

278 _DefaultName = 'cpPhotonTransferCurveSolve' 

279 

280 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

281 """Ensure that the input and output dimensions are passed along. 

282 

283 Parameters 

284 ---------- 

285 butlerQC : `~lsst.daf.butler.QuantumContext` 

286 Butler to operate on. 

287 inputRefs : `~lsst.pipe.base.InputQuantizedConnection` 

288 Input data refs to load. 

289 ouptutRefs : `~lsst.pipe.base.OutputQuantizedConnection` 

290 Output data refs to persist. 

291 """ 

292 inputs = butlerQC.get(inputRefs) 

293 detId = inputRefs.inputCovariances[0].dataId['detector'] 

294 outputs = self.run(inputCovariances=inputs['inputCovariances'], camera=inputs['camera'], detId=detId) 

295 butlerQC.put(outputs, outputRefs) 

296 

297 def run(self, inputCovariances, camera=None, detId=0): 

298 """Fit measured covariances to different models. 

299 

300 Parameters 

301 ---------- 

302 inputCovariances : `list` [`lsst.ip.isr.PhotonTransferCurveDataset`] 

303 List of lsst.ip.isr.PhotonTransferCurveDataset datasets. 

304 camera : `lsst.afw.cameraGeom.Camera`, optional 

305 Input camera. 

306 detId : `int` 

307 Detector ID to locate the detector in the camera and 

308 populate the `lsst.ip.isr.PhotonTransferCurveDataset` 

309 metadata. 

310 Returns 

311 ------- 

312 results : `lsst.pipe.base.Struct` 

313 The resultins structure contains: 

314 

315 ``outputPtcDatset`` 

316 Final PTC dataset, containing information such as the 

317 means, variances, and exposure times 

318 (`lsst.ip.isr.PhotonTransferCurveDataset`). 

319 """ 

320 # Find the ampNames from a non-dummy ptc. 

321 ampNames = [] 

322 for partialPtcDataset in inputCovariances: 

323 if partialPtcDataset.ptcFitType != 'DUMMY': 

324 ampNames = partialPtcDataset.ampNames 

325 break 

326 

327 # Each amp may have a different min and max ADU signal 

328 # specified in the config. 

329 maxMeanSignalDict = {ampName: 1e6 for ampName in ampNames} 

330 minMeanSignalDict = {ampName: 0.0 for ampName in ampNames} 

331 for ampName in ampNames: 

332 if 'ALL_AMPS' in self.config.maxMeanSignal: 

333 maxMeanSignalDict[ampName] = self.config.maxMeanSignal['ALL_AMPS'] 

334 elif ampName in self.config.maxMeanSignal: 

335 maxMeanSignalDict[ampName] = self.config.maxMeanSignal[ampName] 

336 

337 if 'ALL_AMPS' in self.config.minMeanSignal: 

338 minMeanSignalDict[ampName] = self.config.minMeanSignal['ALL_AMPS'] 

339 elif ampName in self.config.minMeanSignal: 

340 minMeanSignalDict[ampName] = self.config.minMeanSignal[ampName] 

341 

342 # Assemble individual PTC datasets into a single PTC dataset. 

343 datasetPtc = PhotonTransferCurveDataset( 

344 ampNames=ampNames, 

345 ptcFitType=self.config.ptcFitType, 

346 covMatrixSide=self.config.maximumRangeCovariancesAstier, 

347 covMatrixSideFullCovFit=self.config.maximumRangeCovariancesAstierFullCovFit) 

348 

349 for partialPtcDataset in inputCovariances: 

350 # Ignore dummy datasets 

351 if partialPtcDataset.ptcFitType == 'DUMMY': 

352 continue 

353 for ampName in ampNames: 

354 # The partial dataset consists of lists of values for each 

355 # quantity. In the case of the input exposure pairs, this is a 

356 # list of tuples. In all cases we only want the first 

357 # (and only) element of the list. 

358 datasetPtc.inputExpIdPairs[ampName].append(partialPtcDataset.inputExpIdPairs[ampName][0]) 

359 datasetPtc.rawExpTimes[ampName] = np.append(datasetPtc.rawExpTimes[ampName], 

360 partialPtcDataset.rawExpTimes[ampName][0]) 

361 datasetPtc.rawMeans[ampName] = np.append(datasetPtc.rawMeans[ampName], 

362 partialPtcDataset.rawMeans[ampName][0]) 

363 datasetPtc.rawVars[ampName] = np.append(datasetPtc.rawVars[ampName], 

364 partialPtcDataset.rawVars[ampName][0]) 

365 datasetPtc.rowMeanVariance[ampName] = np.append(datasetPtc.rowMeanVariance[ampName], 

366 partialPtcDataset.rowMeanVariance[ampName][0]) 

367 datasetPtc.photoCharges[ampName] = np.append(datasetPtc.photoCharges[ampName], 

368 partialPtcDataset.photoCharges[ampName][0]) 

369 datasetPtc.histVars[ampName] = np.append(datasetPtc.histVars[ampName], 

370 partialPtcDataset.histVars[ampName][0]) 

371 datasetPtc.histChi2Dofs[ampName] = np.append(datasetPtc.histChi2Dofs[ampName], 

372 partialPtcDataset.histChi2Dofs[ampName][0]) 

373 datasetPtc.kspValues[ampName] = np.append(datasetPtc.kspValues[ampName], 

374 partialPtcDataset.kspValues[ampName][0]) 

375 datasetPtc.noiseList[ampName] = np.append(datasetPtc.noiseList[ampName], 

376 partialPtcDataset.noise[ampName]) 

377 datasetPtc.covariances[ampName] = np.append( 

378 datasetPtc.covariances[ampName].ravel(), 

379 partialPtcDataset.covariances[ampName].ravel() 

380 ).reshape( 

381 ( 

382 len(datasetPtc.rawExpTimes[ampName]), 

383 datasetPtc.covMatrixSide, 

384 datasetPtc.covMatrixSide, 

385 ) 

386 ) 

387 datasetPtc.covariancesSqrtWeights[ampName] = np.append( 

388 datasetPtc.covariancesSqrtWeights[ampName].ravel(), 

389 partialPtcDataset.covariancesSqrtWeights[ampName].ravel() 

390 ).reshape( 

391 ( 

392 len(datasetPtc.rawExpTimes[ampName]), 

393 datasetPtc.covMatrixSide, 

394 datasetPtc.covMatrixSide, 

395 ) 

396 ) 

397 

398 # Apply min/max masking. 

399 rawMean = partialPtcDataset.rawMeans[ampName][0] 

400 rawVar = partialPtcDataset.rawVars[ampName][0] 

401 expIdMask = partialPtcDataset.expIdMask[ampName][0] 

402 if (rawMean <= minMeanSignalDict[ampName]) or (rawMean >= maxMeanSignalDict[ampName]) \ 

403 or not np.isfinite(rawMean) or not np.isfinite(rawVar): 

404 expIdMask = False 

405 

406 kspValue = partialPtcDataset.kspValues[ampName][0] 

407 if not self.config.doLegacyTurnoffSelection and \ 

408 kspValue < self.config.ksTestMinPvalue: 

409 expIdMask = False 

410 

411 datasetPtc.expIdMask[ampName] = np.append(datasetPtc.expIdMask[ampName], expIdMask) 

412 

413 for key, value in partialPtcDataset.auxValues.items(): 

414 if key in datasetPtc.auxValues: 

415 datasetPtc.auxValues[key] = np.append(datasetPtc.auxValues[key], value) 

416 else: 

417 datasetPtc.auxValues[key] = value 

418 

419 # Sort arrays that are filled so far in the final dataset by 

420 # rawMeans index. 

421 # First compute the mean across all the amps to make sure that they are 

422 # all sorted the same way. 

423 detectorMeans = np.zeros(len(datasetPtc.inputExpIdPairs[ampNames[0]])) 

424 

425 for i in range(len(detectorMeans)): 

426 arr = np.array([datasetPtc.rawMeans[ampName][i] for ampName in ampNames]) 

427 good, = (np.isfinite(arr)).nonzero() 

428 if good.size == 0: 

429 detectorMeans[i] = np.nan 

430 else: 

431 detectorMeans[i] = np.mean(arr[good]) 

432 

433 index = np.argsort(detectorMeans) 

434 

435 for ampName in ampNames: 

436 datasetPtc.inputExpIdPairs[ampName] = np.array( 

437 datasetPtc.inputExpIdPairs[ampName] 

438 )[index].tolist() 

439 datasetPtc.rawExpTimes[ampName] = datasetPtc.rawExpTimes[ampName][index] 

440 datasetPtc.rawMeans[ampName] = datasetPtc.rawMeans[ampName][index] 

441 datasetPtc.rawVars[ampName] = datasetPtc.rawVars[ampName][index] 

442 datasetPtc.rowMeanVariance[ampName] = datasetPtc.rowMeanVariance[ampName][index] 

443 datasetPtc.photoCharges[ampName] = datasetPtc.photoCharges[ampName][index] 

444 datasetPtc.histVars[ampName] = datasetPtc.histVars[ampName][index] 

445 datasetPtc.histChi2Dofs[ampName] = datasetPtc.histChi2Dofs[ampName][index] 

446 datasetPtc.kspValues[ampName] = datasetPtc.kspValues[ampName][index] 

447 datasetPtc.expIdMask[ampName] = datasetPtc.expIdMask[ampName][index] 

448 datasetPtc.covariances[ampName] = datasetPtc.covariances[ampName][index] 

449 datasetPtc.covariancesSqrtWeights[ampName] = datasetPtc.covariancesSqrtWeights[ampName][index] 

450 for key, value in datasetPtc.auxValues.items(): 

451 datasetPtc.auxValues[key] = value[index] 

452 

453 if self.config.ptcFitType == "FULLCOVARIANCE": 

454 # Fit the measured covariances vs mean signal to 

455 # the Astier+19 full model (Eq. 20). Before that 

456 # do a preliminary fit to the variance (C_00) vs mean 

457 # signal (mu) curve using the EXPAPPROXIMATION model 

458 # (Eq. 16 in Astier+19) in order to 

459 # get the flat pairs that are masked. The 

460 # points at these fluxes will also be masked when 

461 # calculating the other elements of the covariance 

462 # matrix, C_ij, i!=j). 

463 

464 # Preliminary fit, usign a temp dataset to get the mask 

465 tempDatasetPtc = copy.copy(datasetPtc) 

466 tempDatasetPtc.ptcFitType = "EXPAPPROXIMATION" 

467 tempDatasetPtc = self.fitMeasurementsToModel(tempDatasetPtc) 

468 

469 # "FULLCOVARIANCE", using the mask obtained from the 

470 # previous fit. 

471 for ampName in datasetPtc.ampNames: 

472 datasetPtc.expIdMask[ampName] = tempDatasetPtc.expIdMask[ampName] 

473 datasetPtc.fitType = "FULLCOVARIANCE" 

474 datasetPtc = self.fitMeasurementsToModel(datasetPtc) 

475 # The other options are: self.config.ptcFitType in 

476 # ("EXPAPPROXIMATION", "POLYNOMIAL") 

477 else: 

478 # Fit the PTC to a polynomial or to Astier+19 exponential 

479 # approximation (Eq. 16). Fill up 

480 # PhotonTransferCurveDataset object. 

481 datasetPtc = self.fitMeasurementsToModel(datasetPtc) 

482 

483 # Initial validation of PTC fit. 

484 for ampName in ampNames: 

485 # These may be all nan (esp. in tests) and will be filtered 

486 # as appropriate later. 

487 with warnings.catch_warnings(): 

488 warnings.simplefilter("ignore") 

489 noise = np.nanmedian(datasetPtc.noiseList[ampName]) 

490 noiseFitted = np.sqrt(datasetPtc.noise[ampName]) 

491 

492 # Check if noise is close to noiseFitted 

493 if not np.isclose(noiseFitted, noise, rtol=0.05, atol=0.0, equal_nan=True): 

494 self.log.warning(f"Read noise from PTC fit ({noiseFitted}) is not consistent " 

495 f"with read noise measured from overscan ({noise}) for " 

496 f"amplifier {ampName}. Try adjusting the fit range.") 

497 

498 if camera: 

499 detector = camera[detId] 

500 else: 

501 detector = None 

502 datasetPtc.updateMetadataFromExposures(inputCovariances) 

503 datasetPtc.updateMetadata(setDate=True, camera=camera, detector=detector) 

504 

505 return pipeBase.Struct( 

506 outputPtcDataset=datasetPtc, 

507 ) 

508 

509 def fitMeasurementsToModel(self, dataset): 

510 """Fit the measured covariances vs mean signal to a 

511 polynomial or one of the models in Astier+19 

512 (Eq. 16 or Eq.20). 

513 

514 Parameters 

515 ---------- 

516 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset` 

517 The dataset containing information such as the means, 

518 (co)variances, and exposure times. 

519 

520 Returns 

521 ------- 

522 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset` 

523 This is the same dataset as the input parameter, however, 

524 it has been modified to include information such as the 

525 fit vectors and the fit parameters. See the class 

526 `PhotonTransferCurveDatase`. 

527 """ 

528 fitType = dataset.ptcFitType 

529 if fitType in ["FULLCOVARIANCE", ]: 

530 # This model uses the full covariance matrix in the fit. 

531 # The PTC is technically defined as variance vs signal, 

532 # with variance = Cov_00 

533 dataset = self.fitDataFullCovariance(dataset) 

534 elif fitType in ["POLYNOMIAL", "EXPAPPROXIMATION"]: 

535 # The PTC is technically defined as variance vs signal 

536 dataset = self.fitPtc(dataset) 

537 else: 

538 raise RuntimeError( 

539 f"Fitting option {fitType} not one of " 

540 "'POLYNOMIAL', 'EXPAPPROXIMATION', or 'FULLCOVARIANCE'" 

541 ) 

542 

543 return dataset 

544 

545 def fitDataFullCovariance(self, dataset): 

546 """Fit measured flat covariances to the full model in 

547 Astier+19 (Eq. 20). 

548 

549 Parameters 

550 ---------- 

551 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset` 

552 The dataset containing information such as the means, 

553 (co)variances, and exposure times. 

554 

555 Returns 

556 ------- 

557 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset` 

558 This is the same dataset as the input parameter, however, 

559 it has been modified to include information such as the 

560 fit vectors and the fit parameters. See the class 

561 `PhotonTransferCurveDatase`. 

562 

563 Notes 

564 ----- 

565 The parameters of the full model for C_ij(mu) ("C_ij" and "mu" 

566 in ADU^2 and ADU, respectively) in Astier+19 (Eq. 20) are: 

567 

568 - "a" coefficients (r by r matrix), units: 1/e 

569 - "b" coefficients (r by r matrix), units: 1/e 

570 - noise matrix (r by r matrix), units: e^2 

571 - gain, units: e/ADU 

572 

573 "b" appears in Eq. 20 only through the "ab" combination, which 

574 is defined in this code as "c=ab". 

575 

576 Total number of parameters: #entries(a) + #entries(c) + #entries(noise) 

577 + 1. This is equivalent to r^2 + r^2 + r^2 + 1, where "r" is the 

578 maximum lag considered for the covariances calculation, and the 

579 extra "1" is the gain. If "b" is 0, then "c" is 0, and len(pInit) will 

580 have r^2 fewer entries. 

581 """ 

582 matrixSide = dataset.covMatrixSide 

583 matrixSideFit = dataset.covMatrixSideFullCovFit 

584 lenParams = matrixSideFit*matrixSideFit 

585 

586 for ampName in dataset.ampNames: 

587 lenInputTimes = len(dataset.rawExpTimes[ampName]) 

588 # Not used when ptcFitType is 'FULLCOVARIANCE' 

589 dataset.ptcFitPars[ampName] = np.array([np.nan]) 

590 dataset.ptcFitParsError[ampName] = np.array([np.nan]) 

591 dataset.ptcFitChiSq[ampName] = np.nan 

592 

593 if ampName in dataset.badAmps: 

594 # Bad amp 

595 # Entries need to have proper dimensions so read/write 

596 # with astropy.Table works. 

597 nanMatrixFit = np.full((matrixSideFit, matrixSideFit), np.nan) 

598 listNanMatrix = np.full((lenInputTimes, matrixSide, matrixSide), np.nan) 

599 listNanMatrixFit = np.full((lenInputTimes, matrixSideFit, matrixSideFit), np.nan) 

600 dataset.covariancesModel[ampName] = listNanMatrixFit 

601 dataset.covariancesSqrtWeights[ampName] = listNanMatrix 

602 dataset.aMatrix[ampName] = nanMatrixFit 

603 dataset.bMatrix[ampName] = nanMatrixFit 

604 dataset.covariancesModelNoB[ampName] = listNanMatrixFit 

605 dataset.aMatrixNoB[ampName] = nanMatrixFit 

606 dataset.noiseMatrix[ampName] = nanMatrixFit 

607 dataset.noiseMatrixNoB[ampName] = nanMatrixFit 

608 

609 dataset.expIdMask[ampName] = np.repeat(False, lenInputTimes) 

610 dataset.gain[ampName] = np.nan 

611 dataset.gainErr[ampName] = np.nan 

612 dataset.noise[ampName] = np.nan 

613 dataset.noiseErr[ampName] = np.nan 

614 dataset.finalVars[ampName] = np.repeat(np.nan, lenInputTimes) 

615 dataset.finalModelVars[ampName] = np.repeat(np.nan, lenInputTimes) 

616 dataset.finalMeans[ampName] = np.repeat(np.nan, lenInputTimes) 

617 continue 

618 

619 muAtAmp = dataset.rawMeans[ampName] 

620 maskAtAmp = dataset.expIdMask[ampName] 

621 if len(maskAtAmp) == 0: 

622 maskAtAmp = np.repeat(True, len(muAtAmp)) 

623 

624 muAtAmpMasked = muAtAmp[maskAtAmp] 

625 covAtAmp = dataset.covariances[ampName] 

626 covAtAmpMasked = np.nan_to_num(covAtAmp)[maskAtAmp] 

627 covSqrtWeightsAtAmp = dataset.covariancesSqrtWeights[ampName] 

628 covSqrtWeightsAtAmpMasked = np.nan_to_num(covSqrtWeightsAtAmp)[maskAtAmp] 

629 

630 # Subtract long-range covariances 

631 if self.config.doSubtractLongRangeCovariances: 

632 startLag = self.config.startLongRangeCovariances 

633 covAtAmpMasked, covSqrtWeightsAtAmpMasked = self.subtractDistantOffset( 

634 muAtAmpMasked, covAtAmpMasked, 

635 covSqrtWeightsAtAmpMasked, 

636 start=startLag, 

637 degree=self.config.polyDegLongRangeCovariances) 

638 

639 # In principle, we could fit to a lag smaller than the measured 

640 # covariances. 

641 r = self.config.maximumRangeCovariancesAstierFullCovFit 

642 covAtAmpForFitMasked = covAtAmpMasked[:, :r, :r] 

643 covSqrtWeightsAtAmpForFitMasked = covSqrtWeightsAtAmpMasked[:, :r, :r] 

644 

645 # Initial fit, to approximate parameters, with c=0 

646 a0, c0, noise0, gain0 = self.initialFitFullCovariance( 

647 muAtAmpMasked, 

648 covAtAmpForFitMasked, 

649 covSqrtWeightsAtAmpForFitMasked 

650 ) 

651 

652 # Fit full model (Eq. 20 of Astier+19) and same model with 

653 # b=0 (c=0 in this code) 

654 pInit = np.concatenate((a0.ravel(), c0.ravel(), noise0.ravel(), np.array(gain0)), axis=None) 

655 functionsDict = {'fullModel': self.funcFullCovarianceModel, 

656 'fullModelNoB': self.funcFullCovarianceModelNoB} 

657 fitResults = {'fullModel': {'a': [], 'c': [], 'noise': [], 'gain': [], 'paramsErr': []}, 

658 'fullModelNoB': {'a': [], 'c': [], 'noise': [], 'gain': [], 'paramsErr': []}} 

659 for key in functionsDict: 

660 params, paramsErr, _ = fitLeastSq(pInit, muAtAmpMasked, 

661 covAtAmpForFitMasked.ravel(), functionsDict[key], 

662 weightsY=covSqrtWeightsAtAmpForFitMasked.ravel()) 

663 a = params[:lenParams].reshape((matrixSideFit, matrixSideFit)) 

664 c = params[lenParams:2*lenParams].reshape((matrixSideFit, matrixSideFit)) 

665 noise = params[2*lenParams:3*lenParams].reshape((matrixSideFit, matrixSideFit)) 

666 gain = params[-1] 

667 

668 fitResults[key]['a'] = a 

669 fitResults[key]['c'] = c 

670 fitResults[key]['noise'] = noise 

671 fitResults[key]['gain'] = gain 

672 fitResults[key]['paramsErr'] = paramsErr 

673 

674 # Put the information in the PTC dataset 

675 

676 # Not used when ptcFitType is 'FULLCOVARIANCE' 

677 dataset.ptcFitPars[ampName] = np.array([np.nan]) 

678 dataset.ptcFitParsError[ampName] = np.array([np.nan]) 

679 dataset.ptcFitChiSq[ampName] = np.nan 

680 

681 # Save full covariances, covariances models, and their weights. 

682 # dataset.expIdMask is already full, but needs to be 

683 # converted to bool. 

684 dataset.expIdMask[ampName] = np.array(dataset.expIdMask[ampName], dtype=bool) 

685 dataset.covariances[ampName] = covAtAmp 

686 # We evaluate the covariance model everywhere, even the 

687 # masked amps. 

688 dataset.covariancesModel[ampName] = self.evalCovModel(muAtAmp, 

689 fitResults['fullModel']['a'], 

690 fitResults['fullModel']['c'], 

691 fitResults['fullModel']['noise'], 

692 fitResults['fullModel']['gain']) 

693 dataset.covariancesSqrtWeights[ampName] = covSqrtWeightsAtAmp 

694 dataset.aMatrix[ampName] = fitResults['fullModel']['a'] 

695 dataset.bMatrix[ampName] = fitResults['fullModel']['c']/fitResults['fullModel']['a'] 

696 dataset.covariancesModelNoB[ampName] = self.evalCovModel(muAtAmp, 

697 fitResults['fullModelNoB']['a'], 

698 fitResults['fullModelNoB']['c'], 

699 fitResults['fullModelNoB']['noise'], 

700 fitResults['fullModelNoB']['gain'], 

701 setBtoZero=True) 

702 dataset.aMatrixNoB[ampName] = fitResults['fullModelNoB']['a'] 

703 dataset.gain[ampName] = fitResults['fullModel']['gain'] 

704 dataset.gainErr[ampName] = fitResults['fullModel']['paramsErr'][-1] 

705 readoutNoise = fitResults['fullModel']['noise'][0][0] 

706 readoutNoiseSqrt = np.sqrt(np.fabs(readoutNoise)) 

707 dataset.noise[ampName] = readoutNoise 

708 readoutNoiseSigma = fitResults['fullModel']['paramsErr'][2*lenParams] 

709 dataset.noiseErr[ampName] = 0.5*(readoutNoiseSigma/np.fabs(readoutNoise))*readoutNoiseSqrt 

710 dataset.noiseMatrix[ampName] = fitResults['fullModel']['noise'] 

711 dataset.noiseMatrixNoB[ampName] = fitResults['fullModelNoB']['noise'] 

712 

713 dataset.finalVars[ampName] = covAtAmp[:, 0, 0] 

714 dataset.finalModelVars[ampName] = dataset.covariancesModel[ampName][:, 0, 0] 

715 dataset.finalMeans[ampName] = muAtAmp 

716 

717 return dataset 

718 

719 def initialFitFullCovariance(self, mu, cov, sqrtW): 

720 """ Performs a crude parabolic fit of the data in order to start 

721 the full fit close to the solution, setting b=0 (c=0) in Eq. 20 

722 of Astier+19. 

723 

724 Parameters 

725 ---------- 

726 mu : `numpy.array`, (N,) 

727 Signal `mu` (ADU) 

728 cov : `numpy.array`, (N, M, M) 

729 Covariance arrays of size `(M, M)` (with 

730 `M = config.maximumRangeCovariancesAstier`), 

731 indexed by mean signal `mu`. 

732 sqrtW : `numpy.array`, (N,) 

733 Covariance weights, defined as 1./sqrt(Variances) 

734 

735 Returns 

736 ------- 

737 a : `numpy.array`, (M, M) 

738 "a" parameter per flux in Eq. 20 of Astier+19. 

739 c : `numpy.array`, (M, M) 

740 "c"="ab" parameter per flux in Eq. 20 of Astier+19. 

741 noise : `numpy.array`, (M, M) 

742 "noise" parameter per flux in Eq. 20 of Astier+19. 

743 gain : `float` 

744 Amplifier gain (e/ADU) 

745 """ 

746 matrixSideFit = self.config.maximumRangeCovariancesAstierFullCovFit 

747 

748 # Initialize fit parameters 

749 a = np.zeros((matrixSideFit, matrixSideFit)) 

750 c = np.zeros((matrixSideFit, matrixSideFit)) 

751 noise = np.zeros((matrixSideFit, matrixSideFit)) 

752 gain = 1. 

753 

754 # iterate the fit to account for higher orders 

755 # the chi2 does not necessarily go down, so one could 

756 # stop when it increases 

757 oldChi2 = 1e30 

758 for _ in range(5): 

759 model = np.nan_to_num(self.evalCovModel(mu, a, c, noise, gain, setBtoZero=True)) 

760 # loop on lags 

761 for i in range(matrixSideFit): 

762 for j in range(matrixSideFit): 

763 # fit a parabola for a given lag 

764 parsFit = np.polyfit(mu, cov[:, i, j] - model[:, i, j], 

765 2, w=sqrtW[:, i, j]) 

766 # model equation (Eq. 20) in Astier+19, with c=a*b=0: 

767 a[i, j] += parsFit[0] 

768 noise[i, j] += parsFit[2] 

769 if i + j == 0: 

770 gain = 1./(1/gain+parsFit[1]) 

771 weightedRes = (model - cov)*sqrtW 

772 chi2 = (weightedRes.flatten()**2).sum() 

773 if chi2 > oldChi2: 

774 break 

775 oldChi2 = chi2 

776 

777 return a, c, noise, gain 

778 

779 def funcFullCovarianceModel(self, params, x): 

780 """Model to fit covariances from flat fields; Equation 20 of 

781 Astier+19. 

782 

783 Parameters 

784 ---------- 

785 params : `list` 

786 Parameters of the model: aMatrix, CMatrix, noiseMatrix, 

787 gain (e/ADU). 

788 x : `numpy.array`, (N,) 

789 Signal `mu` (ADU) 

790 

791 Returns 

792 ------- 

793 y : `numpy.array`, (N,) 

794 Covariance matrix. 

795 """ 

796 matrixSideFit = self.config.maximumRangeCovariancesAstierFullCovFit 

797 lenParams = matrixSideFit*matrixSideFit 

798 aMatrix = params[:lenParams].reshape((matrixSideFit, matrixSideFit)) 

799 cMatrix = params[lenParams:2*lenParams].reshape((matrixSideFit, matrixSideFit)) 

800 noiseMatrix = params[2*lenParams:3*lenParams].reshape((matrixSideFit, matrixSideFit)) 

801 gain = params[-1] 

802 

803 return self.evalCovModel(x, aMatrix, cMatrix, noiseMatrix, gain).flatten() 

804 

805 def funcFullCovarianceModelNoB(self, params, x): 

806 """Model to fit covariances from flat fields; Equation 20 of 

807 Astier+19, with b=0 (equivalent to c=a*b=0 in this code). 

808 

809 Parameters 

810 ---------- 

811 params : `list` 

812 Parameters of the model: aMatrix, CMatrix, noiseMatrix, 

813 gain (e/ADU). 

814 x : `numpy.array`, (N,) 

815 Signal mu (ADU) 

816 

817 Returns 

818 ------- 

819 y : `numpy.array`, (N,) 

820 Covariance matrix. 

821 """ 

822 matrixSideFit = self.config.maximumRangeCovariancesAstierFullCovFit 

823 lenParams = matrixSideFit*matrixSideFit 

824 aMatrix = params[:lenParams].reshape((matrixSideFit, matrixSideFit)) 

825 cMatrix = params[lenParams:2*lenParams].reshape((matrixSideFit, matrixSideFit)) 

826 noiseMatrix = params[2*lenParams:3*lenParams].reshape((matrixSideFit, matrixSideFit)) 

827 gain = params[-1] 

828 

829 return self.evalCovModel(x, aMatrix, cMatrix, noiseMatrix, gain, setBtoZero=True).flatten() 

830 

831 def evalCovModel(self, mu, aMatrix, cMatrix, noiseMatrix, gain, setBtoZero=False): 

832 """Computes full covariances model (Eq. 20 of Astier+19). 

833 

834 Parameters 

835 ---------- 

836 mu : `numpy.array`, (N,) 

837 List of mean signals. 

838 aMatrix : `numpy.array`, (M, M) 

839 "a" parameter per flux in Eq. 20 of Astier+19. 

840 cMatrix : `numpy.array`, (M, M) 

841 "c"="ab" parameter per flux in Eq. 20 of Astier+19. 

842 noiseMatrix : `numpy.array`, (M, M) 

843 "noise" parameter per flux in Eq. 20 of Astier+19. 

844 gain : `float` 

845 Amplifier gain (e/ADU) 

846 setBtoZero=False : `bool`, optional 

847 Set "b" parameter in full model (see Astier+19) to zero. 

848 

849 Returns 

850 ------- 

851 covModel : `numpy.array`, (N, M, M) 

852 Covariances model. 

853 

854 Notes 

855 ----- 

856 By default, computes the covModel for the mu's stored(self.mu). 

857 Returns cov[Nmu, M, M]. The variance for the PTC is 

858 cov[:, 0, 0]. mu and cov are in ADUs and ADUs squared. To use 

859 electrons for both, the gain should be set to 1. This routine 

860 implements the model in Astier+19 (1905.08677). 

861 The parameters of the full model for C_ij(mu) ("C_ij" and "mu" 

862 in ADU^2 and ADU, respectively) in Astier+19 (Eq. 20) are: 

863 

864 - "a" coefficients (M by M matrix), units: 1/e 

865 - "b" coefficients (M by M matrix), units: 1/e 

866 - noise matrix (M by M matrix), units: e^2 

867 - gain, units: e/ADU 

868 

869 "b" appears in Eq. 20 only through the "ab" combination, which 

870 is defined in this code as "c=ab". 

871 """ 

872 matrixSideFit = self.config.maximumRangeCovariancesAstierFullCovFit 

873 sa = (matrixSideFit, matrixSideFit) 

874 # pad a with zeros and symmetrize 

875 aEnlarged = np.zeros((int(sa[0]*1.5)+1, int(sa[1]*1.5)+1)) 

876 aEnlarged[0:sa[0], 0:sa[1]] = aMatrix 

877 aSym = symmetrize(aEnlarged) 

878 # pad c with zeros and symmetrize 

879 cEnlarged = np.zeros((int(sa[0]*1.5)+1, int(sa[1]*1.5)+1)) 

880 cEnlarged[0:sa[0], 0:sa[1]] = cMatrix 

881 cSym = symmetrize(cEnlarged) 

882 a2 = fftconvolve(aSym, aSym, mode='same') 

883 a3 = fftconvolve(a2, aSym, mode='same') 

884 ac = fftconvolve(aSym, cSym, mode='same') 

885 (xc, yc) = np.unravel_index(np.abs(aSym).argmax(), a2.shape) 

886 

887 a1 = aMatrix[np.newaxis, :, :] 

888 a2 = a2[np.newaxis, xc:xc + matrixSideFit, yc:yc + matrixSideFit] 

889 a3 = a3[np.newaxis, xc:xc + matrixSideFit, yc:yc + matrixSideFit] 

890 ac = ac[np.newaxis, xc:xc + matrixSideFit, yc:yc + matrixSideFit] 

891 c1 = cMatrix[np.newaxis, ::] 

892 

893 # assumes that mu is 1d 

894 bigMu = mu[:, np.newaxis, np.newaxis]*gain 

895 # c(=a*b in Astier+19) also has a contribution to the last 

896 # term, that is absent for now. 

897 if setBtoZero: 

898 c1 = np.zeros_like(c1) 

899 ac = np.zeros_like(ac) 

900 covModel = (bigMu/(gain*gain)*(a1*bigMu+2./3.*(bigMu*bigMu)*(a2 + c1) 

901 + (1./3.*a3 + 5./6.*ac)*(bigMu*bigMu*bigMu)) + noiseMatrix[np.newaxis, :, :]/gain**2) 

902 # add the Poisson term, and the read out noise (variance) 

903 covModel[:, 0, 0] += mu/gain 

904 

905 return covModel 

906 

907 def subtractDistantOffset(self, muAtAmpMasked, covAtAmpMasked, covSqrtWeightsAtAmpMasked, 

908 start, degree=1): 

909 """Subtract distant offset from the covariance matrices. 

910 

911 Parameters 

912 ---------- 

913 muAtAmpMasked : `numpy.array` 

914 Masked mean flux array for a particular amplifier. 

915 covAtAmpMasked : `numpy.array` 

916 Masked measured covariances for a particular amplifier. 

917 covSqrtWeightsAtAmpMasked : `numpy.array` 

918 Masked inverse covariance weights for a particular amplifier. 

919 start : int, optional 

920 The starting index to eliminate the core for the fit. 

921 degree : int, optional 

922 Degree of the polynomial fit. 

923 

924 Returns 

925 ------- 

926 covAtAmpMasked : `numpy.array` 

927 Subtracted measured covariances for a particular amplifier. 

928 covSqrtWeightsAtAmpMasked : `numpy.array` 

929 Masked inverse covariance weights for a particular amplifier. 

930 

931 Notes 

932 ----- 

933 Ported from https://gitlab.in2p3.fr/astier/bfptc by P. Astier. 

934 

935 This function subtracts a distant offset from the 

936 covariance matrices using polynomial fitting. The core 

937 of the matrices is eliminated for the fit. 

938 

939 The function modifies the internal state of the object, updating the 

940 covariance matrices and related attributes. 

941 """ 

942 for k in range(len(muAtAmpMasked)): 

943 # Make a copy because it will be altered 

944 w = np.copy(covSqrtWeightsAtAmpMasked[k, ...]) 

945 wShape = w.shape 

946 i, j = np.meshgrid(range(wShape[0]), range(wShape[1]), indexing='ij') 

947 

948 # Eliminate the core for the fit 

949 w[:start, :start] = 0 

950 

951 poly = Pol2D(i, j, covAtAmpMasked[k, ...], degree, w=w) 

952 back = poly.eval(i, j) 

953 

954 covAtAmpMasked[k, ...] -= back 

955 

956 return covAtAmpMasked, covSqrtWeightsAtAmpMasked 

957 

958 # EXPAPPROXIMATION and POLYNOMIAL fit methods 

959 @staticmethod 

960 def _initialParsForPolynomial(order): 

961 assert order >= 2 

962 pars = np.zeros(order, dtype=float) 

963 pars[0] = 10 

964 pars[1] = 1 

965 pars[2:] = 0.0001 

966 return pars 

967 

968 @staticmethod 

969 def _boundsForPolynomial(initialPars, lowers=[], uppers=[]): 

970 if not len(lowers): 

971 lowers = [np.NINF for p in initialPars] 

972 if not len(uppers): 

973 uppers = [np.inf for p in initialPars] 

974 lowers[1] = 0 # no negative gains 

975 return (lowers, uppers) 

976 

977 @staticmethod 

978 def _boundsForAstier(initialPars, lowers=[], uppers=[]): 

979 if not len(lowers): 

980 lowers = [np.NINF for p in initialPars] 

981 if not len(uppers): 

982 uppers = [np.inf for p in initialPars] 

983 return (lowers, uppers) 

984 

985 @staticmethod 

986 def _getInitialGoodPoints(means, variances, minVarPivotSearch, consecutivePointsVarDecreases): 

987 """Return a boolean array to mask bad points. 

988 

989 Parameters 

990 ---------- 

991 means : `numpy.array` 

992 Input array with mean signal values. 

993 variances : `numpy.array` 

994 Input array with variances at each mean value. 

995 minVarPivotSearch : `float` 

996 The variance (in ADU^2), above which, the point 

997 of decreasing variance should be sought. 

998 consecutivePointsVarDecreases : `int` 

999 Required number of consecutive points/fluxes 

1000 in the PTC where the variance 

1001 decreases in order to find a first 

1002 estimate of the PTC turn-off. 

1003 

1004 Returns 

1005 ------ 

1006 goodPoints : `numpy.array` [`bool`] 

1007 Boolean array to select good (`True`) and bad (`False`) 

1008 points. 

1009 

1010 Notes 

1011 ----- 

1012 Eliminate points beyond which the variance decreases. 

1013 """ 

1014 goodPoints = np.ones_like(means, dtype=bool) 

1015 # Variances are sorted and should monotonically increase 

1016 pivotList = np.where(np.array(np.diff(variances)) < 0)[0] 

1017 if len(pivotList) > 0: 

1018 # For small values, sometimes the variance decreases slightly 

1019 # Only look when var > self.config.minVarPivotSearch 

1020 pivotList = [p for p in pivotList if variances[p] > minVarPivotSearch] 

1021 # Require that the varince decreases during 

1022 # consecutivePointsVarDecreases 

1023 # consecutive points. This will give a first 

1024 # estimate of the PTC turn-off, which 

1025 # may be updated (reduced) further in the code. 

1026 if len(pivotList) > 1: 

1027 # enumerate(pivotList) creates tuples (index, value), for 

1028 # each value in pivotList. The lambda function subtracts 

1029 # each value from the index. 

1030 # groupby groups elements by equal key value. 

1031 for k, g in groupby(enumerate(pivotList), lambda x: x[0]-x[1]): 

1032 group = (map(itemgetter(1), g)) 

1033 # Form groups of consecute values from pivotList 

1034 group = list(map(int, group)) 

1035 # values in pivotList are indices where np.diff(variances) 

1036 # is negative, i.e., where the variance starts decreasing. 

1037 # Find the first group of consecutive numbers when 

1038 # variance decreases. 

1039 if len(group) >= consecutivePointsVarDecreases: 

1040 pivotIndex = np.min(group) 

1041 goodPoints[pivotIndex+1:] = False 

1042 break 

1043 

1044 # Finally, we filter out any infinities or NaNs. 

1045 goodPoints[(~np.isfinite(means)) | (~np.isfinite(variances))] = False 

1046 

1047 return goodPoints 

1048 

1049 def _makeZeroSafe(self, array, substituteValue=1e-9): 

1050 """""" 

1051 array = np.array(array) 

1052 nBad = Counter(np.ravel(array))[0] 

1053 if nBad == 0: 

1054 return array 

1055 

1056 index, = np.where(array == 0) 

1057 if len(index): 

1058 msg = f"Found {nBad} zeros in array at elements {index}" 

1059 self.log.warning(msg) 

1060 

1061 array[index] = substituteValue 

1062 

1063 return array 

1064 

1065 def fitPtc(self, dataset): 

1066 """Fit the photon transfer curve to a polynomial or to the 

1067 Astier+19 approximation (Eq. 16). 

1068 

1069 Fit the photon transfer curve with either a polynomial of 

1070 the order specified in the task config (POLNOMIAL), 

1071 or using the exponential approximation in Astier+19 

1072 (Eq. 16, FULLCOVARIANCE). 

1073 

1074 Sigma clipping is performed iteratively for the fit, as 

1075 well as an initial clipping of data points that are more 

1076 than `config.initialNonLinearityExclusionThreshold` away 

1077 from lying on a straight line. This other step is necessary 

1078 because the photon transfer curve turns over catastrophically 

1079 at very high flux (because saturation 

1080 drops the variance to ~0) and these far outliers cause the 

1081 initial fit to fail, meaning the sigma cannot be calculated 

1082 to perform the sigma-clipping. 

1083 

1084 Parameters 

1085 ---------- 

1086 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset` 

1087 The dataset containing the means, variances and 

1088 exposure times. 

1089 

1090 Returns 

1091 ------- 

1092 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset` 

1093 This is the same dataset as the input parameter, however, 

1094 it has been modified to include information such as the 

1095 fit vectors and the fit parameters. See the class 

1096 `PhotonTransferCurveDatase`. 

1097 

1098 Raises 

1099 ------ 

1100 RuntimeError 

1101 Raised if dataset.ptcFitType is None or empty. 

1102 """ 

1103 if dataset.ptcFitType: 

1104 ptcFitType = dataset.ptcFitType 

1105 else: 

1106 raise RuntimeError("ptcFitType is None of empty in PTC dataset.") 

1107 # For FULLCOVARIANCE model fit 

1108 matrixSideFit = dataset.covMatrixSideFullCovFit 

1109 nanMatrixFit = np.empty((matrixSideFit, matrixSideFit)) 

1110 nanMatrixFit[:] = np.nan 

1111 

1112 for amp in dataset.ampNames: 

1113 lenInputTimes = len(dataset.rawExpTimes[amp]) 

1114 listNanMatrixFit = np.empty((lenInputTimes, matrixSideFit, matrixSideFit)) 

1115 listNanMatrixFit[:] = np.nan 

1116 

1117 dataset.covariancesModel[amp] = listNanMatrixFit 

1118 dataset.aMatrix[amp] = nanMatrixFit 

1119 dataset.bMatrix[amp] = nanMatrixFit 

1120 dataset.covariancesModelNoB[amp] = listNanMatrixFit 

1121 dataset.aMatrixNoB[amp] = nanMatrixFit 

1122 dataset.noiseMatrix[amp] = nanMatrixFit 

1123 dataset.noiseMatrixNoB[amp] = nanMatrixFit 

1124 

1125 def errFunc(p, x, y): 

1126 return ptcFunc(p, x) - y 

1127 

1128 sigmaCutPtcOutliers = self.config.sigmaCutPtcOutliers 

1129 maxIterationsPtcOutliers = self.config.maxIterationsPtcOutliers 

1130 

1131 for i, ampName in enumerate(dataset.ampNames): 

1132 meanVecOriginal = dataset.rawMeans[ampName].copy() 

1133 varVecOriginal = dataset.rawVars[ampName].copy() 

1134 varVecOriginal = self._makeZeroSafe(varVecOriginal) 

1135 

1136 # These must be sorted for the given amplifier. 

1137 meanVecSort = np.argsort(meanVecOriginal) 

1138 meanVecSorted = meanVecOriginal[meanVecSort] 

1139 varVecSorted = varVecOriginal[meanVecSort] 

1140 

1141 if self.config.doLegacyTurnoffSelection: 

1142 # Discard points when the variance starts to decrease after two 

1143 # consecutive signal levels 

1144 goodPoints = self._getInitialGoodPoints(meanVecSorted, varVecSorted, 

1145 self.config.minVarPivotSearch, 

1146 self.config.consecutivePointsVarDecreases) 

1147 else: 

1148 # Make sure we have this properly sorted. 

1149 goodPoints = dataset.expIdMask[ampName].copy() 

1150 goodPoints = goodPoints[meanVecSort] 

1151 

1152 # Check if all points are bad from the 'cpExtractPtcTask' 

1153 initialExpIdMask = dataset.expIdMask[ampName].copy() 

1154 

1155 if not (goodPoints.any() and initialExpIdMask.any()): 

1156 msg = (f"SERIOUS: All points in goodPoints: {goodPoints} or " 

1157 f"in initialExpIdMask: {initialExpIdMask} are bad." 

1158 f"Setting {ampName} to BAD.") 

1159 self.log.warning(msg) 

1160 # Fill entries with NaNs 

1161 self.fillBadAmp(dataset, ptcFitType, ampName) 

1162 continue 

1163 

1164 mask = goodPoints.copy() 

1165 

1166 if ptcFitType == 'EXPAPPROXIMATION': 

1167 ptcFunc = funcAstier 

1168 parsIniPtc = [-1e-9, 1.0, 10.] # a00, gain, noise^2 

1169 # lowers and uppers obtained from BOT data studies by 

1170 # C. Lage (UC Davis, 11/2020). 

1171 if self.config.binSize > 1: 

1172 bounds = self._boundsForAstier(parsIniPtc) 

1173 else: 

1174 bounds = self._boundsForAstier(parsIniPtc, lowers=[-1e-4, 0.1, -2000], 

1175 uppers=[1e-4, 10.0, 2000]) 

1176 if ptcFitType == 'POLYNOMIAL': 

1177 ptcFunc = funcPolynomial 

1178 parsIniPtc = self._initialParsForPolynomial(self.config.polynomialFitDegree + 1) 

1179 bounds = self._boundsForPolynomial(parsIniPtc) 

1180 

1181 # We perform an initial (unweighted) fit of variance vs signal 

1182 # (after initial KS test or post-drop selection) to look for 

1183 # outliers, particularly at the high-flux end. The initial fit 

1184 # is performed only for points that are guaranteed to be below 

1185 # the PTC turnoff and then extrapolated to ensure that high 

1186 # flux points that have abnormal variance values can be properly 

1187 # rejected in this phase without biasing the initial fit. 

1188 # This algorithm was initially developed by Seth Digel for 

1189 # the EO Testing pipeline. 

1190 

1191 if self.config.scaleMaxSignalInitialPtcOutlierFit: 

1192 approxGain = np.nanmedian(meanVecSorted/varVecSorted) 

1193 maxADUInitialPtcOutlierFit = self.config.maxSignalInitialPtcOutlierFit/approxGain 

1194 maxDeltaADUInitialPtcOutlierFit = self.config.maxDeltaInitialPtcOutlierFit/approxGain 

1195 self.log.info( 

1196 "Using approximate gain %.3f and ADU signal cutoff of %.1f and delta %.1f " 

1197 "for amplifier %s", 

1198 approxGain, 

1199 maxADUInitialPtcOutlierFit, 

1200 maxDeltaADUInitialPtcOutlierFit, 

1201 ampName, 

1202 ) 

1203 else: 

1204 maxADUInitialPtcOutlierFit = self.config.maxSignalInitialPtcOutlierFit 

1205 maxDeltaADUInitialPtcOutlierFit = self.config.maxDeltaInitialPtcOutlierFit 

1206 

1207 if maxIterationsPtcOutliers == 0: 

1208 # We are not doing any outlier rejection here, but we do want 

1209 # an initial fit. 

1210 res = least_squares( 

1211 errFunc, 

1212 parsIniPtc, 

1213 bounds=bounds, 

1214 args=(meanVecSorted[mask], varVecSorted[mask]), 

1215 ) 

1216 pars = res.x 

1217 newMask = mask.copy() 

1218 else: 

1219 newMask = (mask & (meanVecSorted <= maxADUInitialPtcOutlierFit)) 

1220 

1221 converged = False 

1222 count = 0 

1223 lastMask = mask.copy() 

1224 while count < maxIterationsPtcOutliers: 

1225 res = least_squares( 

1226 errFunc, 

1227 parsIniPtc, 

1228 bounds=bounds, 

1229 args=(meanVecSorted[newMask], varVecSorted[newMask]), 

1230 ) 

1231 pars = res.x 

1232 

1233 sigResids = (varVecSorted - ptcFunc(pars, meanVecSorted))/np.sqrt(varVecSorted) 

1234 # The new mask includes points where the residuals are 

1235 # finite, are less than the cut, and include the original 

1236 # mask of known points that should not be used. 

1237 newMask = ( 

1238 np.isfinite(sigResids) 

1239 & (np.abs(np.nan_to_num(sigResids)) < sigmaCutPtcOutliers) 

1240 & mask 

1241 ) 

1242 # Demand at least 2 points to continue. 

1243 if np.count_nonzero(newMask) < 2: 

1244 msg = (f"SERIOUS: All points after outlier rejection are bad. " 

1245 f"Setting {ampName} to BAD.") 

1246 self.log.warning(msg) 

1247 # Fill entries with NaNs 

1248 self.fillBadAmp(dataset, ptcFitType, ampName) 

1249 break 

1250 

1251 self.log.debug( 

1252 "Iteration %d: Removed %d points in total for %s.", 

1253 count, 

1254 np.count_nonzero(mask) - np.count_nonzero(newMask), 

1255 ampName, 

1256 ) 

1257 

1258 # Loop over all used (True) points. If one of them follows 

1259 # a False point, then it must be within 

1260 # maxDeltaADUInitialPtcOutlierFit of a True point. If it 

1261 # is a large gap, everything above is marked False. 

1262 useMask, = np.where(newMask) 

1263 for useIndex, usePoint in enumerate(useMask): 

1264 if useIndex == 0 or newMask[usePoint - 1]: 

1265 # The previous point was good; continue. 

1266 continue 

1267 deltaADU = meanVecSorted[usePoint] - meanVecSorted[useMask[useIndex - 1]] 

1268 if deltaADU < maxDeltaADUInitialPtcOutlierFit: 

1269 # This jump is fine; continue. 

1270 continue 

1271 

1272 # Mark all further points bad. 

1273 newMask[usePoint:] = False 

1274 break 

1275 

1276 # If the mask hasn't changed then break out. 

1277 if np.all(newMask == lastMask): 

1278 self.log.debug("Convergence at iteration %d; breaking loop for %s.", count, ampName) 

1279 converged = True 

1280 break 

1281 

1282 lastMask = newMask.copy() 

1283 

1284 count += 1 

1285 

1286 if not converged: 

1287 self.log.warning( 

1288 "Outlier detection was not converged prior to %d iteration for %s", 

1289 count, 

1290 ampName 

1291 ) 

1292 

1293 # Set the mask to the new mask, and reset the sorting. 

1294 mask = np.zeros(len(meanVecSort), dtype=np.bool_) 

1295 mask[meanVecSort[newMask]] = True 

1296 maskSorted = newMask.copy() 

1297 

1298 if not mask.any(): 

1299 # We hae already filled the bad amp above, so continue. 

1300 continue 

1301 

1302 dataset.expIdMask[ampName] = mask 

1303 

1304 parsIniPtc = pars 

1305 meanVecFinal = meanVecOriginal[mask] 

1306 varVecFinal = varVecOriginal[mask] 

1307 

1308 # Save the maximum point after outlier detection as the 

1309 # PTC turnoff point. We need to pay attention to the sorting 

1310 # here. 

1311 dataset.ptcTurnoff[ampName] = np.max(meanVecFinal) 

1312 # And compute the ptcTurnoffSamplingError as one half the 

1313 # difference between the previous and next point. 

1314 lastGoodIndex = np.where(maskSorted)[0][-1] 

1315 ptcTurnoffLow = meanVecSorted[lastGoodIndex - 1] 

1316 if lastGoodIndex == (len(meanVecSorted) - 1): 

1317 # If it's the last index, just use the interval. 

1318 ptcTurnoffSamplingError = dataset.ptcTurnoff[ampName] - ptcTurnoffLow 

1319 elif not np.isfinite(meanVecSorted[lastGoodIndex + 1]): 

1320 # If the next index is not finite, just use the interval. 

1321 ptcTurnoffSamplingError = dataset.ptcTurnoff[ampName] - ptcTurnoffLow 

1322 else: 

1323 ptcTurnoffSamplingError = (meanVecSorted[lastGoodIndex + 1] - ptcTurnoffLow)/2. 

1324 dataset.ptcTurnoffSamplingError[ampName] = ptcTurnoffSamplingError 

1325 

1326 if Counter(mask)[False] > 0: 

1327 self.log.info("Number of points discarded in PTC of amplifier %s:" 

1328 " %d out of %d", ampName, Counter(mask)[False], len(meanVecOriginal)) 

1329 

1330 if (len(meanVecFinal) < len(parsIniPtc)): 

1331 msg = (f"SERIOUS: Not enough data points ({len(meanVecFinal)}) compared to the number of " 

1332 f"parameters of the PTC model({len(parsIniPtc)}). Setting {ampName} to BAD.") 

1333 self.log.warning(msg) 

1334 # Fill entries with NaNs 

1335 self.fillBadAmp(dataset, ptcFitType, ampName) 

1336 continue 

1337 # Fit the PTC. 

1338 # The variance of the variance is Var(v)=2*v^2/Npix. This is 

1339 # already calculated in `makeCovArray` of CpPtcExtract. 

1340 # dataset.covariancesSqrtWeights[ampName][:,0,0] 

1341 # has 1/sqrt(Var(v)). 

1342 weightsY = dataset.covariancesSqrtWeights[ampName][:, 0, 0][mask] 

1343 if self.config.doFitBootstrap: 

1344 parsFit, parsFitErr, reducedChiSqPtc = fitBootstrap(parsIniPtc, meanVecFinal, 

1345 varVecFinal, ptcFunc, 

1346 weightsY=weightsY) 

1347 else: 

1348 parsFit, parsFitErr, reducedChiSqPtc = fitLeastSq(parsIniPtc, meanVecFinal, 

1349 varVecFinal, ptcFunc, 

1350 weightsY=weightsY) 

1351 dataset.ptcFitPars[ampName] = parsFit 

1352 dataset.ptcFitParsError[ampName] = parsFitErr 

1353 dataset.ptcFitChiSq[ampName] = reducedChiSqPtc 

1354 

1355 dataset.finalVars[ampName] = varVecOriginal 

1356 dataset.finalVars[ampName][~mask] = np.nan 

1357 dataset.finalModelVars[ampName] = ptcFunc(parsFit, meanVecOriginal) 

1358 dataset.finalModelVars[ampName][~mask] = np.nan 

1359 dataset.finalMeans[ampName] = meanVecOriginal 

1360 dataset.finalMeans[ampName][~mask] = np.nan 

1361 

1362 if ptcFitType == 'EXPAPPROXIMATION': 

1363 ptcGain = parsFit[1] 

1364 ptcGainErr = parsFitErr[1] 

1365 ptcNoise = np.sqrt(np.fabs(parsFit[2])) 

1366 ptcNoiseErr = 0.5*(parsFitErr[2]/np.fabs(parsFit[2]))*np.sqrt(np.fabs(parsFit[2])) 

1367 if ptcFitType == 'POLYNOMIAL': 

1368 ptcGain = 1./parsFit[1] 

1369 ptcGainErr = np.fabs(1./parsFit[1])*(parsFitErr[1]/parsFit[1]) 

1370 ptcNoise = np.sqrt(np.fabs(parsFit[0]))*ptcGain 

1371 ptcNoiseErr = (0.5*(parsFitErr[0]/np.fabs(parsFit[0]))*(np.sqrt(np.fabs(parsFit[0]))))*ptcGain 

1372 dataset.gain[ampName] = ptcGain 

1373 dataset.gainErr[ampName] = ptcGainErr 

1374 dataset.noise[ampName] = ptcNoise 

1375 dataset.noiseErr[ampName] = ptcNoiseErr 

1376 

1377 if not len(dataset.ptcFitType) == 0: 

1378 dataset.ptcFitType = ptcFitType 

1379 if len(dataset.badAmps) == 0: 

1380 dataset.badAmps = [] 

1381 

1382 return dataset 

1383 

1384 def fillBadAmp(self, dataset, ptcFitType, ampName): 

1385 """Fill the dataset with NaNs if there are not enough 

1386 good points. 

1387 

1388 Parameters 

1389 ---------- 

1390 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset` 

1391 The dataset containing the means, variances and 

1392 exposure times. 

1393 ptcFitType : {'POLYNOMIAL', 'EXPAPPROXIMATION'} 

1394 Fit a 'POLYNOMIAL' (degree: 'polynomialFitDegree') or 

1395 'EXPAPPROXIMATION' (Eq. 16 of Astier+19) to the PTC. 

1396 ampName : `str` 

1397 Amplifier name. 

1398 """ 

1399 dataset.badAmps.append(ampName) 

1400 dataset.expIdMask[ampName] = np.repeat(False, len(dataset.rawExpTimes[ampName])) 

1401 dataset.gain[ampName] = np.nan 

1402 dataset.gainErr[ampName] = np.nan 

1403 dataset.noise[ampName] = np.nan 

1404 dataset.noiseErr[ampName] = np.nan 

1405 dataset.ptcFitPars[ampName] = (np.repeat(np.nan, self.config.polynomialFitDegree + 1) if 

1406 ptcFitType in ["POLYNOMIAL", ] else np.repeat(np.nan, 3)) 

1407 dataset.ptcFitParsError[ampName] = (np.repeat(np.nan, self.config.polynomialFitDegree + 1) if 

1408 ptcFitType in ["POLYNOMIAL", ] else np.repeat(np.nan, 3)) 

1409 dataset.ptcFitChiSq[ampName] = np.nan 

1410 dataset.ptcTurnoff[ampName] = np.nan 

1411 dataset.finalVars[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName])) 

1412 dataset.finalModelVars[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName])) 

1413 dataset.finalMeans[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName])) 

1414 

1415 return