Coverage for python/lsst/cp/pipe/ptc/cpSolvePtcTask.py: 11%

491 statements  

« prev     ^ index     » next       coverage.py v7.4.0, created at 2024-01-20 12:45 +0000

1# This file is part of cp_pipe. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21# 

22import numpy as np 

23from collections import Counter 

24 

25import lsst.pex.config as pexConfig 

26import lsst.pipe.base as pipeBase 

27from lsst.cp.pipe.utils import (fitLeastSq, fitBootstrap, funcPolynomial, 

28 funcAstier, symmetrize, Pol2D) 

29 

30from scipy.signal import fftconvolve 

31from scipy.optimize import least_squares 

32from itertools import groupby 

33from operator import itemgetter 

34 

35import lsst.pipe.base.connectionTypes as cT 

36 

37from lsst.ip.isr import PhotonTransferCurveDataset 

38 

39import copy 

40 

41 

42__all__ = ['PhotonTransferCurveSolveConfig', 'PhotonTransferCurveSolveTask'] 

43 

44 

45class PhotonTransferCurveSolveConnections(pipeBase.PipelineTaskConnections, 

46 dimensions=("instrument", "detector")): 

47 inputCovariances = cT.Input( 

48 name="ptcCovariances", 

49 doc="Tuple with measured covariances from flats.", 

50 storageClass="PhotonTransferCurveDataset", 

51 dimensions=("instrument", "exposure", "detector"), 

52 isCalibration=True, 

53 multiple=True, 

54 ) 

55 camera = cT.PrerequisiteInput( 

56 name="camera", 

57 doc="Camera the input data comes from.", 

58 storageClass="Camera", 

59 dimensions=("instrument",), 

60 isCalibration=True, 

61 ) 

62 outputPtcDataset = cT.Output( 

63 name="ptcDatsetProposal", 

64 doc="Output proposed ptc dataset.", 

65 storageClass="PhotonTransferCurveDataset", 

66 dimensions=("instrument", "detector"), 

67 multiple=False, 

68 isCalibration=True, 

69 ) 

70 

71 

72class PhotonTransferCurveSolveConfig(pipeBase.PipelineTaskConfig, 

73 pipelineConnections=PhotonTransferCurveSolveConnections): 

74 """Configuration for fitting measured covariances. 

75 """ 

76 

77 ptcFitType = pexConfig.ChoiceField( 

78 dtype=str, 

79 doc="Fit PTC to Eq. 16, Eq. 20 in Astier+19, or to a polynomial.", 

80 default="POLYNOMIAL", 

81 allowed={ 

82 "POLYNOMIAL": "n-degree polynomial (use 'polynomialFitDegree' to set 'n').", 

83 "EXPAPPROXIMATION": "Approximation in Astier+19 (Eq. 16).", 

84 "FULLCOVARIANCE": "Full covariances model in Astier+19 (Eq. 20)" 

85 } 

86 ) 

87 minMeanSignal = pexConfig.DictField( 

88 keytype=str, 

89 itemtype=float, 

90 doc="Minimum values (inclusive) of mean signal (in ADU) per amp to use." 

91 " The same cut is applied to all amps if this parameter [`dict`] is passed as " 

92 " {'ALL_AMPS': value}", 

93 default={'ALL_AMPS': 0.0}, 

94 ) 

95 maxMeanSignal = pexConfig.DictField( 

96 keytype=str, 

97 itemtype=float, 

98 doc="Maximum values (inclusive) of mean signal (in ADU) below which to consider, per amp." 

99 " The same cut is applied to all amps if this dictionary is of the form" 

100 " {'ALL_AMPS': value}", 

101 default={'ALL_AMPS': 1e6}, 

102 ) 

103 maximumRangeCovariancesAstier = pexConfig.Field( 

104 dtype=int, 

105 doc="Maximum range of measured covariances as in Astier+19", 

106 default=8, 

107 ) 

108 maximumRangeCovariancesAstierFullCovFit = pexConfig.Field( 

109 dtype=int, 

110 doc="Maximum range up to where to fit covariances as in Astier+19, " 

111 "for the FULLCOVARIANCE model." 

112 "This is different from maximumRangeCovariancesAstier." 

113 "It should be less or equal than maximumRangeCovariancesAstier." 

114 "The number of parameters for this model is " 

115 "3*maximumRangeCovariancesAstierFullCovFit^2 + 1, so increase with care " 

116 "so that the fit is not too slow.", 

117 default=8, 

118 ) 

119 doSubtractLongRangeCovariances = pexConfig.Field( 

120 dtype=bool, 

121 doc="Subtract long-range covariances before FULLCOVARIANCE fit, " 

122 "beyond startLongRangeCovariances?", 

123 default=False, 

124 ) 

125 startLongRangeCovariances = pexConfig.Field( 

126 dtype=int, 

127 doc="If doSubtractLongRangeCovariances is True, subtract covariances " 

128 "beyond this range. It should be less than maximumRangeCovariancesAstier. ", 

129 default=4, 

130 ) 

131 polyDegLongRangeCovariances = pexConfig.Field( 

132 dtype=int, 

133 doc="If doSubtractLongRangeCovariances is True, polynomial " 

134 "degree to fit data beyond startLongRangeCovariances.", 

135 default=1, 

136 ) 

137 sigmaClipFullFitCovariancesAstier = pexConfig.Field( 

138 dtype=float, 

139 doc="sigma clip for full model fit for FULLCOVARIANCE ptcFitType ", 

140 default=5.0, 

141 ) 

142 maxIterFullFitCovariancesAstier = pexConfig.Field( 

143 dtype=int, 

144 doc="Maximum number of iterations in full model fit for FULLCOVARIANCE ptcFitType", 

145 default=3, 

146 ) 

147 polynomialFitDegree = pexConfig.Field( 

148 dtype=int, 

149 doc="Degree of polynomial to fit the PTC, when 'ptcFitType'=POLYNOMIAL.", 

150 default=3, 

151 ) 

152 doLegacyTurnoffSelection = pexConfig.Field( 

153 dtype=bool, 

154 doc="Use 'legacy' computation for PTC turnoff selection. If set " 

155 "to False, then the KS test p-value selection will be used instead.", 

156 default=False, 

157 ) 

158 sigmaCutPtcOutliers = pexConfig.Field( 

159 dtype=float, 

160 doc="Sigma cut for outlier rejection in PTC.", 

161 default=5.0, 

162 ) 

163 maxIterationsPtcOutliers = pexConfig.RangeField( 

164 dtype=int, 

165 doc="Maximum number of iterations for outlier rejection in PTC.", 

166 default=2, 

167 min=0 

168 ) 

169 maxSignalInitialPtcOutlierFit = pexConfig.Field( 

170 dtype=float, 

171 doc="Maximum signal considered for intial outlier fit. This should be below " 

172 "the PTC turnoff to ensure accurate outlier rejection. If " 

173 "scaleMaxSignalInitialPtcOutlierFit=True then the units are electrons; " 

174 "otherwise ADU.", 

175 default=50_000., 

176 ) 

177 scaleMaxSignalInitialPtcOutlierFit = pexConfig.Field( 

178 dtype=bool, 

179 doc="Scale maxSignalInitialPtcOutlierFit by approximate gain? If yes then " 

180 "maxSignalInitialPtcOutlierFit is assumed to have units of electrons, " 

181 "otherwise ADU.", 

182 default=True, 

183 ) 

184 minVarPivotSearch = pexConfig.Field( 

185 dtype=float, 

186 doc="The code looks for a pivot signal point after which the variance starts decreasing at high-flux" 

187 " to exclude then from the PTC model fit. However, sometimes at low fluxes, the variance" 

188 " decreases slightly. Set this variable for the variance value, in ADU^2, after which the pivot " 

189 " should be sought. Only used if doLegacyTurnoffSelection is True.", 

190 default=10000, 

191 ) 

192 consecutivePointsVarDecreases = pexConfig.RangeField( 

193 dtype=int, 

194 doc="Required number of consecutive points/fluxes in the PTC where the variance " 

195 "decreases in order to find a first estimate of the PTC turn-off. " 

196 "Only used if doLegacyTurnoffSelection is True.", 

197 default=2, 

198 min=2 

199 ) 

200 ksTestMinPvalue = pexConfig.Field( 

201 dtype=float, 

202 doc="Minimum value of the Gaussian histogram KS test p-value to be used in PTC fit. " 

203 "Only used if doLegacyTurnoffSelection is False.", 

204 default=0.01, 

205 ) 

206 doFitBootstrap = pexConfig.Field( 

207 dtype=bool, 

208 doc="Use bootstrap for the PTC fit parameters and errors?.", 

209 default=False, 

210 ) 

211 binSize = pexConfig.Field( 

212 dtype=int, 

213 doc="Bin the image by this factor in both dimensions.", 

214 default=1, 

215 ) 

216 

217 def validate(self): 

218 super().validate() 

219 fitMatrixSide = self.maximumRangeCovariancesAstierFullCovFit 

220 measureMatrixSide = self.maximumRangeCovariancesAstier 

221 if self.ptcFitType == "FULLCOVARIANCE": 

222 if fitMatrixSide > measureMatrixSide: 

223 raise RuntimeError("Covariance fit size %s is larger than" 

224 "measurement size %s.", 

225 fitMatrixSide, measureMatrixSide) 

226 if self.doSubtractLongRangeCovariances: 

227 startLag = self.startLongRangeCovariances 

228 if measureMatrixSide < startLag: 

229 raise RuntimeError("Covariance measure size %s is smaller than long" 

230 "-range covariance starting point %s.", 

231 measureMatrixSide, startLag) 

232 

233 

234class PhotonTransferCurveSolveTask(pipeBase.PipelineTask): 

235 """Task to fit the PTC from flat covariances. 

236 

237 The first task of the PTC measurement pipeline, 

238 ``PhotonTransferCurveMeasureTask`` (and assumed to have been run 

239 before this task), produced a list of 

240 `~lsst.ip.isr.PhotonTransferCurveDataset` objects. Each dataset 

241 contains the mean signal and covariances of the 

242 difference image of the flat-field images taken at 

243 the same exposure time. The list also contains dummy 

244 datasets (with no measurements), whose purpose is to have 

245 the input and output dimensions of ``PhotonTransferCurveMeasureTask`` 

246 match. 

247 

248 This task, ``PhotonTransferCurveSolveTask``, assembles the list 

249 of individual PTC datasets produced 

250 by ``PhotonTransferCurveMeasureTask`` into one single final PTC 

251 dataset, discarding the dummy datset as appropiate. 

252 The task fits the measured (co)variances to one of three models: 

253 a polynomial model of a given order, or the models described 

254 in equations 16 and 20 of Astier+19. These options are referred 

255 to as ``POLYNOMIAL``, ``EXPAPPROXIMATION``, and ``FULLCOVARIANCE`` 

256 in the configuration options of the task, respectively). 

257 Parameters of interest such as the gain and noise are derived 

258 from the fits. The ``FULLCOVARIANCE`` model is fitted to the 

259 full covariance data (as oppossed to the other two models, which 

260 are fit to the variance vs mean measurements only). 

261 

262 Astier+19: "The Shape of the Photon Transfer Curve 

263 of CCD sensors", arXiv:1905.08677 

264 """ 

265 

266 ConfigClass = PhotonTransferCurveSolveConfig 

267 _DefaultName = 'cpPhotonTransferCurveSolve' 

268 

269 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

270 """Ensure that the input and output dimensions are passed along. 

271 

272 Parameters 

273 ---------- 

274 butlerQC : `~lsst.daf.butler.QuantumContext` 

275 Butler to operate on. 

276 inputRefs : `~lsst.pipe.base.InputQuantizedConnection` 

277 Input data refs to load. 

278 ouptutRefs : `~lsst.pipe.base.OutputQuantizedConnection` 

279 Output data refs to persist. 

280 """ 

281 inputs = butlerQC.get(inputRefs) 

282 detId = inputRefs.inputCovariances[0].dataId['detector'] 

283 outputs = self.run(inputCovariances=inputs['inputCovariances'], camera=inputs['camera'], detId=detId) 

284 butlerQC.put(outputs, outputRefs) 

285 

286 def run(self, inputCovariances, camera=None, detId=0): 

287 """Fit measured covariances to different models. 

288 

289 Parameters 

290 ---------- 

291 inputCovariances : `list` [`lsst.ip.isr.PhotonTransferCurveDataset`] 

292 List of lsst.ip.isr.PhotonTransferCurveDataset datasets. 

293 camera : `lsst.afw.cameraGeom.Camera`, optional 

294 Input camera. 

295 detId : `int` 

296 Detector ID to locate the detector in the camera and 

297 populate the `lsst.ip.isr.PhotonTransferCurveDataset` 

298 metadata. 

299 Returns 

300 ------- 

301 results : `lsst.pipe.base.Struct` 

302 The resultins structure contains: 

303 

304 ``outputPtcDatset`` 

305 Final PTC dataset, containing information such as the 

306 means, variances, and exposure times 

307 (`lsst.ip.isr.PhotonTransferCurveDataset`). 

308 """ 

309 # Find the ampNames from a non-dummy ptc. 

310 ampNames = [] 

311 for partialPtcDataset in inputCovariances: 

312 if partialPtcDataset.ptcFitType != 'DUMMY': 

313 ampNames = partialPtcDataset.ampNames 

314 break 

315 

316 # Each amp may have a different min and max ADU signal 

317 # specified in the config. 

318 maxMeanSignalDict = {ampName: 1e6 for ampName in ampNames} 

319 minMeanSignalDict = {ampName: 0.0 for ampName in ampNames} 

320 for ampName in ampNames: 

321 if 'ALL_AMPS' in self.config.maxMeanSignal: 

322 maxMeanSignalDict[ampName] = self.config.maxMeanSignal['ALL_AMPS'] 

323 elif ampName in self.config.maxMeanSignal: 

324 maxMeanSignalDict[ampName] = self.config.maxMeanSignal[ampName] 

325 

326 if 'ALL_AMPS' in self.config.minMeanSignal: 

327 minMeanSignalDict[ampName] = self.config.minMeanSignal['ALL_AMPS'] 

328 elif ampName in self.config.minMeanSignal: 

329 minMeanSignalDict[ampName] = self.config.minMeanSignal[ampName] 

330 

331 # Assemble individual PTC datasets into a single PTC dataset. 

332 datasetPtc = PhotonTransferCurveDataset( 

333 ampNames=ampNames, 

334 ptcFitType=self.config.ptcFitType, 

335 covMatrixSide=self.config.maximumRangeCovariancesAstier, 

336 covMatrixSideFullCovFit=self.config.maximumRangeCovariancesAstierFullCovFit) 

337 for partialPtcDataset in inputCovariances: 

338 # Ignore dummy datasets 

339 if partialPtcDataset.ptcFitType == 'DUMMY': 

340 continue 

341 for ampName in ampNames: 

342 # The partial dataset consists of lists of values for each 

343 # quantity. In the case of the input exposure pairs, this is a 

344 # list of tuples. In all cases we only want the first 

345 # (and only) element of the list. 

346 datasetPtc.inputExpIdPairs[ampName].append(partialPtcDataset.inputExpIdPairs[ampName][0]) 

347 datasetPtc.rawExpTimes[ampName] = np.append(datasetPtc.rawExpTimes[ampName], 

348 partialPtcDataset.rawExpTimes[ampName][0]) 

349 datasetPtc.rawMeans[ampName] = np.append(datasetPtc.rawMeans[ampName], 

350 partialPtcDataset.rawMeans[ampName][0]) 

351 datasetPtc.rawVars[ampName] = np.append(datasetPtc.rawVars[ampName], 

352 partialPtcDataset.rawVars[ampName][0]) 

353 datasetPtc.photoCharges[ampName] = np.append(datasetPtc.photoCharges[ampName], 

354 partialPtcDataset.photoCharges[ampName][0]) 

355 datasetPtc.histVars[ampName] = np.append(datasetPtc.histVars[ampName], 

356 partialPtcDataset.histVars[ampName][0]) 

357 datasetPtc.histChi2Dofs[ampName] = np.append(datasetPtc.histChi2Dofs[ampName], 

358 partialPtcDataset.histChi2Dofs[ampName][0]) 

359 datasetPtc.kspValues[ampName] = np.append(datasetPtc.kspValues[ampName], 

360 partialPtcDataset.kspValues[ampName][0]) 

361 datasetPtc.covariances[ampName] = np.append( 

362 datasetPtc.covariances[ampName].ravel(), 

363 partialPtcDataset.covariances[ampName].ravel() 

364 ).reshape( 

365 ( 

366 len(datasetPtc.rawExpTimes[ampName]), 

367 datasetPtc.covMatrixSide, 

368 datasetPtc.covMatrixSide, 

369 ) 

370 ) 

371 datasetPtc.covariancesSqrtWeights[ampName] = np.append( 

372 datasetPtc.covariancesSqrtWeights[ampName].ravel(), 

373 partialPtcDataset.covariancesSqrtWeights[ampName].ravel() 

374 ).reshape( 

375 ( 

376 len(datasetPtc.rawExpTimes[ampName]), 

377 datasetPtc.covMatrixSide, 

378 datasetPtc.covMatrixSide, 

379 ) 

380 ) 

381 

382 # Apply min/max masking. 

383 rawMean = partialPtcDataset.rawMeans[ampName][0] 

384 rawVar = partialPtcDataset.rawVars[ampName][0] 

385 expIdMask = partialPtcDataset.expIdMask[ampName][0] 

386 if (rawMean <= minMeanSignalDict[ampName]) or (rawMean >= maxMeanSignalDict[ampName]) \ 

387 or not np.isfinite(rawMean) or not np.isfinite(rawVar): 

388 expIdMask = False 

389 

390 kspValue = partialPtcDataset.kspValues[ampName][0] 

391 if not self.config.doLegacyTurnoffSelection and \ 

392 kspValue < self.config.ksTestMinPvalue: 

393 expIdMask = False 

394 

395 datasetPtc.expIdMask[ampName] = np.append(datasetPtc.expIdMask[ampName], expIdMask) 

396 

397 for key, value in partialPtcDataset.auxValues.items(): 

398 if key in datasetPtc.auxValues: 

399 datasetPtc.auxValues[key] = np.append(datasetPtc.auxValues[key], value) 

400 else: 

401 datasetPtc.auxValues[key] = value 

402 

403 # Sort arrays that are filled so far in the final dataset by 

404 # rawMeans index. 

405 # First compute the mean across all the amps to make sure that they are 

406 # all sorted the same way. 

407 detectorMeans = np.zeros(len(datasetPtc.inputExpIdPairs[ampNames[0]])) 

408 

409 for i in range(len(detectorMeans)): 

410 arr = np.array([datasetPtc.rawMeans[ampName][i] for ampName in ampNames]) 

411 good, = (np.isfinite(arr)).nonzero() 

412 if good.size == 0: 

413 detectorMeans[i] = np.nan 

414 else: 

415 detectorMeans[i] = np.mean(arr[good]) 

416 

417 index = np.argsort(detectorMeans) 

418 

419 for ampName in ampNames: 

420 datasetPtc.inputExpIdPairs[ampName] = np.array( 

421 datasetPtc.inputExpIdPairs[ampName] 

422 )[index].tolist() 

423 datasetPtc.rawExpTimes[ampName] = datasetPtc.rawExpTimes[ampName][index] 

424 datasetPtc.rawMeans[ampName] = datasetPtc.rawMeans[ampName][index] 

425 datasetPtc.rawVars[ampName] = datasetPtc.rawVars[ampName][index] 

426 datasetPtc.photoCharges[ampName] = datasetPtc.photoCharges[ampName][index] 

427 datasetPtc.histVars[ampName] = datasetPtc.histVars[ampName][index] 

428 datasetPtc.histChi2Dofs[ampName] = datasetPtc.histChi2Dofs[ampName][index] 

429 datasetPtc.kspValues[ampName] = datasetPtc.kspValues[ampName][index] 

430 datasetPtc.expIdMask[ampName] = datasetPtc.expIdMask[ampName][index] 

431 datasetPtc.covariances[ampName] = datasetPtc.covariances[ampName][index] 

432 datasetPtc.covariancesSqrtWeights[ampName] = datasetPtc.covariancesSqrtWeights[ampName][index] 

433 for key, value in datasetPtc.auxValues.items(): 

434 datasetPtc.auxValues[key] = value[index] 

435 

436 if self.config.ptcFitType == "FULLCOVARIANCE": 

437 # Fit the measured covariances vs mean signal to 

438 # the Astier+19 full model (Eq. 20). Before that 

439 # do a preliminary fit to the variance (C_00) vs mean 

440 # signal (mu) curve using the EXPAPPROXIMATION model 

441 # (Eq. 16 in Astier+19) in order to 

442 # get the flat pairs that are masked. The 

443 # points at these fluxes will also be masked when 

444 # calculating the other elements of the covariance 

445 # matrix, C_ij, i!=j). 

446 

447 # Preliminary fit, usign a temp dataset to get the mask 

448 tempDatasetPtc = copy.copy(datasetPtc) 

449 tempDatasetPtc.ptcFitType = "EXPAPPROXIMATION" 

450 tempDatasetPtc = self.fitMeasurementsToModel(tempDatasetPtc) 

451 

452 # "FULLCOVARIANCE", using the mask obtained from the 

453 # previous fit. 

454 for ampName in datasetPtc.ampNames: 

455 datasetPtc.expIdMask[ampName] = tempDatasetPtc.expIdMask[ampName] 

456 datasetPtc.fitType = "FULLCOVARIANCE" 

457 datasetPtc = self.fitMeasurementsToModel(datasetPtc) 

458 # The other options are: self.config.ptcFitType in 

459 # ("EXPAPPROXIMATION", "POLYNOMIAL") 

460 else: 

461 # Fit the PTC to a polynomial or to Astier+19 exponential 

462 # approximation (Eq. 16). Fill up 

463 # PhotonTransferCurveDataset object. 

464 datasetPtc = self.fitMeasurementsToModel(datasetPtc) 

465 

466 if camera: 

467 detector = camera[detId] 

468 else: 

469 detector = None 

470 datasetPtc.updateMetadataFromExposures(inputCovariances) 

471 datasetPtc.updateMetadata(setDate=True, camera=camera, detector=detector) 

472 

473 return pipeBase.Struct( 

474 outputPtcDataset=datasetPtc, 

475 ) 

476 

477 def fitMeasurementsToModel(self, dataset): 

478 """Fit the measured covariances vs mean signal to a 

479 polynomial or one of the models in Astier+19 

480 (Eq. 16 or Eq.20). 

481 

482 Parameters 

483 ---------- 

484 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset` 

485 The dataset containing information such as the means, 

486 (co)variances, and exposure times. 

487 

488 Returns 

489 ------- 

490 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset` 

491 This is the same dataset as the input parameter, however, 

492 it has been modified to include information such as the 

493 fit vectors and the fit parameters. See the class 

494 `PhotonTransferCurveDatase`. 

495 """ 

496 fitType = dataset.ptcFitType 

497 if fitType in ["FULLCOVARIANCE", ]: 

498 # This model uses the full covariance matrix in the fit. 

499 # The PTC is technically defined as variance vs signal, 

500 # with variance = Cov_00 

501 dataset = self.fitDataFullCovariance(dataset) 

502 elif fitType in ["POLYNOMIAL", "EXPAPPROXIMATION"]: 

503 # The PTC is technically defined as variance vs signal 

504 dataset = self.fitPtc(dataset) 

505 else: 

506 raise RuntimeError( 

507 f"Fitting option {fitType} not one of " 

508 "'POLYNOMIAL', 'EXPAPPROXIMATION', or 'FULLCOVARIANCE'" 

509 ) 

510 

511 return dataset 

512 

513 def fitDataFullCovariance(self, dataset): 

514 """Fit measured flat covariances to the full model in 

515 Astier+19 (Eq. 20). 

516 

517 Parameters 

518 ---------- 

519 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset` 

520 The dataset containing information such as the means, 

521 (co)variances, and exposure times. 

522 

523 Returns 

524 ------- 

525 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset` 

526 This is the same dataset as the input parameter, however, 

527 it has been modified to include information such as the 

528 fit vectors and the fit parameters. See the class 

529 `PhotonTransferCurveDatase`. 

530 

531 Notes 

532 ----- 

533 The parameters of the full model for C_ij(mu) ("C_ij" and "mu" 

534 in ADU^2 and ADU, respectively) in Astier+19 (Eq. 20) are: 

535 

536 - "a" coefficients (r by r matrix), units: 1/e 

537 - "b" coefficients (r by r matrix), units: 1/e 

538 - noise matrix (r by r matrix), units: e^2 

539 - gain, units: e/ADU 

540 

541 "b" appears in Eq. 20 only through the "ab" combination, which 

542 is defined in this code as "c=ab". 

543 

544 Total number of parameters: #entries(a) + #entries(c) + #entries(noise) 

545 + 1. This is equivalent to r^2 + r^2 + r^2 + 1, where "r" is the 

546 maximum lag considered for the covariances calculation, and the 

547 extra "1" is the gain. If "b" is 0, then "c" is 0, and len(pInit) will 

548 have r^2 fewer entries. 

549 """ 

550 matrixSide = dataset.covMatrixSide 

551 matrixSideFit = dataset.covMatrixSideFullCovFit 

552 lenParams = matrixSideFit*matrixSideFit 

553 

554 for ampName in dataset.ampNames: 

555 lenInputTimes = len(dataset.rawExpTimes[ampName]) 

556 # Not used when ptcFitType is 'FULLCOVARIANCE' 

557 dataset.ptcFitPars[ampName] = np.array([np.nan]) 

558 dataset.ptcFitParsError[ampName] = np.array([np.nan]) 

559 dataset.ptcFitChiSq[ampName] = np.nan 

560 

561 if ampName in dataset.badAmps: 

562 # Bad amp 

563 # Entries need to have proper dimensions so read/write 

564 # with astropy.Table works. 

565 nanMatrixFit = np.full((matrixSideFit, matrixSideFit), np.nan) 

566 listNanMatrix = np.full((lenInputTimes, matrixSide, matrixSide), np.nan) 

567 listNanMatrixFit = np.full((lenInputTimes, matrixSideFit, matrixSideFit), np.nan) 

568 dataset.covariancesModel[ampName] = listNanMatrixFit 

569 dataset.covariancesSqrtWeights[ampName] = listNanMatrix 

570 dataset.aMatrix[ampName] = nanMatrixFit 

571 dataset.bMatrix[ampName] = nanMatrixFit 

572 dataset.covariancesModelNoB[ampName] = listNanMatrixFit 

573 dataset.aMatrixNoB[ampName] = nanMatrixFit 

574 dataset.noiseMatrix[ampName] = nanMatrixFit 

575 dataset.noiseMatrixNoB[ampName] = nanMatrixFit 

576 

577 dataset.expIdMask[ampName] = np.repeat(False, lenInputTimes) 

578 dataset.gain[ampName] = np.nan 

579 dataset.gainErr[ampName] = np.nan 

580 dataset.noise[ampName] = np.nan 

581 dataset.noiseErr[ampName] = np.nan 

582 dataset.finalVars[ampName] = np.repeat(np.nan, lenInputTimes) 

583 dataset.finalModelVars[ampName] = np.repeat(np.nan, lenInputTimes) 

584 dataset.finalMeans[ampName] = np.repeat(np.nan, lenInputTimes) 

585 continue 

586 

587 muAtAmp = dataset.rawMeans[ampName] 

588 maskAtAmp = dataset.expIdMask[ampName] 

589 if len(maskAtAmp) == 0: 

590 maskAtAmp = np.repeat(True, len(muAtAmp)) 

591 

592 muAtAmpMasked = muAtAmp[maskAtAmp] 

593 covAtAmp = dataset.covariances[ampName] 

594 covAtAmpMasked = np.nan_to_num(covAtAmp)[maskAtAmp] 

595 covSqrtWeightsAtAmp = dataset.covariancesSqrtWeights[ampName] 

596 covSqrtWeightsAtAmpMasked = np.nan_to_num(covSqrtWeightsAtAmp)[maskAtAmp] 

597 

598 # Subtract long-range covariances 

599 if self.config.doSubtractLongRangeCovariances: 

600 startLag = self.config.startLongRangeCovariances 

601 covAtAmpMasked, covSqrtWeightsAtAmpMasked = self.subtractDistantOffset( 

602 muAtAmpMasked, covAtAmpMasked, 

603 covSqrtWeightsAtAmpMasked, 

604 start=startLag, 

605 degree=self.config.polyDegLongRangeCovariances) 

606 

607 # In principle, we could fit to a lag smaller than the measured 

608 # covariances. 

609 r = self.config.maximumRangeCovariancesAstierFullCovFit 

610 covAtAmpForFitMasked = covAtAmpMasked[:, :r, :r] 

611 covSqrtWeightsAtAmpForFitMasked = covSqrtWeightsAtAmpMasked[:, :r, :r] 

612 

613 # Initial fit, to approximate parameters, with c=0 

614 a0, c0, noise0, gain0 = self.initialFitFullCovariance( 

615 muAtAmpMasked, 

616 covAtAmpForFitMasked, 

617 covSqrtWeightsAtAmpForFitMasked 

618 ) 

619 

620 # Fit full model (Eq. 20 of Astier+19) and same model with 

621 # b=0 (c=0 in this code) 

622 pInit = np.concatenate((a0.ravel(), c0.ravel(), noise0.ravel(), np.array(gain0)), axis=None) 

623 functionsDict = {'fullModel': self.funcFullCovarianceModel, 

624 'fullModelNoB': self.funcFullCovarianceModelNoB} 

625 fitResults = {'fullModel': {'a': [], 'c': [], 'noise': [], 'gain': [], 'paramsErr': []}, 

626 'fullModelNoB': {'a': [], 'c': [], 'noise': [], 'gain': [], 'paramsErr': []}} 

627 for key in functionsDict: 

628 params, paramsErr, _ = fitLeastSq(pInit, muAtAmpMasked, 

629 covAtAmpForFitMasked.ravel(), functionsDict[key], 

630 weightsY=covSqrtWeightsAtAmpForFitMasked.ravel()) 

631 a = params[:lenParams].reshape((matrixSideFit, matrixSideFit)) 

632 c = params[lenParams:2*lenParams].reshape((matrixSideFit, matrixSideFit)) 

633 noise = params[2*lenParams:3*lenParams].reshape((matrixSideFit, matrixSideFit)) 

634 gain = params[-1] 

635 

636 fitResults[key]['a'] = a 

637 fitResults[key]['c'] = c 

638 fitResults[key]['noise'] = noise 

639 fitResults[key]['gain'] = gain 

640 fitResults[key]['paramsErr'] = paramsErr 

641 

642 # Put the information in the PTC dataset 

643 

644 # Not used when ptcFitType is 'FULLCOVARIANCE' 

645 dataset.ptcFitPars[ampName] = np.array([np.nan]) 

646 dataset.ptcFitParsError[ampName] = np.array([np.nan]) 

647 dataset.ptcFitChiSq[ampName] = np.nan 

648 

649 # Save full covariances, covariances models, and their weights. 

650 # dataset.expIdMask is already full, but needs to be 

651 # converted to bool. 

652 dataset.expIdMask[ampName] = np.array(dataset.expIdMask[ampName], dtype=bool) 

653 dataset.covariances[ampName] = covAtAmp 

654 # We evaluate the covariance model everywhere, even the 

655 # masked amps. 

656 dataset.covariancesModel[ampName] = self.evalCovModel(muAtAmp, 

657 fitResults['fullModel']['a'], 

658 fitResults['fullModel']['c'], 

659 fitResults['fullModel']['noise'], 

660 fitResults['fullModel']['gain']) 

661 dataset.covariancesSqrtWeights[ampName] = covSqrtWeightsAtAmp 

662 dataset.aMatrix[ampName] = fitResults['fullModel']['a'] 

663 dataset.bMatrix[ampName] = fitResults['fullModel']['c']/fitResults['fullModel']['a'] 

664 dataset.covariancesModelNoB[ampName] = self.evalCovModel(muAtAmp, 

665 fitResults['fullModelNoB']['a'], 

666 fitResults['fullModelNoB']['c'], 

667 fitResults['fullModelNoB']['noise'], 

668 fitResults['fullModelNoB']['gain'], 

669 setBtoZero=True) 

670 dataset.aMatrixNoB[ampName] = fitResults['fullModelNoB']['a'] 

671 dataset.gain[ampName] = fitResults['fullModel']['gain'] 

672 dataset.gainErr[ampName] = fitResults['fullModel']['paramsErr'][-1] 

673 readoutNoise = fitResults['fullModel']['noise'][0][0] 

674 readoutNoiseSqrt = np.sqrt(np.fabs(readoutNoise)) 

675 dataset.noise[ampName] = readoutNoise 

676 readoutNoiseSigma = fitResults['fullModel']['paramsErr'][2*lenParams] 

677 dataset.noiseErr[ampName] = 0.5*(readoutNoiseSigma/np.fabs(readoutNoise))*readoutNoiseSqrt 

678 dataset.noiseMatrix[ampName] = fitResults['fullModel']['noise'] 

679 dataset.noiseMatrixNoB[ampName] = fitResults['fullModelNoB']['noise'] 

680 

681 dataset.finalVars[ampName] = covAtAmp[:, 0, 0] 

682 dataset.finalModelVars[ampName] = dataset.covariancesModel[ampName][:, 0, 0] 

683 dataset.finalMeans[ampName] = muAtAmp 

684 

685 return dataset 

686 

687 def initialFitFullCovariance(self, mu, cov, sqrtW): 

688 """ Performs a crude parabolic fit of the data in order to start 

689 the full fit close to the solution, setting b=0 (c=0) in Eq. 20 

690 of Astier+19. 

691 

692 Parameters 

693 ---------- 

694 mu : `numpy.array`, (N,) 

695 Signal `mu` (ADU) 

696 cov : `numpy.array`, (N, M, M) 

697 Covariance arrays of size `(M, M)` (with 

698 `M = config.maximumRangeCovariancesAstier`), 

699 indexed by mean signal `mu`. 

700 sqrtW : `numpy.array`, (N,) 

701 Covariance weights, defined as 1./sqrt(Variances) 

702 

703 Returns 

704 ------- 

705 a : `numpy.array`, (M, M) 

706 "a" parameter per flux in Eq. 20 of Astier+19. 

707 c : `numpy.array`, (M, M) 

708 "c"="ab" parameter per flux in Eq. 20 of Astier+19. 

709 noise : `numpy.array`, (M, M) 

710 "noise" parameter per flux in Eq. 20 of Astier+19. 

711 gain : `float` 

712 Amplifier gain (e/ADU) 

713 """ 

714 matrixSideFit = self.config.maximumRangeCovariancesAstierFullCovFit 

715 

716 # Initialize fit parameters 

717 a = np.zeros((matrixSideFit, matrixSideFit)) 

718 c = np.zeros((matrixSideFit, matrixSideFit)) 

719 noise = np.zeros((matrixSideFit, matrixSideFit)) 

720 gain = 1. 

721 

722 # iterate the fit to account for higher orders 

723 # the chi2 does not necessarily go down, so one could 

724 # stop when it increases 

725 oldChi2 = 1e30 

726 for _ in range(5): 

727 model = np.nan_to_num(self.evalCovModel(mu, a, c, noise, gain, setBtoZero=True)) 

728 # loop on lags 

729 for i in range(matrixSideFit): 

730 for j in range(matrixSideFit): 

731 # fit a parabola for a given lag 

732 parsFit = np.polyfit(mu, cov[:, i, j] - model[:, i, j], 

733 2, w=sqrtW[:, i, j]) 

734 # model equation (Eq. 20) in Astier+19, with c=a*b=0: 

735 a[i, j] += parsFit[0] 

736 noise[i, j] += parsFit[2] 

737 if i + j == 0: 

738 gain = 1./(1/gain+parsFit[1]) 

739 weightedRes = (model - cov)*sqrtW 

740 chi2 = (weightedRes.flatten()**2).sum() 

741 if chi2 > oldChi2: 

742 break 

743 oldChi2 = chi2 

744 

745 return a, c, noise, gain 

746 

747 def funcFullCovarianceModel(self, params, x): 

748 """Model to fit covariances from flat fields; Equation 20 of 

749 Astier+19. 

750 

751 Parameters 

752 ---------- 

753 params : `list` 

754 Parameters of the model: aMatrix, CMatrix, noiseMatrix, 

755 gain (e/ADU). 

756 x : `numpy.array`, (N,) 

757 Signal `mu` (ADU) 

758 

759 Returns 

760 ------- 

761 y : `numpy.array`, (N,) 

762 Covariance matrix. 

763 """ 

764 matrixSideFit = self.config.maximumRangeCovariancesAstierFullCovFit 

765 lenParams = matrixSideFit*matrixSideFit 

766 aMatrix = params[:lenParams].reshape((matrixSideFit, matrixSideFit)) 

767 cMatrix = params[lenParams:2*lenParams].reshape((matrixSideFit, matrixSideFit)) 

768 noiseMatrix = params[2*lenParams:3*lenParams].reshape((matrixSideFit, matrixSideFit)) 

769 gain = params[-1] 

770 

771 return self.evalCovModel(x, aMatrix, cMatrix, noiseMatrix, gain).flatten() 

772 

773 def funcFullCovarianceModelNoB(self, params, x): 

774 """Model to fit covariances from flat fields; Equation 20 of 

775 Astier+19, with b=0 (equivalent to c=a*b=0 in this code). 

776 

777 Parameters 

778 ---------- 

779 params : `list` 

780 Parameters of the model: aMatrix, CMatrix, noiseMatrix, 

781 gain (e/ADU). 

782 x : `numpy.array`, (N,) 

783 Signal mu (ADU) 

784 

785 Returns 

786 ------- 

787 y : `numpy.array`, (N,) 

788 Covariance matrix. 

789 """ 

790 matrixSideFit = self.config.maximumRangeCovariancesAstierFullCovFit 

791 lenParams = matrixSideFit*matrixSideFit 

792 aMatrix = params[:lenParams].reshape((matrixSideFit, matrixSideFit)) 

793 cMatrix = params[lenParams:2*lenParams].reshape((matrixSideFit, matrixSideFit)) 

794 noiseMatrix = params[2*lenParams:3*lenParams].reshape((matrixSideFit, matrixSideFit)) 

795 gain = params[-1] 

796 

797 return self.evalCovModel(x, aMatrix, cMatrix, noiseMatrix, gain, setBtoZero=True).flatten() 

798 

799 def evalCovModel(self, mu, aMatrix, cMatrix, noiseMatrix, gain, setBtoZero=False): 

800 """Computes full covariances model (Eq. 20 of Astier+19). 

801 

802 Parameters 

803 ---------- 

804 mu : `numpy.array`, (N,) 

805 List of mean signals. 

806 aMatrix : `numpy.array`, (M, M) 

807 "a" parameter per flux in Eq. 20 of Astier+19. 

808 cMatrix : `numpy.array`, (M, M) 

809 "c"="ab" parameter per flux in Eq. 20 of Astier+19. 

810 noiseMatrix : `numpy.array`, (M, M) 

811 "noise" parameter per flux in Eq. 20 of Astier+19. 

812 gain : `float` 

813 Amplifier gain (e/ADU) 

814 setBtoZero=False : `bool`, optional 

815 Set "b" parameter in full model (see Astier+19) to zero. 

816 

817 Returns 

818 ------- 

819 covModel : `numpy.array`, (N, M, M) 

820 Covariances model. 

821 

822 Notes 

823 ----- 

824 By default, computes the covModel for the mu's stored(self.mu). 

825 Returns cov[Nmu, M, M]. The variance for the PTC is 

826 cov[:, 0, 0]. mu and cov are in ADUs and ADUs squared. To use 

827 electrons for both, the gain should be set to 1. This routine 

828 implements the model in Astier+19 (1905.08677). 

829 The parameters of the full model for C_ij(mu) ("C_ij" and "mu" 

830 in ADU^2 and ADU, respectively) in Astier+19 (Eq. 20) are: 

831 

832 - "a" coefficients (M by M matrix), units: 1/e 

833 - "b" coefficients (M by M matrix), units: 1/e 

834 - noise matrix (M by M matrix), units: e^2 

835 - gain, units: e/ADU 

836 

837 "b" appears in Eq. 20 only through the "ab" combination, which 

838 is defined in this code as "c=ab". 

839 """ 

840 matrixSideFit = self.config.maximumRangeCovariancesAstierFullCovFit 

841 sa = (matrixSideFit, matrixSideFit) 

842 # pad a with zeros and symmetrize 

843 aEnlarged = np.zeros((int(sa[0]*1.5)+1, int(sa[1]*1.5)+1)) 

844 aEnlarged[0:sa[0], 0:sa[1]] = aMatrix 

845 aSym = symmetrize(aEnlarged) 

846 # pad c with zeros and symmetrize 

847 cEnlarged = np.zeros((int(sa[0]*1.5)+1, int(sa[1]*1.5)+1)) 

848 cEnlarged[0:sa[0], 0:sa[1]] = cMatrix 

849 cSym = symmetrize(cEnlarged) 

850 a2 = fftconvolve(aSym, aSym, mode='same') 

851 a3 = fftconvolve(a2, aSym, mode='same') 

852 ac = fftconvolve(aSym, cSym, mode='same') 

853 (xc, yc) = np.unravel_index(np.abs(aSym).argmax(), a2.shape) 

854 

855 a1 = aMatrix[np.newaxis, :, :] 

856 a2 = a2[np.newaxis, xc:xc + matrixSideFit, yc:yc + matrixSideFit] 

857 a3 = a3[np.newaxis, xc:xc + matrixSideFit, yc:yc + matrixSideFit] 

858 ac = ac[np.newaxis, xc:xc + matrixSideFit, yc:yc + matrixSideFit] 

859 c1 = cMatrix[np.newaxis, ::] 

860 

861 # assumes that mu is 1d 

862 bigMu = mu[:, np.newaxis, np.newaxis]*gain 

863 # c(=a*b in Astier+19) also has a contribution to the last 

864 # term, that is absent for now. 

865 if setBtoZero: 

866 c1 = np.zeros_like(c1) 

867 ac = np.zeros_like(ac) 

868 covModel = (bigMu/(gain*gain)*(a1*bigMu+2./3.*(bigMu*bigMu)*(a2 + c1) 

869 + (1./3.*a3 + 5./6.*ac)*(bigMu*bigMu*bigMu)) + noiseMatrix[np.newaxis, :, :]/gain**2) 

870 # add the Poisson term, and the read out noise (variance) 

871 covModel[:, 0, 0] += mu/gain 

872 

873 return covModel 

874 

875 def subtractDistantOffset(self, muAtAmpMasked, covAtAmpMasked, covSqrtWeightsAtAmpMasked, 

876 start, degree=1): 

877 """Subtract distant offset from the covariance matrices. 

878 

879 Parameters 

880 ---------- 

881 muAtAmpMasked : `numpy.array` 

882 Masked mean flux array for a particular amplifier. 

883 covAtAmpMasked : `numpy.array` 

884 Masked measured covariances for a particular amplifier. 

885 covSqrtWeightsAtAmpMasked : `numpy.array` 

886 Masked inverse covariance weights for a particular amplifier. 

887 start : int, optional 

888 The starting index to eliminate the core for the fit. 

889 degree : int, optional 

890 Degree of the polynomial fit. 

891 

892 Returns 

893 ------- 

894 covAtAmpMasked : `numpy.array` 

895 Subtracted measured covariances for a particular amplifier. 

896 covSqrtWeightsAtAmpMasked : `numpy.array` 

897 Masked inverse covariance weights for a particular amplifier. 

898 

899 Notes 

900 ----- 

901 Ported from https://gitlab.in2p3.fr/astier/bfptc by P. Astier. 

902 

903 This function subtracts a distant offset from the 

904 covariance matrices using polynomial fitting. The core 

905 of the matrices is eliminated for the fit. 

906 

907 The function modifies the internal state of the object, updating the 

908 covariance matrices and related attributes. 

909 """ 

910 for k in range(len(muAtAmpMasked)): 

911 # Make a copy because it will be altered 

912 w = np.copy(covSqrtWeightsAtAmpMasked[k, ...]) 

913 wShape = w.shape 

914 i, j = np.meshgrid(range(wShape[0]), range(wShape[1]), indexing='ij') 

915 

916 # Eliminate the core for the fit 

917 w[:start, :start] = 0 

918 

919 poly = Pol2D(i, j, covAtAmpMasked[k, ...], degree, w=w) 

920 back = poly.eval(i, j) 

921 

922 covAtAmpMasked[k, ...] -= back 

923 

924 return covAtAmpMasked, covSqrtWeightsAtAmpMasked 

925 

926 # EXPAPPROXIMATION and POLYNOMIAL fit methods 

927 @staticmethod 

928 def _initialParsForPolynomial(order): 

929 assert order >= 2 

930 pars = np.zeros(order, dtype=float) 

931 pars[0] = 10 

932 pars[1] = 1 

933 pars[2:] = 0.0001 

934 return pars 

935 

936 @staticmethod 

937 def _boundsForPolynomial(initialPars, lowers=[], uppers=[]): 

938 if not len(lowers): 

939 lowers = [np.NINF for p in initialPars] 

940 if not len(uppers): 

941 uppers = [np.inf for p in initialPars] 

942 lowers[1] = 0 # no negative gains 

943 return (lowers, uppers) 

944 

945 @staticmethod 

946 def _boundsForAstier(initialPars, lowers=[], uppers=[]): 

947 if not len(lowers): 

948 lowers = [np.NINF for p in initialPars] 

949 if not len(uppers): 

950 uppers = [np.inf for p in initialPars] 

951 return (lowers, uppers) 

952 

953 @staticmethod 

954 def _getInitialGoodPoints(means, variances, minVarPivotSearch, consecutivePointsVarDecreases): 

955 """Return a boolean array to mask bad points. 

956 

957 Parameters 

958 ---------- 

959 means : `numpy.array` 

960 Input array with mean signal values. 

961 variances : `numpy.array` 

962 Input array with variances at each mean value. 

963 minVarPivotSearch : `float` 

964 The variance (in ADU^2), above which, the point 

965 of decreasing variance should be sought. 

966 consecutivePointsVarDecreases : `int` 

967 Required number of consecutive points/fluxes 

968 in the PTC where the variance 

969 decreases in order to find a first 

970 estimate of the PTC turn-off. 

971 

972 Returns 

973 ------ 

974 goodPoints : `numpy.array` [`bool`] 

975 Boolean array to select good (`True`) and bad (`False`) 

976 points. 

977 

978 Notes 

979 ----- 

980 Eliminate points beyond which the variance decreases. 

981 """ 

982 goodPoints = np.ones_like(means, dtype=bool) 

983 # Variances are sorted and should monotonically increase 

984 pivotList = np.where(np.array(np.diff(variances)) < 0)[0] 

985 if len(pivotList) > 0: 

986 # For small values, sometimes the variance decreases slightly 

987 # Only look when var > self.config.minVarPivotSearch 

988 pivotList = [p for p in pivotList if variances[p] > minVarPivotSearch] 

989 # Require that the varince decreases during 

990 # consecutivePointsVarDecreases 

991 # consecutive points. This will give a first 

992 # estimate of the PTC turn-off, which 

993 # may be updated (reduced) further in the code. 

994 if len(pivotList) > 1: 

995 # enumerate(pivotList) creates tuples (index, value), for 

996 # each value in pivotList. The lambda function subtracts 

997 # each value from the index. 

998 # groupby groups elements by equal key value. 

999 for k, g in groupby(enumerate(pivotList), lambda x: x[0]-x[1]): 

1000 group = (map(itemgetter(1), g)) 

1001 # Form groups of consecute values from pivotList 

1002 group = list(map(int, group)) 

1003 # values in pivotList are indices where np.diff(variances) 

1004 # is negative, i.e., where the variance starts decreasing. 

1005 # Find the first group of consecutive numbers when 

1006 # variance decreases. 

1007 if len(group) >= consecutivePointsVarDecreases: 

1008 pivotIndex = np.min(group) 

1009 goodPoints[pivotIndex+1:] = False 

1010 break 

1011 

1012 # Finally, we filter out any infinities or NaNs. 

1013 goodPoints[(~np.isfinite(means)) | (~np.isfinite(variances))] = False 

1014 

1015 return goodPoints 

1016 

1017 def _makeZeroSafe(self, array, substituteValue=1e-9): 

1018 """""" 

1019 array = np.array(array) 

1020 nBad = Counter(np.ravel(array))[0] 

1021 if nBad == 0: 

1022 return array 

1023 

1024 index, = np.where(array == 0) 

1025 if len(index): 

1026 msg = f"Found {nBad} zeros in array at elements {index}" 

1027 self.log.warning(msg) 

1028 

1029 array[index] = substituteValue 

1030 

1031 return array 

1032 

1033 def fitPtc(self, dataset): 

1034 """Fit the photon transfer curve to a polynomial or to the 

1035 Astier+19 approximation (Eq. 16). 

1036 

1037 Fit the photon transfer curve with either a polynomial of 

1038 the order specified in the task config (POLNOMIAL), 

1039 or using the exponential approximation in Astier+19 

1040 (Eq. 16, FULLCOVARIANCE). 

1041 

1042 Sigma clipping is performed iteratively for the fit, as 

1043 well as an initial clipping of data points that are more 

1044 than `config.initialNonLinearityExclusionThreshold` away 

1045 from lying on a straight line. This other step is necessary 

1046 because the photon transfer curve turns over catastrophically 

1047 at very high flux (because saturation 

1048 drops the variance to ~0) and these far outliers cause the 

1049 initial fit to fail, meaning the sigma cannot be calculated 

1050 to perform the sigma-clipping. 

1051 

1052 Parameters 

1053 ---------- 

1054 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset` 

1055 The dataset containing the means, variances and 

1056 exposure times. 

1057 

1058 Returns 

1059 ------- 

1060 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset` 

1061 This is the same dataset as the input parameter, however, 

1062 it has been modified to include information such as the 

1063 fit vectors and the fit parameters. See the class 

1064 `PhotonTransferCurveDatase`. 

1065 

1066 Raises 

1067 ------ 

1068 RuntimeError 

1069 Raised if dataset.ptcFitType is None or empty. 

1070 """ 

1071 if dataset.ptcFitType: 

1072 ptcFitType = dataset.ptcFitType 

1073 else: 

1074 raise RuntimeError("ptcFitType is None of empty in PTC dataset.") 

1075 # For FULLCOVARIANCE model fit 

1076 matrixSideFit = dataset.covMatrixSideFullCovFit 

1077 nanMatrixFit = np.empty((matrixSideFit, matrixSideFit)) 

1078 nanMatrixFit[:] = np.nan 

1079 

1080 for amp in dataset.ampNames: 

1081 lenInputTimes = len(dataset.rawExpTimes[amp]) 

1082 listNanMatrixFit = np.empty((lenInputTimes, matrixSideFit, matrixSideFit)) 

1083 listNanMatrixFit[:] = np.nan 

1084 

1085 dataset.covariancesModel[amp] = listNanMatrixFit 

1086 dataset.aMatrix[amp] = nanMatrixFit 

1087 dataset.bMatrix[amp] = nanMatrixFit 

1088 dataset.covariancesModelNoB[amp] = listNanMatrixFit 

1089 dataset.aMatrixNoB[amp] = nanMatrixFit 

1090 dataset.noiseMatrix[amp] = nanMatrixFit 

1091 dataset.noiseMatrixNoB[amp] = nanMatrixFit 

1092 

1093 def errFunc(p, x, y): 

1094 return ptcFunc(p, x) - y 

1095 

1096 sigmaCutPtcOutliers = self.config.sigmaCutPtcOutliers 

1097 maxIterationsPtcOutliers = self.config.maxIterationsPtcOutliers 

1098 

1099 for i, ampName in enumerate(dataset.ampNames): 

1100 meanVecOriginal = dataset.rawMeans[ampName].copy() 

1101 varVecOriginal = dataset.rawVars[ampName].copy() 

1102 varVecOriginal = self._makeZeroSafe(varVecOriginal) 

1103 

1104 if self.config.doLegacyTurnoffSelection: 

1105 # Discard points when the variance starts to decrease after two 

1106 # consecutive signal levels 

1107 goodPoints = self._getInitialGoodPoints(meanVecOriginal, varVecOriginal, 

1108 self.config.minVarPivotSearch, 

1109 self.config.consecutivePointsVarDecreases) 

1110 else: 

1111 goodPoints = dataset.expIdMask[ampName] 

1112 

1113 # Check if all points are bad from the 'cpExtractPtcTask' 

1114 initialExpIdMask = dataset.expIdMask[ampName] 

1115 

1116 if not (goodPoints.any() and initialExpIdMask.any()): 

1117 msg = (f"SERIOUS: All points in goodPoints: {goodPoints} or " 

1118 f"in initialExpIdMask: {initialExpIdMask} are bad." 

1119 f"Setting {ampName} to BAD.") 

1120 self.log.warning(msg) 

1121 # Fill entries with NaNs 

1122 self.fillBadAmp(dataset, ptcFitType, ampName) 

1123 continue 

1124 

1125 mask = goodPoints 

1126 

1127 if ptcFitType == 'EXPAPPROXIMATION': 

1128 ptcFunc = funcAstier 

1129 parsIniPtc = [-1e-9, 1.0, 10.] # a00, gain, noise^2 

1130 # lowers and uppers obtained from BOT data studies by 

1131 # C. Lage (UC Davis, 11/2020). 

1132 if self.config.binSize > 1: 

1133 bounds = self._boundsForAstier(parsIniPtc) 

1134 else: 

1135 bounds = self._boundsForAstier(parsIniPtc, lowers=[-1e-4, 0.1, -2000], 

1136 uppers=[1e-4, 10.0, 2000]) 

1137 if ptcFitType == 'POLYNOMIAL': 

1138 ptcFunc = funcPolynomial 

1139 parsIniPtc = self._initialParsForPolynomial(self.config.polynomialFitDegree + 1) 

1140 bounds = self._boundsForPolynomial(parsIniPtc) 

1141 

1142 # We perform an initial (unweighted) fit of variance vs signal 

1143 # (after initial KS test or post-drop selection) to look for 

1144 # outliers, particularly at the high-flux end. The initial fit 

1145 # is performed only for points that are guaranteed to be below 

1146 # the PTC turnoff and then extrapolated to ensure that high 

1147 # flux points that have abnormal variance values can be properly 

1148 # rejected in this phase without biasing the initial fit. 

1149 # This algorithm was initially developed by Seth Digel for 

1150 # the EO Testing pipeline. 

1151 

1152 if self.config.scaleMaxSignalInitialPtcOutlierFit: 

1153 approxGain = np.nanmedian(meanVecOriginal/varVecOriginal) 

1154 maxADUInitialPtcOutlierFit = self.config.maxSignalInitialPtcOutlierFit/approxGain 

1155 self.log.info( 

1156 "Using approximate gain %.3f and ADU signal cutoff of %.1f for amplifier %s", 

1157 approxGain, 

1158 maxADUInitialPtcOutlierFit, 

1159 ampName, 

1160 ) 

1161 else: 

1162 maxADUInitialPtcOutlierFit = self.config.maxSignalInitialPtcOutlierFit 

1163 

1164 if maxIterationsPtcOutliers == 0: 

1165 # We are not doing any outlier rejection here, but we do want 

1166 # an initial fit. 

1167 res = least_squares( 

1168 errFunc, 

1169 parsIniPtc, 

1170 bounds=bounds, 

1171 args=(meanVecOriginal[mask], varVecOriginal[mask]), 

1172 ) 

1173 pars = res.x 

1174 newMask = mask.copy() 

1175 else: 

1176 newMask = (mask & (meanVecOriginal <= maxADUInitialPtcOutlierFit)) 

1177 

1178 count = 0 

1179 lastMask = mask.copy() 

1180 while count < maxIterationsPtcOutliers: 

1181 res = least_squares( 

1182 errFunc, 

1183 parsIniPtc, 

1184 bounds=bounds, 

1185 args=(meanVecOriginal[newMask], varVecOriginal[newMask]), 

1186 ) 

1187 pars = res.x 

1188 

1189 sigResids = (varVecOriginal - ptcFunc(pars, meanVecOriginal))/np.sqrt(varVecOriginal) 

1190 # The new mask includes points where the residuals are 

1191 # finite, are less than the cut, and include the original 

1192 # mask of known points that should not be used. 

1193 newMask = ( 

1194 np.isfinite(sigResids) 

1195 & (np.abs(np.nan_to_num(sigResids)) < sigmaCutPtcOutliers) 

1196 & mask 

1197 ) 

1198 if np.count_nonzero(newMask) == 0: 

1199 msg = (f"SERIOUS: All points after outlier rejection are bad. " 

1200 f"Setting {ampName} to BAD.") 

1201 self.log.warning(msg) 

1202 # Fill entries with NaNs 

1203 self.fillBadAmp(dataset, ptcFitType, ampName) 

1204 break 

1205 

1206 self.log.debug( 

1207 "Iteration %d: Removed %d points in total for %s.", 

1208 count, 

1209 np.count_nonzero(mask) - np.count_nonzero(newMask), 

1210 ampName, 

1211 ) 

1212 

1213 # If the mask hasn't changed then break out. 

1214 if np.all(newMask == lastMask): 

1215 self.log.debug("Convergence at iteration %d; breaking loop for %s.", count, ampName) 

1216 break 

1217 

1218 lastMask = newMask.copy() 

1219 

1220 count += 1 

1221 

1222 # Set the mask to the new mask 

1223 mask = newMask.copy() 

1224 

1225 if not mask.any(): 

1226 # We hae already filled the bad amp above, so continue. 

1227 continue 

1228 

1229 dataset.expIdMask[ampName] = mask 

1230 

1231 parsIniPtc = pars 

1232 meanVecFinal = meanVecOriginal[mask] 

1233 varVecFinal = varVecOriginal[mask] 

1234 

1235 # Save the maximum point after outlier detection as the 

1236 # PTC turnoff point. 

1237 dataset.ptcTurnoff[ampName] = meanVecFinal[-1] 

1238 

1239 if Counter(mask)[False] > 0: 

1240 self.log.info("Number of points discarded in PTC of amplifier %s:" 

1241 " %d out of %d", ampName, Counter(mask)[False], len(meanVecOriginal)) 

1242 

1243 if (len(meanVecFinal) < len(parsIniPtc)): 

1244 msg = (f"SERIOUS: Not enough data points ({len(meanVecFinal)}) compared to the number of " 

1245 f"parameters of the PTC model({len(parsIniPtc)}). Setting {ampName} to BAD.") 

1246 self.log.warning(msg) 

1247 # Fill entries with NaNs 

1248 self.fillBadAmp(dataset, ptcFitType, ampName) 

1249 continue 

1250 # Fit the PTC. 

1251 # The variance of the variance is Var(v)=2*v^2/Npix. This is 

1252 # already calculated in `makeCovArray` of CpPtcExtract. 

1253 # dataset.covariancesSqrtWeights[ampName][:,0,0] 

1254 # has 1/sqrt(Var(v)). 

1255 weightsY = dataset.covariancesSqrtWeights[ampName][:, 0, 0][mask] 

1256 if self.config.doFitBootstrap: 

1257 parsFit, parsFitErr, reducedChiSqPtc = fitBootstrap(parsIniPtc, meanVecFinal, 

1258 varVecFinal, ptcFunc, 

1259 weightsY=weightsY) 

1260 else: 

1261 parsFit, parsFitErr, reducedChiSqPtc = fitLeastSq(parsIniPtc, meanVecFinal, 

1262 varVecFinal, ptcFunc, 

1263 weightsY=weightsY) 

1264 dataset.ptcFitPars[ampName] = parsFit 

1265 dataset.ptcFitParsError[ampName] = parsFitErr 

1266 dataset.ptcFitChiSq[ampName] = reducedChiSqPtc 

1267 

1268 dataset.finalVars[ampName] = varVecOriginal 

1269 dataset.finalVars[ampName][~mask] = np.nan 

1270 dataset.finalModelVars[ampName] = ptcFunc(parsFit, meanVecOriginal) 

1271 dataset.finalModelVars[ampName][~mask] = np.nan 

1272 dataset.finalMeans[ampName] = meanVecOriginal 

1273 dataset.finalMeans[ampName][~mask] = np.nan 

1274 

1275 if ptcFitType == 'EXPAPPROXIMATION': 

1276 ptcGain = parsFit[1] 

1277 ptcGainErr = parsFitErr[1] 

1278 ptcNoise = np.sqrt(np.fabs(parsFit[2])) 

1279 ptcNoiseErr = 0.5*(parsFitErr[2]/np.fabs(parsFit[2]))*np.sqrt(np.fabs(parsFit[2])) 

1280 if ptcFitType == 'POLYNOMIAL': 

1281 ptcGain = 1./parsFit[1] 

1282 ptcGainErr = np.fabs(1./parsFit[1])*(parsFitErr[1]/parsFit[1]) 

1283 ptcNoise = np.sqrt(np.fabs(parsFit[0]))*ptcGain 

1284 ptcNoiseErr = (0.5*(parsFitErr[0]/np.fabs(parsFit[0]))*(np.sqrt(np.fabs(parsFit[0]))))*ptcGain 

1285 dataset.gain[ampName] = ptcGain 

1286 dataset.gainErr[ampName] = ptcGainErr 

1287 dataset.noise[ampName] = ptcNoise 

1288 dataset.noiseErr[ampName] = ptcNoiseErr 

1289 

1290 if not len(dataset.ptcFitType) == 0: 

1291 dataset.ptcFitType = ptcFitType 

1292 if len(dataset.badAmps) == 0: 

1293 dataset.badAmps = [] 

1294 

1295 return dataset 

1296 

1297 def fillBadAmp(self, dataset, ptcFitType, ampName): 

1298 """Fill the dataset with NaNs if there are not enough 

1299 good points. 

1300 

1301 Parameters 

1302 ---------- 

1303 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset` 

1304 The dataset containing the means, variances and 

1305 exposure times. 

1306 ptcFitType : {'POLYNOMIAL', 'EXPAPPROXIMATION'} 

1307 Fit a 'POLYNOMIAL' (degree: 'polynomialFitDegree') or 

1308 'EXPAPPROXIMATION' (Eq. 16 of Astier+19) to the PTC. 

1309 ampName : `str` 

1310 Amplifier name. 

1311 """ 

1312 dataset.badAmps.append(ampName) 

1313 dataset.expIdMask[ampName] = np.repeat(False, len(dataset.rawExpTimes[ampName])) 

1314 dataset.gain[ampName] = np.nan 

1315 dataset.gainErr[ampName] = np.nan 

1316 dataset.noise[ampName] = np.nan 

1317 dataset.noiseErr[ampName] = np.nan 

1318 dataset.ptcFitPars[ampName] = (np.repeat(np.nan, self.config.polynomialFitDegree + 1) if 

1319 ptcFitType in ["POLYNOMIAL", ] else np.repeat(np.nan, 3)) 

1320 dataset.ptcFitParsError[ampName] = (np.repeat(np.nan, self.config.polynomialFitDegree + 1) if 

1321 ptcFitType in ["POLYNOMIAL", ] else np.repeat(np.nan, 3)) 

1322 dataset.ptcFitChiSq[ampName] = np.nan 

1323 dataset.ptcTurnoff[ampName] = np.nan 

1324 dataset.finalVars[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName])) 

1325 dataset.finalModelVars[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName])) 

1326 dataset.finalMeans[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName])) 

1327 

1328 return