Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of cp_pipe. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21# 

22import numpy as np 

23from collections import Counter 

24 

25import lsst.pex.config as pexConfig 

26import lsst.pipe.base as pipeBase 

27from lsst.cp.pipe.utils import (fitLeastSq, fitBootstrap, funcPolynomial, funcAstier) 

28 

29from scipy.optimize import least_squares 

30 

31import lsst.pipe.base.connectionTypes as cT 

32 

33from .astierCovPtcUtils import fitDataFullCovariance 

34 

35from lsst.ip.isr import PhotonTransferCurveDataset 

36 

37from lsst.cp.pipe._lookupStaticCalibration import lookupStaticCalibration 

38 

39import copy 

40 

41 

42__all__ = ['PhotonTransferCurveSolveConfig', 'PhotonTransferCurveSolveTask'] 

43 

44 

45class PhotonTransferCurveSolveConnections(pipeBase.PipelineTaskConnections, 

46 dimensions=("instrument", "detector")): 

47 inputCovariances = cT.Input( 

48 name="ptcCovariances", 

49 doc="Tuple with measured covariances from flats.", 

50 storageClass="PhotonTransferCurveDataset", 

51 dimensions=("instrument", "exposure", "detector"), 

52 multiple=True, 

53 ) 

54 camera = cT.PrerequisiteInput( 

55 name="camera", 

56 doc="Camera the input data comes from.", 

57 storageClass="Camera", 

58 dimensions=("instrument",), 

59 isCalibration=True, 

60 lookupFunction=lookupStaticCalibration, 

61 ) 

62 outputPtcDataset = cT.Output( 

63 name="ptcDatsetProposal", 

64 doc="Output proposed ptc dataset.", 

65 storageClass="PhotonTransferCurveDataset", 

66 dimensions=("instrument", "detector"), 

67 multiple=False, 

68 isCalibration=True, 

69 ) 

70 

71 

72class PhotonTransferCurveSolveConfig(pipeBase.PipelineTaskConfig, 

73 pipelineConnections=PhotonTransferCurveSolveConnections): 

74 """Configuration for fitting measured covariances. 

75 """ 

76 ptcFitType = pexConfig.ChoiceField( 

77 dtype=str, 

78 doc="Fit PTC to Eq. 16, Eq. 20 in Astier+19, or to a polynomial.", 

79 default="POLYNOMIAL", 

80 allowed={ 

81 "POLYNOMIAL": "n-degree polynomial (use 'polynomialFitDegree' to set 'n').", 

82 "EXPAPPROXIMATION": "Approximation in Astier+19 (Eq. 16).", 

83 "FULLCOVARIANCE": "Full covariances model in Astier+19 (Eq. 20)" 

84 } 

85 ) 

86 maximumRangeCovariancesAstier = pexConfig.Field( 

87 dtype=int, 

88 doc="Maximum range of covariances as in Astier+19", 

89 default=8, 

90 ) 

91 sigmaClipFullFitCovariancesAstier = pexConfig.Field( 

92 dtype=float, 

93 doc="sigma clip for full model fit for FULLCOVARIANCE ptcFitType ", 

94 default=5.0, 

95 ) 

96 maxIterFullFitCovariancesAstier = pexConfig.Field( 

97 dtype=int, 

98 doc="Maximum number of iterations in full model fit for FULLCOVARIANCE ptcFitType", 

99 default=3, 

100 ) 

101 polynomialFitDegree = pexConfig.Field( 

102 dtype=int, 

103 doc="Degree of polynomial to fit the PTC, when 'ptcFitType'=POLYNOMIAL.", 

104 default=3, 

105 ) 

106 sigmaCutPtcOutliers = pexConfig.Field( 

107 dtype=float, 

108 doc="Sigma cut for outlier rejection in PTC.", 

109 default=5.0, 

110 ) 

111 maxIterationsPtcOutliers = pexConfig.Field( 

112 dtype=int, 

113 doc="Maximum number of iterations for outlier rejection in PTC.", 

114 default=2, 

115 ) 

116 initialNonLinearityExclusionThresholdPositive = pexConfig.RangeField( 

117 dtype=float, 

118 doc="Initially exclude data points with a variance that are more than a factor of this from being" 

119 " linear in the positive direction, from the PTC fit. Note that these points will also be" 

120 " excluded from the non-linearity fit. This is done before the iterative outlier rejection," 

121 " to allow an accurate determination of the sigmas for said iterative fit.", 

122 default=0.05, 

123 min=0.0, 

124 max=1.0, 

125 ) 

126 initialNonLinearityExclusionThresholdNegative = pexConfig.RangeField( 

127 dtype=float, 

128 doc="Initially exclude data points with a variance that are more than a factor of this from being" 

129 " linear in the negative direction, from the PTC fit. Note that these points will also be" 

130 " excluded from the non-linearity fit. This is done before the iterative outlier rejection," 

131 " to allow an accurate determination of the sigmas for said iterative fit.", 

132 default=0.25, 

133 min=0.0, 

134 max=1.0, 

135 ) 

136 minMeanRatioTest = pexConfig.Field( 

137 dtype=float, 

138 doc="In the initial test to screen out bad points with a ratio test, points with low" 

139 " flux can get inadvertantly screened. This test only screens out points with flux" 

140 " above this value.", 

141 default=20000, 

142 ) 

143 minVarPivotSearch = pexConfig.Field( 

144 dtype=float, 

145 doc="The code looks for a pivot signal point after which the variance starts decreasing at high-flux" 

146 " to exclude then form the PTC model fit. However, sometimes at low fluxes, the variance" 

147 " decreases slightly. Set this variable for the variance value, in ADU^2, after which the pivot " 

148 " should be sought.", 

149 default=10000, 

150 ) 

151 doFitBootstrap = pexConfig.Field( 

152 dtype=bool, 

153 doc="Use bootstrap for the PTC fit parameters and errors?.", 

154 default=False, 

155 ) 

156 

157 

158class PhotonTransferCurveSolveTask(pipeBase.PipelineTask, 

159 pipeBase.CmdLineTask): 

160 """Task to fit the PTC from flat covariances. 

161 This task assembles the list of individual PTC datasets produced 

162 by `PhotonTransferCurveSolveTask` into one single final PTC dataset. 

163 The task fits the measured (co)variances to a polynomial model or to 

164 the models described in equations 16 and 20 of Astier+19 

165 (referred to as `POLYNOMIAL`, `EXPAPPROXIMATION`, and `FULLCOVARIANCE` 

166 in the configuration options of the task, respectively). Parameters 

167 of interest such as tghe gain and noise are derived from the fits. 

168 

169 Astier+19: "The Shape of the Photon Transfer Curve 

170 of CCD sensors", arXiv:1905.08677 

171 """ 

172 ConfigClass = PhotonTransferCurveSolveConfig 

173 _DefaultName = 'cpPhotonTransferCurveSolve' 

174 

175 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

176 """Ensure that the input and output dimensions are passed along. 

177 

178 Parameters 

179 ---------- 

180 butlerQC : `~lsst.daf.butler.butlerQuantumContext.ButlerQuantumContext` 

181 Butler to operate on. 

182 inputRefs : `~lsst.pipe.base.connections.InputQuantizedConnection` 

183 Input data refs to load. 

184 ouptutRefs : `~lsst.pipe.base.connections.OutputQuantizedConnection` 

185 Output data refs to persist. 

186 """ 

187 inputs = butlerQC.get(inputRefs) 

188 outputs = self.run(inputCovariances=inputs['inputCovariances'], camera=inputs['camera']) 

189 butlerQC.put(outputs, outputRefs) 

190 

191 def run(self, inputCovariances, camera=None, inputExpList=None): 

192 """Fit measure covariances to different models. 

193 

194 Parameters 

195 ---------- 

196 inputCovariances : `list` [`lsst.ip.isr.PhotonTransferCurveDataset`] 

197 List of lsst.ip.isr.PhotonTransferCurveDataset datasets. 

198 

199 camera : `lsst.afw.cameraGeom.Camera`, optional 

200 Input camera. 

201 

202 inputExpList : `list` [`~lsst.afw.image.exposure.exposure.ExposureF`], optional 

203 List of exposures. 

204 

205 Returns 

206 ------- 

207 results : `lsst.pipe.base.Struct` 

208 The results struct containing: 

209 ``outputPtcDatset`` : `lsst.ip.isr.PhotonTransferCurveDataset` 

210 Final PTC dataset, containing information such as the means, variances, 

211 and exposure times. 

212 """ 

213 # Assemble partial PTC datasets into a single dataset. 

214 ampNames = np.unique(inputCovariances[0].ampNames) 

215 datasetPtc = PhotonTransferCurveDataset(ampNames, self.config.ptcFitType, 

216 self.config.maximumRangeCovariancesAstier) 

217 for partialPtcDataset in inputCovariances: 

218 if partialPtcDataset.ptcFitType == 'DUMMY': 

219 continue 

220 for ampName in ampNames: 

221 datasetPtc.inputExpIdPairs[ampName].append(partialPtcDataset.inputExpIdPairs[ampName]) 

222 if type(partialPtcDataset.rawExpTimes[ampName]) is list: 

223 datasetPtc.rawExpTimes[ampName].append(partialPtcDataset.rawExpTimes[ampName][0]) 

224 else: 

225 datasetPtc.rawExpTimes[ampName].append(partialPtcDataset.rawExpTimes[ampName]) 

226 if type(partialPtcDataset.rawMeans[ampName]) is list: 

227 datasetPtc.rawMeans[ampName].append(partialPtcDataset.rawMeans[ampName][0]) 

228 else: 

229 datasetPtc.rawMeans[ampName].append(partialPtcDataset.rawMeans[ampName]) 

230 if type(partialPtcDataset.rawVars[ampName]) is list: 

231 datasetPtc.rawVars[ampName].append(partialPtcDataset.rawVars[ampName][0]) 

232 else: 

233 datasetPtc.rawVars[ampName].append(partialPtcDataset.rawVars[ampName]) 

234 datasetPtc.covariances[ampName].append(np.array(partialPtcDataset.covariances[ampName][0])) 

235 datasetPtc.covariancesSqrtWeights[ampName].append( 

236 np.array(partialPtcDataset.covariancesSqrtWeights[ampName][0])) 

237 # Sort arrays that are filled so far in the final dataset by rawMeans index 

238 for ampName in ampNames: 

239 index = np.argsort(np.ravel(np.array(datasetPtc.rawMeans[ampName]))) 

240 datasetPtc.inputExpIdPairs[ampName] = np.array(datasetPtc.inputExpIdPairs[ampName])[index] 

241 datasetPtc.rawExpTimes[ampName] = np.array(datasetPtc.rawExpTimes[ampName])[index] 

242 datasetPtc.rawMeans[ampName] = np.array(datasetPtc.rawMeans[ampName])[index] 

243 datasetPtc.rawVars[ampName] = np.array(datasetPtc.rawVars[ampName])[index] 

244 datasetPtc.covariances[ampName] = np.array(datasetPtc.covariances[ampName])[index] 

245 datasetPtc.covariancesSqrtWeights[ampName] = np.array( 

246 datasetPtc.covariancesSqrtWeights[ampName])[index] 

247 

248 if self.config.ptcFitType == "FULLCOVARIANCE": 

249 # Calculate covariances and fit them, including the PTC, to Astier+19 full model (Eq. 20) 

250 # First, fit get the flat pairs that are masked, fitting C_00 vs mu to 

251 # the EXPAPPROXIMATION model (Eq. 16 in Astier+19). 

252 # The points at these fluxes will also be masked when calculating the other covariances, C_ij) 

253 tempDatasetPtc = copy.copy(datasetPtc) 

254 tempDatasetPtc.ptcFitType = "EXPAPPROXIMATION" 

255 tempDatasetPtc = self.fitPtc(tempDatasetPtc) 

256 for ampName in datasetPtc.ampNames: 

257 datasetPtc.expIdMask[ampName] = tempDatasetPtc.expIdMask[ampName] 

258 datasetPtc.fitType = "FULLCOVARIANCE" 

259 datasetPtc = self.fitCovariancesAstier(datasetPtc) 

260 # The other options are: self.config.ptcFitType in ("EXPAPPROXIMATION", "POLYNOMIAL") 

261 else: 

262 # Fit the PTC to a polynomial or to Astier+19 exponential approximation (Eq. 16). 

263 # Fill up PhotonTransferCurveDataset object. 

264 datasetPtc = self.fitPtc(datasetPtc) 

265 if inputExpList is not None: 

266 # It should be a list of exposures, to get the detector. 

267 detector = inputExpList[0].getDetector() 

268 else: 

269 detector = None 

270 datasetPtc.updateMetadata(setDate=True, camera=camera, detector=detector) 

271 

272 return pipeBase.Struct( 

273 outputPtcDataset=datasetPtc, 

274 ) 

275 

276 def fitCovariancesAstier(self, dataset): 

277 """Fit measured flat covariances to full model in Astier+19. 

278 

279 Parameters 

280 ---------- 

281 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset` 

282 The dataset containing information such as the means, (co)variances, 

283 and exposure times. 

284 

285 Returns 

286 ------- 

287 dataset: `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset` 

288 This is the same dataset as the input paramter, however, it has been modified 

289 to include information such as the fit vectors and the fit parameters. See 

290 the class `PhotonTransferCurveDatase`. 

291 """ 

292 

293 covFits, covFitsNoB = fitDataFullCovariance(dataset) 

294 dataset = self.getOutputPtcDataCovAstier(dataset, covFits, covFitsNoB) 

295 

296 return dataset 

297 

298 def getOutputPtcDataCovAstier(self, dataset, covFits, covFitsNoB): 

299 """Get output data for PhotonTransferCurveCovAstierDataset from CovFit objects. 

300 

301 Parameters 

302 ---------- 

303 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset` 

304 The dataset containing information such as the means, variances and exposure times. 

305 covFits: `dict` 

306 Dictionary of CovFit objects, with amp names as keys. 

307 covFitsNoB : `dict` 

308 Dictionary of CovFit objects, with amp names as keys, and 'b=0' in Eq. 20 of Astier+19. 

309 

310 Returns 

311 ------- 

312 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset` 

313 This is the same dataset as the input paramter, however, it has been modified 

314 to include extra information such as the mask 1D array, gains, reoudout noise, measured signal, 

315 measured variance, modeled variance, a, and b coefficient matrices (see Astier+19) per amplifier. 

316 See the class `PhotonTransferCurveDatase`. 

317 """ 

318 assert(len(covFits) == len(covFitsNoB)) 

319 

320 for i, amp in enumerate(dataset.ampNames): 

321 lenInputTimes = len(dataset.rawExpTimes[amp]) 

322 # Not used when ptcFitType is 'FULLCOVARIANCE' 

323 dataset.ptcFitPars[amp] = [np.nan] 

324 dataset.ptcFitParsError[amp] = [np.nan] 

325 dataset.ptcFitChiSq[amp] = np.nan 

326 if amp in covFits: 

327 fit = covFits[amp] 

328 fitNoB = covFitsNoB[amp] 

329 # Save full covariances, covariances models, and their weights 

330 # dataset.expIdMask is already full 

331 dataset.covariances[amp] = fit.cov 

332 dataset.covariancesModel[amp] = fit.evalCovModel() 

333 dataset.covariancesSqrtWeights[amp] = fit.sqrtW 

334 dataset.aMatrix[amp] = fit.getA() 

335 dataset.bMatrix[amp] = fit.getB() 

336 dataset.covariancesModelNoB[amp] = fitNoB.evalCovModel() 

337 dataset.aMatrixNoB[amp] = fitNoB.getA() 

338 

339 (meanVecFinal, varVecFinal, varVecModel, 

340 wc, varMask) = fit.getFitData(0, 0, divideByMu=False) 

341 gain = fit.getGain() 

342 

343 dataset.gain[amp] = gain 

344 dataset.gainErr[amp] = fit.getGainErr() 

345 dataset.noise[amp] = np.sqrt(fit.getRon()) 

346 dataset.noiseErr[amp] = fit.getRonErr() 

347 dataset.finalVars[amp] = varVecFinal 

348 dataset.finalModelVars[amp] = varVecModel 

349 dataset.finalMeans[amp] = meanVecFinal 

350 

351 else: 

352 # Bad amp 

353 # Entries need to have proper dimensions so read/write with astropy.Table works. 

354 matrixSide = self.config.maximumRangeCovariancesAstier 

355 nanMatrix = np.full((matrixSide, matrixSide), np.nan) 

356 listNanMatrix = np.full((lenInputTimes, matrixSide, matrixSide), np.nan) 

357 

358 dataset.covariances[amp] = listNanMatrix 

359 dataset.covariancesModel[amp] = listNanMatrix 

360 dataset.covariancesSqrtWeights[amp] = listNanMatrix 

361 dataset.aMatrix[amp] = nanMatrix 

362 dataset.bMatrix[amp] = nanMatrix 

363 dataset.covariancesModelNoB[amp] = listNanMatrix 

364 dataset.aMatrixNoB[amp] = nanMatrix 

365 

366 dataset.expIdMask[amp] = np.repeat(np.nan, lenInputTimes) 

367 dataset.gain[amp] = np.nan 

368 dataset.gainErr[amp] = np.nan 

369 dataset.noise[amp] = np.nan 

370 dataset.noiseErr[amp] = np.nan 

371 dataset.finalVars[amp] = np.repeat(np.nan, lenInputTimes) 

372 dataset.finalModelVars[amp] = np.repeat(np.nan, lenInputTimes) 

373 dataset.finalMeans[amp] = np.repeat(np.nan, lenInputTimes) 

374 

375 return dataset 

376 

377 @staticmethod 

378 def _initialParsForPolynomial(order): 

379 assert(order >= 2) 

380 pars = np.zeros(order, dtype=np.float) 

381 pars[0] = 10 

382 pars[1] = 1 

383 pars[2:] = 0.0001 

384 return pars 

385 

386 @staticmethod 

387 def _boundsForPolynomial(initialPars, lowers=[], uppers=[]): 

388 if not len(lowers): 

389 lowers = [np.NINF for p in initialPars] 

390 if not len(uppers): 

391 uppers = [np.inf for p in initialPars] 

392 lowers[1] = 0 # no negative gains 

393 return (lowers, uppers) 

394 

395 @staticmethod 

396 def _boundsForAstier(initialPars, lowers=[], uppers=[]): 

397 if not len(lowers): 

398 lowers = [np.NINF for p in initialPars] 

399 if not len(uppers): 

400 uppers = [np.inf for p in initialPars] 

401 return (lowers, uppers) 

402 

403 @staticmethod 

404 def _getInitialGoodPoints(means, variances, maxDeviationPositive, maxDeviationNegative, 

405 minMeanRatioTest, minVarPivotSearch): 

406 """Return a boolean array to mask bad points. 

407 

408 Parameters 

409 ---------- 

410 means : `numpy.array` 

411 Input array with mean signal values. 

412 variances : `numpy.array` 

413 Input array with variances at each mean value. 

414 maxDeviationPositive : `float` 

415 Maximum deviation from being constant for the variance/mean 

416 ratio, in the positive direction. 

417 maxDeviationNegative : `float` 

418 Maximum deviation from being constant for the variance/mean 

419 ratio, in the negative direction. 

420 minMeanRatioTest : `float` 

421 Minimum signal value (in ADU) after which to start examining 

422 the ratios var/mean. 

423 minVarPivotSearch : `float` 

424 Minimum variance point (in ADU^2) after which the pivot point 

425 wher the variance starts decreasing should be sought. 

426 

427 Returns 

428 ------ 

429 goodPoints : `numpy.array` [`bool`] 

430 Boolean array to select good (`True`) and bad (`False`) 

431 points. 

432 

433 Notes 

434 ----- 

435 A linear function has a constant ratio, so find the median 

436 value of the ratios, and exclude the points that deviate 

437 from that by more than a factor of maxDeviationPositive/negative. 

438 Asymmetric deviations are supported as we expect the PTC to turn 

439 down as the flux increases, but sometimes it anomalously turns 

440 upwards just before turning over, which ruins the fits, so it 

441 is wise to be stricter about restricting positive outliers than 

442 negative ones. 

443 Too high and points that are so bad that fit will fail will be included 

444 Too low and the non-linear points will be excluded, biasing the NL fit. 

445 This function also masks points after the variance starts decreasing. 

446 """ 

447 

448 assert(len(means) == len(variances)) 

449 ratios = [b/a for (a, b) in zip(means, variances)] 

450 medianRatio = np.nanmedian(ratios) 

451 ratioDeviations = [0.0 if a < minMeanRatioTest else (r/medianRatio)-1 

452 for (a, r) in zip(means, ratios)] 

453 

454 # so that it doesn't matter if the deviation is expressed as positive or negative 

455 maxDeviationPositive = abs(maxDeviationPositive) 

456 maxDeviationNegative = -1. * abs(maxDeviationNegative) 

457 

458 goodPoints = np.array([True if (r < maxDeviationPositive and r > maxDeviationNegative) 

459 else False for r in ratioDeviations]) 

460 

461 # Eliminate points beyond which the variance decreases 

462 pivot = np.where(np.array(np.diff(variances)) < 0)[0] 

463 if len(pivot) > 0: 

464 # For small values, sometimes the variance decreases slightly 

465 # Only look when var > self.config.minVarPivotSearch 

466 pivot = [p for p in pivot if variances[p] > minVarPivotSearch] 

467 if len(pivot) > 0: 

468 pivot = np.min(pivot) 

469 goodPoints[pivot+1:len(goodPoints)] = False 

470 

471 return goodPoints 

472 

473 def _makeZeroSafe(self, array, substituteValue=1e-9): 

474 """""" 

475 array = np.array(array) 

476 nBad = Counter(np.ravel(array))[0] 

477 if nBad == 0: 

478 return array 

479 

480 index, = np.where(array == 0) 

481 if len(index): 

482 msg = f"Found {nBad} zeros in array at elements {index}" 

483 self.log.warn(msg) 

484 

485 array[index] = substituteValue 

486 

487 return array 

488 

489 def fitPtc(self, dataset): 

490 """Fit the photon transfer curve to a polynomial or to Astier+19 approximation. 

491 

492 Fit the photon transfer curve with either a polynomial of the order 

493 specified in the task config, or using the exponential approximation 

494 in Astier+19 (Eq. 16). 

495 

496 Sigma clipping is performed iteratively for the fit, as well as an 

497 initial clipping of data points that are more than 

498 config.initialNonLinearityExclusionThreshold away from lying on a 

499 straight line. This other step is necessary because the photon transfer 

500 curve turns over catastrophically at very high flux (because saturation 

501 drops the variance to ~0) and these far outliers cause the initial fit 

502 to fail, meaning the sigma cannot be calculated to perform the 

503 sigma-clipping. 

504 

505 Parameters 

506 ---------- 

507 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset` 

508 The dataset containing the means, variances and exposure times. 

509 

510 Returns 

511 ------- 

512 dataset: `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset` 

513 This is the same dataset as the input parameter, however, it has been modified 

514 to include information such as the fit vectors and the fit parameters. See 

515 the class `PhotonTransferCurveDatase`. 

516 

517 Raises 

518 ------ 

519 RuntimeError: 

520 Raises if dataset.ptcFitType is None or empty. 

521 """ 

522 if dataset.ptcFitType: 

523 ptcFitType = dataset.ptcFitType 

524 else: 

525 raise RuntimeError("ptcFitType is None of empty in PTC dataset.") 

526 matrixSide = self.config.maximumRangeCovariancesAstier 

527 nanMatrix = np.empty((matrixSide, matrixSide)) 

528 nanMatrix[:] = np.nan 

529 

530 for amp in dataset.ampNames: 

531 lenInputTimes = len(dataset.rawExpTimes[amp]) 

532 listNanMatrix = np.empty((lenInputTimes, matrixSide, matrixSide)) 

533 listNanMatrix[:] = np.nan 

534 

535 dataset.covariancesModel[amp] = listNanMatrix 

536 dataset.aMatrix[amp] = nanMatrix 

537 dataset.bMatrix[amp] = nanMatrix 

538 dataset.covariancesModelNoB[amp] = listNanMatrix 

539 dataset.aMatrixNoB[amp] = nanMatrix 

540 

541 def errFunc(p, x, y): 

542 return ptcFunc(p, x) - y 

543 

544 sigmaCutPtcOutliers = self.config.sigmaCutPtcOutliers 

545 maxIterationsPtcOutliers = self.config.maxIterationsPtcOutliers 

546 

547 for i, ampName in enumerate(dataset.ampNames): 

548 timeVecOriginal = np.ravel(np.array(dataset.rawExpTimes[ampName])) 

549 meanVecOriginal = np.ravel(np.array(dataset.rawMeans[ampName])) 

550 varVecOriginal = np.ravel(np.array(dataset.rawVars[ampName])) 

551 varVecOriginal = self._makeZeroSafe(varVecOriginal) 

552 

553 goodPoints = self._getInitialGoodPoints(meanVecOriginal, varVecOriginal, 

554 self.config.initialNonLinearityExclusionThresholdPositive, 

555 self.config.initialNonLinearityExclusionThresholdNegative, 

556 self.config.minMeanRatioTest, 

557 self.config.minVarPivotSearch) 

558 if not (goodPoints.any()): 

559 msg = (f"SERIOUS: All points in goodPoints: {goodPoints} are bad." 

560 f"Setting {ampName} to BAD.") 

561 self.log.warn(msg) 

562 # Fill entries with NaNs 

563 self.fillBadAmp(dataset, ptcFitType, ampName) 

564 continue 

565 

566 mask = goodPoints 

567 

568 if ptcFitType == 'EXPAPPROXIMATION': 

569 ptcFunc = funcAstier 

570 parsIniPtc = [-1e-9, 1.0, 10.] # a00, gain, noisei^2 

571 # lowers and uppers obtained from BOT data studies by C. Lage (UC Davis, 11/2020). 

572 bounds = self._boundsForAstier(parsIniPtc, lowers=[-1e-4, 0.5, -2000], 

573 uppers=[1e-4, 2.5, 2000]) 

574 if ptcFitType == 'POLYNOMIAL': 

575 ptcFunc = funcPolynomial 

576 parsIniPtc = self._initialParsForPolynomial(self.config.polynomialFitDegree + 1) 

577 bounds = self._boundsForPolynomial(parsIniPtc) 

578 

579 # Before bootstrap fit, do an iterative fit to get rid of outliers 

580 count = 1 

581 while count <= maxIterationsPtcOutliers: 

582 # Note that application of the mask actually shrinks the array 

583 # to size rather than setting elements to zero (as we want) so 

584 # always update mask itself and re-apply to the original data 

585 meanTempVec = meanVecOriginal[mask] 

586 varTempVec = varVecOriginal[mask] 

587 res = least_squares(errFunc, parsIniPtc, bounds=bounds, args=(meanTempVec, varTempVec)) 

588 pars = res.x 

589 

590 # change this to the original from the temp because the masks are ANDed 

591 # meaning once a point is masked it's always masked, and the masks must 

592 # always be the same length for broadcasting 

593 sigResids = (varVecOriginal - ptcFunc(pars, meanVecOriginal))/np.sqrt(varVecOriginal) 

594 newMask = np.array([True if np.abs(r) < sigmaCutPtcOutliers else False for r in sigResids]) 

595 mask = mask & newMask 

596 if not (mask.any() and newMask.any()): 

597 msg = (f"SERIOUS: All points in either mask: {mask} or newMask: {newMask} are bad. " 

598 f"Setting {ampName} to BAD.") 

599 self.log.warn(msg) 

600 # Fill entries with NaNs 

601 self.fillBadAmp(dataset, ptcFitType, ampName) 

602 break 

603 nDroppedTotal = Counter(mask)[False] 

604 self.log.debug(f"Iteration {count}: discarded {nDroppedTotal} points in total for {ampName}") 

605 count += 1 

606 # objects should never shrink 

607 assert (len(mask) == len(timeVecOriginal) == len(meanVecOriginal) == len(varVecOriginal)) 

608 if not (mask.any() and newMask.any()): 

609 continue 

610 dataset.expIdMask[ampName] = mask # store the final mask 

611 parsIniPtc = pars 

612 meanVecFinal = meanVecOriginal[mask] 

613 varVecFinal = varVecOriginal[mask] 

614 

615 if Counter(mask)[False] > 0: 

616 self.log.info((f"Number of points discarded in PTC of amplifier {ampName}:" 

617 f" {Counter(mask)[False]} out of {len(meanVecOriginal)}")) 

618 

619 if (len(meanVecFinal) < len(parsIniPtc)): 

620 msg = (f"SERIOUS: Not enough data points ({len(meanVecFinal)}) compared to the number of " 

621 f"parameters of the PTC model({len(parsIniPtc)}). Setting {ampName} to BAD.") 

622 self.log.warn(msg) 

623 # Fill entries with NaNs 

624 self.fillBadAmp(dataset, ptcFitType, ampName) 

625 continue 

626 # Fit the PTC 

627 if self.config.doFitBootstrap: 

628 parsFit, parsFitErr, reducedChiSqPtc = fitBootstrap(parsIniPtc, meanVecFinal, 

629 varVecFinal, ptcFunc, 

630 weightsY=1./np.sqrt(varVecFinal)) 

631 else: 

632 parsFit, parsFitErr, reducedChiSqPtc = fitLeastSq(parsIniPtc, meanVecFinal, 

633 varVecFinal, ptcFunc, 

634 weightsY=1./np.sqrt(varVecFinal)) 

635 dataset.ptcFitPars[ampName] = parsFit 

636 dataset.ptcFitParsError[ampName] = parsFitErr 

637 dataset.ptcFitChiSq[ampName] = reducedChiSqPtc 

638 # Masked variances (measured and modeled) and means. Need to pad the array so astropy.Table does 

639 # not crash (the mask may vary per amp). 

640 padLength = len(dataset.rawExpTimes[ampName]) - len(varVecFinal) 

641 dataset.finalVars[ampName] = np.pad(varVecFinal, (0, padLength), 'constant', 

642 constant_values=np.nan) 

643 dataset.finalModelVars[ampName] = np.pad(ptcFunc(parsFit, meanVecFinal), (0, padLength), 

644 'constant', constant_values=np.nan) 

645 dataset.finalMeans[ampName] = np.pad(meanVecFinal, (0, padLength), 'constant', 

646 constant_values=np.nan) 

647 if ptcFitType == 'EXPAPPROXIMATION': 

648 ptcGain = parsFit[1] 

649 ptcGainErr = parsFitErr[1] 

650 ptcNoise = np.sqrt(np.fabs(parsFit[2])) 

651 ptcNoiseErr = 0.5*(parsFitErr[2]/np.fabs(parsFit[2]))*np.sqrt(np.fabs(parsFit[2])) 

652 if ptcFitType == 'POLYNOMIAL': 

653 ptcGain = 1./parsFit[1] 

654 ptcGainErr = np.fabs(1./parsFit[1])*(parsFitErr[1]/parsFit[1]) 

655 ptcNoise = np.sqrt(np.fabs(parsFit[0]))*ptcGain 

656 ptcNoiseErr = (0.5*(parsFitErr[0]/np.fabs(parsFit[0]))*(np.sqrt(np.fabs(parsFit[0]))))*ptcGain 

657 dataset.gain[ampName] = ptcGain 

658 dataset.gainErr[ampName] = ptcGainErr 

659 dataset.noise[ampName] = ptcNoise 

660 dataset.noiseErr[ampName] = ptcNoiseErr 

661 

662 if not len(dataset.ptcFitType) == 0: 

663 dataset.ptcFitType = ptcFitType 

664 if len(dataset.badAmps) == 0: 

665 dataset.badAmps = np.repeat(np.nan, len(list(dataset.rawExpTimes.values())[0])) 

666 

667 return dataset 

668 

669 def fillBadAmp(self, dataset, ptcFitType, ampName): 

670 """Fill the dataset with NaNs if there are not enough good points. 

671 

672 Parameters 

673 ---------- 

674 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset` 

675 The dataset containing the means, variances and exposure times. 

676 ptcFitType : `str` 

677 Fit a 'POLYNOMIAL' (degree: 'polynomialFitDegree') or 

678 'EXPAPPROXIMATION' (Eq. 16 of Astier+19) to the PTC. 

679 ampName : `str` 

680 Amplifier name. 

681 """ 

682 dataset.badAmps.append(ampName) 

683 dataset.expIdMask[ampName] = np.repeat(False, len(dataset.rawExpTimes[ampName])) 

684 dataset.gain[ampName] = np.nan 

685 dataset.gainErr[ampName] = np.nan 

686 dataset.noise[ampName] = np.nan 

687 dataset.noiseErr[ampName] = np.nan 

688 dataset.ptcFitPars[ampName] = (np.repeat(np.nan, self.config.polynomialFitDegree + 1) if 

689 ptcFitType in ["POLYNOMIAL", ] else np.repeat(np.nan, 3)) 

690 dataset.ptcFitParsError[ampName] = (np.repeat(np.nan, self.config.polynomialFitDegree + 1) if 

691 ptcFitType in ["POLYNOMIAL", ] else np.repeat(np.nan, 3)) 

692 dataset.ptcFitChiSq[ampName] = np.nan 

693 dataset.finalVars[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName])) 

694 dataset.finalModelVars[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName])) 

695 dataset.finalMeans[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName])) 

696 

697 return