Coverage for python/lsst/cp/pipe/utils.py: 10%

1# This file is part of cp_pipe. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21# 

22 

23__all__ = ['ddict2dict', 'CovFastFourierTransform'] 

24 

25import numpy as np 

26from scipy.optimize import leastsq 

27import numpy.polynomial.polynomial as poly 

28from scipy.stats import median_abs_deviation, norm 

29import logging 

30 

31from lsst.ip.isr import isrMock 

32import lsst.afw.image 

33import lsst.afw.math 

34 

35import galsim 

36 

37 

38def sigmaClipCorrection(nSigClip): 

39 """Correct measured sigma to account for clipping. 

40 

41 If we clip our input data and then measure sigma, then the 

42 measured sigma is smaller than the true value because real 

43 points beyond the clip threshold have been removed. This is a 

44 small (1.5% at nSigClip=3) effect when nSigClip >~ 3, but the 

45 default parameters for measure crosstalk use nSigClip=2.0. 

46 This causes the measured sigma to be about 15% smaller than 

47 real. This formula corrects the issue, for the symmetric case 

48 (upper clip threshold equal to lower clip threshold). 

49 

50 Parameters 

51 ---------- 

52 nSigClip : `float` 

53 Number of sigma the measurement was clipped by. 

54 

55 Returns 

56 ------- 

57 scaleFactor : `float` 

58 Scale factor to increase the measured sigma by. 

59 """ 

60 varFactor = 1.0 - (2 * nSigClip * norm.pdf(nSigClip)) / (norm.cdf(nSigClip) - norm.cdf(-nSigClip)) 

61 return 1.0 / np.sqrt(varFactor) 

62 

63 

64def calculateWeightedReducedChi2(measured, model, weightsMeasured, nData, nParsModel): 

65 """Calculate weighted reduced chi2. 

66 

67 Parameters 

68 ---------- 

69 measured : `list` 

70 List with measured data. 

71 model : `list` 

72 List with modeled data. 

73 weightsMeasured : `list` 

74 List with weights for the measured data. 

75 nData : `int` 

76 Number of data points. 

77 nParsModel : `int` 

78 Number of parameters in the model. 

79 

80 Returns 

81 ------- 

82 redWeightedChi2 : `float` 

83 Reduced weighted chi2. 

84 """ 

85 wRes = (measured - model)*weightsMeasured 

86 return ((wRes*wRes).sum())/(nData-nParsModel) 

87 

88 

89def makeMockFlats(expTime, gain=1.0, readNoiseElectrons=5, fluxElectrons=1000, 

90 randomSeedFlat1=1984, randomSeedFlat2=666, powerLawBfParams=[], 

91 expId1=0, expId2=1): 

92 """Create a pair or mock flats with isrMock. 

93 

94 Parameters 

95 ---------- 

96 expTime : `float` 

97 Exposure time of the flats. 

98 gain : `float`, optional 

99 Gain, in e/ADU. 

100 readNoiseElectrons : `float`, optional 

101 Read noise rms, in electrons. 

102 fluxElectrons : `float`, optional 

103 Flux of flats, in electrons per second. 

104 randomSeedFlat1 : `int`, optional 

105 Random seed for the normal distrubutions for the mean signal 

106 and noise (flat1). 

107 randomSeedFlat2 : `int`, optional 

108 Random seed for the normal distrubutions for the mean signal 

109 and noise (flat2). 

110 powerLawBfParams : `list`, optional 

111 Parameters for `galsim.cdmodel.PowerLawCD` to simulate the 

112 brightter-fatter effect. 

113 expId1 : `int`, optional 

114 Exposure ID for first flat. 

115 expId2 : `int`, optional 

116 Exposure ID for second flat. 

117 

118 Returns 

119 ------- 

120 flatExp1 : `lsst.afw.image.exposure.ExposureF` 

121 First exposure of flat field pair. 

122 flatExp2 : `lsst.afw.image.exposure.ExposureF` 

123 Second exposure of flat field pair. 

124 

125 Notes 

126 ----- 

127 The parameters of `galsim.cdmodel.PowerLawCD` are `n, r0, t0, rx, 

128 tx, r, t, alpha`. For more information about their meaning, see 

129 the Galsim documentation 

130 https://galsim-developers.github.io/GalSim/_build/html/_modules/galsim/cdmodel.html # noqa: W505 

131 and Gruen+15 (1501.02802). 

132 

133 Example: galsim.cdmodel.PowerLawCD(8, 1.1e-7, 1.1e-7, 1.0e-8, 

134 1.0e-8, 1.0e-9, 1.0e-9, 2.0) 

135 """ 

136 flatFlux = fluxElectrons # e/s 

137 flatMean = flatFlux*expTime # e 

138 readNoise = readNoiseElectrons # e 

139 

140 mockImageConfig = isrMock.IsrMock.ConfigClass() 

141 

142 mockImageConfig.flatDrop = 0.99999 

143 mockImageConfig.isTrimmed = True 

144 

145 flatExp1 = isrMock.FlatMock(config=mockImageConfig).run() 

146 flatExp2 = flatExp1.clone() 

147 (shapeY, shapeX) = flatExp1.getDimensions() 

148 flatWidth = np.sqrt(flatMean) 

149 

150 rng1 = np.random.RandomState(randomSeedFlat1) 

151 flatData1 = rng1.normal(flatMean, flatWidth, (shapeX, shapeY)) + rng1.normal(0.0, readNoise, 

152 (shapeX, shapeY)) 

153 rng2 = np.random.RandomState(randomSeedFlat2) 

154 flatData2 = rng2.normal(flatMean, flatWidth, (shapeX, shapeY)) + rng2.normal(0.0, readNoise, 

155 (shapeX, shapeY)) 

156 # Simulate BF with power law model in galsim 

157 if len(powerLawBfParams): 

158 if not len(powerLawBfParams) == 8: 

159 raise RuntimeError("Wrong number of parameters for `galsim.cdmodel.PowerLawCD`. " 

160 f"Expected 8; passed {len(powerLawBfParams)}.") 

161 cd = galsim.cdmodel.PowerLawCD(*powerLawBfParams) 

162 tempFlatData1 = galsim.Image(flatData1) 

163 temp2FlatData1 = cd.applyForward(tempFlatData1) 

164 

165 tempFlatData2 = galsim.Image(flatData2) 

166 temp2FlatData2 = cd.applyForward(tempFlatData2) 

167 

168 flatExp1.image.array[:] = temp2FlatData1.array/gain # ADU 

169 flatExp2.image.array[:] = temp2FlatData2.array/gain # ADU 

170 else: 

171 flatExp1.image.array[:] = flatData1/gain # ADU 

172 flatExp2.image.array[:] = flatData2/gain # ADU 

173 

174 visitInfoExp1 = lsst.afw.image.VisitInfo(exposureTime=expTime) 

175 visitInfoExp2 = lsst.afw.image.VisitInfo(exposureTime=expTime) 

176 

177 flatExp1.info.id = expId1 

178 flatExp1.getInfo().setVisitInfo(visitInfoExp1) 

179 flatExp2.info.id = expId2 

180 flatExp2.getInfo().setVisitInfo(visitInfoExp2) 

181 

182 return flatExp1, flatExp2 

183 

184 

185def irlsFit(initialParams, dataX, dataY, function, weightsY=None, weightType='Cauchy', scaleResidual=True): 

186 """Iteratively reweighted least squares fit. 

187 

188 This uses the `lsst.cp.pipe.utils.fitLeastSq`, but applies weights 

189 based on the Cauchy distribution by default. Other weight options 

190 are implemented. See e.g. Holland and Welsch, 1977, 

191 doi:10.1080/03610927708827533 

192 

193 Parameters 

194 ---------- 

195 initialParams : `list` [`float`] 

196 Starting parameters. 

197 dataX : `numpy.array`, (N,) 

198 Abscissa data. 

199 dataY : `numpy.array`, (N,) 

200 Ordinate data. 

201 function : callable 

202 Function to fit. 

203 weightsY : `numpy.array`, (N,) 

204 Weights to apply to the data. 

205 weightType : `str`, optional 

206 Type of weighting to use. One of Cauchy, Anderson, bisquare, 

207 box, Welsch, Huber, logistic, or Fair. 

208 scaleResidual : `bool`, optional 

209 If true, the residual is scaled by the sqrt of the Y values. 

210 

211 Returns 

212 ------- 

213 polyFit : `list` [`float`] 

214 Final best fit parameters. 

215 polyFitErr : `list` [`float`] 

216 Final errors on fit parameters. 

217 chiSq : `float` 

218 Reduced chi squared. 

219 weightsY : `list` [`float`] 

220 Final weights used for each point. 

221 

222 Raises 

223 ------ 

224 RuntimeError : 

225 Raised if an unknown weightType string is passed. 

226 """ 

227 if not weightsY: 

228 weightsY = np.ones_like(dataX) 

229 

230 polyFit, polyFitErr, chiSq = fitLeastSq(initialParams, dataX, dataY, function, weightsY=weightsY) 

231 for iteration in range(10): 

232 resid = np.abs(dataY - function(polyFit, dataX)) 

233 if scaleResidual: 

234 resid = resid / np.sqrt(dataY) 

235 if weightType == 'Cauchy': 

236 # Use Cauchy weighting. This is a soft weight. 

237 # At [2, 3, 5, 10] sigma, weights are [.59, .39, .19, .05]. 

238 Z = resid / 2.385 

239 weightsY = 1.0 / (1.0 + np.square(Z)) 

240 elif weightType == 'Anderson': 

241 # Anderson+1972 weighting. This is a hard weight. 

242 # At [2, 3, 5, 10] sigma, weights are [.67, .35, 0.0, 0.0]. 

243 Z = resid / (1.339 * np.pi) 

244 weightsY = np.where(Z < 1.0, np.sinc(Z), 0.0) 

245 elif weightType == 'bisquare': 

246 # Beaton and Tukey (1974) biweight. This is a hard weight. 

247 # At [2, 3, 5, 10] sigma, weights are [.81, .59, 0.0, 0.0]. 

248 Z = resid / 4.685 

249 weightsY = np.where(Z < 1.0, 1.0 - np.square(Z), 0.0) 

250 elif weightType == 'box': 

251 # Hinich and Talwar (1975). This is a hard weight. 

252 # At [2, 3, 5, 10] sigma, weights are [1.0, 0.0, 0.0, 0.0]. 

253 weightsY = np.where(resid < 2.795, 1.0, 0.0) 

254 elif weightType == 'Welsch': 

255 # Dennis and Welsch (1976). This is a hard weight. 

256 # At [2, 3, 5, 10] sigma, weights are [.64, .36, .06, 1e-5]. 

257 Z = resid / 2.985 

258 weightsY = np.exp(-1.0 * np.square(Z)) 

259 elif weightType == 'Huber': 

260 # Huber (1964) weighting. This is a soft weight. 

261 # At [2, 3, 5, 10] sigma, weights are [.67, .45, .27, .13]. 

262 Z = resid / 1.345 

263 weightsY = np.where(Z < 1.0, 1.0, 1 / Z) 

264 elif weightType == 'logistic': 

265 # Logistic weighting. This is a soft weight. 

266 # At [2, 3, 5, 10] sigma, weights are [.56, .40, .24, .12]. 

267 Z = resid / 1.205 

268 weightsY = np.tanh(Z) / Z 

269 elif weightType == 'Fair': 

270 # Fair (1974) weighting. This is a soft weight. 

271 # At [2, 3, 5, 10] sigma, weights are [.41, .32, .22, .12]. 

272 Z = resid / 1.4 

273 weightsY = (1.0 / (1.0 + (Z))) 

274 else: 

275 raise RuntimeError(f"Unknown weighting type: {weightType}") 

276 polyFit, polyFitErr, chiSq = fitLeastSq(initialParams, dataX, dataY, function, weightsY=weightsY) 

277 

278 return polyFit, polyFitErr, chiSq, weightsY 

279 

280 

281def fitLeastSq(initialParams, dataX, dataY, function, weightsY=None): 

282 """Do a fit and estimate the parameter errors using using 

283 scipy.optimize.leastq. 

284 

285 optimize.leastsq returns the fractional covariance matrix. To 

286 estimate the standard deviation of the fit parameters, multiply 

287 the entries of this matrix by the unweighted reduced chi squared 

288 and take the square root of the diagonal elements. 

289 

290 Parameters 

291 ---------- 

292 initialParams : `list` [`float`] 

293 initial values for fit parameters. For ptcFitType=POLYNOMIAL, 

294 its length determines the degree of the polynomial. 

295 dataX : `numpy.array`, (N,) 

296 Data in the abscissa axis. 

297 dataY : `numpy.array`, (N,) 

298 Data in the ordinate axis. 

299 function : callable object (function) 

300 Function to fit the data with. 

301 weightsY : `numpy.array`, (N,) 

302 Weights of the data in the ordinate axis. 

303 

304 Return 

305 ------ 

306 pFitSingleLeastSquares : `list` [`float`] 

307 List with fitted parameters. 

308 pErrSingleLeastSquares : `list` [`float`] 

309 List with errors for fitted parameters. 

310 

311 reducedChiSqSingleLeastSquares : `float` 

312 Reduced chi squared, unweighted if weightsY is not provided. 

313 """ 

314 if weightsY is None: 

315 weightsY = np.ones(len(dataX)) 

316 

317 def errFunc(p, x, y, weightsY=None): 

318 if weightsY is None: 

319 weightsY = np.ones(len(x)) 

320 return (function(p, x) - y)*weightsY 

321 

322 pFit, pCov, infoDict, errMessage, success = leastsq(errFunc, initialParams, 

323 args=(dataX, dataY, weightsY), full_output=1, 

324 epsfcn=0.0001) 

325 

326 if (len(dataY) > len(initialParams)) and pCov is not None: 

327 reducedChiSq = calculateWeightedReducedChi2(dataY, function(pFit, dataX), weightsY, len(dataY), 

328 len(initialParams)) 

329 pCov *= reducedChiSq 

330 else: 

331 pCov = np.zeros((len(initialParams), len(initialParams))) 

332 pCov[:, :] = np.nan 

333 reducedChiSq = np.nan 

334 

335 errorVec = [] 

336 for i in range(len(pFit)): 

337 errorVec.append(np.fabs(pCov[i][i])**0.5) 

338 

339 pFitSingleLeastSquares = pFit 

340 pErrSingleLeastSquares = np.array(errorVec) 

341 

342 return pFitSingleLeastSquares, pErrSingleLeastSquares, reducedChiSq 

343 

344 

345def fitBootstrap(initialParams, dataX, dataY, function, weightsY=None, confidenceSigma=1.): 

346 """Do a fit using least squares and bootstrap to estimate parameter errors. 

347 

348 The bootstrap error bars are calculated by fitting 100 random data sets. 

349 

350 Parameters 

351 ---------- 

352 initialParams : `list` [`float`] 

353 initial values for fit parameters. For ptcFitType=POLYNOMIAL, 

354 its length determines the degree of the polynomial. 

355 dataX : `numpy.array`, (N,) 

356 Data in the abscissa axis. 

357 dataY : `numpy.array`, (N,) 

358 Data in the ordinate axis. 

359 function : callable object (function) 

360 Function to fit the data with. 

361 weightsY : `numpy.array`, (N,), optional. 

362 Weights of the data in the ordinate axis. 

363 confidenceSigma : `float`, optional. 

364 Number of sigmas that determine confidence interval for the 

365 bootstrap errors. 

366 

367 Return 

368 ------ 

369 pFitBootstrap : `list` [`float`] 

370 List with fitted parameters. 

371 pErrBootstrap : `list` [`float`] 

372 List with errors for fitted parameters. 

373 reducedChiSqBootstrap : `float` 

374 Reduced chi squared, unweighted if weightsY is not provided. 

375 """ 

376 if weightsY is None: 

377 weightsY = np.ones(len(dataX)) 

378 

379 def errFunc(p, x, y, weightsY): 

380 if weightsY is None: 

381 weightsY = np.ones(len(x)) 

382 return (function(p, x) - y)*weightsY 

383 

384 # Fit first time 

385 pFit, _ = leastsq(errFunc, initialParams, args=(dataX, dataY, weightsY), full_output=0) 

386 

387 # Get the stdev of the residuals 

388 residuals = errFunc(pFit, dataX, dataY, weightsY) 

389 # 100 random data sets are generated and fitted 

390 pars = [] 

391 for i in range(100): 

392 randomDelta = np.random.normal(0., np.fabs(residuals), len(dataY)) 

393 randomDataY = dataY + randomDelta 

394 randomFit, _ = leastsq(errFunc, initialParams, 

395 args=(dataX, randomDataY, weightsY), full_output=0) 

396 pars.append(randomFit) 

397 pars = np.array(pars) 

398 meanPfit = np.mean(pars, 0) 

399 

400 # confidence interval for parameter estimates 

401 errPfit = confidenceSigma*np.std(pars, 0) 

402 pFitBootstrap = meanPfit 

403 pErrBootstrap = errPfit 

404 

405 reducedChiSq = calculateWeightedReducedChi2(dataY, function(pFitBootstrap, dataX), weightsY, len(dataY), 

406 len(initialParams)) 

407 return pFitBootstrap, pErrBootstrap, reducedChiSq 

408 

409 

410def funcPolynomial(pars, x): 

411 """Polynomial function definition 

412 Parameters 

413 ---------- 

414 params : `list` 

415 Polynomial coefficients. Its length determines the polynomial order. 

416 

417 x : `numpy.array`, (N,) 

418 Abscisa array. 

419 

420 Returns 

421 ------- 

422 y : `numpy.array`, (N,) 

423 Ordinate array after evaluating polynomial of order 

424 len(pars)-1 at `x`. 

425 """ 

426 return poly.polyval(x, [*pars]) 

427 

428 

429def funcAstier(pars, x): 

430 """Single brighter-fatter parameter model for PTC; Equation 16 of 

431 Astier+19. 

432 

433 Parameters 

434 ---------- 

435 params : `list` 

436 Parameters of the model: a00 (brightter-fatter), gain (e/ADU), 

437 and noise (e^2). 

438 x : `numpy.array`, (N,) 

439 Signal mu (ADU). 

440 

441 Returns 

442 ------- 

443 y : `numpy.array`, (N,) 

444 C_00 (variance) in ADU^2. 

445 """ 

446 a00, gain, noise = pars 

447 return 0.5/(a00*gain*gain)*(np.exp(2*a00*x*gain)-1) + noise/(gain*gain) # C_00 

448 

449 

450def arrangeFlatsByExpTime(exposureList, exposureIdList): 

451 """Arrange exposures by exposure time. 

452 

453 Parameters 

454 ---------- 

455 exposureList : `list` [`lsst.pipe.base.connections.DeferredDatasetRef`] 

456 Input list of exposure references. 

457 exposureIdList : `list` [`int`] 

458 List of exposure ids as obtained by dataId[`exposure`]. 

459 

460 Returns 

461 ------ 

462 flatsAtExpTime : `dict` [`float`, 

463 `list`[(`lsst.pipe.base.connections.DeferredDatasetRef`, 

464 `int`)]] 

465 Dictionary that groups references to flat-field exposures 

466 (and their IDs) that have the same exposure time (seconds). 

467 """ 

468 flatsAtExpTime = {} 

469 assert len(exposureList) == len(exposureIdList), "Different lengths for exp. list and exp. ID lists" 

470 for expRef, expId in zip(exposureList, exposureIdList): 

471 expTime = expRef.get(component='visitInfo').exposureTime 

472 listAtExpTime = flatsAtExpTime.setdefault(expTime, []) 

473 listAtExpTime.append((expRef, expId)) 

474 

475 return flatsAtExpTime 

476 

477 

478def arrangeFlatsByExpFlux(exposureList, exposureIdList, fluxKeyword): 

479 """Arrange exposures by exposure flux. 

480 

481 Parameters 

482 ---------- 

483 exposureList : `list` [`lsst.pipe.base.connections.DeferredDatasetRef`] 

484 Input list of exposure references. 

485 exposureIdList : `list` [`int`] 

486 List of exposure ids as obtained by dataId[`exposure`]. 

487 fluxKeyword : `str` 

488 Header keyword that contains the flux per exposure. 

489 

490 Returns 

491 ------- 

492 flatsAtFlux : `dict` [`float`, 

493 `list`[(`lsst.pipe.base.connections.DeferredDatasetRef`, 

494 `int`)]] 

495 Dictionary that groups references to flat-field exposures 

496 (and their IDs) that have the same flux. 

497 """ 

498 flatsAtExpFlux = {} 

499 assert len(exposureList) == len(exposureIdList), "Different lengths for exp. list and exp. ID lists" 

500 for expRef, expId in zip(exposureList, exposureIdList): 

501 # Get flux from header, assuming it is in the metadata. 

502 expFlux = expRef.get().getMetadata()[fluxKeyword] 

503 listAtExpFlux = flatsAtExpFlux.setdefault(expFlux, []) 

504 listAtExpFlux.append((expRef, expId)) 

505 

506 return flatsAtExpFlux 

507 

508 

509def arrangeFlatsByExpId(exposureList, exposureIdList): 

510 """Arrange exposures by exposure ID. 

511 

512 There is no guarantee that this will properly group exposures, but 

513 allows a sequence of flats that have different illumination 

514 (despite having the same exposure time) to be processed. 

515 

516 Parameters 

517 ---------- 

518 exposureList : `list`[`lsst.pipe.base.connections.DeferredDatasetRef`] 

519 Input list of exposure references. 

520 exposureIdList : `list`[`int`] 

521 List of exposure ids as obtained by dataId[`exposure`]. 

522 

523 Returns 

524 ------ 

525 flatsAtExpId : `dict` [`float`, 

526 `list`[(`lsst.pipe.base.connections.DeferredDatasetRef`, 

527 `int`)]] 

528 Dictionary that groups references to flat-field exposures (and their 

529 IDs) sequentially by their exposure id. 

530 

531 Notes 

532 ----- 

533 

534 This algorithm sorts the input exposure references by their exposure 

535 id, and then assigns each pair of exposure references (exp_j, exp_{j+1}) 

536 to pair k, such that 2*k = j, where j is the python index of one of the 

537 exposure references (starting from zero). By checking for the IndexError 

538 while appending, we can ensure that there will only ever be fully 

539 populated pairs. 

540 """ 

541 flatsAtExpId = {} 

542 assert len(exposureList) == len(exposureIdList), "Different lengths for exp. list and exp. ID lists" 

543 # Sort exposures by expIds, which are in the second list `exposureIdList`. 

544 sortedExposures = sorted(zip(exposureList, exposureIdList), key=lambda pair: pair[1]) 

545 

546 for jPair, expTuple in enumerate(sortedExposures): 

547 if (jPair + 1) % 2: 

548 kPair = jPair // 2 

549 listAtExpId = flatsAtExpId.setdefault(kPair, []) 

550 try: 

551 listAtExpId.append(expTuple) 

552 listAtExpId.append(sortedExposures[jPair + 1]) 

553 except IndexError: 

554 pass 

555 

556 return flatsAtExpId 

557 

558 

559class CovFastFourierTransform: 

560 """A class to compute (via FFT) the nearby pixels correlation function. 

561 

562 Implements appendix of Astier+19. 

563 

564 Parameters 

565 ---------- 

566 diff : `numpy.array` 

567 Image where to calculate the covariances (e.g., the difference 

568 image of two flats). 

569 w : `numpy.array` 

570 Weight image (mask): it should consist of 1's (good pixel) and 

571 0's (bad pixels). 

572 fftShape : `tuple` 

573 2d-tuple with the shape of the FFT 

574 maxRangeCov : `int` 

575 Maximum range for the covariances. 

576 """ 

577 

578 def __init__(self, diff, w, fftShape, maxRangeCov): 

579 # check that the zero padding implied by "fft_shape" 

580 # is large enough for the required correlation range 

581 assert(fftShape[0] > diff.shape[0]+maxRangeCov+1) 

582 assert(fftShape[1] > diff.shape[1]+maxRangeCov+1) 

583 # for some reason related to numpy.fft.rfftn, 

584 # the second dimension should be even, so 

585 if fftShape[1]%2 == 1: 

586 fftShape = (fftShape[0], fftShape[1]+1) 

587 tIm = np.fft.rfft2(diff*w, fftShape) 

588 tMask = np.fft.rfft2(w, fftShape) 

589 # sum of "squares" 

590 self.pCov = np.fft.irfft2(tIm*tIm.conjugate()) 

591 # sum of values 

592 self.pMean = np.fft.irfft2(tIm*tMask.conjugate()) 

593 # number of w!=0 pixels. 

594 self.pCount = np.fft.irfft2(tMask*tMask.conjugate()) 

595 

596 def cov(self, dx, dy): 

597 """Covariance for dx,dy averaged with dx,-dy if both non zero. 

598 

599 Implements appendix of Astier+19. 

600 

601 Parameters 

602 ---------- 

603 dx : `int` 

604 Lag in x 

605 dy : `int` 

606 Lag in y 

607 

608 Returns 

609 ------- 

610 0.5*(cov1+cov2) : `float` 

611 Covariance at (dx, dy) lag 

612 npix1+npix2 : `int` 

613 Number of pixels used in covariance calculation. 

614 

615 Raises 

616 ------ 

617 ValueError if number of pixels for a given lag is 0. 

618 """ 

619 # compensate rounding errors 

620 nPix1 = int(round(self.pCount[dy, dx])) 

621 if nPix1 == 0: 

622 raise ValueError(f"Could not compute covariance term {dy}, {dx}, as there are no good pixels.") 

623 cov1 = self.pCov[dy, dx]/nPix1-self.pMean[dy, dx]*self.pMean[-dy, -dx]/(nPix1*nPix1) 

624 if (dx == 0 or dy == 0): 

625 return cov1, nPix1 

626 nPix2 = int(round(self.pCount[-dy, dx])) 

627 if nPix2 == 0: 

628 raise ValueError("Could not compute covariance term {dy}, {dx} as there are no good pixels.") 

629 cov2 = self.pCov[-dy, dx]/nPix2-self.pMean[-dy, dx]*self.pMean[dy, -dx]/(nPix2*nPix2) 

630 return 0.5*(cov1+cov2), nPix1+nPix2 

631 

632 def reportCovFastFourierTransform(self, maxRange): 

633 """Produce a list of tuples with covariances. 

634 

635 Implements appendix of Astier+19. 

636 

637 Parameters 

638 ---------- 

639 maxRange : `int` 

640 Maximum range of covariances. 

641 

642 Returns 

643 ------- 

644 tupleVec : `list` 

645 List with covariance tuples. 

646 """ 

647 tupleVec = [] 

648 # (dy,dx) = (0,0) has to be first 

649 for dy in range(maxRange+1): 

650 for dx in range(maxRange+1): 

651 cov, npix = self.cov(dx, dy) 

652 if (dx == 0 and dy == 0): 

653 var = cov 

654 tupleVec.append((dx, dy, var, cov, npix)) 

655 return tupleVec 

656 

657 

658def getFitDataFromCovariances(i, j, mu, fullCov, fullCovModel, fullCovSqrtWeights, gain=1.0, 

659 divideByMu=False, returnMasked=False): 

660 """Get measured signal and covariance, cov model, weigths, and mask at 

661 covariance lag (i, j). 

662 

663 Parameters 

664 ---------- 

665 i : `int` 

666 Lag for covariance matrix. 

667 j : `int` 

668 Lag for covariance matrix. 

669 mu : `list` 

670 Mean signal values. 

671 fullCov : `list` of `numpy.array` 

672 Measured covariance matrices at each mean signal level in mu. 

673 fullCovSqrtWeights : `list` of `numpy.array` 

674 List of square root of measured covariances at each mean 

675 signal level in mu. 

676 fullCovModel : `list` of `numpy.array` 

677 List of modeled covariances at each mean signal level in mu. 

678 gain : `float`, optional 

679 Gain, in e-/ADU. If other than 1.0 (default), the returned 

680 quantities will be in electrons or powers of electrons. 

681 divideByMu : `bool`, optional 

682 Divide returned covariance, model, and weights by the mean 

683 signal mu? 

684 returnMasked : `bool`, optional 

685 Use mask (based on weights) in returned arrays (mu, 

686 covariance, and model)? 

687 

688 Returns 

689 ------- 

690 mu : `numpy.array` 

691 list of signal values at (i, j). 

692 covariance : `numpy.array` 

693 Covariance at (i, j) at each mean signal mu value (fullCov[:, i, j]). 

694 covarianceModel : `numpy.array` 

695 Covariance model at (i, j). 

696 weights : `numpy.array` 

697 Weights at (i, j). 

698 maskFromWeights : `numpy.array`, optional 

699 Boolean mask of the covariance at (i,j), where the weights 

700 differ from 0. 

701 """ 

702 mu = np.array(mu) 

703 fullCov = np.array(fullCov) 

704 fullCovModel = np.array(fullCovModel) 

705 fullCovSqrtWeights = np.array(fullCovSqrtWeights) 

706 covariance = fullCov[:, i, j]*(gain**2) 

707 covarianceModel = fullCovModel[:, i, j]*(gain**2) 

708 weights = fullCovSqrtWeights[:, i, j]/(gain**2) 

709 

710 maskFromWeights = weights != 0 

711 if returnMasked: 

712 weights = weights[maskFromWeights] 

713 covarianceModel = covarianceModel[maskFromWeights] 

714 mu = mu[maskFromWeights] 

715 covariance = covariance[maskFromWeights] 

716 

717 if divideByMu: 

718 covariance /= mu 

719 covarianceModel /= mu 

720 weights *= mu 

721 return mu, covariance, covarianceModel, weights, maskFromWeights 

722 

723 

724def symmetrize(inputArray): 

725 """ Copy array over 4 quadrants prior to convolution. 

726 

727 Parameters 

728 ---------- 

729 inputarray : `numpy.array` 

730 Input array to symmetrize. 

731 

732 Returns 

733 ------- 

734 aSym : `numpy.array` 

735 Symmetrized array. 

736 """ 

737 targetShape = list(inputArray.shape) 

738 r1, r2 = inputArray.shape[-1], inputArray.shape[-2] 

739 targetShape[-1] = 2*r1-1 

740 targetShape[-2] = 2*r2-1 

741 aSym = np.ndarray(tuple(targetShape)) 

742 aSym[..., r2-1:, r1-1:] = inputArray 

743 aSym[..., r2-1:, r1-1::-1] = inputArray 

744 aSym[..., r2-1::-1, r1-1::-1] = inputArray 

745 aSym[..., r2-1::-1, r1-1:] = inputArray 

746 

747 return aSym 

748 

749 

750def ddict2dict(d): 

751 """Convert nested default dictionaries to regular dictionaries. 

752 

753 This is needed to prevent yaml persistence issues. 

754 

755 Parameters 

756 ---------- 

757 d : `defaultdict` 

758 A possibly nested set of `defaultdict`. 

759 

760 Returns 

761 ------- 

762 dict : `dict` 

763 A possibly nested set of `dict`. 

764 """ 

765 for k, v in d.items(): 

766 if isinstance(v, dict): 

767 d[k] = ddict2dict(v) 

768 return dict(d) 

769 

770 

771class AstierSplineLinearityFitter: 

772 """Class to fit the Astier spline linearity model. 

773 

774 This is a spline fit with photodiode data based on a model 

775 from Pierre Astier, referenced in June 2023 from 

776 https://me.lsst.eu/astier/bot/7224D/model_nonlin.py 

777 

778 This model fits a spline with (optional) nuisance parameters 

779 to allow for different linearity coefficients with different 

780 photodiode settings. The minimization is a least-squares 

781 fit with the residual of 

782 Sum[(S(mu_i) + mu_i)/(k_j * D_i) - 1]**2, where S(mu_i) is 

783 an Akima Spline function of mu_i, the observed flat-pair 

784 mean; D_j is the photo-diode measurement corresponding to 

785 that flat-pair; and k_j is a constant of proportionality 

786 which is over index j as it is allowed to 

787 be different based on different photodiode settings (e.g. 

788 CCOBCURR). 

789 

790 The fit has additional constraints to ensure that the spline 

791 goes through the (0, 0) point, as well as a normalization 

792 condition so that the average of the spline over the full 

793 range is 0. The normalization ensures that the spline only 

794 fits deviations from linearity, rather than the linear 

795 function itself which is degenerate with the gain. 

796 

797 Parameters 

798 ---------- 

799 nodes : `np.ndarray` (N,) 

800 Array of spline node locations. 

801 grouping_values : `np.ndarray` (M,) 

802 Array of values to group values for different proportionality 

803 constants (e.g. CCOBCURR). 

804 pd : `np.ndarray` (M,) 

805 Array of photodiode measurements. 

806 mu : `np.ndarray` (M,) 

807 Array of flat mean values. 

808 mask : `np.ndarray` (M,), optional 

809 Input mask (True is good point, False is bad point). 

810 log : `logging.logger`, optional 

811 Logger object to use for logging. 

812 """ 

813 def __init__(self, nodes, grouping_values, pd, mu, mask=None, log=None): 

814 self._pd = pd 

815 self._mu = mu 

816 self._grouping_values = grouping_values 

817 self.log = log if log else logging.getLogger(__name__) 

818 

819 self._nodes = nodes 

820 if nodes[0] != 0.0: 

821 raise ValueError("First node must be 0.0") 

822 if not np.all(np.diff(nodes) > 0): 

823 raise ValueError("Nodes must be sorted with no repeats.") 

824 

825 # Check if sorted (raise otherwise) 

826 if not np.all(np.diff(self._grouping_values) >= 0): 

827 raise ValueError("Grouping values must be sorted.") 

828 

829 _, uindex, ucounts = np.unique(self._grouping_values, return_index=True, return_counts=True) 

830 self.ngroup = len(uindex) 

831 

832 self.group_indices = [] 

833 for i in range(self.ngroup): 

834 self.group_indices.append(np.arange(uindex[i], uindex[i] + ucounts[i])) 

835 

836 # Values to regularize spline fit. 

837 self._x_regularize = np.linspace(self._mu.min(), self._mu.max(), 100) 

838 

839 # Outlier weight values. Will be 1 (in) or 0 (out). 

840 self._w = np.ones(len(self._pd)) 

841 

842 if mask is not None: 

843 self._w[~mask] = 0.0 

844 

845 def estimate_p0(self): 

846 """Estimate initial fit parameters. 

847 

848 Returns 

849 ------- 

850 p0 : `np.ndarray` 

851 Parameter array, with spline values (one for each node) followed 

852 by proportionality constants (one for each group). 

853 """ 

854 npt = len(self._nodes) + self.ngroup 

855 p0 = np.zeros(npt) 

856 

857 # Do a simple linear fit and set all the constants to this. 

858 linfit = np.polyfit(self._pd, self._mu, 1) 

859 p0[-self.ngroup:] = linfit[0] 

860 

861 # Look at the residuals... 

862 ratio_model = self.compute_ratio_model( 

863 self._nodes, 

864 self.group_indices, 

865 p0, 

866 self._pd, 

867 self._mu, 

868 ) 

869 # ...and adjust the linear parameters accordingly. 

870 p0[-self.ngroup:] *= np.median(ratio_model) 

871 

872 # Re-compute the residuals. 

873 ratio_model2 = self.compute_ratio_model( 

874 self._nodes, 

875 self.group_indices, 

876 p0, 

877 self._pd, 

878 self._mu, 

879 ) 

880 

881 # And compute a first guess of the spline nodes. 

882 bins = np.searchsorted(self._nodes, self._mu) 

883 tot_arr = np.zeros(len(self._nodes)) 

884 n_arr = np.zeros(len(self._nodes), dtype=int) 

885 np.add.at(tot_arr, bins, ratio_model2) 

886 np.add.at(n_arr, bins, 1) 

887 

888 ratio = np.ones(len(self._nodes)) 

889 ratio[n_arr > 0] = tot_arr[n_arr > 0]/n_arr[n_arr > 0] 

890 ratio[0] = 1.0 

891 p0[0: len(self._nodes)] = (ratio - 1) * self._nodes 

892 

893 return p0 

894 

895 @staticmethod 

896 def compute_ratio_model(nodes, group_indices, pars, pd, mu, return_spline=False): 

897 """Compute the ratio model values. 

898 

899 Parameters 

900 ---------- 

901 nodes : `np.ndarray` (M,) 

902 Array of node positions. 

903 group_indices : `list` [`np.ndarray`] 

904 List of group indices, one array for each group. 

905 pars : `np.ndarray` 

906 Parameter array, with spline values (one for each node) followed 

907 by proportionality constants (one for each group.) 

908 pd : `np.ndarray` (N,) 

909 Array of photodiode measurements. 

910 mu : `np.ndarray` (N,) 

911 Array of flat means. 

912 return_spline : `bool`, optional 

913 Return the spline interpolation as well as the model ratios? 

914 

915 Returns 

916 ------- 

917 ratio_models : `np.ndarray` (N,) 

918 Model ratio, (mu_i - S(mu_i))/(k_j * D_i)