Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of cp_pipe. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import numpy as np 

23from .astierCovPtcFit import CovFit 

24 

25__all__ = ['CovFft'] 

26 

27 

28class CovFft: 

29 """A class to compute (via FFT) the nearby pixels correlation function. 

30 

31 Implements appendix of Astier+19. 

32 

33 Parameters 

34 ---------- 

35 diff: `numpy.array` 

36 Image where to calculate the covariances (e.g., the difference image of two flats). 

37 

38 w: `numpy.array` 

39 Weight image (mask): it should consist of 1's (good pixel) and 0's (bad pixels). 

40 

41 fftShape: `tuple` 

42 2d-tuple with the shape of the FFT 

43 

44 maxRangeCov: `int` 

45 Maximum range for the covariances. 

46 """ 

47 

48 def __init__(self, diff, w, fftShape, maxRangeCov): 

49 # check that the zero padding implied by "fft_shape" 

50 # is large enough for the required correlation range 

51 assert(fftShape[0] > diff.shape[0]+maxRangeCov+1) 

52 assert(fftShape[1] > diff.shape[1]+maxRangeCov+1) 

53 # for some reason related to numpy.fft.rfftn, 

54 # the second dimension should be even, so 

55 if fftShape[1]%2 == 1: 

56 fftShape = (fftShape[0], fftShape[1]+1) 

57 tIm = np.fft.rfft2(diff*w, fftShape) 

58 tMask = np.fft.rfft2(w, fftShape) 

59 # sum of "squares" 

60 self.pCov = np.fft.irfft2(tIm*tIm.conjugate()) 

61 # sum of values 

62 self.pMean = np.fft.irfft2(tIm*tMask.conjugate()) 

63 # number of w!=0 pixels. 

64 self.pCount = np.fft.irfft2(tMask*tMask.conjugate()) 

65 

66 def cov(self, dx, dy): 

67 """Covariance for dx,dy averaged with dx,-dy if both non zero. 

68 

69 Implements appendix of Astier+19. 

70 

71 Parameters 

72 ---------- 

73 dx: `int` 

74 Lag in x 

75 

76 dy: `int 

77 Lag in y 

78 

79 Returns 

80 ------- 

81 0.5*(cov1+cov2): `float` 

82 Covariance at (dx, dy) lag 

83 

84 npix1+npix2: `int` 

85 Number of pixels used in covariance calculation. 

86 """ 

87 # compensate rounding errors 

88 nPix1 = int(round(self.pCount[dy, dx])) 

89 cov1 = self.pCov[dy, dx]/nPix1-self.pMean[dy, dx]*self.pMean[-dy, -dx]/(nPix1*nPix1) 

90 if (dx == 0 or dy == 0): 

91 return cov1, nPix1 

92 nPix2 = int(round(self.pCount[-dy, dx])) 

93 cov2 = self.pCov[-dy, dx]/nPix2-self.pMean[-dy, dx]*self.pMean[dy, -dx]/(nPix2*nPix2) 

94 return 0.5*(cov1+cov2), nPix1+nPix2 

95 

96 def reportCovFft(self, maxRange): 

97 """Produce a list of tuples with covariances. 

98 

99 Implements appendix of Astier+19. 

100 

101 Parameters 

102 ---------- 

103 maxRange: `int` 

104 Maximum range of covariances. 

105 

106 Returns 

107 ------- 

108 tupleVec: `list` 

109 List with covariance tuples. 

110 """ 

111 tupleVec = [] 

112 # (dy,dx) = (0,0) has to be first 

113 for dy in range(maxRange+1): 

114 for dx in range(maxRange+1): 

115 cov, npix = self.cov(dx, dy) 

116 if (dx == 0 and dy == 0): 

117 var = cov 

118 tupleVec.append((dx, dy, var, cov, npix)) 

119 return tupleVec 

120 

121 

122def fftSize(s): 

123 """Calculate the size fof one dimension for the FFT""" 

124 x = int(np.log(s)/np.log(2.)) 

125 return int(2**(x+1)) 

126 

127 

128def computeCovDirect(diffImage, weightImage, maxRange): 

129 """Compute covariances of diffImage in real space. 

130 

131 For lags larger than ~25, it is slower than the FFT way. 

132 Taken from https://github.com/PierreAstier/bfptc/ 

133 

134 Parameters 

135 ---------- 

136 diffImage : `numpy.array` 

137 Image to compute the covariance of. 

138 

139 weightImage : `numpy.array` 

140 Weight image of diffImage (1's and 0's for good and bad pixels, respectively). 

141 

142 maxRange : `int` 

143 Last index of the covariance to be computed. 

144 

145 Returns 

146 ------- 

147 outList : `list` 

148 List with tuples of the form (dx, dy, var, cov, npix), where: 

149 dx : `int` 

150 Lag in x 

151 dy : `int` 

152 Lag in y 

153 var : `float` 

154 Variance at (dx, dy). 

155 cov : `float` 

156 Covariance at (dx, dy). 

157 nPix : `int` 

158 Number of pixel pairs used to evaluate var and cov. 

159 """ 

160 outList = [] 

161 var = 0 

162 # (dy,dx) = (0,0) has to be first 

163 for dy in range(maxRange + 1): 

164 for dx in range(0, maxRange + 1): 

165 if (dx*dy > 0): 

166 cov1, nPix1 = covDirectValue(diffImage, weightImage, dx, dy) 

167 cov2, nPix2 = covDirectValue(diffImage, weightImage, dx, -dy) 

168 cov = 0.5*(cov1 + cov2) 

169 nPix = nPix1 + nPix2 

170 else: 

171 cov, nPix = covDirectValue(diffImage, weightImage, dx, dy) 

172 if (dx == 0 and dy == 0): 

173 var = cov 

174 outList.append((dx, dy, var, cov, nPix)) 

175 

176 return outList 

177 

178 

179def covDirectValue(diffImage, weightImage, dx, dy): 

180 """Compute covariances of diffImage in real space at lag (dx, dy). 

181 

182 Taken from https://github.com/PierreAstier/bfptc/ (c.f., appendix of Astier+19). 

183 

184 Parameters 

185 ---------- 

186 diffImage : `numpy.array` 

187 Image to compute the covariance of. 

188 

189 weightImage : `numpy.array` 

190 Weight image of diffImage (1's and 0's for good and bad pixels, respectively). 

191 

192 dx : `int` 

193 Lag in x. 

194 

195 dy : `int` 

196 Lag in y. 

197 

198 Returns 

199 ------- 

200 cov : `float` 

201 Covariance at (dx, dy) 

202 

203 nPix : `int` 

204 Number of pixel pairs used to evaluate var and cov. 

205 """ 

206 (nCols, nRows) = diffImage.shape 

207 # switching both signs does not change anything: 

208 # it just swaps im1 and im2 below 

209 if (dx < 0): 

210 (dx, dy) = (-dx, -dy) 

211 # now, we have dx >0. We have to distinguish two cases 

212 # depending on the sign of dy 

213 if dy >= 0: 

214 im1 = diffImage[dy:, dx:] 

215 w1 = weightImage[dy:, dx:] 

216 im2 = diffImage[:nCols - dy, :nRows - dx] 

217 w2 = weightImage[:nCols - dy, :nRows - dx] 

218 else: 

219 im1 = diffImage[:nCols + dy, dx:] 

220 w1 = weightImage[:nCols + dy, dx:] 

221 im2 = diffImage[-dy:, :nRows - dx] 

222 w2 = weightImage[-dy:, :nRows - dx] 

223 # use the same mask for all 3 calculations 

224 wAll = w1*w2 

225 # do not use mean() because weightImage=0 pixels would then count 

226 nPix = wAll.sum() 

227 im1TimesW = im1*wAll 

228 s1 = im1TimesW.sum()/nPix 

229 s2 = (im2*wAll).sum()/nPix 

230 p = (im1TimesW*im2).sum()/nPix 

231 cov = p - s1*s2 

232 

233 return cov, nPix 

234 

235 

236class LoadParams: 

237 """ 

238 A class to prepare covariances for the PTC fit. 

239 

240 Parameters 

241 ---------- 

242 r: `int`, optional 

243 Maximum lag considered (e.g., to eliminate data beyond a separation "r": ignored in the fit). 

244 

245 subtractDistantValue: `bool`, optional 

246 Subtract a background to the measured covariances (mandatory for HSC flat pairs)? 

247 

248 start: `int`, optional 

249 Distance beyond which the subtractDistant model is fitted. 

250 

251 offsetDegree: `int` 

252 Polynomial degree for the subtraction model. 

253 

254 Notes 

255 ----- 

256 params = LoadParams(). "params" drives what happens in he fit. LoadParams provides default values. 

257 """ 

258 def __init__(self): 

259 self.r = 8 

260 self.subtractDistantValue = False 

261 self.start = 5 

262 self.offsetDegree = 1 

263 

264 

265def loadData(tupleName, params): 

266 """ Returns a list of CovFit objects, indexed by amp number. 

267 

268 Params 

269 ------ 

270 tupleName: `numpy.recarray` 

271 Recarray with rows with at least ( mu1, mu2, cov ,var, i, j, npix), where: 

272 mu1: mean value of flat1 

273 mu2: mean value of flat2 

274 cov: covariance value at lag (i, j) 

275 var: variance (covariance value at lag (0, 0)) 

276 i: lag dimension 

277 j: lag dimension 

278 npix: number of pixels used for covariance calculation. 

279 

280 params: `covAstierptcUtil.LoadParams` 

281 Object with values to drive the bahaviour of fits. 

282 

283 Returns 

284 ------- 

285 covFitList: `dict` 

286 Dictionary with amps as keys, and CovFit objects as values. 

287 """ 

288 

289 exts = np.array(np.unique(tupleName['ampName']), dtype=str) 

290 covFitList = {} 

291 for ext in exts: 

292 ntext = tupleName[tupleName['ampName'] == ext] 

293 if params.subtractDistantValue: 

294 c = CovFit(ntext, params.r) 

295 c.subtractDistantOffset(params.r, params.start, params.offsetDegree) 

296 else: 

297 c = CovFit(ntext, params.r) 

298 

299 cc = c.copy() 

300 cc.initFit() # allows to get a crude gain. 

301 covFitList[ext] = cc 

302 

303 return covFitList 

304 

305 

306def fitData(tupleName, r=8, nSigmaFullFit=5.5, maxIterFullFit=3): 

307 """Fit data to models in Astier+19. 

308 

309 Parameters 

310 ---------- 

311 tupleName: `numpy.recarray` 

312 Recarray with rows with at least ( mu1, mu2, cov ,var, i, j, npix), where: 

313 mu1: mean value of flat1 

314 mu2: mean value of flat2 

315 cov: covariance value at lag (i, j) 

316 var: variance (covariance value at lag (0, 0)) 

317 i: lag dimension 

318 j: lag dimension 

319 npix: number of pixels used for covariance calculation. 

320 

321 r: `int`, optional 

322 Maximum lag considered (e.g., to eliminate data beyond a separation "r": ignored in the fit). 

323 

324 nSigmaFullFit : `float`, optional 

325 Sigma cut to get rid of outliers in full model fit. 

326 

327 maxIterFullFit : `int`, optional 

328 Number of iterations for full model fit. 

329 

330 Returns 

331 ------- 

332 covFitList: `dict` 

333 Dictionary of CovFit objects, with amp names as keys. 

334 

335 covFitNoBList: `dict` 

336 Dictionary of CovFit objects, with amp names as keys (b=0 in Eq. 20 of Astier+19). 

337 

338 Notes 

339 ----- 

340 The parameters of the full model for C_ij(mu) ("C_ij" and "mu" in ADU^2 and ADU, respectively) 

341 in Astier+19 (Eq. 20) are: 

342 

343 "a" coefficients (r by r matrix), units: 1/e 

344 "b" coefficients (r by r matrix), units: 1/e 

345 noise matrix (r by r matrix), units: e^2 

346 gain, units: e/ADU 

347 

348 "b" appears in Eq. 20 only through the "ab" combination, which is defined in this code as "c=ab". 

349 """ 

350 

351 lparams = LoadParams() 

352 lparams.subtractDistantValue = False 

353 lparams.r = r 

354 covFitList = loadData(tupleName, lparams) 

355 covFitNoBList = {} # [None]*(exts[-1]+1) 

356 for ext, c in covFitList.items(): 

357 c.fitFullModel(nSigma=nSigmaFullFit, maxFitIter=maxIterFullFit) 

358 covFitNoBList[ext] = c.copy() 

359 c.params['c'].release() 

360 c.fitFullModel(nSigma=nSigmaFullFit, maxFitIter=maxIterFullFit) 

361 return covFitList, covFitNoBList 

362 

363 

364def getFitDataFromCovariances(i, j, mu, fullCov, fullCovModel, fullCovSqrtWeights, gain=1.0, 

365 divideByMu=False, returnMasked=False): 

366 """Get measured signal and covariance, cov model, weigths, and mask at covariance lag (i, j). 

367 

368 Parameters 

369 ---------- 

370 i : `int` 

371 Lag for covariance matrix. 

372 

373 j: `int` 

374 Lag for covariance matrix. 

375 

376 mu : `list` 

377 Mean signal values. 

378 

379 fullCov: `list` of `numpy.array` 

380 Measured covariance matrices at each mean signal level in mu. 

381 

382 fullCovSqrtWeights: `list` of `numpy.array` 

383 List of square root of measured covariances at each mean signal level in mu. 

384 

385 fullCovModel : `list` of `numpy.array` 

386 List of modeled covariances at each mean signal level in mu. 

387 

388 gain : `float`, optional 

389 Gain, in e-/ADU. If other than 1.0 (default), the returned quantities will be in 

390 electrons or powers of electrons. 

391 

392 divideByMu: `bool`, optional 

393 Divide returned covariance, model, and weights by the mean signal mu? 

394 

395 returnMasked : `bool`, optional 

396 Use mask (based on weights) in returned arrays (mu, covariance, and model)? 

397 

398 Returns 

399 ------- 

400 mu : `numpy.array` 

401 list of signal values at (i, j). 

402 

403 covariance : `numpy.array` 

404 Covariance at (i, j) at each mean signal mu value (fullCov[:, i, j]). 

405 

406 covarianceModel : `numpy.array` 

407 Covariance model at (i, j). 

408 

409 weights : `numpy.array` 

410 Weights at (i, j). 

411 

412 mask : `numpy.array`, optional 

413 Boolean mask of the covariance at (i,j). 

414 

415 Notes 

416 ----- 

417 This function is a method of the `CovFit` class. 

418 """ 

419 mu = np.array(mu) 

420 fullCov = np.array(fullCov) 

421 fullCovModel = np.array(fullCovModel) 

422 fullCovSqrtWeights = np.array(fullCovSqrtWeights) 

423 covariance = fullCov[:, i, j]*(gain**2) 

424 covarianceModel = fullCovModel[:, i, j]*(gain**2) 

425 weights = fullCovSqrtWeights[:, i, j]/(gain**2) 

426 

427 # select data used for the fit 

428 mask = weights != 0 

429 if returnMasked: 

430 weights = weights[mask] 

431 covarianceModel = covarianceModel[mask] 

432 mu = mu[mask] 

433 covariance = covariance[mask] 

434 

435 if divideByMu: 

436 covariance /= mu 

437 covarianceModel /= mu 

438 weights *= mu 

439 return mu, covariance, covarianceModel, weights, mask