Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of cp_pipe. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import numpy as np 

23from .astierCovPtcFit import CovFit 

24 

25__all__ = ['CovFft'] 

26 

27 

28class CovFft: 

29 """A class to compute (via FFT) the nearby pixels correlation function. 

30 

31 Implements appendix of Astier+19. 

32 

33 Parameters 

34 ---------- 

35 diff: `numpy.array` 

36 Image where to calculate the covariances (e.g., the difference image of two flats). 

37 

38 w: `numpy.array` 

39 Weight image (mask): it should consist of 1's (good pixel) and 0's (bad pixels). 

40 

41 fftShape: `tuple` 

42 2d-tuple with the shape of the FFT 

43 

44 maxRangeCov: `int` 

45 Maximum range for the covariances. 

46 """ 

47 

48 def __init__(self, diff, w, fftShape, maxRangeCov): 

49 # check that the zero padding implied by "fft_shape" 

50 # is large enough for the required correlation range 

51 assert(fftShape[0] > diff.shape[0]+maxRangeCov+1) 

52 assert(fftShape[1] > diff.shape[1]+maxRangeCov+1) 

53 # for some reason related to numpy.fft.rfftn, 

54 # the second dimension should be even, so 

55 if fftShape[1]%2 == 1: 

56 fftShape = (fftShape[0], fftShape[1]+1) 

57 tIm = np.fft.rfft2(diff*w, fftShape) 

58 tMask = np.fft.rfft2(w, fftShape) 

59 # sum of "squares" 

60 self.pCov = np.fft.irfft2(tIm*tIm.conjugate()) 

61 # sum of values 

62 self.pMean = np.fft.irfft2(tIm*tMask.conjugate()) 

63 # number of w!=0 pixels. 

64 self.pCount = np.fft.irfft2(tMask*tMask.conjugate()) 

65 

66 def cov(self, dx, dy): 

67 """Covariance for dx,dy averaged with dx,-dy if both non zero. 

68 

69 Implements appendix of Astier+19. 

70 

71 Parameters 

72 ---------- 

73 dx: `int` 

74 Lag in x 

75 

76 dy: `int 

77 Lag in y 

78 

79 Returns 

80 ------- 

81 0.5*(cov1+cov2): `float` 

82 Covariance at (dx, dy) lag 

83 

84 npix1+npix2: `int` 

85 Number of pixels used in covariance calculation. 

86 """ 

87 # compensate rounding errors 

88 nPix1 = int(round(self.pCount[dy, dx])) 

89 cov1 = self.pCov[dy, dx]/nPix1-self.pMean[dy, dx]*self.pMean[-dy, -dx]/(nPix1*nPix1) 

90 if (dx == 0 or dy == 0): 

91 return cov1, nPix1 

92 nPix2 = int(round(self.pCount[-dy, dx])) 

93 cov2 = self.pCov[-dy, dx]/nPix2-self.pMean[-dy, dx]*self.pMean[dy, -dx]/(nPix2*nPix2) 

94 return 0.5*(cov1+cov2), nPix1+nPix2 

95 

96 def reportCovFft(self, maxRange): 

97 """Produce a list of tuples with covariances. 

98 

99 Implements appendix of Astier+19. 

100 

101 Parameters 

102 ---------- 

103 maxRange: `int` 

104 Maximum range of covariances. 

105 

106 Returns 

107 ------- 

108 tupleVec: `list` 

109 List with covariance tuples. 

110 """ 

111 tupleVec = [] 

112 # (dy,dx) = (0,0) has to be first 

113 for dy in range(maxRange+1): 

114 for dx in range(maxRange+1): 

115 cov, npix = self.cov(dx, dy) 

116 if (dx == 0 and dy == 0): 

117 var = cov 

118 tupleVec.append((dx, dy, var, cov, npix)) 

119 return tupleVec 

120 

121 

122def fftSize(s): 

123 """Calculate the size fof one dimension for the FFT""" 

124 x = int(np.log(s)/np.log(2.)) 

125 return int(2**(x+1)) 

126 

127 

128def computeCovDirect(diffImage, weightImage, maxRange): 

129 """Compute covariances of diffImage in real space. 

130 

131 For lags larger than ~25, it is slower than the FFT way. 

132 Taken from https://github.com/PierreAstier/bfptc/ 

133 

134 Parameters 

135 ---------- 

136 diffImage : `numpy.array` 

137 Image to compute the covariance of. 

138 

139 weightImage : `numpy.array` 

140 Weight image of diffImage (1's and 0's for good and bad pixels, respectively). 

141 

142 maxRange : `int` 

143 Last index of the covariance to be computed. 

144 

145 Returns 

146 ------- 

147 outList : `list` 

148 List with tuples of the form (dx, dy, var, cov, npix), where: 

149 dx : `int` 

150 Lag in x 

151 dy : `int` 

152 Lag in y 

153 var : `float` 

154 Variance at (dx, dy). 

155 cov : `float` 

156 Covariance at (dx, dy). 

157 nPix : `int` 

158 Number of pixel pairs used to evaluate var and cov. 

159 """ 

160 outList = [] 

161 var = 0 

162 # (dy,dx) = (0,0) has to be first 

163 for dy in range(maxRange + 1): 

164 for dx in range(0, maxRange + 1): 

165 if (dx*dy > 0): 

166 cov1, nPix1 = covDirectValue(diffImage, weightImage, dx, dy) 

167 cov2, nPix2 = covDirectValue(diffImage, weightImage, dx, -dy) 

168 cov = 0.5*(cov1 + cov2) 

169 nPix = nPix1 + nPix2 

170 else: 

171 cov, nPix = covDirectValue(diffImage, weightImage, dx, dy) 

172 if (dx == 0 and dy == 0): 

173 var = cov 

174 outList.append((dx, dy, var, cov, nPix)) 

175 

176 return outList 

177 

178 

179def covDirectValue(diffImage, weightImage, dx, dy): 

180 """Compute covariances of diffImage in real space at lag (dx, dy). 

181 

182 Taken from https://github.com/PierreAstier/bfptc/ (c.f., appendix of Astier+19). 

183 

184 Parameters 

185 ---------- 

186 diffImage : `numpy.array` 

187 Image to compute the covariance of. 

188 

189 weightImage : `numpy.array` 

190 Weight image of diffImage (1's and 0's for good and bad pixels, respectively). 

191 

192 dx : `int` 

193 Lag in x. 

194 

195 dy : `int` 

196 Lag in y. 

197 

198 Returns 

199 ------- 

200 cov : `float` 

201 Covariance at (dx, dy) 

202 

203 nPix : `int` 

204 Number of pixel pairs used to evaluate var and cov. 

205 """ 

206 (nCols, nRows) = diffImage.shape 

207 # switching both signs does not change anything: 

208 # it just swaps im1 and im2 below 

209 if (dx < 0): 

210 (dx, dy) = (-dx, -dy) 

211 # now, we have dx >0. We have to distinguish two cases 

212 # depending on the sign of dy 

213 if dy >= 0: 

214 im1 = diffImage[dy:, dx:] 

215 w1 = weightImage[dy:, dx:] 

216 im2 = diffImage[:nCols - dy, :nRows - dx] 

217 w2 = weightImage[:nCols - dy, :nRows - dx] 

218 else: 

219 im1 = diffImage[:nCols + dy, dx:] 

220 w1 = weightImage[:nCols + dy, dx:] 

221 im2 = diffImage[-dy:, :nRows - dx] 

222 w2 = weightImage[-dy:, :nRows - dx] 

223 # use the same mask for all 3 calculations 

224 wAll = w1*w2 

225 # do not use mean() because weightImage=0 pixels would then count 

226 nPix = wAll.sum() 

227 im1TimesW = im1*wAll 

228 s1 = im1TimesW.sum()/nPix 

229 s2 = (im2*wAll).sum()/nPix 

230 p = (im1TimesW*im2).sum()/nPix 

231 cov = p - s1*s2 

232 

233 return cov, nPix 

234 

235 

236class LoadParams: 

237 """ 

238 A class to prepare covariances for the PTC fit. 

239 

240 Parameters 

241 ---------- 

242 r: `int`, optional 

243 Maximum lag considered (e.g., to eliminate data beyond a separation "r": ignored in the fit). 

244 

245 subtractDistantValue: `bool`, optional 

246 Subtract a background to the measured covariances (mandatory for HSC flat pairs)? 

247 

248 start: `int`, optional 

249 Distance beyond which the subtractDistant model is fitted. 

250 

251 offsetDegree: `int` 

252 Polynomial degree for the subtraction model. 

253 

254 Notes 

255 ----- 

256 params = LoadParams(). "params" drives what happens in he fit. LoadParams provides default values. 

257 """ 

258 def __init__(self): 

259 self.r = 8 

260 self.subtractDistantValue = False 

261 self.start = 5 

262 self.offsetDegree = 1 

263 

264 

265def loadData(tupleName, params, expIdMask): 

266 """ Returns a list of CovFit objects, indexed by amp number. 

267 

268 Params 

269 ------ 

270 tupleName: `numpy.recarray` 

271 Recarray with rows with at least ( mu1, mu2, cov ,var, i, j, npix), where: 

272 mu1: mean value of flat1 

273 mu2: mean value of flat2 

274 cov: covariance value at lag (i, j) 

275 var: variance (covariance value at lag (0, 0)) 

276 i: lag dimension 

277 j: lag dimension 

278 npix: number of pixels used for covariance calculation. 

279 

280 params: `covAstierptcUtil.LoadParams` 

281 Object with values to drive the bahaviour of fits. 

282 

283 expIdMask : `dict`, [`str`, `list`] 

284 Dictionary keyed by amp names containing the masked exposure pairs. 

285 

286 Returns 

287 ------- 

288 covFitList: `dict` 

289 Dictionary with amps as keys, and CovFit objects as values. 

290 """ 

291 

292 exts = np.array(np.unique(tupleName['ampName']), dtype=str) 

293 covFitList = {} 

294 for ext in exts: 

295 ntext = tupleName[tupleName['ampName'] == ext] 

296 maskExt = expIdMask[ext] 

297 if params.subtractDistantValue: 

298 c = CovFit(ntext, params.r, maskExt) 

299 c.subtractDistantOffset(params.r, params.start, params.offsetDegree) 

300 else: 

301 c = CovFit(ntext, params.r, maskExt) 

302 

303 cc = c.copy() 

304 cc.initFit() # allows to get a crude gain. 

305 covFitList[ext] = cc 

306 

307 return covFitList 

308 

309 

310def fitData(tupleName, expIdMask, r=8): 

311 """Fit data to models in Astier+19. 

312 

313 Parameters 

314 ---------- 

315 tupleName: `numpy.recarray` 

316 Recarray with rows with at least ( mu1, mu2, cov ,var, i, j, npix), where: 

317 mu1: mean value of flat1 

318 mu2: mean value of flat2 

319 cov: covariance value at lag (i, j) 

320 var: variance (covariance value at lag (0, 0)) 

321 i: lag dimension 

322 j: lag dimension 

323 npix: number of pixels used for covariance calculation. 

324 

325 expIdMask : `dict`, [`str`, `list`] 

326 Dictionary keyed by amp names containing the masked exposure pairs. 

327 

328 r: `int`, optional 

329 Maximum lag considered (e.g., to eliminate data beyond a separation "r": ignored in the fit). 

330 

331 Returns 

332 ------- 

333 covFitList: `dict` 

334 Dictionary of CovFit objects, with amp names as keys. 

335 

336 covFitNoBList: `dict` 

337 Dictionary of CovFit objects, with amp names as keys (b=0 in Eq. 20 of Astier+19). 

338 

339 Notes 

340 ----- 

341 The parameters of the full model for C_ij(mu) ("C_ij" and "mu" in ADU^2 and ADU, respectively) 

342 in Astier+19 (Eq. 20) are: 

343 

344 "a" coefficients (r by r matrix), units: 1/e 

345 "b" coefficients (r by r matrix), units: 1/e 

346 noise matrix (r by r matrix), units: e^2 

347 gain, units: e/ADU 

348 

349 "b" appears in Eq. 20 only through the "ab" combination, which is defined in this code as "c=ab". 

350 """ 

351 

352 lparams = LoadParams() 

353 lparams.subtractDistantValue = False 

354 lparams.r = r 

355 covFitList = loadData(tupleName, lparams, expIdMask=expIdMask) 

356 covFitNoBList = {} # [None]*(exts[-1]+1) 

357 for ext, c in covFitList.items(): 

358 c.fitFullModel() 

359 covFitNoBList[ext] = c.copy() 

360 c.params['c'].release() 

361 c.fitFullModel() 

362 return covFitList, covFitNoBList 

363 

364 

365def getFitDataFromCovariances(i, j, mu, fullCov, fullCovModel, fullCovSqrtWeights, gain=1.0, 

366 divideByMu=False, returnMasked=False): 

367 """Get measured signal and covariance, cov model, weigths, and mask at covariance lag (i, j). 

368 

369 Parameters 

370 ---------- 

371 i : `int` 

372 Lag for covariance matrix. 

373 

374 j: `int` 

375 Lag for covariance matrix. 

376 

377 mu : `list` 

378 Mean signal values. 

379 

380 fullCov: `list` of `numpy.array` 

381 Measured covariance matrices at each mean signal level in mu. 

382 

383 fullCovSqrtWeights: `list` of `numpy.array` 

384 List of square root of measured covariances at each mean signal level in mu. 

385 

386 fullCovModel : `list` of `numpy.array` 

387 List of modeled covariances at each mean signal level in mu. 

388 

389 gain : `float`, optional 

390 Gain, in e-/ADU. If other than 1.0 (default), the returned quantities will be in 

391 electrons or powers of electrons. 

392 

393 divideByMu: `bool`, optional 

394 Divide returned covariance, model, and weights by the mean signal mu? 

395 

396 returnMasked : `bool`, optional 

397 Use mask (based on weights) in returned arrays (mu, covariance, and model)? 

398 

399 Returns 

400 ------- 

401 mu : `numpy.array` 

402 list of signal values at (i, j). 

403 

404 covariance : `numpy.array` 

405 Covariance at (i, j) at each mean signal mu value (fullCov[:, i, j]). 

406 

407 covarianceModel : `numpy.array` 

408 Covariance model at (i, j). 

409 

410 weights : `numpy.array` 

411 Weights at (i, j). 

412 

413 maskFromWeights : `numpy.array`, optional 

414 Boolean mask of the covariance at (i,j), where the weights differ from 0. 

415 

416 Notes 

417 ----- 

418 This function is a method of the `CovFit` class. 

419 """ 

420 mu = np.array(mu) 

421 fullCov = np.array(fullCov) 

422 fullCovModel = np.array(fullCovModel) 

423 fullCovSqrtWeights = np.array(fullCovSqrtWeights) 

424 covariance = fullCov[:, i, j]*(gain**2) 

425 covarianceModel = fullCovModel[:, i, j]*(gain**2) 

426 weights = fullCovSqrtWeights[:, i, j]/(gain**2) 

427 

428 maskFromWeights = weights != 0 

429 if returnMasked: 

430 weights = weights[maskFromWeights] 

431 covarianceModel = covarianceModel[maskFromWeights] 

432 mu = mu[maskFromWeights] 

433 covariance = covariance[maskFromWeights] 

434 

435 if divideByMu: 

436 covariance /= mu 

437 covarianceModel /= mu 

438 weights *= mu 

439 return mu, covariance, covarianceModel, weights, maskFromWeights