Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of cp_pipe. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import numpy as np 

23from .astierCovPtcFit import CovFit 

24 

25__all__ = ['CovFastFourierTransform'] 

26 

27 

28class CovFastFourierTransform: 

29 """A class to compute (via FFT) the nearby pixels correlation function. 

30 

31 Implements appendix of Astier+19. 

32 

33 Parameters 

34 ---------- 

35 diff: `numpy.array` 

36 Image where to calculate the covariances (e.g., the difference image of two flats). 

37 

38 w: `numpy.array` 

39 Weight image (mask): it should consist of 1's (good pixel) and 0's (bad pixels). 

40 

41 fftShape: `tuple` 

42 2d-tuple with the shape of the FFT 

43 

44 maxRangeCov: `int` 

45 Maximum range for the covariances. 

46 """ 

47 

48 def __init__(self, diff, w, fftShape, maxRangeCov): 

49 # check that the zero padding implied by "fft_shape" 

50 # is large enough for the required correlation range 

51 assert(fftShape[0] > diff.shape[0]+maxRangeCov+1) 

52 assert(fftShape[1] > diff.shape[1]+maxRangeCov+1) 

53 # for some reason related to numpy.fft.rfftn, 

54 # the second dimension should be even, so 

55 if fftShape[1]%2 == 1: 

56 fftShape = (fftShape[0], fftShape[1]+1) 

57 tIm = np.fft.rfft2(diff*w, fftShape) 

58 tMask = np.fft.rfft2(w, fftShape) 

59 # sum of "squares" 

60 self.pCov = np.fft.irfft2(tIm*tIm.conjugate()) 

61 # sum of values 

62 self.pMean = np.fft.irfft2(tIm*tMask.conjugate()) 

63 # number of w!=0 pixels. 

64 self.pCount = np.fft.irfft2(tMask*tMask.conjugate()) 

65 

66 def cov(self, dx, dy): 

67 """Covariance for dx,dy averaged with dx,-dy if both non zero. 

68 

69 Implements appendix of Astier+19. 

70 

71 Parameters 

72 ---------- 

73 dx: `int` 

74 Lag in x 

75 

76 dy: `int 

77 Lag in y 

78 

79 Returns 

80 ------- 

81 0.5*(cov1+cov2): `float` 

82 Covariance at (dx, dy) lag 

83 

84 npix1+npix2: `int` 

85 Number of pixels used in covariance calculation. 

86 """ 

87 # compensate rounding errors 

88 nPix1 = int(round(self.pCount[dy, dx])) 

89 cov1 = self.pCov[dy, dx]/nPix1-self.pMean[dy, dx]*self.pMean[-dy, -dx]/(nPix1*nPix1) 

90 if (dx == 0 or dy == 0): 

91 return cov1, nPix1 

92 nPix2 = int(round(self.pCount[-dy, dx])) 

93 cov2 = self.pCov[-dy, dx]/nPix2-self.pMean[-dy, dx]*self.pMean[dy, -dx]/(nPix2*nPix2) 

94 return 0.5*(cov1+cov2), nPix1+nPix2 

95 

96 def reportCovFastFourierTransform(self, maxRange): 

97 """Produce a list of tuples with covariances. 

98 

99 Implements appendix of Astier+19. 

100 

101 Parameters 

102 ---------- 

103 maxRange: `int` 

104 Maximum range of covariances. 

105 

106 Returns 

107 ------- 

108 tupleVec: `list` 

109 List with covariance tuples. 

110 """ 

111 tupleVec = [] 

112 # (dy,dx) = (0,0) has to be first 

113 for dy in range(maxRange+1): 

114 for dx in range(maxRange+1): 

115 cov, npix = self.cov(dx, dy) 

116 if (dx == 0 and dy == 0): 

117 var = cov 

118 tupleVec.append((dx, dy, var, cov, npix)) 

119 return tupleVec 

120 

121 

122def computeCovDirect(diffImage, weightImage, maxRange): 

123 """Compute covariances of diffImage in real space. 

124 

125 For lags larger than ~25, it is slower than the FFT way. 

126 Taken from https://github.com/PierreAstier/bfptc/ 

127 

128 Parameters 

129 ---------- 

130 diffImage : `numpy.array` 

131 Image to compute the covariance of. 

132 

133 weightImage : `numpy.array` 

134 Weight image of diffImage (1's and 0's for good and bad pixels, respectively). 

135 

136 maxRange : `int` 

137 Last index of the covariance to be computed. 

138 

139 Returns 

140 ------- 

141 outList : `list` 

142 List with tuples of the form (dx, dy, var, cov, npix), where: 

143 dx : `int` 

144 Lag in x 

145 dy : `int` 

146 Lag in y 

147 var : `float` 

148 Variance at (dx, dy). 

149 cov : `float` 

150 Covariance at (dx, dy). 

151 nPix : `int` 

152 Number of pixel pairs used to evaluate var and cov. 

153 """ 

154 outList = [] 

155 var = 0 

156 # (dy,dx) = (0,0) has to be first 

157 for dy in range(maxRange + 1): 

158 for dx in range(0, maxRange + 1): 

159 if (dx*dy > 0): 

160 cov1, nPix1 = covDirectValue(diffImage, weightImage, dx, dy) 

161 cov2, nPix2 = covDirectValue(diffImage, weightImage, dx, -dy) 

162 cov = 0.5*(cov1 + cov2) 

163 nPix = nPix1 + nPix2 

164 else: 

165 cov, nPix = covDirectValue(diffImage, weightImage, dx, dy) 

166 if (dx == 0 and dy == 0): 

167 var = cov 

168 outList.append((dx, dy, var, cov, nPix)) 

169 

170 return outList 

171 

172 

173def covDirectValue(diffImage, weightImage, dx, dy): 

174 """Compute covariances of diffImage in real space at lag (dx, dy). 

175 

176 Taken from https://github.com/PierreAstier/bfptc/ (c.f., appendix of Astier+19). 

177 

178 Parameters 

179 ---------- 

180 diffImage : `numpy.array` 

181 Image to compute the covariance of. 

182 

183 weightImage : `numpy.array` 

184 Weight image of diffImage (1's and 0's for good and bad pixels, respectively). 

185 

186 dx : `int` 

187 Lag in x. 

188 

189 dy : `int` 

190 Lag in y. 

191 

192 Returns 

193 ------- 

194 cov : `float` 

195 Covariance at (dx, dy) 

196 

197 nPix : `int` 

198 Number of pixel pairs used to evaluate var and cov. 

199 """ 

200 (nCols, nRows) = diffImage.shape 

201 # switching both signs does not change anything: 

202 # it just swaps im1 and im2 below 

203 if (dx < 0): 

204 (dx, dy) = (-dx, -dy) 

205 # now, we have dx >0. We have to distinguish two cases 

206 # depending on the sign of dy 

207 if dy >= 0: 

208 im1 = diffImage[dy:, dx:] 

209 w1 = weightImage[dy:, dx:] 

210 im2 = diffImage[:nCols - dy, :nRows - dx] 

211 w2 = weightImage[:nCols - dy, :nRows - dx] 

212 else: 

213 im1 = diffImage[:nCols + dy, dx:] 

214 w1 = weightImage[:nCols + dy, dx:] 

215 im2 = diffImage[-dy:, :nRows - dx] 

216 w2 = weightImage[-dy:, :nRows - dx] 

217 # use the same mask for all 3 calculations 

218 wAll = w1*w2 

219 # do not use mean() because weightImage=0 pixels would then count 

220 nPix = wAll.sum() 

221 im1TimesW = im1*wAll 

222 s1 = im1TimesW.sum()/nPix 

223 s2 = (im2*wAll).sum()/nPix 

224 p = (im1TimesW*im2).sum()/nPix 

225 cov = p - s1*s2 

226 

227 return cov, nPix 

228 

229 

230def parseData(dataset): 

231 """ Returns a list of CovFit objects, indexed by amp number. 

232 

233 Params 

234 ------ 

235 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset` 

236 The PTC dataset containing the means, variances, and 

237 exposure times. 

238 

239 Returns 

240 ------- 

241 covFitDict: `dict` 

242 Dictionary with amps as keys, and CovFit objects as values. 

243 """ 

244 

245 covFitDict = {} 

246 for ampName in dataset.ampNames: 

247 # If there is a bad amp, don't fit it 

248 if ampName in dataset.badAmps: 

249 continue 

250 maskAtAmp = dataset.expIdMask[ampName] 

251 muAtAmp = dataset.rawMeans[ampName] 

252 covAtAmp = dataset.covariances[ampName] 

253 covSqrtWeightsAtAmp = dataset.covariancesSqrtWeights[ampName] 

254 

255 c = CovFit(muAtAmp, covAtAmp, covSqrtWeightsAtAmp, dataset.covMatrixSide, maskAtAmp) 

256 cc = c.copy() 

257 cc.initFit() # allows to get a crude gain. 

258 covFitDict[ampName] = cc 

259 

260 return covFitDict 

261 

262 

263def fitDataFullCovariance(dataset): 

264 """Fit data to model in Astier+19 (Eq. 20). 

265 

266 Parameters 

267 ---------- 

268 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset` 

269 The dataset containing the means, (co)variances, and exposure times. 

270 

271 Returns 

272 ------- 

273 covFitDict: `dict` 

274 Dictionary of CovFit objects, with amp names as keys. 

275 

276 covFitNoBDict: `dict` 

277 Dictionary of CovFit objects, with amp names as keys (b=0 in Eq. 20 of Astier+19). 

278 

279 Notes 

280 ----- 

281 The parameters of the full model for C_ij(mu) ("C_ij" and "mu" in ADU^2 and ADU, respectively) 

282 in Astier+19 (Eq. 20) are: 

283 

284 "a" coefficients (r by r matrix), units: 1/e 

285 "b" coefficients (r by r matrix), units: 1/e 

286 noise matrix (r by r matrix), units: e^2 

287 gain, units: e/ADU 

288 

289 "b" appears in Eq. 20 only through the "ab" combination, which is defined in this code as "c=ab". 

290 """ 

291 

292 covFitDict = parseData(dataset) 

293 covFitNoBDict = {} 

294 for ext, c in covFitDict.items(): 

295 c.fitFullModel() 

296 covFitNoBDict[ext] = c.copy() 

297 c.params['c'].release() 

298 c.fitFullModel() 

299 return covFitDict, covFitNoBDict 

300 

301 

302def getFitDataFromCovariances(i, j, mu, fullCov, fullCovModel, fullCovSqrtWeights, gain=1.0, 

303 divideByMu=False, returnMasked=False): 

304 """Get measured signal and covariance, cov model, weigths, and mask at covariance lag (i, j). 

305 

306 Parameters 

307 ---------- 

308 i : `int` 

309 Lag for covariance matrix. 

310 

311 j: `int` 

312 Lag for covariance matrix. 

313 

314 mu : `list` 

315 Mean signal values. 

316 

317 fullCov: `list` of `numpy.array` 

318 Measured covariance matrices at each mean signal level in mu. 

319 

320 fullCovSqrtWeights: `list` of `numpy.array` 

321 List of square root of measured covariances at each mean signal level in mu. 

322 

323 fullCovModel : `list` of `numpy.array` 

324 List of modeled covariances at each mean signal level in mu. 

325 

326 gain : `float`, optional 

327 Gain, in e-/ADU. If other than 1.0 (default), the returned quantities will be in 

328 electrons or powers of electrons. 

329 

330 divideByMu: `bool`, optional 

331 Divide returned covariance, model, and weights by the mean signal mu? 

332 

333 returnMasked : `bool`, optional 

334 Use mask (based on weights) in returned arrays (mu, covariance, and model)? 

335 

336 Returns 

337 ------- 

338 mu : `numpy.array` 

339 list of signal values at (i, j). 

340 

341 covariance : `numpy.array` 

342 Covariance at (i, j) at each mean signal mu value (fullCov[:, i, j]). 

343 

344 covarianceModel : `numpy.array` 

345 Covariance model at (i, j). 

346 

347 weights : `numpy.array` 

348 Weights at (i, j). 

349 

350 maskFromWeights : `numpy.array`, optional 

351 Boolean mask of the covariance at (i,j), where the weights differ from 0. 

352 

353 Notes 

354 ----- 

355 This function is a method of the `CovFit` class. 

356 """ 

357 mu = np.array(mu) 

358 fullCov = np.array(fullCov) 

359 fullCovModel = np.array(fullCovModel) 

360 fullCovSqrtWeights = np.array(fullCovSqrtWeights) 

361 covariance = fullCov[:, i, j]*(gain**2) 

362 covarianceModel = fullCovModel[:, i, j]*(gain**2) 

363 weights = fullCovSqrtWeights[:, i, j]/(gain**2) 

364 

365 maskFromWeights = weights != 0 

366 if returnMasked: 

367 weights = weights[maskFromWeights] 

368 covarianceModel = covarianceModel[maskFromWeights] 

369 mu = mu[maskFromWeights] 

370 covariance = covariance[maskFromWeights] 

371 

372 if divideByMu: 

373 covariance /= mu 

374 covarianceModel /= mu 

375 weights *= mu 

376 return mu, covariance, covarianceModel, weights, maskFromWeights