Coverage for python/lsst/cp/pipe/ptc/astierCovPtcUtils.py: 10%

Shortcuts on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

109 statements  

1# This file is part of cp_pipe. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import numpy as np 

23from .astierCovPtcFit import CovFit 

24 

25__all__ = ['CovFastFourierTransform'] 

26 

27 

28class CovFastFourierTransform: 

29 """A class to compute (via FFT) the nearby pixels correlation function. 

30 

31 Implements appendix of Astier+19. 

32 

33 Parameters 

34 ---------- 

35 diff : `numpy.array` 

36 Image where to calculate the covariances (e.g., the difference 

37 image of two flats). 

38 

39 w : `numpy.array` 

40 Weight image (mask): it should consist of 1's (good pixel) and 

41 0's (bad pixels). 

42 

43 fftShape : `tuple` 

44 2d-tuple with the shape of the FFT 

45 

46 maxRangeCov : `int` 

47 Maximum range for the covariances. 

48 """ 

49 

50 def __init__(self, diff, w, fftShape, maxRangeCov): 

51 # check that the zero padding implied by "fft_shape" 

52 # is large enough for the required correlation range 

53 assert(fftShape[0] > diff.shape[0]+maxRangeCov+1) 

54 assert(fftShape[1] > diff.shape[1]+maxRangeCov+1) 

55 # for some reason related to numpy.fft.rfftn, 

56 # the second dimension should be even, so 

57 if fftShape[1]%2 == 1: 

58 fftShape = (fftShape[0], fftShape[1]+1) 

59 tIm = np.fft.rfft2(diff*w, fftShape) 

60 tMask = np.fft.rfft2(w, fftShape) 

61 # sum of "squares" 

62 self.pCov = np.fft.irfft2(tIm*tIm.conjugate()) 

63 # sum of values 

64 self.pMean = np.fft.irfft2(tIm*tMask.conjugate()) 

65 # number of w!=0 pixels. 

66 self.pCount = np.fft.irfft2(tMask*tMask.conjugate()) 

67 

68 def cov(self, dx, dy): 

69 """Covariance for dx,dy averaged with dx,-dy if both non zero. 

70 

71 Implements appendix of Astier+19. 

72 

73 Parameters 

74 ---------- 

75 dx : `int` 

76 Lag in x 

77 

78 dy : `int 

79 Lag in y 

80 

81 Returns 

82 ------- 

83 0.5*(cov1+cov2) : `float` 

84 Covariance at (dx, dy) lag 

85 

86 npix1+npix2 : `int` 

87 Number of pixels used in covariance calculation. 

88 """ 

89 # compensate rounding errors 

90 nPix1 = int(round(self.pCount[dy, dx])) 

91 cov1 = self.pCov[dy, dx]/nPix1-self.pMean[dy, dx]*self.pMean[-dy, -dx]/(nPix1*nPix1) 

92 if (dx == 0 or dy == 0): 

93 return cov1, nPix1 

94 nPix2 = int(round(self.pCount[-dy, dx])) 

95 cov2 = self.pCov[-dy, dx]/nPix2-self.pMean[-dy, dx]*self.pMean[dy, -dx]/(nPix2*nPix2) 

96 return 0.5*(cov1+cov2), nPix1+nPix2 

97 

98 def reportCovFastFourierTransform(self, maxRange): 

99 """Produce a list of tuples with covariances. 

100 

101 Implements appendix of Astier+19. 

102 

103 Parameters 

104 ---------- 

105 maxRange : `int` 

106 Maximum range of covariances. 

107 

108 Returns 

109 ------- 

110 tupleVec : `list` 

111 List with covariance tuples. 

112 """ 

113 tupleVec = [] 

114 # (dy,dx) = (0,0) has to be first 

115 for dy in range(maxRange+1): 

116 for dx in range(maxRange+1): 

117 cov, npix = self.cov(dx, dy) 

118 if (dx == 0 and dy == 0): 

119 var = cov 

120 tupleVec.append((dx, dy, var, cov, npix)) 

121 return tupleVec 

122 

123 

124def computeCovDirect(diffImage, weightImage, maxRange): 

125 """Compute covariances of diffImage in real space. 

126 

127 For lags larger than ~25, it is slower than the FFT way. 

128 Taken from https://github.com/PierreAstier/bfptc/ 

129 

130 Parameters 

131 ---------- 

132 diffImage : `numpy.array` 

133 Image to compute the covariance of. 

134 

135 weightImage : `numpy.array` 

136 Weight image of diffImage (1's and 0's for good and bad 

137 pixels, respectively). 

138 

139 maxRange : `int` 

140 Last index of the covariance to be computed. 

141 

142 Returns 

143 ------- 

144 outList : `list` 

145 List with tuples of the form (dx, dy, var, cov, npix), where: 

146 dx : `int` 

147 Lag in x 

148 dy : `int` 

149 Lag in y 

150 var : `float` 

151 Variance at (dx, dy). 

152 cov : `float` 

153 Covariance at (dx, dy). 

154 nPix : `int` 

155 Number of pixel pairs used to evaluate var and cov. 

156 """ 

157 outList = [] 

158 var = 0 

159 # (dy,dx) = (0,0) has to be first 

160 for dy in range(maxRange + 1): 

161 for dx in range(0, maxRange + 1): 

162 if (dx*dy > 0): 

163 cov1, nPix1 = covDirectValue(diffImage, weightImage, dx, dy) 

164 cov2, nPix2 = covDirectValue(diffImage, weightImage, dx, -dy) 

165 cov = 0.5*(cov1 + cov2) 

166 nPix = nPix1 + nPix2 

167 else: 

168 cov, nPix = covDirectValue(diffImage, weightImage, dx, dy) 

169 if (dx == 0 and dy == 0): 

170 var = cov 

171 outList.append((dx, dy, var, cov, nPix)) 

172 

173 return outList 

174 

175 

176def covDirectValue(diffImage, weightImage, dx, dy): 

177 """Compute covariances of diffImage in real space at lag (dx, dy). 

178 

179 Taken from https://github.com/PierreAstier/bfptc/ (c.f., appendix 

180 of Astier+19). 

181 

182 Parameters 

183 ---------- 

184 diffImage : `numpy.array` 

185 Image to compute the covariance of. 

186 

187 weightImage : `numpy.array` 

188 Weight image of diffImage (1's and 0's for good and bad 

189 pixels, respectively). 

190 

191 dx : `int` 

192 Lag in x. 

193 

194 dy : `int` 

195 Lag in y. 

196 

197 Returns 

198 ------- 

199 cov : `float` 

200 Covariance at (dx, dy) 

201 

202 nPix : `int` 

203 Number of pixel pairs used to evaluate var and cov. 

204 """ 

205 (nCols, nRows) = diffImage.shape 

206 # switching both signs does not change anything: 

207 # it just swaps im1 and im2 below 

208 if (dx < 0): 

209 (dx, dy) = (-dx, -dy) 

210 # now, we have dx >0. We have to distinguish two cases 

211 # depending on the sign of dy 

212 if dy >= 0: 

213 im1 = diffImage[dy:, dx:] 

214 w1 = weightImage[dy:, dx:] 

215 im2 = diffImage[:nCols - dy, :nRows - dx] 

216 w2 = weightImage[:nCols - dy, :nRows - dx] 

217 else: 

218 im1 = diffImage[:nCols + dy, dx:] 

219 w1 = weightImage[:nCols + dy, dx:] 

220 im2 = diffImage[-dy:, :nRows - dx] 

221 w2 = weightImage[-dy:, :nRows - dx] 

222 # use the same mask for all 3 calculations 

223 wAll = w1*w2 

224 # do not use mean() because weightImage=0 pixels would then count 

225 nPix = wAll.sum() 

226 im1TimesW = im1*wAll 

227 s1 = im1TimesW.sum()/nPix 

228 s2 = (im2*wAll).sum()/nPix 

229 p = (im1TimesW*im2).sum()/nPix 

230 cov = p - s1*s2 

231 

232 return cov, nPix 

233 

234 

235def parseData(dataset): 

236 """ Returns a list of CovFit objects, indexed by amp number. 

237 

238 Params 

239 ------ 

240 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset` 

241 The PTC dataset containing the means, variances, and 

242 exposure times. 

243 

244 Returns 

245 ------- 

246 covFitDict : `dict` 

247 Dictionary with amps as keys, and CovFit objects as values. 

248 """ 

249 covFitDict = {} 

250 for ampName in dataset.ampNames: 

251 # If there is a bad amp, don't fit it 

252 if ampName in dataset.badAmps: 

253 continue 

254 maskAtAmp = dataset.expIdMask[ampName] 

255 muAtAmp = dataset.rawMeans[ampName] 

256 covAtAmp = dataset.covariances[ampName] 

257 covSqrtWeightsAtAmp = dataset.covariancesSqrtWeights[ampName] 

258 

259 c = CovFit(muAtAmp, covAtAmp, covSqrtWeightsAtAmp, dataset.covMatrixSide, maskAtAmp) 

260 cc = c.copy() 

261 cc.initFit() # allows to get a crude gain. 

262 covFitDict[ampName] = cc 

263 

264 return covFitDict 

265 

266 

267def fitDataFullCovariance(dataset): 

268 """Fit data to model in Astier+19 (Eq. 20). 

269 

270 Parameters 

271 ---------- 

272 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset` 

273 The dataset containing the means, (co)variances, and exposure times. 

274 

275 Returns 

276 ------- 

277 covFitDict : `dict` 

278 Dictionary of CovFit objects, with amp names as keys. 

279 

280 covFitNoBDict : `dict` 

281 Dictionary of CovFit objects, with amp names as keys (b=0 in 

282 Eq. 20 of Astier+19). 

283 

284 Notes 

285 ----- 

286 The parameters of the full model for C_ij(mu) ("C_ij" and "mu" in 

287 ADU^2 and ADU, respectively) in Astier+19 (Eq. 20) are: 

288 

289 "a" coefficients (r by r matrix), units: 1/e 

290 "b" coefficients (r by r matrix), units: 1/e 

291 noise matrix (r by r matrix), units: e^2 

292 gain, units: e/ADU 

293 

294 "b" appears in Eq. 20 only through the "ab" combination, which is 

295 defined in this code as "c=ab". 

296 """ 

297 covFitDict = parseData(dataset) 

298 covFitNoBDict = {} 

299 for ext, c in covFitDict.items(): 

300 c.fitFullModel() 

301 covFitNoBDict[ext] = c.copy() 

302 c.params['c'].release() 

303 c.fitFullModel() 

304 return covFitDict, covFitNoBDict 

305 

306 

307def getFitDataFromCovariances(i, j, mu, fullCov, fullCovModel, fullCovSqrtWeights, gain=1.0, 

308 divideByMu=False, returnMasked=False): 

309 """Get measured signal and covariance, cov model, weigths, and mask at 

310 covariance lag (i, j). 

311 

312 Parameters 

313 ---------- 

314 i : `int` 

315 Lag for covariance matrix. 

316 

317 j : `int` 

318 Lag for covariance matrix. 

319 

320 mu : `list` 

321 Mean signal values. 

322 

323 fullCov : `list` of `numpy.array` 

324 Measured covariance matrices at each mean signal level in mu. 

325 

326 fullCovSqrtWeights : `list` of `numpy.array` 

327 List of square root of measured covariances at each mean 

328 signal level in mu. 

329 

330 fullCovModel : `list` of `numpy.array` 

331 List of modeled covariances at each mean signal level in mu. 

332 

333 gain : `float`, optional 

334 Gain, in e-/ADU. If other than 1.0 (default), the returned 

335 quantities will be in electrons or powers of electrons. 

336 

337 divideByMu : `bool`, optional 

338 Divide returned covariance, model, and weights by the mean 

339 signal mu? 

340 

341 returnMasked : `bool`, optional 

342 Use mask (based on weights) in returned arrays (mu, 

343 covariance, and model)? 

344 

345 Returns 

346 ------- 

347 mu : `numpy.array` 

348 list of signal values at (i, j). 

349 

350 covariance : `numpy.array` 

351 Covariance at (i, j) at each mean signal mu value (fullCov[:, i, j]). 

352 

353 covarianceModel : `numpy.array` 

354 Covariance model at (i, j). 

355 

356 weights : `numpy.array` 

357 Weights at (i, j). 

358 

359 maskFromWeights : `numpy.array`, optional 

360 Boolean mask of the covariance at (i,j), where the weights 

361 differ from 0. 

362 

363 Notes 

364 ----- 

365 This function is a method of the `CovFit` class. 

366 """ 

367 mu = np.array(mu) 

368 fullCov = np.array(fullCov) 

369 fullCovModel = np.array(fullCovModel) 

370 fullCovSqrtWeights = np.array(fullCovSqrtWeights) 

371 covariance = fullCov[:, i, j]*(gain**2) 

372 covarianceModel = fullCovModel[:, i, j]*(gain**2) 

373 weights = fullCovSqrtWeights[:, i, j]/(gain**2) 

374 

375 maskFromWeights = weights != 0 

376 if returnMasked: 

377 weights = weights[maskFromWeights] 

378 covarianceModel = covarianceModel[maskFromWeights] 

379 mu = mu[maskFromWeights] 

380 covariance = covariance[maskFromWeights] 

381 

382 if divideByMu: 

383 covariance /= mu 

384 covarianceModel /= mu 

385 weights *= mu 

386 return mu, covariance, covarianceModel, weights, maskFromWeights