Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of cp_pipe. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import numpy as np 

23from .astierCovPtcFit import CovFit 

24 

25__all__ = ['CovFft'] 

26 

27 

28class CovFft: 

29 """A class to compute (via FFT) the nearby pixels correlation function. 

30 

31 Implements appendix of Astier+19. 

32 

33 Parameters 

34 ---------- 

35 diff: `numpy.array` 

36 Image where to calculate the covariances (e.g., the difference image of two flats). 

37 

38 w: `numpy.array` 

39 Weight image (mask): it should consist of 1's (good pixel) and 0's (bad pixels). 

40 

41 fftShape: `tuple` 

42 2d-tuple with the shape of the FFT 

43 

44 maxRangeCov: `int` 

45 Maximum range for the covariances. 

46 """ 

47 

48 def __init__(self, diff, w, fftShape, maxRangeCov): 

49 # check that the zero padding implied by "fft_shape" 

50 # is large enough for the required correlation range 

51 assert(fftShape[0] > diff.shape[0]+maxRangeCov+1) 

52 assert(fftShape[1] > diff.shape[1]+maxRangeCov+1) 

53 # for some reason related to numpy.fft.rfftn, 

54 # the second dimension should be even, so 

55 if fftShape[1]%2 == 1: 

56 fftShape = (fftShape[0], fftShape[1]+1) 

57 tIm = np.fft.rfft2(diff*w, fftShape) 

58 tMask = np.fft.rfft2(w, fftShape) 

59 # sum of "squares" 

60 self.pCov = np.fft.irfft2(tIm*tIm.conjugate()) 

61 # sum of values 

62 self.pMean = np.fft.irfft2(tIm*tMask.conjugate()) 

63 # number of w!=0 pixels. 

64 self.pCount = np.fft.irfft2(tMask*tMask.conjugate()) 

65 

66 def cov(self, dx, dy): 

67 """Covariance for dx,dy averaged with dx,-dy if both non zero. 

68 

69 Implements appendix of Astier+19. 

70 

71 Parameters 

72 ---------- 

73 dx: `int` 

74 Lag in x 

75 

76 dy: `int 

77 Lag in y 

78 

79 Returns 

80 ------- 

81 0.5*(cov1+cov2): `float` 

82 Covariance at (dx, dy) lag 

83 

84 npix1+npix2: `int` 

85 Number of pixels used in covariance calculation. 

86 """ 

87 # compensate rounding errors 

88 nPix1 = int(round(self.pCount[dy, dx])) 

89 cov1 = self.pCov[dy, dx]/nPix1-self.pMean[dy, dx]*self.pMean[-dy, -dx]/(nPix1*nPix1) 

90 if (dx == 0 or dy == 0): 

91 return cov1, nPix1 

92 nPix2 = int(round(self.pCount[-dy, dx])) 

93 cov2 = self.pCov[-dy, dx]/nPix2-self.pMean[-dy, dx]*self.pMean[dy, -dx]/(nPix2*nPix2) 

94 return 0.5*(cov1+cov2), nPix1+nPix2 

95 

96 def reportCovFft(self, maxRange): 

97 """Produce a list of tuples with covariances. 

98 

99 Implements appendix of Astier+19. 

100 

101 Parameters 

102 ---------- 

103 maxRange: `int` 

104 Maximum range of covariances. 

105 

106 Returns 

107 ------- 

108 tupleVec: `list` 

109 List with covariance tuples. 

110 """ 

111 tupleVec = [] 

112 # (dy,dx) = (0,0) has to be first 

113 for dy in range(maxRange+1): 

114 for dx in range(maxRange+1): 

115 cov, npix = self.cov(dx, dy) 

116 if (dx == 0 and dy == 0): 

117 var = cov 

118 tupleVec.append((dx, dy, var, cov, npix)) 

119 return tupleVec 

120 

121 

122def fftSize(s): 

123 """Calculate the size fof one dimension for the FFT""" 

124 x = int(np.log(s)/np.log(2.)) 

125 return int(2**(x+1)) 

126 

127 

128def computeCovDirect(diffImage, weightImage, maxRange): 

129 """Compute covariances of diffImage in real space. 

130 

131 For lags larger than ~25, it is slower than the FFT way. 

132 Taken from https://github.com/PierreAstier/bfptc/ 

133 

134 Parameters 

135 ---------- 

136 diffImage : `numpy.array` 

137 Image to compute the covariance of. 

138 

139 weightImage : `numpy.array` 

140 Weight image of diffImage (1's and 0's for good and bad pixels, respectively). 

141 

142 maxRange : `int` 

143 Last index of the covariance to be computed. 

144 

145 Returns 

146 ------- 

147 outList : `list` 

148 List with tuples of the form (dx, dy, var, cov, npix), where: 

149 dx : `int` 

150 Lag in x 

151 dy : `int` 

152 Lag in y 

153 var : `float` 

154 Variance at (dx, dy). 

155 cov : `float` 

156 Covariance at (dx, dy). 

157 nPix : `int` 

158 Number of pixel pairs used to evaluate var and cov. 

159 """ 

160 outList = [] 

161 var = 0 

162 # (dy,dx) = (0,0) has to be first 

163 for dy in range(maxRange + 1): 

164 for dx in range(0, maxRange + 1): 

165 if (dx*dy > 0): 

166 cov1, nPix1 = covDirectValue(diffImage, weightImage, dx, dy) 

167 cov2, nPix2 = covDirectValue(diffImage, weightImage, dx, -dy) 

168 cov = 0.5*(cov1 + cov2) 

169 nPix = nPix1 + nPix2 

170 else: 

171 cov, nPix = covDirectValue(diffImage, weightImage, dx, dy) 

172 if (dx == 0 and dy == 0): 

173 var = cov 

174 outList.append((dx, dy, var, cov, nPix)) 

175 

176 return outList 

177 

178 

179def covDirectValue(diffImage, weightImage, dx, dy): 

180 """Compute covariances of diffImage in real space at lag (dx, dy). 

181 

182 Taken from https://github.com/PierreAstier/bfptc/ (c.f., appendix of Astier+19). 

183 

184 Parameters 

185 ---------- 

186 diffImage : `numpy.array` 

187 Image to compute the covariance of. 

188 

189 weightImage : `numpy.array` 

190 Weight image of diffImage (1's and 0's for good and bad pixels, respectively). 

191 

192 dx : `int` 

193 Lag in x. 

194 

195 dy : `int` 

196 Lag in y. 

197 

198 Returns 

199 ------- 

200 cov : `float` 

201 Covariance at (dx, dy) 

202 

203 nPix : `int` 

204 Number of pixel pairs used to evaluate var and cov. 

205 """ 

206 (nCols, nRows) = diffImage.shape 

207 # switching both signs does not change anything: 

208 # it just swaps im1 and im2 below 

209 if (dx < 0): 

210 (dx, dy) = (-dx, -dy) 

211 # now, we have dx >0. We have to distinguish two cases 

212 # depending on the sign of dy 

213 if dy >= 0: 

214 im1 = diffImage[dy:, dx:] 

215 w1 = weightImage[dy:, dx:] 

216 im2 = diffImage[:nCols - dy, :nRows - dx] 

217 w2 = weightImage[:nCols - dy, :nRows - dx] 

218 else: 

219 im1 = diffImage[:nCols + dy, dx:] 

220 w1 = weightImage[:nCols + dy, dx:] 

221 im2 = diffImage[-dy:, :nRows - dx] 

222 w2 = weightImage[-dy:, :nRows - dx] 

223 # use the same mask for all 3 calculations 

224 wAll = w1*w2 

225 # do not use mean() because weightImage=0 pixels would then count 

226 nPix = wAll.sum() 

227 im1TimesW = im1*wAll 

228 s1 = im1TimesW.sum()/nPix 

229 s2 = (im2*wAll).sum()/nPix 

230 p = (im1TimesW*im2).sum()/nPix 

231 cov = p - s1*s2 

232 

233 return cov, nPix 

234 

235 

236class LoadParams: 

237 """ 

238 A class to prepare covariances for the PTC fit. 

239 

240 Parameters 

241 ---------- 

242 r: `int`, optional 

243 Maximum lag considered (e.g., to eliminate data beyond a separation "r": ignored in the fit). 

244 

245 subtractDistantValue: `bool`, optional 

246 Subtract a background to the measured covariances (mandatory for HSC flat pairs)? 

247 

248 start: `int`, optional 

249 Distance beyond which the subtractDistant model is fitted. 

250 

251 offsetDegree: `int` 

252 Polynomial degree for the subtraction model. 

253 

254 Notes 

255 ----- 

256 params = LoadParams(). "params" drives what happens in he fit. LoadParams provides default values. 

257 """ 

258 def __init__(self): 

259 self.r = 8 

260 self.subtractDistantValue = False 

261 self.start = 5 

262 self.offsetDegree = 1 

263 

264 

265def parseData(dataset, params): 

266 """ Returns a list of CovFit objects, indexed by amp number. 

267 

268 Params 

269 ------ 

270 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset` 

271 The PTC dataset containing the means, variances, and 

272 exposure times. 

273 

274 params: `covAstierptcUtil.LoadParams` 

275 Object with values to drive the bahaviour of fits. 

276 

277 Returns 

278 ------- 

279 covFitList: `dict` 

280 Dictionary with amps as keys, and CovFit objects as values. 

281 """ 

282 

283 covFitList = {} 

284 for ampName in dataset.ampNames: 

285 # If there is a bad amp, don't fit it 

286 if ampName in dataset.badAmps: 

287 continue 

288 maskAtAmp = dataset.expIdMask[ampName] 

289 muAtAmp = dataset.rawMeans[ampName] 

290 covAtAmp = dataset.covariances[ampName] 

291 covSqrtWeightsAtAmp = dataset.covariancesSqrtWeights[ampName] 

292 

293 if params.subtractDistantValue: 

294 c = CovFit(muAtAmp, covAtAmp, covSqrtWeightsAtAmp, params.r, maskAtAmp) 

295 c.subtractDistantOffset(params.r, params.start, params.offsetDegree) 

296 else: 

297 c = CovFit(muAtAmp, covAtAmp, covSqrtWeightsAtAmp, params.r, maskAtAmp) 

298 

299 cc = c.copy() 

300 cc.initFit() # allows to get a crude gain. 

301 covFitList[ampName] = cc 

302 

303 return covFitList 

304 

305 

306def fitData(dataset, r=8): 

307 """Fit data to models in Astier+19. 

308 

309 Parameters 

310 ---------- 

311 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset` 

312 The dataset containing the means, variances, and exposure times. 

313 

314 r : `int`, optional 

315 Maximum lag considered (e.g., to eliminate data beyond a separation "r": ignored in the fit). 

316 

317 Returns 

318 ------- 

319 covFitList: `dict` 

320 Dictionary of CovFit objects, with amp names as keys. 

321 

322 covFitNoBList: `dict` 

323 Dictionary of CovFit objects, with amp names as keys (b=0 in Eq. 20 of Astier+19). 

324 

325 Notes 

326 ----- 

327 The parameters of the full model for C_ij(mu) ("C_ij" and "mu" in ADU^2 and ADU, respectively) 

328 in Astier+19 (Eq. 20) are: 

329 

330 "a" coefficients (r by r matrix), units: 1/e 

331 "b" coefficients (r by r matrix), units: 1/e 

332 noise matrix (r by r matrix), units: e^2 

333 gain, units: e/ADU 

334 

335 "b" appears in Eq. 20 only through the "ab" combination, which is defined in this code as "c=ab". 

336 """ 

337 

338 lparams = LoadParams() 

339 lparams.subtractDistantValue = False 

340 lparams.r = r 

341 covFitList = parseData(dataset, lparams) 

342 covFitNoBList = {} # [None]*(exts[-1]+1) 

343 for ext, c in covFitList.items(): 

344 c.fitFullModel() 

345 covFitNoBList[ext] = c.copy() 

346 c.params['c'].release() 

347 c.fitFullModel() 

348 return covFitList, covFitNoBList 

349 

350 

351def getFitDataFromCovariances(i, j, mu, fullCov, fullCovModel, fullCovSqrtWeights, gain=1.0, 

352 divideByMu=False, returnMasked=False): 

353 """Get measured signal and covariance, cov model, weigths, and mask at covariance lag (i, j). 

354 

355 Parameters 

356 ---------- 

357 i : `int` 

358 Lag for covariance matrix. 

359 

360 j: `int` 

361 Lag for covariance matrix. 

362 

363 mu : `list` 

364 Mean signal values. 

365 

366 fullCov: `list` of `numpy.array` 

367 Measured covariance matrices at each mean signal level in mu. 

368 

369 fullCovSqrtWeights: `list` of `numpy.array` 

370 List of square root of measured covariances at each mean signal level in mu. 

371 

372 fullCovModel : `list` of `numpy.array` 

373 List of modeled covariances at each mean signal level in mu. 

374 

375 gain : `float`, optional 

376 Gain, in e-/ADU. If other than 1.0 (default), the returned quantities will be in 

377 electrons or powers of electrons. 

378 

379 divideByMu: `bool`, optional 

380 Divide returned covariance, model, and weights by the mean signal mu? 

381 

382 returnMasked : `bool`, optional 

383 Use mask (based on weights) in returned arrays (mu, covariance, and model)? 

384 

385 Returns 

386 ------- 

387 mu : `numpy.array` 

388 list of signal values at (i, j). 

389 

390 covariance : `numpy.array` 

391 Covariance at (i, j) at each mean signal mu value (fullCov[:, i, j]). 

392 

393 covarianceModel : `numpy.array` 

394 Covariance model at (i, j). 

395 

396 weights : `numpy.array` 

397 Weights at (i, j). 

398 

399 maskFromWeights : `numpy.array`, optional 

400 Boolean mask of the covariance at (i,j), where the weights differ from 0. 

401 

402 Notes 

403 ----- 

404 This function is a method of the `CovFit` class. 

405 """ 

406 mu = np.array(mu) 

407 fullCov = np.array(fullCov) 

408 fullCovModel = np.array(fullCovModel) 

409 fullCovSqrtWeights = np.array(fullCovSqrtWeights) 

410 covariance = fullCov[:, i, j]*(gain**2) 

411 covarianceModel = fullCovModel[:, i, j]*(gain**2) 

412 weights = fullCovSqrtWeights[:, i, j]/(gain**2) 

413 

414 maskFromWeights = weights != 0 

415 if returnMasked: 

416 weights = weights[maskFromWeights] 

417 covarianceModel = covarianceModel[maskFromWeights] 

418 mu = mu[maskFromWeights] 

419 covariance = covariance[maskFromWeights] 

420 

421 if divideByMu: 

422 covariance /= mu 

423 covarianceModel /= mu 

424 weights *= mu 

425 return mu, covariance, covarianceModel, weights, maskFromWeights