23 from .astierCovPtcFit
import CovFit
29 """A class to compute (via FFT) the nearby pixels correlation function.
31 Implements appendix of Astier+19.
36 Image where to calculate the covariances (e.g., the difference image of two flats).
39 Weight image (mask): it should consist of 1's (good pixel) and 0's (bad pixels).
42 2d-tuple with the shape of the FFT
45 Maximum range for the covariances.
48 def __init__(self, diff, w, fftShape, maxRangeCov):
51 assert(fftShape[0] > diff.shape[0]+maxRangeCov+1)
52 assert(fftShape[1] > diff.shape[1]+maxRangeCov+1)
55 if fftShape[1]%2 == 1:
56 fftShape = (fftShape[0], fftShape[1]+1)
57 tIm = np.fft.rfft2(diff*w, fftShape)
58 tMask = np.fft.rfft2(w, fftShape)
60 self.
pCov = np.fft.irfft2(tIm*tIm.conjugate())
62 self.
pMean = np.fft.irfft2(tIm*tMask.conjugate())
64 self.
pCount = np.fft.irfft2(tMask*tMask.conjugate())
66 def cov(self, dx, dy):
67 """Covariance for dx,dy averaged with dx,-dy if both non zero.
69 Implements appendix of Astier+19.
81 0.5*(cov1+cov2): `float`
82 Covariance at (dx, dy) lag
85 Number of pixels used in covariance calculation.
88 nPix1 = int(round(self.
pCount[dy, dx]))
89 cov1 = self.
pCov[dy, dx]/nPix1-self.
pMean[dy, dx]*self.
pMean[-dy, -dx]/(nPix1*nPix1)
90 if (dx == 0
or dy == 0):
92 nPix2 = int(round(self.
pCount[-dy, dx]))
93 cov2 = self.
pCov[-dy, dx]/nPix2-self.
pMean[-dy, dx]*self.
pMean[dy, -dx]/(nPix2*nPix2)
94 return 0.5*(cov1+cov2), nPix1+nPix2
97 """Produce a list of tuples with covariances.
99 Implements appendix of Astier+19.
104 Maximum range of covariances.
109 List with covariance tuples.
113 for dy
in range(maxRange+1):
114 for dx
in range(maxRange+1):
115 cov, npix = self.
cov(dx, dy)
116 if (dx == 0
and dy == 0):
118 tupleVec.append((dx, dy, var, cov, npix))
123 """Calculate the size fof one dimension for the FFT"""
124 x = int(np.log(s)/np.log(2.))
129 """Compute covariances of diffImage in real space.
131 For lags larger than ~25, it is slower than the FFT way.
132 Taken from https://github.com/PierreAstier/bfptc/
136 diffImage : `numpy.array`
137 Image to compute the covariance of.
139 weightImage : `numpy.array`
140 Weight image of diffImage (1's and 0's for good and bad pixels, respectively).
143 Last index of the covariance to be computed.
148 List with tuples of the form (dx, dy, var, cov, npix), where:
154 Variance at (dx, dy).
156 Covariance at (dx, dy).
158 Number of pixel pairs used to evaluate var and cov.
163 for dy
in range(maxRange + 1):
164 for dx
in range(0, maxRange + 1):
168 cov = 0.5*(cov1 + cov2)
172 if (dx == 0
and dy == 0):
174 outList.append((dx, dy, var, cov, nPix))
180 """Compute covariances of diffImage in real space at lag (dx, dy).
182 Taken from https://github.com/PierreAstier/bfptc/ (c.f., appendix of Astier+19).
186 diffImage : `numpy.array`
187 Image to compute the covariance of.
189 weightImage : `numpy.array`
190 Weight image of diffImage (1's and 0's for good and bad pixels, respectively).
201 Covariance at (dx, dy)
204 Number of pixel pairs used to evaluate var and cov.
206 (nCols, nRows) = diffImage.shape
210 (dx, dy) = (-dx, -dy)
214 im1 = diffImage[dy:, dx:]
215 w1 = weightImage[dy:, dx:]
216 im2 = diffImage[:nCols - dy, :nRows - dx]
217 w2 = weightImage[:nCols - dy, :nRows - dx]
219 im1 = diffImage[:nCols + dy, dx:]
220 w1 = weightImage[:nCols + dy, dx:]
221 im2 = diffImage[-dy:, :nRows - dx]
222 w2 = weightImage[-dy:, :nRows - dx]
228 s1 = im1TimesW.sum()/nPix
229 s2 = (im2*wAll).sum()/nPix
230 p = (im1TimesW*im2).sum()/nPix
238 A class to prepare covariances for the PTC fit.
243 Maximum lag considered (e.g., to eliminate data beyond a separation "r": ignored in the fit).
245 maxMu: `float`, optional
246 Maximum signal, in ADU (e.g., to eliminate data beyond saturation).
248 maxMuElectrons: `float`, optional
249 Maximum signal in electrons.
251 subtractDistantValue: `bool`, optional
252 Subtract a background to the measured covariances (mandatory for HSC flat pairs)?
254 start: `int`, optional
255 Distance beyond which the subtractDistant model is fitted.
258 Polynomial degree for the subtraction model.
262 params = LoadParams(). "params" drives what happens in he fit. LoadParams provides default values.
274 """ Returns a list of CovFit objects, indexed by amp number.
278 tupleName: `numpy.recarray`
279 Recarray with rows with at least ( mu1, mu2, cov ,var, i, j, npix), where:
280 mu1: mean value of flat1
281 mu2: mean value of flat2
282 cov: covariance value at lag (i, j)
283 var: variance (covariance value at lag (0, 0))
286 npix: number of pixels used for covariance calculation.
288 params: `covAstierptcUtil.LoadParams`
289 Object with values to drive the bahaviour of fits.
294 Dictionary with amps as keys, and CovFit objects as values.
297 exts = np.array(np.unique(tupleName[
'ampName']), dtype=str)
300 ntext = tupleName[tupleName[
'ampName'] == ext]
301 if params.subtractDistantValue:
302 c =
CovFit(ntext, params.r)
303 c.subtractDistantOffset(params.r, params.start, params.offsetDegree)
305 c =
CovFit(ntext, params.r)
306 thisMaxMu = params.maxMu
308 for iter
in range(3):
310 cc.setMaxMu(thisMaxMu)
313 if (thisMaxMu*gain < params.maxMuElectrons):
314 thisMaxMu = params.maxMuElectrons/gain
316 cc.setMaxMuElectrons(params.maxMuElectrons)
323 def fitData(tupleName, maxMu=1e9, r=8, nSigmaFullFit=5.5, maxIterFullFit=3):
324 """Fit data to models in Astier+19.
328 tupleName: `numpy.recarray`
329 Recarray with rows with at least ( mu1, mu2, cov ,var, i, j, npix), where:
330 mu1: mean value of flat1
331 mu2: mean value of flat2
332 cov: covariance value at lag (i, j)
333 var: variance (covariance value at lag (0, 0))
336 npix: number of pixels used for covariance calculation.
339 Maximum lag considered (e.g., to eliminate data beyond a separation "r": ignored in the fit).
341 maxMu: `float`, optional
342 Maximum signal, in ADU (e.g., to eliminate data beyond saturation).
344 nSigmaFullFit : `float`, optional
345 Sigma cut to get rid of outliers in full model fit.
347 maxIterFullFit : `int`, optional
348 Number of iterations for full model fit.
353 Dictionary of CovFit objects, with amp names as keys.
355 covFitNoBList: `dict`
356 Dictionary of CovFit objects, with amp names as keys (b=0 in Eq. 20 of Astier+19).
360 The parameters of the full model for C_ij(mu) ("C_ij" and "mu" in ADU^2 and ADU, respectively)
361 in Astier+19 (Eq. 20) are:
363 "a" coefficients (r by r matrix), units: 1/e
364 "b" coefficients (r by r matrix), units: 1/e
365 noise matrix (r by r matrix), units: e^2
368 "b" appears in Eq. 20 only through the "ab" combination, which is defined in this code as "c=ab".
372 lparams.subtractDistantValue =
False
373 lparams.maxMu = maxMu
376 covFitList =
loadData(tupleName, lparams)
378 for ext, c
in covFitList.items():
379 c.fitFullModel(nSigma=nSigmaFullFit, maxFitIter=maxIterFullFit)
380 covFitNoBList[ext] = c.copy()
381 c.params[
'c'].release()
382 c.fitFullModel(nSigma=nSigmaFullFit, maxFitIter=maxIterFullFit)
383 return covFitList, covFitNoBList
387 divideByMu=False, returnMasked=False):
388 """Get measured signal and covariance, cov model, weigths, and mask at covariance lag (i, j).
393 Lag for covariance matrix.
396 Lag for covariance matrix.
401 fullCov: `list` of `numpy.array`
402 Measured covariance matrices at each mean signal level in mu.
404 fullCovSqrtWeights: `list` of `numpy.array`
405 List of square root of measured covariances at each mean signal level in mu.
407 fullCovModel : `list` of `numpy.array`
408 List of modeled covariances at each mean signal level in mu.
410 gain : `float`, optional
411 Gain, in e-/ADU. If other than 1.0 (default), the returned quantities will be in
412 electrons or powers of electrons.
414 divideByMu: `bool`, optional
415 Divide returned covariance, model, and weights by the mean signal mu?
417 returnMasked : `bool`, optional
418 Use mask (based on weights) in returned arrays (mu, covariance, and model)?
423 list of signal values at (i, j).
425 covariance : `numpy.array`
426 Covariance at (i, j) at each mean signal mu value (fullCov[:, i, j]).
428 covarianceModel : `numpy.array`
429 Covariance model at (i, j).
431 weights : `numpy.array`
434 mask : `numpy.array`, optional
435 Boolean mask of the covariance at (i,j).
439 This function is a method of the `CovFit` class.
442 fullCov = np.array(fullCov)
443 fullCovModel = np.array(fullCovModel)
444 fullCovSqrtWeights = np.array(fullCovSqrtWeights)
445 covariance = fullCov[:, i, j]*(gain**2)
446 covarianceModel = fullCovModel[:, i, j]*(gain**2)
447 weights = fullCovSqrtWeights[:, i, j]/(gain**2)
452 weights = weights[mask]
453 covarianceModel = covarianceModel[mask]
455 covariance = covariance[mask]
459 covarianceModel /= mu
462 return mu, covariance, covarianceModel, weights, mask