25 from scipy.signal
import fftconvolve
26 from scipy.optimize
import leastsq
27 from .astierCovFitParameters
import FitParameters
35 """Compute the "a" coefficients of the Antilogus+14 (1402.0725) model as in
36 Guyonnet+15 (1501.01577, eq. 16, the slope of cov/var at a given flux mu in electrons).
38 Eq. 16 of 1501.01577 is an approximation to the more complete model in Astier+19 (1905.08677).
43 Covariance model from Eq. 20 in Astier+19.
46 Mean signal in electrons
53 aCoeffsOld: `numpy.array`
54 Slope of cov/var at a given flux mu in electrons.
58 Returns the "a" array, computed this way, to be compared to the actual a_array from the full model
61 covModel = np.array(covModel)
62 var = covModel[0, 0, 0]
64 return covModel[0, :, :]/(var*muEl)
68 """Make covariances array from tuple.
72 inputTuple: `numpy.recarray`
73 Recarray with rows with at least (mu, cov, var, i, j, npix), where:
74 mu : 0.5*(m1 + m2), where:
75 mu1: mean value of flat1
76 mu2: mean value of flat2
77 cov: covariance value at lag(i, j)
78 var: variance(covariance value at lag(0, 0))
81 npix: number of pixels used for covariance calculation.
83 maxRangeFromTuple: `int`
84 Maximum range to select from tuple.
89 Covariance arrays, indexed by mean signal mu.
92 Variance arrays, indexed by mean signal mu.
95 List of mean signal values.
100 The input tuple should contain the following rows:
101 (mu, cov, var, i, j, npix), with one entry per lag, and image pair.
102 Different lags(i.e. different i and j) from the same
103 image pair have the same values of mu1 and mu2. When i==j==0, cov
106 If the input tuple contains several video channels, one should
107 select the data of a given channel *before* entering this
108 routine, as well as apply(e.g.) saturation cuts.
110 The routine returns cov[k_mu, j, i], vcov[(same indices)], and mu[k]
111 where the first index of cov matches the one in mu.
113 This routine implements the loss of variance due to
114 clipping cuts when measuring variances and covariance, but this should happen inside
115 the measurement code, where the cuts are readily available.
118 if maxRangeFromTuple
is not None:
119 cut = (inputTuple[
'i'] < maxRangeFromTuple) & (inputTuple[
'j'] < maxRangeFromTuple)
120 cutTuple = inputTuple[cut]
122 cutTuple = inputTuple
124 muTemp = cutTuple[
'mu']
125 ind = np.argsort(muTemp)
127 cutTuple = cutTuple[ind]
130 xx = np.hstack(([mu[0]], mu))
131 delta = xx[1:] - xx[:-1]
132 steps, = np.where(delta > 0)
133 ind = np.zeros_like(mu, dtype=int)
137 muVals = np.array(np.unique(mu))
138 i = cutTuple[
'i'].astype(int)
139 j = cutTuple[
'j'].astype(int)
140 c = 0.5*cutTuple[
'cov']
142 v = 0.5*cutTuple[
'var']
144 cov = np.ndarray((len(muVals), np.max(i)+1, np.max(j)+1))
145 var = np.zeros_like(cov)
147 var[ind, i, j] = v**2/n
156 return cov, var, muVals
160 """ Copy array over 4 quadrants prior to convolution.
164 inputarray: `numpy.array`
165 Input array to symmetrize.
173 targetShape = list(inputArray.shape)
174 r1, r2 = inputArray.shape[-1], inputArray.shape[-2]
175 targetShape[-1] = 2*r1-1
176 targetShape[-2] = 2*r2-1
177 aSym = np.ndarray(tuple(targetShape))
178 aSym[..., r2-1:, r1-1:] = inputArray
179 aSym[..., r2-1:, r1-1::-1] = inputArray
180 aSym[..., r2-1::-1, r1-1::-1] = inputArray
181 aSym[..., r2-1::-1, r1-1:] = inputArray
187 """A class to calculate 2D polynomials"""
194 self.coeff, _, rank, _ = np.linalg.lstsq(G, z.ravel())
196 self.coeff, _, rank, _ = np.linalg.lstsq((w.ravel()*G.T).T, z.ravel()*w.ravel())
200 G = np.zeros(x.shape + (ncols,))
201 ij = itertools.product(range(self.
orderx+1), range(self.
ordery+1))
202 for k, (i, j)
in enumerate(ij):
203 G[..., k] = x**i * y**j
210 return np.dot(G, self.coeff)
214 """A class to fit the models in Astier+19 to flat covariances.
216 This code implements the model(and the fit thereof) described in
217 Astier+19: https://arxiv.org/pdf/1905.08677.pdf
218 For the time being it uses as input a numpy recarray (tuple with named tags) which
219 contains one row per covariance and per pair: see the routine makeCovArray.
223 inputTuple: `numpy.recarray`
224 Tuple with at least (mu, cov, var, i, j, npix), where:
225 mu : 0.5*(m1 + m2), where:
226 mu1: mean value of flat1
227 mu2: mean value of flat2
228 cov: covariance value at lag(i, j)
229 var: variance(covariance value at lag(0, 0))
232 npix: number of pixels used for covariance calculation.
234 maxRangeFromTuple: `int`, optional
235 Maximum range to select from tuple.
237 meanSignalMask: `list`[`bool`], optional
238 Mask of mean signal 1D array. Use all entries if empty.
241 def __init__(self, inputTuple, maxRangeFromTuple=8, meanSignalMask=[]):
246 self.
logger = lsstLog.Log.getDefaultLogger()
247 if len(meanSignalMask):
250 self.
maskMu = np.repeat(
True, len(self.
mu))
253 """Subtract a background/offset to the measured covariances.
258 Maximum lag considered
261 First lag from where to start the offset subtraction.
264 Degree of 2D polynomial to fit to covariance to define offse to be subtracted.
266 assert(startLag < self.
r)
267 for k
in range(len(self.
mu)):
269 w = self.
sqrtW[k, ...] + 0.
271 i, j = np.meshgrid(range(sh[0]), range(sh[1]), indexing=
'ij')
273 w[:startLag, :startLag] = 0
274 poly =
Pol2d(i, j, self.
cov[k, ...], polDegree+1, w=w)
275 back = poly.eval(i, j)
276 self.
cov[k, ...] -= back
285 """Make a copy of params"""
286 cop = copy.deepcopy(self)
288 if hasattr(self,
'params'):
293 """ Performs a crude parabolic fit of the data in order to start
294 the full fit close to the solution.
302 self.
params[
'c'].fix(val=0.)
304 a = self.
params[
'a'].full.reshape(self.
r, self.
r)
305 noise = self.
params[
'noise'].full.reshape(self.
r, self.
r)
306 gain = self.
params[
'gain'].full[0]
315 for i
in range(self.
r):
316 for j
in range(self.
r):
318 parsFit = np.polyfit(self.
mu, self.
cov[:, i, j] - model[:, i, j],
319 2, w=self.
sqrtW[:, i, j])
321 a[i, j] += parsFit[0]
322 noise[i, j] += parsFit[2]*gain*gain
324 gain = 1./(1/gain+parsFit[1])
325 self.
params[
'gain'].full[0] = gain
334 """Return an array of free parameter values (it is a copy)."""
335 return self.
params.free + 0.
338 """Set parameter values."""
343 """Computes full covariances model (Eq. 20 of Astier+19).
347 mu: `numpy.array`, optional
348 List of mean signals.
352 covModel: `numpy.array`
357 By default, computes the covModel for the mu's stored(self.mu).
359 Returns cov[Nmu, self.r, self.r]. The variance for the PTC is cov[:, 0, 0].
360 mu and cov are in ADUs and ADUs squared. To use electrons for both,
361 the gain should be set to 1. This routine implements the model in Astier+19 (1905.08677).
363 The parameters of the full model for C_ij(mu) ("C_ij" and "mu" in ADU^2 and ADU, respectively)
364 in Astier+19 (Eq. 20) are:
366 "a" coefficients (r by r matrix), units: 1/e
367 "b" coefficients (r by r matrix), units: 1/e
368 noise matrix (r by r matrix), units: e^2
371 "b" appears in Eq. 20 only through the "ab" combination, which is defined in this code as "c=ab".
373 sa = (self.
r, self.
r)
374 a = self.
params[
'a'].full.reshape(sa)
375 c = self.
params[
'c'].full.reshape(sa)
376 gain = self.
params[
'gain'].full[0]
377 noise = self.
params[
'noise'].full.reshape(sa)
379 aEnlarged = np.zeros((int(sa[0]*1.5)+1, int(sa[1]*1.5)+1))
380 aEnlarged[0:sa[0], 0:sa[1]] = a
383 cEnlarged = np.zeros((int(sa[0]*1.5)+1, int(sa[1]*1.5)+1))
384 cEnlarged[0:sa[0], 0:sa[1]] = c
386 a2 = fftconvolve(aSym, aSym, mode=
'same')
387 a3 = fftconvolve(a2, aSym, mode=
'same')
388 ac = fftconvolve(aSym, cSym, mode=
'same')
389 (xc, yc) = np.unravel_index(np.abs(aSym).argmax(), a2.shape)
391 a1 = a[np.newaxis, :, :]
392 a2 = a2[np.newaxis, xc:xc + range, yc:yc + range]
393 a3 = a3[np.newaxis, xc:xc + range, yc:yc + range]
394 ac = ac[np.newaxis, xc:xc + range, yc:yc + range]
395 c1 = c[np.newaxis, ::]
399 bigMu = mu[:, np.newaxis, np.newaxis]*gain
401 covModel = (bigMu/(gain*gain)*(a1*bigMu+2./3.*(bigMu*bigMu)*(a2 + c1) +
402 (1./3.*a3 + 5./6.*ac)*(bigMu*bigMu*bigMu)) + noise[np.newaxis, :, :]/gain**2)
404 covModel[:, 0, 0] += mu/gain
409 """'a' matrix from Astier+19(e.g., Eq. 20)"""
410 return self.
params[
'a'].full.reshape(self.
r, self.
r)
413 """'b' matrix from Astier+19(e.g., Eq. 20)"""
414 return self.
params[
'c'].full.reshape(self.
r, self.
r)/self.
getA()
417 """'c'='ab' matrix from Astier+19(e.g., Eq. 20)"""
418 return np.array(self.
params[
'c'].full.reshape(self.
r, self.
r))
420 def _getCovParams(self, what):
421 """Get covariance matrix of parameters from fit"""
422 indices = self.
params[what].indexof()
423 i1 = indices[:, np.newaxis]
424 i2 = indices[np.newaxis, :]
432 """Get covariance matrix of "a" coefficients from fit"""
440 """Square root of diagonal of the parameter covariance of the fitted "a" matrix"""
442 sigA = np.sqrt(self.
_getCovParams(
'a').diagonal()).reshape((self.
r, self.
r))
448 """Get covariance matrix of "a" coefficients from fit
452 aval = self.
getA().flatten()
453 factor = np.outer(aval, aval)
455 return covb.reshape((self.
r, self.
r, self.
r, self.
r))
458 """Get covariance matrix of "c" coefficients from fit"""
460 return cova.reshape((self.
r, self.
r, self.
r, self.
r))
463 """Get error on fitted gain parameter"""
471 """Get covariances of noise matrix from fit"""
473 return covNoise.reshape((self.
r, self.
r, self.
r, self.
r))
476 """Square root of diagonal of the parameter covariance of the fitted "noise" matrix"""
479 noise = np.sqrt(covNoise.diagonal()).reshape((self.
r, self.
r))
485 """Get gain (e/ADU)"""
486 return self.
params[
'gain'].full[0]
489 """Get readout noise (e^2)"""
490 return self.
params[
'noise'].full[0]
493 """Get error on readout noise parameter"""
494 ronSqrt = np.sqrt(np.fabs(self.
getRon()))
497 ronErr = 0.5*(noiseSigma/np.fabs(self.
getRon()))*ronSqrt
503 """Get noise matrix"""
504 return self.
params[
'noise'].full.reshape(self.
r, self.
r)
507 """Get mask of Cov[i,j]"""
508 weights = self.
sqrtW[:, i, j]
513 """Set "a" and "b" coeffcients forfull Astier+19 model (Eq. 20). "c=a*b"."""
514 self.
params[
'a'].full = a.flatten()
515 self.
params[
'c'].full = a.flatten()*b.flatten()
519 """Calculate weighted chi2 of full-model fit."""
523 """To be used in weightedRes"""
524 if params
is not None:
527 weightedRes = (covModel-self.
cov)*self.
sqrtW
528 maskedWeightedRes = weightedRes[self.
maskMu]
530 return maskedWeightedRes
533 """Weighted residuas.
540 coeffs, cov, _, mesg, ierr = leastsq(c.weightedRes, c.getParamValues(), full_output=True)
542 return self.
wres(params).flatten()
545 """Fit measured covariances to full model in Astier+19 (Eq. 20)
550 Initial parameters of the fit.
551 len(pInit) = #entries(a) + #entries(c) + #entries(noise) + 1
552 len(pInit) = r^2 + r^2 + r^2 + 1, where "r" is the maximum lag considered for the
553 covariances calculation, and the extra "1" is the gain.
554 If "b" is 0, then "c" is 0, and len(pInit) will have r^2 fewer entries.
559 Fit parameters (see "Notes" below).
563 The parameters of the full model for C_ij(mu) ("C_ij" and "mu" in ADU^2 and ADU, respectively)
564 in Astier+19 (Eq. 20) are:
566 "a" coefficients (r by r matrix), units: 1/e
567 "b" coefficients (r by r matrix), units: 1/e
568 noise matrix (r by r matrix), units: e^2
571 "b" appears in Eq. 20 only through the "ab" combination, which is defined in this code as "c=ab".
577 params, paramsCov, _, mesg, ierr = leastsq(self.
weightedRes, pInit, full_output=
True)
583 """Number of degrees of freedom
587 mask.sum() - len(self.params.free): `int`
588 Number of usable pixels - number of parameters of fit.
590 mask = self.
sqrtW != 0
592 return mask.sum() - len(self.
params.free)
594 def getFitData(self, i, j, divideByMu=False, unitsElectrons=False, returnMasked=False):
595 """Get measured signal and covariance, cov model, weigths, and mask at covariance lag (i, j).
600 Lag for covariance matrix.
603 Lag for covariance matrix.
605 divideByMu: `bool`, optional
606 Divide covariance, model, and weights by signal mu?
608 unitsElectrons : `bool`, optional
609 mu, covariance, and model are in ADU (or powers of ADU) If tthis parameter is true, these are
610 multiplied by the adequte factors of the gain to return quantities in electrons
611 (or powers of electrons).
613 returnMasked : `bool`, optional
614 Use mask (based on weights) in returned arrays (mu, covariance, and model)?
619 list of signal values (mu).
621 covariance: `numpy.array`
622 Covariance arrays, indexed by mean signal mu (self.cov[:, i, j]).
624 covarianceModel: `numpy.array`
625 Covariance model (model).
627 weights: `numpy.array`
630 mask : `numpy.array`, optional
631 Boolean mask of the covariance at (i,j).
635 Using a CovFit object, selects from (i, j) and returns
636 mu*gain, self.cov[:, i, j]*gain**2 model*gain**2, and self.sqrtW/gain**2
637 in electrons or ADU if unitsElectrons=False.
645 covariance = self.
cov[:, i, j]*(gain**2)
646 covarianceModel = self.
evalCovModel()[:, i, j]*(gain**2)
647 weights = self.
sqrtW[:, i, j]/(gain**2)
652 weights = weights[mask]
653 covarianceModel = covarianceModel[mask]
655 covariance = covariance[mask]
659 covarianceModel /= mu
662 return mu, covariance, covarianceModel, weights, mask