25 from scipy.stats
import median_absolute_deviation
as mad
26 from scipy.signal
import fftconvolve
27 from scipy.optimize
import leastsq
28 from .astierCovFitParameters
import FitParameters
35 """Compute the "a" coefficients of the Antilogus+14 (1402.0725) model as in
36 Guyonnet+15 (1501.01577, eq. 16, the slope of cov/var at a given flux mu in electrons).
38 Eq. 16 of 1501.01577 is an approximation to the more complete model in Astier+19 (1905.08677).
42 fit: `lsst.cp.pipe.astierCovPtcFit.CovFit`
47 aCoeffsOld: `numpy.array`
48 Slope of cov/var at a given flux mu in electrons.
52 Returns the "a" array, computed this way, to be compared to the actual a_array from the full model
57 muAdu = np.array([muEl/gain])
58 model = fit.evalCovModel(muAdu)
62 return model[0, :, :]/(var*muEl)
66 """Make covariances array from tuple.
70 inputTuple: `numpy.recarray`
71 Recarray with rows with at least (mu, cov, var, i, j, npix), where:
72 mu : 0.5*(m1 + m2), where:
73 mu1: mean value of flat1
74 mu2: mean value of flat2
75 cov: covariance value at lag(i, j)
76 var: variance(covariance value at lag(0, 0))
79 npix: number of pixels used for covariance calculation.
81 maxRangeFromTuple: `int`
82 Maximum range to select from tuple.
87 Covariance arrays, indexed by mean signal mu.
90 Variance arrays, indexed by mean signal mu.
93 List of mean signal values.
98 The input tuple should contain the following rows:
99 (mu, cov, var, i, j, npix), with one entry per lag, and image pair.
100 Different lags(i.e. different i and j) from the same
101 image pair have the same values of mu1 and mu2. When i==j==0, cov
104 If the input tuple contains several video channels, one should
105 select the data of a given channel *before* entering this
106 routine, as well as apply(e.g.) saturation cuts.
108 The routine returns cov[k_mu, j, i], vcov[(same indices)], and mu[k]
109 where the first index of cov matches the one in mu.
111 This routine implements the loss of variance due to
112 clipping cuts when measuring variances and covariance, but this should happen inside
113 the measurement code, where the cuts are readily available.
116 if maxRangeFromTuple
is not None:
117 cut = (inputTuple[
'i'] < maxRangeFromTuple) & (inputTuple[
'j'] < maxRangeFromTuple)
118 cutTuple = inputTuple[cut]
120 cutTuple = inputTuple
122 muTemp = cutTuple[
'mu']
123 ind = np.argsort(muTemp)
125 cutTuple = cutTuple[ind]
128 xx = np.hstack(([mu[0]], mu))
129 delta = xx[1:] - xx[:-1]
130 steps, = np.where(delta > 0)
131 ind = np.zeros_like(mu, dtype=int)
135 muVals = np.array(np.unique(mu))
136 i = cutTuple[
'i'].astype(int)
137 j = cutTuple[
'j'].astype(int)
138 c = 0.5*cutTuple[
'cov']
140 v = 0.5*cutTuple[
'var']
142 cov = np.ndarray((len(muVals), np.max(i)+1, np.max(j)+1))
143 var = np.zeros_like(cov)
145 var[ind, i, j] = v**2/n
154 return cov, var, muVals
158 """ Copy array over 4 quadrants prior to convolution.
162 inputarray: `numpy.array`
163 Input array to symmetrize.
171 targetShape = list(inputArray.shape)
172 r1, r2 = inputArray.shape[-1], inputArray.shape[-2]
173 targetShape[-1] = 2*r1-1
174 targetShape[-2] = 2*r2-1
175 aSym = np.ndarray(tuple(targetShape))
176 aSym[..., r2-1:, r1-1:] = inputArray
177 aSym[..., r2-1:, r1-1::-1] = inputArray
178 aSym[..., r2-1::-1, r1-1::-1] = inputArray
179 aSym[..., r2-1::-1, r1-1:] = inputArray
185 """A class to calculate 2D polynomials"""
192 self.coeff, _, rank, _ = np.linalg.lstsq(G, z.ravel())
194 self.coeff, _, rank, _ = np.linalg.lstsq((w.ravel()*G.T).T, z.ravel()*w.ravel())
198 G = np.zeros(x.shape + (ncols,))
199 ij = itertools.product(range(self.
orderx+1), range(self.
ordery+1))
200 for k, (i, j)
in enumerate(ij):
201 G[..., k] = x**i * y**j
208 return np.dot(G, self.coeff)
212 """A class to fit the models in Astier+19 to flat covariances.
214 This code implements the model(and the fit thereof) described in
215 Astier+19: https://arxiv.org/pdf/1905.08677.pdf
216 For the time being it uses as input a numpy recarray (tuple with named tags) which
217 contains one row per covariance and per pair: see the routine makeCovArray.
221 inputTuple: `numpy.recarray`
222 Tuple with at least (mu, cov, var, i, j, npix), where:
223 mu : 0.5*(m1 + m2), where:
224 mu1: mean value of flat1
225 mu2: mean value of flat2
226 cov: covariance value at lag(i, j)
227 var: variance(covariance value at lag(0, 0))
230 npix: number of pixels used for covariance calculation.
232 maxRangeFromTuple: `int`
233 Maximum range to select from tuple.
236 def __init__(self, inputTuple, maxRangeFromTuple=8):
242 """Subtract a background/offset to the measured covariances.
247 Maximum lag considered
250 First lag from where to start the offset subtraction.
253 Degree of 2D polynomial to fit to covariance to define offse to be subtracted.
255 assert(startLag < self.
r)
256 for k
in range(len(self.
mu)):
258 w = self.
sqrtW[k, ...] + 0.
260 i, j = np.meshgrid(range(sh[0]), range(sh[1]), indexing=
'ij')
262 w[:startLag, :startLag] = 0
263 poly =
Pol2d(i, j, self.
cov[k, ...], polDegree+1, w=w)
264 back = poly.eval(i, j)
265 self.
cov[k, ...] -= back
274 """Select signal level based on max average signal in ADU"""
276 index = self.
mu < maxMu
278 self.
mu = self.
mu[:k]
279 self.
cov = self.
cov[:k, ...]
286 """Select signal level based on max average signal in electrons"""
288 kill = (self.
mu*g > maxMuEl)
289 self.
sqrtW[kill, :, :] = 0
294 """Make a copy of params"""
295 cop = copy.deepcopy(self)
297 if hasattr(self,
'params'):
302 """ Performs a crude parabolic fit of the data in order to start
303 the full fit close to the solution.
311 self.
params[
'c'].fix(val=0.)
313 a = self.
params[
'a'].full.reshape(self.
r, self.
r)
314 noise = self.
params[
'noise'].full.reshape(self.
r, self.
r)
315 gain = self.
params[
'gain'].full[0]
324 for i
in range(self.
r):
325 for j
in range(self.
r):
327 parsFit = np.polyfit(self.
mu, self.
cov[:, i, j] - model[:, i, j],
328 2, w=self.
sqrtW[:, i, j])
330 a[i, j] += parsFit[0]
331 noise[i, j] += parsFit[2]*gain*gain
333 gain = 1./(1/gain+parsFit[1])
334 self.
params[
'gain'].full[0] = gain
343 """Return an array of free parameter values (it is a copy)."""
344 return self.
params.free + 0.
347 """Set parameter values."""
352 """Computes full covariances model (Eq. 20 of Astier+19).
356 mu: `numpy.array`, optional
357 List of mean signals.
361 covModel: `numpy.array`
366 By default, computes the covModel for the mu's stored(self.mu).
368 Returns cov[Nmu, self.r, self.r]. The variance for the PTC is cov[:, 0, 0].
369 mu and cov are in ADUs and ADUs squared. To use electrons for both,
370 the gain should be set to 1. This routine implements the model in Astier+19 (1905.08677).
372 The parameters of the full model for C_ij(mu) ("C_ij" and "mu" in ADU^2 and ADU, respectively)
373 in Astier+19 (Eq. 20) are:
375 "a" coefficients (r by r matrix), units: 1/e
376 "b" coefficients (r by r matrix), units: 1/e
377 noise matrix (r by r matrix), units: e^2
380 "b" appears in Eq. 20 only through the "ab" combination, which is defined in this code as "c=ab".
382 sa = (self.
r, self.
r)
383 a = self.
params[
'a'].full.reshape(sa)
384 c = self.
params[
'c'].full.reshape(sa)
385 gain = self.
params[
'gain'].full[0]
386 noise = self.
params[
'noise'].full.reshape(sa)
388 aEnlarged = np.zeros((int(sa[0]*1.5)+1, int(sa[1]*1.5)+1))
389 aEnlarged[0:sa[0], 0:sa[1]] = a
392 cEnlarged = np.zeros((int(sa[0]*1.5)+1, int(sa[1]*1.5)+1))
393 cEnlarged[0:sa[0], 0:sa[1]] = c
395 a2 = fftconvolve(aSym, aSym, mode=
'same')
396 a3 = fftconvolve(a2, aSym, mode=
'same')
397 ac = fftconvolve(aSym, cSym, mode=
'same')
398 (xc, yc) = np.unravel_index(np.abs(aSym).argmax(), a2.shape)
400 a1 = a[np.newaxis, :, :]
401 a2 = a2[np.newaxis, xc:xc + range, yc:yc + range]
402 a3 = a3[np.newaxis, xc:xc + range, yc:yc + range]
403 ac = ac[np.newaxis, xc:xc + range, yc:yc + range]
404 c1 = c[np.newaxis, ::]
408 bigMu = mu[:, np.newaxis, np.newaxis]*gain
410 covModel = (bigMu/(gain*gain)*(a1*bigMu+2./3.*(bigMu*bigMu)*(a2 + c1) +
411 (1./3.*a3 + 5./6.*ac)*(bigMu*bigMu*bigMu)) + noise[np.newaxis, :, :]/gain**2)
413 covModel[:, 0, 0] += mu/gain
418 """'a' matrix from Astier+19(e.g., Eq. 20)"""
419 return self.
params[
'a'].full.reshape(self.
r, self.
r)
422 """'b' matrix from Astier+19(e.g., Eq. 20)"""
423 return self.
params[
'c'].full.reshape(self.
r, self.
r)/self.
getA()
426 """'c'='ab' matrix from Astier+19(e.g., Eq. 20)"""
427 return self.
params[
'c'].full.reshape(self.
r, self.
r)
429 def _getCovParams(self, what):
430 """Get covariance matrix of parameters from fit"""
431 indices = self.
params[what].indexof()
432 i1 = indices[:, np.newaxis]
433 i2 = indices[np.newaxis, :]
438 """Get covariance matrix of "a" coefficients from fit"""
440 return cova.reshape((self.
r, self.
r, self.
r, self.
r))
443 """Square root of diagonal of the parameter covariance of the fitted "a" matrix"""
445 return np.sqrt(cova.diagonal()).reshape((self.
r, self.
r))
448 """Get covariance matrix of "a" coefficients from fit
452 aval = self.
getA().flatten()
453 factor = np.outer(aval, aval)
455 return covb.reshape((self.
r, self.
r, self.
r, self.
r))
458 """Get covariance matrix of "c" coefficients from fit"""
460 return cova.reshape((self.
r, self.
r, self.
r, self.
r))
463 """Get error on fitted gain parameter"""
471 """Get covariances of noise matrix from fit"""
473 return covNoise.reshape((self.
r, self.
r, self.
r, self.
r))
476 """Square root of diagonal of the parameter covariance of the fitted "noise" matrix"""
478 return np.sqrt(covNoise.diagonal()).reshape((self.
r, self.
r))
481 """Get gain (e/ADU)"""
482 return self.
params[
'gain'].full[0]
485 """Get readout noise (e^2)"""
486 return self.
params[
'noise'].full[0]
489 """Get error on readout noise parameter"""
490 ronSqrt = np.sqrt(np.fabs(self.
getRon()))
492 return 0.5*(noiseSigma/np.fabs(self.
getRon()))*ronSqrt
495 """Get noise matrix"""
496 return self.
params[
'noise'].full.reshape(self.
r, self.
r)
499 """Get mask of var = cov[0,0]"""
501 weights = self.
sqrtW[:, 0, 0]/(gain**2)
506 """Set "a" and "b" coeffcients forfull Astier+19 model (Eq. 20). "c=a*b"."""
507 self.
params[
'a'].full = a.flatten()
508 self.
params[
'c'].full = a.flatten()*b.flatten()
512 """Calculate weighte chi2 of full-model fit."""
516 """To be used in weightedRes"""
517 if params
is not None:
520 return((covModel-self.
cov)*self.
sqrtW)
523 """Weighted residuas.
530 coeffs, cov, _, mesg, ierr = leastsq(c.weightedRes, c.getParamValues(), full_output=True )
532 return self.
wres(params).flatten()
535 """Fit measured covariances to full model in Astier+19 (Eq. 20)
540 Initial parameters of the fit.
541 len(pInit) = #entries(a) + #entries(c) + #entries(noise) + 1
542 len(pInit) = r^2 + r^2 + r^2 + 1, where "r" is the maximum lag considered for the
543 covariances calculation, and the extra "1" is the gain.
544 If "b" is 0, then "c" is 0, and len(pInit) will have r^2 fewer entries.
547 Sigma cut to get rid of outliers.
552 Fit parameters (see "Notes" below).
556 The parameters of the full model for C_ij(mu) ("C_ij" and "mu" in ADU^2 and ADU, respectively)
557 in Astier+19 (Eq. 20) are:
559 "a" coefficients (r by r matrix), units: 1/e
560 "b" coefficients (r by r matrix), units: 1/e
561 noise matrix (r by r matrix), units: e^2
564 "b" appears in Eq. 20 only through the "ab" combination, which is defined in this code as "c=ab".
572 while nOutliers != 0:
573 params, paramsCov, _, mesg, ierr = leastsq(self.
weightedRes, pInit, full_output=
True)
576 sig = mad(wres[wres != 0])
577 mask = (np.abs(wres) > (nSigma * sig))
578 self.
sqrtW.flat[mask] = 0
579 nOutliers = mask.sum()
581 if counter == maxFitIter:
584 if ierr
not in [1, 2, 3, 4]:
585 raise RuntimeError(
"Minimization failed: " + mesg)
591 """Number of degrees of freedom
595 mask.sum() - len(self.params.free): `int`
596 Number of usable pixels - number of parameters of fit.
598 mask = self.
sqrtW != 0
600 return mask.sum() - len(self.
params.free)
603 """Get measured signal and covariance, cov model and wigths
613 divideByMu: `bool`, optional
614 Divide covariance, model, and weights by signal mu?
619 list of signal values(mu*gain).
621 covariance: `numpy.array`
622 Covariance arrays, indexed by mean signal mu(self.cov[:, i, j]*gain**2).
625 Covariance model(model*gain**2)
627 weights: `numpy.array`
628 Weights(self.sqrtW/gain**2)
632 Using a CovFit object, selects from(i, j) and returns
633 mu*gain, self.cov[:, i, j]*gain**2 model*gain**2, and self.sqrtW/gain**2
637 covariance = self.
cov[:, i, j]*(gain**2)
639 weights = self.
sqrtW[:, i, j]/(gain**2)
644 weights = weights[mask]
648 covariance = covariance[mask]
654 return mu, covariance, model, weights