25 from scipy.stats
import median_absolute_deviation
as mad
26 from scipy.signal
import fftconvolve
27 from scipy.optimize
import leastsq
28 from .astierCovFitParameters
import FitParameters
35 """Compute the "a" coefficients of the Antilogus+14 (1402.0725) model as in
36 Guyonnet+15 (1501.01577, eq. 16, the slope of cov/var at a given flux mu in electrons).
38 Eq. 16 of 1501.01577 is an approximation to the more complete model in Astier+19 (1905.08677).
42 fit: `lsst.cp.pipe.astierCovPtcFit.CovFit`
47 aCoeffsOld: `numpy.array`
48 Slope of cov/var at a given flux mu in electrons.
52 Returns the "a" array, computed this way, to be compared to the actual a_array from the full model
57 muAdu = np.array([muEl/gain])
58 model = fit.evalCovModel(muAdu)
62 return model[0, :, :]/(var*muEl)
66 """Make covariances array from tuple.
70 inputTuple: `numpy.recarray`
71 Recarray with rows with at least (mu, cov, var, i, j, npix), where:
72 mu : 0.5*(m1 + m2), where:
73 mu1: mean value of flat1
74 mu2: mean value of flat2
75 cov: covariance value at lag(i, j)
76 var: variance(covariance value at lag(0, 0))
79 npix: number of pixels used for covariance calculation.
81 maxRangeFromTuple: `int`
82 Maximum range to select from tuple.
87 Covariance arrays, indexed by mean signal mu.
90 Variance arrays, indexed by mean signal mu.
93 List of mean signal values.
98 The input tuple should contain the following rows:
99 (mu, cov, var, i, j, npix), with one entry per lag, and image pair.
100 Different lags(i.e. different i and j) from the same
101 image pair have the same values of mu1 and mu2. When i==j==0, cov
104 If the input tuple contains several video channels, one should
105 select the data of a given channel *before* entering this
106 routine, as well as apply(e.g.) saturation cuts.
108 The routine returns cov[k_mu, j, i], vcov[(same indices)], and mu[k]
109 where the first index of cov matches the one in mu.
111 This routine implements the loss of variance due to
112 clipping cuts when measuring variances and covariance, but this should happen inside
113 the measurement code, where the cuts are readily available.
116 if maxRangeFromTuple
is not None:
117 cut = (inputTuple[
'i'] < maxRangeFromTuple) & (inputTuple[
'j'] < maxRangeFromTuple)
118 cutTuple = inputTuple[cut]
120 cutTuple = inputTuple
122 muTemp = cutTuple[
'mu']
123 ind = np.argsort(muTemp)
125 cutTuple = cutTuple[ind]
128 xx = np.hstack(([mu[0]], mu))
129 delta = xx[1:] - xx[:-1]
130 steps, = np.where(delta > 0)
131 ind = np.zeros_like(mu, dtype=int)
135 muVals = np.array(np.unique(mu))
136 i = cutTuple[
'i'].astype(int)
137 j = cutTuple[
'j'].astype(int)
138 c = 0.5*cutTuple[
'cov']
140 v = 0.5*cutTuple[
'var']
142 cov = np.ndarray((len(muVals), np.max(i)+1, np.max(j)+1))
143 var = np.zeros_like(cov)
145 var[ind, i, j] = v**2/n
154 return cov, var, muVals
158 """ Copy array over 4 quadrants prior to convolution.
162 inputarray: `numpy.array`
163 Input array to symmetrize.
171 targetShape = list(inputArray.shape)
172 r1, r2 = inputArray.shape[-1], inputArray.shape[-2]
173 targetShape[-1] = 2*r1-1
174 targetShape[-2] = 2*r2-1
175 aSym = np.ndarray(tuple(targetShape))
176 aSym[..., r2-1:, r1-1:] = inputArray
177 aSym[..., r2-1:, r1-1::-1] = inputArray
178 aSym[..., r2-1::-1, r1-1::-1] = inputArray
179 aSym[..., r2-1::-1, r1-1:] = inputArray
185 """A class to calculate 2D polynomials"""
192 self.coeff, _, rank, _ = np.linalg.lstsq(G, z.ravel())
194 self.coeff, _, rank, _ = np.linalg.lstsq((w.ravel()*G.T).T, z.ravel()*w.ravel())
198 G = np.zeros(x.shape + (ncols,))
199 ij = itertools.product(range(self.
orderx+1), range(self.
ordery+1))
200 for k, (i, j)
in enumerate(ij):
201 G[..., k] = x**i * y**j
208 return np.dot(G, self.coeff)
212 """A class to fit the models in Astier+19 to flat covariances.
214 This code implements the model(and the fit thereof) described in
215 Astier+19: https://arxiv.org/pdf/1905.08677.pdf
216 For the time being it uses as input a numpy recarray (tuple with named tags) which
217 contains one row per covariance and per pair: see the routine makeCovArray.
221 inputTuple: `numpy.recarray`
222 Tuple with at least (mu, cov, var, i, j, npix), where:
223 mu : 0.5*(m1 + m2), where:
224 mu1: mean value of flat1
225 mu2: mean value of flat2
226 cov: covariance value at lag(i, j)
227 var: variance(covariance value at lag(0, 0))
230 npix: number of pixels used for covariance calculation.
232 maxRangeFromTuple: `int`
233 Maximum range to select from tuple.
236 def __init__(self, inputTuple, maxRangeFromTuple=8):
242 """Subtract a background/offset to the measured covariances.
247 Maximum lag considered
250 First lag from where to start the offset subtraction.
253 Degree of 2D polynomial to fit to covariance to define offse to be subtracted.
255 assert(startLag < self.
r)
256 for k
in range(len(self.
mu)):
258 w = self.
sqrtW[k, ...] + 0.
260 i, j = np.meshgrid(range(sh[0]), range(sh[1]), indexing=
'ij')
262 w[:startLag, :startLag] = 0
263 poly =
Pol2d(i, j, self.
cov[k, ...], polDegree+1, w=w)
264 back = poly.eval(i, j)
265 self.
cov[k, ...] -= back
274 """Select signal level based on max average signal in ADU"""
276 index = self.
mu < maxMu
278 self.
mu = self.
mu[:k]
279 self.
cov = self.
cov[:k, ...]
286 """Select signal level based on max average signal in electrons"""
288 kill = (self.
mu*g > maxMuEl)
289 self.
sqrtW[kill, :, :] = 0
294 """Make a copy of params"""
295 cop = copy.deepcopy(self)
297 if hasattr(self,
'params'):
302 """ Performs a crude parabolic fit of the data in order to start
303 the full fit close to the solution.
311 self.
params[
'c'].fix(val=0.)
313 a = self.
params[
'a'].full.reshape(self.
r, self.
r)
314 noise = self.
params[
'noise'].full.reshape(self.
r, self.
r)
315 gain = self.
params[
'gain'].full[0]
324 for i
in range(self.
r):
325 for j
in range(self.
r):
327 parsFit = np.polyfit(self.
mu, self.
cov[:, i, j] - model[:, i, j],
328 2, w=self.
sqrtW[:, i, j])
330 a[i, j] += parsFit[0]
331 noise[i, j] += parsFit[2]*gain*gain
333 gain = 1./(1/gain+parsFit[1])
334 self.
params[
'gain'].full[0] = gain
343 """Return an array of free parameter values (it is a copy)."""
344 return self.
params.free + 0.
347 """Set parameter values."""
352 """Computes full covariances model (Eq. 20 of Astier+19).
356 mu: `numpy.array`, optional
357 List of mean signals.
361 covModel: `numpy.array`
366 By default, computes the covModel for the mu's stored(self.mu).
368 Returns cov[Nmu, self.r, self.r]. The variance for the PTC is cov[:, 0, 0].
369 mu and cov are in ADUs and ADUs squared. To use electrons for both,
370 the gain should be set to 1. This routine implements the model in Astier+19 (1905.08677).
372 The parameters of the full model for C_ij(mu) ("C_ij" and "mu" in ADU^2 and ADU, respectively)
373 in Astier+19 (Eq. 20) are:
375 "a" coefficients (r by r matrix), units: 1/e
376 "b" coefficients (r by r matrix), units: 1/e
377 noise matrix (r by r matrix), units: e^2
380 "b" appears in Eq. 20 only through the "ab" combination, which is defined in this code as "c=ab".
382 sa = (self.
r, self.
r)
383 a = self.
params[
'a'].full.reshape(sa)
384 c = self.
params[
'c'].full.reshape(sa)
385 gain = self.
params[
'gain'].full[0]
386 noise = self.
params[
'noise'].full.reshape(sa)
388 aEnlarged = np.zeros((int(sa[0]*1.5)+1, int(sa[1]*1.5)+1))
389 aEnlarged[0:sa[0], 0:sa[1]] = a
392 cEnlarged = np.zeros((int(sa[0]*1.5)+1, int(sa[1]*1.5)+1))
393 cEnlarged[0:sa[0], 0:sa[1]] = c
395 a2 = fftconvolve(aSym, aSym, mode=
'same')
396 a3 = fftconvolve(a2, aSym, mode=
'same')
397 ac = fftconvolve(aSym, cSym, mode=
'same')
398 (xc, yc) = np.unravel_index(np.abs(aSym).argmax(), a2.shape)
400 a1 = a[np.newaxis, :, :]
401 a2 = a2[np.newaxis, xc:xc + range, yc:yc + range]
402 a3 = a3[np.newaxis, xc:xc + range, yc:yc + range]
403 ac = ac[np.newaxis, xc:xc + range, yc:yc + range]
404 c1 = c[np.newaxis, ::]
408 bigMu = mu[:, np.newaxis, np.newaxis]*gain
410 covModel = (bigMu/(gain*gain)*(a1*bigMu+2./3.*(bigMu*bigMu)*(a2 + c1) +
411 (1./3.*a3 + 5./6.*ac)*(bigMu*bigMu*bigMu)) + noise[np.newaxis, :, :]/gain**2)
413 covModel[:, 0, 0] += mu/gain
418 """'a' matrix from Astier+19(e.g., Eq. 20)"""
419 return self.
params[
'a'].full.reshape(self.
r, self.
r)
422 """'b' matrix from Astier+19(e.g., Eq. 20)"""
423 return self.
params[
'c'].full.reshape(self.
r, self.
r)/self.
getA()
426 """'c'='ab' matrix from Astier+19(e.g., Eq. 20)"""
427 return self.
params[
'c'].full.reshape(self.
r, self.
r)
429 def _getCovParams(self, what):
430 """Get covariance matrix of parameters from fit"""
431 indices = self.
params[what].indexof()
432 i1 = indices[:, np.newaxis]
433 i2 = indices[np.newaxis, :]
441 """Get covariance matrix of "a" coefficients from fit"""
449 """Square root of diagonal of the parameter covariance of the fitted "a" matrix"""
451 sigA = np.sqrt(self.
_getCovParams(
'a').diagonal()).reshape((self.
r, self.
r))
457 """Get covariance matrix of "a" coefficients from fit
461 aval = self.
getA().flatten()
462 factor = np.outer(aval, aval)
464 return covb.reshape((self.
r, self.
r, self.
r, self.
r))
467 """Get covariance matrix of "c" coefficients from fit"""
469 return cova.reshape((self.
r, self.
r, self.
r, self.
r))
472 """Get error on fitted gain parameter"""
480 """Get covariances of noise matrix from fit"""
482 return covNoise.reshape((self.
r, self.
r, self.
r, self.
r))
485 """Square root of diagonal of the parameter covariance of the fitted "noise" matrix"""
488 noise = np.sqrt(covNoise.diagonal()).reshape((self.
r, self.
r))
494 """Get gain (e/ADU)"""
495 return self.
params[
'gain'].full[0]
498 """Get readout noise (e^2)"""
499 return self.
params[
'noise'].full[0]
502 """Get error on readout noise parameter"""
503 ronSqrt = np.sqrt(np.fabs(self.
getRon()))
506 ronErr = 0.5*(noiseSigma/np.fabs(self.
getRon()))*ronSqrt
512 """Get noise matrix"""
513 return self.
params[
'noise'].full.reshape(self.
r, self.
r)
516 """Get mask of Cov[i,j]"""
517 weights = self.
sqrtW[:, i, j]
522 """Set "a" and "b" coeffcients forfull Astier+19 model (Eq. 20). "c=a*b"."""
523 self.
params[
'a'].full = a.flatten()
524 self.
params[
'c'].full = a.flatten()*b.flatten()
528 """Calculate weighte chi2 of full-model fit."""
532 """To be used in weightedRes"""
533 if params
is not None:
536 return((covModel-self.
cov)*self.
sqrtW)
539 """Weighted residuas.
546 coeffs, cov, _, mesg, ierr = leastsq(c.weightedRes, c.getParamValues(), full_output=True )
548 return self.
wres(params).flatten()
551 """Fit measured covariances to full model in Astier+19 (Eq. 20)
556 Initial parameters of the fit.
557 len(pInit) = #entries(a) + #entries(c) + #entries(noise) + 1
558 len(pInit) = r^2 + r^2 + r^2 + 1, where "r" is the maximum lag considered for the
559 covariances calculation, and the extra "1" is the gain.
560 If "b" is 0, then "c" is 0, and len(pInit) will have r^2 fewer entries.
562 nSigma : `float`, optional
563 Sigma cut to get rid of outliers.
565 maxFitIter : `int`, optional
566 Number of iterations for full model fit.
571 Fit parameters (see "Notes" below).
575 The parameters of the full model for C_ij(mu) ("C_ij" and "mu" in ADU^2 and ADU, respectively)
576 in Astier+19 (Eq. 20) are:
578 "a" coefficients (r by r matrix), units: 1/e
579 "b" coefficients (r by r matrix), units: 1/e
580 noise matrix (r by r matrix), units: e^2
583 "b" appears in Eq. 20 only through the "ab" combination, which is defined in this code as "c=ab".
590 while nOutliers != 0:
591 params, paramsCov, _, mesg, ierr = leastsq(self.
weightedRes, pInit, full_output=
True)
594 sig = mad(wres[wres != 0])
595 mask = (np.abs(wres) > (nSigma*sig))
596 self.
sqrtW.flat[mask] = 0
597 nOutliers = mask.sum()
599 if counter == maxFitIter:
602 if ierr
not in [1, 2, 3, 4]:
603 raise RuntimeError(
"Minimization failed: " + mesg)
609 """Number of degrees of freedom
613 mask.sum() - len(self.params.free): `int`
614 Number of usable pixels - number of parameters of fit.
616 mask = self.
sqrtW != 0
618 return mask.sum() - len(self.
params.free)
620 def getFitData(self, i, j, divideByMu=False, unitsElectrons=False, returnMasked=False):
621 """Get measured signal and covariance, cov model, weigths, and mask.
631 divideByMu: `bool`, optional
632 Divide covariance, model, and weights by signal mu?
634 unitsElectrons : `bool`, optional
635 mu, covariance, and model are in ADU (or powers of ADU) If tthis parameter is true, these are
636 multiplied by the adequte factors of the gain to return quantities in electrons
637 (or powers of electrons).
639 returneMasked : `bool`, optional
640 Use mask (based on weights) in returned arrays (mu, covariance, and model)?
645 list of signal values (mu).
647 covariance: `numpy.array`
648 Covariance arrays, indexed by mean signal mu (self.cov[:, i, j]).
651 Covariance model (model).
653 weights: `numpy.array`
656 mask : `numpy.array`, optional
657 Boolean mask of the covariance at (i,j).
661 Using a CovFit object, selects from (i, j) and returns
662 mu*gain, self.cov[:, i, j]*gain**2 model*gain**2, and self.sqrtW/gain**2
663 in electrons or ADU if unitsElectrons=False.
671 covariance = self.
cov[:, i, j]*(gain**2)
673 weights = self.
sqrtW[:, i, j]/(gain**2)
678 weights = weights[mask]
681 covariance = covariance[mask]
688 return mu, covariance, model, weights, mask