25 from scipy.signal
import fftconvolve
26 from scipy.optimize
import leastsq
27 from .astierCovFitParameters
import FitParameters
35 """Compute the "a" coefficients of the Antilogus+14 (1402.0725) model as in
36 Guyonnet+15 (1501.01577, eq. 16, the slope of cov/var at a given flux mu in electrons).
38 Eq. 16 of 1501.01577 is an approximation to the more complete model in Astier+19 (1905.08677).
43 Covariance model from Eq. 20 in Astier+19.
46 Mean signal in electrons
53 aCoeffsOld: `numpy.array`
54 Slope of cov/var at a given flux mu in electrons.
58 Returns the "a" array, computed this way, to be compared to the actual a_array from the full model
61 covModel = np.array(covModel)
62 var = covModel[0, 0, 0]
64 return covModel[0, :, :]/(var*muEl)
68 """Make covariances array from tuple.
72 inputTuple: `numpy.ndarray`
73 Structured array with rows with at least
74 (mu, afwVar, cov, var, i, j, npix), where:
76 mu : 0.5*(m1 + m2), where:
77 mu1: mean value of flat1
78 mu2: mean value of flat2
79 afwVar: variance of difference flat, calculated with afw
80 cov: covariance value at lag(i, j)
81 var: variance(covariance value at lag(0, 0))
84 npix: number of pixels used for covariance calculation.
86 maxRangeFromTuple: `int`
87 Maximum range to select from tuple.
92 Covariance arrays, indexed by mean signal mu.
95 Variance arrays, indexed by mean signal mu.
98 List of mean signal values.
103 The input tuple should contain the following rows:
104 (mu, cov, var, i, j, npix), with one entry per lag, and image pair.
105 Different lags(i.e. different i and j) from the same
106 image pair have the same values of mu1 and mu2. When i==j==0, cov
109 If the input tuple contains several video channels, one should
110 select the data of a given channel *before* entering this
111 routine, as well as apply(e.g.) saturation cuts.
113 The routine returns cov[k_mu, j, i], vcov[(same indices)], and mu[k]
114 where the first index of cov matches the one in mu.
116 This routine implements the loss of variance due to
117 clipping cuts when measuring variances and covariance, but this should happen inside
118 the measurement code, where the cuts are readily available.
121 if maxRangeFromTuple
is not None:
122 cut = (inputTuple[
'i'] < maxRangeFromTuple) & (inputTuple[
'j'] < maxRangeFromTuple)
123 cutTuple = inputTuple[cut]
125 cutTuple = inputTuple
127 muTemp = cutTuple[
'mu']
128 ind = np.argsort(muTemp)
130 cutTuple = cutTuple[ind]
133 xx = np.hstack(([mu[0]], mu))
134 delta = xx[1:] - xx[:-1]
135 steps, = np.where(delta > 0)
136 ind = np.zeros_like(mu, dtype=int)
140 muVals = np.array(np.unique(mu))
141 i = cutTuple[
'i'].astype(int)
142 j = cutTuple[
'j'].astype(int)
143 c = 0.5*cutTuple[
'cov']
145 v = 0.5*cutTuple[
'var']
147 cov = np.ndarray((len(muVals), np.max(i)+1, np.max(j)+1))
148 var = np.zeros_like(cov)
150 var[ind, i, j] = v**2/n
159 return cov, var, muVals
163 """ Copy array over 4 quadrants prior to convolution.
167 inputarray: `numpy.array`
168 Input array to symmetrize.
176 targetShape = list(inputArray.shape)
177 r1, r2 = inputArray.shape[-1], inputArray.shape[-2]
178 targetShape[-1] = 2*r1-1
179 targetShape[-2] = 2*r2-1
180 aSym = np.ndarray(tuple(targetShape))
181 aSym[..., r2-1:, r1-1:] = inputArray
182 aSym[..., r2-1:, r1-1::-1] = inputArray
183 aSym[..., r2-1::-1, r1-1::-1] = inputArray
184 aSym[..., r2-1::-1, r1-1:] = inputArray
190 """A class to calculate 2D polynomials"""
197 self.coeff, _, rank, _ = np.linalg.lstsq(G, z.ravel())
199 self.coeff, _, rank, _ = np.linalg.lstsq((w.ravel()*G.T).T, z.ravel()*w.ravel())
203 G = np.zeros(x.shape + (ncols,))
204 ij = itertools.product(range(self.
orderx+1), range(self.
ordery+1))
205 for k, (i, j)
in enumerate(ij):
206 G[..., k] = x**i * y**j
213 return np.dot(G, self.coeff)
217 """A class to fit the models in Astier+19 to flat covariances.
219 This code implements the model(and the fit thereof) described in
220 Astier+19: https://arxiv.org/pdf/1905.08677.pdf
224 meanSignals : `list`[`float`]
225 List with means of the difference image of two flats,
226 for a particular amplifier in the detector.
228 covariances : `list`[`numpy.array`]
229 List with 2D covariance arrays at a given mean signal.
231 covsSqrtWeights : `list`[`numpy.array`]
232 List with 2D arrays with weights from `vcov as defined in
233 `makeCovArray`: weight = 1/sqrt(vcov).
235 maxRangeFromTuple: `int`, optional
236 Maximum range to select from tuple.
238 meanSignalMask: `list`[`bool`], optional
239 Mask of mean signal 1D array. Use all entries if empty.
242 def __init__(self, meanSignals, covariances, covsSqrtWeights, maxRangeFromTuple=8, meanSignalsMask=[]):
243 assert (len(meanSignals) == len(covariances))
244 assert (len(covariances) == len(covsSqrtWeights))
245 if len(meanSignalsMask) == 0:
246 meanSignalsMask = np.repeat(
True, len(meanSignals))
247 self.
mu = meanSignals[meanSignalsMask]
248 self.
cov = np.nan_to_num(covariances)[meanSignalsMask]
250 self.
sqrtW = np.nan_to_num(covsSqrtWeights)[meanSignalsMask]
251 self.
r = maxRangeFromTuple
252 self.
logger = lsstLog.Log.getDefaultLogger()
255 """Subtract a background/offset to the measured covariances.
260 Maximum lag considered
263 First lag from where to start the offset subtraction.
266 Degree of 2D polynomial to fit to covariance to define offse to be subtracted.
268 assert(startLag < self.
r)
269 for k
in range(len(self.
mu)):
271 w = self.
sqrtW[k, ...] + 0.
273 i, j = np.meshgrid(range(sh[0]), range(sh[1]), indexing=
'ij')
275 w[:startLag, :startLag] = 0
276 poly =
Pol2d(i, j, self.
cov[k, ...], polDegree+1, w=w)
277 back = poly.eval(i, j)
278 self.
cov[k, ...] -= back
280 self.
cov = self.
cov[:, :maxLag, :maxLag]
286 """Make a copy of params"""
287 cop = copy.deepcopy(self)
289 if hasattr(self,
'params'):
294 """ Performs a crude parabolic fit of the data in order to start
295 the full fit close to the solution.
303 self.
params[
'c'].fix(val=0.)
305 a = self.
params[
'a'].full.reshape(self.
r, self.
r)
306 noise = self.
params[
'noise'].full.reshape(self.
r, self.
r)
307 gain = self.
params[
'gain'].full[0]
316 for i
in range(self.
r):
317 for j
in range(self.
r):
319 parsFit = np.polyfit(self.
mu, self.
cov[:, i, j] - model[:, i, j],
320 2, w=self.
sqrtW[:, i, j])
322 a[i, j] += parsFit[0]
323 noise[i, j] += parsFit[2]*gain*gain
325 gain = 1./(1/gain+parsFit[1])
326 self.
params[
'gain'].full[0] = gain
335 """Return an array of free parameter values (it is a copy)."""
336 return self.
params.free + 0.
339 """Set parameter values."""
344 """Computes full covariances model (Eq. 20 of Astier+19).
348 mu: `numpy.array`, optional
349 List of mean signals.
353 covModel: `numpy.array`
358 By default, computes the covModel for the mu's stored(self.mu).
360 Returns cov[Nmu, self.r, self.r]. The variance for the PTC is cov[:, 0, 0].
361 mu and cov are in ADUs and ADUs squared. To use electrons for both,
362 the gain should be set to 1. This routine implements the model in Astier+19 (1905.08677).
364 The parameters of the full model for C_ij(mu) ("C_ij" and "mu" in ADU^2 and ADU, respectively)
365 in Astier+19 (Eq. 20) are:
367 "a" coefficients (r by r matrix), units: 1/e
368 "b" coefficients (r by r matrix), units: 1/e
369 noise matrix (r by r matrix), units: e^2
372 "b" appears in Eq. 20 only through the "ab" combination, which is defined in this code as "c=ab".
374 sa = (self.
r, self.
r)
375 a = self.
params[
'a'].full.reshape(sa)
376 c = self.
params[
'c'].full.reshape(sa)
377 gain = self.
params[
'gain'].full[0]
378 noise = self.
params[
'noise'].full.reshape(sa)
380 aEnlarged = np.zeros((int(sa[0]*1.5)+1, int(sa[1]*1.5)+1))
381 aEnlarged[0:sa[0], 0:sa[1]] = a
384 cEnlarged = np.zeros((int(sa[0]*1.5)+1, int(sa[1]*1.5)+1))
385 cEnlarged[0:sa[0], 0:sa[1]] = c
387 a2 = fftconvolve(aSym, aSym, mode=
'same')
388 a3 = fftconvolve(a2, aSym, mode=
'same')
389 ac = fftconvolve(aSym, cSym, mode=
'same')
390 (xc, yc) = np.unravel_index(np.abs(aSym).argmax(), a2.shape)
392 a1 = a[np.newaxis, :, :]
393 a2 = a2[np.newaxis, xc:xc + range, yc:yc + range]
394 a3 = a3[np.newaxis, xc:xc + range, yc:yc + range]
395 ac = ac[np.newaxis, xc:xc + range, yc:yc + range]
396 c1 = c[np.newaxis, ::]
400 bigMu = mu[:, np.newaxis, np.newaxis]*gain
402 covModel = (bigMu/(gain*gain)*(a1*bigMu+2./3.*(bigMu*bigMu)*(a2 + c1) +
403 (1./3.*a3 + 5./6.*ac)*(bigMu*bigMu*bigMu)) + noise[np.newaxis, :, :]/gain**2)
405 covModel[:, 0, 0] += mu/gain
410 """'a' matrix from Astier+19(e.g., Eq. 20)"""
411 return self.
params[
'a'].full.reshape(self.
r, self.
r)
414 """'b' matrix from Astier+19(e.g., Eq. 20)"""
415 return self.
params[
'c'].full.reshape(self.
r, self.
r)/self.
getA()
418 """'c'='ab' matrix from Astier+19(e.g., Eq. 20)"""
419 return np.array(self.
params[
'c'].full.reshape(self.
r, self.
r))
421 def _getCovParams(self, what):
422 """Get covariance matrix of parameters from fit"""
423 indices = self.
params[what].indexof()
424 i1 = indices[:, np.newaxis]
425 i2 = indices[np.newaxis, :]
433 """Get covariance matrix of "a" coefficients from fit"""
441 """Square root of diagonal of the parameter covariance of the fitted "a" matrix"""
443 sigA = np.sqrt(self.
_getCovParams(
'a').diagonal()).reshape((self.
r, self.
r))
449 """Get covariance matrix of "a" coefficients from fit
453 aval = self.
getA().flatten()
454 factor = np.outer(aval, aval)
456 return covb.reshape((self.
r, self.
r, self.
r, self.
r))
459 """Get covariance matrix of "c" coefficients from fit"""
461 return cova.reshape((self.
r, self.
r, self.
r, self.
r))
464 """Get error on fitted gain parameter"""
472 """Get covariances of noise matrix from fit"""
474 return covNoise.reshape((self.
r, self.
r, self.
r, self.
r))
477 """Square root of diagonal of the parameter covariance of the fitted "noise" matrix"""
480 noise = np.sqrt(covNoise.diagonal()).reshape((self.
r, self.
r))
486 """Get gain (e/ADU)"""
487 return self.
params[
'gain'].full[0]
490 """Get readout noise (e^2)"""
491 return self.
params[
'noise'].full[0]
494 """Get error on readout noise parameter"""
495 ronSqrt = np.sqrt(np.fabs(self.
getRon()))
498 ronErr = 0.5*(noiseSigma/np.fabs(self.
getRon()))*ronSqrt
504 """Get noise matrix"""
505 return self.
params[
'noise'].full.reshape(self.
r, self.
r)
508 """Get mask of Cov[i,j]"""
509 weights = self.
sqrtW[:, i, j]
514 """Set "a" and "b" coeffcients forfull Astier+19 model (Eq. 20). "c=a*b"."""
515 self.
params[
'a'].full = a.flatten()
516 self.
params[
'c'].full = a.flatten()*b.flatten()
520 """Calculate weighted chi2 of full-model fit."""
524 """To be used in weightedRes"""
525 if params
is not None:
528 weightedRes = (covModel-self.
cov)*self.
sqrtW
529 maskedWeightedRes = weightedRes
531 return maskedWeightedRes
534 """Weighted residuals.
539 c = CovFit(meanSignals, covariances, covsSqrtWeights)
541 coeffs, cov, _, mesg, ierr = leastsq(c.weightedRes, c.getParamValues(), full_output=True)
543 return self.
wres(params).flatten()
546 """Fit measured covariances to full model in Astier+19 (Eq. 20)
551 Initial parameters of the fit.
552 len(pInit) = #entries(a) + #entries(c) + #entries(noise) + 1
553 len(pInit) = r^2 + r^2 + r^2 + 1, where "r" is the maximum lag considered for the
554 covariances calculation, and the extra "1" is the gain.
555 If "b" is 0, then "c" is 0, and len(pInit) will have r^2 fewer entries.
560 Fit parameters (see "Notes" below).
564 The parameters of the full model for C_ij(mu) ("C_ij" and "mu" in ADU^2 and ADU, respectively)
565 in Astier+19 (Eq. 20) are:
567 "a" coefficients (r by r matrix), units: 1/e
568 "b" coefficients (r by r matrix), units: 1/e
569 noise matrix (r by r matrix), units: e^2
572 "b" appears in Eq. 20 only through the "ab" combination, which is defined in this code as "c=ab".
577 params, paramsCov, _, mesg, ierr = leastsq(self.
weightedRes, pInit, full_output=
True)
583 """Number of degrees of freedom
587 mask.sum() - len(self.params.free): `int`
588 Number of usable pixels - number of parameters of fit.
590 mask = self.
sqrtW != 0
592 return mask.sum() - len(self.
params.free)
594 def getFitData(self, i, j, divideByMu=False, unitsElectrons=False, returnMasked=False):
595 """Get measured signal and covariance, cov model, weigths, and mask at covariance lag (i, j).
600 Lag for covariance matrix.
603 Lag for covariance matrix.
605 divideByMu: `bool`, optional
606 Divide covariance, model, and weights by signal mu?
608 unitsElectrons : `bool`, optional
609 mu, covariance, and model are in ADU (or powers of ADU) If tthis parameter is true, these are
610 multiplied by the adequte factors of the gain to return quantities in electrons
611 (or powers of electrons).
613 returnMasked : `bool`, optional
614 Use mask (based on weights) in returned arrays (mu, covariance, and model)?
619 list of signal values (mu).
621 covariance: `numpy.array`
622 Covariance arrays, indexed by mean signal mu (self.cov[:, i, j]).
624 covarianceModel: `numpy.array`
625 Covariance model (model).
627 weights: `numpy.array`
630 mask : `numpy.array`, optional
631 Boolean mask of the covariance at (i,j).
635 Using a CovFit object, selects from (i, j) and returns
636 mu*gain, self.cov[:, i, j]*gain**2 model*gain**2, and self.sqrtW/gain**2
637 in electrons or ADU if unitsElectrons=False.
645 covariance = self.
cov[:, i, j]*(gain**2)
646 covarianceModel = self.
evalCovModel()[:, i, j]*(gain**2)
647 weights = self.
sqrtW[:, i, j]/(gain**2)
652 weights = weights[mask]
653 covarianceModel = covarianceModel[mask]
655 covariance = covariance[mask]
659 covarianceModel /= mu
662 return mu, covariance, covarianceModel, weights, mask