Coverage for python/lsst/cp/pipe/astierCovPtcUtils.py : 10%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of cp_pipe.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22import numpy as np
23from .astierCovPtcFit import CovFit
25__all__ = ['CovFft']
28class CovFft:
29 """A class to compute (via FFT) the nearby pixels correlation function.
31 Implements appendix of Astier+19.
33 Parameters
34 ----------
35 diff: `numpy.array`
36 Image where to calculate the covariances (e.g., the difference image of two flats).
38 w: `numpy.array`
39 Weight image (mask): it should consist of 1's (good pixel) and 0's (bad pixels).
41 fftShape: `tuple`
42 2d-tuple with the shape of the FFT
44 maxRangeCov: `int`
45 Maximum range for the covariances.
46 """
48 def __init__(self, diff, w, fftShape, maxRangeCov):
49 # check that the zero padding implied by "fft_shape"
50 # is large enough for the required correlation range
51 assert(fftShape[0] > diff.shape[0]+maxRangeCov+1)
52 assert(fftShape[1] > diff.shape[1]+maxRangeCov+1)
53 # for some reason related to numpy.fft.rfftn,
54 # the second dimension should be even, so
55 if fftShape[1]%2 == 1:
56 fftShape = (fftShape[0], fftShape[1]+1)
57 tIm = np.fft.rfft2(diff*w, fftShape)
58 tMask = np.fft.rfft2(w, fftShape)
59 # sum of "squares"
60 self.pCov = np.fft.irfft2(tIm*tIm.conjugate())
61 # sum of values
62 self.pMean = np.fft.irfft2(tIm*tMask.conjugate())
63 # number of w!=0 pixels.
64 self.pCount = np.fft.irfft2(tMask*tMask.conjugate())
66 def cov(self, dx, dy):
67 """Covariance for dx,dy averaged with dx,-dy if both non zero.
69 Implements appendix of Astier+19.
71 Parameters
72 ----------
73 dx: `int`
74 Lag in x
76 dy: `int
77 Lag in y
79 Returns
80 -------
81 0.5*(cov1+cov2): `float`
82 Covariance at (dx, dy) lag
84 npix1+npix2: `int`
85 Number of pixels used in covariance calculation.
86 """
87 # compensate rounding errors
88 nPix1 = int(round(self.pCount[dy, dx]))
89 cov1 = self.pCov[dy, dx]/nPix1-self.pMean[dy, dx]*self.pMean[-dy, -dx]/(nPix1*nPix1)
90 if (dx == 0 or dy == 0):
91 return cov1, nPix1
92 nPix2 = int(round(self.pCount[-dy, dx]))
93 cov2 = self.pCov[-dy, dx]/nPix2-self.pMean[-dy, dx]*self.pMean[dy, -dx]/(nPix2*nPix2)
94 return 0.5*(cov1+cov2), nPix1+nPix2
96 def reportCovFft(self, maxRange):
97 """Produce a list of tuples with covariances.
99 Implements appendix of Astier+19.
101 Parameters
102 ----------
103 maxRange: `int`
104 Maximum range of covariances.
106 Returns
107 -------
108 tupleVec: `list`
109 List with covariance tuples.
110 """
111 tupleVec = []
112 # (dy,dx) = (0,0) has to be first
113 for dy in range(maxRange+1):
114 for dx in range(maxRange+1):
115 cov, npix = self.cov(dx, dy)
116 if (dx == 0 and dy == 0):
117 var = cov
118 tupleVec.append((dx, dy, var, cov, npix))
119 return tupleVec
122def fftSize(s):
123 """Calculate the size fof one dimension for the FFT"""
124 x = int(np.log(s)/np.log(2.))
125 return int(2**(x+1))
128def computeCovDirect(diffImage, weightImage, maxRange):
129 """Compute covariances of diffImage in real space.
131 For lags larger than ~25, it is slower than the FFT way.
132 Taken from https://github.com/PierreAstier/bfptc/
134 Parameters
135 ----------
136 diffImage : `numpy.array`
137 Image to compute the covariance of.
139 weightImage : `numpy.array`
140 Weight image of diffImage (1's and 0's for good and bad pixels, respectively).
142 maxRange : `int`
143 Last index of the covariance to be computed.
145 Returns
146 -------
147 outList : `list`
148 List with tuples of the form (dx, dy, var, cov, npix), where:
149 dx : `int`
150 Lag in x
151 dy : `int`
152 Lag in y
153 var : `float`
154 Variance at (dx, dy).
155 cov : `float`
156 Covariance at (dx, dy).
157 nPix : `int`
158 Number of pixel pairs used to evaluate var and cov.
159 """
160 outList = []
161 var = 0
162 # (dy,dx) = (0,0) has to be first
163 for dy in range(maxRange + 1):
164 for dx in range(0, maxRange + 1):
165 if (dx*dy > 0):
166 cov1, nPix1 = covDirectValue(diffImage, weightImage, dx, dy)
167 cov2, nPix2 = covDirectValue(diffImage, weightImage, dx, -dy)
168 cov = 0.5*(cov1 + cov2)
169 nPix = nPix1 + nPix2
170 else:
171 cov, nPix = covDirectValue(diffImage, weightImage, dx, dy)
172 if (dx == 0 and dy == 0):
173 var = cov
174 outList.append((dx, dy, var, cov, nPix))
176 return outList
179def covDirectValue(diffImage, weightImage, dx, dy):
180 """Compute covariances of diffImage in real space at lag (dx, dy).
182 Taken from https://github.com/PierreAstier/bfptc/ (c.f., appendix of Astier+19).
184 Parameters
185 ----------
186 diffImage : `numpy.array`
187 Image to compute the covariance of.
189 weightImage : `numpy.array`
190 Weight image of diffImage (1's and 0's for good and bad pixels, respectively).
192 dx : `int`
193 Lag in x.
195 dy : `int`
196 Lag in y.
198 Returns
199 -------
200 cov : `float`
201 Covariance at (dx, dy)
203 nPix : `int`
204 Number of pixel pairs used to evaluate var and cov.
205 """
206 (nCols, nRows) = diffImage.shape
207 # switching both signs does not change anything:
208 # it just swaps im1 and im2 below
209 if (dx < 0):
210 (dx, dy) = (-dx, -dy)
211 # now, we have dx >0. We have to distinguish two cases
212 # depending on the sign of dy
213 if dy >= 0:
214 im1 = diffImage[dy:, dx:]
215 w1 = weightImage[dy:, dx:]
216 im2 = diffImage[:nCols - dy, :nRows - dx]
217 w2 = weightImage[:nCols - dy, :nRows - dx]
218 else:
219 im1 = diffImage[:nCols + dy, dx:]
220 w1 = weightImage[:nCols + dy, dx:]
221 im2 = diffImage[-dy:, :nRows - dx]
222 w2 = weightImage[-dy:, :nRows - dx]
223 # use the same mask for all 3 calculations
224 wAll = w1*w2
225 # do not use mean() because weightImage=0 pixels would then count
226 nPix = wAll.sum()
227 im1TimesW = im1*wAll
228 s1 = im1TimesW.sum()/nPix
229 s2 = (im2*wAll).sum()/nPix
230 p = (im1TimesW*im2).sum()/nPix
231 cov = p - s1*s2
233 return cov, nPix
236class LoadParams:
237 """
238 A class to prepare covariances for the PTC fit.
240 Parameters
241 ----------
242 r: `int`, optional
243 Maximum lag considered (e.g., to eliminate data beyond a separation "r": ignored in the fit).
245 maxMu: `float`, optional
246 Maximum signal, in ADU (e.g., to eliminate data beyond saturation).
248 maxMuElectrons: `float`, optional
249 Maximum signal in electrons.
251 subtractDistantValue: `bool`, optional
252 Subtract a background to the measured covariances (mandatory for HSC flat pairs)?
254 start: `int`, optional
255 Distance beyond which the subtractDistant model is fitted.
257 offsetDegree: `int`
258 Polynomial degree for the subtraction model.
260 Notes
261 -----
262 params = LoadParams(). "params" drives what happens in he fit. LoadParams provides default values.
263 """
264 def __init__(self):
265 self.r = 8
266 self.maxMu = 1e9
267 self.maxMuElectrons = 1e9
268 self.subtractDistantValue = False
269 self.start = 5
270 self.offsetDegree = 1
273def loadData(tupleName, params):
274 """ Returns a list of CovFit objects, indexed by amp number.
276 Params
277 ------
278 tupleName: `numpy.recarray`
279 Recarray with rows with at least ( mu1, mu2, cov ,var, i, j, npix), where:
280 mu1: mean value of flat1
281 mu2: mean value of flat2
282 cov: covariance value at lag (i, j)
283 var: variance (covariance value at lag (0, 0))
284 i: lag dimension
285 j: lag dimension
286 npix: number of pixels used for covariance calculation.
288 params: `covAstierptcUtil.LoadParams`
289 Object with values to drive the bahaviour of fits.
291 Returns
292 -------
293 covFitList: `dict`
294 Dictionary with amps as keys, and CovFit objects as values.
295 """
297 exts = np.array(np.unique(tupleName['ampName']), dtype=str)
298 covFitList = {}
299 for ext in exts:
300 ntext = tupleName[tupleName['ampName'] == ext]
301 if params.subtractDistantValue:
302 c = CovFit(ntext, params.r)
303 c.subtractDistantOffset(params.r, params.start, params.offsetDegree)
304 else:
305 c = CovFit(ntext, params.r)
306 thisMaxMu = params.maxMu
307 # Tune the maxMuElectrons cut
308 for iter in range(3):
309 cc = c.copy()
310 cc.setMaxMu(thisMaxMu)
311 cc.initFit() # allows to get a crude gain.
312 gain = cc.getGain()
313 if (thisMaxMu*gain < params.maxMuElectrons):
314 thisMaxMu = params.maxMuElectrons/gain
315 continue
316 cc.setMaxMuElectrons(params.maxMuElectrons)
317 break
318 covFitList[ext] = cc
320 return covFitList
323def fitData(tupleName, maxMu=1e9, r=8, nSigmaFullFit=5.5, maxIterFullFit=3):
324 """Fit data to models in Astier+19.
326 Parameters
327 ----------
328 tupleName: `numpy.recarray`
329 Recarray with rows with at least ( mu1, mu2, cov ,var, i, j, npix), where:
330 mu1: mean value of flat1
331 mu2: mean value of flat2
332 cov: covariance value at lag (i, j)
333 var: variance (covariance value at lag (0, 0))
334 i: lag dimension
335 j: lag dimension
336 npix: number of pixels used for covariance calculation.
338 r: `int`, optional
339 Maximum lag considered (e.g., to eliminate data beyond a separation "r": ignored in the fit).
341 maxMu: `float`, optional
342 Maximum signal, in ADU (e.g., to eliminate data beyond saturation).
344 nSigmaFullFit : `float`, optional
345 Sigma cut to get rid of outliers in full model fit.
347 maxIterFullFit : `int`, optional
348 Number of iterations for full model fit.
350 Returns
351 -------
352 covFitList: `dict`
353 Dictionary of CovFit objects, with amp names as keys.
355 covFitNoBList: `dict`
356 Dictionary of CovFit objects, with amp names as keys (b=0 in Eq. 20 of Astier+19).
358 Notes
359 -----
360 The parameters of the full model for C_ij(mu) ("C_ij" and "mu" in ADU^2 and ADU, respectively)
361 in Astier+19 (Eq. 20) are:
363 "a" coefficients (r by r matrix), units: 1/e
364 "b" coefficients (r by r matrix), units: 1/e
365 noise matrix (r by r matrix), units: e^2
366 gain, units: e/ADU
368 "b" appears in Eq. 20 only through the "ab" combination, which is defined in this code as "c=ab".
369 """
371 lparams = LoadParams()
372 lparams.subtractDistantValue = False
373 lparams.maxMu = maxMu
374 lparams.r = r
376 covFitList = loadData(tupleName, lparams)
377 covFitNoBList = {} # [None]*(exts[-1]+1)
378 for ext, c in covFitList.items():
379 c.fitFullModel(nSigma=nSigmaFullFit, maxFitIter=maxIterFullFit)
380 covFitNoBList[ext] = c.copy()
381 c.params['c'].release()
382 c.fitFullModel(nSigma=nSigmaFullFit, maxFitIter=maxIterFullFit)
383 return covFitList, covFitNoBList