lsst.cp.pipe  20.0.0-7-g3c4151b
astierCovPtcUtils.py
Go to the documentation of this file.
1 # This file is part of cp_pipe.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 
22 import numpy as np
23 from .astierCovPtcFit import CovFit
24 
25 __all__ = ['CovFft']
26 
27 
28 class CovFft:
29  """A class to compute (via FFT) the nearby pixels correlation function.
30 
31  Implements appendix of Astier+19.
32 
33  Parameters
34  ----------
35  diff: `numpy.array`
36  Image where to calculate the covariances (e.g., the difference image of two flats).
37 
38  w: `numpy.array`
39  Weight image (mask): it should consist of 1's (good pixel) and 0's (bad pixels).
40 
41  fftShape: `tuple`
42  2d-tuple with the shape of the FFT
43 
44  maxRangeCov: `int`
45  Maximum range for the covariances.
46  """
47 
48  def __init__(self, diff, w, fftShape, maxRangeCov):
49  # check that the zero padding implied by "fft_shape"
50  # is large enough for the required correlation range
51  assert(fftShape[0] > diff.shape[0]+maxRangeCov+1)
52  assert(fftShape[1] > diff.shape[1]+maxRangeCov+1)
53  # for some reason related to numpy.fft.rfftn,
54  # the second dimension should be even, so
55  if fftShape[1]%2 == 1:
56  fftShape = (fftShape[0], fftShape[1]+1)
57  tIm = np.fft.rfft2(diff*w, fftShape)
58  tMask = np.fft.rfft2(w, fftShape)
59  # sum of "squares"
60  self.pCov = np.fft.irfft2(tIm*tIm.conjugate())
61  # sum of values
62  self.pMean = np.fft.irfft2(tIm*tMask.conjugate())
63  # number of w!=0 pixels.
64  self.pCount = np.fft.irfft2(tMask*tMask.conjugate())
65 
66  def cov(self, dx, dy):
67  """Covariance for dx,dy averaged with dx,-dy if both non zero.
68 
69  Implements appendix of Astier+19.
70 
71  Parameters
72  ----------
73  dx: `int`
74  Lag in x
75 
76  dy: `int
77  Lag in y
78 
79  Returns
80  -------
81  0.5*(cov1+cov2): `float`
82  Covariance at (dx, dy) lag
83 
84  npix1+npix2: `int`
85  Number of pixels used in covariance calculation.
86  """
87  # compensate rounding errors
88  nPix1 = int(round(self.pCount[dy, dx]))
89  cov1 = self.pCov[dy, dx]/nPix1-self.pMean[dy, dx]*self.pMean[-dy, -dx]/(nPix1*nPix1)
90  if (dx == 0 or dy == 0):
91  return cov1, nPix1
92  nPix2 = int(round(self.pCount[-dy, dx]))
93  cov2 = self.pCov[-dy, dx]/nPix2-self.pMean[-dy, dx]*self.pMean[dy, -dx]/(nPix2*nPix2)
94  return 0.5*(cov1+cov2), nPix1+nPix2
95 
96  def reportCovFft(self, maxRange):
97  """Produce a list of tuples with covariances.
98 
99  Implements appendix of Astier+19.
100 
101  Parameters
102  ----------
103  maxRange: `int`
104  Maximum range of covariances.
105 
106  Returns
107  -------
108  tupleVec: `list`
109  List with covariance tuples.
110  """
111  tupleVec = []
112  # (dy,dx) = (0,0) has to be first
113  for dy in range(maxRange+1):
114  for dx in range(maxRange+1):
115  cov, npix = self.cov(dx, dy)
116  if (dx == 0 and dy == 0):
117  var = cov
118  tupleVec.append((dx, dy, var, cov, npix))
119  return tupleVec
120 
121 
122 def fftSize(s):
123  """Calculate the size fof one dimension for the FFT"""
124  x = int(np.log(s)/np.log(2.))
125  return int(2**(x+1))
126 
127 
128 def computeCovDirect(diffImage, weightImage, maxRange):
129  """Compute covariances of diffImage in real space.
130 
131  For lags larger than ~25, it is slower than the FFT way.
132  Taken from https://github.com/PierreAstier/bfptc/
133 
134  Parameters
135  ----------
136  diffImage : `numpy.array`
137  Image to compute the covariance of.
138 
139  weightImage : `numpy.array`
140  Weight image of diffImage (1's and 0's for good and bad pixels, respectively).
141 
142  maxRange : `int`
143  Last index of the covariance to be computed.
144 
145  Returns
146  -------
147  outList : `list`
148  List with tuples of the form (dx, dy, var, cov, npix), where:
149  dx : `int`
150  Lag in x
151  dy : `int`
152  Lag in y
153  var : `float`
154  Variance at (dx, dy).
155  cov : `float`
156  Covariance at (dx, dy).
157  nPix : `int`
158  Number of pixel pairs used to evaluate var and cov.
159  """
160  outList = []
161  var = 0
162  # (dy,dx) = (0,0) has to be first
163  for dy in range(maxRange + 1):
164  for dx in range(0, maxRange + 1):
165  if (dx*dy > 0):
166  cov1, nPix1 = covDirectValue(diffImage, weightImage, dx, dy)
167  cov2, nPix2 = covDirectValue(diffImage, weightImage, dx, -dy)
168  cov = 0.5*(cov1 + cov2)
169  nPix = nPix1 + nPix2
170  else:
171  cov, nPix = covDirectValue(diffImage, weightImage, dx, dy)
172  if (dx == 0 and dy == 0):
173  var = cov
174  outList.append((dx, dy, var, cov, nPix))
175 
176  return outList
177 
178 
179 def covDirectValue(diffImage, weightImage, dx, dy):
180  """Compute covariances of diffImage in real space at lag (dx, dy).
181 
182  Taken from https://github.com/PierreAstier/bfptc/ (c.f., appendix of Astier+19).
183 
184  Parameters
185  ----------
186  diffImage : `numpy.array`
187  Image to compute the covariance of.
188 
189  weightImage : `numpy.array`
190  Weight image of diffImage (1's and 0's for good and bad pixels, respectively).
191 
192  dx : `int`
193  Lag in x.
194 
195  dy : `int`
196  Lag in y.
197 
198  Returns
199  -------
200  cov : `float`
201  Covariance at (dx, dy)
202 
203  nPix : `int`
204  Number of pixel pairs used to evaluate var and cov.
205  """
206  (nCols, nRows) = diffImage.shape
207  # switching both signs does not change anything:
208  # it just swaps im1 and im2 below
209  if (dx < 0):
210  (dx, dy) = (-dx, -dy)
211  # now, we have dx >0. We have to distinguish two cases
212  # depending on the sign of dy
213  if dy >= 0:
214  im1 = diffImage[dy:, dx:]
215  w1 = weightImage[dy:, dx:]
216  im2 = diffImage[:nCols - dy, :nRows - dx]
217  w2 = weightImage[:nCols - dy, :nRows - dx]
218  else:
219  im1 = diffImage[:nCols + dy, dx:]
220  w1 = weightImage[:nCols + dy, dx:]
221  im2 = diffImage[-dy:, :nRows - dx]
222  w2 = weightImage[-dy:, :nRows - dx]
223  # use the same mask for all 3 calculations
224  wAll = w1*w2
225  # do not use mean() because weightImage=0 pixels would then count
226  nPix = wAll.sum()
227  im1TimesW = im1*wAll
228  s1 = im1TimesW.sum()/nPix
229  s2 = (im2*wAll).sum()/nPix
230  p = (im1TimesW*im2).sum()/nPix
231  cov = p - s1*s2
232 
233  return cov, nPix
234 
235 
237  """
238  A class to prepare covariances for the PTC fit.
239 
240  Parameters
241  ----------
242  r: `int`, optional
243  Maximum lag considered (e.g., to eliminate data beyond a separation "r": ignored in the fit).
244 
245  maxMu: `float`, optional
246  Maximum signal, in ADU (e.g., to eliminate data beyond saturation).
247 
248  maxMuElectrons: `float`, optional
249  Maximum signal in electrons.
250 
251  subtractDistantValue: `bool`, optional
252  Subtract a background to the measured covariances (mandatory for HSC flat pairs)?
253 
254  start: `int`, optional
255  Distance beyond which the subtractDistant model is fitted.
256 
257  offsetDegree: `int`
258  Polynomial degree for the subtraction model.
259 
260  Notes
261  -----
262  params = LoadParams(). "params" drives what happens in he fit. LoadParams provides default values.
263  """
264  def __init__(self):
265  self.r = 8
266  self.maxMu = 1e9
267  self.maxMuElectrons = 1e9
268  self.subtractDistantValue = False
269  self.start = 5
270  self.offsetDegree = 1
271 
272 
273 def loadData(tupleName, params):
274  """ Returns a list of CovFit objects, indexed by amp number.
275 
276  Params
277  ------
278  tupleName: `numpy.recarray`
279  Recarray with rows with at least ( mu1, mu2, cov ,var, i, j, npix), where:
280  mu1: mean value of flat1
281  mu2: mean value of flat2
282  cov: covariance value at lag (i, j)
283  var: variance (covariance value at lag (0, 0))
284  i: lag dimension
285  j: lag dimension
286  npix: number of pixels used for covariance calculation.
287 
288  params: `covAstierptcUtil.LoadParams`
289  Object with values to drive the bahaviour of fits.
290 
291  Returns
292  -------
293  covFitList: `dict`
294  Dictionary with amps as keys, and CovFit objects as values.
295  """
296 
297  exts = np.array(np.unique(tupleName['ampName']), dtype=str)
298  covFitList = {}
299  for ext in exts:
300  ntext = tupleName[tupleName['ampName'] == ext]
301  if params.subtractDistantValue:
302  c = CovFit(ntext, params.r)
303  c.subtractDistantOffset(params.r, params.start, params.offsetDegree)
304  else:
305  c = CovFit(ntext, params.r)
306  thisMaxMu = params.maxMu
307  # Tune the maxMuElectrons cut
308  for iter in range(3):
309  cc = c.copy()
310  cc.setMaxMu(thisMaxMu)
311  cc.initFit() # allows to get a crude gain.
312  gain = cc.getGain()
313  if (thisMaxMu*gain < params.maxMuElectrons):
314  thisMaxMu = params.maxMuElectrons/gain
315  continue
316  cc.setMaxMuElectrons(params.maxMuElectrons)
317  break
318  covFitList[ext] = cc
319 
320  return covFitList
321 
322 
323 def fitData(tupleName, maxMu=1e9, r=8):
324  """Fit data to models in Astier+19.
325 
326  Parameters
327  ----------
328  tupleName: `numpy.recarray`
329  Recarray with rows with at least ( mu1, mu2, cov ,var, i, j, npix), where:
330  mu1: mean value of flat1
331  mu2: mean value of flat2
332  cov: covariance value at lag (i, j)
333  var: variance (covariance value at lag (0, 0))
334  i: lag dimension
335  j: lag dimension
336  npix: number of pixels used for covariance calculation.
337 
338  r: `int`, optional
339  Maximum lag considered (e.g., to eliminate data beyond a separation "r": ignored in the fit).
340 
341  maxMu: `float`, optional
342  Maximum signal, in ADU (e.g., to eliminate data beyond saturation).
343 
344  Returns
345  -------
346  covFitList: `dict`
347  Dictionary of CovFit objects, with amp names as keys.
348 
349  covFitNoBList: `dict`
350  Dictionary of CovFit objects, with amp names as keys (b=0 in Eq. 20 of Astier+19).
351 
352  Notes
353  -----
354  The parameters of the full model for C_ij(mu) ("C_ij" and "mu" in ADU^2 and ADU, respectively)
355  in Astier+19 (Eq. 20) are:
356 
357  "a" coefficients (r by r matrix), units: 1/e
358  "b" coefficients (r by r matrix), units: 1/e
359  noise matrix (r by r matrix), units: e^2
360  gain, units: e/ADU
361 
362  "b" appears in Eq. 20 only through the "ab" combination, which is defined in this code as "c=ab".
363  """
364 
365  lparams = LoadParams()
366  lparams.subtractDistantValue = False
367  lparams.maxMu = maxMu
368  lparams.r = r
369 
370  covFitList = loadData(tupleName, lparams)
371  covFitNoBList = {} # [None]*(exts[-1]+1)
372  for ext, c in covFitList.items():
373  c.fitFullModel()
374  covFitNoBList[ext] = c.copy()
375  c.params['c'].release()
376  c.fitFullModel()
377  return covFitList, covFitNoBList
lsst.cp.pipe.astierCovPtcUtils.CovFft.reportCovFft
def reportCovFft(self, maxRange)
Definition: astierCovPtcUtils.py:96
lsst.cp.pipe.astierCovPtcUtils.LoadParams.subtractDistantValue
subtractDistantValue
Definition: astierCovPtcUtils.py:268
lsst.cp.pipe.astierCovPtcUtils.fftSize
def fftSize(s)
Definition: astierCovPtcUtils.py:122
lsst.cp.pipe.astierCovPtcUtils.CovFft.__init__
def __init__(self, diff, w, fftShape, maxRangeCov)
Definition: astierCovPtcUtils.py:48
lsst.cp.pipe.astierCovPtcUtils.CovFft.pMean
pMean
Definition: astierCovPtcUtils.py:62
lsst.cp.pipe.astierCovPtcUtils.covDirectValue
def covDirectValue(diffImage, weightImage, dx, dy)
Definition: astierCovPtcUtils.py:179
lsst.cp.pipe.astierCovPtcUtils.LoadParams.start
start
Definition: astierCovPtcUtils.py:269
lsst.cp.pipe.astierCovPtcUtils.computeCovDirect
def computeCovDirect(diffImage, weightImage, maxRange)
Definition: astierCovPtcUtils.py:128
lsst.cp.pipe.astierCovPtcUtils.LoadParams.maxMu
maxMu
Definition: astierCovPtcUtils.py:266
lsst.cp.pipe.astierCovPtcUtils.CovFft.cov
def cov(self, dx, dy)
Definition: astierCovPtcUtils.py:66
lsst.cp.pipe.astierCovPtcUtils.CovFft.pCount
pCount
Definition: astierCovPtcUtils.py:64
lsst.cp.pipe.astierCovPtcFit.CovFit
Definition: astierCovPtcFit.py:211
lsst.cp.pipe.astierCovPtcUtils.LoadParams.r
r
Definition: astierCovPtcUtils.py:265
lsst.cp.pipe.astierCovPtcUtils.CovFft.pCov
pCov
Definition: astierCovPtcUtils.py:60
lsst.cp.pipe.astierCovPtcUtils.fitData
def fitData(tupleName, maxMu=1e9, r=8)
Definition: astierCovPtcUtils.py:323
lsst.cp.pipe.astierCovPtcUtils.CovFft
Definition: astierCovPtcUtils.py:28
lsst.cp.pipe.astierCovPtcUtils.LoadParams
Definition: astierCovPtcUtils.py:236
lsst.cp.pipe.astierCovPtcUtils.LoadParams.offsetDegree
offsetDegree
Definition: astierCovPtcUtils.py:270
lsst.cp.pipe.astierCovPtcUtils.LoadParams.maxMuElectrons
maxMuElectrons
Definition: astierCovPtcUtils.py:267
lsst.cp.pipe.astierCovPtcUtils.LoadParams.__init__
def __init__(self)
Definition: astierCovPtcUtils.py:264
lsst.cp.pipe.astierCovPtcUtils.loadData
def loadData(tupleName, params)
Definition: astierCovPtcUtils.py:273