lsst.cp.pipe  20.0.0-31-g42917e2+deb446c958
astierCovPtcUtils.py
Go to the documentation of this file.
1 # This file is part of cp_pipe.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 
22 import numpy as np
23 from .astierCovPtcFit import CovFit
24 
25 __all__ = ['CovFft']
26 
27 
28 class CovFft:
29  """A class to compute (via FFT) the nearby pixels correlation function.
30 
31  Implements appendix of Astier+19.
32 
33  Parameters
34  ----------
35  diff: `numpy.array`
36  Image where to calculate the covariances (e.g., the difference image of two flats).
37 
38  w: `numpy.array`
39  Weight image (mask): it should consist of 1's (good pixel) and 0's (bad pixels).
40 
41  fftShape: `tuple`
42  2d-tuple with the shape of the FFT
43 
44  maxRangeCov: `int`
45  Maximum range for the covariances.
46  """
47 
48  def __init__(self, diff, w, fftShape, maxRangeCov):
49  # check that the zero padding implied by "fft_shape"
50  # is large enough for the required correlation range
51  assert(fftShape[0] > diff.shape[0]+maxRangeCov+1)
52  assert(fftShape[1] > diff.shape[1]+maxRangeCov+1)
53  # for some reason related to numpy.fft.rfftn,
54  # the second dimension should be even, so
55  if fftShape[1]%2 == 1:
56  fftShape = (fftShape[0], fftShape[1]+1)
57  tIm = np.fft.rfft2(diff*w, fftShape)
58  tMask = np.fft.rfft2(w, fftShape)
59  # sum of "squares"
60  self.pCov = np.fft.irfft2(tIm*tIm.conjugate())
61  # sum of values
62  self.pMean = np.fft.irfft2(tIm*tMask.conjugate())
63  # number of w!=0 pixels.
64  self.pCount = np.fft.irfft2(tMask*tMask.conjugate())
65 
66  def cov(self, dx, dy):
67  """Covariance for dx,dy averaged with dx,-dy if both non zero.
68 
69  Implements appendix of Astier+19.
70 
71  Parameters
72  ----------
73  dx: `int`
74  Lag in x
75 
76  dy: `int
77  Lag in y
78 
79  Returns
80  -------
81  0.5*(cov1+cov2): `float`
82  Covariance at (dx, dy) lag
83 
84  npix1+npix2: `int`
85  Number of pixels used in covariance calculation.
86  """
87  # compensate rounding errors
88  nPix1 = int(round(self.pCount[dy, dx]))
89  cov1 = self.pCov[dy, dx]/nPix1-self.pMean[dy, dx]*self.pMean[-dy, -dx]/(nPix1*nPix1)
90  if (dx == 0 or dy == 0):
91  return cov1, nPix1
92  nPix2 = int(round(self.pCount[-dy, dx]))
93  cov2 = self.pCov[-dy, dx]/nPix2-self.pMean[-dy, dx]*self.pMean[dy, -dx]/(nPix2*nPix2)
94  return 0.5*(cov1+cov2), nPix1+nPix2
95 
96  def reportCovFft(self, maxRange):
97  """Produce a list of tuples with covariances.
98 
99  Implements appendix of Astier+19.
100 
101  Parameters
102  ----------
103  maxRange: `int`
104  Maximum range of covariances.
105 
106  Returns
107  -------
108  tupleVec: `list`
109  List with covariance tuples.
110  """
111  tupleVec = []
112  # (dy,dx) = (0,0) has to be first
113  for dy in range(maxRange+1):
114  for dx in range(maxRange+1):
115  cov, npix = self.cov(dx, dy)
116  if (dx == 0 and dy == 0):
117  var = cov
118  tupleVec.append((dx, dy, var, cov, npix))
119  return tupleVec
120 
121 
122 def fftSize(s):
123  """Calculate the size fof one dimension for the FFT"""
124  x = int(np.log(s)/np.log(2.))
125  return int(2**(x+1))
126 
127 
128 def computeCovDirect(diffImage, weightImage, maxRange):
129  """Compute covariances of diffImage in real space.
130 
131  For lags larger than ~25, it is slower than the FFT way.
132  Taken from https://github.com/PierreAstier/bfptc/
133 
134  Parameters
135  ----------
136  diffImage : `numpy.array`
137  Image to compute the covariance of.
138 
139  weightImage : `numpy.array`
140  Weight image of diffImage (1's and 0's for good and bad pixels, respectively).
141 
142  maxRange : `int`
143  Last index of the covariance to be computed.
144 
145  Returns
146  -------
147  outList : `list`
148  List with tuples of the form (dx, dy, var, cov, npix), where:
149  dx : `int`
150  Lag in x
151  dy : `int`
152  Lag in y
153  var : `float`
154  Variance at (dx, dy).
155  cov : `float`
156  Covariance at (dx, dy).
157  nPix : `int`
158  Number of pixel pairs used to evaluate var and cov.
159  """
160  outList = []
161  var = 0
162  # (dy,dx) = (0,0) has to be first
163  for dy in range(maxRange + 1):
164  for dx in range(0, maxRange + 1):
165  if (dx*dy > 0):
166  cov1, nPix1 = covDirectValue(diffImage, weightImage, dx, dy)
167  cov2, nPix2 = covDirectValue(diffImage, weightImage, dx, -dy)
168  cov = 0.5*(cov1 + cov2)
169  nPix = nPix1 + nPix2
170  else:
171  cov, nPix = covDirectValue(diffImage, weightImage, dx, dy)
172  if (dx == 0 and dy == 0):
173  var = cov
174  outList.append((dx, dy, var, cov, nPix))
175 
176  return outList
177 
178 
179 def covDirectValue(diffImage, weightImage, dx, dy):
180  """Compute covariances of diffImage in real space at lag (dx, dy).
181 
182  Taken from https://github.com/PierreAstier/bfptc/ (c.f., appendix of Astier+19).
183 
184  Parameters
185  ----------
186  diffImage : `numpy.array`
187  Image to compute the covariance of.
188 
189  weightImage : `numpy.array`
190  Weight image of diffImage (1's and 0's for good and bad pixels, respectively).
191 
192  dx : `int`
193  Lag in x.
194 
195  dy : `int`
196  Lag in y.
197 
198  Returns
199  -------
200  cov : `float`
201  Covariance at (dx, dy)
202 
203  nPix : `int`
204  Number of pixel pairs used to evaluate var and cov.
205  """
206  (nCols, nRows) = diffImage.shape
207  # switching both signs does not change anything:
208  # it just swaps im1 and im2 below
209  if (dx < 0):
210  (dx, dy) = (-dx, -dy)
211  # now, we have dx >0. We have to distinguish two cases
212  # depending on the sign of dy
213  if dy >= 0:
214  im1 = diffImage[dy:, dx:]
215  w1 = weightImage[dy:, dx:]
216  im2 = diffImage[:nCols - dy, :nRows - dx]
217  w2 = weightImage[:nCols - dy, :nRows - dx]
218  else:
219  im1 = diffImage[:nCols + dy, dx:]
220  w1 = weightImage[:nCols + dy, dx:]
221  im2 = diffImage[-dy:, :nRows - dx]
222  w2 = weightImage[-dy:, :nRows - dx]
223  # use the same mask for all 3 calculations
224  wAll = w1*w2
225  # do not use mean() because weightImage=0 pixels would then count
226  nPix = wAll.sum()
227  im1TimesW = im1*wAll
228  s1 = im1TimesW.sum()/nPix
229  s2 = (im2*wAll).sum()/nPix
230  p = (im1TimesW*im2).sum()/nPix
231  cov = p - s1*s2
232 
233  return cov, nPix
234 
235 
237  """
238  A class to prepare covariances for the PTC fit.
239 
240  Parameters
241  ----------
242  r: `int`, optional
243  Maximum lag considered (e.g., to eliminate data beyond a separation "r": ignored in the fit).
244 
245  subtractDistantValue: `bool`, optional
246  Subtract a background to the measured covariances (mandatory for HSC flat pairs)?
247 
248  start: `int`, optional
249  Distance beyond which the subtractDistant model is fitted.
250 
251  offsetDegree: `int`
252  Polynomial degree for the subtraction model.
253 
254  Notes
255  -----
256  params = LoadParams(). "params" drives what happens in he fit. LoadParams provides default values.
257  """
258  def __init__(self):
259  self.r = 8
260  self.subtractDistantValue = False
261  self.start = 5
262  self.offsetDegree = 1
263 
264 
265 def loadData(tupleName, params, expIdMask):
266  """ Returns a list of CovFit objects, indexed by amp number.
267 
268  Params
269  ------
270  tupleName: `numpy.recarray`
271  Recarray with rows with at least ( mu1, mu2, cov ,var, i, j, npix), where:
272  mu1: mean value of flat1
273  mu2: mean value of flat2
274  cov: covariance value at lag (i, j)
275  var: variance (covariance value at lag (0, 0))
276  i: lag dimension
277  j: lag dimension
278  npix: number of pixels used for covariance calculation.
279 
280  params: `covAstierptcUtil.LoadParams`
281  Object with values to drive the bahaviour of fits.
282 
283  expIdMask : `dict`, [`str`, `list`]
284  Dictionary keyed by amp names containing the masked exposure pairs.
285 
286  Returns
287  -------
288  covFitList: `dict`
289  Dictionary with amps as keys, and CovFit objects as values.
290  """
291 
292  exts = np.array(np.unique(tupleName['ampName']), dtype=str)
293  covFitList = {}
294  for ext in exts:
295  ntext = tupleName[tupleName['ampName'] == ext]
296  maskExt = expIdMask[ext]
297  if params.subtractDistantValue:
298  c = CovFit(ntext, params.r, maskExt)
299  c.subtractDistantOffset(params.r, params.start, params.offsetDegree)
300  else:
301  c = CovFit(ntext, params.r, maskExt)
302 
303  cc = c.copy()
304  cc.initFit() # allows to get a crude gain.
305  covFitList[ext] = cc
306 
307  return covFitList
308 
309 
310 def fitData(tupleName, expIdMask, r=8):
311  """Fit data to models in Astier+19.
312 
313  Parameters
314  ----------
315  tupleName: `numpy.recarray`
316  Recarray with rows with at least ( mu1, mu2, cov ,var, i, j, npix), where:
317  mu1: mean value of flat1
318  mu2: mean value of flat2
319  cov: covariance value at lag (i, j)
320  var: variance (covariance value at lag (0, 0))
321  i: lag dimension
322  j: lag dimension
323  npix: number of pixels used for covariance calculation.
324 
325  expIdMask : `dict`, [`str`, `list`]
326  Dictionary keyed by amp names containing the masked exposure pairs.
327 
328  r: `int`, optional
329  Maximum lag considered (e.g., to eliminate data beyond a separation "r": ignored in the fit).
330 
331  Returns
332  -------
333  covFitList: `dict`
334  Dictionary of CovFit objects, with amp names as keys.
335 
336  covFitNoBList: `dict`
337  Dictionary of CovFit objects, with amp names as keys (b=0 in Eq. 20 of Astier+19).
338 
339  Notes
340  -----
341  The parameters of the full model for C_ij(mu) ("C_ij" and "mu" in ADU^2 and ADU, respectively)
342  in Astier+19 (Eq. 20) are:
343 
344  "a" coefficients (r by r matrix), units: 1/e
345  "b" coefficients (r by r matrix), units: 1/e
346  noise matrix (r by r matrix), units: e^2
347  gain, units: e/ADU
348 
349  "b" appears in Eq. 20 only through the "ab" combination, which is defined in this code as "c=ab".
350  """
351 
352  lparams = LoadParams()
353  lparams.subtractDistantValue = False
354  lparams.r = r
355  covFitList = loadData(tupleName, lparams, expIdMask=expIdMask)
356  covFitNoBList = {} # [None]*(exts[-1]+1)
357  for ext, c in covFitList.items():
358  c.fitFullModel()
359  covFitNoBList[ext] = c.copy()
360  c.params['c'].release()
361  c.fitFullModel()
362  return covFitList, covFitNoBList
363 
364 
365 def getFitDataFromCovariances(i, j, mu, fullCov, fullCovModel, fullCovSqrtWeights, gain=1.0,
366  divideByMu=False, returnMasked=False):
367  """Get measured signal and covariance, cov model, weigths, and mask at covariance lag (i, j).
368 
369  Parameters
370  ----------
371  i : `int`
372  Lag for covariance matrix.
373 
374  j: `int`
375  Lag for covariance matrix.
376 
377  mu : `list`
378  Mean signal values.
379 
380  fullCov: `list` of `numpy.array`
381  Measured covariance matrices at each mean signal level in mu.
382 
383  fullCovSqrtWeights: `list` of `numpy.array`
384  List of square root of measured covariances at each mean signal level in mu.
385 
386  fullCovModel : `list` of `numpy.array`
387  List of modeled covariances at each mean signal level in mu.
388 
389  gain : `float`, optional
390  Gain, in e-/ADU. If other than 1.0 (default), the returned quantities will be in
391  electrons or powers of electrons.
392 
393  divideByMu: `bool`, optional
394  Divide returned covariance, model, and weights by the mean signal mu?
395 
396  returnMasked : `bool`, optional
397  Use mask (based on weights) in returned arrays (mu, covariance, and model)?
398 
399  Returns
400  -------
401  mu : `numpy.array`
402  list of signal values at (i, j).
403 
404  covariance : `numpy.array`
405  Covariance at (i, j) at each mean signal mu value (fullCov[:, i, j]).
406 
407  covarianceModel : `numpy.array`
408  Covariance model at (i, j).
409 
410  weights : `numpy.array`
411  Weights at (i, j).
412 
413  maskFromWeights : `numpy.array`, optional
414  Boolean mask of the covariance at (i,j), where the weights differ from 0.
415 
416  Notes
417  -----
418  This function is a method of the `CovFit` class.
419  """
420  mu = np.array(mu)
421  fullCov = np.array(fullCov)
422  fullCovModel = np.array(fullCovModel)
423  fullCovSqrtWeights = np.array(fullCovSqrtWeights)
424  covariance = fullCov[:, i, j]*(gain**2)
425  covarianceModel = fullCovModel[:, i, j]*(gain**2)
426  weights = fullCovSqrtWeights[:, i, j]/(gain**2)
427 
428  maskFromWeights = weights != 0
429  if returnMasked:
430  weights = weights[maskFromWeights]
431  covarianceModel = covarianceModel[maskFromWeights]
432  mu = mu[maskFromWeights]
433  covariance = covariance[maskFromWeights]
434 
435  if divideByMu:
436  covariance /= mu
437  covarianceModel /= mu
438  weights *= mu
439  return mu, covariance, covarianceModel, weights, maskFromWeights
lsst.cp.pipe.astierCovPtcUtils.CovFft.reportCovFft
def reportCovFft(self, maxRange)
Definition: astierCovPtcUtils.py:96
lsst.cp.pipe.astierCovPtcUtils.LoadParams.subtractDistantValue
subtractDistantValue
Definition: astierCovPtcUtils.py:260
lsst.cp.pipe.astierCovPtcUtils.fftSize
def fftSize(s)
Definition: astierCovPtcUtils.py:122
lsst.cp.pipe.astierCovPtcUtils.CovFft.__init__
def __init__(self, diff, w, fftShape, maxRangeCov)
Definition: astierCovPtcUtils.py:48
lsst.cp.pipe.astierCovPtcUtils.CovFft.pMean
pMean
Definition: astierCovPtcUtils.py:62
lsst.cp.pipe.astierCovPtcUtils.covDirectValue
def covDirectValue(diffImage, weightImage, dx, dy)
Definition: astierCovPtcUtils.py:179
lsst.cp.pipe.astierCovPtcUtils.LoadParams.start
start
Definition: astierCovPtcUtils.py:261
lsst.cp.pipe.astierCovPtcUtils.computeCovDirect
def computeCovDirect(diffImage, weightImage, maxRange)
Definition: astierCovPtcUtils.py:128
lsst.cp.pipe.astierCovPtcUtils.fitData
def fitData(tupleName, expIdMask, r=8)
Definition: astierCovPtcUtils.py:310
lsst.cp.pipe.astierCovPtcUtils.CovFft.cov
def cov(self, dx, dy)
Definition: astierCovPtcUtils.py:66
lsst.cp.pipe.astierCovPtcUtils.CovFft.pCount
pCount
Definition: astierCovPtcUtils.py:64
lsst.cp.pipe.astierCovPtcFit.CovFit
Definition: astierCovPtcFit.py:213
lsst.cp.pipe.astierCovPtcUtils.LoadParams.r
r
Definition: astierCovPtcUtils.py:259
lsst.cp.pipe.astierCovPtcUtils.getFitDataFromCovariances
def getFitDataFromCovariances(i, j, mu, fullCov, fullCovModel, fullCovSqrtWeights, gain=1.0, divideByMu=False, returnMasked=False)
Definition: astierCovPtcUtils.py:365
lsst.cp.pipe.astierCovPtcUtils.CovFft.pCov
pCov
Definition: astierCovPtcUtils.py:60
lsst.cp.pipe.astierCovPtcUtils.CovFft
Definition: astierCovPtcUtils.py:28
lsst.cp.pipe.astierCovPtcUtils.LoadParams
Definition: astierCovPtcUtils.py:236
lsst.cp.pipe.astierCovPtcUtils.LoadParams.offsetDegree
offsetDegree
Definition: astierCovPtcUtils.py:262
lsst.cp.pipe.astierCovPtcUtils.loadData
def loadData(tupleName, params, expIdMask)
Definition: astierCovPtcUtils.py:265
lsst.cp.pipe.astierCovPtcUtils.LoadParams.__init__
def __init__(self)
Definition: astierCovPtcUtils.py:258