lsst.cp.pipe  21.0.0-10-gcf60f90+26854609b8
astierCovPtcUtils.py
Go to the documentation of this file.
1 # This file is part of cp_pipe.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 
22 import numpy as np
23 from .astierCovPtcFit import CovFit
24 
25 __all__ = ['CovFft']
26 
27 
28 class CovFft:
29  """A class to compute (via FFT) the nearby pixels correlation function.
30 
31  Implements appendix of Astier+19.
32 
33  Parameters
34  ----------
35  diff: `numpy.array`
36  Image where to calculate the covariances (e.g., the difference image of two flats).
37 
38  w: `numpy.array`
39  Weight image (mask): it should consist of 1's (good pixel) and 0's (bad pixels).
40 
41  fftShape: `tuple`
42  2d-tuple with the shape of the FFT
43 
44  maxRangeCov: `int`
45  Maximum range for the covariances.
46  """
47 
48  def __init__(self, diff, w, fftShape, maxRangeCov):
49  # check that the zero padding implied by "fft_shape"
50  # is large enough for the required correlation range
51  assert(fftShape[0] > diff.shape[0]+maxRangeCov+1)
52  assert(fftShape[1] > diff.shape[1]+maxRangeCov+1)
53  # for some reason related to numpy.fft.rfftn,
54  # the second dimension should be even, so
55  if fftShape[1]%2 == 1:
56  fftShape = (fftShape[0], fftShape[1]+1)
57  tIm = np.fft.rfft2(diff*w, fftShape)
58  tMask = np.fft.rfft2(w, fftShape)
59  # sum of "squares"
60  self.pCovpCov = np.fft.irfft2(tIm*tIm.conjugate())
61  # sum of values
62  self.pMeanpMean = np.fft.irfft2(tIm*tMask.conjugate())
63  # number of w!=0 pixels.
64  self.pCountpCount = np.fft.irfft2(tMask*tMask.conjugate())
65 
66  def cov(self, dx, dy):
67  """Covariance for dx,dy averaged with dx,-dy if both non zero.
68 
69  Implements appendix of Astier+19.
70 
71  Parameters
72  ----------
73  dx: `int`
74  Lag in x
75 
76  dy: `int
77  Lag in y
78 
79  Returns
80  -------
81  0.5*(cov1+cov2): `float`
82  Covariance at (dx, dy) lag
83 
84  npix1+npix2: `int`
85  Number of pixels used in covariance calculation.
86  """
87  # compensate rounding errors
88  nPix1 = int(round(self.pCountpCount[dy, dx]))
89  cov1 = self.pCovpCov[dy, dx]/nPix1-self.pMeanpMean[dy, dx]*self.pMeanpMean[-dy, -dx]/(nPix1*nPix1)
90  if (dx == 0 or dy == 0):
91  return cov1, nPix1
92  nPix2 = int(round(self.pCountpCount[-dy, dx]))
93  cov2 = self.pCovpCov[-dy, dx]/nPix2-self.pMeanpMean[-dy, dx]*self.pMeanpMean[dy, -dx]/(nPix2*nPix2)
94  return 0.5*(cov1+cov2), nPix1+nPix2
95 
96  def reportCovFft(self, maxRange):
97  """Produce a list of tuples with covariances.
98 
99  Implements appendix of Astier+19.
100 
101  Parameters
102  ----------
103  maxRange: `int`
104  Maximum range of covariances.
105 
106  Returns
107  -------
108  tupleVec: `list`
109  List with covariance tuples.
110  """
111  tupleVec = []
112  # (dy,dx) = (0,0) has to be first
113  for dy in range(maxRange+1):
114  for dx in range(maxRange+1):
115  cov, npix = self.covcov(dx, dy)
116  if (dx == 0 and dy == 0):
117  var = cov
118  tupleVec.append((dx, dy, var, cov, npix))
119  return tupleVec
120 
121 
122 def fftSize(s):
123  """Calculate the size fof one dimension for the FFT"""
124  x = int(np.log(s)/np.log(2.))
125  return int(2**(x+1))
126 
127 
128 def computeCovDirect(diffImage, weightImage, maxRange):
129  """Compute covariances of diffImage in real space.
130 
131  For lags larger than ~25, it is slower than the FFT way.
132  Taken from https://github.com/PierreAstier/bfptc/
133 
134  Parameters
135  ----------
136  diffImage : `numpy.array`
137  Image to compute the covariance of.
138 
139  weightImage : `numpy.array`
140  Weight image of diffImage (1's and 0's for good and bad pixels, respectively).
141 
142  maxRange : `int`
143  Last index of the covariance to be computed.
144 
145  Returns
146  -------
147  outList : `list`
148  List with tuples of the form (dx, dy, var, cov, npix), where:
149  dx : `int`
150  Lag in x
151  dy : `int`
152  Lag in y
153  var : `float`
154  Variance at (dx, dy).
155  cov : `float`
156  Covariance at (dx, dy).
157  nPix : `int`
158  Number of pixel pairs used to evaluate var and cov.
159  """
160  outList = []
161  var = 0
162  # (dy,dx) = (0,0) has to be first
163  for dy in range(maxRange + 1):
164  for dx in range(0, maxRange + 1):
165  if (dx*dy > 0):
166  cov1, nPix1 = covDirectValue(diffImage, weightImage, dx, dy)
167  cov2, nPix2 = covDirectValue(diffImage, weightImage, dx, -dy)
168  cov = 0.5*(cov1 + cov2)
169  nPix = nPix1 + nPix2
170  else:
171  cov, nPix = covDirectValue(diffImage, weightImage, dx, dy)
172  if (dx == 0 and dy == 0):
173  var = cov
174  outList.append((dx, dy, var, cov, nPix))
175 
176  return outList
177 
178 
179 def covDirectValue(diffImage, weightImage, dx, dy):
180  """Compute covariances of diffImage in real space at lag (dx, dy).
181 
182  Taken from https://github.com/PierreAstier/bfptc/ (c.f., appendix of Astier+19).
183 
184  Parameters
185  ----------
186  diffImage : `numpy.array`
187  Image to compute the covariance of.
188 
189  weightImage : `numpy.array`
190  Weight image of diffImage (1's and 0's for good and bad pixels, respectively).
191 
192  dx : `int`
193  Lag in x.
194 
195  dy : `int`
196  Lag in y.
197 
198  Returns
199  -------
200  cov : `float`
201  Covariance at (dx, dy)
202 
203  nPix : `int`
204  Number of pixel pairs used to evaluate var and cov.
205  """
206  (nCols, nRows) = diffImage.shape
207  # switching both signs does not change anything:
208  # it just swaps im1 and im2 below
209  if (dx < 0):
210  (dx, dy) = (-dx, -dy)
211  # now, we have dx >0. We have to distinguish two cases
212  # depending on the sign of dy
213  if dy >= 0:
214  im1 = diffImage[dy:, dx:]
215  w1 = weightImage[dy:, dx:]
216  im2 = diffImage[:nCols - dy, :nRows - dx]
217  w2 = weightImage[:nCols - dy, :nRows - dx]
218  else:
219  im1 = diffImage[:nCols + dy, dx:]
220  w1 = weightImage[:nCols + dy, dx:]
221  im2 = diffImage[-dy:, :nRows - dx]
222  w2 = weightImage[-dy:, :nRows - dx]
223  # use the same mask for all 3 calculations
224  wAll = w1*w2
225  # do not use mean() because weightImage=0 pixels would then count
226  nPix = wAll.sum()
227  im1TimesW = im1*wAll
228  s1 = im1TimesW.sum()/nPix
229  s2 = (im2*wAll).sum()/nPix
230  p = (im1TimesW*im2).sum()/nPix
231  cov = p - s1*s2
232 
233  return cov, nPix
234 
235 
237  """
238  A class to prepare covariances for the PTC fit.
239 
240  Parameters
241  ----------
242  r: `int`, optional
243  Maximum lag considered (e.g., to eliminate data beyond a separation "r": ignored in the fit).
244 
245  subtractDistantValue: `bool`, optional
246  Subtract a background to the measured covariances (mandatory for HSC flat pairs)?
247 
248  start: `int`, optional
249  Distance beyond which the subtractDistant model is fitted.
250 
251  offsetDegree: `int`
252  Polynomial degree for the subtraction model.
253 
254  Notes
255  -----
256  params = LoadParams(). "params" drives what happens in he fit. LoadParams provides default values.
257  """
258  def __init__(self):
259  self.rr = 8
260  self.subtractDistantValuesubtractDistantValue = False
261  self.startstart = 5
262  self.offsetDegreeoffsetDegree = 1
263 
264 
265 def parseData(dataset, params):
266  """ Returns a list of CovFit objects, indexed by amp number.
267 
268  Params
269  ------
270  dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset`
271  The PTC dataset containing the means, variances, and
272  exposure times.
273 
274  params: `covAstierptcUtil.LoadParams`
275  Object with values to drive the bahaviour of fits.
276 
277  Returns
278  -------
279  covFitList: `dict`
280  Dictionary with amps as keys, and CovFit objects as values.
281  """
282 
283  covFitList = {}
284  for ampName in dataset.ampNames:
285  # If there is a bad amp, don't fit it
286  if ampName in dataset.badAmps:
287  continue
288  maskAtAmp = dataset.expIdMask[ampName]
289  muAtAmp = dataset.rawMeans[ampName]
290  covAtAmp = dataset.covariances[ampName]
291  covSqrtWeightsAtAmp = dataset.covariancesSqrtWeights[ampName]
292 
293  if params.subtractDistantValue:
294  c = CovFit(muAtAmp, covAtAmp, covSqrtWeightsAtAmp, params.r, maskAtAmp)
295  c.subtractDistantOffset(params.r, params.start, params.offsetDegree)
296  else:
297  c = CovFit(muAtAmp, covAtAmp, covSqrtWeightsAtAmp, params.r, maskAtAmp)
298 
299  cc = c.copy()
300  cc.initFit() # allows to get a crude gain.
301  covFitList[ampName] = cc
302 
303  return covFitList
304 
305 
306 def fitData(dataset, r=8):
307  """Fit data to models in Astier+19.
308 
309  Parameters
310  ----------
311  dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset`
312  The dataset containing the means, variances, and exposure times.
313 
314  r : `int`, optional
315  Maximum lag considered (e.g., to eliminate data beyond a separation "r": ignored in the fit).
316 
317  Returns
318  -------
319  covFitList: `dict`
320  Dictionary of CovFit objects, with amp names as keys.
321 
322  covFitNoBList: `dict`
323  Dictionary of CovFit objects, with amp names as keys (b=0 in Eq. 20 of Astier+19).
324 
325  Notes
326  -----
327  The parameters of the full model for C_ij(mu) ("C_ij" and "mu" in ADU^2 and ADU, respectively)
328  in Astier+19 (Eq. 20) are:
329 
330  "a" coefficients (r by r matrix), units: 1/e
331  "b" coefficients (r by r matrix), units: 1/e
332  noise matrix (r by r matrix), units: e^2
333  gain, units: e/ADU
334 
335  "b" appears in Eq. 20 only through the "ab" combination, which is defined in this code as "c=ab".
336  """
337 
338  lparams = LoadParams()
339  lparams.subtractDistantValue = False
340  lparams.r = r
341  covFitList = parseData(dataset, lparams)
342  covFitNoBList = {} # [None]*(exts[-1]+1)
343  for ext, c in covFitList.items():
344  c.fitFullModel()
345  covFitNoBList[ext] = c.copy()
346  c.params['c'].release()
347  c.fitFullModel()
348  return covFitList, covFitNoBList
349 
350 
351 def getFitDataFromCovariances(i, j, mu, fullCov, fullCovModel, fullCovSqrtWeights, gain=1.0,
352  divideByMu=False, returnMasked=False):
353  """Get measured signal and covariance, cov model, weigths, and mask at covariance lag (i, j).
354 
355  Parameters
356  ----------
357  i : `int`
358  Lag for covariance matrix.
359 
360  j: `int`
361  Lag for covariance matrix.
362 
363  mu : `list`
364  Mean signal values.
365 
366  fullCov: `list` of `numpy.array`
367  Measured covariance matrices at each mean signal level in mu.
368 
369  fullCovSqrtWeights: `list` of `numpy.array`
370  List of square root of measured covariances at each mean signal level in mu.
371 
372  fullCovModel : `list` of `numpy.array`
373  List of modeled covariances at each mean signal level in mu.
374 
375  gain : `float`, optional
376  Gain, in e-/ADU. If other than 1.0 (default), the returned quantities will be in
377  electrons or powers of electrons.
378 
379  divideByMu: `bool`, optional
380  Divide returned covariance, model, and weights by the mean signal mu?
381 
382  returnMasked : `bool`, optional
383  Use mask (based on weights) in returned arrays (mu, covariance, and model)?
384 
385  Returns
386  -------
387  mu : `numpy.array`
388  list of signal values at (i, j).
389 
390  covariance : `numpy.array`
391  Covariance at (i, j) at each mean signal mu value (fullCov[:, i, j]).
392 
393  covarianceModel : `numpy.array`
394  Covariance model at (i, j).
395 
396  weights : `numpy.array`
397  Weights at (i, j).
398 
399  maskFromWeights : `numpy.array`, optional
400  Boolean mask of the covariance at (i,j), where the weights differ from 0.
401 
402  Notes
403  -----
404  This function is a method of the `CovFit` class.
405  """
406  mu = np.array(mu)
407  fullCov = np.array(fullCov)
408  fullCovModel = np.array(fullCovModel)
409  fullCovSqrtWeights = np.array(fullCovSqrtWeights)
410  covariance = fullCov[:, i, j]*(gain**2)
411  covarianceModel = fullCovModel[:, i, j]*(gain**2)
412  weights = fullCovSqrtWeights[:, i, j]/(gain**2)
413 
414  maskFromWeights = weights != 0
415  if returnMasked:
416  weights = weights[maskFromWeights]
417  covarianceModel = covarianceModel[maskFromWeights]
418  mu = mu[maskFromWeights]
419  covariance = covariance[maskFromWeights]
420 
421  if divideByMu:
422  covariance /= mu
423  covarianceModel /= mu
424  weights *= mu
425  return mu, covariance, covarianceModel, weights, maskFromWeights
def __init__(self, diff, w, fftShape, maxRangeCov)
def getFitDataFromCovariances(i, j, mu, fullCov, fullCovModel, fullCovSqrtWeights, gain=1.0, divideByMu=False, returnMasked=False)
def computeCovDirect(diffImage, weightImage, maxRange)
def covDirectValue(diffImage, weightImage, dx, dy)