Coverage for python/lsst/cp/pipe/ptc/astierCovPtcUtils.py: 9%
109 statements
« prev ^ index » next coverage.py v7.1.0, created at 2023-02-05 18:59 -0800
« prev ^ index » next coverage.py v7.1.0, created at 2023-02-05 18:59 -0800
1# This file is part of cp_pipe.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22import numpy as np
23from .astierCovPtcFit import CovFit
25__all__ = ['CovFastFourierTransform']
28class CovFastFourierTransform:
29 """A class to compute (via FFT) the nearby pixels correlation function.
31 Implements appendix of Astier+19.
33 Parameters
34 ----------
35 diff : `numpy.array`
36 Image where to calculate the covariances (e.g., the difference
37 image of two flats).
39 w : `numpy.array`
40 Weight image (mask): it should consist of 1's (good pixel) and
41 0's (bad pixels).
43 fftShape : `tuple`
44 2d-tuple with the shape of the FFT
46 maxRangeCov : `int`
47 Maximum range for the covariances.
48 """
50 def __init__(self, diff, w, fftShape, maxRangeCov):
51 # check that the zero padding implied by "fft_shape"
52 # is large enough for the required correlation range
53 assert(fftShape[0] > diff.shape[0]+maxRangeCov+1)
54 assert(fftShape[1] > diff.shape[1]+maxRangeCov+1)
55 # for some reason related to numpy.fft.rfftn,
56 # the second dimension should be even, so
57 if fftShape[1]%2 == 1:
58 fftShape = (fftShape[0], fftShape[1]+1)
59 tIm = np.fft.rfft2(diff*w, fftShape)
60 tMask = np.fft.rfft2(w, fftShape)
61 # sum of "squares"
62 self.pCov = np.fft.irfft2(tIm*tIm.conjugate())
63 # sum of values
64 self.pMean = np.fft.irfft2(tIm*tMask.conjugate())
65 # number of w!=0 pixels.
66 self.pCount = np.fft.irfft2(tMask*tMask.conjugate())
68 def cov(self, dx, dy):
69 """Covariance for dx,dy averaged with dx,-dy if both non zero.
71 Implements appendix of Astier+19.
73 Parameters
74 ----------
75 dx : `int`
76 Lag in x
78 dy : `int
79 Lag in y
81 Returns
82 -------
83 0.5*(cov1+cov2) : `float`
84 Covariance at (dx, dy) lag
86 npix1+npix2 : `int`
87 Number of pixels used in covariance calculation.
88 """
89 # compensate rounding errors
90 nPix1 = int(round(self.pCount[dy, dx]))
91 cov1 = self.pCov[dy, dx]/nPix1-self.pMean[dy, dx]*self.pMean[-dy, -dx]/(nPix1*nPix1)
92 if (dx == 0 or dy == 0):
93 return cov1, nPix1
94 nPix2 = int(round(self.pCount[-dy, dx]))
95 cov2 = self.pCov[-dy, dx]/nPix2-self.pMean[-dy, dx]*self.pMean[dy, -dx]/(nPix2*nPix2)
96 return 0.5*(cov1+cov2), nPix1+nPix2
98 def reportCovFastFourierTransform(self, maxRange):
99 """Produce a list of tuples with covariances.
101 Implements appendix of Astier+19.
103 Parameters
104 ----------
105 maxRange : `int`
106 Maximum range of covariances.
108 Returns
109 -------
110 tupleVec : `list`
111 List with covariance tuples.
112 """
113 tupleVec = []
114 # (dy,dx) = (0,0) has to be first
115 for dy in range(maxRange+1):
116 for dx in range(maxRange+1):
117 cov, npix = self.cov(dx, dy)
118 if (dx == 0 and dy == 0):
119 var = cov
120 tupleVec.append((dx, dy, var, cov, npix))
121 return tupleVec
124def computeCovDirect(diffImage, weightImage, maxRange):
125 """Compute covariances of diffImage in real space.
127 For lags larger than ~25, it is slower than the FFT way.
128 Taken from https://github.com/PierreAstier/bfptc/
130 Parameters
131 ----------
132 diffImage : `numpy.array`
133 Image to compute the covariance of.
135 weightImage : `numpy.array`
136 Weight image of diffImage (1's and 0's for good and bad
137 pixels, respectively).
139 maxRange : `int`
140 Last index of the covariance to be computed.
142 Returns
143 -------
144 outList : `list`
145 List with tuples of the form (dx, dy, var, cov, npix), where:
146 dx : `int`
147 Lag in x
148 dy : `int`
149 Lag in y
150 var : `float`
151 Variance at (dx, dy).
152 cov : `float`
153 Covariance at (dx, dy).
154 nPix : `int`
155 Number of pixel pairs used to evaluate var and cov.
156 """
157 outList = []
158 var = 0
159 # (dy,dx) = (0,0) has to be first
160 for dy in range(maxRange + 1):
161 for dx in range(0, maxRange + 1):
162 if (dx*dy > 0):
163 cov1, nPix1 = covDirectValue(diffImage, weightImage, dx, dy)
164 cov2, nPix2 = covDirectValue(diffImage, weightImage, dx, -dy)
165 cov = 0.5*(cov1 + cov2)
166 nPix = nPix1 + nPix2
167 else:
168 cov, nPix = covDirectValue(diffImage, weightImage, dx, dy)
169 if (dx == 0 and dy == 0):
170 var = cov
171 outList.append((dx, dy, var, cov, nPix))
173 return outList
176def covDirectValue(diffImage, weightImage, dx, dy):
177 """Compute covariances of diffImage in real space at lag (dx, dy).
179 Taken from https://github.com/PierreAstier/bfptc/ (c.f., appendix
180 of Astier+19).
182 Parameters
183 ----------
184 diffImage : `numpy.array`
185 Image to compute the covariance of.
187 weightImage : `numpy.array`
188 Weight image of diffImage (1's and 0's for good and bad
189 pixels, respectively).
191 dx : `int`
192 Lag in x.
194 dy : `int`
195 Lag in y.
197 Returns
198 -------
199 cov : `float`
200 Covariance at (dx, dy)
202 nPix : `int`
203 Number of pixel pairs used to evaluate var and cov.
204 """
205 (nCols, nRows) = diffImage.shape
206 # switching both signs does not change anything:
207 # it just swaps im1 and im2 below
208 if (dx < 0):
209 (dx, dy) = (-dx, -dy)
210 # now, we have dx >0. We have to distinguish two cases
211 # depending on the sign of dy
212 if dy >= 0:
213 im1 = diffImage[dy:, dx:]
214 w1 = weightImage[dy:, dx:]
215 im2 = diffImage[:nCols - dy, :nRows - dx]
216 w2 = weightImage[:nCols - dy, :nRows - dx]
217 else:
218 im1 = diffImage[:nCols + dy, dx:]
219 w1 = weightImage[:nCols + dy, dx:]
220 im2 = diffImage[-dy:, :nRows - dx]
221 w2 = weightImage[-dy:, :nRows - dx]
222 # use the same mask for all 3 calculations
223 wAll = w1*w2
224 # do not use mean() because weightImage=0 pixels would then count
225 nPix = wAll.sum()
226 im1TimesW = im1*wAll
227 s1 = im1TimesW.sum()/nPix
228 s2 = (im2*wAll).sum()/nPix
229 p = (im1TimesW*im2).sum()/nPix
230 cov = p - s1*s2
232 return cov, nPix
235def parseData(dataset):
236 """ Returns a list of CovFit objects, indexed by amp number.
238 Params
239 ------
240 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset`
241 The PTC dataset containing the means, variances, and
242 exposure times.
244 Returns
245 -------
246 covFitDict : `dict`
247 Dictionary with amps as keys, and CovFit objects as values.
248 """
249 covFitDict = {}
250 for ampName in dataset.ampNames:
251 # If there is a bad amp, don't fit it
252 if ampName in dataset.badAmps:
253 continue
254 maskAtAmp = dataset.expIdMask[ampName]
255 muAtAmp = dataset.rawMeans[ampName]
256 covAtAmp = dataset.covariances[ampName]
257 covSqrtWeightsAtAmp = dataset.covariancesSqrtWeights[ampName]
259 c = CovFit(muAtAmp, covAtAmp, covSqrtWeightsAtAmp, dataset.covMatrixSide, maskAtAmp)
260 cc = c.copy()
261 cc.initFit() # allows to get a crude gain.
262 covFitDict[ampName] = cc
264 return covFitDict
267def fitDataFullCovariance(dataset):
268 """Fit data to model in Astier+19 (Eq. 20).
270 Parameters
271 ----------
272 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset`
273 The dataset containing the means, (co)variances, and exposure times.
275 Returns
276 -------
277 covFitDict : `dict`
278 Dictionary of CovFit objects, with amp names as keys.
280 covFitNoBDict : `dict`
281 Dictionary of CovFit objects, with amp names as keys (b=0 in
282 Eq. 20 of Astier+19).
284 Notes
285 -----
286 The parameters of the full model for C_ij(mu) ("C_ij" and "mu" in
287 ADU^2 and ADU, respectively) in Astier+19 (Eq. 20) are:
289 "a" coefficients (r by r matrix), units: 1/e
290 "b" coefficients (r by r matrix), units: 1/e
291 noise matrix (r by r matrix), units: e^2
292 gain, units: e/ADU
294 "b" appears in Eq. 20 only through the "ab" combination, which is
295 defined in this code as "c=ab".
296 """
297 covFitDict = parseData(dataset)
298 covFitNoBDict = {}
299 for ext, c in covFitDict.items():
300 c.fitFullModel()
301 covFitNoBDict[ext] = c.copy()
302 c.params['c'].release()
303 c.fitFullModel()
304 return covFitDict, covFitNoBDict
307def getFitDataFromCovariances(i, j, mu, fullCov, fullCovModel, fullCovSqrtWeights, gain=1.0,
308 divideByMu=False, returnMasked=False):
309 """Get measured signal and covariance, cov model, weigths, and mask at
310 covariance lag (i, j).
312 Parameters
313 ----------
314 i : `int`
315 Lag for covariance matrix.
317 j : `int`
318 Lag for covariance matrix.
320 mu : `list`
321 Mean signal values.
323 fullCov : `list` of `numpy.array`
324 Measured covariance matrices at each mean signal level in mu.
326 fullCovSqrtWeights : `list` of `numpy.array`
327 List of square root of measured covariances at each mean
328 signal level in mu.
330 fullCovModel : `list` of `numpy.array`
331 List of modeled covariances at each mean signal level in mu.
333 gain : `float`, optional
334 Gain, in e-/ADU. If other than 1.0 (default), the returned
335 quantities will be in electrons or powers of electrons.
337 divideByMu : `bool`, optional
338 Divide returned covariance, model, and weights by the mean
339 signal mu?
341 returnMasked : `bool`, optional
342 Use mask (based on weights) in returned arrays (mu,
343 covariance, and model)?
345 Returns
346 -------
347 mu : `numpy.array`
348 list of signal values at (i, j).
350 covariance : `numpy.array`
351 Covariance at (i, j) at each mean signal mu value (fullCov[:, i, j]).
353 covarianceModel : `numpy.array`
354 Covariance model at (i, j).
356 weights : `numpy.array`
357 Weights at (i, j).
359 maskFromWeights : `numpy.array`, optional
360 Boolean mask of the covariance at (i,j), where the weights
361 differ from 0.
363 Notes
364 -----
365 This function is a method of the `CovFit` class.
366 """
367 mu = np.array(mu)
368 fullCov = np.array(fullCov)
369 fullCovModel = np.array(fullCovModel)
370 fullCovSqrtWeights = np.array(fullCovSqrtWeights)
371 covariance = fullCov[:, i, j]*(gain**2)
372 covarianceModel = fullCovModel[:, i, j]*(gain**2)
373 weights = fullCovSqrtWeights[:, i, j]/(gain**2)
375 maskFromWeights = weights != 0
376 if returnMasked:
377 weights = weights[maskFromWeights]
378 covarianceModel = covarianceModel[maskFromWeights]
379 mu = mu[maskFromWeights]
380 covariance = covariance[maskFromWeights]
382 if divideByMu:
383 covariance /= mu
384 covarianceModel /= mu
385 weights *= mu
386 return mu, covariance, covarianceModel, weights, maskFromWeights