Coverage for python/lsst/cp/pipe/ptc/astierCovPtcUtils.py : 9%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of cp_pipe.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22import numpy as np
23from .astierCovPtcFit import CovFit
25__all__ = ['CovFastFourierTransform']
28class CovFastFourierTransform:
29 """A class to compute (via FFT) the nearby pixels correlation function.
31 Implements appendix of Astier+19.
33 Parameters
34 ----------
35 diff: `numpy.array`
36 Image where to calculate the covariances (e.g., the difference image of two flats).
38 w: `numpy.array`
39 Weight image (mask): it should consist of 1's (good pixel) and 0's (bad pixels).
41 fftShape: `tuple`
42 2d-tuple with the shape of the FFT
44 maxRangeCov: `int`
45 Maximum range for the covariances.
46 """
48 def __init__(self, diff, w, fftShape, maxRangeCov):
49 # check that the zero padding implied by "fft_shape"
50 # is large enough for the required correlation range
51 assert(fftShape[0] > diff.shape[0]+maxRangeCov+1)
52 assert(fftShape[1] > diff.shape[1]+maxRangeCov+1)
53 # for some reason related to numpy.fft.rfftn,
54 # the second dimension should be even, so
55 if fftShape[1]%2 == 1:
56 fftShape = (fftShape[0], fftShape[1]+1)
57 tIm = np.fft.rfft2(diff*w, fftShape)
58 tMask = np.fft.rfft2(w, fftShape)
59 # sum of "squares"
60 self.pCov = np.fft.irfft2(tIm*tIm.conjugate())
61 # sum of values
62 self.pMean = np.fft.irfft2(tIm*tMask.conjugate())
63 # number of w!=0 pixels.
64 self.pCount = np.fft.irfft2(tMask*tMask.conjugate())
66 def cov(self, dx, dy):
67 """Covariance for dx,dy averaged with dx,-dy if both non zero.
69 Implements appendix of Astier+19.
71 Parameters
72 ----------
73 dx: `int`
74 Lag in x
76 dy: `int
77 Lag in y
79 Returns
80 -------
81 0.5*(cov1+cov2): `float`
82 Covariance at (dx, dy) lag
84 npix1+npix2: `int`
85 Number of pixels used in covariance calculation.
86 """
87 # compensate rounding errors
88 nPix1 = int(round(self.pCount[dy, dx]))
89 cov1 = self.pCov[dy, dx]/nPix1-self.pMean[dy, dx]*self.pMean[-dy, -dx]/(nPix1*nPix1)
90 if (dx == 0 or dy == 0):
91 return cov1, nPix1
92 nPix2 = int(round(self.pCount[-dy, dx]))
93 cov2 = self.pCov[-dy, dx]/nPix2-self.pMean[-dy, dx]*self.pMean[dy, -dx]/(nPix2*nPix2)
94 return 0.5*(cov1+cov2), nPix1+nPix2
96 def reportCovFastFourierTransform(self, maxRange):
97 """Produce a list of tuples with covariances.
99 Implements appendix of Astier+19.
101 Parameters
102 ----------
103 maxRange: `int`
104 Maximum range of covariances.
106 Returns
107 -------
108 tupleVec: `list`
109 List with covariance tuples.
110 """
111 tupleVec = []
112 # (dy,dx) = (0,0) has to be first
113 for dy in range(maxRange+1):
114 for dx in range(maxRange+1):
115 cov, npix = self.cov(dx, dy)
116 if (dx == 0 and dy == 0):
117 var = cov
118 tupleVec.append((dx, dy, var, cov, npix))
119 return tupleVec
122def computeCovDirect(diffImage, weightImage, maxRange):
123 """Compute covariances of diffImage in real space.
125 For lags larger than ~25, it is slower than the FFT way.
126 Taken from https://github.com/PierreAstier/bfptc/
128 Parameters
129 ----------
130 diffImage : `numpy.array`
131 Image to compute the covariance of.
133 weightImage : `numpy.array`
134 Weight image of diffImage (1's and 0's for good and bad pixels, respectively).
136 maxRange : `int`
137 Last index of the covariance to be computed.
139 Returns
140 -------
141 outList : `list`
142 List with tuples of the form (dx, dy, var, cov, npix), where:
143 dx : `int`
144 Lag in x
145 dy : `int`
146 Lag in y
147 var : `float`
148 Variance at (dx, dy).
149 cov : `float`
150 Covariance at (dx, dy).
151 nPix : `int`
152 Number of pixel pairs used to evaluate var and cov.
153 """
154 outList = []
155 var = 0
156 # (dy,dx) = (0,0) has to be first
157 for dy in range(maxRange + 1):
158 for dx in range(0, maxRange + 1):
159 if (dx*dy > 0):
160 cov1, nPix1 = covDirectValue(diffImage, weightImage, dx, dy)
161 cov2, nPix2 = covDirectValue(diffImage, weightImage, dx, -dy)
162 cov = 0.5*(cov1 + cov2)
163 nPix = nPix1 + nPix2
164 else:
165 cov, nPix = covDirectValue(diffImage, weightImage, dx, dy)
166 if (dx == 0 and dy == 0):
167 var = cov
168 outList.append((dx, dy, var, cov, nPix))
170 return outList
173def covDirectValue(diffImage, weightImage, dx, dy):
174 """Compute covariances of diffImage in real space at lag (dx, dy).
176 Taken from https://github.com/PierreAstier/bfptc/ (c.f., appendix of Astier+19).
178 Parameters
179 ----------
180 diffImage : `numpy.array`
181 Image to compute the covariance of.
183 weightImage : `numpy.array`
184 Weight image of diffImage (1's and 0's for good and bad pixels, respectively).
186 dx : `int`
187 Lag in x.
189 dy : `int`
190 Lag in y.
192 Returns
193 -------
194 cov : `float`
195 Covariance at (dx, dy)
197 nPix : `int`
198 Number of pixel pairs used to evaluate var and cov.
199 """
200 (nCols, nRows) = diffImage.shape
201 # switching both signs does not change anything:
202 # it just swaps im1 and im2 below
203 if (dx < 0):
204 (dx, dy) = (-dx, -dy)
205 # now, we have dx >0. We have to distinguish two cases
206 # depending on the sign of dy
207 if dy >= 0:
208 im1 = diffImage[dy:, dx:]
209 w1 = weightImage[dy:, dx:]
210 im2 = diffImage[:nCols - dy, :nRows - dx]
211 w2 = weightImage[:nCols - dy, :nRows - dx]
212 else:
213 im1 = diffImage[:nCols + dy, dx:]
214 w1 = weightImage[:nCols + dy, dx:]
215 im2 = diffImage[-dy:, :nRows - dx]
216 w2 = weightImage[-dy:, :nRows - dx]
217 # use the same mask for all 3 calculations
218 wAll = w1*w2
219 # do not use mean() because weightImage=0 pixels would then count
220 nPix = wAll.sum()
221 im1TimesW = im1*wAll
222 s1 = im1TimesW.sum()/nPix
223 s2 = (im2*wAll).sum()/nPix
224 p = (im1TimesW*im2).sum()/nPix
225 cov = p - s1*s2
227 return cov, nPix
230def parseData(dataset):
231 """ Returns a list of CovFit objects, indexed by amp number.
233 Params
234 ------
235 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset`
236 The PTC dataset containing the means, variances, and
237 exposure times.
239 Returns
240 -------
241 covFitDict: `dict`
242 Dictionary with amps as keys, and CovFit objects as values.
243 """
245 covFitDict = {}
246 for ampName in dataset.ampNames:
247 # If there is a bad amp, don't fit it
248 if ampName in dataset.badAmps:
249 continue
250 maskAtAmp = dataset.expIdMask[ampName]
251 muAtAmp = dataset.rawMeans[ampName]
252 covAtAmp = dataset.covariances[ampName]
253 covSqrtWeightsAtAmp = dataset.covariancesSqrtWeights[ampName]
255 c = CovFit(muAtAmp, covAtAmp, covSqrtWeightsAtAmp, dataset.covMatrixSide, maskAtAmp)
256 cc = c.copy()
257 cc.initFit() # allows to get a crude gain.
258 covFitDict[ampName] = cc
260 return covFitDict
263def fitDataFullCovariance(dataset):
264 """Fit data to model in Astier+19 (Eq. 20).
266 Parameters
267 ----------
268 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset`
269 The dataset containing the means, (co)variances, and exposure times.
271 Returns
272 -------
273 covFitDict: `dict`
274 Dictionary of CovFit objects, with amp names as keys.
276 covFitNoBDict: `dict`
277 Dictionary of CovFit objects, with amp names as keys (b=0 in Eq. 20 of Astier+19).
279 Notes
280 -----
281 The parameters of the full model for C_ij(mu) ("C_ij" and "mu" in ADU^2 and ADU, respectively)
282 in Astier+19 (Eq. 20) are:
284 "a" coefficients (r by r matrix), units: 1/e
285 "b" coefficients (r by r matrix), units: 1/e
286 noise matrix (r by r matrix), units: e^2
287 gain, units: e/ADU
289 "b" appears in Eq. 20 only through the "ab" combination, which is defined in this code as "c=ab".
290 """
292 covFitDict = parseData(dataset)
293 covFitNoBDict = {}
294 for ext, c in covFitDict.items():
295 c.fitFullModel()
296 covFitNoBDict[ext] = c.copy()
297 c.params['c'].release()
298 c.fitFullModel()
299 return covFitDict, covFitNoBDict
302def getFitDataFromCovariances(i, j, mu, fullCov, fullCovModel, fullCovSqrtWeights, gain=1.0,
303 divideByMu=False, returnMasked=False):
304 """Get measured signal and covariance, cov model, weigths, and mask at covariance lag (i, j).
306 Parameters
307 ----------
308 i : `int`
309 Lag for covariance matrix.
311 j: `int`
312 Lag for covariance matrix.
314 mu : `list`
315 Mean signal values.
317 fullCov: `list` of `numpy.array`
318 Measured covariance matrices at each mean signal level in mu.
320 fullCovSqrtWeights: `list` of `numpy.array`
321 List of square root of measured covariances at each mean signal level in mu.
323 fullCovModel : `list` of `numpy.array`
324 List of modeled covariances at each mean signal level in mu.
326 gain : `float`, optional
327 Gain, in e-/ADU. If other than 1.0 (default), the returned quantities will be in
328 electrons or powers of electrons.
330 divideByMu: `bool`, optional
331 Divide returned covariance, model, and weights by the mean signal mu?
333 returnMasked : `bool`, optional
334 Use mask (based on weights) in returned arrays (mu, covariance, and model)?
336 Returns
337 -------
338 mu : `numpy.array`
339 list of signal values at (i, j).
341 covariance : `numpy.array`
342 Covariance at (i, j) at each mean signal mu value (fullCov[:, i, j]).
344 covarianceModel : `numpy.array`
345 Covariance model at (i, j).
347 weights : `numpy.array`
348 Weights at (i, j).
350 maskFromWeights : `numpy.array`, optional
351 Boolean mask of the covariance at (i,j), where the weights differ from 0.
353 Notes
354 -----
355 This function is a method of the `CovFit` class.
356 """
357 mu = np.array(mu)
358 fullCov = np.array(fullCov)
359 fullCovModel = np.array(fullCovModel)
360 fullCovSqrtWeights = np.array(fullCovSqrtWeights)
361 covariance = fullCov[:, i, j]*(gain**2)
362 covarianceModel = fullCovModel[:, i, j]*(gain**2)
363 weights = fullCovSqrtWeights[:, i, j]/(gain**2)
365 maskFromWeights = weights != 0
366 if returnMasked:
367 weights = weights[maskFromWeights]
368 covarianceModel = covarianceModel[maskFromWeights]
369 mu = mu[maskFromWeights]
370 covariance = covariance[maskFromWeights]
372 if divideByMu:
373 covariance /= mu
374 covarianceModel /= mu
375 weights *= mu
376 return mu, covariance, covarianceModel, weights, maskFromWeights