Coverage for tests/test_ptc.py : 10%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1#!/usr/bin/env python
3#
4# LSST Data Management System
5#
6# Copyright 2008-2017 AURA/LSST.
7#
8# This product includes software developed by the
9# LSST Project (http://www.lsst.org/).
10#
11# This program is free software: you can redistribute it and/or modify
12# it under the terms of the GNU General Public License as published by
13# the Free Software Foundation, either version 3 of the License, or
14# (at your option) any later version.
15#
16# This program is distributed in the hope that it will be useful,
17# but WITHOUT ANY WARRANTY; without even the implied warranty of
18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19# GNU General Public License for more details.
20#
21# You should have received a copy of the LSST License Statement and
22# the GNU General Public License along with this program. If not,
23# see <https://www.lsstcorp.org/LegalNotices/>.
24#
25"""Test cases for cp_pipe."""
27from __future__ import absolute_import, division, print_function
28import unittest
29import numpy as np
30import copy
32import lsst.utils
33import lsst.utils.tests
35import lsst.cp.pipe as cpPipe
36import lsst.ip.isr.isrMock as isrMock
37from lsst.cp.pipe.ptc import PhotonTransferCurveDataset
38from lsst.cp.pipe.astierCovPtcUtils import fitData
39from lsst.cp.pipe.utils import funcPolynomial
42class MeasurePhotonTransferCurveTaskTestCase(lsst.utils.tests.TestCase):
43 """A test case for the PTC task."""
45 def setUp(self):
46 self.defaultConfig = cpPipe.ptc.MeasurePhotonTransferCurveTask.ConfigClass()
47 self.defaultTask = cpPipe.ptc.MeasurePhotonTransferCurveTask(config=self.defaultConfig)
49 self.flatMean = 2000
50 self.readNoiseAdu = 10
51 mockImageConfig = isrMock.IsrMock.ConfigClass()
53 # flatDrop is not really relevant as we replace the data
54 # but good to note it in case we change how this image is made
55 mockImageConfig.flatDrop = 0.99999
56 mockImageConfig.isTrimmed = True
58 self.flatExp1 = isrMock.FlatMock(config=mockImageConfig).run()
59 self.flatExp2 = self.flatExp1.clone()
60 (shapeY, shapeX) = self.flatExp1.getDimensions()
62 self.flatWidth = np.sqrt(self.flatMean) + self.readNoiseAdu
64 self.rng1 = np.random.RandomState(1984)
65 flatData1 = self.rng1.normal(self.flatMean, self.flatWidth, (shapeX, shapeY))
66 self.rng2 = np.random.RandomState(666)
67 flatData2 = self.rng2.normal(self.flatMean, self.flatWidth, (shapeX, shapeY))
69 self.flatExp1.image.array[:] = flatData1
70 self.flatExp2.image.array[:] = flatData2
72 # create fake PTC data to see if fit works, for one amp ('amp')
73 self.flux = 1000. # ADU/sec
74 timeVec = np.arange(1., 201.)
75 self.k2NonLinearity = -5e-6
76 muVec = self.flux*timeVec + self.k2NonLinearity*timeVec**2 # quadratic signal-chain non-linearity
77 self.gain = 1.5 # e-/ADU
78 self.c1 = 1./self.gain
79 self.noiseSq = 5*self.gain # 7.5 (e-)^2
80 self.a00 = -1.2e-6
81 self.c2 = -1.5e-6
82 self.c3 = -4.7e-12 # tuned so that it turns over for 200k mean
84 self.ampNames = [amp.getName() for amp in self.flatExp1.getDetector().getAmplifiers()]
85 self.dataset = PhotonTransferCurveDataset(self.ampNames, " ") # pack raw data for fitting
87 for ampName in self.ampNames: # just the expTimes and means here - vars vary per function
88 self.dataset.rawExpTimes[ampName] = timeVec
89 self.dataset.rawMeans[ampName] = muVec
91 def makeMockFlats(self, expTime, gain=1.0, readNoiseElectrons=5, fluxElectrons=1000):
92 flatFlux = fluxElectrons # e/s
93 flatMean = flatFlux*expTime # e
94 readNoise = readNoiseElectrons # e
96 mockImageConfig = isrMock.IsrMock.ConfigClass()
98 mockImageConfig.flatDrop = 0.99999
99 mockImageConfig.isTrimmed = True
101 flatExp1 = isrMock.FlatMock(config=mockImageConfig).run()
102 flatExp2 = flatExp1.clone()
103 (shapeY, shapeX) = flatExp1.getDimensions()
104 flatWidth = np.sqrt(flatMean)
106 rng1 = np.random.RandomState(1984)
107 flatData1 = (rng1.normal(flatMean, flatWidth, (shapeX, shapeY)) +
108 rng1.normal(0.0, readNoise, (shapeX, shapeY)))
109 rng2 = np.random.RandomState(666)
110 flatData2 = (rng2.normal(flatMean, flatWidth, (shapeX, shapeY)) +
111 rng2.normal(0.0, readNoise, (shapeX, shapeY)))
113 flatExp1.image.array[:] = flatData1/gain # ADU
114 flatExp2.image.array[:] = flatData2/gain # ADU
116 return flatExp1, flatExp2
118 def test_covAstier(self):
119 """Test to check getCovariancesAstier
121 We check that the gain is the same as the imput gain from the mock data, that
122 the covariances via FFT (as it is in MeasurePhotonTransferCurveTask when
123 doCovariancesAstier=True) are the same as calculated in real space, and that
124 Cov[0, 0] (i.e., the variances) are similar to the variances calculated with the standard
125 method (when doCovariancesAstier=false),
126 """
127 localDataset = copy.copy(self.dataset)
128 config = copy.copy(self.defaultConfig)
129 task = cpPipe.ptc.MeasurePhotonTransferCurveTask(config=config)
131 expTimes = np.arange(5, 170, 5)
132 tupleRecords = []
133 allTags = []
134 muStandard, varStandard = {}, {}
135 for expTime in expTimes:
136 mockExp1, mockExp2 = self.makeMockFlats(expTime, gain=0.75)
137 tupleRows = []
139 for ampNumber, amp in enumerate(self.ampNames):
140 # cov has (i, j, var, cov, npix)
141 muDiff, varDiff, covAstier = task.measureMeanVarCov(mockExp1, mockExp2)
142 muStandard.setdefault(amp, []).append(muDiff)
143 varStandard.setdefault(amp, []).append(varDiff)
145 # Calculate covariances in an independent way: direct space
146 _, _, covsDirect = task.measureMeanVarCov(mockExp1, mockExp2, covAstierRealSpace=True)
148 # Test that the arrays "covs" (FFT) and "covDirect" (direct space) are the same
149 for row1, row2 in zip(covAstier, covsDirect):
150 for a, b in zip(row1, row2):
151 self.assertAlmostEqual(a, b)
152 tupleRows += [(muDiff, ) + covRow + (ampNumber, expTime, amp) for covRow in covAstier]
153 tags = ['mu', 'i', 'j', 'var', 'cov', 'npix', 'ext', 'expTime', 'ampName']
154 allTags += tags
155 tupleRecords += tupleRows
156 covariancesWithTags = np.core.records.fromrecords(tupleRecords, names=allTags)
157 covFits, _ = fitData(covariancesWithTags)
158 localDataset = task.getOutputPtcDataCovAstier(localDataset, covFits)
160 # Chek the gain and that the ratio of the variance caclulated via cov Astier (FFT) and
161 # that calculated with the standard PTC is close to 1.
162 for amp in self.ampNames:
163 self.assertAlmostEqual(localDataset.gain[amp], 0.75, places=2)
164 for v1, v2 in zip(varStandard[amp], localDataset.finalVars[amp][0]):
165 self.assertAlmostEqual(v1/v2, 1.0, places=4)
167 def ptcFitAndCheckPtc(self, order=None, fitType='', doTableArray=False):
168 localDataset = copy.copy(self.dataset)
169 config = copy.copy(self.defaultConfig)
170 if fitType == 'POLYNOMIAL':
171 if order not in [2, 3]:
172 RuntimeError("Enter a valid polynomial order for this test: 2 or 3")
173 if order == 2:
174 for ampName in self.ampNames:
175 localDataset.rawVars[ampName] = [self.noiseSq + self.c1*mu + self.c2*mu**2 for
176 mu in localDataset.rawMeans[ampName]]
177 config.polynomialFitDegree = 2
178 if order == 3:
179 for ampName in self.ampNames:
180 localDataset.rawVars[ampName] = [self.noiseSq + self.c1*mu + self.c2*mu**2 + self.c3*mu**3
181 for mu in localDataset.rawMeans[ampName]]
182 config.polynomialFitDegree = 3
183 elif fitType == 'EXPAPPROXIMATION':
184 g = self.gain
185 for ampName in self.ampNames:
186 localDataset.rawVars[ampName] = [(0.5/(self.a00*g**2)*(np.exp(2*self.a00*mu*g)-1) +
187 self.noiseSq/(g*g)) for mu in localDataset.rawMeans[ampName]]
188 else:
189 RuntimeError("Enter a fit function type: 'POLYNOMIAL' or 'EXPAPPROXIMATION'")
191 config.maxAduForLookupTableLinearizer = 200000 # Max ADU in input mock flats
192 task = cpPipe.ptc.MeasurePhotonTransferCurveTask(config=config)
194 if doTableArray:
195 # Non-linearity
196 numberAmps = len(self.ampNames)
197 numberAduValues = config.maxAduForLookupTableLinearizer
198 lookupTableArray = np.zeros((numberAmps, numberAduValues), dtype=np.float32)
199 # localDataset: PTC dataset (lsst.cp.pipe.ptc.PhotonTransferCurveDataset)
200 localDataset = task.fitPtc(localDataset, ptcFitType=fitType)
201 # linDataset: Dictionary of `lsst.cp.pipe.ptc.LinearityResidualsAndLinearizersDataset`
202 linDataset = task.fitNonLinearity(localDataset, tableArray=lookupTableArray)
203 else:
204 localDataset = task.fitPtc(localDataset, ptcFitType=fitType)
205 linDataset = task.fitNonLinearity(localDataset)
207 if doTableArray:
208 # check that the linearizer table has been filled out properly
209 for i in np.arange(numberAmps):
210 tMax = (config.maxAduForLookupTableLinearizer)/self.flux
211 timeRange = np.linspace(0., tMax, config.maxAduForLookupTableLinearizer)
212 signalIdeal = timeRange*self.flux
213 signalUncorrected = funcPolynomial(np.array([0.0, self.flux, self.k2NonLinearity]),
214 timeRange)
215 linearizerTableRow = signalIdeal - signalUncorrected
216 self.assertEqual(len(linearizerTableRow), len(lookupTableArray[i, :]))
217 for j in np.arange(len(linearizerTableRow)):
218 self.assertAlmostEqual(linearizerTableRow[j], lookupTableArray[i, :][j], places=6)
220 # check entries in localDataset, which was modified by the function
221 for ampName in self.ampNames:
222 maskAmp = localDataset.visitMask[ampName]
223 finalMuVec = localDataset.rawMeans[ampName][maskAmp]
224 finalTimeVec = localDataset.rawExpTimes[ampName][maskAmp]
225 linearPart = self.flux*finalTimeVec
226 inputFracNonLinearityResiduals = 100*(linearPart - finalMuVec)/linearPart
227 self.assertEqual(fitType, localDataset.ptcFitType)
228 self.assertAlmostEqual(self.gain, localDataset.gain[ampName])
229 if fitType == 'POLYNOMIAL':
230 self.assertAlmostEqual(self.c1, localDataset.ptcFitPars[ampName][1])
231 self.assertAlmostEqual(np.sqrt(self.noiseSq)*self.gain, localDataset.noise[ampName])
232 if fitType == 'EXPAPPROXIMATION':
233 self.assertAlmostEqual(self.a00, localDataset.ptcFitPars[ampName][0])
234 # noise already in electrons for 'EXPAPPROXIMATION' fit
235 self.assertAlmostEqual(np.sqrt(self.noiseSq), localDataset.noise[ampName])
237 # check entries in returned dataset (a dict of , for nonlinearity)
238 for ampName in self.ampNames:
239 maskAmp = localDataset.visitMask[ampName]
240 finalMuVec = localDataset.rawMeans[ampName][maskAmp]
241 finalTimeVec = localDataset.rawExpTimes[ampName][maskAmp]
242 linearPart = self.flux*finalTimeVec
243 inputFracNonLinearityResiduals = 100*(linearPart - finalMuVec)/linearPart
245 # Nonlinearity fit parameters
246 self.assertAlmostEqual(0.0, linDataset[ampName].meanSignalVsTimePolyFitPars[0])
247 self.assertAlmostEqual(self.flux, linDataset[ampName].meanSignalVsTimePolyFitPars[1])
248 self.assertAlmostEqual(self.k2NonLinearity, linDataset[ampName].meanSignalVsTimePolyFitPars[2])
250 # Non-linearity coefficient for linearizer
251 self.assertAlmostEqual(-self.k2NonLinearity/(self.flux**2),
252 linDataset[ampName].quadraticPolynomialLinearizerCoefficient)
254 linearPartModel = linDataset[ampName].meanSignalVsTimePolyFitPars[1]*finalTimeVec
255 outputFracNonLinearityResiduals = 100*(linearPartModel - finalMuVec)/linearPartModel
256 # Fractional nonlinearity residuals
257 self.assertEqual(len(outputFracNonLinearityResiduals), len(inputFracNonLinearityResiduals))
258 for calc, truth in zip(outputFracNonLinearityResiduals, inputFracNonLinearityResiduals):
259 self.assertAlmostEqual(calc, truth)
261 # check calls to calculateLinearityResidualAndLinearizers
262 datasetLinResAndLinearizers = task.calculateLinearityResidualAndLinearizers(
263 localDataset.rawExpTimes[ampName], localDataset.rawMeans[ampName])
265 self.assertAlmostEqual(-self.k2NonLinearity/(self.flux**2),
266 datasetLinResAndLinearizers.quadraticPolynomialLinearizerCoefficient)
267 self.assertAlmostEqual(0.0, datasetLinResAndLinearizers.meanSignalVsTimePolyFitPars[0])
268 self.assertAlmostEqual(self.flux, datasetLinResAndLinearizers.meanSignalVsTimePolyFitPars[1])
269 self.assertAlmostEqual(self.k2NonLinearity,
270 datasetLinResAndLinearizers.meanSignalVsTimePolyFitPars[2])
272 def test_ptcFit(self):
273 for createArray in [True, False]:
274 for (fitType, order) in [('POLYNOMIAL', 2), ('POLYNOMIAL', 3), ('EXPAPPROXIMATION', None)]:
275 self.ptcFitAndCheckPtc(fitType=fitType, order=order, doTableArray=createArray)
277 def test_meanVarMeasurement(self):
278 task = self.defaultTask
279 mu, varDiff, _ = task.measureMeanVarCov(self.flatExp1, self.flatExp2)
281 self.assertLess(self.flatWidth - np.sqrt(varDiff), 1)
282 self.assertLess(self.flatMean - mu, 1)
284 def test_getInitialGoodPoints(self):
285 xs = [1, 2, 3, 4, 5, 6]
286 ys = [2*x for x in xs]
287 points = self.defaultTask._getInitialGoodPoints(xs, ys, 0.1, 0.25)
288 assert np.all(points) == np.all(np.array([True for x in xs]))
290 ys[-1] = 30
291 points = self.defaultTask._getInitialGoodPoints(xs, ys, 0.1, 0.25)
292 assert np.all(points) == np.all(np.array([True, True, True, True, False]))
294 ys = [2*x for x in xs]
295 newYs = copy.copy(ys)
296 results = [False, True, True, False, False]
297 for i, factor in enumerate([-0.5, -0.1, 0, 0.1, 0.5]):
298 newYs[-1] = ys[-1] + (factor*ys[-1])
299 points = self.defaultTask._getInitialGoodPoints(xs, newYs, 0.05, 0.25)
300 assert (np.all(points[0:-1]) == True) # noqa: E712 - flake8 is wrong here because of numpy.bool
301 assert points[-1] == results[i]
303 def test_getVisitsUsed(self):
304 localDataset = copy.copy(self.dataset)
306 for pair in [(12, 34), (56, 78), (90, 10)]:
307 localDataset.inputVisitPairs["C:0,0"].append(pair)
308 localDataset.visitMask["C:0,0"] = np.array([True, False, True])
309 self.assertTrue(np.all(localDataset.getVisitsUsed("C:0,0") == [(12, 34), (90, 10)]))
311 localDataset.visitMask["C:0,0"] = np.array([True, False, True, True]) # wrong length now
312 with self.assertRaises(AssertionError):
313 localDataset.getVisitsUsed("C:0,0")
315 def test_getGoodAmps(self):
316 dataset = self.dataset
318 self.assertTrue(dataset.ampNames == self.ampNames)
319 dataset.badAmps.append("C:0,1")
320 self.assertTrue(dataset.getGoodAmps() == [amp for amp in self.ampNames if amp != "C:0,1"])
323class MeasurePhotonTransferCurveDatasetTestCase(lsst.utils.tests.TestCase):
324 def setUp(self):
325 self.ptcData = PhotonTransferCurveDataset(['C00', 'C01'], " ")
326 self.ptcData.inputVisitPairs = {'C00': [(123, 234), (345, 456), (567, 678)],
327 'C01': [(123, 234), (345, 456), (567, 678)]}
329 def test_generalBehaviour(self):
330 test = PhotonTransferCurveDataset(['C00', 'C01'], " ")
331 test.inputVisitPairs = {'C00': [(123, 234), (345, 456), (567, 678)],
332 'C01': [(123, 234), (345, 456), (567, 678)]}
334 with self.assertRaises(AttributeError):
335 test.newItem = 1
338class TestMemory(lsst.utils.tests.MemoryTestCase):
339 pass
342def setup_module(module):
343 lsst.utils.tests.init()
346if __name__ == "__main__": 346 ↛ 347line 346 didn't jump to line 347, because the condition on line 346 was never true
347 lsst.utils.tests.init()
348 unittest.main()