Coverage for tests/test_ptc.py : 10%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1#!/usr/bin/env python
3#
4# LSST Data Management System
5#
6# Copyright 2008-2017 AURA/LSST.
7#
8# This product includes software developed by the
9# LSST Project (http://www.lsst.org/).
10#
11# This program is free software: you can redistribute it and/or modify
12# it under the terms of the GNU General Public License as published by
13# the Free Software Foundation, either version 3 of the License, or
14# (at your option) any later version.
15#
16# This program is distributed in the hope that it will be useful,
17# but WITHOUT ANY WARRANTY; without even the implied warranty of
18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19# GNU General Public License for more details.
20#
21# You should have received a copy of the LSST License Statement and
22# the GNU General Public License along with this program. If not,
23# see <https://www.lsstcorp.org/LegalNotices/>.
24#
25"""Test cases for cp_pipe."""
27from __future__ import absolute_import, division, print_function
28import unittest
29import numpy as np
30import copy
32import lsst.utils
33import lsst.utils.tests
35import lsst.cp.pipe as cpPipe
36import lsst.ip.isr.isrMock as isrMock
37from lsst.cp.pipe.ptc import PhotonTransferCurveDataset
40class MeasurePhotonTransferCurveTaskTestCase(lsst.utils.tests.TestCase):
41 """A test case for the PTC task."""
43 def setUp(self):
44 self.defaultConfig = cpPipe.ptc.MeasurePhotonTransferCurveTask.ConfigClass()
45 self.defaultConfig.isr.doFlat = False
46 self.defaultConfig.isr.doFringe = False
47 self.defaultConfig.isr.doCrosstalk = False
48 self.defaultConfig.isr.doUseOpticsTransmission = False
49 self.defaultConfig.isr.doUseFilterTransmission = False
50 self.defaultConfig.isr.doUseSensorTransmission = False
51 self.defaultConfig.isr.doUseAtmosphereTransmission = False
52 self.defaultConfig.isr.doAttachTransmissionCurve = False
54 self.defaultTask = cpPipe.ptc.MeasurePhotonTransferCurveTask(config=self.defaultConfig)
56 self.flatMean = 2000
57 self.readNoiseAdu = 10
58 mockImageConfig = isrMock.IsrMock.ConfigClass()
60 # flatDrop is not really relevant as we replace the data
61 # but good to note it in case we change how this image is made
62 mockImageConfig.flatDrop = 0.99999
63 mockImageConfig.isTrimmed = True
65 self.flatExp1 = isrMock.FlatMock(config=mockImageConfig).run()
66 self.flatExp2 = self.flatExp1.clone()
67 (shapeY, shapeX) = self.flatExp1.getDimensions()
69 self.flatWidth = np.sqrt(self.flatMean) + self.readNoiseAdu
71 self.rng1 = np.random.RandomState(1984)
72 flatData1 = self.rng1.normal(self.flatMean, self.flatWidth, (shapeX, shapeY))
73 self.rng2 = np.random.RandomState(666)
74 flatData2 = self.rng2.normal(self.flatMean, self.flatWidth, (shapeX, shapeY))
76 self.flatExp1.image.array[:] = flatData1
77 self.flatExp2.image.array[:] = flatData2
79 # create fake PTC data to see if fit works, for one amp ('amp')
80 self.flux = 1000. # ADU/sec
81 timeVec = np.arange(1., 201.)
82 self.k2NonLinearity = -5e-6
83 muVec = self.flux*timeVec + self.k2NonLinearity*timeVec**2 # quadratic signal-chain non-linearity
84 self.gain = 1.5 # e-/ADU
85 self.c1 = 1./self.gain
86 self.noiseSq = 5*self.gain # 7.5 (e-)^2
87 self.a00 = -1.2e-6
88 self.c2 = -1.5e-6
89 self.c3 = -4.7e-12 # tuned so that it turns over for 200k mean
91 self.ampNames = [amp.getName() for amp in self.flatExp1.getDetector().getAmplifiers()]
92 self.dataset = PhotonTransferCurveDataset(self.ampNames) # pack raw data for fitting
94 for ampName in self.ampNames: # just the expTimes and means here - vars vary per function
95 self.dataset.rawExpTimes[ampName] = timeVec
96 self.dataset.rawMeans[ampName] = muVec
98 def ptcFitAndCheckPtc(self, order=None, fitType='', doTableArray=False):
99 localDataset = copy.copy(self.dataset)
100 config = copy.copy(self.defaultConfig)
101 if fitType == 'POLYNOMIAL':
102 if order not in [2, 3]:
103 RuntimeError("Enter a valid polynomial order for this test: 2 or 3")
104 if order == 2:
105 for ampName in self.ampNames:
106 localDataset.rawVars[ampName] = [self.noiseSq + self.c1*mu + self.c2*mu**2 for
107 mu in localDataset.rawMeans[ampName]]
108 config.polynomialFitDegree = 2
109 if order == 3:
110 for ampName in self.ampNames:
111 localDataset.rawVars[ampName] = [self.noiseSq + self.c1*mu + self.c2*mu**2 + self.c3*mu**3
112 for mu in localDataset.rawMeans[ampName]]
113 config.polynomialFitDegree = 3
114 elif fitType == 'ASTIERAPPROXIMATION':
115 g = self.gain
116 for ampName in self.ampNames:
117 localDataset.rawVars[ampName] = [(0.5/(self.a00*g**2)*(np.exp(2*self.a00*mu*g)-1) +
118 self.noiseSq/(g*g)) for mu in localDataset.rawMeans[ampName]]
119 else:
120 RuntimeError("Enter a fit function type: 'POLYNOMIAL' or 'ASTIERAPPROXIMATION'")
122 config.maxAduForLookupTableLinearizer = 200000 # Max ADU in input mock flats
123 task = cpPipe.ptc.MeasurePhotonTransferCurveTask(config=config)
125 if doTableArray:
126 numberAmps = len(self.ampNames)
127 numberAduValues = config.maxAduForLookupTableLinearizer
128 lookupTableArray = np.zeros((numberAmps, numberAduValues), dtype=np.float32)
129 returnedDataset = task.fitPtcAndNonLinearity(localDataset, ptcFitType=fitType,
130 tableArray=lookupTableArray)
131 else:
132 returnedDataset = task.fitPtcAndNonLinearity(localDataset, ptcFitType=fitType)
134 if doTableArray:
135 # check that the linearizer table has been filled out properly
136 for i in np.arange(numberAmps):
137 tMax = (config.maxAduForLookupTableLinearizer)/self.flux
138 timeRange = np.linspace(0., tMax, config.maxAduForLookupTableLinearizer)
139 signalIdeal = timeRange*self.flux
140 signalUncorrected = task.funcPolynomial(np.array([0.0, self.flux, self.k2NonLinearity]),
141 timeRange)
142 linearizerTableRow = signalIdeal - signalUncorrected
143 self.assertEqual(len(linearizerTableRow), len(lookupTableArray[i, :]))
144 for j in np.arange(len(linearizerTableRow)):
145 self.assertAlmostEqual(linearizerTableRow[j], lookupTableArray[i, :][j], places=6)
147 # check entries in localDataset, which was modified by the function
148 for ampName in self.ampNames:
149 maskAmp = localDataset.visitMask[ampName]
150 finalMuVec = localDataset.rawMeans[ampName][maskAmp]
151 finalTimeVec = localDataset.rawExpTimes[ampName][maskAmp]
152 inputNonLinearityResiduals = 100*(1 - ((finalMuVec[2]/finalTimeVec[2])/(finalMuVec/finalTimeVec)))
153 linearPart = self.flux*finalTimeVec
154 inputFracNonLinearityResiduals = 100*(linearPart - finalMuVec)/linearPart
156 self.assertEqual(fitType, localDataset.ptcFitType[ampName])
157 self.assertAlmostEqual(self.gain, localDataset.gain[ampName])
158 if fitType == 'POLYNOMIAL':
159 self.assertAlmostEqual(self.c1, localDataset.ptcFitPars[ampName][1])
160 self.assertAlmostEqual(np.sqrt(self.noiseSq)*self.gain, localDataset.noise[ampName])
161 else:
162 self.assertAlmostEqual(self.a00, localDataset.ptcFitPars[ampName][0])
163 # noise already in electrons for 'ASTIERAPPROXIMATION' fit
164 self.assertAlmostEqual(np.sqrt(self.noiseSq), localDataset.noise[ampName])
165 # Nonlinearity fit parameters
166 self.assertAlmostEqual(0.0, localDataset.nonLinearity[ampName][0])
167 self.assertAlmostEqual(self.flux, localDataset.nonLinearity[ampName][1])
168 self.assertAlmostEqual(self.k2NonLinearity, localDataset.nonLinearity[ampName][2])
170 # Non-linearity coefficient for quadratic linearizer
171 self.assertAlmostEqual(-self.k2NonLinearity/(self.flux**2),
172 localDataset.coefficientLinearizeSquared[ampName])
174 # Linearity residuals
175 self.assertEqual(len(localDataset.nonLinearityResiduals[ampName]),
176 len(inputNonLinearityResiduals))
177 for calc, truth in zip(localDataset.nonLinearityResiduals[ampName],
178 inputNonLinearityResiduals):
179 self.assertAlmostEqual(calc, truth)
181 # Fractional nonlinearity residuals
182 self.assertEqual(len(localDataset.fractionalNonLinearityResiduals[ampName]),
183 len(inputFracNonLinearityResiduals))
184 for calc, truth in zip(localDataset.fractionalNonLinearityResiduals[ampName],
185 inputFracNonLinearityResiduals):
186 self.assertAlmostEqual(calc, truth)
188 # check calls to calculateLinearityResidualAndLinearizers
189 datasetLinResAndLinearizers = task.calculateLinearityResidualAndLinearizers(
190 localDataset.rawExpTimes[ampName], localDataset.rawMeans[ampName])
192 self.assertAlmostEqual(-self.k2NonLinearity/(self.flux**2),
193 datasetLinResAndLinearizers.quadraticPolynomialLinearizerCoefficient)
194 self.assertAlmostEqual(0.0, datasetLinResAndLinearizers.meanSignalVsTimePolyFitPars[0])
195 self.assertAlmostEqual(self.flux, datasetLinResAndLinearizers.meanSignalVsTimePolyFitPars[1])
196 self.assertAlmostEqual(self.k2NonLinearity,
197 datasetLinResAndLinearizers.meanSignalVsTimePolyFitPars[2])
199 # check entries in returned dataset (should be the same as localDataset after calling the function)
200 for ampName in self.ampNames:
201 maskAmp = returnedDataset.visitMask[ampName]
202 finalMuVec = returnedDataset.rawMeans[ampName][maskAmp]
203 finalTimeVec = returnedDataset.rawExpTimes[ampName][maskAmp]
204 inputNonLinearityResiduals = 100*(1 - ((finalMuVec[2]/finalTimeVec[2])/(finalMuVec/finalTimeVec)))
205 linearPart = self.flux*finalTimeVec
206 inputFracNonLinearityResiduals = 100*(linearPart - finalMuVec)/linearPart
208 self.assertEqual(fitType, returnedDataset.ptcFitType[ampName])
209 self.assertAlmostEqual(self.gain, returnedDataset.gain[ampName])
210 if fitType == 'POLYNOMIAL':
211 self.assertAlmostEqual(self.c1, returnedDataset.ptcFitPars[ampName][1])
212 self.assertAlmostEqual(np.sqrt(self.noiseSq)*self.gain, returnedDataset.noise[ampName])
213 else:
214 self.assertAlmostEqual(self.a00, returnedDataset.ptcFitPars[ampName][0])
215 # noise already in electrons for 'ASTIERAPPROXIMATION' fit
216 self.assertAlmostEqual(np.sqrt(self.noiseSq), returnedDataset.noise[ampName])
218 # Nonlinearity fit parameters
219 self.assertAlmostEqual(0.0, returnedDataset.nonLinearity[ampName][0])
220 self.assertAlmostEqual(self.flux, returnedDataset.nonLinearity[ampName][1])
221 self.assertAlmostEqual(self.k2NonLinearity, returnedDataset.nonLinearity[ampName][2])
223 # Non-linearity coefficient for linearizer
224 self.assertAlmostEqual(-self.k2NonLinearity/(self.flux**2),
225 returnedDataset.coefficientLinearizeSquared[ampName])
227 # Linearity residuals
228 self.assertEqual(len(returnedDataset.nonLinearityResiduals[ampName]),
229 len(inputNonLinearityResiduals))
230 for calc, truth in zip(returnedDataset.nonLinearityResiduals[ampName],
231 inputNonLinearityResiduals):
232 self.assertAlmostEqual(calc, truth)
234 # Fractional nonlinearity residuals
235 self.assertEqual(len(returnedDataset.fractionalNonLinearityResiduals[ampName]),
236 len(inputFracNonLinearityResiduals))
237 for calc, truth in zip(returnedDataset.fractionalNonLinearityResiduals[ampName],
238 inputFracNonLinearityResiduals):
239 self.assertAlmostEqual(calc, truth)
241 # check calls to calculateLinearityResidualAndLinearizers
242 datasetLinResAndLinearizers = task.calculateLinearityResidualAndLinearizers(
243 returnedDataset.rawExpTimes[ampName], returnedDataset.rawMeans[ampName])
245 self.assertAlmostEqual(-self.k2NonLinearity/(self.flux**2),
246 datasetLinResAndLinearizers.quadraticPolynomialLinearizerCoefficient)
247 self.assertAlmostEqual(0.0, datasetLinResAndLinearizers.meanSignalVsTimePolyFitPars[0])
248 self.assertAlmostEqual(self.flux, datasetLinResAndLinearizers.meanSignalVsTimePolyFitPars[1])
249 self.assertAlmostEqual(self.k2NonLinearity,
250 datasetLinResAndLinearizers.meanSignalVsTimePolyFitPars[2])
252 def test_ptcFit(self):
253 for createArray in [True, False]:
254 for typeAndOrder in [('POLYNOMIAL', 2), ('POLYNOMIAL', 3), ('ASTIERAPPROXIMATION', None)]:
255 self.ptcFitAndCheckPtc(fitType=typeAndOrder[0], order=typeAndOrder[1],
256 doTableArray=createArray)
258 def test_meanVarMeasurement(self):
259 task = self.defaultTask
260 mu, varDiff = task.measureMeanVarPair(self.flatExp1, self.flatExp2)
262 self.assertLess(self.flatWidth - np.sqrt(varDiff), 1)
263 self.assertLess(self.flatMean - mu, 1)
265 def test_getInitialGoodPoints(self):
266 xs = [1, 2, 3, 4, 5, 6]
267 ys = [2*x for x in xs]
268 points = self.defaultTask._getInitialGoodPoints(xs, ys, 0.1, 0.25)
269 assert np.all(points) == np.all(np.array([True for x in xs]))
271 ys[-1] = 30
272 points = self.defaultTask._getInitialGoodPoints(xs, ys, 0.1, 0.25)
273 assert np.all(points) == np.all(np.array([True, True, True, True, False]))
275 ys = [2*x for x in xs]
276 newYs = copy.copy(ys)
277 results = [False, True, True, False, False]
278 for i, factor in enumerate([-0.5, -0.1, 0, 0.1, 0.5]):
279 newYs[-1] = ys[-1] + (factor*ys[-1])
280 points = self.defaultTask._getInitialGoodPoints(xs, newYs, 0.05, 0.25)
281 assert (np.all(points[0:-1]) == True) # noqa: E712 - flake8 is wrong here because of numpy.bool
282 assert points[-1] == results[i]
284 def test_getVisitsUsed(self):
285 localDataset = copy.copy(self.dataset)
287 for pair in [(12, 34), (56, 78), (90, 10)]:
288 localDataset.inputVisitPairs["C:0,0"].append(pair)
289 localDataset.visitMask["C:0,0"] = np.array([True, False, True])
290 self.assertTrue(np.all(localDataset.getVisitsUsed("C:0,0") == [(12, 34), (90, 10)]))
292 localDataset.visitMask["C:0,0"] = np.array([True, False, True, True]) # wrong length now
293 with self.assertRaises(AssertionError):
294 localDataset.getVisitsUsed("C:0,0")
296 def test_getGoodAmps(self):
297 dataset = self.dataset
299 self.assertTrue(dataset.ampNames == self.ampNames)
300 dataset.badAmps.append("C:0,1")
301 self.assertTrue(dataset.getGoodAmps() == [amp for amp in self.ampNames if amp != "C:0,1"])
304class MeasurePhotonTransferCurveDatasetTestCase(lsst.utils.tests.TestCase):
305 def setUp(self):
306 self.ptcData = PhotonTransferCurveDataset(['C00', 'C01'])
307 self.ptcData.inputVisitPairs = {'C00': [(123, 234), (345, 456), (567, 678)],
308 'C01': [(123, 234), (345, 456), (567, 678)]}
310 def test_generalBehaviour(self):
311 test = PhotonTransferCurveDataset(['C00', 'C01'])
312 test.inputVisitPairs = {'C00': [(123, 234), (345, 456), (567, 678)],
313 'C01': [(123, 234), (345, 456), (567, 678)]}
315 with self.assertRaises(AttributeError):
316 test.newItem = 1
319class TestMemory(lsst.utils.tests.MemoryTestCase):
320 pass
323def setup_module(module):
324 lsst.utils.tests.init()
327if __name__ == "__main__": 327 ↛ 328line 327 didn't jump to line 328, because the condition on line 327 was never true
328 lsst.utils.tests.init()
329 unittest.main()