Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1#!/usr/bin/env python 

2 

3# 

4# LSST Data Management System 

5# 

6# Copyright 2008-2017 AURA/LSST. 

7# 

8# This product includes software developed by the 

9# LSST Project (http://www.lsst.org/). 

10# 

11# This program is free software: you can redistribute it and/or modify 

12# it under the terms of the GNU General Public License as published by 

13# the Free Software Foundation, either version 3 of the License, or 

14# (at your option) any later version. 

15# 

16# This program is distributed in the hope that it will be useful, 

17# but WITHOUT ANY WARRANTY; without even the implied warranty of 

18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

19# GNU General Public License for more details. 

20# 

21# You should have received a copy of the LSST License Statement and 

22# the GNU General Public License along with this program. If not, 

23# see <https://www.lsstcorp.org/LegalNotices/>. 

24# 

25"""Test cases for cp_pipe.""" 

26 

27from __future__ import absolute_import, division, print_function 

28import unittest 

29import numpy as np 

30import copy 

31 

32import lsst.utils 

33import lsst.utils.tests 

34 

35import lsst.cp.pipe as cpPipe 

36import lsst.ip.isr.isrMock as isrMock 

37from lsst.cp.pipe.ptc import PhotonTransferCurveDataset 

38 

39 

40class MeasurePhotonTransferCurveTaskTestCase(lsst.utils.tests.TestCase): 

41 """A test case for the PTC task.""" 

42 

43 def setUp(self): 

44 self.defaultConfig = cpPipe.ptc.MeasurePhotonTransferCurveTask.ConfigClass() 

45 self.defaultConfig.isr.doFlat = False 

46 self.defaultConfig.isr.doFringe = False 

47 self.defaultConfig.isr.doCrosstalk = False 

48 self.defaultConfig.isr.doAddDistortionModel = False 

49 self.defaultConfig.isr.doUseOpticsTransmission = False 

50 self.defaultConfig.isr.doUseFilterTransmission = False 

51 self.defaultConfig.isr.doUseSensorTransmission = False 

52 self.defaultConfig.isr.doUseAtmosphereTransmission = False 

53 self.defaultConfig.isr.doAttachTransmissionCurve = False 

54 

55 self.defaultTask = cpPipe.ptc.MeasurePhotonTransferCurveTask(config=self.defaultConfig) 

56 

57 self.flatMean = 2000 

58 self.readNoiseAdu = 10 

59 mockImageConfig = isrMock.IsrMock.ConfigClass() 

60 

61 # flatDrop is not really relevant as we replace the data 

62 # but good to note it in case we change how this image is made 

63 mockImageConfig.flatDrop = 0.99999 

64 mockImageConfig.isTrimmed = True 

65 

66 self.flatExp1 = isrMock.FlatMock(config=mockImageConfig).run() 

67 self.flatExp2 = self.flatExp1.clone() 

68 (shapeY, shapeX) = self.flatExp1.getDimensions() 

69 

70 self.flatWidth = np.sqrt(self.flatMean) + self.readNoiseAdu 

71 

72 self.rng1 = np.random.RandomState(1984) 

73 flatData1 = self.rng1.normal(self.flatMean, self.flatWidth, (shapeX, shapeY)) 

74 self.rng2 = np.random.RandomState(666) 

75 flatData2 = self.rng2.normal(self.flatMean, self.flatWidth, (shapeX, shapeY)) 

76 

77 self.flatExp1.image.array[:] = flatData1 

78 self.flatExp2.image.array[:] = flatData2 

79 

80 # create fake PTC data to see if fit works, for one amp ('amp') 

81 self.flux = 1000 # ADU/sec 

82 timeVec = np.arange(1., 201.) 

83 self.k2NonLinearity = -5e-6 

84 muVec = self.flux*timeVec + self.k2NonLinearity*timeVec**2 # quadratic signal-chain non-linearity 

85 # Assumes parameter linResidualTimeIndex = 2 (default) in PTC task 

86 self.inputNonLinearityResiduals = 100*(1 - ((muVec[2]/timeVec[2])/(muVec/timeVec))) 

87 

88 self.gain = 1.5 # e-/ADU 

89 self.c1 = 1./self.gain 

90 self.noiseSq = 5*self.gain # 7.5 (e-)^2 

91 self.a00 = -1.2e-6 

92 self.c2 = -1.5e-6 

93 self.c3 = -4.7e-12 # tuned so that it turns over for 200k mean 

94 

95 self.ampNames = [amp.getName() for amp in self.flatExp1.getDetector().getAmplifiers()] 

96 self.dataset = PhotonTransferCurveDataset(self.ampNames) # pack raw data for fitting 

97 

98 for ampName in self.ampNames: # just the expTimes and means here - vars vary per function 

99 self.dataset.rawExpTimes[ampName] = timeVec 

100 self.dataset.rawMeans[ampName] = muVec 

101 

102 def ptcFitAndCheckPtc(self, order=None, fitType='', doTableArray=False): 

103 localDataset = copy.copy(self.dataset) 

104 config = copy.copy(self.defaultConfig) 

105 if fitType == 'POLYNOMIAL': 

106 if order not in [2, 3]: 

107 RuntimeError("Enter a valid polynomial order for this test: 2 or 3") 

108 if order == 2: 

109 for ampName in self.ampNames: 

110 localDataset.rawVars[ampName] = [self.noiseSq + self.c1*mu + self.c2*mu**2 for 

111 mu in localDataset.rawMeans[ampName]] 

112 config.polynomialFitDegree = 2 

113 if order == 3: 

114 for ampName in self.ampNames: 

115 localDataset.rawVars[ampName] = [self.noiseSq + self.c1*mu + self.c2*mu**2 + self.c3*mu**3 

116 for mu in localDataset.rawMeans[ampName]] 

117 config.polynomialFitDegree = 3 

118 elif fitType == 'ASTIERAPPROXIMATION': 

119 g = self.gain 

120 for ampName in self.ampNames: 

121 localDataset.rawVars[ampName] = [(0.5/(self.a00*g**2)*(np.exp(2*self.a00*mu*g)-1) + 

122 self.noiseSq/(g*g)) for mu in localDataset.rawMeans[ampName]] 

123 else: 

124 RuntimeError("Enter a fit function type: 'POLYNOMIAL' or 'ASTIERAPPROXIMATION'") 

125 

126 config.maxAduForLookupTableLinearizer = 200000 # Max ADU in input mock flats 

127 task = cpPipe.ptc.MeasurePhotonTransferCurveTask(config=config) 

128 

129 if doTableArray: 

130 numberAmps = len(self.ampNames) 

131 numberAduValues = config.maxAduForLookupTableLinearizer 

132 lookupTableArray = np.zeros((numberAmps, numberAduValues), dtype=np.int) 

133 returnedDataset = task.fitPtcAndNonLinearity(localDataset, ptcFitType=fitType, 

134 tableArray=lookupTableArray) 

135 else: 

136 returnedDataset = task.fitPtcAndNonLinearity(localDataset, ptcFitType=fitType) 

137 

138 if doTableArray: 

139 # check that the linearizer table has been filled out properly 

140 for i in np.arange(numberAmps): 

141 tMax = (config.maxAduForLookupTableLinearizer)/self.flux 

142 timeRange = np.linspace(0., tMax, config.maxAduForLookupTableLinearizer) 

143 signalIdeal = timeRange*self.flux 

144 signalUncorrected = task.funcPolynomial(np.array([0.0, self.flux, self.k2NonLinearity]), 

145 timeRange) 

146 linearizerTableRow = signalIdeal.astype(int) - signalUncorrected.astype(int) 

147 self.assertEqual(len(linearizerTableRow), len(lookupTableArray[i, :])) 

148 for j in np.arange(len(linearizerTableRow)): 

149 self.assertAlmostEqual(linearizerTableRow[j], lookupTableArray[i, :][j], places=6) 

150 

151 # check entries in localDataset, which was modified by the function 

152 for ampName in self.ampNames: 

153 self.assertEqual(fitType, localDataset.ptcFitType[ampName]) 

154 self.assertAlmostEqual(self.gain, localDataset.gain[ampName]) 

155 if fitType == 'POLYNOMIAL': 

156 self.assertAlmostEqual(self.c1, localDataset.ptcFitPars[ampName][1]) 

157 self.assertAlmostEqual(np.sqrt(self.noiseSq)*self.gain, localDataset.noise[ampName]) 

158 else: 

159 self.assertAlmostEqual(self.a00, localDataset.ptcFitPars[ampName][0]) 

160 # noise already in electrons for 'ASTIERAPPROXIMATION' fit 

161 self.assertAlmostEqual(np.sqrt(self.noiseSq), localDataset.noise[ampName]) 

162 # Nonlinearity fit parameters 

163 self.assertAlmostEqual(0.0, localDataset.nonLinearity[ampName][0]) 

164 self.assertAlmostEqual(self.flux, localDataset.nonLinearity[ampName][1]) 

165 self.assertAlmostEqual(self.k2NonLinearity, localDataset.nonLinearity[ampName][2]) 

166 

167 # Non-linearity coefficient for quadratic linearizer 

168 self.assertAlmostEqual(-self.k2NonLinearity/(self.flux**2), 

169 localDataset.coefficientLinearizeSquared[ampName]) 

170 

171 # Linearity residuals 

172 self.assertEqual(len(localDataset.nonLinearityResiduals[ampName]), 

173 len(self.inputNonLinearityResiduals[localDataset.visitMask[ampName]])) 

174 

175 for i in np.arange(len(localDataset.nonLinearityResiduals[ampName])): 

176 self.assertAlmostEqual(localDataset.nonLinearityResiduals[ampName][i], 

177 self.inputNonLinearityResiduals[i]) 

178 

179 # check entries in returned dataset (should be the same as localDataset after calling the funciton) 

180 for ampName in self.ampNames: 

181 self.assertEqual(fitType, returnedDataset.ptcFitType[ampName]) 

182 self.assertAlmostEqual(self.gain, returnedDataset.gain[ampName]) 

183 if fitType == 'POLYNOMIAL': 

184 self.assertAlmostEqual(self.c1, returnedDataset.ptcFitPars[ampName][1]) 

185 self.assertAlmostEqual(np.sqrt(self.noiseSq)*self.gain, returnedDataset.noise[ampName]) 

186 else: 

187 self.assertAlmostEqual(self.a00, returnedDataset.ptcFitPars[ampName][0]) 

188 # noise already in electrons for 'ASTIERAPPROXIMATION' fit 

189 self.assertAlmostEqual(np.sqrt(self.noiseSq), returnedDataset.noise[ampName]) 

190 

191 # Nonlinearity fit parameters 

192 self.assertAlmostEqual(0.0, localDataset.nonLinearity[ampName][0]) 

193 self.assertAlmostEqual(self.flux, localDataset.nonLinearity[ampName][1]) 

194 self.assertAlmostEqual(self.k2NonLinearity, localDataset.nonLinearity[ampName][2]) 

195 

196 # Non-linearity coefficient for linearizer 

197 self.assertAlmostEqual(-self.k2NonLinearity/(self.flux**2), 

198 localDataset.coefficientLinearizeSquared[ampName]) 

199 

200 # Linearity residuals 

201 self.assertEqual(len(localDataset.nonLinearityResiduals[ampName]), 

202 len(self.inputNonLinearityResiduals[localDataset.visitMask[ampName]])) 

203 

204 for i in np.arange(len(localDataset.nonLinearityResiduals[ampName])): 

205 self.assertAlmostEqual(localDataset.nonLinearityResiduals[ampName][i], 

206 self.inputNonLinearityResiduals[i]) 

207 

208 def test_ptcFit(self): 

209 for createArray in [True, False]: 

210 for typeAndOrder in [('POLYNOMIAL', 2), ('POLYNOMIAL', 3), ('ASTIERAPPROXIMATION', None)]: 

211 self.ptcFitAndCheckPtc(fitType=typeAndOrder[0], order=typeAndOrder[1], 

212 doTableArray=createArray) 

213 

214 def test_meanVarMeasurement(self): 

215 task = self.defaultTask 

216 mu, varDiff = task.measureMeanVarPair(self.flatExp1, self.flatExp2) 

217 

218 self.assertLess(self.flatWidth - np.sqrt(varDiff), 1) 

219 self.assertLess(self.flatMean - mu, 1) 

220 

221 def test_getInitialGoodPoints(self): 

222 xs = [1, 2, 3, 4, 5, 6] 

223 ys = [2*x for x in xs] 

224 points = self.defaultTask._getInitialGoodPoints(xs, ys, 0.1, 0.25) 

225 assert np.all(points) == np.all(np.array([True for x in xs])) 

226 

227 ys[-1] = 30 

228 points = self.defaultTask._getInitialGoodPoints(xs, ys, 0.1, 0.25) 

229 assert np.all(points) == np.all(np.array([True, True, True, True, False])) 

230 

231 ys = [2*x for x in xs] 

232 newYs = copy.copy(ys) 

233 results = [False, True, True, False, False] 

234 for i, factor in enumerate([-0.5, -0.1, 0, 0.1, 0.5]): 

235 newYs[-1] = ys[-1] + (factor*ys[-1]) 

236 points = self.defaultTask._getInitialGoodPoints(xs, newYs, 0.05, 0.25) 

237 assert (np.all(points[0:-1]) == True) # noqa: E712 - flake8 is wrong here because of numpy.bool 

238 assert points[-1] == results[i] 

239 

240 def test_getVisitsUsed(self): 

241 localDataset = copy.copy(self.dataset) 

242 

243 for pair in [(12, 34), (56, 78), (90, 10)]: 

244 localDataset.inputVisitPairs["C:0,0"].append(pair) 

245 localDataset.visitMask["C:0,0"] = np.array([True, False, True]) 

246 self.assertTrue(np.all(localDataset.getVisitsUsed("C:0,0") == [(12, 34), (90, 10)])) 

247 

248 localDataset.visitMask["C:0,0"] = np.array([True, False, True, True]) # wrong length now 

249 with self.assertRaises(AssertionError): 

250 localDataset.getVisitsUsed("C:0,0") 

251 

252 def test_getGoodAmps(self): 

253 dataset = self.dataset 

254 

255 self.assertTrue(dataset.ampNames == self.ampNames) 

256 dataset.badAmps.append("C:0,1") 

257 self.assertTrue(dataset.getGoodAmps() == [amp for amp in self.ampNames if amp != "C:0,1"]) 

258 

259 

260class MeasurePhotonTransferCurveDatasetTestCase(lsst.utils.tests.TestCase): 

261 def setUp(self): 

262 self.ptcData = PhotonTransferCurveDataset(['C00', 'C01']) 

263 self.ptcData.inputVisitPairs = {'C00': [(123, 234), (345, 456), (567, 678)], 

264 'C01': [(123, 234), (345, 456), (567, 678)]} 

265 

266 def test_generalBehaviour(self): 

267 test = PhotonTransferCurveDataset(['C00', 'C01']) 

268 test.inputVisitPairs = {'C00': [(123, 234), (345, 456), (567, 678)], 

269 'C01': [(123, 234), (345, 456), (567, 678)]} 

270 

271 with self.assertRaises(AttributeError): 

272 test.newItem = 1 

273 

274 

275class TestMemory(lsst.utils.tests.MemoryTestCase): 

276 pass 

277 

278 

279def setup_module(module): 

280 lsst.utils.tests.init() 

281 

282 

283if __name__ == "__main__": 283 ↛ 284line 283 didn't jump to line 284, because the condition on line 283 was never true

284 lsst.utils.tests.init() 

285 unittest.main()