Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1#!/usr/bin/env python 

2 

3# 

4# LSST Data Management System 

5# 

6# Copyright 2008-2017 AURA/LSST. 

7# 

8# This product includes software developed by the 

9# LSST Project (http://www.lsst.org/). 

10# 

11# This program is free software: you can redistribute it and/or modify 

12# it under the terms of the GNU General Public License as published by 

13# the Free Software Foundation, either version 3 of the License, or 

14# (at your option) any later version. 

15# 

16# This program is distributed in the hope that it will be useful, 

17# but WITHOUT ANY WARRANTY; without even the implied warranty of 

18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

19# GNU General Public License for more details. 

20# 

21# You should have received a copy of the LSST License Statement and 

22# the GNU General Public License along with this program. If not, 

23# see <https://www.lsstcorp.org/LegalNotices/>. 

24# 

25"""Test cases for cp_pipe.""" 

26 

27from __future__ import absolute_import, division, print_function 

28import unittest 

29import numpy as np 

30import copy 

31 

32import lsst.utils 

33import lsst.utils.tests 

34 

35import lsst.cp.pipe as cpPipe 

36import lsst.ip.isr.isrMock as isrMock 

37from lsst.cp.pipe.ptc import PhotonTransferCurveDataset 

38 

39 

40class MeasurePhotonTransferCurveTaskTestCase(lsst.utils.tests.TestCase): 

41 """A test case for the PTC task.""" 

42 

43 def setUp(self): 

44 self.defaultConfig = cpPipe.ptc.MeasurePhotonTransferCurveTask.ConfigClass() 

45 self.defaultConfig.isr.doFlat = False 

46 self.defaultConfig.isr.doFringe = False 

47 self.defaultConfig.isr.doCrosstalk = False 

48 self.defaultConfig.isr.doUseOpticsTransmission = False 

49 self.defaultConfig.isr.doUseFilterTransmission = False 

50 self.defaultConfig.isr.doUseSensorTransmission = False 

51 self.defaultConfig.isr.doUseAtmosphereTransmission = False 

52 self.defaultConfig.isr.doAttachTransmissionCurve = False 

53 

54 self.defaultTask = cpPipe.ptc.MeasurePhotonTransferCurveTask(config=self.defaultConfig) 

55 

56 self.flatMean = 2000 

57 self.readNoiseAdu = 10 

58 mockImageConfig = isrMock.IsrMock.ConfigClass() 

59 

60 # flatDrop is not really relevant as we replace the data 

61 # but good to note it in case we change how this image is made 

62 mockImageConfig.flatDrop = 0.99999 

63 mockImageConfig.isTrimmed = True 

64 

65 self.flatExp1 = isrMock.FlatMock(config=mockImageConfig).run() 

66 self.flatExp2 = self.flatExp1.clone() 

67 (shapeY, shapeX) = self.flatExp1.getDimensions() 

68 

69 self.flatWidth = np.sqrt(self.flatMean) + self.readNoiseAdu 

70 

71 self.rng1 = np.random.RandomState(1984) 

72 flatData1 = self.rng1.normal(self.flatMean, self.flatWidth, (shapeX, shapeY)) 

73 self.rng2 = np.random.RandomState(666) 

74 flatData2 = self.rng2.normal(self.flatMean, self.flatWidth, (shapeX, shapeY)) 

75 

76 self.flatExp1.image.array[:] = flatData1 

77 self.flatExp2.image.array[:] = flatData2 

78 

79 # create fake PTC data to see if fit works, for one amp ('amp') 

80 self.flux = 1000. # ADU/sec 

81 timeVec = np.arange(1., 201.) 

82 self.k2NonLinearity = -5e-6 

83 muVec = self.flux*timeVec + self.k2NonLinearity*timeVec**2 # quadratic signal-chain non-linearity 

84 self.gain = 1.5 # e-/ADU 

85 self.c1 = 1./self.gain 

86 self.noiseSq = 5*self.gain # 7.5 (e-)^2 

87 self.a00 = -1.2e-6 

88 self.c2 = -1.5e-6 

89 self.c3 = -4.7e-12 # tuned so that it turns over for 200k mean 

90 

91 self.ampNames = [amp.getName() for amp in self.flatExp1.getDetector().getAmplifiers()] 

92 self.dataset = PhotonTransferCurveDataset(self.ampNames) # pack raw data for fitting 

93 

94 for ampName in self.ampNames: # just the expTimes and means here - vars vary per function 

95 self.dataset.rawExpTimes[ampName] = timeVec 

96 self.dataset.rawMeans[ampName] = muVec 

97 

98 def ptcFitAndCheckPtc(self, order=None, fitType='', doTableArray=False): 

99 localDataset = copy.copy(self.dataset) 

100 config = copy.copy(self.defaultConfig) 

101 if fitType == 'POLYNOMIAL': 

102 if order not in [2, 3]: 

103 RuntimeError("Enter a valid polynomial order for this test: 2 or 3") 

104 if order == 2: 

105 for ampName in self.ampNames: 

106 localDataset.rawVars[ampName] = [self.noiseSq + self.c1*mu + self.c2*mu**2 for 

107 mu in localDataset.rawMeans[ampName]] 

108 config.polynomialFitDegree = 2 

109 if order == 3: 

110 for ampName in self.ampNames: 

111 localDataset.rawVars[ampName] = [self.noiseSq + self.c1*mu + self.c2*mu**2 + self.c3*mu**3 

112 for mu in localDataset.rawMeans[ampName]] 

113 config.polynomialFitDegree = 3 

114 elif fitType == 'ASTIERAPPROXIMATION': 

115 g = self.gain 

116 for ampName in self.ampNames: 

117 localDataset.rawVars[ampName] = [(0.5/(self.a00*g**2)*(np.exp(2*self.a00*mu*g)-1) + 

118 self.noiseSq/(g*g)) for mu in localDataset.rawMeans[ampName]] 

119 else: 

120 RuntimeError("Enter a fit function type: 'POLYNOMIAL' or 'ASTIERAPPROXIMATION'") 

121 

122 config.maxAduForLookupTableLinearizer = 200000 # Max ADU in input mock flats 

123 task = cpPipe.ptc.MeasurePhotonTransferCurveTask(config=config) 

124 

125 if doTableArray: 

126 numberAmps = len(self.ampNames) 

127 numberAduValues = config.maxAduForLookupTableLinearizer 

128 lookupTableArray = np.zeros((numberAmps, numberAduValues), dtype=np.float32) 

129 returnedDataset = task.fitPtcAndNonLinearity(localDataset, ptcFitType=fitType, 

130 tableArray=lookupTableArray) 

131 else: 

132 returnedDataset = task.fitPtcAndNonLinearity(localDataset, ptcFitType=fitType) 

133 

134 if doTableArray: 

135 # check that the linearizer table has been filled out properly 

136 for i in np.arange(numberAmps): 

137 tMax = (config.maxAduForLookupTableLinearizer)/self.flux 

138 timeRange = np.linspace(0., tMax, config.maxAduForLookupTableLinearizer) 

139 signalIdeal = timeRange*self.flux 

140 signalUncorrected = task.funcPolynomial(np.array([0.0, self.flux, self.k2NonLinearity]), 

141 timeRange) 

142 linearizerTableRow = signalIdeal - signalUncorrected 

143 self.assertEqual(len(linearizerTableRow), len(lookupTableArray[i, :])) 

144 for j in np.arange(len(linearizerTableRow)): 

145 self.assertAlmostEqual(linearizerTableRow[j], lookupTableArray[i, :][j], places=6) 

146 

147 # check entries in localDataset, which was modified by the function 

148 for ampName in self.ampNames: 

149 maskAmp = localDataset.visitMask[ampName] 

150 finalMuVec = localDataset.rawMeans[ampName][maskAmp] 

151 finalTimeVec = localDataset.rawExpTimes[ampName][maskAmp] 

152 inputNonLinearityResiduals = 100*(1 - ((finalMuVec[2]/finalTimeVec[2])/(finalMuVec/finalTimeVec))) 

153 linearPart = self.flux*finalTimeVec 

154 inputFracNonLinearityResiduals = 100*(linearPart - finalMuVec)/linearPart 

155 

156 self.assertEqual(fitType, localDataset.ptcFitType[ampName]) 

157 self.assertAlmostEqual(self.gain, localDataset.gain[ampName]) 

158 if fitType == 'POLYNOMIAL': 

159 self.assertAlmostEqual(self.c1, localDataset.ptcFitPars[ampName][1]) 

160 self.assertAlmostEqual(np.sqrt(self.noiseSq)*self.gain, localDataset.noise[ampName]) 

161 else: 

162 self.assertAlmostEqual(self.a00, localDataset.ptcFitPars[ampName][0]) 

163 # noise already in electrons for 'ASTIERAPPROXIMATION' fit 

164 self.assertAlmostEqual(np.sqrt(self.noiseSq), localDataset.noise[ampName]) 

165 # Nonlinearity fit parameters 

166 self.assertAlmostEqual(0.0, localDataset.nonLinearity[ampName][0]) 

167 self.assertAlmostEqual(self.flux, localDataset.nonLinearity[ampName][1]) 

168 self.assertAlmostEqual(self.k2NonLinearity, localDataset.nonLinearity[ampName][2]) 

169 

170 # Non-linearity coefficient for quadratic linearizer 

171 self.assertAlmostEqual(-self.k2NonLinearity/(self.flux**2), 

172 localDataset.coefficientLinearizeSquared[ampName]) 

173 

174 # Linearity residuals 

175 self.assertEqual(len(localDataset.nonLinearityResiduals[ampName]), 

176 len(inputNonLinearityResiduals)) 

177 for calc, truth in zip(localDataset.nonLinearityResiduals[ampName], 

178 inputNonLinearityResiduals): 

179 self.assertAlmostEqual(calc, truth) 

180 

181 # Fractional nonlinearity residuals 

182 self.assertEqual(len(localDataset.fractionalNonLinearityResiduals[ampName]), 

183 len(inputFracNonLinearityResiduals)) 

184 for calc, truth in zip(localDataset.fractionalNonLinearityResiduals[ampName], 

185 inputFracNonLinearityResiduals): 

186 self.assertAlmostEqual(calc, truth) 

187 

188 # check calls to calculateLinearityResidualAndLinearizers 

189 datasetLinResAndLinearizers = task.calculateLinearityResidualAndLinearizers( 

190 localDataset.rawExpTimes[ampName], localDataset.rawMeans[ampName]) 

191 

192 self.assertAlmostEqual(-self.k2NonLinearity/(self.flux**2), 

193 datasetLinResAndLinearizers.quadraticPolynomialLinearizerCoefficient) 

194 self.assertAlmostEqual(0.0, datasetLinResAndLinearizers.meanSignalVsTimePolyFitPars[0]) 

195 self.assertAlmostEqual(self.flux, datasetLinResAndLinearizers.meanSignalVsTimePolyFitPars[1]) 

196 self.assertAlmostEqual(self.k2NonLinearity, 

197 datasetLinResAndLinearizers.meanSignalVsTimePolyFitPars[2]) 

198 

199 # check entries in returned dataset (should be the same as localDataset after calling the function) 

200 for ampName in self.ampNames: 

201 maskAmp = returnedDataset.visitMask[ampName] 

202 finalMuVec = returnedDataset.rawMeans[ampName][maskAmp] 

203 finalTimeVec = returnedDataset.rawExpTimes[ampName][maskAmp] 

204 inputNonLinearityResiduals = 100*(1 - ((finalMuVec[2]/finalTimeVec[2])/(finalMuVec/finalTimeVec))) 

205 linearPart = self.flux*finalTimeVec 

206 inputFracNonLinearityResiduals = 100*(linearPart - finalMuVec)/linearPart 

207 

208 self.assertEqual(fitType, returnedDataset.ptcFitType[ampName]) 

209 self.assertAlmostEqual(self.gain, returnedDataset.gain[ampName]) 

210 if fitType == 'POLYNOMIAL': 

211 self.assertAlmostEqual(self.c1, returnedDataset.ptcFitPars[ampName][1]) 

212 self.assertAlmostEqual(np.sqrt(self.noiseSq)*self.gain, returnedDataset.noise[ampName]) 

213 else: 

214 self.assertAlmostEqual(self.a00, returnedDataset.ptcFitPars[ampName][0]) 

215 # noise already in electrons for 'ASTIERAPPROXIMATION' fit 

216 self.assertAlmostEqual(np.sqrt(self.noiseSq), returnedDataset.noise[ampName]) 

217 

218 # Nonlinearity fit parameters 

219 self.assertAlmostEqual(0.0, returnedDataset.nonLinearity[ampName][0]) 

220 self.assertAlmostEqual(self.flux, returnedDataset.nonLinearity[ampName][1]) 

221 self.assertAlmostEqual(self.k2NonLinearity, returnedDataset.nonLinearity[ampName][2]) 

222 

223 # Non-linearity coefficient for linearizer 

224 self.assertAlmostEqual(-self.k2NonLinearity/(self.flux**2), 

225 returnedDataset.coefficientLinearizeSquared[ampName]) 

226 

227 # Linearity residuals 

228 self.assertEqual(len(returnedDataset.nonLinearityResiduals[ampName]), 

229 len(inputNonLinearityResiduals)) 

230 for calc, truth in zip(returnedDataset.nonLinearityResiduals[ampName], 

231 inputNonLinearityResiduals): 

232 self.assertAlmostEqual(calc, truth) 

233 

234 # Fractional nonlinearity residuals 

235 self.assertEqual(len(returnedDataset.fractionalNonLinearityResiduals[ampName]), 

236 len(inputFracNonLinearityResiduals)) 

237 for calc, truth in zip(returnedDataset.fractionalNonLinearityResiduals[ampName], 

238 inputFracNonLinearityResiduals): 

239 self.assertAlmostEqual(calc, truth) 

240 

241 # check calls to calculateLinearityResidualAndLinearizers 

242 datasetLinResAndLinearizers = task.calculateLinearityResidualAndLinearizers( 

243 returnedDataset.rawExpTimes[ampName], returnedDataset.rawMeans[ampName]) 

244 

245 self.assertAlmostEqual(-self.k2NonLinearity/(self.flux**2), 

246 datasetLinResAndLinearizers.quadraticPolynomialLinearizerCoefficient) 

247 self.assertAlmostEqual(0.0, datasetLinResAndLinearizers.meanSignalVsTimePolyFitPars[0]) 

248 self.assertAlmostEqual(self.flux, datasetLinResAndLinearizers.meanSignalVsTimePolyFitPars[1]) 

249 self.assertAlmostEqual(self.k2NonLinearity, 

250 datasetLinResAndLinearizers.meanSignalVsTimePolyFitPars[2]) 

251 

252 def test_ptcFit(self): 

253 for createArray in [True, False]: 

254 for typeAndOrder in [('POLYNOMIAL', 2), ('POLYNOMIAL', 3), ('ASTIERAPPROXIMATION', None)]: 

255 self.ptcFitAndCheckPtc(fitType=typeAndOrder[0], order=typeAndOrder[1], 

256 doTableArray=createArray) 

257 

258 def test_meanVarMeasurement(self): 

259 task = self.defaultTask 

260 mu, varDiff = task.measureMeanVarPair(self.flatExp1, self.flatExp2) 

261 

262 self.assertLess(self.flatWidth - np.sqrt(varDiff), 1) 

263 self.assertLess(self.flatMean - mu, 1) 

264 

265 def test_getInitialGoodPoints(self): 

266 xs = [1, 2, 3, 4, 5, 6] 

267 ys = [2*x for x in xs] 

268 points = self.defaultTask._getInitialGoodPoints(xs, ys, 0.1, 0.25) 

269 assert np.all(points) == np.all(np.array([True for x in xs])) 

270 

271 ys[-1] = 30 

272 points = self.defaultTask._getInitialGoodPoints(xs, ys, 0.1, 0.25) 

273 assert np.all(points) == np.all(np.array([True, True, True, True, False])) 

274 

275 ys = [2*x for x in xs] 

276 newYs = copy.copy(ys) 

277 results = [False, True, True, False, False] 

278 for i, factor in enumerate([-0.5, -0.1, 0, 0.1, 0.5]): 

279 newYs[-1] = ys[-1] + (factor*ys[-1]) 

280 points = self.defaultTask._getInitialGoodPoints(xs, newYs, 0.05, 0.25) 

281 assert (np.all(points[0:-1]) == True) # noqa: E712 - flake8 is wrong here because of numpy.bool 

282 assert points[-1] == results[i] 

283 

284 def test_getVisitsUsed(self): 

285 localDataset = copy.copy(self.dataset) 

286 

287 for pair in [(12, 34), (56, 78), (90, 10)]: 

288 localDataset.inputVisitPairs["C:0,0"].append(pair) 

289 localDataset.visitMask["C:0,0"] = np.array([True, False, True]) 

290 self.assertTrue(np.all(localDataset.getVisitsUsed("C:0,0") == [(12, 34), (90, 10)])) 

291 

292 localDataset.visitMask["C:0,0"] = np.array([True, False, True, True]) # wrong length now 

293 with self.assertRaises(AssertionError): 

294 localDataset.getVisitsUsed("C:0,0") 

295 

296 def test_getGoodAmps(self): 

297 dataset = self.dataset 

298 

299 self.assertTrue(dataset.ampNames == self.ampNames) 

300 dataset.badAmps.append("C:0,1") 

301 self.assertTrue(dataset.getGoodAmps() == [amp for amp in self.ampNames if amp != "C:0,1"]) 

302 

303 

304class MeasurePhotonTransferCurveDatasetTestCase(lsst.utils.tests.TestCase): 

305 def setUp(self): 

306 self.ptcData = PhotonTransferCurveDataset(['C00', 'C01']) 

307 self.ptcData.inputVisitPairs = {'C00': [(123, 234), (345, 456), (567, 678)], 

308 'C01': [(123, 234), (345, 456), (567, 678)]} 

309 

310 def test_generalBehaviour(self): 

311 test = PhotonTransferCurveDataset(['C00', 'C01']) 

312 test.inputVisitPairs = {'C00': [(123, 234), (345, 456), (567, 678)], 

313 'C01': [(123, 234), (345, 456), (567, 678)]} 

314 

315 with self.assertRaises(AttributeError): 

316 test.newItem = 1 

317 

318 

319class TestMemory(lsst.utils.tests.MemoryTestCase): 

320 pass 

321 

322 

323def setup_module(module): 

324 lsst.utils.tests.init() 

325 

326 

327if __name__ == "__main__": 327 ↛ 328line 327 didn't jump to line 328, because the condition on line 327 was never true

328 lsst.utils.tests.init() 

329 unittest.main()