Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1#!/usr/bin/env python 

2 

3# 

4# LSST Data Management System 

5# 

6# Copyright 2008-2017 AURA/LSST. 

7# 

8# This product includes software developed by the 

9# LSST Project (http://www.lsst.org/). 

10# 

11# This program is free software: you can redistribute it and/or modify 

12# it under the terms of the GNU General Public License as published by 

13# the Free Software Foundation, either version 3 of the License, or 

14# (at your option) any later version. 

15# 

16# This program is distributed in the hope that it will be useful, 

17# but WITHOUT ANY WARRANTY; without even the implied warranty of 

18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

19# GNU General Public License for more details. 

20# 

21# You should have received a copy of the LSST License Statement and 

22# the GNU General Public License along with this program. If not, 

23# see <https://www.lsstcorp.org/LegalNotices/>. 

24# 

25"""Test cases for cp_pipe.""" 

26 

27from __future__ import absolute_import, division, print_function 

28import unittest 

29import numpy as np 

30import copy 

31 

32import lsst.utils 

33import lsst.utils.tests 

34 

35import lsst.cp.pipe as cpPipe 

36import lsst.ip.isr.isrMock as isrMock 

37from lsst.cp.pipe.ptc import PhotonTransferCurveDataset 

38from lsst.cp.pipe.astierCovPtcUtils import fitData 

39from lsst.cp.pipe.utils import funcPolynomial 

40 

41 

42class MeasurePhotonTransferCurveTaskTestCase(lsst.utils.tests.TestCase): 

43 """A test case for the PTC task.""" 

44 

45 def setUp(self): 

46 self.defaultConfig = cpPipe.ptc.MeasurePhotonTransferCurveTask.ConfigClass() 

47 self.defaultTask = cpPipe.ptc.MeasurePhotonTransferCurveTask(config=self.defaultConfig) 

48 

49 self.flatMean = 2000 

50 self.readNoiseAdu = 10 

51 mockImageConfig = isrMock.IsrMock.ConfigClass() 

52 

53 # flatDrop is not really relevant as we replace the data 

54 # but good to note it in case we change how this image is made 

55 mockImageConfig.flatDrop = 0.99999 

56 mockImageConfig.isTrimmed = True 

57 

58 self.flatExp1 = isrMock.FlatMock(config=mockImageConfig).run() 

59 self.flatExp2 = self.flatExp1.clone() 

60 (shapeY, shapeX) = self.flatExp1.getDimensions() 

61 

62 self.flatWidth = np.sqrt(self.flatMean) + self.readNoiseAdu 

63 

64 self.rng1 = np.random.RandomState(1984) 

65 flatData1 = self.rng1.normal(self.flatMean, self.flatWidth, (shapeX, shapeY)) 

66 self.rng2 = np.random.RandomState(666) 

67 flatData2 = self.rng2.normal(self.flatMean, self.flatWidth, (shapeX, shapeY)) 

68 

69 self.flatExp1.image.array[:] = flatData1 

70 self.flatExp2.image.array[:] = flatData2 

71 

72 # create fake PTC data to see if fit works, for one amp ('amp') 

73 self.flux = 1000. # ADU/sec 

74 timeVec = np.arange(1., 201.) 

75 self.k2NonLinearity = -5e-6 

76 muVec = self.flux*timeVec + self.k2NonLinearity*timeVec**2 # quadratic signal-chain non-linearity 

77 self.gain = 1.5 # e-/ADU 

78 self.c1 = 1./self.gain 

79 self.noiseSq = 5*self.gain # 7.5 (e-)^2 

80 self.a00 = -1.2e-6 

81 self.c2 = -1.5e-6 

82 self.c3 = -4.7e-12 # tuned so that it turns over for 200k mean 

83 

84 self.ampNames = [amp.getName() for amp in self.flatExp1.getDetector().getAmplifiers()] 

85 self.dataset = PhotonTransferCurveDataset(self.ampNames, " ") # pack raw data for fitting 

86 

87 for ampName in self.ampNames: # just the expTimes and means here - vars vary per function 

88 self.dataset.rawExpTimes[ampName] = timeVec 

89 self.dataset.rawMeans[ampName] = muVec 

90 

91 def makeMockFlats(self, expTime, gain=1.0, readNoiseElectrons=5, fluxElectrons=1000): 

92 flatFlux = fluxElectrons # e/s 

93 flatMean = flatFlux*expTime # e 

94 readNoise = readNoiseElectrons # e 

95 

96 mockImageConfig = isrMock.IsrMock.ConfigClass() 

97 

98 mockImageConfig.flatDrop = 0.99999 

99 mockImageConfig.isTrimmed = True 

100 

101 flatExp1 = isrMock.FlatMock(config=mockImageConfig).run() 

102 flatExp2 = flatExp1.clone() 

103 (shapeY, shapeX) = flatExp1.getDimensions() 

104 flatWidth = np.sqrt(flatMean) 

105 

106 rng1 = np.random.RandomState(1984) 

107 flatData1 = (rng1.normal(flatMean, flatWidth, (shapeX, shapeY)) + 

108 rng1.normal(0.0, readNoise, (shapeX, shapeY))) 

109 rng2 = np.random.RandomState(666) 

110 flatData2 = (rng2.normal(flatMean, flatWidth, (shapeX, shapeY)) + 

111 rng2.normal(0.0, readNoise, (shapeX, shapeY))) 

112 

113 flatExp1.image.array[:] = flatData1/gain # ADU 

114 flatExp2.image.array[:] = flatData2/gain # ADU 

115 

116 return flatExp1, flatExp2 

117 

118 def test_covAstier(self): 

119 """Test to check getCovariancesAstier 

120 

121 We check that the gain is the same as the imput gain from the mock data, that 

122 the covariances via FFT (as it is in MeasurePhotonTransferCurveTask when 

123 doCovariancesAstier=True) are the same as calculated in real space, and that 

124 Cov[0, 0] (i.e., the variances) are similar to the variances calculated with the standard 

125 method (when doCovariancesAstier=false), 

126 """ 

127 localDataset = copy.copy(self.dataset) 

128 config = copy.copy(self.defaultConfig) 

129 task = cpPipe.ptc.MeasurePhotonTransferCurveTask(config=config) 

130 

131 expTimes = np.arange(5, 170, 5) 

132 tupleRecords = [] 

133 allTags = [] 

134 muStandard, varStandard = {}, {} 

135 for expTime in expTimes: 

136 mockExp1, mockExp2 = self.makeMockFlats(expTime, gain=0.75) 

137 tupleRows = [] 

138 

139 for ampNumber, amp in enumerate(self.ampNames): 

140 # cov has (i, j, var, cov, npix) 

141 muDiff, varDiff, covAstier = task.measureMeanVarCov(mockExp1, mockExp2) 

142 muStandard.setdefault(amp, []).append(muDiff) 

143 varStandard.setdefault(amp, []).append(varDiff) 

144 

145 # Calculate covariances in an independent way: direct space 

146 _, _, covsDirect = task.measureMeanVarCov(mockExp1, mockExp2, covAstierRealSpace=True) 

147 

148 # Test that the arrays "covs" (FFT) and "covDirect" (direct space) are the same 

149 for row1, row2 in zip(covAstier, covsDirect): 

150 for a, b in zip(row1, row2): 

151 self.assertAlmostEqual(a, b) 

152 tupleRows += [(muDiff, ) + covRow + (ampNumber, expTime, amp) for covRow in covAstier] 

153 tags = ['mu', 'i', 'j', 'var', 'cov', 'npix', 'ext', 'expTime', 'ampName'] 

154 allTags += tags 

155 tupleRecords += tupleRows 

156 covariancesWithTags = np.core.records.fromrecords(tupleRecords, names=allTags) 

157 covFits, _ = fitData(covariancesWithTags) 

158 localDataset = task.getOutputPtcDataCovAstier(localDataset, covFits) 

159 

160 # Chek the gain and that the ratio of the variance caclulated via cov Astier (FFT) and 

161 # that calculated with the standard PTC is close to 1. 

162 for amp in self.ampNames: 

163 self.assertAlmostEqual(localDataset.gain[amp], 0.75, places=2) 

164 for v1, v2 in zip(varStandard[amp], localDataset.finalVars[amp][0]): 

165 self.assertAlmostEqual(v1/v2, 1.0, places=4) 

166 

167 def ptcFitAndCheckPtc(self, order=None, fitType='', doTableArray=False): 

168 localDataset = copy.copy(self.dataset) 

169 config = copy.copy(self.defaultConfig) 

170 if fitType == 'POLYNOMIAL': 

171 if order not in [2, 3]: 

172 RuntimeError("Enter a valid polynomial order for this test: 2 or 3") 

173 if order == 2: 

174 for ampName in self.ampNames: 

175 localDataset.rawVars[ampName] = [self.noiseSq + self.c1*mu + self.c2*mu**2 for 

176 mu in localDataset.rawMeans[ampName]] 

177 config.polynomialFitDegree = 2 

178 if order == 3: 

179 for ampName in self.ampNames: 

180 localDataset.rawVars[ampName] = [self.noiseSq + self.c1*mu + self.c2*mu**2 + self.c3*mu**3 

181 for mu in localDataset.rawMeans[ampName]] 

182 config.polynomialFitDegree = 3 

183 elif fitType == 'EXPAPPROXIMATION': 

184 g = self.gain 

185 for ampName in self.ampNames: 

186 localDataset.rawVars[ampName] = [(0.5/(self.a00*g**2)*(np.exp(2*self.a00*mu*g)-1) + 

187 self.noiseSq/(g*g)) for mu in localDataset.rawMeans[ampName]] 

188 else: 

189 RuntimeError("Enter a fit function type: 'POLYNOMIAL' or 'EXPAPPROXIMATION'") 

190 

191 config.maxAduForLookupTableLinearizer = 200000 # Max ADU in input mock flats 

192 task = cpPipe.ptc.MeasurePhotonTransferCurveTask(config=config) 

193 

194 if doTableArray: 

195 # Non-linearity 

196 numberAmps = len(self.ampNames) 

197 numberAduValues = config.maxAduForLookupTableLinearizer 

198 lookupTableArray = np.zeros((numberAmps, numberAduValues), dtype=np.float32) 

199 # localDataset: PTC dataset (lsst.cp.pipe.ptc.PhotonTransferCurveDataset) 

200 localDataset = task.fitPtc(localDataset, ptcFitType=fitType) 

201 # linDataset: Dictionary of `lsst.cp.pipe.ptc.LinearityResidualsAndLinearizersDataset` 

202 linDataset = task.fitNonLinearity(localDataset, tableArray=lookupTableArray) 

203 else: 

204 localDataset = task.fitPtc(localDataset, ptcFitType=fitType) 

205 linDataset = task.fitNonLinearity(localDataset) 

206 

207 if doTableArray: 

208 # check that the linearizer table has been filled out properly 

209 for i in np.arange(numberAmps): 

210 tMax = (config.maxAduForLookupTableLinearizer)/self.flux 

211 timeRange = np.linspace(0., tMax, config.maxAduForLookupTableLinearizer) 

212 signalIdeal = timeRange*self.flux 

213 signalUncorrected = funcPolynomial(np.array([0.0, self.flux, self.k2NonLinearity]), 

214 timeRange) 

215 linearizerTableRow = signalIdeal - signalUncorrected 

216 self.assertEqual(len(linearizerTableRow), len(lookupTableArray[i, :])) 

217 for j in np.arange(len(linearizerTableRow)): 

218 self.assertAlmostEqual(linearizerTableRow[j], lookupTableArray[i, :][j], places=6) 

219 

220 # check entries in localDataset, which was modified by the function 

221 for ampName in self.ampNames: 

222 maskAmp = localDataset.visitMask[ampName] 

223 finalMuVec = localDataset.rawMeans[ampName][maskAmp] 

224 finalTimeVec = localDataset.rawExpTimes[ampName][maskAmp] 

225 linearPart = self.flux*finalTimeVec 

226 inputFracNonLinearityResiduals = 100*(linearPart - finalMuVec)/linearPart 

227 self.assertEqual(fitType, localDataset.ptcFitType) 

228 self.assertAlmostEqual(self.gain, localDataset.gain[ampName]) 

229 if fitType == 'POLYNOMIAL': 

230 self.assertAlmostEqual(self.c1, localDataset.ptcFitPars[ampName][1]) 

231 self.assertAlmostEqual(np.sqrt(self.noiseSq)*self.gain, localDataset.noise[ampName]) 

232 if fitType == 'EXPAPPROXIMATION': 

233 self.assertAlmostEqual(self.a00, localDataset.ptcFitPars[ampName][0]) 

234 # noise already in electrons for 'EXPAPPROXIMATION' fit 

235 self.assertAlmostEqual(np.sqrt(self.noiseSq), localDataset.noise[ampName]) 

236 

237 # check entries in returned dataset (a dict of , for nonlinearity) 

238 for ampName in self.ampNames: 

239 maskAmp = localDataset.visitMask[ampName] 

240 finalMuVec = localDataset.rawMeans[ampName][maskAmp] 

241 finalTimeVec = localDataset.rawExpTimes[ampName][maskAmp] 

242 linearPart = self.flux*finalTimeVec 

243 inputFracNonLinearityResiduals = 100*(linearPart - finalMuVec)/linearPart 

244 

245 # Nonlinearity fit parameters 

246 self.assertAlmostEqual(0.0, linDataset[ampName].meanSignalVsTimePolyFitPars[0]) 

247 self.assertAlmostEqual(self.flux, linDataset[ampName].meanSignalVsTimePolyFitPars[1]) 

248 self.assertAlmostEqual(self.k2NonLinearity, linDataset[ampName].meanSignalVsTimePolyFitPars[2]) 

249 

250 # Non-linearity coefficient for linearizer 

251 self.assertAlmostEqual(-self.k2NonLinearity/(self.flux**2), 

252 linDataset[ampName].quadraticPolynomialLinearizerCoefficient) 

253 

254 linearPartModel = linDataset[ampName].meanSignalVsTimePolyFitPars[1]*finalTimeVec 

255 outputFracNonLinearityResiduals = 100*(linearPartModel - finalMuVec)/linearPartModel 

256 # Fractional nonlinearity residuals 

257 self.assertEqual(len(outputFracNonLinearityResiduals), len(inputFracNonLinearityResiduals)) 

258 for calc, truth in zip(outputFracNonLinearityResiduals, inputFracNonLinearityResiduals): 

259 self.assertAlmostEqual(calc, truth) 

260 

261 # check calls to calculateLinearityResidualAndLinearizers 

262 datasetLinResAndLinearizers = task.calculateLinearityResidualAndLinearizers( 

263 localDataset.rawExpTimes[ampName], localDataset.rawMeans[ampName]) 

264 

265 self.assertAlmostEqual(-self.k2NonLinearity/(self.flux**2), 

266 datasetLinResAndLinearizers.quadraticPolynomialLinearizerCoefficient) 

267 self.assertAlmostEqual(0.0, datasetLinResAndLinearizers.meanSignalVsTimePolyFitPars[0]) 

268 self.assertAlmostEqual(self.flux, datasetLinResAndLinearizers.meanSignalVsTimePolyFitPars[1]) 

269 self.assertAlmostEqual(self.k2NonLinearity, 

270 datasetLinResAndLinearizers.meanSignalVsTimePolyFitPars[2]) 

271 

272 def test_ptcFit(self): 

273 for createArray in [True, False]: 

274 for (fitType, order) in [('POLYNOMIAL', 2), ('POLYNOMIAL', 3), ('EXPAPPROXIMATION', None)]: 

275 self.ptcFitAndCheckPtc(fitType=fitType, order=order, doTableArray=createArray) 

276 

277 def test_meanVarMeasurement(self): 

278 task = self.defaultTask 

279 mu, varDiff, _ = task.measureMeanVarCov(self.flatExp1, self.flatExp2) 

280 

281 self.assertLess(self.flatWidth - np.sqrt(varDiff), 1) 

282 self.assertLess(self.flatMean - mu, 1) 

283 

284 def test_getInitialGoodPoints(self): 

285 xs = [1, 2, 3, 4, 5, 6] 

286 ys = [2*x for x in xs] 

287 points = self.defaultTask._getInitialGoodPoints(xs, ys, 0.1, 0.25) 

288 assert np.all(points) == np.all(np.array([True for x in xs])) 

289 

290 ys[-1] = 30 

291 points = self.defaultTask._getInitialGoodPoints(xs, ys, 0.1, 0.25) 

292 assert np.all(points) == np.all(np.array([True, True, True, True, False])) 

293 

294 ys = [2*x for x in xs] 

295 newYs = copy.copy(ys) 

296 results = [False, True, True, False, False] 

297 for i, factor in enumerate([-0.5, -0.1, 0, 0.1, 0.5]): 

298 newYs[-1] = ys[-1] + (factor*ys[-1]) 

299 points = self.defaultTask._getInitialGoodPoints(xs, newYs, 0.05, 0.25) 

300 assert (np.all(points[0:-1]) == True) # noqa: E712 - flake8 is wrong here because of numpy.bool 

301 assert points[-1] == results[i] 

302 

303 def test_getVisitsUsed(self): 

304 localDataset = copy.copy(self.dataset) 

305 

306 for pair in [(12, 34), (56, 78), (90, 10)]: 

307 localDataset.inputVisitPairs["C:0,0"].append(pair) 

308 localDataset.visitMask["C:0,0"] = np.array([True, False, True]) 

309 self.assertTrue(np.all(localDataset.getVisitsUsed("C:0,0") == [(12, 34), (90, 10)])) 

310 

311 localDataset.visitMask["C:0,0"] = np.array([True, False, True, True]) # wrong length now 

312 with self.assertRaises(AssertionError): 

313 localDataset.getVisitsUsed("C:0,0") 

314 

315 def test_getGoodAmps(self): 

316 dataset = self.dataset 

317 

318 self.assertTrue(dataset.ampNames == self.ampNames) 

319 dataset.badAmps.append("C:0,1") 

320 self.assertTrue(dataset.getGoodAmps() == [amp for amp in self.ampNames if amp != "C:0,1"]) 

321 

322 

323class MeasurePhotonTransferCurveDatasetTestCase(lsst.utils.tests.TestCase): 

324 def setUp(self): 

325 self.ptcData = PhotonTransferCurveDataset(['C00', 'C01'], " ") 

326 self.ptcData.inputVisitPairs = {'C00': [(123, 234), (345, 456), (567, 678)], 

327 'C01': [(123, 234), (345, 456), (567, 678)]} 

328 

329 def test_generalBehaviour(self): 

330 test = PhotonTransferCurveDataset(['C00', 'C01'], " ") 

331 test.inputVisitPairs = {'C00': [(123, 234), (345, 456), (567, 678)], 

332 'C01': [(123, 234), (345, 456), (567, 678)]} 

333 

334 with self.assertRaises(AttributeError): 

335 test.newItem = 1 

336 

337 

338class TestMemory(lsst.utils.tests.MemoryTestCase): 

339 pass 

340 

341 

342def setup_module(module): 

343 lsst.utils.tests.init() 

344 

345 

346if __name__ == "__main__": 346 ↛ 347line 346 didn't jump to line 347, because the condition on line 346 was never true

347 lsst.utils.tests.init() 

348 unittest.main()