Coverage for tests/test_blockUtils.py: 34%

126 statements  

« prev     ^ index     » next       coverage.py v7.4.1, created at 2024-02-18 12:42 +0000

1# This file is part of summit_utils. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22"""Test cases for utils.""" 

23 

24import os 

25import unittest 

26import pandas as pd 

27import numpy as np 

28import asyncio 

29import json 

30 

31import lsst.utils.tests 

32from lsst.utils import getPackageDir 

33from lsst.summit.utils.efdUtils import makeEfdClient 

34from lsst.summit.utils.blockUtils import ( 

35 BlockParser, 

36) 

37 

38from utils import getVcr 

39 

40__all__ = ( 

41 "writeNewBlockInfoTestTruthValues", 

42) 

43 

44HAS_EFD_CLIENT = True 

45try: 

46 import lsst_efd_client # noqa: F401 just need to check this is available 

47except ImportError: 

48 HAS_EFD_CLIENT = False 

49 

50vcr = getVcr() 

51 

52DELIMITER = "||" # don't use a comma, as str(list) will naturally contain commas 

53 

54 

55def getBlockInfoTestTruthValues(dataFilename=None): 

56 """Get the current truth values for the block information. 

57 

58 Parameters 

59 ---------- 

60 dataFilename : `str`, optional 

61 The filename to read the truth values from. If not provided, the 

62 default is to read from the file in the tests/data directory. 

63 

64 Returns 

65 ------- 

66 data : `dict` [`tuple` [`int`, `int`], `str`] 

67 The block info truth data. 

68 """ 

69 if dataFilename is None: 

70 packageDir = getPackageDir("summit_utils") 

71 dataFilename = os.path.join(packageDir, "tests", "data", "blockInfoData.json") 

72 

73 with open(dataFilename, 'r') as f: 

74 loaded = json.loads(f.read()) 

75 

76 data = {} 

77 for dayObsSeqNumStr, line in loaded.items(): 

78 dayObs = int(dayObsSeqNumStr.split(f'{DELIMITER}')[0]) 

79 seqNum = int(dayObsSeqNumStr.split(f'{DELIMITER}')[1]) 

80 data[dayObs, seqNum] = line 

81 return data 

82 

83 

84def writeNewBlockInfoTestTruthValues(): 

85 """This function is used to write out the truth values for the test cases. 

86 

87 If bugs are found in the parsing, it's possible these values could change, 

88 and would need to be updated. If that happens, run this function, and check 

89 the new values into git. 

90 """ 

91 dayObs = 20230615 

92 blockParser = BlockParser(dayObs) 

93 

94 data = {} 

95 for block in blockParser.getBlockNums(): 

96 seqNums = blockParser.getSeqNums(block) 

97 for seqNum in seqNums: 

98 blockInfo = blockParser.getBlockInfo(block, seqNum) 

99 line = ( 

100 f"{blockInfo.blockId}{DELIMITER}" 

101 f"{blockInfo.begin}{DELIMITER}" 

102 f"{blockInfo.end}{DELIMITER}" 

103 f"{blockInfo.salIndices}{DELIMITER}" 

104 f"{blockInfo.tickets}{DELIMITER}" 

105 f"{len(blockInfo.states)}" 

106 ) 

107 # must store as string not tuple for json serialization 

108 data[f'{block}{DELIMITER}{seqNum}'] = line 

109 

110 packageDir = getPackageDir("summit_utils") 

111 dataFilename = os.path.join(packageDir, "tests", "data", "blockInfoData.json") 

112 with open(dataFilename, 'w') as f: 

113 json.dump(data, f) 

114 

115 

116@unittest.skipIf(not HAS_EFD_CLIENT, "No EFD client available") 

117@vcr.use_cassette() 

118class BlockParserTestCase(lsst.utils.tests.TestCase): 

119 @classmethod 

120 @vcr.use_cassette() 

121 def setUpClass(cls): 

122 try: 

123 cls.client = makeEfdClient(testing=True) 

124 except RuntimeError: 

125 raise unittest.SkipTest("Could not instantiate an EFD client") 

126 

127 cls.dayObs = 20230615 

128 cls.dayObsNoBlocks = 20230531 # contains data but no blocks 

129 cls.blockParser = BlockParser(dayObs=cls.dayObs, client=cls.client) 

130 cls.blockNums = cls.blockParser.getBlockNums() 

131 cls.blockDict = {} 

132 for block in cls.blockNums: 

133 cls.blockDict[block] = cls.blockParser.getSeqNums(block) 

134 

135 @vcr.use_cassette() 

136 def tearDown(self): 

137 loop = asyncio.get_event_loop() 

138 loop.run_until_complete(self.client.influx_client.close()) 

139 

140 @vcr.use_cassette() 

141 def test_parsing(self): 

142 blockNums = self.blockParser.getBlockNums() 

143 self.assertTrue(all(np.issubdtype(n, int)) for n in blockNums) 

144 self.assertEqual(blockNums, list(self.blockDict.keys())) 

145 

146 for block, seqNums in self.blockDict.items(): 

147 self.assertTrue(np.issubdtype(block, int)) 

148 self.assertIsInstance(seqNums, list) 

149 self.assertTrue(all(np.issubdtype(s, int)) for s in seqNums) 

150 

151 found = self.blockParser.getSeqNums(block) 

152 self.assertTrue(all(np.issubdtype(s, int) for s in found)) 

153 self.assertEqual(found, seqNums) 

154 self.blockParser.printBlockEvolution(block) 

155 

156 for seqNum in seqNums: 

157 data = self.blockParser.getRows(block, seqNum) 

158 self.assertIsInstance(data, pd.DataFrame) 

159 self.assertGreater(len(data), 0) 

160 self.blockParser.getBlockInfo(block=block, seqNum=seqNum) 

161 self.blockParser.printBlockEvolution(block, seqNum=seqNum) 

162 

163 @vcr.use_cassette() 

164 def test_notFoundBehavior(self): 

165 # no block data on this day so check init doesn't raise 

166 blockParser = BlockParser(dayObs=self.dayObsNoBlocks, client=self.client) 

167 self.assertIsInstance(blockParser, BlockParser) 

168 

169 # check the queries which return nothing give nothing back gracefully 

170 blocks = blockParser.getBlockNums() 

171 self.assertIsInstance(blocks, list) 

172 self.assertEqual(len(blocks), 0) 

173 

174 seqNums = blockParser.getSeqNums(block=123) 

175 self.assertIsInstance(seqNums, list) 

176 self.assertEqual(len(seqNums), 0) 

177 

178 # just check this doesn't raise 

179 blockParser.getBlockInfo(block=1, seqNum=1) 

180 

181 # now switch back to one with data, and make sure the same is true 

182 # when there is data present 

183 blockParser = self.blockParser 

184 seqNums = blockParser.getSeqNums(block=9999999) 

185 self.assertIsInstance(seqNums, list) 

186 self.assertEqual(len(seqNums), 0) 

187 

188 # just check this doesn't raise 

189 blockParser.getBlockInfo(block=9999999, seqNum=9999999) 

190 

191 @vcr.use_cassette() 

192 def test_actualValues(self): 

193 data = getBlockInfoTestTruthValues() 

194 

195 dayObs = 20230615 

196 blockParser = BlockParser(dayObs, client=self.client) 

197 

198 for block in blockParser.getBlockNums(): 

199 seqNums = blockParser.getSeqNums(block) 

200 for seqNum in seqNums: 

201 blockInfo = blockParser.getBlockInfo(block, seqNum) 

202 line = data[blockInfo.blockNumber, blockInfo.seqNum] 

203 items = line.split(f'{DELIMITER}') 

204 self.assertEqual(items[0], blockInfo.blockId) 

205 self.assertEqual(items[1], str(blockInfo.begin.value)) 

206 self.assertEqual(items[2], str(blockInfo.end.value)) 

207 self.assertEqual(items[3], str(blockInfo.salIndices)) 

208 self.assertEqual(items[4], str(blockInfo.tickets)) 

209 self.assertEqual(items[5], str(len(blockInfo.states))) 

210 

211 

212class TestMemory(lsst.utils.tests.MemoryTestCase): 

213 pass 

214 

215 

216def setup_module(module): 

217 lsst.utils.tests.init() 

218 

219 

220if __name__ == "__main__": 220 ↛ 221line 220 didn't jump to line 221, because the condition on line 220 was never true

221 lsst.utils.tests.init() 

222 unittest.main()