Coverage for tests/test_blockUtils.py: 35%

123 statements  

« prev     ^ index     » next       coverage.py v7.4.4, created at 2024-04-10 05:49 -0700

1# This file is part of summit_utils. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22"""Test cases for utils.""" 

23 

24import asyncio 

25import json 

26import os 

27import unittest 

28 

29import numpy as np 

30import pandas as pd 

31from utils import getVcr 

32 

33import lsst.utils.tests 

34from lsst.summit.utils.blockUtils import BlockParser 

35from lsst.summit.utils.efdUtils import makeEfdClient 

36 

37__all__ = ("writeNewBlockInfoTestTruthValues",) 

38 

39HAS_EFD_CLIENT = True 

40try: 

41 import lsst_efd_client # noqa: F401 just need to check this is available 

42except ImportError: 

43 HAS_EFD_CLIENT = False 

44 

45vcr = getVcr() 

46 

47DELIMITER = "||" # don't use a comma, as str(list) will naturally contain commas 

48TESTDIR = os.path.abspath(os.path.dirname(__file__)) 

49 

50 

51def getBlockInfoTestTruthValues(dataFilename=None): 

52 """Get the current truth values for the block information. 

53 

54 Parameters 

55 ---------- 

56 dataFilename : `str`, optional 

57 The filename to read the truth values from. If not provided, the 

58 default is to read from the file in the tests/data directory. 

59 

60 Returns 

61 ------- 

62 data : `dict` [`tuple` [`int`, `int`], `str`] 

63 The block info truth data. 

64 """ 

65 if dataFilename is None: 

66 dataFilename = os.path.join(TESTDIR, "data", "blockInfoData.json") 

67 

68 with open(dataFilename, "r") as f: 

69 loaded = json.loads(f.read()) 

70 

71 data = {} 

72 for dayObsSeqNumStr, line in loaded.items(): 

73 dayObs = int(dayObsSeqNumStr.split(f"{DELIMITER}")[0]) 

74 seqNum = int(dayObsSeqNumStr.split(f"{DELIMITER}")[1]) 

75 data[dayObs, seqNum] = line 

76 return data 

77 

78 

79def writeNewBlockInfoTestTruthValues(): 

80 """This function is used to write out the truth values for the test cases. 

81 

82 If bugs are found in the parsing, it's possible these values could change, 

83 and would need to be updated. If that happens, run this function, and check 

84 the new values into git. 

85 """ 

86 dayObs = 20230615 

87 blockParser = BlockParser(dayObs) 

88 

89 data = {} 

90 for block in blockParser.getBlockNums(): 

91 seqNums = blockParser.getSeqNums(block) 

92 for seqNum in seqNums: 

93 blockInfo = blockParser.getBlockInfo(block, seqNum) 

94 line = ( 

95 f"{blockInfo.blockId}{DELIMITER}" 

96 f"{blockInfo.begin}{DELIMITER}" 

97 f"{blockInfo.end}{DELIMITER}" 

98 f"{blockInfo.salIndices}{DELIMITER}" 

99 f"{blockInfo.tickets}{DELIMITER}" 

100 f"{len(blockInfo.states)}" 

101 ) 

102 # must store as string not tuple for json serialization 

103 data[f"{block}{DELIMITER}{seqNum}"] = line 

104 

105 dataFilename = os.path.join(TESTDIR, "data", "blockInfoData.json") 

106 with open(dataFilename, "w") as f: 

107 json.dump(data, f) 

108 

109 

110@unittest.skipIf(not HAS_EFD_CLIENT, "No EFD client available") 

111@vcr.use_cassette() 

112class BlockParserTestCase(lsst.utils.tests.TestCase): 

113 @classmethod 

114 @vcr.use_cassette() 

115 def setUpClass(cls): 

116 try: 

117 cls.client = makeEfdClient(testing=True) 

118 except RuntimeError: 

119 raise unittest.SkipTest("Could not instantiate an EFD client") 

120 

121 cls.dayObs = 20230615 

122 cls.dayObsNoBlocks = 20230531 # contains data but no blocks 

123 cls.blockParser = BlockParser(dayObs=cls.dayObs, client=cls.client) 

124 cls.blockNums = cls.blockParser.getBlockNums() 

125 cls.blockDict = {} 

126 for block in cls.blockNums: 

127 cls.blockDict[block] = cls.blockParser.getSeqNums(block) 

128 

129 @vcr.use_cassette() 

130 def tearDown(self): 

131 loop = asyncio.get_event_loop() 

132 loop.run_until_complete(self.client.influx_client.close()) 

133 

134 @vcr.use_cassette() 

135 def test_parsing(self): 

136 blockNums = self.blockParser.getBlockNums() 

137 self.assertTrue(all(np.issubdtype(n, int)) for n in blockNums) 

138 self.assertEqual(blockNums, list(self.blockDict.keys())) 

139 

140 for block, seqNums in self.blockDict.items(): 

141 self.assertTrue(np.issubdtype(block, int)) 

142 self.assertIsInstance(seqNums, list) 

143 self.assertTrue(all(np.issubdtype(s, int)) for s in seqNums) 

144 

145 found = self.blockParser.getSeqNums(block) 

146 self.assertTrue(all(np.issubdtype(s, int) for s in found)) 

147 self.assertEqual(found, seqNums) 

148 self.blockParser.printBlockEvolution(block) 

149 

150 for seqNum in seqNums: 

151 data = self.blockParser.getRows(block, seqNum) 

152 self.assertIsInstance(data, pd.DataFrame) 

153 self.assertGreater(len(data), 0) 

154 self.blockParser.getBlockInfo(block=block, seqNum=seqNum) 

155 self.blockParser.printBlockEvolution(block, seqNum=seqNum) 

156 

157 @vcr.use_cassette() 

158 def test_notFoundBehavior(self): 

159 # no block data on this day so check init doesn't raise 

160 blockParser = BlockParser(dayObs=self.dayObsNoBlocks, client=self.client) 

161 self.assertIsInstance(blockParser, BlockParser) 

162 

163 # check the queries which return nothing give nothing back gracefully 

164 blocks = blockParser.getBlockNums() 

165 self.assertIsInstance(blocks, list) 

166 self.assertEqual(len(blocks), 0) 

167 

168 seqNums = blockParser.getSeqNums(block=123) 

169 self.assertIsInstance(seqNums, list) 

170 self.assertEqual(len(seqNums), 0) 

171 

172 # just check this doesn't raise 

173 blockParser.getBlockInfo(block=1, seqNum=1) 

174 

175 # now switch back to one with data, and make sure the same is true 

176 # when there is data present 

177 blockParser = self.blockParser 

178 seqNums = blockParser.getSeqNums(block=9999999) 

179 self.assertIsInstance(seqNums, list) 

180 self.assertEqual(len(seqNums), 0) 

181 

182 # just check this doesn't raise 

183 blockParser.getBlockInfo(block=9999999, seqNum=9999999) 

184 

185 @vcr.use_cassette() 

186 def test_actualValues(self): 

187 data = getBlockInfoTestTruthValues() 

188 

189 blockParser = BlockParser(self.dayObs, client=self.client) 

190 

191 for block in blockParser.getBlockNums(): 

192 seqNums = blockParser.getSeqNums(block) 

193 for seqNum in seqNums: 

194 blockInfo = blockParser.getBlockInfo(block, seqNum) 

195 line = data[blockInfo.blockNumber, blockInfo.seqNum] 

196 items = line.split(f"{DELIMITER}") 

197 self.assertEqual(items[0], blockInfo.blockId) 

198 self.assertEqual(items[1], str(blockInfo.begin.value)) 

199 self.assertEqual(items[2], str(blockInfo.end.value)) 

200 self.assertEqual(items[3], str(blockInfo.salIndices)) 

201 self.assertEqual(items[4], str(blockInfo.tickets)) 

202 self.assertEqual(items[5], str(len(blockInfo.states))) 

203 

204 

205class TestMemory(lsst.utils.tests.MemoryTestCase): 

206 pass 

207 

208 

209def setup_module(module): 

210 lsst.utils.tests.init() 

211 

212 

213if __name__ == "__main__": 213 ↛ 214line 213 didn't jump to line 214, because the condition on line 213 was never true

214 lsst.utils.tests.init() 

215 unittest.main()