Coverage for tests/test_blockUtils.py: 34%
126 statements
« prev ^ index » next coverage.py v7.4.1, created at 2024-02-01 15:39 +0000
« prev ^ index » next coverage.py v7.4.1, created at 2024-02-01 15:39 +0000
1# This file is part of summit_utils.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22"""Test cases for utils."""
24import os
25import unittest
26import pandas as pd
27import numpy as np
28import asyncio
29import json
31import lsst.utils.tests
32from lsst.utils import getPackageDir
33from lsst.summit.utils.efdUtils import makeEfdClient
34from lsst.summit.utils.blockUtils import (
35 BlockParser,
36)
38from utils import getVcr
40__all__ = (
41 "writeNewBlockInfoTestTruthValues",
42)
44HAS_EFD_CLIENT = True
45try:
46 import lsst_efd_client # noqa: F401 just need to check this is available
47except ImportError:
48 HAS_EFD_CLIENT = False
50vcr = getVcr()
52DELIMITER = "||" # don't use a comma, as str(list) will naturally contain commas
55def getBlockInfoTestTruthValues(dataFilename=None):
56 """Get the current truth values for the block information.
58 Parameters
59 ----------
60 dataFilename : `str`, optional
61 The filename to read the truth values from. If not provided, the
62 default is to read from the file in the tests/data directory.
64 Returns
65 -------
66 data : `dict` [`tuple` [`int`, `int`], `str`]
67 The block info truth data.
68 """
69 if dataFilename is None:
70 packageDir = getPackageDir("summit_utils")
71 dataFilename = os.path.join(packageDir, "tests", "data", "blockInfoData.json")
73 with open(dataFilename, 'r') as f:
74 loaded = json.loads(f.read())
76 data = {}
77 for dayObsSeqNumStr, line in loaded.items():
78 dayObs = int(dayObsSeqNumStr.split(f'{DELIMITER}')[0])
79 seqNum = int(dayObsSeqNumStr.split(f'{DELIMITER}')[1])
80 data[dayObs, seqNum] = line
81 return data
84def writeNewBlockInfoTestTruthValues():
85 """This function is used to write out the truth values for the test cases.
87 If bugs are found in the parsing, it's possible these values could change,
88 and would need to be updated. If that happens, run this function, and check
89 the new values into git.
90 """
91 dayObs = 20230615
92 blockParser = BlockParser(dayObs)
94 data = {}
95 for block in blockParser.getBlockNums():
96 seqNums = blockParser.getSeqNums(block)
97 for seqNum in seqNums:
98 blockInfo = blockParser.getBlockInfo(block, seqNum)
99 line = (
100 f"{blockInfo.blockId}{DELIMITER}"
101 f"{blockInfo.begin}{DELIMITER}"
102 f"{blockInfo.end}{DELIMITER}"
103 f"{blockInfo.salIndices}{DELIMITER}"
104 f"{blockInfo.tickets}{DELIMITER}"
105 f"{len(blockInfo.states)}"
106 )
107 # must store as string not tuple for json serialization
108 data[f'{block}{DELIMITER}{seqNum}'] = line
110 packageDir = getPackageDir("summit_utils")
111 dataFilename = os.path.join(packageDir, "tests", "data", "blockInfoData.json")
112 with open(dataFilename, 'w') as f:
113 json.dump(data, f)
116@unittest.skipIf(not HAS_EFD_CLIENT, "No EFD client available")
117@vcr.use_cassette()
118class BlockParserTestCase(lsst.utils.tests.TestCase):
119 @classmethod
120 @vcr.use_cassette()
121 def setUpClass(cls):
122 try:
123 cls.client = makeEfdClient(testing=True)
124 except RuntimeError:
125 raise unittest.SkipTest("Could not instantiate an EFD client")
127 cls.dayObs = 20230615
128 cls.dayObsNoBlocks = 20230531 # contains data but no blocks
129 cls.blockParser = BlockParser(dayObs=cls.dayObs, client=cls.client)
130 cls.blockNums = cls.blockParser.getBlockNums()
131 cls.blockDict = {}
132 for block in cls.blockNums:
133 cls.blockDict[block] = cls.blockParser.getSeqNums(block)
135 @vcr.use_cassette()
136 def tearDown(self):
137 loop = asyncio.get_event_loop()
138 loop.run_until_complete(self.client.influx_client.close())
140 @vcr.use_cassette()
141 def test_parsing(self):
142 blockNums = self.blockParser.getBlockNums()
143 self.assertTrue(all(np.issubdtype(n, int)) for n in blockNums)
144 self.assertEqual(blockNums, list(self.blockDict.keys()))
146 for block, seqNums in self.blockDict.items():
147 self.assertTrue(np.issubdtype(block, int))
148 self.assertIsInstance(seqNums, list)
149 self.assertTrue(all(np.issubdtype(s, int)) for s in seqNums)
151 found = self.blockParser.getSeqNums(block)
152 self.assertTrue(all(np.issubdtype(s, int) for s in found))
153 self.assertEqual(found, seqNums)
154 self.blockParser.printBlockEvolution(block)
156 for seqNum in seqNums:
157 data = self.blockParser.getRows(block, seqNum)
158 self.assertIsInstance(data, pd.DataFrame)
159 self.assertGreater(len(data), 0)
160 self.blockParser.getBlockInfo(block=block, seqNum=seqNum)
161 self.blockParser.printBlockEvolution(block, seqNum=seqNum)
163 @vcr.use_cassette()
164 def test_notFoundBehavior(self):
165 # no block data on this day so check init doesn't raise
166 blockParser = BlockParser(dayObs=self.dayObsNoBlocks, client=self.client)
167 self.assertIsInstance(blockParser, BlockParser)
169 # check the queries which return nothing give nothing back gracefully
170 blocks = blockParser.getBlockNums()
171 self.assertIsInstance(blocks, list)
172 self.assertEqual(len(blocks), 0)
174 seqNums = blockParser.getSeqNums(block=123)
175 self.assertIsInstance(seqNums, list)
176 self.assertEqual(len(seqNums), 0)
178 # just check this doesn't raise
179 blockParser.getBlockInfo(block=1, seqNum=1)
181 # now switch back to one with data, and make sure the same is true
182 # when there is data present
183 blockParser = self.blockParser
184 seqNums = blockParser.getSeqNums(block=9999999)
185 self.assertIsInstance(seqNums, list)
186 self.assertEqual(len(seqNums), 0)
188 # just check this doesn't raise
189 blockParser.getBlockInfo(block=9999999, seqNum=9999999)
191 @vcr.use_cassette()
192 def test_actualValues(self):
193 data = getBlockInfoTestTruthValues()
195 dayObs = 20230615
196 blockParser = BlockParser(dayObs, client=self.client)
198 for block in blockParser.getBlockNums():
199 seqNums = blockParser.getSeqNums(block)
200 for seqNum in seqNums:
201 blockInfo = blockParser.getBlockInfo(block, seqNum)
202 line = data[blockInfo.blockNumber, blockInfo.seqNum]
203 items = line.split(f'{DELIMITER}')
204 self.assertEqual(items[0], blockInfo.blockId)
205 self.assertEqual(items[1], str(blockInfo.begin.value))
206 self.assertEqual(items[2], str(blockInfo.end.value))
207 self.assertEqual(items[3], str(blockInfo.salIndices))
208 self.assertEqual(items[4], str(blockInfo.tickets))
209 self.assertEqual(items[5], str(len(blockInfo.states)))
212class TestMemory(lsst.utils.tests.MemoryTestCase):
213 pass
216def setup_module(module):
217 lsst.utils.tests.init()
220if __name__ == "__main__": 220 ↛ 221line 220 didn't jump to line 221, because the condition on line 220 was never true
221 lsst.utils.tests.init()
222 unittest.main()