Coverage for tests/test_extract_metricvalues.py: 15%
98 statements
« prev ^ index » next coverage.py v6.4.2, created at 2022-07-27 09:57 +0000
« prev ^ index » next coverage.py v6.4.2, created at 2022-07-27 09:57 +0000
1# This file is part of verify.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22import contextlib
23import io
24import os
25import unittest
27import lsst.daf.butler
28from lsst.verify import extract_metricvalues
31class ExtractMetricvaluesTest(unittest.TestCase):
32 """Tests of methods in extract_metricvalues.py that are called by the
33 commandline interface.
35 These tests use the two repos created by
36 ``data/make_metricvalue_butlers.py``
37 """
38 def setUp(self):
39 path = os.path.join(os.path.dirname(__file__), "data")
40 self.repo1 = os.path.join(path, "metricvalue_repo1")
41 self.repo2 = os.path.join(path, "metricvalue_repo2")
43 def test_load_from_butler(self):
44 butler = lsst.daf.butler.Butler(self.repo1, collections="testrun")
46 result = extract_metricvalues.load_from_butler(butler, "metricvalue_verify_other*")
47 self.assertEqual(len(result), 2)
48 for value in result.values():
49 self.assertEqual(value.metric_name.metric, "other")
51 # Check that reject_suffix correctly restricts the list.
52 result = extract_metricvalues.load_from_butler(butler, "metricvalue_verify_testing*",
53 reject_suffix="Time")
54 self.assertEqual(len(result), 5)
55 for value in result.values():
56 self.assertNotIn("Time", value.metric_name.metric)
58 # Check that reject suffix behaves correctly with tuples.
59 result = extract_metricvalues.load_from_butler(butler, "metricvalue_verify_testing*",
60 reject_suffix=("Time", "ing"))
61 self.assertEqual(len(result.keys()), 2)
62 for value in result.values():
63 self.assertNotIn("Time", value.metric_name.metric)
64 self.assertNotIn("Objects", value.metric_name.metric)
65 self.assertIn("Memory", value.metric_name.metric)
67 # Check that we can reject everything.
68 result = extract_metricvalues.load_from_butler(butler, "metricvalue_verify_testing*",
69 reject_suffix=("Time", "ing", "Memory"))
70 self.assertEqual(len(result.keys()), 0)
72 def test_load_value(self):
73 butler = lsst.daf.butler.Butler(self.repo1, collections="testrun")
75 result = extract_metricvalues.load_value(butler)
76 self.assertEqual(len(result.keys()), 6)
77 for value in result.values():
78 self.assertNotIn("Time", value.metric_name.metric)
79 self.assertNotIn("Memory", value.metric_name.metric)
81 def test_load_timing(self):
82 butler = lsst.daf.butler.Butler(self.repo1, collections="testrun")
84 result = extract_metricvalues.load_timing(butler)
85 self.assertEqual(len(result.keys()), 4)
86 for value in result.values():
87 self.assertIn("Time", value.metric_name.metric)
89 def test_load_memory(self):
90 butler = lsst.daf.butler.Butler(self.repo1, collections="testrun")
92 result = extract_metricvalues.load_memory(butler)
93 self.assertEqual(len(result.keys()), 4)
94 for value in result.values():
95 self.assertIn("Memory", value.metric_name.metric)
97 def test_print_metrics(self):
98 """Test what is printed to stdout for various ``print_metrics`` args.
99 """
100 butler = lsst.daf.butler.Butler(self.repo1, collections="testrun")
102 def check_stdout(kind, n, last_line, contained, **kwargs):
103 """Test the the correct number of lines are printed, and that
104 the last line is as expected.
106 Parameters
107 ----------
108 kind : `str`
109 What kind of metrics to load; passed to ``print_metrics()``.
110 n : `int`
111 How many non-empty lines should have been printed?
112 last_line : `str`
113 Expected last non-empty line in the printed output.
114 contained : `str`
115 A full-line string expected to be contained in the output.
116 **kwargs
117 Other arguments to pass to ``print_metrics()``.
118 """
119 with contextlib.redirect_stdout(io.StringIO()) as stdout:
120 extract_metricvalues.print_metrics(butler, kind, **kwargs)
121 # Split up the lines, and remove empty ones.
122 result = list(filter(None, stdout.getvalue().split("\n")))
123 self.assertEqual(len(result), n)
124 self.assertEqual(last_line, result[-1])
125 self.assertIn(contained, result)
127 # default call with no kwargs
128 contained = "{instrument: 'TestCam', detector: 25, visit: 54321, ...}"
129 last = "verify.testing: 42.0"
130 check_stdout("value", 9, last, contained)
132 # restrict the number of items returned to only the last detector
133 check_stdout("value", 3, last, contained, data_id_restriction={"detector": 25})
135 # only print part of the dataIds
136 contained = "detector: 25, visit: 54321"
137 check_stdout("value", 9, last, contained, data_id_keys=("detector", "visit"))
139 # Get the timings instead
140 contained = "{instrument: 'TestCam', detector: 25, visit: 54321, ...}"
141 last = "verify.testingTime: 19.0 s"
142 check_stdout("timing", 6, last, contained)
144 # Get the timings and print a partial dataId
145 contained = "detector: 25, visit: 54321"
146 check_stdout("timing", 6, last, contained, data_id_keys=("detector", "visit"))
148 # Get the memory values instead
149 contained = "{instrument: 'TestCam', detector: 25, visit: 54321, ...}"
150 last = "verify.testingMemory: 190.73 Mibyte"
151 check_stdout("memory", 6, last, contained)
153 # Get the memory values and print a partial dataId
154 contained = "detector: 25, visit: 54321"
155 check_stdout("memory", 6, last, contained, data_id_keys=("detector", "visit"))
157 with self.assertRaisesRegex(RuntimeError, "Cannot handle kind=blah"):
158 extract_metricvalues.print_metrics(butler, "blah")
160 def test_print_diff_values(self):
161 butler1 = lsst.daf.butler.Butler(self.repo1, collections="testrun")
162 butler2 = lsst.daf.butler.Butler(self.repo2, collections="testrun")
164 def check_stdout(n, last_line, **kwargs):
165 """Test the the correct number of lines are printed, and that
166 the last line is as expected.
168 Parameters
169 ----------
170 n : `int`
171 How many non-empty lines should have been printed?
172 last_line : `str`
173 Expected last non-empty line in the printed output.
174 **kwargs
175 Other arguments to pass to ``print_diff_metrics()``.
176 """
177 with contextlib.redirect_stdout(io.StringIO()) as stdout:
178 extract_metricvalues.print_diff_metrics(butler1, butler2, **kwargs)
179 # Split up the lines, and remove empty ones.
180 result = list(filter(None, stdout.getvalue().split("\n")))
181 self.assertEqual(len(result), n)
182 self.assertEqual(last_line, result[-1])
183 return result
185 last_line = "Number of metrics that are the same in both runs: 0 / 6"
186 result = check_stdout(10, last_line)
187 expect = "{instrument: 'TestCam', detector: 12, visit: 12345, ...}"
188 self.assertIn(expect, result)
189 expect = "verify.another: 1.0 mas / 3.0 mas"
190 self.assertIn(expect, result)
192 result = check_stdout(10, last_line, data_id_keys=("detector", "visit"))
193 expect = "detector: 12, visit: 12345"
194 self.assertIn(expect, result)
195 expect = "verify.another: 1.0 mas / 3.0 mas"
196 self.assertIn(expect, result)
199if __name__ == "__main__": 199 ↛ 200line 199 didn't jump to line 200, because the condition on line 199 was never true
200 unittest.main()