Coverage for tests/test_metricset.py: 21%
116 statements
« prev ^ index » next coverage.py v6.5.0, created at 2023-02-10 02:09 -0800
« prev ^ index » next coverage.py v6.5.0, created at 2023-02-10 02:09 -0800
1# This file is part of verify.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22import os
23import unittest
25from lsst.verify import Metric, MetricSet, Name
28class MetricsPackageTestCase(unittest.TestCase):
29 """Test creating a MetricSet from a mock verification framework
30 metric package located at data/ relative test modules.
32 These tests are coupled to the data/metrics/*.yaml files.
33 """
35 def setUp(self):
36 self.metrics_yaml_dirname = os.path.join(
37 os.path.dirname(__file__), 'data')
39 self.metric_set = MetricSet.load_metrics_package(
40 self.metrics_yaml_dirname)
42 def test_len(self):
43 self.assertEqual(len(self.metric_set), 4)
45 def test_contains(self):
46 for key in ['testing.PA1',
47 'testing.PF1',
48 'testing.PA2',
49 'testing.AM1']:
50 self.assertIn(key, self.metric_set, msg=key)
52 def test_iter(self):
53 """Test __iter__ over keys (Name instances of metrics)."""
54 keys = [k for k in self.metric_set]
55 self.assertEqual(len(keys), 4)
56 for k in keys:
57 self.assertIsInstance(k, Name)
59 def test_items(self):
60 """Test the items iterator."""
61 count = 0
62 for key, value in self.metric_set.items():
63 count += 1
64 self.assertIsInstance(key, Name)
65 self.assertIsInstance(value, Metric)
66 self.assertEqual(count, 4)
68 def test_tags(self):
69 """Both single string and tag lists are present in YAML."""
70 # Parsing this metric required putting the single tag inside a list
71 self.assertEqual(
72 len(self.metric_set['testing.PA1'].tags),
73 1)
74 self.assertIn(
75 'photometric',
76 self.metric_set['testing.PA1'].tags)
78 self.assertEqual(
79 len(self.metric_set['testing.AM1'].tags),
80 2)
81 self.assertIn(
82 'astrometric',
83 self.metric_set['testing.AM1'].tags)
84 self.assertIn(
85 'random-tag',
86 self.metric_set['testing.AM1'].tags)
88 def test_setitem_delitem(self):
89 """Test adding and deleting metrics."""
90 m1 = Metric('validate_drp.test',
91 'test', '',
92 reference_url='example.com',
93 reference_doc='Doc', reference_page=1)
94 metric_set = MetricSet()
95 self.assertEqual(len(metric_set), 0)
97 metric_set['validate_drp.test'] = m1
98 self.assertEqual(len(metric_set), 1)
99 self.assertEqual(metric_set['validate_drp.test'], m1)
101 with self.assertRaises(KeyError):
102 # inconsistent metric names
103 metric_set['validate_drp.new_test'] = m1
105 with self.assertRaises(TypeError):
106 # Not a metric name
107 n = Name('validate_drp')
108 m2 = Metric(n, 'test', '')
109 metric_set[n] = m2
111 del metric_set['validate_drp.test']
112 self.assertEqual(len(metric_set), 0)
114 def test_insert(self):
115 """Test MetricSet.insert."""
116 m1 = Metric('validate_drp.test',
117 'test', '',
118 reference_url='example.com',
119 reference_doc='Doc', reference_page=1)
120 metric_set = MetricSet()
122 metric_set.insert(m1)
123 self.assertEqual(m1, metric_set['validate_drp.test'])
125 def test_update(self):
126 """Test MetricSet.update."""
127 m1 = Metric('validate_drp.test',
128 'test', '',
129 reference_url='example.com',
130 reference_doc='Doc', reference_page=1)
131 new_metric_set = MetricSet([m1])
133 self.metric_set.update(new_metric_set)
135 self.assertIn('validate_drp.test', self.metric_set)
136 self.assertIn('testing.PA1', self.metric_set)
137 self.assertIn('testing.PF1', self.metric_set)
138 self.assertIn('testing.PA2', self.metric_set)
139 self.assertIn('testing.AM1', self.metric_set)
141 def test_iadd(self):
142 """Test __iadd__ to merging metric sets."""
143 m1 = Metric('validate_drp.test',
144 'test', '',
145 reference_url='example.com',
146 reference_doc='Doc', reference_page=1)
147 new_metric_set = MetricSet([m1])
149 self.metric_set += new_metric_set
151 self.assertIn('validate_drp.test', self.metric_set)
152 self.assertIn('testing.PA1', self.metric_set)
153 self.assertIn('testing.PF1', self.metric_set)
154 self.assertIn('testing.PA2', self.metric_set)
155 self.assertIn('testing.AM1', self.metric_set)
158class VerifyMetricsParsingTestCase(unittest.TestCase):
159 """Test parsing metrics from verify_metrics (an EUPS package)."""
161 def setUp(self):
162 self.metric_set = MetricSet.load_metrics_package('verify_metrics')
164 def test_len(self):
165 """Just verify that we got metrics without raising an exception"""
166 self.assertTrue(len(self.metric_set) > 0)
168 def test_nonexistent_package(self):
169 """Test handling of non-existing metrics packages/directories."""
170 with self.assertRaises(OSError):
171 MetricSet.load_metrics_package('nonexistent_metrics')
174class MetricSetSubsetTestCase(unittest.TestCase):
175 """Test case for MetricSet.subset."""
177 def setUp(self):
178 self.m1 = Metric('pkgA.m1', 'In pkgA', '', tags='testing')
179 self.m2 = Metric('pkgA.m2', 'In pkgA', '', tags='other')
180 self.m3 = Metric('pkgB.m3', 'In pkgB', '', tags='testing')
181 self.metric_set = MetricSet([self.m1, self.m2, self.m3])
183 def test_subset_A(self):
184 subset = self.metric_set.subset('pkgA')
185 self.assertEqual(len(subset), 2)
186 self.assertIn(self.m1.name, subset)
187 self.assertIn(self.m2.name, subset)
188 self.assertNotIn(self.m3.name, subset)
190 def test_subset_B(self):
191 subset = self.metric_set.subset('pkgB')
192 self.assertEqual(len(subset), 1)
193 self.assertNotIn(self.m1.name, subset)
194 self.assertNotIn(self.m2.name, subset)
195 self.assertIn(self.m3.name, subset)
197 def test_subset_testing_tag(self):
198 subset = self.metric_set.subset(tags=['testing'])
199 self.assertEqual(len(subset), 2)
200 self.assertIn(self.m1.name, subset)
201 self.assertNotIn(self.m2.name, subset)
202 self.assertIn(self.m3.name, subset)
204 def test_subset_A_testing_tag(self):
205 subset = self.metric_set.subset(package='pkgA', tags=['testing'])
206 self.assertEqual(len(subset), 1)
207 self.assertIn(self.m1.name, subset)
208 self.assertNotIn(self.m2.name, subset)
209 self.assertNotIn(self.m3.name, subset)
212class MetricSetSerializationTestCase(unittest.TestCase):
213 """Test JSON serialization and deserialization for MetricSets."""
215 def setUp(self):
216 self.m1 = Metric('pkgA.m1', 'In pkgA', '', tags=['testing'])
217 self.m2 = Metric('pkgA.m2', 'In pkgA', '', tags=['other'])
218 self.m3 = Metric('pkgB.m3', 'In pkgB', '', tags=['testing'])
219 self.metric_set = MetricSet([self.m1, self.m2, self.m3])
221 def test_serialization(self):
222 json_doc = self.metric_set.json
223 new_metric_set = MetricSet.deserialize(json_doc)
224 self.assertEqual(self.metric_set, new_metric_set)