Coverage for tests/test_metricset.py: 21%

116 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2023-03-24 03:18 -0700

1# This file is part of verify. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import os 

23import unittest 

24 

25from lsst.verify import Metric, MetricSet, Name 

26 

27 

28class MetricsPackageTestCase(unittest.TestCase): 

29 """Test creating a MetricSet from a mock verification framework 

30 metric package located at data/ relative test modules. 

31 

32 These tests are coupled to the data/metrics/*.yaml files. 

33 """ 

34 

35 def setUp(self): 

36 self.metrics_yaml_dirname = os.path.join( 

37 os.path.dirname(__file__), 'data') 

38 

39 self.metric_set = MetricSet.load_metrics_package( 

40 self.metrics_yaml_dirname) 

41 

42 def test_len(self): 

43 self.assertEqual(len(self.metric_set), 4) 

44 

45 def test_contains(self): 

46 for key in ['testing.PA1', 

47 'testing.PF1', 

48 'testing.PA2', 

49 'testing.AM1']: 

50 self.assertIn(key, self.metric_set, msg=key) 

51 

52 def test_iter(self): 

53 """Test __iter__ over keys (Name instances of metrics).""" 

54 keys = [k for k in self.metric_set] 

55 self.assertEqual(len(keys), 4) 

56 for k in keys: 

57 self.assertIsInstance(k, Name) 

58 

59 def test_items(self): 

60 """Test the items iterator.""" 

61 count = 0 

62 for key, value in self.metric_set.items(): 

63 count += 1 

64 self.assertIsInstance(key, Name) 

65 self.assertIsInstance(value, Metric) 

66 self.assertEqual(count, 4) 

67 

68 def test_tags(self): 

69 """Both single string and tag lists are present in YAML.""" 

70 # Parsing this metric required putting the single tag inside a list 

71 self.assertEqual( 

72 len(self.metric_set['testing.PA1'].tags), 

73 1) 

74 self.assertIn( 

75 'photometric', 

76 self.metric_set['testing.PA1'].tags) 

77 

78 self.assertEqual( 

79 len(self.metric_set['testing.AM1'].tags), 

80 2) 

81 self.assertIn( 

82 'astrometric', 

83 self.metric_set['testing.AM1'].tags) 

84 self.assertIn( 

85 'random-tag', 

86 self.metric_set['testing.AM1'].tags) 

87 

88 def test_setitem_delitem(self): 

89 """Test adding and deleting metrics.""" 

90 m1 = Metric('validate_drp.test', 

91 'test', '', 

92 reference_url='example.com', 

93 reference_doc='Doc', reference_page=1) 

94 metric_set = MetricSet() 

95 self.assertEqual(len(metric_set), 0) 

96 

97 metric_set['validate_drp.test'] = m1 

98 self.assertEqual(len(metric_set), 1) 

99 self.assertEqual(metric_set['validate_drp.test'], m1) 

100 

101 with self.assertRaises(KeyError): 

102 # inconsistent metric names 

103 metric_set['validate_drp.new_test'] = m1 

104 

105 with self.assertRaises(TypeError): 

106 # Not a metric name 

107 n = Name('validate_drp') 

108 m2 = Metric(n, 'test', '') 

109 metric_set[n] = m2 

110 

111 del metric_set['validate_drp.test'] 

112 self.assertEqual(len(metric_set), 0) 

113 

114 def test_insert(self): 

115 """Test MetricSet.insert.""" 

116 m1 = Metric('validate_drp.test', 

117 'test', '', 

118 reference_url='example.com', 

119 reference_doc='Doc', reference_page=1) 

120 metric_set = MetricSet() 

121 

122 metric_set.insert(m1) 

123 self.assertEqual(m1, metric_set['validate_drp.test']) 

124 

125 def test_update(self): 

126 """Test MetricSet.update.""" 

127 m1 = Metric('validate_drp.test', 

128 'test', '', 

129 reference_url='example.com', 

130 reference_doc='Doc', reference_page=1) 

131 new_metric_set = MetricSet([m1]) 

132 

133 self.metric_set.update(new_metric_set) 

134 

135 self.assertIn('validate_drp.test', self.metric_set) 

136 self.assertIn('testing.PA1', self.metric_set) 

137 self.assertIn('testing.PF1', self.metric_set) 

138 self.assertIn('testing.PA2', self.metric_set) 

139 self.assertIn('testing.AM1', self.metric_set) 

140 

141 def test_iadd(self): 

142 """Test __iadd__ to merging metric sets.""" 

143 m1 = Metric('validate_drp.test', 

144 'test', '', 

145 reference_url='example.com', 

146 reference_doc='Doc', reference_page=1) 

147 new_metric_set = MetricSet([m1]) 

148 

149 self.metric_set += new_metric_set 

150 

151 self.assertIn('validate_drp.test', self.metric_set) 

152 self.assertIn('testing.PA1', self.metric_set) 

153 self.assertIn('testing.PF1', self.metric_set) 

154 self.assertIn('testing.PA2', self.metric_set) 

155 self.assertIn('testing.AM1', self.metric_set) 

156 

157 

158class VerifyMetricsParsingTestCase(unittest.TestCase): 

159 """Test parsing metrics from verify_metrics (an EUPS package).""" 

160 

161 def setUp(self): 

162 self.metric_set = MetricSet.load_metrics_package('verify_metrics') 

163 

164 def test_len(self): 

165 """Just verify that we got metrics without raising an exception""" 

166 self.assertTrue(len(self.metric_set) > 0) 

167 

168 def test_nonexistent_package(self): 

169 """Test handling of non-existing metrics packages/directories.""" 

170 with self.assertRaises(OSError): 

171 MetricSet.load_metrics_package('nonexistent_metrics') 

172 

173 

174class MetricSetSubsetTestCase(unittest.TestCase): 

175 """Test case for MetricSet.subset.""" 

176 

177 def setUp(self): 

178 self.m1 = Metric('pkgA.m1', 'In pkgA', '', tags='testing') 

179 self.m2 = Metric('pkgA.m2', 'In pkgA', '', tags='other') 

180 self.m3 = Metric('pkgB.m3', 'In pkgB', '', tags='testing') 

181 self.metric_set = MetricSet([self.m1, self.m2, self.m3]) 

182 

183 def test_subset_A(self): 

184 subset = self.metric_set.subset('pkgA') 

185 self.assertEqual(len(subset), 2) 

186 self.assertIn(self.m1.name, subset) 

187 self.assertIn(self.m2.name, subset) 

188 self.assertNotIn(self.m3.name, subset) 

189 

190 def test_subset_B(self): 

191 subset = self.metric_set.subset('pkgB') 

192 self.assertEqual(len(subset), 1) 

193 self.assertNotIn(self.m1.name, subset) 

194 self.assertNotIn(self.m2.name, subset) 

195 self.assertIn(self.m3.name, subset) 

196 

197 def test_subset_testing_tag(self): 

198 subset = self.metric_set.subset(tags=['testing']) 

199 self.assertEqual(len(subset), 2) 

200 self.assertIn(self.m1.name, subset) 

201 self.assertNotIn(self.m2.name, subset) 

202 self.assertIn(self.m3.name, subset) 

203 

204 def test_subset_A_testing_tag(self): 

205 subset = self.metric_set.subset(package='pkgA', tags=['testing']) 

206 self.assertEqual(len(subset), 1) 

207 self.assertIn(self.m1.name, subset) 

208 self.assertNotIn(self.m2.name, subset) 

209 self.assertNotIn(self.m3.name, subset) 

210 

211 

212class MetricSetSerializationTestCase(unittest.TestCase): 

213 """Test JSON serialization and deserialization for MetricSets.""" 

214 

215 def setUp(self): 

216 self.m1 = Metric('pkgA.m1', 'In pkgA', '', tags=['testing']) 

217 self.m2 = Metric('pkgA.m2', 'In pkgA', '', tags=['other']) 

218 self.m3 = Metric('pkgB.m3', 'In pkgB', '', tags=['testing']) 

219 self.metric_set = MetricSet([self.m1, self.m2, self.m3]) 

220 

221 def test_serialization(self): 

222 json_doc = self.metric_set.json 

223 new_metric_set = MetricSet.deserialize(json_doc) 

224 self.assertEqual(self.metric_set, new_metric_set)