Coverage for tests/test_inspectjob.py: 23%

1# This file is part of verify. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22from io import StringIO 

23import re 

24import unittest.mock 

25 

26import astropy.units as u 

27 

28from lsst.verify import Job, Metric, Measurement, ThresholdSpecification 

29from lsst.verify.bin.inspectjob import inspect_job 

30 

31 

32@unittest.mock.patch("sys.stdout", new_callable=StringIO) 

33class InspectJobTestCase(unittest.TestCase): 

34 

35 @classmethod 

36 def setUpClass(cls): 

37 # Do not use re.DOTALL; some tests assume it's not set 

38 cls.regex_flags = re.IGNORECASE 

39 

40 def setUp(self): 

41 self.job = Job() 

42 self.job.metrics.insert(Metric("foo.boringmetric", "", 

43 u.percent, 

44 tags=["redundant"])) 

45 self.job.metrics.insert(Metric("foo.fancymetric", "", 

46 u.meter, 

47 tags=["vital"])) 

48 self.job.measurements.insert(Measurement("foo.fancymetric", 

49 2.0 * u.meter)) 

50 self.job.measurements.insert(Measurement("foo.fanciermetric", 

51 3.5 * u.second)) 

52 self.job.measurements["foo.fanciermetric"].notes["fanciness"] \ 

53 = "moderate" 

54 self.job.measurements.insert(Measurement("foo.fanciestmetric", 

55 3.1415927 * u.kilogram)) 

56 self.job.meta["bar"] = "high" 

57 self.job.meta["shape"] = "rotund" 

58 self.job.specs.insert(ThresholdSpecification("utterly_ridiculous", 

59 1e10 * u.meter, 

60 ">")) 

61 # MUST run inspect_job inside test case to capture output 

62 

63 def test_metrics(self, mock_stdout): 

64 """Test that inspect_job only mentions metrics with measurements." 

65 """ 

66 inspect_job(self.job) 

67 self.assertNotIn("foo.boringmetric", mock_stdout.getvalue()) 

68 self.assertIn("foo.fancymetric", mock_stdout.getvalue()) 

69 self.assertIn("foo.fanciermetric", mock_stdout.getvalue()) 

70 self.assertIn("foo.fanciestmetric", mock_stdout.getvalue()) 

71 

72 def _check_measurement(self, measurement, output): 

73 # Test for metric name, followed by value and units 

74 # None of the examples are dimensionless, so can ignore that case 

75 regex = r"%s\W+?(?P<value>[\d.-]+ \w+)" % (measurement.metric_name) 

76 match = re.search(regex, output, flags=self.regex_flags) 

77 

78 error = "Can't find %s and value on same row." \ 

79 % measurement.metric_name 

80 self.assertIsNotNone(match, msg=error) 

81 

82 value = match.group("value") 

83 try: 

84 trailing = re.match(r"\d+\.(\d+)", value, flags=self.regex_flags) 

85 decimals = len(trailing.group(1)) 

86 except TypeError: 

87 decimals = 0 

88 # Don't test # of decimal places; trailing zeros may be dropped 

89 self.assertEqual(str(measurement.quantity.round(decimals)), value) 

90 

91 def test_measurements(self, mock_stdout): 

92 """Test that inspect_job dumps measurements with and without metadata. 

93 """ 

94 inspect_job(self.job) 

95 output = mock_stdout.getvalue() 

96 # MeasurementSet.values does not exist 

97 for _, measurement in self.job.measurements.items(): 

98 self._check_measurement(measurement, output) 

99 

100 def test_measurement_metadata(self, mock_stdout): 

101 """Test that inspect_job dumps measurement-level metadata on the same 

102 line as their measurement. 

103 """ 

104 inspect_job(self.job) 

105 output = mock_stdout.getvalue() 

106 for metric_name, measurement in self.job.measurements.items(): 

107 line = re.search("^.*%s.*$" % metric_name, 

108 output, 

109 flags=self.regex_flags | re.MULTILINE) 

110 error = "Can't find measurement %s" % metric_name 

111 self.assertIsNotNone(line, msg=error) 

112 line = line.group() 

113 

114 for key in measurement.notes: 

115 regex = r"(?P<keyname>[\w\.]+)\W+%s" % (measurement.notes[key]) 

116 match = re.search(regex, line, flags=self.regex_flags) 

117 self.assertIsNotNone(match, 

118 msg="Can't find metadata %s." % key) 

119 reportedMetadataName = match.group('keyname') 

120 fullMetadataName = "%s.%s" % (measurement.metric_name, 

121 reportedMetadataName) 

122 self.assertEqual(fullMetadataName, key) 

123 

124 def _check_metadata(self, key, value, output): 

125 regex = r"%s.+%s" % (key, value) 

126 match = re.search(regex, output, flags=self.regex_flags) 

127 self.assertIsNotNone(match, msg="Can't find metadata %s" % key) 

128 

129 def test_top_metadata(self, mock_stdout): 

130 """Test that inspect_job dumps top-level metadata. 

131 """ 

132 inspect_job(self.job) 

133 output = mock_stdout.getvalue() 

134 for key, value in [("bar", "high"), 

135 ("shape", "rotund")]: 

136 self._check_metadata(key, value, output) 

137 

138 def test_specs(self, mock_stdout): 

139 """Test that inspect_job does not dump specifications." 

140 """ 

141 self.assertNotIn("utterly_ridiculous", mock_stdout.getvalue()) 

142 

143 def test_empty(self, mock_stdout): 

144 """Test that inspect_job can handle files with neither metrics nor 

145 metadata. 

146 """ 

147 inspect_job(Job()) 

148 # No specific output expected, so test passes if inspect_job 

149 # didn't raise. 

150 

151 def test_metadataonly(self, mock_stdout): 

152 """Test that inspect_job can handle files with metadata but no metrics. 

153 """ 

154 # Job and its components were not designed to support deletion, so 

155 # create a new Job from scratch to ensure it's a valid object. 

156 job = Job() 

157 job.metrics.insert(Metric("foo.boringmetric", "", 

158 u.percent, 

159 tags=["redundant"])) 

160 job.metrics.insert(Metric("foo.fancymetric", "", 

161 u.meter, 

162 tags=["vital"])) 

163 job.meta["bar"] = "high" 

164 job.meta["shape"] = "rotund" 

165 job.specs.insert(ThresholdSpecification("utterly_ridiculous", 

166 1e10 * u.meter, 

167 ">")) 

168 

169 inspect_job(job) 

170 output = mock_stdout.getvalue() 

171 for key, value in [("bar", "high"), 

172 ("shape", "rotund")]: 

173 self._check_metadata(key, value, output) 

174 

175 def test_metricsonly(self, mock_stdout): 

176 """Test that inspect_job can handle files with metrics but no metadata. 

177 """ 

178 # Job and its components were not designed to support deletion, so 

179 # create a new Job from scratch to ensure it's a valid object. 

180 job = Job() 

181 job.metrics.insert(Metric("foo.boringmetric", "", 

182 u.percent, 

183 tags=["redundant"])) 

184 job.metrics.insert(Metric("foo.fancymetric", "", 

185 u.meter, 

186 tags=["vital"])) 

187 job.measurements.insert(Measurement("foo.fancymetric", 

188 2.0 * u.meter)) 

189 job.measurements.insert(Measurement("foo.fanciermetric", 

190 3.5 * u.second)) 

191 job.measurements["foo.fanciermetric"].notes["fanciness"] = "moderate" 

192 job.measurements.insert(Measurement("foo.fanciestmetric", 

193 3.1415927 * u.kilogram)) 

194 

195 inspect_job(job) 

196 output = mock_stdout.getvalue() 

197 # MeasurementSet.values does not exist 

198 for _, measurement in job.measurements.items(): 

199 self._check_measurement(measurement, output) 

200 

201 

202if __name__ == "__main__": 202 ↛ 203line 202 didn't jump to line 203, because the condition on line 202 was never true

203 unittest.main()