Coverage for tests/test_commonMetrics.py: 27%
176 statements
« prev ^ index » next coverage.py v6.4.2, created at 2022-07-16 10:09 +0000
« prev ^ index » next coverage.py v6.4.2, created at 2022-07-16 10:09 +0000
1# This file is part of verify.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22import time
23import unittest
24import warnings
26import astropy.units as u
28import lsst.utils.tests
29import lsst.pipe.base.testUtils
30from lsst.pex.config import Config
31from lsst.pipe.base import Task
32from lsst.utils.timer import timeMethod
34from lsst.verify import Measurement, Name
35from lsst.verify.gen2tasks.testUtils import MetricTaskTestCase
36from lsst.verify.tasks import MetricComputationError, TimingMetricTask, \
37 MemoryMetricTask
38from lsst.verify.tasks.testUtils import MetadataMetricTestCase
41class DummyTask(Task):
42 ConfigClass = Config
43 _DefaultName = "NotARealTask"
44 taskLength = 0.1
46 @timeMethod
47 def run(self):
48 time.sleep(self.taskLength)
51class TimingMetricTestSuite(MetadataMetricTestCase):
52 @classmethod
53 def makeTask(cls):
54 return TimingMetricTask(config=cls._standardConfig())
56 @staticmethod
57 def _standardConfig():
58 config = TimingMetricTask.ConfigClass()
59 config.connections.labelName = DummyTask._DefaultName
60 config.target = DummyTask._DefaultName + ".run"
61 config.connections.package = "verify"
62 config.connections.metric = "DummyTime"
63 return config
65 def setUp(self):
66 super().setUp()
67 self.config = TimingMetricTestSuite._standardConfig()
68 self.metric = Name("verify.DummyTime")
70 self.scienceTask = DummyTask()
71 self.scienceTask.run()
73 def testValid(self):
74 result = self.task.run(self.scienceTask.getFullMetadata())
75 lsst.pipe.base.testUtils.assertValidOutput(self.task, result)
76 meas = result.measurement
78 self.assertIsInstance(meas, Measurement)
79 self.assertEqual(meas.metric_name, self.metric)
80 self.assertGreater(meas.quantity, 0.0 * u.second)
81 self.assertLess(meas.quantity, 2 * DummyTask.taskLength * u.second)
83 def testNoMetric(self):
84 self.config.connections.package = "foo.bar"
85 self.config.connections.metric = "FooBarTime"
86 task = TimingMetricTask(config=self.config)
87 with self.assertRaises(TypeError):
88 task.run(self.scienceTask.getFullMetadata())
90 def testMissingData(self):
91 result = self.task.run(None)
92 lsst.pipe.base.testUtils.assertValidOutput(self.task, result)
93 meas = result.measurement
94 self.assertIsNone(meas)
96 def testRunDifferentMethod(self):
97 self.config.target = DummyTask._DefaultName + ".runDataRef"
98 task = TimingMetricTask(config=self.config)
99 result = task.run(self.scienceTask.getFullMetadata())
100 lsst.pipe.base.testUtils.assertValidOutput(task, result)
101 meas = result.measurement
102 self.assertIsNone(meas)
104 def testNonsenseKeys(self):
105 metadata = self.scienceTask.getFullMetadata()
106 startKeys = [key
107 for key in metadata.paramNames(topLevelOnly=False)
108 if "StartCpuTime" in key]
109 for key in startKeys:
110 del metadata[key]
112 task = TimingMetricTask(config=self.config)
113 with self.assertRaises(MetricComputationError):
114 task.run(metadata)
116 def testBadlyTypedKeys(self):
117 metadata = self.scienceTask.getFullMetadata()
118 endKeys = [key
119 for key in metadata.paramNames(topLevelOnly=False)
120 if "EndCpuTime" in key]
121 for key in endKeys:
122 metadata[key] = str(float(metadata[key]))
124 task = TimingMetricTask(config=self.config)
125 with self.assertRaises(MetricComputationError):
126 task.run(metadata)
128 def testDeprecated(self):
129 with warnings.catch_warnings(record=True):
130 self.config.metric = "verify.DummyTime"
131 self.config.connections.package = ""
132 self.config.connections.metric = ""
133 with warnings.catch_warnings(record=True) as emitted:
134 self.config.validate()
135 self.assertEqual(len(emitted), 1)
136 self.assertEqual(emitted[0].category, FutureWarning)
137 self.assertEqual(self.config.connections.package, "verify")
138 self.assertEqual(self.config.connections.metric, "DummyTime")
141class MemoryMetricTestSuite(MetadataMetricTestCase):
142 @classmethod
143 def makeTask(cls):
144 return MemoryMetricTask(config=cls._standardConfig())
146 @staticmethod
147 def _standardConfig():
148 config = MemoryMetricTask.ConfigClass()
149 config.connections.labelName = DummyTask._DefaultName
150 config.target = DummyTask._DefaultName + ".run"
151 config.connections.package = "verify"
152 config.connections.metric = "DummyMemory"
153 return config
155 def setUp(self):
156 super().setUp()
157 self.config = self._standardConfig()
158 self.metric = Name("verify.DummyMemory")
160 self.scienceTask = DummyTask()
161 self.scienceTask.run()
163 def testValid(self):
164 result = self.task.run(self.scienceTask.getFullMetadata())
165 lsst.pipe.base.testUtils.assertValidOutput(self.task, result)
166 meas = result.measurement
168 self.assertIsInstance(meas, Measurement)
169 self.assertEqual(meas.metric_name, self.metric)
170 self.assertGreater(meas.quantity, 0.0 * u.byte)
172 def testNoMetric(self):
173 self.config.connections.package = "foo.bar"
174 self.config.connections.metric = "FooBarMemory"
175 task = MemoryMetricTask(config=self.config)
176 with self.assertRaises(TypeError):
177 task.run(self.scienceTask.getFullMetadata())
179 def testMissingData(self):
180 result = self.task.run(None)
181 lsst.pipe.base.testUtils.assertValidOutput(self.task, result)
182 meas = result.measurement
183 self.assertIsNone(meas)
185 def testRunDifferentMethod(self):
186 self.config.target = DummyTask._DefaultName + ".runDataRef"
187 task = MemoryMetricTask(config=self.config)
188 result = task.run(self.scienceTask.getFullMetadata())
189 lsst.pipe.base.testUtils.assertValidOutput(task, result)
190 meas = result.measurement
191 self.assertIsNone(meas)
193 def testBadlyTypedKeys(self):
194 metadata = self.scienceTask.getFullMetadata()
195 endKeys = [key
196 for key in metadata.paramNames(topLevelOnly=False)
197 if "EndMaxResidentSetSize" in key]
198 for key in endKeys:
199 metadata[key] = str(float(metadata[key]))
201 task = MemoryMetricTask(config=self.config)
202 with self.assertRaises(MetricComputationError):
203 task.run(metadata)
205 def testOldMetadata(self):
206 """Test compatibility with version 0 metadata
208 This can't actually test differences in unit handling between version 0
209 and version 1, but at least verifies that the code didn't choke on
210 old-style metadata.
211 """
212 newMetadata = self.scienceTask.getFullMetadata()
213 oldMetadata = newMetadata.copy()
214 for key in newMetadata.names(topLevelOnly=False):
215 if "__version__" in key:
216 oldMetadata.remove(key)
218 result = self.task.run(oldMetadata)
219 lsst.pipe.base.testUtils.assertValidOutput(self.task, result)
220 meas = result.measurement
222 self.assertIsInstance(meas, Measurement)
223 self.assertEqual(meas.metric_name, self.metric)
225 # Since new style is always bytes, old-style will be less or equal
226 newResult = self.task.run(newMetadata)
227 self.assertGreater(meas.quantity, 0.0 * u.byte)
228 self.assertLessEqual(meas.quantity, newResult.measurement.quantity)
230 def testDeprecated(self):
231 with warnings.catch_warnings(record=True):
232 self.config.metric = "verify.DummyMemory"
233 self.config.connections.package = ""
234 self.config.connections.metric = ""
235 with warnings.catch_warnings(record=True) as emitted:
236 self.config.validate()
237 self.assertEqual(len(emitted), 1)
238 self.assertEqual(emitted[0].category, FutureWarning)
239 self.assertEqual(self.config.connections.package, "verify")
240 self.assertEqual(self.config.connections.metric, "DummyMemory")
243# Hack around unittest's hacky test setup system
244del MetricTaskTestCase
245del MetadataMetricTestCase
248class MemoryTester(lsst.utils.tests.MemoryTestCase):
249 pass
252def setup_module(module):
253 lsst.utils.tests.init()
256if __name__ == "__main__": 256 ↛ 257line 256 didn't jump to line 257, because the condition on line 256 was never true
257 lsst.utils.tests.init()
258 unittest.main()