Coverage for tests/test_commonMetrics.py: 30%
152 statements
« prev ^ index » next coverage.py v6.4.4, created at 2022-08-19 19:10 +0000
« prev ^ index » next coverage.py v6.4.4, created at 2022-08-19 19:10 +0000
1# This file is part of verify.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22import time
23import unittest
25import astropy.units as u
27import lsst.utils.tests
28import lsst.pipe.base.testUtils
29from lsst.pex.config import Config
30from lsst.pipe.base import Task
31from lsst.utils.timer import timeMethod
33from lsst.verify import Measurement, Name
34from lsst.verify.tasks import MetricComputationError, TimingMetricTask, \
35 MemoryMetricTask
36from lsst.verify.tasks.testUtils import MetricTaskTestCase, MetadataMetricTestCase
39class DummyTask(Task):
40 ConfigClass = Config
41 _DefaultName = "NotARealTask"
42 taskLength = 0.1
44 @timeMethod
45 def run(self):
46 time.sleep(self.taskLength)
49class TimingMetricTestSuite(MetadataMetricTestCase):
50 @classmethod
51 def makeTask(cls):
52 return TimingMetricTask(config=cls._standardConfig())
54 @staticmethod
55 def _standardConfig():
56 config = TimingMetricTask.ConfigClass()
57 config.connections.labelName = DummyTask._DefaultName
58 config.target = DummyTask._DefaultName + ".run"
59 config.connections.package = "verify"
60 config.connections.metric = "DummyTime"
61 return config
63 def setUp(self):
64 super().setUp()
65 self.config = TimingMetricTestSuite._standardConfig()
66 self.metric = Name("verify.DummyTime")
68 self.scienceTask = DummyTask()
69 self.scienceTask.run()
71 def testValid(self):
72 result = self.task.run(self.scienceTask.getFullMetadata())
73 lsst.pipe.base.testUtils.assertValidOutput(self.task, result)
74 meas = result.measurement
76 self.assertIsInstance(meas, Measurement)
77 self.assertEqual(meas.metric_name, self.metric)
78 self.assertGreater(meas.quantity, 0.0 * u.second)
79 self.assertLess(meas.quantity, 2 * DummyTask.taskLength * u.second)
81 def testNoMetric(self):
82 self.config.connections.package = "foo.bar"
83 self.config.connections.metric = "FooBarTime"
84 task = TimingMetricTask(config=self.config)
85 with self.assertRaises(TypeError):
86 task.run(self.scienceTask.getFullMetadata())
88 def testMissingData(self):
89 result = self.task.run(None)
90 lsst.pipe.base.testUtils.assertValidOutput(self.task, result)
91 meas = result.measurement
92 self.assertIsNone(meas)
94 def testRunDifferentMethod(self):
95 self.config.target = DummyTask._DefaultName + ".runDataRef"
96 task = TimingMetricTask(config=self.config)
97 result = task.run(self.scienceTask.getFullMetadata())
98 lsst.pipe.base.testUtils.assertValidOutput(task, result)
99 meas = result.measurement
100 self.assertIsNone(meas)
102 def testNonsenseKeys(self):
103 metadata = self.scienceTask.getFullMetadata()
104 startKeys = [key
105 for key in metadata.paramNames(topLevelOnly=False)
106 if "StartCpuTime" in key]
107 for key in startKeys:
108 del metadata[key]
110 task = TimingMetricTask(config=self.config)
111 with self.assertRaises(MetricComputationError):
112 task.run(metadata)
114 def testBadlyTypedKeys(self):
115 metadata = self.scienceTask.getFullMetadata()
116 endKeys = [key
117 for key in metadata.paramNames(topLevelOnly=False)
118 if "EndCpuTime" in key]
119 for key in endKeys:
120 metadata[key] = str(float(metadata[key]))
122 task = TimingMetricTask(config=self.config)
123 with self.assertRaises(MetricComputationError):
124 task.run(metadata)
127class MemoryMetricTestSuite(MetadataMetricTestCase):
128 @classmethod
129 def makeTask(cls):
130 return MemoryMetricTask(config=cls._standardConfig())
132 @staticmethod
133 def _standardConfig():
134 config = MemoryMetricTask.ConfigClass()
135 config.connections.labelName = DummyTask._DefaultName
136 config.target = DummyTask._DefaultName + ".run"
137 config.connections.package = "verify"
138 config.connections.metric = "DummyMemory"
139 return config
141 def setUp(self):
142 super().setUp()
143 self.config = self._standardConfig()
144 self.metric = Name("verify.DummyMemory")
146 self.scienceTask = DummyTask()
147 self.scienceTask.run()
149 def testValid(self):
150 result = self.task.run(self.scienceTask.getFullMetadata())
151 lsst.pipe.base.testUtils.assertValidOutput(self.task, result)
152 meas = result.measurement
154 self.assertIsInstance(meas, Measurement)
155 self.assertEqual(meas.metric_name, self.metric)
156 self.assertGreater(meas.quantity, 0.0 * u.byte)
158 def testNoMetric(self):
159 self.config.connections.package = "foo.bar"
160 self.config.connections.metric = "FooBarMemory"
161 task = MemoryMetricTask(config=self.config)
162 with self.assertRaises(TypeError):
163 task.run(self.scienceTask.getFullMetadata())
165 def testMissingData(self):
166 result = self.task.run(None)
167 lsst.pipe.base.testUtils.assertValidOutput(self.task, result)
168 meas = result.measurement
169 self.assertIsNone(meas)
171 def testRunDifferentMethod(self):
172 self.config.target = DummyTask._DefaultName + ".runDataRef"
173 task = MemoryMetricTask(config=self.config)
174 result = task.run(self.scienceTask.getFullMetadata())
175 lsst.pipe.base.testUtils.assertValidOutput(task, result)
176 meas = result.measurement
177 self.assertIsNone(meas)
179 def testBadlyTypedKeys(self):
180 metadata = self.scienceTask.getFullMetadata()
181 endKeys = [key
182 for key in metadata.paramNames(topLevelOnly=False)
183 if "EndMaxResidentSetSize" in key]
184 for key in endKeys:
185 metadata[key] = str(float(metadata[key]))
187 task = MemoryMetricTask(config=self.config)
188 with self.assertRaises(MetricComputationError):
189 task.run(metadata)
191 def testOldMetadata(self):
192 """Test compatibility with version 0 metadata
194 This can't actually test differences in unit handling between version 0
195 and version 1, but at least verifies that the code didn't choke on
196 old-style metadata.
197 """
198 newMetadata = self.scienceTask.getFullMetadata()
199 oldMetadata = newMetadata.copy()
200 for key in newMetadata.names(topLevelOnly=False):
201 if "__version__" in key:
202 del oldMetadata[key]
204 result = self.task.run(oldMetadata)
205 lsst.pipe.base.testUtils.assertValidOutput(self.task, result)
206 meas = result.measurement
208 self.assertIsInstance(meas, Measurement)
209 self.assertEqual(meas.metric_name, self.metric)
211 # Since new style is always bytes, old-style will be less or equal
212 newResult = self.task.run(newMetadata)
213 self.assertGreater(meas.quantity, 0.0 * u.byte)
214 self.assertLessEqual(meas.quantity, newResult.measurement.quantity)
217# Hack around unittest's hacky test setup system
218del MetricTaskTestCase
219del MetadataMetricTestCase
222class MemoryTester(lsst.utils.tests.MemoryTestCase):
223 pass
226def setup_module(module):
227 lsst.utils.tests.init()
230if __name__ == "__main__": 230 ↛ 231line 230 didn't jump to line 231, because the condition on line 230 was never true
231 lsst.utils.tests.init()
232 unittest.main()