Coverage for tests/test_task.py: 30%
186 statements
« prev ^ index » next coverage.py v7.4.3, created at 2024-02-28 11:05 +0000
« prev ^ index » next coverage.py v7.4.3, created at 2024-02-28 11:05 +0000
1#
2# LSST Data Management System
3# Copyright 2008, 2009, 2010 LSST Corporation.
4#
5# This product includes software developed by the
6# LSST Project (http://www.lsst.org/).
7#
8# This program is free software: you can redistribute it and/or modify
9# it under the terms of the GNU General Public License as published by
10# the Free Software Foundation, either version 3 of the License, or
11# (at your option) any later version.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the LSST License Statement and
19# the GNU General Public License along with this program. If not,
20# see <http://www.lsstcorp.org/LegalNotices/>.
21#
22import json
23import logging
24import numbers
25import time
26import unittest
28import lsst.pex.config as pexConfig
29import lsst.pipe.base as pipeBase
30import lsst.utils.tests
31import yaml
33# Whilst in transition the test can't tell which type is
34# going to be used for metadata.
35from lsst.pipe.base.task import _TASK_METADATA_TYPE
36from lsst.utils.timer import timeMethod
39class AddConfig(pexConfig.Config):
40 """Config for AddTask."""
42 addend = pexConfig.Field(doc="amount to add", dtype=float, default=3.1)
45class AddTask(pipeBase.Task):
46 """Example task to add two values."""
48 ConfigClass = AddConfig
50 @timeMethod
51 def run(self, val):
52 self.metadata.add("add", self.config.addend)
53 return pipeBase.Struct(
54 val=val + self.config.addend,
55 )
58class MultConfig(pexConfig.Config):
59 """Config for MultTask."""
61 multiplicand = pexConfig.Field(doc="amount by which to multiply", dtype=float, default=2.5)
64class MultTask(pipeBase.Task):
65 """Task to multiply."""
67 ConfigClass = MultConfig
69 @timeMethod
70 def run(self, val):
71 self.metadata.add("mult", self.config.multiplicand)
72 return pipeBase.Struct(
73 val=val * self.config.multiplicand,
74 )
77# prove that registry fields can also be used to hold subtasks
78# by using a registry to hold MultTask
79multRegistry = pexConfig.makeRegistry("Registry for Mult-like tasks")
80multRegistry.register("stdMult", MultTask)
83class AddMultConfig(pexConfig.Config):
84 """Config for AddMult."""
86 add = AddTask.makeField("add task")
87 mult = multRegistry.makeField("mult task", default="stdMult")
90class AddMultTask(pipeBase.Task):
91 """Test Task with subtasks."""
93 ConfigClass = AddMultConfig
94 _DefaultName = "addMult"
95 _add_module_logger_prefix = False
97 """First add, then multiply."""
99 def __init__(self, **keyArgs):
100 pipeBase.Task.__init__(self, **keyArgs)
101 self.makeSubtask("add")
102 self.makeSubtask("mult")
104 @timeMethod
105 def run(self, val):
106 with self.timer("context"):
107 addRet = self.add.run(val)
108 multRet = self.mult.run(addRet.val)
109 self.metadata.add("addmult", multRet.val)
110 return pipeBase.Struct(
111 val=multRet.val,
112 )
114 @timeMethod
115 def failDec(self):
116 """Fail with a decorator."""
117 raise RuntimeError("failDec intentional error")
119 def failCtx(self):
120 """Fail inside a context manager."""
121 with self.timer("failCtx"):
122 raise RuntimeError("failCtx intentional error")
125class AddMultTask2(AddMultTask):
126 """Subclass that gets an automatic logger prefix."""
128 _add_module_logger_prefix = True
131class AddTwiceTask(AddTask):
132 """Variant of AddTask that adds twice the addend."""
134 def run(self, val):
135 addend = self.config.addend
136 return pipeBase.Struct(val=val + (2 * addend))
139class TaskTestCase(unittest.TestCase):
140 """A test case for Task."""
142 def setUp(self):
143 self.valDict = dict()
145 def tearDown(self):
146 self.valDict = None
148 def testBasics(self):
149 """Test basic construction and use of a task."""
150 for addend in (1.1, -3.5):
151 for multiplicand in (0.9, -45.0):
152 config = AddMultTask.ConfigClass()
153 config.add.addend = addend
154 config.mult["stdMult"].multiplicand = multiplicand
155 # make sure both ways of accessing the registry work and give
156 # the same result
157 self.assertEqual(config.mult.active.multiplicand, multiplicand)
158 addMultTask = AddMultTask(config=config)
159 for val in (-1.0, 0.0, 17.5):
160 ret = addMultTask.run(val=val)
161 self.assertAlmostEqual(ret.val, (val + addend) * multiplicand)
163 def testNames(self):
164 """Test getName() and getFullName()."""
165 addMultTask = AddMultTask()
166 self.assertEqual(addMultTask.getName(), "addMult")
167 self.assertEqual(addMultTask.add.getName(), "add")
168 self.assertEqual(addMultTask.mult.getName(), "mult")
170 self.assertEqual(addMultTask._name, "addMult")
171 self.assertEqual(addMultTask.add._name, "add")
172 self.assertEqual(addMultTask.mult._name, "mult")
174 self.assertEqual(addMultTask.getFullName(), "addMult")
175 self.assertEqual(addMultTask.add.getFullName(), "addMult.add")
176 self.assertEqual(addMultTask.mult.getFullName(), "addMult.mult")
178 self.assertEqual(addMultTask._fullName, "addMult")
179 self.assertEqual(addMultTask.add._fullName, "addMult.add")
180 self.assertEqual(addMultTask.mult._fullName, "addMult.mult")
182 def testLog(self):
183 """Test the Task's logger."""
184 addMultTask = AddMultTask()
185 self.assertEqual(addMultTask.log.name, "addMult")
186 self.assertEqual(addMultTask.add.log.name, "addMult.add")
188 log = logging.getLogger("tester")
189 addMultTask = AddMultTask(log=log)
190 self.assertEqual(addMultTask.log.name, "tester.addMult")
191 self.assertEqual(addMultTask.add.log.name, "tester.addMult.add")
193 addMultTask2 = AddMultTask2()
194 self.assertEqual(addMultTask2.log.name, f"{__name__}.addMult")
196 def testGetFullMetadata(self):
197 """Test getFullMetadata()."""
198 addMultTask = AddMultTask()
199 addMultTask.run(val=1.234) # Add some metadata
200 fullMetadata = addMultTask.getFullMetadata()
201 self.assertIsInstance(fullMetadata["addMult"], _TASK_METADATA_TYPE)
202 self.assertIsInstance(fullMetadata["addMult:add"], _TASK_METADATA_TYPE)
203 self.assertIsInstance(fullMetadata["addMult:mult"], _TASK_METADATA_TYPE)
204 self.assertEqual(set(fullMetadata), {"addMult", "addMult:add", "addMult:mult"})
206 all_names = fullMetadata.names()
207 self.assertIn("addMult", all_names)
208 self.assertIn("addMult.runStartUtc", all_names)
210 param_names = fullMetadata.paramNames(topLevelOnly=True)
211 # No top level keys without hierarchy
212 self.assertEqual(set(param_names), set())
214 param_names = fullMetadata.paramNames(topLevelOnly=False)
215 self.assertNotIn("addMult", param_names)
216 self.assertIn("addMult.runStartUtc", param_names)
217 self.assertIn("addMult:add.runStartCpuTime", param_names)
219 def testEmptyMetadata(self):
220 task = AddMultTask()
221 task.run(val=1.2345)
222 task.emptyMetadata()
223 fullMetadata = task.getFullMetadata()
224 self.assertEqual(len(fullMetadata["addMult"]), 0)
225 self.assertEqual(len(fullMetadata["addMult:add"]), 0)
226 self.assertEqual(len(fullMetadata["addMult:mult"]), 0)
228 def testReplace(self):
229 """Test replacing one subtask with another."""
230 for addend in (1.1, -3.5):
231 for multiplicand in (0.9, -45.0):
232 config = AddMultTask.ConfigClass()
233 config.add.retarget(AddTwiceTask)
234 config.add.addend = addend
235 config.mult["stdMult"].multiplicand = multiplicand
236 addMultTask = AddMultTask(config=config)
237 for val in (-1.0, 0.0, 17.5):
238 ret = addMultTask.run(val=val)
239 self.assertAlmostEqual(ret.val, (val + (2 * addend)) * multiplicand)
241 def testFail(self):
242 """Test timers when the code they are timing fails."""
243 addMultTask = AddMultTask()
244 try:
245 addMultTask.failDec()
246 self.fail("Expected RuntimeError")
247 except RuntimeError:
248 self.assertIn("failDecEndCpuTime", addMultTask.metadata)
249 try:
250 addMultTask.failCtx()
251 self.fail("Expected RuntimeError")
252 except RuntimeError:
253 self.assertIn("failCtxEndCpuTime", addMultTask.metadata)
255 def testTimeMethod(self):
256 """Test that the timer is adding the right metadata."""
257 addMultTask = AddMultTask()
259 # Run twice to ensure we are additive.
260 addMultTask.run(val=1.1)
261 addMultTask.run(val=2.0)
262 # Check existence and type
263 for key, keyType in (
264 ("Utc", str),
265 ("CpuTime", float),
266 ("UserTime", float),
267 ("SystemTime", float),
268 ("MaxResidentSetSize", numbers.Integral),
269 ("MinorPageFaults", numbers.Integral),
270 ("MajorPageFaults", numbers.Integral),
271 ("BlockInputs", numbers.Integral),
272 ("BlockOutputs", numbers.Integral),
273 ("VoluntaryContextSwitches", numbers.Integral),
274 ("InvoluntaryContextSwitches", numbers.Integral),
275 ):
276 for when in ("Start", "End"):
277 for method in ("run", "context"):
278 name = method + when + key
279 self.assertIn(name, addMultTask.metadata, name + " is missing from task metadata")
280 self.assertIsInstance(
281 addMultTask.metadata.getScalar(name),
282 keyType,
283 f"{name} is not of the right type "
284 f"({keyType} vs {type(addMultTask.metadata.getScalar(name))})",
285 )
286 # Some basic sanity checks
287 currCpuTime = time.process_time()
288 self.assertLessEqual(
289 addMultTask.metadata.getScalar("runStartCpuTime"),
290 addMultTask.metadata.getScalar("runEndCpuTime"),
291 )
292 self.assertLessEqual(addMultTask.metadata.getScalar("runEndCpuTime"), currCpuTime)
293 self.assertLessEqual(
294 addMultTask.metadata.getScalar("contextStartCpuTime"),
295 addMultTask.metadata.getScalar("contextEndCpuTime"),
296 )
297 self.assertLessEqual(addMultTask.metadata.getScalar("contextEndCpuTime"), currCpuTime)
298 self.assertLessEqual(
299 addMultTask.add.metadata.getScalar("runStartCpuTime"),
300 addMultTask.metadata.getScalar("runEndCpuTime"),
301 )
302 self.assertLessEqual(addMultTask.add.metadata.getScalar("runEndCpuTime"), currCpuTime)
304 # Add some explicit values for serialization test.
305 addMultTask.metadata["comment"] = "A comment"
306 addMultTask.metadata["integer"] = 5
307 addMultTask.metadata["float"] = 3.14
308 addMultTask.metadata["bool"] = False
309 addMultTask.metadata.add("commentList", "comment1")
310 addMultTask.metadata.add("commentList", "comment1")
311 addMultTask.metadata.add("intList", 6)
312 addMultTask.metadata.add("intList", 7)
313 addMultTask.metadata.add("boolList", False)
314 addMultTask.metadata.add("boolList", True)
315 addMultTask.metadata.add("floatList", 6.6)
316 addMultTask.metadata.add("floatList", 7.8)
318 # TaskMetadata can serialize to JSON but not YAML
319 # and PropertySet can serialize to YAML and not JSON.
320 if hasattr(addMultTask.metadata, "json"):
321 j = addMultTask.metadata.model_dump_json()
322 new_meta = pipeBase.TaskMetadata.model_validate(json.loads(j))
323 else:
324 y = yaml.dump(addMultTask.metadata)
325 new_meta = yaml.safe_load(y)
326 self.assertEqual(new_meta, addMultTask.metadata)
329class MyMemoryTestCase(lsst.utils.tests.MemoryTestCase):
330 """Run file leak tests."""
333def setup_module(module):
334 """Configure pytest."""
335 lsst.utils.tests.init()
338if __name__ == "__main__":
339 lsst.utils.tests.init()
340 unittest.main()