Coverage for tests/test_task.py: 34%
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1#
2# LSST Data Management System
3# Copyright 2008, 2009, 2010 LSST Corporation.
4#
5# This product includes software developed by the
6# LSST Project (http://www.lsst.org/).
7#
8# This program is free software: you can redistribute it and/or modify
9# it under the terms of the GNU General Public License as published by
10# the Free Software Foundation, either version 3 of the License, or
11# (at your option) any later version.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the LSST License Statement and
19# the GNU General Public License along with this program. If not,
20# see <http://www.lsstcorp.org/LegalNotices/>.
21#
22import logging
23import time
24import unittest
25import numbers
26import json
27import yaml
29import lsst.utils.tests
30import lsst.pex.config as pexConfig
31import lsst.pipe.base as pipeBase
32from lsst.utils.timer import timeMethod
34# Whilst in transition the test can't tell which type is
35# going to be used for metadata.
36from lsst.pipe.base.task import _TASK_METADATA_TYPE
39class AddConfig(pexConfig.Config):
40 addend = pexConfig.Field(doc="amount to add", dtype=float, default=3.1)
43class AddTask(pipeBase.Task):
44 ConfigClass = AddConfig
46 @timeMethod
47 def run(self, val):
48 self.metadata.add("add", self.config.addend)
49 return pipeBase.Struct(
50 val=val + self.config.addend,
51 )
54class MultConfig(pexConfig.Config):
55 multiplicand = pexConfig.Field(doc="amount by which to multiply", dtype=float, default=2.5)
58class MultTask(pipeBase.Task):
59 ConfigClass = MultConfig
61 @timeMethod
62 def run(self, val):
63 self.metadata.add("mult", self.config.multiplicand)
64 return pipeBase.Struct(
65 val=val * self.config.multiplicand,
66 )
69# prove that registry fields can also be used to hold subtasks
70# by using a registry to hold MultTask
71multRegistry = pexConfig.makeRegistry("Registry for Mult-like tasks")
72multRegistry.register("stdMult", MultTask)
75class AddMultConfig(pexConfig.Config):
76 add = AddTask.makeField("add task")
77 mult = multRegistry.makeField("mult task", default="stdMult")
80class AddMultTask(pipeBase.Task):
81 ConfigClass = AddMultConfig
82 _DefaultName = "addMult"
83 _add_module_logger_prefix = False
85 """First add, then multiply"""
87 def __init__(self, **keyArgs):
88 pipeBase.Task.__init__(self, **keyArgs)
89 self.makeSubtask("add")
90 self.makeSubtask("mult")
92 @timeMethod
93 def run(self, val):
94 with self.timer("context"):
95 addRet = self.add.run(val)
96 multRet = self.mult.run(addRet.val)
97 self.metadata.add("addmult", multRet.val)
98 return pipeBase.Struct(
99 val=multRet.val,
100 )
102 @timeMethod
103 def failDec(self):
104 """A method that fails with a decorator
105 """
106 raise RuntimeError("failDec intentional error")
108 def failCtx(self):
109 """A method that fails inside a context manager
110 """
111 with self.timer("failCtx"):
112 raise RuntimeError("failCtx intentional error")
115class AddMultTask2(AddMultTask):
116 """Subclass that gets an automatic logger prefix."""
117 _add_module_logger_prefix = True
120class AddTwiceTask(AddTask):
121 """Variant of AddTask that adds twice the addend"""
123 def run(self, val):
124 addend = self.config.addend
125 return pipeBase.Struct(val=val + (2 * addend))
128class TaskTestCase(unittest.TestCase):
129 """A test case for Task
130 """
132 def setUp(self):
133 self.valDict = dict()
135 def tearDown(self):
136 self.valDict = None
138 def testBasics(self):
139 """Test basic construction and use of a task
140 """
141 for addend in (1.1, -3.5):
142 for multiplicand in (0.9, -45.0):
143 config = AddMultTask.ConfigClass()
144 config.add.addend = addend
145 config.mult["stdMult"].multiplicand = multiplicand
146 # make sure both ways of accessing the registry work and give
147 # the same result
148 self.assertEqual(config.mult.active.multiplicand, multiplicand)
149 addMultTask = AddMultTask(config=config)
150 for val in (-1.0, 0.0, 17.5):
151 ret = addMultTask.run(val=val)
152 self.assertAlmostEqual(ret.val, (val + addend) * multiplicand)
154 def testNames(self):
155 """Test getName() and getFullName()
156 """
157 addMultTask = AddMultTask()
158 self.assertEqual(addMultTask.getName(), "addMult")
159 self.assertEqual(addMultTask.add.getName(), "add")
160 self.assertEqual(addMultTask.mult.getName(), "mult")
162 self.assertEqual(addMultTask._name, "addMult")
163 self.assertEqual(addMultTask.add._name, "add")
164 self.assertEqual(addMultTask.mult._name, "mult")
166 self.assertEqual(addMultTask.getFullName(), "addMult")
167 self.assertEqual(addMultTask.add.getFullName(), "addMult.add")
168 self.assertEqual(addMultTask.mult.getFullName(), "addMult.mult")
170 self.assertEqual(addMultTask._fullName, "addMult")
171 self.assertEqual(addMultTask.add._fullName, "addMult.add")
172 self.assertEqual(addMultTask.mult._fullName, "addMult.mult")
174 def testLog(self):
175 """Test the Task's logger
176 """
177 addMultTask = AddMultTask()
178 self.assertEqual(addMultTask.log.name, "addMult")
179 self.assertEqual(addMultTask.add.log.name, "addMult.add")
181 log = logging.getLogger("tester")
182 addMultTask = AddMultTask(log=log)
183 self.assertEqual(addMultTask.log.name, "tester.addMult")
184 self.assertEqual(addMultTask.add.log.name, "tester.addMult.add")
186 addMultTask2 = AddMultTask2()
187 self.assertEqual(addMultTask2.log.name, f"{__name__}.addMult")
189 def testGetFullMetadata(self):
190 """Test getFullMetadata()
191 """
192 addMultTask = AddMultTask()
193 addMultTask.run(val=1.234) # Add some metadata
194 fullMetadata = addMultTask.getFullMetadata()
195 self.assertIsInstance(fullMetadata["addMult"], _TASK_METADATA_TYPE)
196 self.assertIsInstance(fullMetadata["addMult:add"], _TASK_METADATA_TYPE)
197 self.assertIsInstance(fullMetadata["addMult:mult"], _TASK_METADATA_TYPE)
198 self.assertEqual(set(fullMetadata), {"addMult", "addMult:add", "addMult:mult"})
200 all_names = fullMetadata.names(topLevelOnly=False)
201 self.assertIn("addMult", all_names)
202 self.assertIn("addMult.runStartUtc", all_names)
204 param_names = fullMetadata.paramNames(topLevelOnly=True)
205 # No top level keys without hierarchy
206 self.assertEqual(set(param_names), set())
208 param_names = fullMetadata.paramNames(topLevelOnly=False)
209 self.assertNotIn("addMult", param_names)
210 self.assertIn("addMult.runStartUtc", param_names)
211 self.assertIn("addMult:add.runStartCpuTime", param_names)
213 def testEmptyMetadata(self):
214 task = AddMultTask()
215 task.run(val=1.2345)
216 task.emptyMetadata()
217 fullMetadata = task.getFullMetadata()
218 self.assertEqual(len(fullMetadata["addMult"]), 0)
219 self.assertEqual(len(fullMetadata["addMult:add"]), 0)
220 self.assertEqual(len(fullMetadata["addMult:mult"]), 0)
222 def testReplace(self):
223 """Test replacing one subtask with another
224 """
225 for addend in (1.1, -3.5):
226 for multiplicand in (0.9, -45.0):
227 config = AddMultTask.ConfigClass()
228 config.add.retarget(AddTwiceTask)
229 config.add.addend = addend
230 config.mult["stdMult"].multiplicand = multiplicand
231 addMultTask = AddMultTask(config=config)
232 for val in (-1.0, 0.0, 17.5):
233 ret = addMultTask.run(val=val)
234 self.assertAlmostEqual(ret.val, (val + (2 * addend)) * multiplicand)
236 def testFail(self):
237 """Test timers when the code they are timing fails
238 """
239 addMultTask = AddMultTask()
240 try:
241 addMultTask.failDec()
242 self.fail("Expected RuntimeError")
243 except RuntimeError:
244 self.assertIn("failDecEndCpuTime", addMultTask.metadata)
245 try:
246 addMultTask.failCtx()
247 self.fail("Expected RuntimeError")
248 except RuntimeError:
249 self.assertIn("failCtxEndCpuTime", addMultTask.metadata)
251 def testTimeMethod(self):
252 """Test that the timer is adding the right metadata
253 """
254 addMultTask = AddMultTask()
256 # Run twice to ensure we are additive.
257 addMultTask.run(val=1.1)
258 addMultTask.run(val=2.0)
259 # Check existence and type
260 for key, keyType in (("Utc", str),
261 ("CpuTime", float),
262 ("UserTime", float),
263 ("SystemTime", float),
264 ("MaxResidentSetSize", numbers.Integral),
265 ("MinorPageFaults", numbers.Integral),
266 ("MajorPageFaults", numbers.Integral),
267 ("BlockInputs", numbers.Integral),
268 ("BlockOutputs", numbers.Integral),
269 ("VoluntaryContextSwitches", numbers.Integral),
270 ("InvoluntaryContextSwitches", numbers.Integral),
271 ):
272 for when in ("Start", "End"):
273 for method in ("run", "context"):
274 name = method + when + key
275 self.assertIn(name, addMultTask.metadata,
276 name + " is missing from task metadata")
277 self.assertIsInstance(addMultTask.metadata.getScalar(name), keyType,
278 f"{name} is not of the right type "
279 f"({keyType} vs {type(addMultTask.metadata.getScalar(name))})")
280 # Some basic sanity checks
281 currCpuTime = time.process_time()
282 self.assertLessEqual(
283 addMultTask.metadata.getScalar("runStartCpuTime"),
284 addMultTask.metadata.getScalar("runEndCpuTime"),
285 )
286 self.assertLessEqual(addMultTask.metadata.getScalar("runEndCpuTime"), currCpuTime)
287 self.assertLessEqual(
288 addMultTask.metadata.getScalar("contextStartCpuTime"),
289 addMultTask.metadata.getScalar("contextEndCpuTime"),
290 )
291 self.assertLessEqual(addMultTask.metadata.getScalar("contextEndCpuTime"), currCpuTime)
292 self.assertLessEqual(
293 addMultTask.add.metadata.getScalar("runStartCpuTime"),
294 addMultTask.metadata.getScalar("runEndCpuTime"),
295 )
296 self.assertLessEqual(addMultTask.add.metadata.getScalar("runEndCpuTime"), currCpuTime)
298 # Add some explicit values for serialization test.
299 addMultTask.metadata["comment"] = "A comment"
300 addMultTask.metadata["integer"] = 5
301 addMultTask.metadata["float"] = 3.14
302 addMultTask.metadata["bool"] = False
303 addMultTask.metadata.add("commentList", "comment1")
304 addMultTask.metadata.add("commentList", "comment1")
305 addMultTask.metadata.add("intList", 6)
306 addMultTask.metadata.add("intList", 7)
307 addMultTask.metadata.add("boolList", False)
308 addMultTask.metadata.add("boolList", True)
309 addMultTask.metadata.add("floatList", 6.6)
310 addMultTask.metadata.add("floatList", 7.8)
312 # TaskMetadata can serialize to JSON but not YAML
313 # and PropertySet can serialize to YAML and not JSON.
314 if hasattr(addMultTask.metadata, "json"):
315 j = addMultTask.metadata.json()
316 new_meta = pipeBase.TaskMetadata.parse_obj(json.loads(j))
317 else:
318 y = yaml.dump(addMultTask.metadata)
319 new_meta = yaml.safe_load(y)
320 self.assertEqual(new_meta, addMultTask.metadata)
323class MyMemoryTestCase(lsst.utils.tests.MemoryTestCase):
324 pass
327def setup_module(module):
328 lsst.utils.tests.init()
331if __name__ == "__main__": 331 ↛ 332line 331 didn't jump to line 332, because the condition on line 331 was never true
332 lsst.utils.tests.init()
333 unittest.main()