Coverage for tests/test_cmdLineFwk.py : 22%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of ctrl_mpexec.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22"""Simple unit test for cmdLineFwk module.
23"""
25import argparse
26import contextlib
27import logging
28import os
29import pickle
30import tempfile
31import unittest
33from lsst.ctrl.mpexec.cmdLineFwk import CmdLineFwk
34from lsst.ctrl.mpexec.cmdLineParser import (_ACTION_ADD_TASK, _ACTION_CONFIG,
35 _ACTION_CONFIG_FILE, _ACTION_ADD_INSTRUMENT)
36from lsst.daf.butler import Quantum, Config
37import lsst.pex.config as pexConfig
38from lsst.pipe.base import (Pipeline, PipelineTask, PipelineTaskConfig,
39 QuantumGraph, QuantumGraphTaskNodes,
40 TaskDef, TaskFactory, PipelineTaskConnections)
41import lsst.pipe.base.connectionTypes as cT
42import lsst.utils.tests
43from testUtil import (AddTask, AddTaskFactoryMock, makeSimpleQGraph)
46logging.basicConfig(level=logging.INFO)
49@contextlib.contextmanager
50def makeTmpFile(contents=None):
51 """Context manager for generating temporary file name.
53 Temporary file is deleted on exiting context.
55 Parameters
56 ----------
57 contents : `bytes`
58 Data to write into a file.
59 """
60 fd, tmpname = tempfile.mkstemp()
61 if contents:
62 os.write(fd, contents)
63 os.close(fd)
64 yield tmpname
65 with contextlib.suppress(OSError):
66 os.remove(tmpname)
69class SimpleConnections(PipelineTaskConnections, dimensions=(),
70 defaultTemplates={"template": "simple"}):
71 schema = cT.InitInput(doc="Schema",
72 name="{template}schema",
73 storageClass="SourceCatalog")
76class SimpleConfig(PipelineTaskConfig, pipelineConnections=SimpleConnections):
77 field = pexConfig.Field(dtype=str, doc="arbitrary string")
79 def setDefaults(self):
80 PipelineTaskConfig.setDefaults(self)
83class TaskOne(PipelineTask):
84 ConfigClass = SimpleConfig
85 _DefaultName = "taskOne"
88class TaskTwo(PipelineTask):
89 ConfigClass = SimpleConfig
90 _DefaultName = "taskTwo"
93class TaskFactoryMock(TaskFactory):
94 def loadTaskClass(self, taskName):
95 if taskName == "TaskOne":
96 return TaskOne, "TaskOne"
97 elif taskName == "TaskTwo":
98 return TaskTwo, "TaskTwo"
100 def makeTask(self, taskClass, config, overrides, butler):
101 if config is None:
102 config = taskClass.ConfigClass()
103 if overrides:
104 overrides.applyTo(config)
105 return taskClass(config=config, butler=butler)
108def _makeArgs(pipeline=None, qgraph=None, pipeline_actions=(), order_pipeline=False,
109 save_pipeline="", save_qgraph="", save_single_quanta="", pipeline_dot="", qgraph_dot=""):
110 """Return parsed command line arguments.
112 Parameters
113 ----------
114 pipeline : `str`, optional
115 Name of the YAML file with pipeline.
116 qgraph : `str`, optional
117 Name of the pickle file with QGraph.
118 pipeline_actions : itrable of `cmdLinePArser._PipelineAction`, optional
119 order_pipeline : `bool`
120 save_pipeline : `str`
121 Name of the YAML file to store pipeline.
122 save_qgraph : `str`
123 Name of the pickle file to store QGraph.
124 save_single_quanta : `str`
125 Name of the pickle file pattern to store individual QGraph.
126 pipeline_dot : `str`
127 Name of the DOT file to write pipeline graph.
128 qgraph_dot : `str`
129 Name of the DOT file to write QGraph representation.
130 """
131 args = argparse.Namespace()
132 args.butler_config = Config()
133 # The default datastore has a relocatable root, so we need to specify
134 # some root here for it to use
135 args.butler_config.configFile = "."
136 args.pipeline = pipeline
137 args.qgraph = qgraph
138 args.pipeline_actions = pipeline_actions
139 args.order_pipeline = order_pipeline
140 args.save_pipeline = save_pipeline
141 args.save_qgraph = save_qgraph
142 args.save_single_quanta = save_single_quanta
143 args.pipeline_dot = pipeline_dot
144 args.qgraph_dot = qgraph_dot
145 args.output = {}
146 args.register_dataset_types = False
147 args.skip_init_writes = False
148 args.no_versions = False
149 args.skip_existing = False
150 args.init_only = False
151 args.processes = 1
152 args.profile = None
153 args.enableLsstDebug = False
154 args.graph_fixup = None
155 return args
158def _makeQGraph():
159 """Make a trivial QuantumGraph with one quantum.
161 The only thing that we need to do with this quantum graph is to pickle
162 it, the quanta in this graph are not usable for anything else.
164 Returns
165 -------
166 qgraph : `~lsst.pipe.base.QuantumGraph`
167 """
168 taskDef = TaskDef(taskName="taskOne", config=SimpleConfig())
169 quanta = [Quantum()]
170 taskNodes = QuantumGraphTaskNodes(taskDef=taskDef, quanta=quanta, initInputs={}, initOutputs={})
171 qgraph = QuantumGraph([taskNodes])
172 return qgraph
175class CmdLineFwkTestCase(unittest.TestCase):
176 """A test case for CmdLineFwk
177 """
179 def testMakePipeline(self):
180 """Tests for CmdLineFwk.makePipeline method
181 """
182 fwk = CmdLineFwk()
184 # make empty pipeline
185 args = _makeArgs()
186 pipeline = fwk.makePipeline(args)
187 self.assertIsInstance(pipeline, Pipeline)
188 self.assertEqual(len(pipeline), 0)
190 # few tests with serialization
191 with makeTmpFile() as tmpname:
192 # make empty pipeline and store it in a file
193 args = _makeArgs(save_pipeline=tmpname)
194 pipeline = fwk.makePipeline(args)
195 self.assertIsInstance(pipeline, Pipeline)
197 # read pipeline from a file
198 args = _makeArgs(pipeline=tmpname)
199 pipeline = fwk.makePipeline(args)
200 self.assertIsInstance(pipeline, Pipeline)
201 self.assertEqual(len(pipeline), 0)
203 # single task pipeline
204 actions = [
205 _ACTION_ADD_TASK("TaskOne:task1")
206 ]
207 args = _makeArgs(pipeline_actions=actions)
208 pipeline = fwk.makePipeline(args)
209 self.assertIsInstance(pipeline, Pipeline)
210 self.assertEqual(len(pipeline), 1)
212 # many task pipeline
213 actions = [
214 _ACTION_ADD_TASK("TaskOne:task1a"),
215 _ACTION_ADD_TASK("TaskTwo:task2"),
216 _ACTION_ADD_TASK("TaskOne:task1b")
217 ]
218 args = _makeArgs(pipeline_actions=actions)
219 pipeline = fwk.makePipeline(args)
220 self.assertIsInstance(pipeline, Pipeline)
221 self.assertEqual(len(pipeline), 3)
223 # single task pipeline with config overrides, cannot use TaskOne, need
224 # something that can be imported with `doImport()`
225 actions = [
226 _ACTION_ADD_TASK("testUtil.AddTask:task"),
227 _ACTION_CONFIG("task:addend=100")
228 ]
229 args = _makeArgs(pipeline_actions=actions)
230 pipeline = fwk.makePipeline(args)
231 taskDefs = list(pipeline.toExpandedPipeline())
232 self.assertEqual(len(taskDefs), 1)
233 self.assertEqual(taskDefs[0].config.addend, 100)
235 overrides = b"config.addend = 1000\n"
236 with makeTmpFile(overrides) as tmpname:
237 actions = [
238 _ACTION_ADD_TASK("testUtil.AddTask:task"),
239 _ACTION_CONFIG_FILE("task:" + tmpname)
240 ]
241 args = _makeArgs(pipeline_actions=actions)
242 pipeline = fwk.makePipeline(args)
243 taskDefs = list(pipeline.toExpandedPipeline())
244 self.assertEqual(len(taskDefs), 1)
245 self.assertEqual(taskDefs[0].config.addend, 1000)
247 # Check --instrument option, for now it only checks that it does not crash
248 actions = [
249 _ACTION_ADD_TASK("testUtil.AddTask:task"),
250 _ACTION_ADD_INSTRUMENT("Instrument")
251 ]
252 args = _makeArgs(pipeline_actions=actions)
253 pipeline = fwk.makePipeline(args)
255 def testMakeGraphFromPickle(self):
256 """Tests for CmdLineFwk.makeGraph method.
258 Only most trivial case is tested that does not do actual graph
259 building.
260 """
261 fwk = CmdLineFwk()
263 with makeTmpFile() as tmpname:
265 # make non-empty graph and store it in a file
266 qgraph = _makeQGraph()
267 with open(tmpname, "wb") as pickleFile:
268 pickle.dump(qgraph, pickleFile)
269 args = _makeArgs(qgraph=tmpname)
270 qgraph = fwk.makeGraph(None, args)
271 self.assertIsInstance(qgraph, QuantumGraph)
272 self.assertEqual(len(qgraph), 1)
274 # pickle with wrong object type
275 with open(tmpname, "wb") as pickleFile:
276 pickle.dump({}, pickleFile)
277 args = _makeArgs(qgraph=tmpname)
278 with self.assertRaises(TypeError):
279 fwk.makeGraph(None, args)
281 # reading empty graph from pickle should return None
282 qgraph = QuantumGraph()
283 with open(tmpname, "wb") as pickleFile:
284 pickle.dump(qgraph, pickleFile)
285 args = _makeArgs(qgraph=tmpname)
286 with self.assertWarnsRegex(UserWarning, "QuantumGraph is empty"):
287 # this also tests that warning is generated for empty graph
288 qgraph = fwk.makeGraph(None, args)
289 self.assertIs(qgraph, None)
291 def testSimpleQGraph(self):
292 """Test successfull execution of trivial quantum graph.
293 """
295 nQuanta = 5
296 butler, qgraph = makeSimpleQGraph(nQuanta)
298 # should have one task and number of quanta
299 self.assertEqual(len(qgraph), 1)
300 self.assertEqual(len(list(qgraph.quanta())), nQuanta)
302 args = _makeArgs()
303 fwk = CmdLineFwk()
304 taskFactory = AddTaskFactoryMock()
306 # run whole thing
307 AddTask.countExec = 0
308 fwk.runPipeline(qgraph, taskFactory, args, butler=butler)
309 self.assertEqual(AddTask.countExec, nQuanta)
311 def testSimpleQGraphSkipExisting(self):
312 """Test continuing execution of trivial quantum graph with --skip-existing.
313 """
315 nQuanta = 5
316 butler, qgraph = makeSimpleQGraph(nQuanta)
318 # should have one task and number of quanta
319 self.assertEqual(len(qgraph), 1)
320 self.assertEqual(len(list(qgraph.quanta())), nQuanta)
322 args = _makeArgs()
323 fwk = CmdLineFwk()
324 taskFactory = AddTaskFactoryMock()
326 # run whole thing
327 AddTask.countExec = 0
328 AddTask.stopAt = 3
329 with self.assertRaises(RuntimeError):
330 fwk.runPipeline(qgraph, taskFactory, args, butler=butler)
331 self.assertEqual(AddTask.countExec, 3)
333 AddTask.stopAt = -1
334 args.skip_existing = True
335 args.no_versions = True
336 fwk.runPipeline(qgraph, taskFactory, args, butler=butler)
337 self.assertEqual(AddTask.countExec, nQuanta)
339 def testShowPipeline(self):
340 """Test for --show options for pipeline.
341 """
342 fwk = CmdLineFwk()
344 actions = [
345 _ACTION_ADD_TASK("testUtil.AddTask:task"),
346 _ACTION_CONFIG("task:addend=100")
347 ]
348 args = _makeArgs(pipeline_actions=actions)
349 pipeline = fwk.makePipeline(args)
351 args.show = ["pipeline"]
352 fwk.showInfo(args, pipeline)
353 args.show = ["config"]
354 fwk.showInfo(args, pipeline)
355 args.show = ["history=task::addend"]
356 fwk.showInfo(args, pipeline)
357 args.show = ["tasks"]
358 fwk.showInfo(args, pipeline)
360 def testShowGraph(self):
361 """Test for --show options for quantum graph.
362 """
363 fwk = CmdLineFwk()
365 nQuanta = 2
366 butler, qgraph = makeSimpleQGraph(nQuanta)
368 args = _makeArgs()
369 args.show = ["graph"]
370 fwk.showInfo(args, pipeline=None, graph=qgraph)
371 # TODO: cannot test "workflow" option presently, it instanciates
372 # butler from command line options and there is no way to pass butler
373 # mock to that code.
376class MyMemoryTestCase(lsst.utils.tests.MemoryTestCase):
377 pass
380def setup_module(module):
381 lsst.utils.tests.init()
384if __name__ == "__main__": 384 ↛ 385line 384 didn't jump to line 385, because the condition on line 384 was never true
385 lsst.utils.tests.init()
386 unittest.main()