Coverage for tests/test_cmdLineFwk.py : 23%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
# This file is part of ctrl_mpexec. # # Developed for the LSST Data Management System. # This product includes software developed by the LSST Project # (https://www.lsst.org). # See the COPYRIGHT file at the top-level directory of this distribution # for details of code ownership. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
QuantumGraph, QuantumGraphTaskNodes, TaskDef, TaskFactory, PipelineTaskConnections)
"""Context manager for generating temporary file name.
Temporary file is deleted on exiting context.
Parameters ---------- contents : `bytes` Data to write into a file. """ fd, tmpname = tempfile.mkstemp() if contents: os.write(fd, contents) os.close(fd) yield tmpname with contextlib.suppress(OSError): os.remove(tmpname)
defaultTemplates={"template": "simple"}): name="{template}schema", storageClass="SourceCatalog")
PipelineTaskConfig.setDefaults(self)
if taskName == "TaskOne": return TaskOne, "TaskOne" elif taskName == "TaskTwo": return TaskTwo, "TaskTwo"
if config is None: config = taskClass.ConfigClass() if overrides: overrides.applyTo(config) return taskClass(config=config, butler=butler)
save_pipeline="", save_qgraph="", save_single_quanta="", pipeline_dot="", qgraph_dot=""): """Return parsed command line arguments.
Parameters ---------- pipeline : `str`, optional Name of the YAML file with pipeline. qgraph : `str`, optional Name of the pickle file with QGraph. pipeline_actions : itrable of `cmdLinePArser._PipelineAction`, optional order_pipeline : `bool` save_pipeline : `str` Name of the YAML file to store pipeline. save_qgraph : `str` Name of the pickle file to store QGraph. save_single_quanta : `str` Name of the pickle file pattern to store individual QGraph. pipeline_dot : `str` Name of the DOT file to write pipeline graph. qgraph_dot : `str` Name of the DOT file to write QGrpah representation. """ args = argparse.Namespace() args.butler_config = None args.pipeline = pipeline args.qgraph = qgraph args.pipeline_actions = pipeline_actions args.order_pipeline = order_pipeline args.save_pipeline = save_pipeline args.save_qgraph = save_qgraph args.save_single_quanta = save_single_quanta args.pipeline_dot = pipeline_dot args.qgraph_dot = qgraph_dot args.output = {} args.register_dataset_types = False args.skip_init_writes = False args.skip_existing = False args.clobber_output = False args.init_only = False args.processes = 1 args.profile = None return args
"""Make a trivial QuantumGraph with one quantum.
The only thing that we need to do with this quantum graph is to pickle it, the quanta in this graph are not usable for anything else.
Returns ------- qgraph : `~lsst.pipe.base.QuantumGraph` """ taskDef = TaskDef(taskName="taskOne", config=SimpleConfig()) quanta = [Quantum()] taskNodes = QuantumGraphTaskNodes(taskDef=taskDef, quanta=quanta, initInputs={}, initOutputs={}) qgraph = QuantumGraph([taskNodes]) return qgraph
"""A test case for CmdLineFwk """
"""Tests for CmdLineFwk.makePipeline method """ fwk = CmdLineFwk()
# make empty pipeline args = _makeArgs() pipeline = fwk.makePipeline(args) self.assertIsInstance(pipeline, Pipeline) self.assertEqual(len(pipeline), 0)
# few tests with serialization with makeTmpFile() as tmpname: # make empty pipeline and store it in a file args = _makeArgs(save_pipeline=tmpname) pipeline = fwk.makePipeline(args) self.assertIsInstance(pipeline, Pipeline)
# read pipeline from a file args = _makeArgs(pipeline=tmpname) pipeline = fwk.makePipeline(args) self.assertIsInstance(pipeline, Pipeline) self.assertEqual(len(pipeline), 0)
# single task pipeline actions = [ _PipelineAction(action="new_task", label="task1", value="TaskOne") ] args = _makeArgs(pipeline_actions=actions) pipeline = fwk.makePipeline(args) self.assertIsInstance(pipeline, Pipeline) self.assertEqual(len(pipeline), 1)
# many task pipeline actions = [ _PipelineAction(action="new_task", label="task1a", value="TaskOne"), _PipelineAction(action="new_task", label="task2", value="TaskTwo"), _PipelineAction(action="new_task", label="task1b", value="TaskOne") ] args = _makeArgs(pipeline_actions=actions) pipeline = fwk.makePipeline(args) self.assertIsInstance(pipeline, Pipeline) self.assertEqual(len(pipeline), 3)
# single task pipeline with config overrides, cannot use TaskOne, need # something that can be imported with `doImport()` actions = [ _PipelineAction(action="new_task", label="task", value="testUtil.AddTask"), _PipelineAction(action="config", label="task", value="addend=100") ] args = _makeArgs(pipeline_actions=actions) pipeline = fwk.makePipeline(args) taskDefs = list(pipeline.toExpandedPipeline()) self.assertEqual(len(taskDefs), 1) self.assertEqual(taskDefs[0].config.addend, 100)
overrides = b"config.addend = 1000\n" with makeTmpFile(overrides) as tmpname: actions = [ _PipelineAction(action="new_task", label="task", value="testUtil.AddTask"), _PipelineAction(action="configfile", label="task", value=tmpname) ] args = _makeArgs(pipeline_actions=actions) pipeline = fwk.makePipeline(args) taskDefs = list(pipeline.toExpandedPipeline()) self.assertEqual(len(taskDefs), 1) self.assertEqual(taskDefs[0].config.addend, 1000)
"""Tests for CmdLineFwk.makeGraph method.
Only most trivial case is tested that does not do actual graph building. """ fwk = CmdLineFwk()
with makeTmpFile() as tmpname:
# make non-empty graph and store it in a file qgraph = _makeQGraph() with open(tmpname, "wb") as pickleFile: pickle.dump(qgraph, pickleFile) args = _makeArgs(qgraph=tmpname) qgraph = fwk.makeGraph(None, args) self.assertIsInstance(qgraph, QuantumGraph) self.assertEqual(len(qgraph), 1)
# pickle with wrong object type with open(tmpname, "wb") as pickleFile: pickle.dump({}, pickleFile) args = _makeArgs(qgraph=tmpname) with self.assertRaises(TypeError): fwk.makeGraph(None, args)
# reading empty graph from pickle should return None qgraph = QuantumGraph() with open(tmpname, "wb") as pickleFile: pickle.dump(qgraph, pickleFile) args = _makeArgs(qgraph=tmpname) with self.assertWarnsRegex(UserWarning, "QuantumGraph is empty"): # this also tests that warning is generated for empty graph qgraph = fwk.makeGraph(None, args) self.assertIs(qgraph, None)
"""Test successfull execution of trivial quantum graph. """
nQuanta = 5 butler, qgraph = makeSimpleQGraph(nQuanta)
# should have one task and number of quanta self.assertEqual(len(qgraph), 1) self.assertEqual(len(list(qgraph.quanta())), nQuanta)
args = _makeArgs() fwk = CmdLineFwk() taskFactory = AddTaskFactoryMock()
# run whole thing AddTask.countExec = 0 fwk.runPipeline(qgraph, taskFactory, args, butler=butler) self.assertEqual(AddTask.countExec, nQuanta)
"""Test continuing execution of trivial quantum graph with --skip-existing. """
nQuanta = 5 butler, qgraph = makeSimpleQGraph(nQuanta)
# should have one task and number of quanta self.assertEqual(len(qgraph), 1) self.assertEqual(len(list(qgraph.quanta())), nQuanta)
args = _makeArgs() fwk = CmdLineFwk() taskFactory = AddTaskFactoryMock()
# run whole thing AddTask.countExec = 0 AddTask.stopAt = 3 with self.assertRaises(RuntimeError): fwk.runPipeline(qgraph, taskFactory, args, butler=butler) self.assertEqual(AddTask.countExec, 3)
AddTask.stopAt = -1 args.skip_existing = True fwk.runPipeline(qgraph, taskFactory, args, butler=butler) self.assertEqual(AddTask.countExec, nQuanta)
"""Test re-execution of trivial quantum graph with --clobber-output. """
nQuanta = 5 butler, qgraph = makeSimpleQGraph(nQuanta)
# should have one task and number of quanta self.assertEqual(len(qgraph), 1) self.assertEqual(qgraph.countQuanta(), nQuanta)
args = _makeArgs() fwk = CmdLineFwk() taskFactory = AddTaskFactoryMock()
# run whole thing AddTask.stopAt = -1 AddTask.countExec = 0 fwk.runPipeline(qgraph, taskFactory, args, butler=butler) self.assertEqual(AddTask.countExec, nQuanta)
# and repeat args.clobber_output = True fwk.runPipeline(qgraph, taskFactory, args, butler=butler) self.assertEqual(AddTask.countExec, 2*nQuanta)
# rebuild graph with clobber option, should make same graph butler, qgraph = makeSimpleQGraph(nQuanta, butler=butler, clobberExisting=True) self.assertEqual(len(qgraph), 1) self.assertEqual(qgraph.countQuanta(), nQuanta)
lsst.utils.tests.init()
lsst.utils.tests.init() unittest.main() |