Coverage for tests/test_cmdLineFwk.py: 20%
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of ctrl_mpexec.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22"""Simple unit test for cmdLineFwk module.
23"""
25import click
26from types import SimpleNamespace
27import contextlib
28import copy
29from dataclasses import dataclass
30import logging
31import os
32import pickle
33import re
34import shutil
35import tempfile
36from typing import NamedTuple
37import unittest
39from lsst.ctrl.mpexec.cmdLineFwk import CmdLineFwk
40from lsst.ctrl.mpexec.cli.opt import run_options
41from lsst.ctrl.mpexec.cli.utils import (
42 _ACTION_ADD_TASK,
43 _ACTION_CONFIG,
44 _ACTION_CONFIG_FILE,
45 _ACTION_ADD_INSTRUMENT,
46 PipetaskCommand,
47)
48from lsst.daf.butler import Config, Quantum, Registry
49from lsst.daf.butler.registry import RegistryConfig
50from lsst.obs.base import Instrument
51import lsst.pex.config as pexConfig
52from lsst.pipe.base import (Pipeline, PipelineTaskConfig, QuantumGraph, TaskDef, PipelineTaskConnections)
53from lsst.pipe.base.graphBuilder import DatasetQueryConstraintVariant as DQCVariant
54import lsst.pipe.base.connectionTypes as cT
55import lsst.utils.tests
56from lsst.pipe.base.tests.simpleQGraph import (
57 AddTaskFactoryMock,
58 makeSimpleButler,
59 makeSimplePipeline,
60 makeSimpleQGraph,
61 populateButler)
62from lsst.utils.tests import temporaryDirectory
65logging.basicConfig(level=getattr(logging, os.environ.get("UNIT_TEST_LOGGING_LEVEL", "INFO"), logging.INFO))
67# Have to monkey-patch Instrument.fromName() to not retrieve non-existing
68# instrument from registry, these tests can run fine without actual instrument
69# and implementing full mock for Instrument is too complicated.
70Instrument.fromName = lambda name, reg: None 70 ↛ exitline 70 didn't run the lambda on line 70
73@contextlib.contextmanager
74def makeTmpFile(contents=None, suffix=None):
75 """Context manager for generating temporary file name.
77 Temporary file is deleted on exiting context.
79 Parameters
80 ----------
81 contents : `bytes`
82 Data to write into a file.
83 """
84 fd, tmpname = tempfile.mkstemp(suffix=suffix)
85 if contents:
86 os.write(fd, contents)
87 os.close(fd)
88 yield tmpname
89 with contextlib.suppress(OSError):
90 os.remove(tmpname)
93@contextlib.contextmanager
94def makeSQLiteRegistry(create=True):
95 """Context manager to create new empty registry database.
97 Yields
98 ------
99 config : `RegistryConfig`
100 Registry configuration for initialized registry database.
101 """
102 with temporaryDirectory() as tmpdir:
103 uri = f"sqlite:///{tmpdir}/gen3.sqlite"
104 config = RegistryConfig()
105 config["db"] = uri
106 if create:
107 Registry.createFromConfig(config)
108 yield config
111class SimpleConnections(PipelineTaskConnections, dimensions=(),
112 defaultTemplates={"template": "simple"}):
113 schema = cT.InitInput(doc="Schema",
114 name="{template}schema",
115 storageClass="SourceCatalog")
118class SimpleConfig(PipelineTaskConfig, pipelineConnections=SimpleConnections):
119 field = pexConfig.Field(dtype=str, doc="arbitrary string")
121 def setDefaults(self):
122 PipelineTaskConfig.setDefaults(self)
125def _makeArgs(registryConfig=None, **kwargs):
126 """Return parsed command line arguments.
128 By default butler_config is set to `Config` populated with some defaults,
129 it can be overridden completely by keyword argument.
131 Parameters
132 ----------
133 cmd : `str`, optional
134 Produce arguments for this pipetask command.
135 registryConfig : `RegistryConfig`, optional
136 Override for registry configuration.
137 **kwargs
138 Overrides for other arguments.
139 """
140 # Use a mock to get the default value of arguments to 'run'.
142 mock = unittest.mock.Mock()
144 @click.command(cls=PipetaskCommand)
145 @run_options()
146 def fake_run(ctx, **kwargs):
147 """Fake "pipetask run" command for gathering input arguments.
149 The arguments & options should always match the arguments & options in
150 the "real" command function `lsst.ctrl.mpexec.cli.cmd.run`.
151 """
152 mock(**kwargs)
154 runner = click.testing.CliRunner()
155 # --butler-config is the only required option
156 result = runner.invoke(fake_run, "--butler-config /")
157 if result.exit_code != 0:
158 raise RuntimeError(f"Failure getting default args from 'fake_run': {result}")
159 mock.assert_called_once()
160 args = mock.call_args[1]
161 args["enableLsstDebug"] = args.pop("debug")
162 args["execution_butler_location"] = args.pop("save_execution_butler")
163 if "pipeline_actions" not in args:
164 args["pipeline_actions"] = []
165 args = SimpleNamespace(**args)
167 # override butler_config with our defaults
168 if "butler_config" not in kwargs:
169 args.butler_config = Config()
170 if registryConfig:
171 args.butler_config["registry"] = registryConfig
172 # The default datastore has a relocatable root, so we need to specify
173 # some root here for it to use
174 args.butler_config.configFile = "."
176 # override arguments from keyword parameters
177 for key, value in kwargs.items():
178 setattr(args, key, value)
179 args.dataset_query_constraint = DQCVariant.fromExpression(args.dataset_query_constraint)
180 return args
183class FakeDSType(NamedTuple):
184 name: str
187@dataclass(frozen=True)
188class FakeDSRef:
189 datasetType: str
190 dataId: tuple
192 def isComponent(self):
193 return False
196# Task class name used by tests, needs to be importable
197_TASK_CLASS = "lsst.pipe.base.tests.simpleQGraph.AddTask"
200def _makeQGraph():
201 """Make a trivial QuantumGraph with one quantum.
203 The only thing that we need to do with this quantum graph is to pickle
204 it, the quanta in this graph are not usable for anything else.
206 Returns
207 -------
208 qgraph : `~lsst.pipe.base.QuantumGraph`
209 """
210 taskDef = TaskDef(taskName=_TASK_CLASS, config=SimpleConfig(), label="test")
211 quanta = [Quantum(taskName=_TASK_CLASS,
212 inputs={FakeDSType("A"): [FakeDSRef("A", (1, 2))]})] # type: ignore
213 qgraph = QuantumGraph({taskDef: set(quanta)})
214 return qgraph
217class CmdLineFwkTestCase(unittest.TestCase):
218 """A test case for CmdLineFwk
219 """
221 def testMakePipeline(self):
222 """Tests for CmdLineFwk.makePipeline method
223 """
224 fwk = CmdLineFwk()
226 # make empty pipeline
227 args = _makeArgs()
228 pipeline = fwk.makePipeline(args)
229 self.assertIsInstance(pipeline, Pipeline)
230 self.assertEqual(len(pipeline), 0)
232 # few tests with serialization
233 with makeTmpFile() as tmpname:
234 # make empty pipeline and store it in a file
235 args = _makeArgs(save_pipeline=tmpname)
236 pipeline = fwk.makePipeline(args)
237 self.assertIsInstance(pipeline, Pipeline)
239 # read pipeline from a file
240 args = _makeArgs(pipeline=tmpname)
241 pipeline = fwk.makePipeline(args)
242 self.assertIsInstance(pipeline, Pipeline)
243 self.assertEqual(len(pipeline), 0)
245 # single task pipeline, task name can be anything here
246 actions = [
247 _ACTION_ADD_TASK("TaskOne:task1")
248 ]
249 args = _makeArgs(pipeline_actions=actions)
250 pipeline = fwk.makePipeline(args)
251 self.assertIsInstance(pipeline, Pipeline)
252 self.assertEqual(len(pipeline), 1)
254 # many task pipeline
255 actions = [
256 _ACTION_ADD_TASK("TaskOne:task1a"),
257 _ACTION_ADD_TASK("TaskTwo:task2"),
258 _ACTION_ADD_TASK("TaskOne:task1b")
259 ]
260 args = _makeArgs(pipeline_actions=actions)
261 pipeline = fwk.makePipeline(args)
262 self.assertIsInstance(pipeline, Pipeline)
263 self.assertEqual(len(pipeline), 3)
265 # single task pipeline with config overrides, need real task class
266 actions = [
267 _ACTION_ADD_TASK(f"{_TASK_CLASS}:task"),
268 _ACTION_CONFIG("task:addend=100")
269 ]
270 args = _makeArgs(pipeline_actions=actions)
271 pipeline = fwk.makePipeline(args)
272 taskDefs = list(pipeline.toExpandedPipeline())
273 self.assertEqual(len(taskDefs), 1)
274 self.assertEqual(taskDefs[0].config.addend, 100)
276 overrides = b"config.addend = 1000\n"
277 with makeTmpFile(overrides) as tmpname:
278 actions = [
279 _ACTION_ADD_TASK(f"{_TASK_CLASS}:task"),
280 _ACTION_CONFIG_FILE("task:" + tmpname)
281 ]
282 args = _makeArgs(pipeline_actions=actions)
283 pipeline = fwk.makePipeline(args)
284 taskDefs = list(pipeline.toExpandedPipeline())
285 self.assertEqual(len(taskDefs), 1)
286 self.assertEqual(taskDefs[0].config.addend, 1000)
288 # Check --instrument option, for now it only checks that it does not
289 # crash.
290 actions = [
291 _ACTION_ADD_TASK(f"{_TASK_CLASS}:task"),
292 _ACTION_ADD_INSTRUMENT("Instrument")
293 ]
294 args = _makeArgs(pipeline_actions=actions)
295 pipeline = fwk.makePipeline(args)
297 def testMakeGraphFromSave(self):
298 """Tests for CmdLineFwk.makeGraph method.
300 Only most trivial case is tested that does not do actual graph
301 building.
302 """
303 fwk = CmdLineFwk()
305 with makeTmpFile(suffix=".qgraph") as tmpname, makeSQLiteRegistry() as registryConfig:
307 # make non-empty graph and store it in a file
308 qgraph = _makeQGraph()
309 with open(tmpname, "wb") as saveFile:
310 qgraph.save(saveFile)
311 args = _makeArgs(qgraph=tmpname, registryConfig=registryConfig, execution_butler_location=None)
312 qgraph = fwk.makeGraph(None, args)
313 self.assertIsInstance(qgraph, QuantumGraph)
314 self.assertEqual(len(qgraph), 1)
316 # will fail if graph id does not match
317 args = _makeArgs(
318 qgraph=tmpname,
319 qgraph_id="R2-D2 is that you?",
320 registryConfig=registryConfig,
321 execution_butler_location=None
322 )
323 with self.assertRaisesRegex(ValueError, "graphID does not match"):
324 fwk.makeGraph(None, args)
326 # save with wrong object type
327 with open(tmpname, "wb") as saveFile:
328 pickle.dump({}, saveFile)
329 args = _makeArgs(qgraph=tmpname, registryConfig=registryConfig, execution_butler_location=None)
330 with self.assertRaises(ValueError):
331 fwk.makeGraph(None, args)
333 # reading empty graph from pickle should work but makeGraph()
334 # will return None and make a warning
335 qgraph = QuantumGraph(dict())
336 with open(tmpname, "wb") as saveFile:
337 qgraph.save(saveFile)
338 args = _makeArgs(qgraph=tmpname, registryConfig=registryConfig, execution_butler_location=None)
339 with self.assertWarnsRegex(UserWarning, "QuantumGraph is empty"):
340 # this also tests that warning is generated for empty graph
341 qgraph = fwk.makeGraph(None, args)
342 self.assertIs(qgraph, None)
344 def testShowPipeline(self):
345 """Test for --show options for pipeline.
346 """
347 fwk = CmdLineFwk()
349 actions = [
350 _ACTION_ADD_TASK(f"{_TASK_CLASS}:task"),
351 _ACTION_CONFIG("task:addend=100")
352 ]
353 args = _makeArgs(pipeline_actions=actions)
354 pipeline = fwk.makePipeline(args)
356 args.show = ["pipeline"]
357 fwk.showInfo(args, pipeline)
358 args.show = ["config"]
359 fwk.showInfo(args, pipeline)
360 args.show = ["history=task::addend"]
361 fwk.showInfo(args, pipeline)
362 args.show = ["tasks"]
363 fwk.showInfo(args, pipeline)
366class CmdLineFwkTestCaseWithButler(unittest.TestCase):
367 """A test case for CmdLineFwk
368 """
370 def setUp(self):
371 super().setUpClass()
372 self.root = tempfile.mkdtemp()
373 self.nQuanta = 5
374 self.pipeline = makeSimplePipeline(nQuanta=self.nQuanta)
376 def tearDown(self):
377 shutil.rmtree(self.root, ignore_errors=True)
378 super().tearDownClass()
380 def testSimpleQGraph(self):
381 """Test successfull execution of trivial quantum graph.
382 """
383 args = _makeArgs(butler_config=self.root, input="test", output="output")
384 butler = makeSimpleButler(self.root, run=args.input, inMemory=False)
385 populateButler(self.pipeline, butler)
387 fwk = CmdLineFwk()
388 taskFactory = AddTaskFactoryMock()
390 qgraph = fwk.makeGraph(self.pipeline, args)
391 self.assertEqual(len(qgraph.taskGraph), self.nQuanta)
392 self.assertEqual(len(qgraph), self.nQuanta)
394 # run whole thing
395 fwk.runPipeline(qgraph, taskFactory, args)
396 self.assertEqual(taskFactory.countExec, self.nQuanta)
398 def testSimpleQGraphNoSkipExisting_inputs(self):
399 """Test for case when output data for one task already appears in
400 _input_ collection, but no ``--extend-run`` or ``-skip-existing``
401 option is present.
402 """
403 args = _makeArgs(
404 butler_config=self.root,
405 input="test",
406 output="output",
407 )
408 butler = makeSimpleButler(self.root, run=args.input, inMemory=False)
409 populateButler(
410 self.pipeline, butler,
411 datasetTypes={args.input: [
412 "add_dataset0",
413 "add_dataset1", "add2_dataset1",
414 "add_init_output1",
415 "task0_config",
416 "task0_metadata",
417 "task0_log",
418 ]}
419 )
421 fwk = CmdLineFwk()
422 taskFactory = AddTaskFactoryMock()
424 qgraph = fwk.makeGraph(self.pipeline, args)
425 self.assertEqual(len(qgraph.taskGraph), self.nQuanta)
426 # With current implementation graph has all nQuanta quanta, but when
427 # executing one quantum is skipped.
428 self.assertEqual(len(qgraph), self.nQuanta)
430 # run whole thing
431 fwk.runPipeline(qgraph, taskFactory, args)
432 self.assertEqual(taskFactory.countExec, self.nQuanta)
434 def testSimpleQGraphSkipExisting_inputs(self):
435 """Test for ``--skip-existing`` with output data for one task already
436 appears in _input_ collection. No ``--extend-run`` option is needed
437 for this case.
438 """
439 args = _makeArgs(
440 butler_config=self.root,
441 input="test",
442 output="output",
443 skip_existing_in=("test", ),
444 )
445 butler = makeSimpleButler(self.root, run=args.input, inMemory=False)
446 populateButler(
447 self.pipeline, butler,
448 datasetTypes={args.input: [
449 "add_dataset0",
450 "add_dataset1", "add2_dataset1",
451 "add_init_output1",
452 "task0_config",
453 "task0_metadata",
454 "task0_log",
455 ]}
456 )
458 fwk = CmdLineFwk()
459 taskFactory = AddTaskFactoryMock()
461 qgraph = fwk.makeGraph(self.pipeline, args)
462 self.assertEqual(len(qgraph.taskGraph), self.nQuanta)
463 self.assertEqual(len(qgraph), self.nQuanta - 1)
465 # run whole thing
466 fwk.runPipeline(qgraph, taskFactory, args)
467 self.assertEqual(taskFactory.countExec, self.nQuanta - 1)
469 def testSimpleQGraphSkipExisting_outputs(self):
470 """Test for ``--skip-existing`` with output data for one task already
471 appears in _output_ collection. The ``--extend-run`` option is needed
472 for this case.
473 """
474 args = _makeArgs(
475 butler_config=self.root,
476 input="test",
477 output_run="output/run",
478 skip_existing_in=("output/run", ),
479 )
480 butler = makeSimpleButler(self.root, run=args.input, inMemory=False)
481 populateButler(
482 self.pipeline, butler, datasetTypes={
483 args.input: ["add_dataset0"],
484 args.output_run: [
485 "add_dataset1", "add2_dataset1",
486 "add_init_output1",
487 "task0_metadata",
488 "task0_log",
489 ]
490 }
491 )
493 fwk = CmdLineFwk()
494 taskFactory = AddTaskFactoryMock()
496 # fails without --extend-run
497 with self.assertRaisesRegex(ValueError, "--extend-run was not given"):
498 qgraph = fwk.makeGraph(self.pipeline, args)
500 # retry with --extend-run
501 args.extend_run = True
502 qgraph = fwk.makeGraph(self.pipeline, args)
504 self.assertEqual(len(qgraph.taskGraph), self.nQuanta)
505 # Graph does not include quantum for first task
506 self.assertEqual(len(qgraph), self.nQuanta - 1)
508 # run whole thing
509 fwk.runPipeline(qgraph, taskFactory, args)
510 self.assertEqual(taskFactory.countExec, self.nQuanta - 1)
512 def testSimpleQGraphOutputsFail(self):
513 """Test continuing execution of trivial quantum graph with partial
514 outputs.
515 """
516 args = _makeArgs(butler_config=self.root, input="test", output="output")
517 butler = makeSimpleButler(self.root, run=args.input, inMemory=False)
518 populateButler(self.pipeline, butler)
520 fwk = CmdLineFwk()
521 taskFactory = AddTaskFactoryMock(stopAt=3)
523 qgraph = fwk.makeGraph(self.pipeline, args)
524 self.assertEqual(len(qgraph), self.nQuanta)
526 # run first three quanta
527 with self.assertRaises(RuntimeError):
528 fwk.runPipeline(qgraph, taskFactory, args)
529 self.assertEqual(taskFactory.countExec, 3)
531 butler.registry.refresh()
533 # drop one of the two outputs from one task
534 ref1 = butler.registry.findDataset("add2_dataset2", collections=args.output,
535 instrument="INSTR", detector=0)
536 self.assertIsNotNone(ref1)
537 # also drop the metadata output
538 ref2 = butler.registry.findDataset("task1_metadata", collections=args.output,
539 instrument="INSTR", detector=0)
540 self.assertIsNotNone(ref2)
541 butler.pruneDatasets([ref1, ref2], disassociate=True, unstore=True, purge=True)
543 taskFactory.stopAt = -1
544 args.skip_existing_in = (args.output, )
545 args.extend_run = True
546 args.no_versions = True
547 excRe = "Registry inconsistency while checking for existing outputs.*"
548 with self.assertRaisesRegex(RuntimeError, excRe):
549 fwk.runPipeline(qgraph, taskFactory, args)
551 def testSimpleQGraphClobberOutputs(self):
552 """Test continuing execution of trivial quantum graph with
553 --clobber-outputs.
554 """
555 args = _makeArgs(butler_config=self.root, input="test", output="output")
556 butler = makeSimpleButler(self.root, run=args.input, inMemory=False)
557 populateButler(self.pipeline, butler)
559 fwk = CmdLineFwk()
560 taskFactory = AddTaskFactoryMock(stopAt=3)
562 qgraph = fwk.makeGraph(self.pipeline, args)
564 # should have one task and number of quanta
565 self.assertEqual(len(qgraph), self.nQuanta)
567 # run first three quanta
568 with self.assertRaises(RuntimeError):
569 fwk.runPipeline(qgraph, taskFactory, args)
570 self.assertEqual(taskFactory.countExec, 3)
572 butler.registry.refresh()
574 # drop one of the two outputs from one task
575 ref1 = butler.registry.findDataset("add2_dataset2", collections=args.output,
576 dataId=dict(instrument="INSTR", detector=0))
577 self.assertIsNotNone(ref1)
578 # also drop the metadata output
579 ref2 = butler.registry.findDataset("task1_metadata", collections=args.output,
580 dataId=dict(instrument="INSTR", detector=0))
581 self.assertIsNotNone(ref2)
582 butler.pruneDatasets([ref1, ref2], disassociate=True, unstore=True, purge=True)
584 taskFactory.stopAt = -1
585 args.skip_existing = True
586 args.extend_run = True
587 args.clobber_outputs = True
588 args.no_versions = True
589 fwk.runPipeline(qgraph, taskFactory, args)
590 # number of executed quanta is incremented
591 self.assertEqual(taskFactory.countExec, self.nQuanta + 1)
593 def testSimpleQGraphReplaceRun(self):
594 """Test repeated execution of trivial quantum graph with
595 --replace-run.
596 """
597 args = _makeArgs(
598 butler_config=self.root,
599 input="test",
600 output="output",
601 output_run="output/run1")
602 butler = makeSimpleButler(self.root, run=args.input, inMemory=False)
603 populateButler(self.pipeline, butler)
605 fwk = CmdLineFwk()
606 taskFactory = AddTaskFactoryMock()
608 qgraph = fwk.makeGraph(self.pipeline, args)
610 # should have one task and number of quanta
611 self.assertEqual(len(qgraph), self.nQuanta)
613 # deep copy is needed because quanta are updated in place
614 fwk.runPipeline(copy.deepcopy(qgraph), taskFactory, args)
615 self.assertEqual(taskFactory.countExec, self.nQuanta)
617 # need to refresh collections explicitly (or make new butler/registry)
618 butler.registry.refresh()
619 collections = set(butler.registry.queryCollections(...))
620 self.assertEqual(collections, {"test", "output", "output/run1"})
622 # number of datasets written by pipeline:
623 # - nQuanta of init_outputs
624 # - nQuanta of configs
625 # - packages (single dataset)
626 # - nQuanta * two output datasets
627 # - nQuanta of metadata
628 # - nQuanta of log output
629 n_outputs = self.nQuanta * 6 + 1
630 refs = butler.registry.queryDatasets(..., collections="output/run1")
631 self.assertEqual(len(list(refs)), n_outputs)
633 # re-run with --replace-run (--inputs is ignored, as long as it hasn't
634 # changed)
635 args.replace_run = True
636 args.output_run = "output/run2"
637 fwk.runPipeline(copy.deepcopy(qgraph), taskFactory, args)
639 butler.registry.refresh()
640 collections = set(butler.registry.queryCollections(...))
641 self.assertEqual(collections, {"test", "output", "output/run1", "output/run2"})
643 # new output collection
644 refs = butler.registry.queryDatasets(..., collections="output/run2")
645 self.assertEqual(len(list(refs)), n_outputs)
647 # old output collection is still there
648 refs = butler.registry.queryDatasets(..., collections="output/run1")
649 self.assertEqual(len(list(refs)), n_outputs)
651 # re-run with --replace-run and --prune-replaced=unstore
652 args.replace_run = True
653 args.prune_replaced = "unstore"
654 args.output_run = "output/run3"
655 fwk.runPipeline(copy.deepcopy(qgraph), taskFactory, args)
657 butler.registry.refresh()
658 collections = set(butler.registry.queryCollections(...))
659 self.assertEqual(collections, {"test", "output", "output/run1", "output/run2", "output/run3"})
661 # new output collection
662 refs = butler.registry.queryDatasets(..., collections="output/run3")
663 self.assertEqual(len(list(refs)), n_outputs)
665 # old output collection is still there, and it has all datasets but
666 # non-InitOutputs are not in datastore
667 refs = butler.registry.queryDatasets(..., collections="output/run2")
668 refs = list(refs)
669 self.assertEqual(len(refs), n_outputs)
670 initOutNameRe = re.compile("packages|task.*_config|add_init_output.*")
671 for ref in refs:
672 if initOutNameRe.fullmatch(ref.datasetType.name):
673 butler.get(ref, collections="output/run2")
674 else:
675 with self.assertRaises(FileNotFoundError):
676 butler.get(ref, collections="output/run2")
678 # re-run with --replace-run and --prune-replaced=purge
679 # This time also remove --input; passing the same inputs that we
680 # started with and not passing inputs at all should be equivalent.
681 args.input = None
682 args.replace_run = True
683 args.prune_replaced = "purge"
684 args.output_run = "output/run4"
685 fwk.runPipeline(copy.deepcopy(qgraph), taskFactory, args)
687 butler.registry.refresh()
688 collections = set(butler.registry.queryCollections(...))
689 # output/run3 should disappear now
690 self.assertEqual(collections, {"test", "output", "output/run1", "output/run2", "output/run4"})
692 # new output collection
693 refs = butler.registry.queryDatasets(..., collections="output/run4")
694 self.assertEqual(len(list(refs)), n_outputs)
696 # Trying to run again with inputs that aren't exactly what we started
697 # with is an error, and the kind that should not modify the data repo.
698 with self.assertRaises(ValueError):
699 args.input = ["test", "output/run2"]
700 args.prune_replaced = None
701 args.replace_run = True
702 args.output_run = "output/run5"
703 fwk.runPipeline(copy.deepcopy(qgraph), taskFactory, args)
704 butler.registry.refresh()
705 collections = set(butler.registry.queryCollections(...))
706 self.assertEqual(collections, {"test", "output", "output/run1", "output/run2", "output/run4"})
707 with self.assertRaises(ValueError):
708 args.input = ["output/run2", "test"]
709 args.prune_replaced = None
710 args.replace_run = True
711 args.output_run = "output/run6"
712 fwk.runPipeline(copy.deepcopy(qgraph), taskFactory, args)
713 butler.registry.refresh()
714 collections = set(butler.registry.queryCollections(...))
715 self.assertEqual(collections, {"test", "output", "output/run1", "output/run2", "output/run4"})
717 def testSubgraph(self):
718 """Test successfull execution of trivial quantum graph.
719 """
720 args = _makeArgs(butler_config=self.root, input="test", output="output")
721 butler = makeSimpleButler(self.root, run=args.input, inMemory=False)
722 populateButler(self.pipeline, butler)
724 fwk = CmdLineFwk()
725 qgraph = fwk.makeGraph(self.pipeline, args)
727 # Select first two nodes for execution. This depends on node ordering
728 # which I assume is the same as execution order.
729 nNodes = 2
730 nodeIds = [node.nodeId.number for node in qgraph]
731 nodeIds = nodeIds[:nNodes]
733 self.assertEqual(len(qgraph.taskGraph), self.nQuanta)
734 self.assertEqual(len(qgraph), self.nQuanta)
736 with makeTmpFile(suffix=".qgraph") as tmpname, makeSQLiteRegistry() as registryConfig:
737 with open(tmpname, "wb") as saveFile:
738 qgraph.save(saveFile)
740 args = _makeArgs(qgraph=tmpname, qgraph_node_id=nodeIds, registryConfig=registryConfig,
741 execution_butler_location=None)
742 fwk = CmdLineFwk()
744 # load graph, should only read a subset
745 qgraph = fwk.makeGraph(pipeline=None, args=args)
746 self.assertEqual(len(qgraph), nNodes)
748 def testShowGraph(self):
749 """Test for --show options for quantum graph.
750 """
751 fwk = CmdLineFwk()
753 nQuanta = 2
754 butler, qgraph = makeSimpleQGraph(nQuanta, root=self.root)
756 args = _makeArgs(show=["graph"])
757 fwk.showInfo(args, pipeline=None, graph=qgraph)
759 def testShowGraphWorkflow(self):
760 fwk = CmdLineFwk()
762 nQuanta = 2
763 butler, qgraph = makeSimpleQGraph(nQuanta, root=self.root)
765 args = _makeArgs(show=["workflow"])
766 fwk.showInfo(args, pipeline=None, graph=qgraph)
768 # TODO: cannot test "uri" option presently, it instanciates
769 # butler from command line options and there is no way to pass butler
770 # mock to that code.
773class MyMemoryTestCase(lsst.utils.tests.MemoryTestCase):
774 pass
777def setup_module(module):
778 lsst.utils.tests.init()
781if __name__ == "__main__": 781 ↛ 782line 781 didn't jump to line 782, because the condition on line 781 was never true
782 lsst.utils.tests.init()
783 unittest.main()