Coverage for python/lsst/ctrl/mpexec/cmdLineFwk.py : 12%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of ctrl_mpexec.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22"""Module defining CmdLineFwk class and related methods.
23"""
25__all__ = ['CmdLineFwk']
27# -------------------------------
28# Imports of standard modules --
29# -------------------------------
30import argparse
31import fnmatch
32import logging
33import re
34import sys
35from typing import List, Optional, Tuple
36import warnings
38# -----------------------------
39# Imports for other modules --
40# -----------------------------
41from lsst.daf.butler import (
42 Butler,
43 CollectionSearch,
44 CollectionType,
45 DatasetTypeRestriction,
46 Registry,
47)
48from lsst.daf.butler.registry import MissingCollectionError
49import lsst.pex.config as pexConfig
50from lsst.pipe.base import GraphBuilder, Pipeline, QuantumGraph
51from lsst.obs.base import Instrument
52from .dotTools import graph2dot, pipeline2dot
53from .executionGraphFixup import ExecutionGraphFixup
54from .mpGraphExecutor import MPGraphExecutor
55from .preExecInit import PreExecInit
56from .singleQuantumExecutor import SingleQuantumExecutor
57from . import util
58from lsst.utils import doImport
60# ----------------------------------
61# Local non-exported definitions --
62# ----------------------------------
64_LOG = logging.getLogger(__name__.partition(".")[2])
67class _OutputChainedCollectionInfo:
68 """A helper class for handling command-line arguments related to an output
69 `~lsst.daf.butler.CollectionType.CHAINED` collection.
71 Parameters
72 ----------
73 registry : `lsst.daf.butler.Registry`
74 Butler registry that collections will be added to and/or queried from.
75 name : `str`
76 Name of the collection given on the command line.
77 """
78 def __init__(self, registry: Registry, name: str):
79 self.name = name
80 try:
81 self.chain = list(registry.getCollectionChain(name))
82 self.exists = True
83 except MissingCollectionError:
84 self.chain = []
85 self.exists = False
87 def __str__(self):
88 return self.name
90 name: str
91 """Name of the collection provided on the command line (`str`).
92 """
94 exists: bool
95 """Whether this collection already exists in the registry (`bool`).
96 """
98 chain: List[str]
99 """The definition of the collection, if it already exists (`list`).
101 Empty if the collection does not alredy exist.
102 """
105class _OutputRunCollectionInfo:
106 """A helper class for handling command-line arguments related to an output
107 `~lsst.daf.butler.CollectionType.RUN` collection.
109 Parameters
110 ----------
111 registry : `lsst.daf.butler.Registry`
112 Butler registry that collections will be added to and/or queried from.
113 name : `str`
114 Name of the collection given on the command line.
115 """
116 def __init__(self, registry: Registry, name: str):
117 self.name = name
118 try:
119 actualType = registry.getCollectionType(name)
120 if actualType is not CollectionType.RUN:
121 raise TypeError(f"Collection '{name}' exists but has type {actualType.name}, not RUN.")
122 self.exists = True
123 except MissingCollectionError:
124 self.exists = False
126 name: str
127 """Name of the collection provided on the command line (`str`).
128 """
130 exists: bool
131 """Whether this collection already exists in the registry (`bool`).
132 """
135class _ButlerFactory:
136 """A helper class for processing command-line arguments related to input
137 and output collections.
139 Parameters
140 ----------
141 registry : `lsst.daf.butler.Registry`
142 Butler registry that collections will be added to and/or queried from.
144 args : `argparse.Namespace`
145 Parsed command-line arguments. The following attributes are used,
146 either at construction or in later methods.
148 ``output``
149 The name of a `~lsst.daf.butler.CollectionType.CHAINED`
150 input/output collection.
152 ``output_run``
153 The name of a `~lsst.daf.butler.CollectionType.RUN` input/output
154 collection.
156 ``extend_run``
157 A boolean indicating whether ``output_run`` should already exist
158 and be extended.
160 ``replace_run``
161 A boolean indicating that (if `True`) ``output_run`` should already
162 exist but will be removed from the output chained collection and
163 replaced with a new one.
165 ``prune_replaced``
166 A boolean indicating whether to prune the replaced run (requires
167 ``replace_run``).
169 ``inputs``
170 Input collections of any type; may be any type handled by
171 `lsst.daf.butler.registry.CollectionSearch.fromExpression`.
173 ``butler_config``
174 Path to a data repository root or configuration file.
176 writeable : `bool`
177 If `True`, a `Butler` is being initialized in a context where actual
178 writes should happens, and hence no output run is necessary.
180 Raises
181 ------
182 ValueError
183 Raised if ``writeable is True`` but there are no output collections.
184 """
185 def __init__(self, registry: Registry, args: argparse.Namespace, writeable: bool):
186 if args.output is not None:
187 self.output = _OutputChainedCollectionInfo(registry, args.output)
188 else:
189 self.output = None
190 if args.output_run is not None:
191 self.outputRun = _OutputRunCollectionInfo(registry, args.output_run)
192 elif self.output is not None:
193 if args.extend_run:
194 runName, _ = self.output.chain[0]
195 else:
196 runName = "{}/{}".format(self.output, Instrument.makeCollectionTimestamp())
197 self.outputRun = _OutputRunCollectionInfo(registry, runName)
198 elif not writeable:
199 # If we're not writing yet, ok to have no output run.
200 self.outputRun = None
201 else:
202 raise ValueError("Cannot write without at least one of (--output, --output-run).")
203 self.inputs = list(CollectionSearch.fromExpression(args.input)) if args.input else []
205 def check(self, args: argparse.Namespace):
206 """Check command-line options for consistency with each other and the
207 data repository.
209 Parameters
210 ----------
211 args : `argparse.Namespace`
212 Parsed command-line arguments. See class documentation for the
213 construction parameter of the same name.
214 """
215 assert not (args.extend_run and args.replace_run), "In mutually-exclusive group in ArgumentParser."
216 if self.inputs and self.output is not None and self.output.exists:
217 raise ValueError("Cannot use --output with existing collection with --inputs.")
218 if args.extend_run and self.outputRun is None:
219 raise ValueError("Cannot --extend-run when no output collection is given.")
220 if args.extend_run and not self.outputRun.exists:
221 raise ValueError(f"Cannot --extend-run; output collection "
222 f"'{self.outputRun.name}' does not exist.")
223 if not args.extend_run and self.outputRun is not None and self.outputRun.exists:
224 raise ValueError(f"Output run '{self.outputRun.name}' already exists, but "
225 f"--extend-run was not given.")
226 if args.prune_replaced and not args.replace_run:
227 raise ValueError("--prune-replaced requires --replace-run.")
228 if args.replace_run and (self.output is None or not self.output.exists):
229 raise ValueError("--output must point to an existing CHAINED collection for --replace-run.")
231 @classmethod
232 def _makeReadParts(cls, args: argparse.Namespace):
233 """Common implementation for `makeReadButler` and
234 `makeRegistryAndCollections`.
236 Parameters
237 ----------
238 args : `argparse.Namespace`
239 Parsed command-line arguments. See class documentation for the
240 construction parameter of the same name.
242 Returns
243 -------
244 butler : `lsst.daf.butler.Butler`
245 A read-only butler constructed from the repo at
246 ``args.butler_config``, but with no default collections.
247 inputs : `lsst.daf.butler.registry.CollectionSearch`
248 A collection search path constructed according to ``args``.
249 self : `_ButlerFactory`
250 A new `_ButlerFactory` instance representing the processed version
251 of ``args``.
252 """
253 butler = Butler(args.butler_config, writeable=False)
254 self = cls(butler.registry, args, writeable=False)
255 self.check(args)
256 if self.output and self.output.exists:
257 if args.replace_run:
258 replaced = self.output.chain[0]
259 inputs = self.output.chain[1:]
260 _LOG.debug("Simulating collection search in '%s' after removing '%s'.",
261 self.output.name, replaced)
262 else:
263 inputs = [self.output.name]
264 else:
265 inputs = list(self.inputs)
266 if args.extend_run:
267 inputs.insert(0, self.outputRun.name)
268 inputs = CollectionSearch.fromExpression(inputs)
269 return butler, inputs, self
271 @classmethod
272 def makeReadButler(cls, args: argparse.Namespace) -> Butler:
273 """Construct a read-only butler according to the given command-line
274 arguments.
276 Parameters
277 ----------
278 args : `argparse.Namespace`
279 Parsed command-line arguments. See class documentation for the
280 construction parameter of the same name.
282 Returns
283 -------
284 butler : `lsst.daf.butler.Butler`
285 A read-only butler initialized with the collections specified by
286 ``args``.
287 """
288 butler, inputs, _ = cls._makeReadParts(args)
289 _LOG.debug("Preparing butler to read from %s.", inputs)
290 return Butler(butler=butler, collections=inputs)
292 @classmethod
293 def makeRegistryAndCollections(cls, args: argparse.Namespace) -> \
294 Tuple[Registry, CollectionSearch, Optional[str]]:
295 """Return a read-only registry, a collection search path, and the name
296 of the run to be used for future writes.
298 Parameters
299 ----------
300 args : `argparse.Namespace`
301 Parsed command-line arguments. See class documentation for the
302 construction parameter of the same name.
304 Returns
305 -------
306 registry : `lsst.daf.butler.Registry`
307 Butler registry that collections will be added to and/or queried
308 from.
309 inputs : `lsst.daf.butler.registry.CollectionSearch`
310 Collections to search for datasets.
311 run : `str` or `None`
312 Name of the output `~lsst.daf.butler.CollectionType.RUN` collection
313 if it already exists, or `None` if it does not.
314 """
315 butler, inputs, self = cls._makeReadParts(args)
316 run = self.outputRun.name if args.extend_run else None
317 _LOG.debug("Preparing registry to read from %s and expect future writes to '%s'.", inputs, run)
318 return butler.registry, inputs, run
320 @classmethod
321 def makeWriteButler(cls, args: argparse.Namespace) -> Butler:
322 """Return a read-write butler initialized to write to and read from
323 the collections specified by the given command-line arguments.
325 Parameters
326 ----------
327 args : `argparse.Namespace`
328 Parsed command-line arguments. See class documentation for the
329 construction parameter of the same name.
331 Returns
332 -------
333 butler : `lsst.daf.butler.Butler`
334 A read-write butler initialized according to the given arguments.
335 """
336 butler = Butler(args.butler_config, writeable=True)
337 self = cls(butler.registry, args, writeable=True)
338 self.check(args)
339 if self.output is not None:
340 chainDefinition = list(self.output.chain if self.output.exists else self.inputs)
341 if args.replace_run:
342 replaced = chainDefinition.pop(0)
343 if args.prune_replaced == "unstore":
344 # Remove datasets from datastore
345 with butler.transaction():
346 refs = butler.registry.queryDatasets(..., collections=replaced)
347 butler.pruneDatasets(refs, unstore=True, run=replaced, disassociate=False)
348 elif args.prune_replaced == "purge":
349 # Erase entire collection and all datasets, need to remove
350 # collection from its chain collection first.
351 with butler.transaction():
352 butler.registry.setCollectionChain(self.output.name, chainDefinition)
353 butler.pruneCollection(replaced, purge=True, unstore=True)
354 elif args.prune_replaced is not None:
355 raise NotImplementedError(
356 f"Unsupported --prune-replaced option '{args.prune_replaced}'."
357 )
358 chainDefinition.insert(0, self.outputRun.name)
359 chainDefinition = CollectionSearch.fromExpression(chainDefinition)
360 _LOG.debug("Preparing butler to write to '%s' and read from '%s'=%s",
361 self.outputRun.name, self.output.name, chainDefinition)
362 return Butler(butler=butler, run=self.outputRun.name, collections=self.output.name,
363 chains={self.output.name: chainDefinition})
364 else:
365 inputs = CollectionSearch.fromExpression([self.outputRun.name] + self.inputs)
366 _LOG.debug("Preparing butler to write to '%s' and read from %s.", self.outputRun.name, inputs)
367 return Butler(butler=butler, run=self.outputRun.name, collections=inputs)
369 output: Optional[_OutputChainedCollectionInfo]
370 """Information about the output chained collection, if there is or will be
371 one (`_OutputChainedCollectionInfo` or `None`).
372 """
374 outputRun: Optional[_OutputRunCollectionInfo]
375 """Information about the output run collection, if there is or will be
376 one (`_OutputRunCollectionInfo` or `None`).
377 """
379 inputs: List[Tuple[str, DatasetTypeRestriction]]
380 """Input collections, including those also used for outputs and any
381 restrictions on dataset types (`list`).
382 """
385class _FilteredStream:
386 """A file-like object that filters some config fields.
388 Note
389 ----
390 This class depends on implementation details of ``Config.saveToStream``
391 methods, in particular that that method uses single call to write()
392 method to save information about single config field, and that call
393 combines comments string(s) for a field and field path and value.
394 This class will not work reliably on the "import" strings, so imports
395 should be disabled by passing ``skipImports=True`` to ``saveToStream()``.
396 """
397 def __init__(self, pattern):
398 # obey case if pattern isn't lowercase or requests NOIGNORECASE
399 mat = re.search(r"(.*):NOIGNORECASE$", pattern)
401 if mat:
402 pattern = mat.group(1)
403 self._pattern = re.compile(fnmatch.translate(pattern))
404 else:
405 if pattern != pattern.lower():
406 print(f"Matching \"{pattern}\" without regard to case "
407 "(append :NOIGNORECASE to prevent this)", file=sys.stdout)
408 self._pattern = re.compile(fnmatch.translate(pattern), re.IGNORECASE)
410 def write(self, showStr):
411 # Strip off doc string line(s) and cut off at "=" for string matching
412 matchStr = showStr.rstrip().split("\n")[-1].split("=")[0]
413 if self._pattern.search(matchStr):
414 sys.stdout.write(showStr)
416# ------------------------
417# Exported definitions --
418# ------------------------
421class CmdLineFwk:
422 """PipelineTask framework which executes tasks from command line.
424 In addition to executing tasks this activator provides additional methods
425 for task management like dumping configuration or execution chain.
426 """
428 MP_TIMEOUT = 9999 # Default timeout (sec) for multiprocessing
430 def __init__(self):
431 pass
433 def makePipeline(self, args):
434 """Build a pipeline from command line arguments.
436 Parameters
437 ----------
438 args : `argparse.Namespace`
439 Parsed command line
441 Returns
442 -------
443 pipeline : `~lsst.pipe.base.Pipeline`
444 """
445 if args.pipeline:
446 pipeline = Pipeline.fromFile(args.pipeline)
447 else:
448 pipeline = Pipeline("anonymous")
450 # loop over all pipeline actions and apply them in order
451 for action in args.pipeline_actions:
452 if action.action == "add_instrument":
454 pipeline.addInstrument(action.value)
456 elif action.action == "new_task":
458 pipeline.addTask(action.value, action.label)
460 elif action.action == "delete_task":
462 pipeline.removeTask(action.label)
464 elif action.action == "config":
466 # action value string is "field=value", split it at '='
467 field, _, value = action.value.partition("=")
468 pipeline.addConfigOverride(action.label, field, value)
470 elif action.action == "configfile":
472 pipeline.addConfigFile(action.label, action.value)
474 else:
476 raise ValueError(f"Unexpected pipeline action: {action.action}")
478 if args.save_pipeline:
479 pipeline.toFile(args.save_pipeline)
481 if args.pipeline_dot:
482 pipeline2dot(pipeline, args.pipeline_dot)
484 return pipeline
486 def makeGraph(self, pipeline, args):
487 """Build a graph from command line arguments.
489 Parameters
490 ----------
491 pipeline : `~lsst.pipe.base.Pipeline`
492 Pipeline, can be empty or ``None`` if graph is read from a file.
493 args : `argparse.Namespace`
494 Parsed command line
496 Returns
497 -------
498 graph : `~lsst.pipe.base.QuantumGraph` or `None`
499 If resulting graph is empty then `None` is returned.
500 """
502 registry, collections, run = _ButlerFactory.makeRegistryAndCollections(args)
504 if args.qgraph:
505 qgraph = QuantumGraph.loadUri(args.qgraph, registry.dimensions)
507 # pipeline can not be provided in this case
508 if pipeline:
509 raise ValueError("Pipeline must not be given when quantum graph is read from file.")
511 else:
513 # make execution plan (a.k.a. DAG) for pipeline
514 graphBuilder = GraphBuilder(registry,
515 skipExisting=args.skip_existing)
516 qgraph = graphBuilder.makeGraph(pipeline, collections, run, args.data_query)
518 # count quanta in graph and give a warning if it's empty and return None
519 nQuanta = len(qgraph)
520 if nQuanta == 0:
521 warnings.warn("QuantumGraph is empty", stacklevel=2)
522 return None
523 else:
524 _LOG.info("QuantumGraph contains %d quanta for %d tasks",
525 nQuanta, len(qgraph.taskGraph))
527 if args.save_qgraph:
528 qgraph.saveUri(args.save_qgraph)
530 if args.save_single_quanta:
531 for quantumNode in qgraph:
532 sqgraph = qgraph.subset(quantumNode)
533 uri = args.save_single_quanta.format(quantumNode.nodeId.number)
534 sqgraph.saveUri(uri)
536 if args.qgraph_dot:
537 graph2dot(qgraph, args.qgraph_dot)
539 return qgraph
541 def runPipeline(self, graph, taskFactory, args, butler=None):
542 """Execute complete QuantumGraph.
544 Parameters
545 ----------
546 graph : `QuantumGraph`
547 Execution graph.
548 taskFactory : `~lsst.pipe.base.TaskFactory`
549 Task factory
550 args : `argparse.Namespace`
551 Parsed command line
552 butler : `~lsst.daf.butler.Butler`, optional
553 Data Butler instance, if not defined then new instance is made
554 using command line options.
555 """
556 # make butler instance
557 if butler is None:
558 butler = _ButlerFactory.makeWriteButler(args)
560 # Enable lsstDebug debugging. Note that this is done once in the
561 # main process before PreExecInit and it is also repeated before
562 # running each task in SingleQuantumExecutor (which may not be
563 # needed if `multipocessing` always uses fork start method).
564 if args.enableLsstDebug:
565 try:
566 _LOG.debug("Will try to import debug.py")
567 import debug # noqa:F401
568 except ImportError:
569 _LOG.warn("No 'debug' module found.")
571 preExecInit = PreExecInit(butler, taskFactory, args.skip_existing)
572 preExecInit.initialize(graph,
573 saveInitOutputs=not args.skip_init_writes,
574 registerDatasetTypes=args.register_dataset_types,
575 saveVersions=not args.no_versions)
577 if not args.init_only:
578 graphFixup = self._importGraphFixup(args)
579 quantumExecutor = SingleQuantumExecutor(taskFactory,
580 skipExisting=args.skip_existing,
581 clobberPartialOutputs=args.clobber_partial_outputs,
582 enableLsstDebug=args.enableLsstDebug)
583 timeout = self.MP_TIMEOUT if args.timeout is None else args.timeout
584 executor = MPGraphExecutor(numProc=args.processes, timeout=timeout,
585 startMethod=args.start_method,
586 quantumExecutor=quantumExecutor,
587 failFast=args.fail_fast,
588 executionGraphFixup=graphFixup)
589 with util.profile(args.profile, _LOG):
590 executor.execute(graph, butler)
592 def showInfo(self, args, pipeline, graph=None):
593 """Display useful info about pipeline and environment.
595 Parameters
596 ----------
597 args : `argparse.Namespace`
598 Parsed command line
599 pipeline : `Pipeline`
600 Pipeline definition
601 graph : `QuantumGraph`, optional
602 Execution graph
603 """
604 showOpts = args.show
605 for what in showOpts:
606 showCommand, _, showArgs = what.partition("=")
608 if showCommand in ["pipeline", "config", "history", "tasks"]:
609 if not pipeline:
610 _LOG.warning("Pipeline is required for --show=%s", showCommand)
611 continue
613 if showCommand in ["graph", "workflow", "uri"]:
614 if not graph:
615 _LOG.warning("QuantumGraph is required for --show=%s", showCommand)
616 continue
618 if showCommand == "pipeline":
619 print(pipeline)
620 elif showCommand == "config":
621 self._showConfig(pipeline, showArgs, False)
622 elif showCommand == "dump-config":
623 self._showConfig(pipeline, showArgs, True)
624 elif showCommand == "history":
625 self._showConfigHistory(pipeline, showArgs)
626 elif showCommand == "tasks":
627 self._showTaskHierarchy(pipeline)
628 elif showCommand == "graph":
629 if graph:
630 self._showGraph(graph)
631 elif showCommand == "uri":
632 if graph:
633 self._showUri(graph, args)
634 elif showCommand == "workflow":
635 if graph:
636 self._showWorkflow(graph, args)
637 else:
638 print("Unknown value for show: %s (choose from '%s')" %
639 (what, "', '".join("pipeline config[=XXX] history=XXX tasks graph".split())),
640 file=sys.stderr)
641 sys.exit(1)
643 def _showConfig(self, pipeline, showArgs, dumpFullConfig):
644 """Show task configuration
646 Parameters
647 ----------
648 pipeline : `Pipeline`
649 Pipeline definition
650 showArgs : `str`
651 Defines what to show
652 dumpFullConfig : `bool`
653 If true then dump complete task configuration with all imports.
654 """
655 stream = sys.stdout
656 if dumpFullConfig:
657 # Task label can be given with this option
658 taskName = showArgs
659 else:
660 # The argument can have form [TaskLabel::][pattern:NOIGNORECASE]
661 matConfig = re.search(r"^(?:(\w+)::)?(?:config.)?(.+)?", showArgs)
662 taskName = matConfig.group(1)
663 pattern = matConfig.group(2)
664 if pattern:
665 stream = _FilteredStream(pattern)
667 tasks = util.filterTasks(pipeline, taskName)
668 if not tasks:
669 print("Pipeline has no tasks named {}".format(taskName), file=sys.stderr)
670 sys.exit(1)
672 for taskDef in tasks:
673 print("### Configuration for task `{}'".format(taskDef.label))
674 taskDef.config.saveToStream(stream, root="config", skipImports=not dumpFullConfig)
676 def _showConfigHistory(self, pipeline, showArgs):
677 """Show history for task configuration
679 Parameters
680 ----------
681 pipeline : `Pipeline`
682 Pipeline definition
683 showArgs : `str`
684 Defines what to show
685 """
687 taskName = None
688 pattern = None
689 matHistory = re.search(r"^(?:(\w+)::)?(?:config[.])?(.+)", showArgs)
690 if matHistory:
691 taskName = matHistory.group(1)
692 pattern = matHistory.group(2)
693 if not pattern:
694 print("Please provide a value with --show history (e.g. history=Task::param)", file=sys.stderr)
695 sys.exit(1)
697 tasks = util.filterTasks(pipeline, taskName)
698 if not tasks:
699 print(f"Pipeline has no tasks named {taskName}", file=sys.stderr)
700 sys.exit(1)
702 found = False
703 for taskDef in tasks:
705 config = taskDef.config
707 # Look for any matches in the config hierarchy for this name
708 for nmatch, thisName in enumerate(fnmatch.filter(config.names(), pattern)):
709 if nmatch > 0:
710 print("")
712 cpath, _, cname = thisName.rpartition(".")
713 try:
714 if not cpath:
715 # looking for top-level field
716 hconfig = taskDef.config
717 else:
718 hconfig = eval("config." + cpath, {}, {"config": config})
719 except AttributeError:
720 print(f"Error: Unable to extract attribute {cpath} from task {taskDef.label}",
721 file=sys.stderr)
722 hconfig = None
724 # Sometimes we end up with a non-Config so skip those
725 if isinstance(hconfig, (pexConfig.Config, pexConfig.ConfigurableInstance)) and \
726 hasattr(hconfig, cname):
727 print(f"### Configuration field for task `{taskDef.label}'")
728 print(pexConfig.history.format(hconfig, cname))
729 found = True
731 if not found:
732 print(f"None of the tasks has field matching {pattern}", file=sys.stderr)
733 sys.exit(1)
735 def _showTaskHierarchy(self, pipeline):
736 """Print task hierarchy to stdout
738 Parameters
739 ----------
740 pipeline: `Pipeline`
741 """
742 for taskDef in pipeline.toExpandedPipeline():
743 print("### Subtasks for task `{}'".format(taskDef.taskName))
745 for configName, taskName in util.subTaskIter(taskDef.config):
746 print("{}: {}".format(configName, taskName))
748 def _showGraph(self, graph):
749 """Print quanta information to stdout
751 Parameters
752 ----------
753 graph : `QuantumGraph`
754 Execution graph.
755 """
756 for taskNode in graph.taskGraph:
757 print(taskNode)
759 for iq, quantum in enumerate(graph.getQuantaForTask(taskNode)):
760 print(" Quantum {}:".format(iq))
761 print(" inputs:")
762 for key, refs in quantum.inputs.items():
763 dataIds = ["DataId({})".format(ref.dataId) for ref in refs]
764 print(" {}: [{}]".format(key, ", ".join(dataIds)))
765 print(" outputs:")
766 for key, refs in quantum.outputs.items():
767 dataIds = ["DataId({})".format(ref.dataId) for ref in refs]
768 print(" {}: [{}]".format(key, ", ".join(dataIds)))
770 def _showWorkflow(self, graph, args):
771 """Print quanta information and dependency to stdout
773 Parameters
774 ----------
775 graph : `QuantumGraph`
776 Execution graph.
777 args : `argparse.Namespace`
778 Parsed command line
779 """
780 for node in graph:
781 print(f"Quantum {node.nodeId.number}: {node.taskDef.taskName}")
782 for parent in graph.determineInputsToQuantumNode(node):
783 print(f"Parent Quantum {parent.nodeId.number} - Child Quantum {node.nodeId.number}")
785 def _showUri(self, graph, args):
786 """Print input and predicted output URIs to stdout
788 Parameters
789 ----------
790 graph : `QuantumGraph`
791 Execution graph
792 args : `argparse.Namespace`
793 Parsed command line
794 """
795 def dumpURIs(thisRef):
796 primary, components = butler.getURIs(thisRef, predict=True, run="TBD")
797 if primary:
798 print(f" {primary}")
799 else:
800 print(" (disassembled artifact)")
801 for compName, compUri in components.items():
802 print(f" {compName}: {compUri}")
804 butler = _ButlerFactory.makeReadButler(args)
805 for node in graph:
806 print(f"Quantum {node.nodeId.number}: {node.taskDef.taskName}")
807 print(" inputs:")
808 for key, refs in node.quantum.inputs.items():
809 for ref in refs:
810 dumpURIs(ref)
811 print(" outputs:")
812 for key, refs in node.quantum.outputs.items():
813 for ref in refs:
814 dumpURIs(ref)
816 def _importGraphFixup(self, args):
817 """Import/instantiate graph fixup object.
819 Parameters
820 ----------
821 args : `argparse.Namespace`
822 Parsed command line.
824 Returns
825 -------
826 fixup : `ExecutionGraphFixup` or `None`
828 Raises
829 ------
830 ValueError
831 Raised if import fails, method call raises exception, or returned
832 instance has unexpected type.
833 """
834 if args.graph_fixup:
835 try:
836 factory = doImport(args.graph_fixup)
837 except Exception as exc:
838 raise ValueError("Failed to import graph fixup class/method") from exc
839 try:
840 fixup = factory()
841 except Exception as exc:
842 raise ValueError("Failed to make instance of graph fixup") from exc
843 if not isinstance(fixup, ExecutionGraphFixup):
844 raise ValueError("Graph fixup is not an instance of ExecutionGraphFixup class")
845 return fixup