Coverage for python/lsst/ctrl/mpexec/cmdLineFwk.py : 12%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of ctrl_mpexec.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22"""Module defining CmdLineFwk class and related methods.
23"""
25__all__ = ['CmdLineFwk']
27# -------------------------------
28# Imports of standard modules --
29# -------------------------------
30import argparse
31import fnmatch
32import logging
33import re
34import sys
35from typing import List, Optional, Tuple
36import warnings
38# -----------------------------
39# Imports for other modules --
40# -----------------------------
41from lsst.daf.butler import (
42 Butler,
43 CollectionSearch,
44 CollectionType,
45 DatasetTypeRestriction,
46 Registry,
47)
48from lsst.daf.butler.registry import MissingCollectionError
49import lsst.pex.config as pexConfig
50from lsst.pipe.base import GraphBuilder, Pipeline, QuantumGraph
51from lsst.obs.base import Instrument
52from .dotTools import graph2dot, pipeline2dot
53from .executionGraphFixup import ExecutionGraphFixup
54from .mpGraphExecutor import MPGraphExecutor
55from .preExecInit import PreExecInit
56from .singleQuantumExecutor import SingleQuantumExecutor
57from . import util
58from lsst.utils import doImport
60# ----------------------------------
61# Local non-exported definitions --
62# ----------------------------------
64_LOG = logging.getLogger(__name__.partition(".")[2])
67class _OutputChainedCollectionInfo:
68 """A helper class for handling command-line arguments related to an output
69 `~lsst.daf.butler.CollectionType.CHAINED` collection.
71 Parameters
72 ----------
73 registry : `lsst.daf.butler.Registry`
74 Butler registry that collections will be added to and/or queried from.
75 name : `str`
76 Name of the collection given on the command line.
77 """
78 def __init__(self, registry: Registry, name: str):
79 self.name = name
80 try:
81 self.chain = list(registry.getCollectionChain(name))
82 self.exists = True
83 except MissingCollectionError:
84 self.chain = []
85 self.exists = False
87 def __str__(self):
88 return self.name
90 name: str
91 """Name of the collection provided on the command line (`str`).
92 """
94 exists: bool
95 """Whether this collection already exists in the registry (`bool`).
96 """
98 chain: List[str]
99 """The definition of the collection, if it already exists (`list`).
101 Empty if the collection does not alredy exist.
102 """
105class _OutputRunCollectionInfo:
106 """A helper class for handling command-line arguments related to an output
107 `~lsst.daf.butler.CollectionType.RUN` collection.
109 Parameters
110 ----------
111 registry : `lsst.daf.butler.Registry`
112 Butler registry that collections will be added to and/or queried from.
113 name : `str`
114 Name of the collection given on the command line.
115 """
116 def __init__(self, registry: Registry, name: str):
117 self.name = name
118 try:
119 actualType = registry.getCollectionType(name)
120 if actualType is not CollectionType.RUN:
121 raise TypeError(f"Collection '{name}' exists but has type {actualType.name}, not RUN.")
122 self.exists = True
123 except MissingCollectionError:
124 self.exists = False
126 name: str
127 """Name of the collection provided on the command line (`str`).
128 """
130 exists: bool
131 """Whether this collection already exists in the registry (`bool`).
132 """
135class _ButlerFactory:
136 """A helper class for processing command-line arguments related to input
137 and output collections.
139 Parameters
140 ----------
141 registry : `lsst.daf.butler.Registry`
142 Butler registry that collections will be added to and/or queried from.
144 args : `argparse.Namespace`
145 Parsed command-line arguments. The following attributes are used,
146 either at construction or in later methods.
148 ``output``
149 The name of a `~lsst.daf.butler.CollectionType.CHAINED`
150 input/output collection.
152 ``output_run``
153 The name of a `~lsst.daf.butler.CollectionType.RUN` input/output
154 collection.
156 ``extend_run``
157 A boolean indicating whether ``output_run`` should already exist
158 and be extended.
160 ``replace_run``
161 A boolean indicating that (if `True`) ``output_run`` should already
162 exist but will be removed from the output chained collection and
163 replaced with a new one.
165 ``prune_replaced``
166 A boolean indicating whether to prune the replaced run (requires
167 ``replace_run``).
169 ``inputs``
170 Input collections of any type; may be any type handled by
171 `lsst.daf.butler.registry.CollectionSearch.fromExpression`.
173 ``butler_config``
174 Path to a data repository root or configuration file.
176 writeable : `bool`
177 If `True`, a `Butler` is being initialized in a context where actual
178 writes should happens, and hence no output run is necessary.
180 Raises
181 ------
182 ValueError
183 Raised if ``writeable is True`` but there are no output collections.
184 """
185 def __init__(self, registry: Registry, args: argparse.Namespace, writeable: bool):
186 if args.output is not None:
187 self.output = _OutputChainedCollectionInfo(registry, args.output)
188 else:
189 self.output = None
190 if args.output_run is not None:
191 self.outputRun = _OutputRunCollectionInfo(registry, args.output_run)
192 elif self.output is not None:
193 if args.extend_run:
194 runName = self.output.chain[0]
195 else:
196 runName = "{}/{}".format(self.output, Instrument.makeCollectionTimestamp())
197 self.outputRun = _OutputRunCollectionInfo(registry, runName)
198 elif not writeable:
199 # If we're not writing yet, ok to have no output run.
200 self.outputRun = None
201 else:
202 raise ValueError("Cannot write without at least one of (--output, --output-run).")
203 self.inputs = list(CollectionSearch.fromExpression(args.input)) if args.input else []
205 def check(self, args: argparse.Namespace):
206 """Check command-line options for consistency with each other and the
207 data repository.
209 Parameters
210 ----------
211 args : `argparse.Namespace`
212 Parsed command-line arguments. See class documentation for the
213 construction parameter of the same name.
214 """
215 assert not (args.extend_run and args.replace_run), "In mutually-exclusive group in ArgumentParser."
216 if self.inputs and self.output is not None and self.output.exists:
217 raise ValueError("Cannot use --output with existing collection with --inputs.")
218 if args.extend_run and self.outputRun is None:
219 raise ValueError("Cannot --extend-run when no output collection is given.")
220 if args.extend_run and not self.outputRun.exists:
221 raise ValueError(f"Cannot --extend-run; output collection "
222 f"'{self.outputRun.name}' does not exist.")
223 if not args.extend_run and self.outputRun is not None and self.outputRun.exists:
224 raise ValueError(f"Output run '{self.outputRun.name}' already exists, but "
225 f"--extend-run was not given.")
226 if args.prune_replaced and not args.replace_run:
227 raise ValueError("--prune-replaced requires --replace-run.")
228 if args.replace_run and (self.output is None or not self.output.exists):
229 raise ValueError("--output must point to an existing CHAINED collection for --replace-run.")
231 @classmethod
232 def _makeReadParts(cls, args: argparse.Namespace):
233 """Common implementation for `makeReadButler` and
234 `makeRegistryAndCollections`.
236 Parameters
237 ----------
238 args : `argparse.Namespace`
239 Parsed command-line arguments. See class documentation for the
240 construction parameter of the same name.
242 Returns
243 -------
244 butler : `lsst.daf.butler.Butler`
245 A read-only butler constructed from the repo at
246 ``args.butler_config``, but with no default collections.
247 inputs : `lsst.daf.butler.registry.CollectionSearch`
248 A collection search path constructed according to ``args``.
249 self : `_ButlerFactory`
250 A new `_ButlerFactory` instance representing the processed version
251 of ``args``.
252 """
253 butler = Butler(args.butler_config, writeable=False)
254 self = cls(butler.registry, args, writeable=False)
255 self.check(args)
256 if self.output and self.output.exists:
257 if args.replace_run:
258 replaced = self.output.chain[0]
259 inputs = self.output.chain[1:]
260 _LOG.debug("Simulating collection search in '%s' after removing '%s'.",
261 self.output.name, replaced)
262 else:
263 inputs = [self.output.name]
264 else:
265 inputs = list(self.inputs)
266 if args.extend_run:
267 inputs.insert(0, self.outputRun.name)
268 inputs = CollectionSearch.fromExpression(inputs)
269 return butler, inputs, self
271 @classmethod
272 def makeReadButler(cls, args: argparse.Namespace) -> Butler:
273 """Construct a read-only butler according to the given command-line
274 arguments.
276 Parameters
277 ----------
278 args : `argparse.Namespace`
279 Parsed command-line arguments. See class documentation for the
280 construction parameter of the same name.
282 Returns
283 -------
284 butler : `lsst.daf.butler.Butler`
285 A read-only butler initialized with the collections specified by
286 ``args``.
287 """
288 butler, inputs, _ = cls._makeReadParts(args)
289 _LOG.debug("Preparing butler to read from %s.", inputs)
290 return Butler(butler=butler, collections=inputs)
292 @classmethod
293 def makeRegistryAndCollections(cls, args: argparse.Namespace) -> \
294 Tuple[Registry, CollectionSearch, Optional[str]]:
295 """Return a read-only registry, a collection search path, and the name
296 of the run to be used for future writes.
298 Parameters
299 ----------
300 args : `argparse.Namespace`
301 Parsed command-line arguments. See class documentation for the
302 construction parameter of the same name.
304 Returns
305 -------
306 registry : `lsst.daf.butler.Registry`
307 Butler registry that collections will be added to and/or queried
308 from.
309 inputs : `lsst.daf.butler.registry.CollectionSearch`
310 Collections to search for datasets.
311 run : `str` or `None`
312 Name of the output `~lsst.daf.butler.CollectionType.RUN` collection
313 if it already exists, or `None` if it does not.
314 """
315 butler, inputs, self = cls._makeReadParts(args)
316 run = self.outputRun.name if args.extend_run else None
317 _LOG.debug("Preparing registry to read from %s and expect future writes to '%s'.", inputs, run)
318 return butler.registry, inputs, run
320 @classmethod
321 def makeWriteButler(cls, args: argparse.Namespace) -> Butler:
322 """Return a read-write butler initialized to write to and read from
323 the collections specified by the given command-line arguments.
325 Parameters
326 ----------
327 args : `argparse.Namespace`
328 Parsed command-line arguments. See class documentation for the
329 construction parameter of the same name.
331 Returns
332 -------
333 butler : `lsst.daf.butler.Butler`
334 A read-write butler initialized according to the given arguments.
335 """
336 butler = Butler(args.butler_config, writeable=True)
337 self = cls(butler.registry, args, writeable=True)
338 self.check(args)
339 if self.output is not None:
340 chainDefinition = list(self.output.chain if self.output.exists else self.inputs)
341 if args.replace_run:
342 replaced = chainDefinition.pop(0)
343 if args.prune_replaced == "unstore":
344 # Remove datasets from datastore
345 with butler.transaction():
346 refs = butler.registry.queryDatasets(..., collections=replaced)
347 butler.pruneDatasets(refs, unstore=True, run=replaced, disassociate=False)
348 elif args.prune_replaced == "purge":
349 # Erase entire collection and all datasets, need to remove
350 # collection from its chain collection first.
351 with butler.transaction():
352 butler.registry.setCollectionChain(self.output.name, chainDefinition)
353 butler.pruneCollection(replaced, purge=True, unstore=True)
354 elif args.prune_replaced is not None:
355 raise NotImplementedError(
356 f"Unsupported --prune-replaced option '{args.prune_replaced}'."
357 )
358 chainDefinition.insert(0, self.outputRun.name)
359 chainDefinition = CollectionSearch.fromExpression(chainDefinition)
360 _LOG.debug("Preparing butler to write to '%s' and read from '%s'=%s",
361 self.outputRun.name, self.output.name, chainDefinition)
362 return Butler(butler=butler, run=self.outputRun.name, collections=self.output.name,
363 chains={self.output.name: chainDefinition})
364 else:
365 inputs = CollectionSearch.fromExpression([self.outputRun.name] + self.inputs)
366 _LOG.debug("Preparing butler to write to '%s' and read from %s.", self.outputRun.name, inputs)
367 return Butler(butler=butler, run=self.outputRun.name, collections=inputs)
369 output: Optional[_OutputChainedCollectionInfo]
370 """Information about the output chained collection, if there is or will be
371 one (`_OutputChainedCollectionInfo` or `None`).
372 """
374 outputRun: Optional[_OutputRunCollectionInfo]
375 """Information about the output run collection, if there is or will be
376 one (`_OutputRunCollectionInfo` or `None`).
377 """
379 inputs: List[Tuple[str, DatasetTypeRestriction]]
380 """Input collections, including those also used for outputs and any
381 restrictions on dataset types (`list`).
382 """
385class _FilteredStream:
386 """A file-like object that filters some config fields.
388 Note
389 ----
390 This class depends on implementation details of ``Config.saveToStream``
391 methods, in particular that that method uses single call to write()
392 method to save information about single config field, and that call
393 combines comments string(s) for a field and field path and value.
394 This class will not work reliably on the "import" strings, so imports
395 should be disabled by passing ``skipImports=True`` to ``saveToStream()``.
396 """
397 def __init__(self, pattern):
398 # obey case if pattern isn't lowercase or requests NOIGNORECASE
399 mat = re.search(r"(.*):NOIGNORECASE$", pattern)
401 if mat:
402 pattern = mat.group(1)
403 self._pattern = re.compile(fnmatch.translate(pattern))
404 else:
405 if pattern != pattern.lower():
406 print(f"Matching \"{pattern}\" without regard to case "
407 "(append :NOIGNORECASE to prevent this)", file=sys.stdout)
408 self._pattern = re.compile(fnmatch.translate(pattern), re.IGNORECASE)
410 def write(self, showStr):
411 # Strip off doc string line(s) and cut off at "=" for string matching
412 matchStr = showStr.rstrip().split("\n")[-1].split("=")[0]
413 if self._pattern.search(matchStr):
414 sys.stdout.write(showStr)
416# ------------------------
417# Exported definitions --
418# ------------------------
421class CmdLineFwk:
422 """PipelineTask framework which executes tasks from command line.
424 In addition to executing tasks this activator provides additional methods
425 for task management like dumping configuration or execution chain.
426 """
428 MP_TIMEOUT = 9999 # Default timeout (sec) for multiprocessing
430 def __init__(self):
431 pass
433 def makePipeline(self, args):
434 """Build a pipeline from command line arguments.
436 Parameters
437 ----------
438 args : `argparse.Namespace`
439 Parsed command line
441 Returns
442 -------
443 pipeline : `~lsst.pipe.base.Pipeline`
444 """
445 if args.pipeline:
446 pipeline = Pipeline.fromFile(args.pipeline)
447 else:
448 pipeline = Pipeline("anonymous")
450 # loop over all pipeline actions and apply them in order
451 for action in args.pipeline_actions:
452 if action.action == "add_instrument":
454 pipeline.addInstrument(action.value)
456 elif action.action == "new_task":
458 pipeline.addTask(action.value, action.label)
460 elif action.action == "delete_task":
462 pipeline.removeTask(action.label)
464 elif action.action == "config":
466 # action value string is "field=value", split it at '='
467 field, _, value = action.value.partition("=")
468 pipeline.addConfigOverride(action.label, field, value)
470 elif action.action == "configfile":
472 pipeline.addConfigFile(action.label, action.value)
474 else:
476 raise ValueError(f"Unexpected pipeline action: {action.action}")
478 if args.save_pipeline:
479 pipeline.toFile(args.save_pipeline)
481 if args.pipeline_dot:
482 pipeline2dot(pipeline, args.pipeline_dot)
484 return pipeline
486 def makeGraph(self, pipeline, args):
487 """Build a graph from command line arguments.
489 Parameters
490 ----------
491 pipeline : `~lsst.pipe.base.Pipeline`
492 Pipeline, can be empty or ``None`` if graph is read from a file.
493 args : `argparse.Namespace`
494 Parsed command line
496 Returns
497 -------
498 graph : `~lsst.pipe.base.QuantumGraph` or `None`
499 If resulting graph is empty then `None` is returned.
500 """
502 registry, collections, run = _ButlerFactory.makeRegistryAndCollections(args)
504 if args.qgraph:
505 # click passes empty tuple as default value for qgraph_node_id
506 nodes = args.qgraph_node_id or None
507 qgraph = QuantumGraph.loadUri(args.qgraph, registry.dimensions,
508 nodes=nodes, graphID=args.qgraph_id)
510 # pipeline can not be provided in this case
511 if pipeline:
512 raise ValueError("Pipeline must not be given when quantum graph is read from file.")
514 else:
516 # make execution plan (a.k.a. DAG) for pipeline
517 graphBuilder = GraphBuilder(registry,
518 skipExisting=args.skip_existing)
519 qgraph = graphBuilder.makeGraph(pipeline, collections, run, args.data_query)
521 # count quanta in graph and give a warning if it's empty and return None
522 nQuanta = len(qgraph)
523 if nQuanta == 0:
524 warnings.warn("QuantumGraph is empty", stacklevel=2)
525 return None
526 else:
527 _LOG.info("QuantumGraph contains %d quanta for %d tasks, graph ID: %r",
528 nQuanta, len(qgraph.taskGraph), qgraph.graphID)
530 if args.save_qgraph:
531 qgraph.saveUri(args.save_qgraph)
533 if args.save_single_quanta:
534 for quantumNode in qgraph:
535 sqgraph = qgraph.subset(quantumNode)
536 uri = args.save_single_quanta.format(quantumNode.nodeId.number)
537 sqgraph.saveUri(uri)
539 if args.qgraph_dot:
540 graph2dot(qgraph, args.qgraph_dot)
542 return qgraph
544 def runPipeline(self, graph, taskFactory, args, butler=None):
545 """Execute complete QuantumGraph.
547 Parameters
548 ----------
549 graph : `QuantumGraph`
550 Execution graph.
551 taskFactory : `~lsst.pipe.base.TaskFactory`
552 Task factory
553 args : `argparse.Namespace`
554 Parsed command line
555 butler : `~lsst.daf.butler.Butler`, optional
556 Data Butler instance, if not defined then new instance is made
557 using command line options.
558 """
559 # make butler instance
560 if butler is None:
561 butler = _ButlerFactory.makeWriteButler(args)
563 # Enable lsstDebug debugging. Note that this is done once in the
564 # main process before PreExecInit and it is also repeated before
565 # running each task in SingleQuantumExecutor (which may not be
566 # needed if `multipocessing` always uses fork start method).
567 if args.enableLsstDebug:
568 try:
569 _LOG.debug("Will try to import debug.py")
570 import debug # noqa:F401
571 except ImportError:
572 _LOG.warn("No 'debug' module found.")
574 preExecInit = PreExecInit(butler, taskFactory, args.skip_existing)
575 preExecInit.initialize(graph,
576 saveInitOutputs=not args.skip_init_writes,
577 registerDatasetTypes=args.register_dataset_types,
578 saveVersions=not args.no_versions)
580 if not args.init_only:
581 graphFixup = self._importGraphFixup(args)
582 quantumExecutor = SingleQuantumExecutor(taskFactory,
583 skipExisting=args.skip_existing,
584 clobberPartialOutputs=args.clobber_partial_outputs,
585 enableLsstDebug=args.enableLsstDebug)
586 timeout = self.MP_TIMEOUT if args.timeout is None else args.timeout
587 executor = MPGraphExecutor(numProc=args.processes, timeout=timeout,
588 startMethod=args.start_method,
589 quantumExecutor=quantumExecutor,
590 failFast=args.fail_fast,
591 executionGraphFixup=graphFixup)
592 with util.profile(args.profile, _LOG):
593 executor.execute(graph, butler)
595 def showInfo(self, args, pipeline, graph=None):
596 """Display useful info about pipeline and environment.
598 Parameters
599 ----------
600 args : `argparse.Namespace`
601 Parsed command line
602 pipeline : `Pipeline`
603 Pipeline definition
604 graph : `QuantumGraph`, optional
605 Execution graph
606 """
607 showOpts = args.show
608 for what in showOpts:
609 showCommand, _, showArgs = what.partition("=")
611 if showCommand in ["pipeline", "config", "history", "tasks"]:
612 if not pipeline:
613 _LOG.warning("Pipeline is required for --show=%s", showCommand)
614 continue
616 if showCommand in ["graph", "workflow", "uri"]:
617 if not graph:
618 _LOG.warning("QuantumGraph is required for --show=%s", showCommand)
619 continue
621 if showCommand == "pipeline":
622 print(pipeline)
623 elif showCommand == "config":
624 self._showConfig(pipeline, showArgs, False)
625 elif showCommand == "dump-config":
626 self._showConfig(pipeline, showArgs, True)
627 elif showCommand == "history":
628 self._showConfigHistory(pipeline, showArgs)
629 elif showCommand == "tasks":
630 self._showTaskHierarchy(pipeline)
631 elif showCommand == "graph":
632 if graph:
633 self._showGraph(graph)
634 elif showCommand == "uri":
635 if graph:
636 self._showUri(graph, args)
637 elif showCommand == "workflow":
638 if graph:
639 self._showWorkflow(graph, args)
640 else:
641 print("Unknown value for show: %s (choose from '%s')" %
642 (what, "', '".join("pipeline config[=XXX] history=XXX tasks graph".split())),
643 file=sys.stderr)
644 sys.exit(1)
646 def _showConfig(self, pipeline, showArgs, dumpFullConfig):
647 """Show task configuration
649 Parameters
650 ----------
651 pipeline : `Pipeline`
652 Pipeline definition
653 showArgs : `str`
654 Defines what to show
655 dumpFullConfig : `bool`
656 If true then dump complete task configuration with all imports.
657 """
658 stream = sys.stdout
659 if dumpFullConfig:
660 # Task label can be given with this option
661 taskName = showArgs
662 else:
663 # The argument can have form [TaskLabel::][pattern:NOIGNORECASE]
664 matConfig = re.search(r"^(?:(\w+)::)?(?:config.)?(.+)?", showArgs)
665 taskName = matConfig.group(1)
666 pattern = matConfig.group(2)
667 if pattern:
668 stream = _FilteredStream(pattern)
670 tasks = util.filterTasks(pipeline, taskName)
671 if not tasks:
672 print("Pipeline has no tasks named {}".format(taskName), file=sys.stderr)
673 sys.exit(1)
675 for taskDef in tasks:
676 print("### Configuration for task `{}'".format(taskDef.label))
677 taskDef.config.saveToStream(stream, root="config", skipImports=not dumpFullConfig)
679 def _showConfigHistory(self, pipeline, showArgs):
680 """Show history for task configuration
682 Parameters
683 ----------
684 pipeline : `Pipeline`
685 Pipeline definition
686 showArgs : `str`
687 Defines what to show
688 """
690 taskName = None
691 pattern = None
692 matHistory = re.search(r"^(?:(\w+)::)?(?:config[.])?(.+)", showArgs)
693 if matHistory:
694 taskName = matHistory.group(1)
695 pattern = matHistory.group(2)
696 if not pattern:
697 print("Please provide a value with --show history (e.g. history=Task::param)", file=sys.stderr)
698 sys.exit(1)
700 tasks = util.filterTasks(pipeline, taskName)
701 if not tasks:
702 print(f"Pipeline has no tasks named {taskName}", file=sys.stderr)
703 sys.exit(1)
705 found = False
706 for taskDef in tasks:
708 config = taskDef.config
710 # Look for any matches in the config hierarchy for this name
711 for nmatch, thisName in enumerate(fnmatch.filter(config.names(), pattern)):
712 if nmatch > 0:
713 print("")
715 cpath, _, cname = thisName.rpartition(".")
716 try:
717 if not cpath:
718 # looking for top-level field
719 hconfig = taskDef.config
720 else:
721 hconfig = eval("config." + cpath, {}, {"config": config})
722 except AttributeError:
723 print(f"Error: Unable to extract attribute {cpath} from task {taskDef.label}",
724 file=sys.stderr)
725 hconfig = None
727 # Sometimes we end up with a non-Config so skip those
728 if isinstance(hconfig, (pexConfig.Config, pexConfig.ConfigurableInstance)) and \
729 hasattr(hconfig, cname):
730 print(f"### Configuration field for task `{taskDef.label}'")
731 print(pexConfig.history.format(hconfig, cname))
732 found = True
734 if not found:
735 print(f"None of the tasks has field matching {pattern}", file=sys.stderr)
736 sys.exit(1)
738 def _showTaskHierarchy(self, pipeline):
739 """Print task hierarchy to stdout
741 Parameters
742 ----------
743 pipeline: `Pipeline`
744 """
745 for taskDef in pipeline.toExpandedPipeline():
746 print("### Subtasks for task `{}'".format(taskDef.taskName))
748 for configName, taskName in util.subTaskIter(taskDef.config):
749 print("{}: {}".format(configName, taskName))
751 def _showGraph(self, graph):
752 """Print quanta information to stdout
754 Parameters
755 ----------
756 graph : `QuantumGraph`
757 Execution graph.
758 """
759 for taskNode in graph.taskGraph:
760 print(taskNode)
762 for iq, quantum in enumerate(graph.getQuantaForTask(taskNode)):
763 print(" Quantum {}:".format(iq))
764 print(" inputs:")
765 for key, refs in quantum.inputs.items():
766 dataIds = ["DataId({})".format(ref.dataId) for ref in refs]
767 print(" {}: [{}]".format(key, ", ".join(dataIds)))
768 print(" outputs:")
769 for key, refs in quantum.outputs.items():
770 dataIds = ["DataId({})".format(ref.dataId) for ref in refs]
771 print(" {}: [{}]".format(key, ", ".join(dataIds)))
773 def _showWorkflow(self, graph, args):
774 """Print quanta information and dependency to stdout
776 Parameters
777 ----------
778 graph : `QuantumGraph`
779 Execution graph.
780 args : `argparse.Namespace`
781 Parsed command line
782 """
783 for node in graph:
784 print(f"Quantum {node.nodeId.number}: {node.taskDef.taskName}")
785 for parent in graph.determineInputsToQuantumNode(node):
786 print(f"Parent Quantum {parent.nodeId.number} - Child Quantum {node.nodeId.number}")
788 def _showUri(self, graph, args):
789 """Print input and predicted output URIs to stdout
791 Parameters
792 ----------
793 graph : `QuantumGraph`
794 Execution graph
795 args : `argparse.Namespace`
796 Parsed command line
797 """
798 def dumpURIs(thisRef):
799 primary, components = butler.getURIs(thisRef, predict=True, run="TBD")
800 if primary:
801 print(f" {primary}")
802 else:
803 print(" (disassembled artifact)")
804 for compName, compUri in components.items():
805 print(f" {compName}: {compUri}")
807 butler = _ButlerFactory.makeReadButler(args)
808 for node in graph:
809 print(f"Quantum {node.nodeId.number}: {node.taskDef.taskName}")
810 print(" inputs:")
811 for key, refs in node.quantum.inputs.items():
812 for ref in refs:
813 dumpURIs(ref)
814 print(" outputs:")
815 for key, refs in node.quantum.outputs.items():
816 for ref in refs:
817 dumpURIs(ref)
819 def _importGraphFixup(self, args):
820 """Import/instantiate graph fixup object.
822 Parameters
823 ----------
824 args : `argparse.Namespace`
825 Parsed command line.
827 Returns
828 -------
829 fixup : `ExecutionGraphFixup` or `None`
831 Raises
832 ------
833 ValueError
834 Raised if import fails, method call raises exception, or returned
835 instance has unexpected type.
836 """
837 if args.graph_fixup:
838 try:
839 factory = doImport(args.graph_fixup)
840 except Exception as exc:
841 raise ValueError("Failed to import graph fixup class/method") from exc
842 try:
843 fixup = factory()
844 except Exception as exc:
845 raise ValueError("Failed to make instance of graph fixup") from exc
846 if not isinstance(fixup, ExecutionGraphFixup):
847 raise ValueError("Graph fixup is not an instance of ExecutionGraphFixup class")
848 return fixup