Coverage for python/lsst/ctrl/mpexec/cmdLineFwk.py : 12%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of ctrl_mpexec.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22"""Module defining CmdLineFwk class and related methods.
23"""
25__all__ = ['CmdLineFwk']
27# -------------------------------
28# Imports of standard modules --
29# -------------------------------
30import argparse
31import fnmatch
32import logging
33import re
34import sys
35from typing import Optional, Tuple
36import warnings
38# -----------------------------
39# Imports for other modules --
40# -----------------------------
41from lsst.daf.butler import (
42 Butler,
43 CollectionSearch,
44 CollectionType,
45 Registry,
46)
47from lsst.daf.butler.registry import MissingCollectionError, RegistryDefaults
48import lsst.pex.config as pexConfig
49from lsst.pipe.base import GraphBuilder, Pipeline, QuantumGraph
50from lsst.obs.base import Instrument
51from .dotTools import graph2dot, pipeline2dot
52from .executionGraphFixup import ExecutionGraphFixup
53from .mpGraphExecutor import MPGraphExecutor
54from .preExecInit import PreExecInit
55from .singleQuantumExecutor import SingleQuantumExecutor
56from . import util
57from lsst.utils import doImport
59# ----------------------------------
60# Local non-exported definitions --
61# ----------------------------------
63_LOG = logging.getLogger(__name__.partition(".")[2])
66class _OutputChainedCollectionInfo:
67 """A helper class for handling command-line arguments related to an output
68 `~lsst.daf.butler.CollectionType.CHAINED` collection.
70 Parameters
71 ----------
72 registry : `lsst.daf.butler.Registry`
73 Butler registry that collections will be added to and/or queried from.
74 name : `str`
75 Name of the collection given on the command line.
76 """
77 def __init__(self, registry: Registry, name: str):
78 self.name = name
79 try:
80 self.chain = tuple(registry.getCollectionChain(name))
81 self.exists = True
82 except MissingCollectionError:
83 self.chain = ()
84 self.exists = False
86 def __str__(self):
87 return self.name
89 name: str
90 """Name of the collection provided on the command line (`str`).
91 """
93 exists: bool
94 """Whether this collection already exists in the registry (`bool`).
95 """
97 chain: Tuple[str, ...]
98 """The definition of the collection, if it already exists (`tuple` [`str`]).
100 Empty if the collection does not already exist.
101 """
104class _OutputRunCollectionInfo:
105 """A helper class for handling command-line arguments related to an output
106 `~lsst.daf.butler.CollectionType.RUN` collection.
108 Parameters
109 ----------
110 registry : `lsst.daf.butler.Registry`
111 Butler registry that collections will be added to and/or queried from.
112 name : `str`
113 Name of the collection given on the command line.
114 """
115 def __init__(self, registry: Registry, name: str):
116 self.name = name
117 try:
118 actualType = registry.getCollectionType(name)
119 if actualType is not CollectionType.RUN:
120 raise TypeError(f"Collection '{name}' exists but has type {actualType.name}, not RUN.")
121 self.exists = True
122 except MissingCollectionError:
123 self.exists = False
125 name: str
126 """Name of the collection provided on the command line (`str`).
127 """
129 exists: bool
130 """Whether this collection already exists in the registry (`bool`).
131 """
134class _ButlerFactory:
135 """A helper class for processing command-line arguments related to input
136 and output collections.
138 Parameters
139 ----------
140 registry : `lsst.daf.butler.Registry`
141 Butler registry that collections will be added to and/or queried from.
143 args : `argparse.Namespace`
144 Parsed command-line arguments. The following attributes are used,
145 either at construction or in later methods.
147 ``output``
148 The name of a `~lsst.daf.butler.CollectionType.CHAINED`
149 input/output collection.
151 ``output_run``
152 The name of a `~lsst.daf.butler.CollectionType.RUN` input/output
153 collection.
155 ``extend_run``
156 A boolean indicating whether ``output_run`` should already exist
157 and be extended.
159 ``replace_run``
160 A boolean indicating that (if `True`) ``output_run`` should already
161 exist but will be removed from the output chained collection and
162 replaced with a new one.
164 ``prune_replaced``
165 A boolean indicating whether to prune the replaced run (requires
166 ``replace_run``).
168 ``inputs``
169 Input collections of any type; may be any type handled by
170 `lsst.daf.butler.registry.CollectionSearch.fromExpression`.
172 ``butler_config``
173 Path to a data repository root or configuration file.
175 writeable : `bool`
176 If `True`, a `Butler` is being initialized in a context where actual
177 writes should happens, and hence no output run is necessary.
179 Raises
180 ------
181 ValueError
182 Raised if ``writeable is True`` but there are no output collections.
183 """
184 def __init__(self, registry: Registry, args: argparse.Namespace, writeable: bool):
185 if args.output is not None:
186 self.output = _OutputChainedCollectionInfo(registry, args.output)
187 else:
188 self.output = None
189 if args.output_run is not None:
190 self.outputRun = _OutputRunCollectionInfo(registry, args.output_run)
191 elif self.output is not None:
192 if args.extend_run:
193 runName = self.output.chain[0]
194 else:
195 runName = "{}/{}".format(self.output, Instrument.makeCollectionTimestamp())
196 self.outputRun = _OutputRunCollectionInfo(registry, runName)
197 elif not writeable:
198 # If we're not writing yet, ok to have no output run.
199 self.outputRun = None
200 else:
201 raise ValueError("Cannot write without at least one of (--output, --output-run).")
202 # Recursively flatten any input CHAINED collections. We do this up
203 # front so we can tell if the user passes the same inputs on subsequent
204 # calls, even though we also flatten when we define the output CHAINED
205 # collection.
206 self.inputs = tuple(registry.queryCollections(args.input, flattenChains=True)) if args.input else ()
208 def check(self, args: argparse.Namespace):
209 """Check command-line options for consistency with each other and the
210 data repository.
212 Parameters
213 ----------
214 args : `argparse.Namespace`
215 Parsed command-line arguments. See class documentation for the
216 construction parameter of the same name.
217 """
218 assert not (args.extend_run and args.replace_run), "In mutually-exclusive group in ArgumentParser."
219 if self.inputs and self.output is not None and self.output.exists:
220 # Passing the same inputs that were used to initialize the output
221 # collection is allowed; this means they must _end_ with the same
222 # collections, because we push new runs to the front of the chain.
223 for c1, c2 in zip(self.inputs[::-1], self.output.chain[::-1]):
224 if c1 != c2:
225 raise ValueError(
226 f"Output CHAINED collection {self.output.name!r} exists, but it ends with "
227 "a different sequence of input collections than those given: "
228 f"{c1!r} != {c2!r} in inputs={self.inputs} vs "
229 f"{self.output.name}={self.output.chain}."
230 )
231 if len(self.inputs) > len(self.output.chain):
232 nNew = len(self.inputs) - len(self.output.chain)
233 raise ValueError(
234 f"Cannot add new input collections {self.inputs[:nNew]} after "
235 "output collection is first created."
236 )
237 if args.extend_run and self.outputRun is None:
238 raise ValueError("Cannot --extend-run when no output collection is given.")
239 if args.extend_run and not self.outputRun.exists:
240 raise ValueError(f"Cannot --extend-run; output collection "
241 f"'{self.outputRun.name}' does not exist.")
242 if not args.extend_run and self.outputRun is not None and self.outputRun.exists:
243 raise ValueError(f"Output run '{self.outputRun.name}' already exists, but "
244 f"--extend-run was not given.")
245 if args.prune_replaced and not args.replace_run:
246 raise ValueError("--prune-replaced requires --replace-run.")
247 if args.replace_run and (self.output is None or not self.output.exists):
248 raise ValueError("--output must point to an existing CHAINED collection for --replace-run.")
250 @classmethod
251 def _makeReadParts(cls, args: argparse.Namespace):
252 """Common implementation for `makeReadButler` and
253 `makeRegistryAndCollections`.
255 Parameters
256 ----------
257 args : `argparse.Namespace`
258 Parsed command-line arguments. See class documentation for the
259 construction parameter of the same name.
261 Returns
262 -------
263 butler : `lsst.daf.butler.Butler`
264 A read-only butler constructed from the repo at
265 ``args.butler_config``, but with no default collections.
266 inputs : `lsst.daf.butler.registry.CollectionSearch`
267 A collection search path constructed according to ``args``.
268 self : `_ButlerFactory`
269 A new `_ButlerFactory` instance representing the processed version
270 of ``args``.
271 """
272 butler = Butler(args.butler_config, writeable=False)
273 self = cls(butler.registry, args, writeable=False)
274 self.check(args)
275 if self.output and self.output.exists:
276 if args.replace_run:
277 replaced = self.output.chain[0]
278 inputs = self.output.chain[1:]
279 _LOG.debug("Simulating collection search in '%s' after removing '%s'.",
280 self.output.name, replaced)
281 else:
282 inputs = [self.output.name]
283 else:
284 inputs = list(self.inputs)
285 if args.extend_run:
286 inputs.insert(0, self.outputRun.name)
287 inputs = CollectionSearch.fromExpression(inputs)
288 return butler, inputs, self
290 @classmethod
291 def makeReadButler(cls, args: argparse.Namespace) -> Butler:
292 """Construct a read-only butler according to the given command-line
293 arguments.
295 Parameters
296 ----------
297 args : `argparse.Namespace`
298 Parsed command-line arguments. See class documentation for the
299 construction parameter of the same name.
301 Returns
302 -------
303 butler : `lsst.daf.butler.Butler`
304 A read-only butler initialized with the collections specified by
305 ``args``.
306 """
307 butler, inputs, _ = cls._makeReadParts(args)
308 _LOG.debug("Preparing butler to read from %s.", inputs)
309 return Butler(butler=butler, collections=inputs)
311 @classmethod
312 def makeRegistryAndCollections(cls, args: argparse.Namespace) -> \
313 Tuple[Registry, CollectionSearch, Optional[str]]:
314 """Return a read-only registry, a collection search path, and the name
315 of the run to be used for future writes.
317 Parameters
318 ----------
319 args : `argparse.Namespace`
320 Parsed command-line arguments. See class documentation for the
321 construction parameter of the same name.
323 Returns
324 -------
325 registry : `lsst.daf.butler.Registry`
326 Butler registry that collections will be added to and/or queried
327 from.
328 inputs : `lsst.daf.butler.registry.CollectionSearch`
329 Collections to search for datasets.
330 run : `str` or `None`
331 Name of the output `~lsst.daf.butler.CollectionType.RUN` collection
332 if it already exists, or `None` if it does not.
333 """
334 butler, inputs, self = cls._makeReadParts(args)
335 run = self.outputRun.name if args.extend_run else None
336 _LOG.debug("Preparing registry to read from %s and expect future writes to '%s'.", inputs, run)
337 return butler.registry, inputs, run
339 @classmethod
340 def makeWriteButler(cls, args: argparse.Namespace) -> Butler:
341 """Return a read-write butler initialized to write to and read from
342 the collections specified by the given command-line arguments.
344 Parameters
345 ----------
346 args : `argparse.Namespace`
347 Parsed command-line arguments. See class documentation for the
348 construction parameter of the same name.
350 Returns
351 -------
352 butler : `lsst.daf.butler.Butler`
353 A read-write butler initialized according to the given arguments.
354 """
355 butler = Butler(args.butler_config, writeable=True)
356 self = cls(butler.registry, args, writeable=True)
357 self.check(args)
358 if self.output is not None:
359 chainDefinition = list(self.output.chain if self.output.exists else self.inputs)
360 if args.replace_run:
361 replaced = chainDefinition.pop(0)
362 if args.prune_replaced == "unstore":
363 # Remove datasets from datastore
364 with butler.transaction():
365 refs = butler.registry.queryDatasets(..., collections=replaced)
366 butler.pruneDatasets(refs, unstore=True, run=replaced, disassociate=False)
367 elif args.prune_replaced == "purge":
368 # Erase entire collection and all datasets, need to remove
369 # collection from its chain collection first.
370 with butler.transaction():
371 butler.registry.setCollectionChain(self.output.name, chainDefinition, flatten=True)
372 butler.pruneCollection(replaced, purge=True, unstore=True)
373 elif args.prune_replaced is not None:
374 raise NotImplementedError(
375 f"Unsupported --prune-replaced option '{args.prune_replaced}'."
376 )
377 if not self.output.exists:
378 butler.registry.registerCollection(self.output.name, CollectionType.CHAINED)
379 if not args.extend_run:
380 butler.registry.registerCollection(self.outputRun.name, CollectionType.RUN)
381 chainDefinition.insert(0, self.outputRun.name)
382 butler.registry.setCollectionChain(self.output.name, chainDefinition, flatten=True)
383 _LOG.debug("Preparing butler to write to '%s' and read from '%s'=%s",
384 self.outputRun.name, self.output.name, chainDefinition)
385 butler.registry.defaults = RegistryDefaults(run=self.outputRun.name, collections=self.output.name)
386 else:
387 inputs = CollectionSearch.fromExpression((self.outputRun.name,) + self.inputs)
388 _LOG.debug("Preparing butler to write to '%s' and read from %s.", self.outputRun.name, inputs)
389 butler.registry.defaults = RegistryDefaults(run=self.outputRun.name, collections=inputs)
390 return butler
392 output: Optional[_OutputChainedCollectionInfo]
393 """Information about the output chained collection, if there is or will be
394 one (`_OutputChainedCollectionInfo` or `None`).
395 """
397 outputRun: Optional[_OutputRunCollectionInfo]
398 """Information about the output run collection, if there is or will be
399 one (`_OutputRunCollectionInfo` or `None`).
400 """
402 inputs: Tuple[str, ...]
403 """Input collections provided directly by the user (`tuple` [ `str` ]).
404 """
407class _FilteredStream:
408 """A file-like object that filters some config fields.
410 Note
411 ----
412 This class depends on implementation details of ``Config.saveToStream``
413 methods, in particular that that method uses single call to write()
414 method to save information about single config field, and that call
415 combines comments string(s) for a field and field path and value.
416 This class will not work reliably on the "import" strings, so imports
417 should be disabled by passing ``skipImports=True`` to ``saveToStream()``.
418 """
419 def __init__(self, pattern):
420 # obey case if pattern isn't lowercase or requests NOIGNORECASE
421 mat = re.search(r"(.*):NOIGNORECASE$", pattern)
423 if mat:
424 pattern = mat.group(1)
425 self._pattern = re.compile(fnmatch.translate(pattern))
426 else:
427 if pattern != pattern.lower():
428 print(f"Matching \"{pattern}\" without regard to case "
429 "(append :NOIGNORECASE to prevent this)", file=sys.stdout)
430 self._pattern = re.compile(fnmatch.translate(pattern), re.IGNORECASE)
432 def write(self, showStr):
433 # Strip off doc string line(s) and cut off at "=" for string matching
434 matchStr = showStr.rstrip().split("\n")[-1].split("=")[0]
435 if self._pattern.search(matchStr):
436 sys.stdout.write(showStr)
438# ------------------------
439# Exported definitions --
440# ------------------------
443class CmdLineFwk:
444 """PipelineTask framework which executes tasks from command line.
446 In addition to executing tasks this activator provides additional methods
447 for task management like dumping configuration or execution chain.
448 """
450 MP_TIMEOUT = 9999 # Default timeout (sec) for multiprocessing
452 def __init__(self):
453 pass
455 def makePipeline(self, args):
456 """Build a pipeline from command line arguments.
458 Parameters
459 ----------
460 args : `argparse.Namespace`
461 Parsed command line
463 Returns
464 -------
465 pipeline : `~lsst.pipe.base.Pipeline`
466 """
467 if args.pipeline:
468 pipeline = Pipeline.from_uri(args.pipeline)
469 else:
470 pipeline = Pipeline("anonymous")
472 # loop over all pipeline actions and apply them in order
473 for action in args.pipeline_actions:
474 if action.action == "add_instrument":
476 pipeline.addInstrument(action.value)
478 elif action.action == "new_task":
480 pipeline.addTask(action.value, action.label)
482 elif action.action == "delete_task":
484 pipeline.removeTask(action.label)
486 elif action.action == "config":
488 # action value string is "field=value", split it at '='
489 field, _, value = action.value.partition("=")
490 pipeline.addConfigOverride(action.label, field, value)
492 elif action.action == "configfile":
494 pipeline.addConfigFile(action.label, action.value)
496 else:
498 raise ValueError(f"Unexpected pipeline action: {action.action}")
500 if args.save_pipeline:
501 pipeline.write_to_uri(args.save_pipeline)
503 if args.pipeline_dot:
504 pipeline2dot(pipeline, args.pipeline_dot)
506 return pipeline
508 def makeGraph(self, pipeline, args):
509 """Build a graph from command line arguments.
511 Parameters
512 ----------
513 pipeline : `~lsst.pipe.base.Pipeline`
514 Pipeline, can be empty or ``None`` if graph is read from a file.
515 args : `argparse.Namespace`
516 Parsed command line
518 Returns
519 -------
520 graph : `~lsst.pipe.base.QuantumGraph` or `None`
521 If resulting graph is empty then `None` is returned.
522 """
524 registry, collections, run = _ButlerFactory.makeRegistryAndCollections(args)
526 if args.qgraph:
527 # click passes empty tuple as default value for qgraph_node_id
528 nodes = args.qgraph_node_id or None
529 qgraph = QuantumGraph.loadUri(args.qgraph, registry.dimensions,
530 nodes=nodes, graphID=args.qgraph_id)
532 # pipeline can not be provided in this case
533 if pipeline:
534 raise ValueError("Pipeline must not be given when quantum graph is read from file.")
536 else:
538 # make execution plan (a.k.a. DAG) for pipeline
539 graphBuilder = GraphBuilder(registry,
540 skipExisting=args.skip_existing)
541 qgraph = graphBuilder.makeGraph(pipeline, collections, run, args.data_query)
543 # count quanta in graph and give a warning if it's empty and return None
544 nQuanta = len(qgraph)
545 if nQuanta == 0:
546 warnings.warn("QuantumGraph is empty", stacklevel=2)
547 return None
548 else:
549 _LOG.info("QuantumGraph contains %d quanta for %d tasks, graph ID: %r",
550 nQuanta, len(qgraph.taskGraph), qgraph.graphID)
552 if args.save_qgraph:
553 qgraph.saveUri(args.save_qgraph)
555 if args.save_single_quanta:
556 for quantumNode in qgraph:
557 sqgraph = qgraph.subset(quantumNode)
558 uri = args.save_single_quanta.format(quantumNode.nodeId.number)
559 sqgraph.saveUri(uri)
561 if args.qgraph_dot:
562 graph2dot(qgraph, args.qgraph_dot)
564 return qgraph
566 def runPipeline(self, graph, taskFactory, args, butler=None):
567 """Execute complete QuantumGraph.
569 Parameters
570 ----------
571 graph : `QuantumGraph`
572 Execution graph.
573 taskFactory : `~lsst.pipe.base.TaskFactory`
574 Task factory
575 args : `argparse.Namespace`
576 Parsed command line
577 butler : `~lsst.daf.butler.Butler`, optional
578 Data Butler instance, if not defined then new instance is made
579 using command line options.
580 """
581 # make butler instance
582 if butler is None:
583 butler = _ButlerFactory.makeWriteButler(args)
585 # Enable lsstDebug debugging. Note that this is done once in the
586 # main process before PreExecInit and it is also repeated before
587 # running each task in SingleQuantumExecutor (which may not be
588 # needed if `multipocessing` always uses fork start method).
589 if args.enableLsstDebug:
590 try:
591 _LOG.debug("Will try to import debug.py")
592 import debug # noqa:F401
593 except ImportError:
594 _LOG.warn("No 'debug' module found.")
596 preExecInit = PreExecInit(butler, taskFactory, args.skip_existing)
597 preExecInit.initialize(graph,
598 saveInitOutputs=not args.skip_init_writes,
599 registerDatasetTypes=args.register_dataset_types,
600 saveVersions=not args.no_versions)
602 if not args.init_only:
603 graphFixup = self._importGraphFixup(args)
604 quantumExecutor = SingleQuantumExecutor(taskFactory,
605 skipExisting=args.skip_existing,
606 clobberPartialOutputs=args.clobber_partial_outputs,
607 enableLsstDebug=args.enableLsstDebug)
608 timeout = self.MP_TIMEOUT if args.timeout is None else args.timeout
609 executor = MPGraphExecutor(numProc=args.processes, timeout=timeout,
610 startMethod=args.start_method,
611 quantumExecutor=quantumExecutor,
612 failFast=args.fail_fast,
613 executionGraphFixup=graphFixup)
614 with util.profile(args.profile, _LOG):
615 executor.execute(graph, butler)
617 def showInfo(self, args, pipeline, graph=None):
618 """Display useful info about pipeline and environment.
620 Parameters
621 ----------
622 args : `argparse.Namespace`
623 Parsed command line
624 pipeline : `Pipeline`
625 Pipeline definition
626 graph : `QuantumGraph`, optional
627 Execution graph
628 """
629 showOpts = args.show
630 for what in showOpts:
631 showCommand, _, showArgs = what.partition("=")
633 if showCommand in ["pipeline", "config", "history", "tasks"]:
634 if not pipeline:
635 _LOG.warning("Pipeline is required for --show=%s", showCommand)
636 continue
638 if showCommand in ["graph", "workflow", "uri"]:
639 if not graph:
640 _LOG.warning("QuantumGraph is required for --show=%s", showCommand)
641 continue
643 if showCommand == "pipeline":
644 print(pipeline)
645 elif showCommand == "config":
646 self._showConfig(pipeline, showArgs, False)
647 elif showCommand == "dump-config":
648 self._showConfig(pipeline, showArgs, True)
649 elif showCommand == "history":
650 self._showConfigHistory(pipeline, showArgs)
651 elif showCommand == "tasks":
652 self._showTaskHierarchy(pipeline)
653 elif showCommand == "graph":
654 if graph:
655 self._showGraph(graph)
656 elif showCommand == "uri":
657 if graph:
658 self._showUri(graph, args)
659 elif showCommand == "workflow":
660 if graph:
661 self._showWorkflow(graph, args)
662 else:
663 print("Unknown value for show: %s (choose from '%s')" %
664 (what, "', '".join("pipeline config[=XXX] history=XXX tasks graph".split())),
665 file=sys.stderr)
666 sys.exit(1)
668 def _showConfig(self, pipeline, showArgs, dumpFullConfig):
669 """Show task configuration
671 Parameters
672 ----------
673 pipeline : `Pipeline`
674 Pipeline definition
675 showArgs : `str`
676 Defines what to show
677 dumpFullConfig : `bool`
678 If true then dump complete task configuration with all imports.
679 """
680 stream = sys.stdout
681 if dumpFullConfig:
682 # Task label can be given with this option
683 taskName = showArgs
684 else:
685 # The argument can have form [TaskLabel::][pattern:NOIGNORECASE]
686 matConfig = re.search(r"^(?:(\w+)::)?(?:config.)?(.+)?", showArgs)
687 taskName = matConfig.group(1)
688 pattern = matConfig.group(2)
689 if pattern:
690 stream = _FilteredStream(pattern)
692 tasks = util.filterTasks(pipeline, taskName)
693 if not tasks:
694 print("Pipeline has no tasks named {}".format(taskName), file=sys.stderr)
695 sys.exit(1)
697 for taskDef in tasks:
698 print("### Configuration for task `{}'".format(taskDef.label))
699 taskDef.config.saveToStream(stream, root="config", skipImports=not dumpFullConfig)
701 def _showConfigHistory(self, pipeline, showArgs):
702 """Show history for task configuration
704 Parameters
705 ----------
706 pipeline : `Pipeline`
707 Pipeline definition
708 showArgs : `str`
709 Defines what to show
710 """
712 taskName = None
713 pattern = None
714 matHistory = re.search(r"^(?:(\w+)::)?(?:config[.])?(.+)", showArgs)
715 if matHistory:
716 taskName = matHistory.group(1)
717 pattern = matHistory.group(2)
718 if not pattern:
719 print("Please provide a value with --show history (e.g. history=Task::param)", file=sys.stderr)
720 sys.exit(1)
722 tasks = util.filterTasks(pipeline, taskName)
723 if not tasks:
724 print(f"Pipeline has no tasks named {taskName}", file=sys.stderr)
725 sys.exit(1)
727 found = False
728 for taskDef in tasks:
730 config = taskDef.config
732 # Look for any matches in the config hierarchy for this name
733 for nmatch, thisName in enumerate(fnmatch.filter(config.names(), pattern)):
734 if nmatch > 0:
735 print("")
737 cpath, _, cname = thisName.rpartition(".")
738 try:
739 if not cpath:
740 # looking for top-level field
741 hconfig = taskDef.config
742 else:
743 hconfig = eval("config." + cpath, {}, {"config": config})
744 except AttributeError:
745 print(f"Error: Unable to extract attribute {cpath} from task {taskDef.label}",
746 file=sys.stderr)
747 hconfig = None
749 # Sometimes we end up with a non-Config so skip those
750 if isinstance(hconfig, (pexConfig.Config, pexConfig.ConfigurableInstance)) and \
751 hasattr(hconfig, cname):
752 print(f"### Configuration field for task `{taskDef.label}'")
753 print(pexConfig.history.format(hconfig, cname))
754 found = True
756 if not found:
757 print(f"None of the tasks has field matching {pattern}", file=sys.stderr)
758 sys.exit(1)
760 def _showTaskHierarchy(self, pipeline):
761 """Print task hierarchy to stdout
763 Parameters
764 ----------
765 pipeline: `Pipeline`
766 """
767 for taskDef in pipeline.toExpandedPipeline():
768 print("### Subtasks for task `{}'".format(taskDef.taskName))
770 for configName, taskName in util.subTaskIter(taskDef.config):
771 print("{}: {}".format(configName, taskName))
773 def _showGraph(self, graph):
774 """Print quanta information to stdout
776 Parameters
777 ----------
778 graph : `QuantumGraph`
779 Execution graph.
780 """
781 for taskNode in graph.taskGraph:
782 print(taskNode)
784 for iq, quantum in enumerate(graph.getQuantaForTask(taskNode)):
785 print(" Quantum {}:".format(iq))
786 print(" inputs:")
787 for key, refs in quantum.inputs.items():
788 dataIds = ["DataId({})".format(ref.dataId) for ref in refs]
789 print(" {}: [{}]".format(key, ", ".join(dataIds)))
790 print(" outputs:")
791 for key, refs in quantum.outputs.items():
792 dataIds = ["DataId({})".format(ref.dataId) for ref in refs]
793 print(" {}: [{}]".format(key, ", ".join(dataIds)))
795 def _showWorkflow(self, graph, args):
796 """Print quanta information and dependency to stdout
798 Parameters
799 ----------
800 graph : `QuantumGraph`
801 Execution graph.
802 args : `argparse.Namespace`
803 Parsed command line
804 """
805 for node in graph:
806 print(f"Quantum {node.nodeId.number}: {node.taskDef.taskName}")
807 for parent in graph.determineInputsToQuantumNode(node):
808 print(f"Parent Quantum {parent.nodeId.number} - Child Quantum {node.nodeId.number}")
810 def _showUri(self, graph, args):
811 """Print input and predicted output URIs to stdout
813 Parameters
814 ----------
815 graph : `QuantumGraph`
816 Execution graph
817 args : `argparse.Namespace`
818 Parsed command line
819 """
820 def dumpURIs(thisRef):
821 primary, components = butler.getURIs(thisRef, predict=True, run="TBD")
822 if primary:
823 print(f" {primary}")
824 else:
825 print(" (disassembled artifact)")
826 for compName, compUri in components.items():
827 print(f" {compName}: {compUri}")
829 butler = _ButlerFactory.makeReadButler(args)
830 for node in graph:
831 print(f"Quantum {node.nodeId.number}: {node.taskDef.taskName}")
832 print(" inputs:")
833 for key, refs in node.quantum.inputs.items():
834 for ref in refs:
835 dumpURIs(ref)
836 print(" outputs:")
837 for key, refs in node.quantum.outputs.items():
838 for ref in refs:
839 dumpURIs(ref)
841 def _importGraphFixup(self, args):
842 """Import/instantiate graph fixup object.
844 Parameters
845 ----------
846 args : `argparse.Namespace`
847 Parsed command line.
849 Returns
850 -------
851 fixup : `ExecutionGraphFixup` or `None`
853 Raises
854 ------
855 ValueError
856 Raised if import fails, method call raises exception, or returned
857 instance has unexpected type.
858 """
859 if args.graph_fixup:
860 try:
861 factory = doImport(args.graph_fixup)
862 except Exception as exc:
863 raise ValueError("Failed to import graph fixup class/method") from exc
864 try:
865 fixup = factory()
866 except Exception as exc:
867 raise ValueError("Failed to make instance of graph fixup") from exc
868 if not isinstance(fixup, ExecutionGraphFixup):
869 raise ValueError("Graph fixup is not an instance of ExecutionGraphFixup class")
870 return fixup