Coverage for python/lsst/ctrl/mpexec/cmdLineFwk.py : 12%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of ctrl_mpexec.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22"""Module defining CmdLineFwk class and related methods.
23"""
25__all__ = ['CmdLineFwk']
27# -------------------------------
28# Imports of standard modules --
29# -------------------------------
30import argparse
31import copy
32import fnmatch
33import logging
34import re
35import sys
36from typing import Optional, Tuple
37import warnings
39# -----------------------------
40# Imports for other modules --
41# -----------------------------
42from lsst.daf.butler import (
43 Butler,
44 CollectionSearch,
45 CollectionType,
46 Registry,
47)
48from lsst.daf.butler.registry import MissingCollectionError, RegistryDefaults
49import lsst.pex.config as pexConfig
50from lsst.pipe.base import GraphBuilder, Pipeline, QuantumGraph, buildExecutionButler
51from lsst.obs.base import Instrument
52from .dotTools import graph2dot, pipeline2dot
53from .executionGraphFixup import ExecutionGraphFixup
54from .mpGraphExecutor import MPGraphExecutor
55from .preExecInit import PreExecInit
56from .singleQuantumExecutor import SingleQuantumExecutor
57from . import util
58from lsst.utils import doImport
60# ----------------------------------
61# Local non-exported definitions --
62# ----------------------------------
64_LOG = logging.getLogger(__name__.partition(".")[2])
67class _OutputChainedCollectionInfo:
68 """A helper class for handling command-line arguments related to an output
69 `~lsst.daf.butler.CollectionType.CHAINED` collection.
71 Parameters
72 ----------
73 registry : `lsst.daf.butler.Registry`
74 Butler registry that collections will be added to and/or queried from.
75 name : `str`
76 Name of the collection given on the command line.
77 """
78 def __init__(self, registry: Registry, name: str):
79 self.name = name
80 try:
81 self.chain = tuple(registry.getCollectionChain(name))
82 self.exists = True
83 except MissingCollectionError:
84 self.chain = ()
85 self.exists = False
87 def __str__(self):
88 return self.name
90 name: str
91 """Name of the collection provided on the command line (`str`).
92 """
94 exists: bool
95 """Whether this collection already exists in the registry (`bool`).
96 """
98 chain: Tuple[str, ...]
99 """The definition of the collection, if it already exists (`tuple` [`str`]).
101 Empty if the collection does not already exist.
102 """
105class _OutputRunCollectionInfo:
106 """A helper class for handling command-line arguments related to an output
107 `~lsst.daf.butler.CollectionType.RUN` collection.
109 Parameters
110 ----------
111 registry : `lsst.daf.butler.Registry`
112 Butler registry that collections will be added to and/or queried from.
113 name : `str`
114 Name of the collection given on the command line.
115 """
116 def __init__(self, registry: Registry, name: str):
117 self.name = name
118 try:
119 actualType = registry.getCollectionType(name)
120 if actualType is not CollectionType.RUN:
121 raise TypeError(f"Collection '{name}' exists but has type {actualType.name}, not RUN.")
122 self.exists = True
123 except MissingCollectionError:
124 self.exists = False
126 name: str
127 """Name of the collection provided on the command line (`str`).
128 """
130 exists: bool
131 """Whether this collection already exists in the registry (`bool`).
132 """
135class _ButlerFactory:
136 """A helper class for processing command-line arguments related to input
137 and output collections.
139 Parameters
140 ----------
141 registry : `lsst.daf.butler.Registry`
142 Butler registry that collections will be added to and/or queried from.
144 args : `argparse.Namespace`
145 Parsed command-line arguments. The following attributes are used,
146 either at construction or in later methods.
148 ``output``
149 The name of a `~lsst.daf.butler.CollectionType.CHAINED`
150 input/output collection.
152 ``output_run``
153 The name of a `~lsst.daf.butler.CollectionType.RUN` input/output
154 collection.
156 ``extend_run``
157 A boolean indicating whether ``output_run`` should already exist
158 and be extended.
160 ``replace_run``
161 A boolean indicating that (if `True`) ``output_run`` should already
162 exist but will be removed from the output chained collection and
163 replaced with a new one.
165 ``prune_replaced``
166 A boolean indicating whether to prune the replaced run (requires
167 ``replace_run``).
169 ``inputs``
170 Input collections of any type; may be any type handled by
171 `lsst.daf.butler.registry.CollectionSearch.fromExpression`.
173 ``butler_config``
174 Path to a data repository root or configuration file.
176 writeable : `bool`
177 If `True`, a `Butler` is being initialized in a context where actual
178 writes should happens, and hence no output run is necessary.
180 Raises
181 ------
182 ValueError
183 Raised if ``writeable is True`` but there are no output collections.
184 """
185 def __init__(self, registry: Registry, args: argparse.Namespace, writeable: bool):
186 if args.output is not None:
187 self.output = _OutputChainedCollectionInfo(registry, args.output)
188 else:
189 self.output = None
190 if args.output_run is not None:
191 self.outputRun = _OutputRunCollectionInfo(registry, args.output_run)
192 elif self.output is not None:
193 if args.extend_run:
194 runName = self.output.chain[0]
195 else:
196 runName = "{}/{}".format(self.output, Instrument.makeCollectionTimestamp())
197 self.outputRun = _OutputRunCollectionInfo(registry, runName)
198 elif not writeable:
199 # If we're not writing yet, ok to have no output run.
200 self.outputRun = None
201 else:
202 raise ValueError("Cannot write without at least one of (--output, --output-run).")
203 # Recursively flatten any input CHAINED collections. We do this up
204 # front so we can tell if the user passes the same inputs on subsequent
205 # calls, even though we also flatten when we define the output CHAINED
206 # collection.
207 self.inputs = tuple(registry.queryCollections(args.input, flattenChains=True)) if args.input else ()
209 def check(self, args: argparse.Namespace):
210 """Check command-line options for consistency with each other and the
211 data repository.
213 Parameters
214 ----------
215 args : `argparse.Namespace`
216 Parsed command-line arguments. See class documentation for the
217 construction parameter of the same name.
218 """
219 assert not (args.extend_run and args.replace_run), "In mutually-exclusive group in ArgumentParser."
220 if self.inputs and self.output is not None and self.output.exists:
221 # Passing the same inputs that were used to initialize the output
222 # collection is allowed; this means they must _end_ with the same
223 # collections, because we push new runs to the front of the chain.
224 for c1, c2 in zip(self.inputs[::-1], self.output.chain[::-1]):
225 if c1 != c2:
226 raise ValueError(
227 f"Output CHAINED collection {self.output.name!r} exists, but it ends with "
228 "a different sequence of input collections than those given: "
229 f"{c1!r} != {c2!r} in inputs={self.inputs} vs "
230 f"{self.output.name}={self.output.chain}."
231 )
232 if len(self.inputs) > len(self.output.chain):
233 nNew = len(self.inputs) - len(self.output.chain)
234 raise ValueError(
235 f"Cannot add new input collections {self.inputs[:nNew]} after "
236 "output collection is first created."
237 )
238 if args.extend_run and self.outputRun is None:
239 raise ValueError("Cannot --extend-run when no output collection is given.")
240 if args.extend_run and not self.outputRun.exists:
241 raise ValueError(f"Cannot --extend-run; output collection "
242 f"'{self.outputRun.name}' does not exist.")
243 if not args.extend_run and self.outputRun is not None and self.outputRun.exists:
244 raise ValueError(f"Output run '{self.outputRun.name}' already exists, but "
245 f"--extend-run was not given.")
246 if args.prune_replaced and not args.replace_run:
247 raise ValueError("--prune-replaced requires --replace-run.")
248 if args.replace_run and (self.output is None or not self.output.exists):
249 raise ValueError("--output must point to an existing CHAINED collection for --replace-run.")
251 @classmethod
252 def _makeReadParts(cls, args: argparse.Namespace):
253 """Common implementation for `makeReadButler` and
254 `makeRegistryAndCollections`.
256 Parameters
257 ----------
258 args : `argparse.Namespace`
259 Parsed command-line arguments. See class documentation for the
260 construction parameter of the same name.
262 Returns
263 -------
264 butler : `lsst.daf.butler.Butler`
265 A read-only butler constructed from the repo at
266 ``args.butler_config``, but with no default collections.
267 inputs : `lsst.daf.butler.registry.CollectionSearch`
268 A collection search path constructed according to ``args``.
269 self : `_ButlerFactory`
270 A new `_ButlerFactory` instance representing the processed version
271 of ``args``.
272 """
273 butler = Butler(args.butler_config, writeable=False)
274 self = cls(butler.registry, args, writeable=False)
275 self.check(args)
276 if self.output and self.output.exists:
277 if args.replace_run:
278 replaced = self.output.chain[0]
279 inputs = self.output.chain[1:]
280 _LOG.debug("Simulating collection search in '%s' after removing '%s'.",
281 self.output.name, replaced)
282 else:
283 inputs = [self.output.name]
284 else:
285 inputs = list(self.inputs)
286 if args.extend_run:
287 inputs.insert(0, self.outputRun.name)
288 inputs = CollectionSearch.fromExpression(inputs)
289 return butler, inputs, self
291 @classmethod
292 def makeReadButler(cls, args: argparse.Namespace) -> Butler:
293 """Construct a read-only butler according to the given command-line
294 arguments.
296 Parameters
297 ----------
298 args : `argparse.Namespace`
299 Parsed command-line arguments. See class documentation for the
300 construction parameter of the same name.
302 Returns
303 -------
304 butler : `lsst.daf.butler.Butler`
305 A read-only butler initialized with the collections specified by
306 ``args``.
307 """
308 butler, inputs, _ = cls._makeReadParts(args)
309 _LOG.debug("Preparing butler to read from %s.", inputs)
310 return Butler(butler=butler, collections=inputs)
312 @classmethod
313 def makeRegistryAndCollections(cls, args: argparse.Namespace) -> \
314 Tuple[Registry, CollectionSearch, Optional[str]]:
315 """Return a read-only registry, a collection search path, and the name
316 of the run to be used for future writes.
318 Parameters
319 ----------
320 args : `argparse.Namespace`
321 Parsed command-line arguments. See class documentation for the
322 construction parameter of the same name.
324 Returns
325 -------
326 registry : `lsst.daf.butler.Registry`
327 Butler registry that collections will be added to and/or queried
328 from.
329 inputs : `lsst.daf.butler.registry.CollectionSearch`
330 Collections to search for datasets.
331 run : `str` or `None`
332 Name of the output `~lsst.daf.butler.CollectionType.RUN` collection
333 if it already exists, or `None` if it does not.
334 """
335 butler, inputs, self = cls._makeReadParts(args)
336 run = self.outputRun.name if args.extend_run else None
337 _LOG.debug("Preparing registry to read from %s and expect future writes to '%s'.", inputs, run)
338 return butler.registry, inputs, run
340 @classmethod
341 def makeWriteButler(cls, args: argparse.Namespace) -> Butler:
342 """Return a read-write butler initialized to write to and read from
343 the collections specified by the given command-line arguments.
345 Parameters
346 ----------
347 args : `argparse.Namespace`
348 Parsed command-line arguments. See class documentation for the
349 construction parameter of the same name.
351 Returns
352 -------
353 butler : `lsst.daf.butler.Butler`
354 A read-write butler initialized according to the given arguments.
355 """
356 butler = Butler(args.butler_config, writeable=True)
357 self = cls(butler.registry, args, writeable=True)
358 self.check(args)
359 if self.output is not None:
360 chainDefinition = list(self.output.chain if self.output.exists else self.inputs)
361 if args.replace_run:
362 replaced = chainDefinition.pop(0)
363 if args.prune_replaced == "unstore":
364 # Remove datasets from datastore
365 with butler.transaction():
366 refs = butler.registry.queryDatasets(..., collections=replaced)
367 butler.pruneDatasets(refs, unstore=True, run=replaced, disassociate=False)
368 elif args.prune_replaced == "purge":
369 # Erase entire collection and all datasets, need to remove
370 # collection from its chain collection first.
371 with butler.transaction():
372 butler.registry.setCollectionChain(self.output.name, chainDefinition, flatten=True)
373 butler.pruneCollection(replaced, purge=True, unstore=True)
374 elif args.prune_replaced is not None:
375 raise NotImplementedError(
376 f"Unsupported --prune-replaced option '{args.prune_replaced}'."
377 )
378 if not self.output.exists:
379 butler.registry.registerCollection(self.output.name, CollectionType.CHAINED)
380 if not args.extend_run:
381 butler.registry.registerCollection(self.outputRun.name, CollectionType.RUN)
382 chainDefinition.insert(0, self.outputRun.name)
383 butler.registry.setCollectionChain(self.output.name, chainDefinition, flatten=True)
384 _LOG.debug("Preparing butler to write to '%s' and read from '%s'=%s",
385 self.outputRun.name, self.output.name, chainDefinition)
386 butler.registry.defaults = RegistryDefaults(run=self.outputRun.name, collections=self.output.name)
387 else:
388 inputs = CollectionSearch.fromExpression((self.outputRun.name,) + self.inputs)
389 _LOG.debug("Preparing butler to write to '%s' and read from %s.", self.outputRun.name, inputs)
390 butler.registry.defaults = RegistryDefaults(run=self.outputRun.name, collections=inputs)
391 return butler
393 output: Optional[_OutputChainedCollectionInfo]
394 """Information about the output chained collection, if there is or will be
395 one (`_OutputChainedCollectionInfo` or `None`).
396 """
398 outputRun: Optional[_OutputRunCollectionInfo]
399 """Information about the output run collection, if there is or will be
400 one (`_OutputRunCollectionInfo` or `None`).
401 """
403 inputs: Tuple[str, ...]
404 """Input collections provided directly by the user (`tuple` [ `str` ]).
405 """
408class _FilteredStream:
409 """A file-like object that filters some config fields.
411 Note
412 ----
413 This class depends on implementation details of ``Config.saveToStream``
414 methods, in particular that that method uses single call to write()
415 method to save information about single config field, and that call
416 combines comments string(s) for a field and field path and value.
417 This class will not work reliably on the "import" strings, so imports
418 should be disabled by passing ``skipImports=True`` to ``saveToStream()``.
419 """
420 def __init__(self, pattern):
421 # obey case if pattern isn't lowercase or requests NOIGNORECASE
422 mat = re.search(r"(.*):NOIGNORECASE$", pattern)
424 if mat:
425 pattern = mat.group(1)
426 self._pattern = re.compile(fnmatch.translate(pattern))
427 else:
428 if pattern != pattern.lower():
429 print(f"Matching \"{pattern}\" without regard to case "
430 "(append :NOIGNORECASE to prevent this)", file=sys.stdout)
431 self._pattern = re.compile(fnmatch.translate(pattern), re.IGNORECASE)
433 def write(self, showStr):
434 # Strip off doc string line(s) and cut off at "=" for string matching
435 matchStr = showStr.rstrip().split("\n")[-1].split("=")[0]
436 if self._pattern.search(matchStr):
437 sys.stdout.write(showStr)
439# ------------------------
440# Exported definitions --
441# ------------------------
444class CmdLineFwk:
445 """PipelineTask framework which executes tasks from command line.
447 In addition to executing tasks this activator provides additional methods
448 for task management like dumping configuration or execution chain.
449 """
451 MP_TIMEOUT = 9999 # Default timeout (sec) for multiprocessing
453 def __init__(self):
454 pass
456 def makePipeline(self, args):
457 """Build a pipeline from command line arguments.
459 Parameters
460 ----------
461 args : `argparse.Namespace`
462 Parsed command line
464 Returns
465 -------
466 pipeline : `~lsst.pipe.base.Pipeline`
467 """
468 if args.pipeline:
469 pipeline = Pipeline.from_uri(args.pipeline)
470 else:
471 pipeline = Pipeline("anonymous")
473 # loop over all pipeline actions and apply them in order
474 for action in args.pipeline_actions:
475 if action.action == "add_instrument":
477 pipeline.addInstrument(action.value)
479 elif action.action == "new_task":
481 pipeline.addTask(action.value, action.label)
483 elif action.action == "delete_task":
485 pipeline.removeTask(action.label)
487 elif action.action == "config":
489 # action value string is "field=value", split it at '='
490 field, _, value = action.value.partition("=")
491 pipeline.addConfigOverride(action.label, field, value)
493 elif action.action == "configfile":
495 pipeline.addConfigFile(action.label, action.value)
497 else:
499 raise ValueError(f"Unexpected pipeline action: {action.action}")
501 if args.save_pipeline:
502 pipeline.write_to_uri(args.save_pipeline)
504 if args.pipeline_dot:
505 pipeline2dot(pipeline, args.pipeline_dot)
507 return pipeline
509 def makeGraph(self, pipeline, args):
510 """Build a graph from command line arguments.
512 Parameters
513 ----------
514 pipeline : `~lsst.pipe.base.Pipeline`
515 Pipeline, can be empty or ``None`` if graph is read from a file.
516 args : `argparse.Namespace`
517 Parsed command line
519 Returns
520 -------
521 graph : `~lsst.pipe.base.QuantumGraph` or `None`
522 If resulting graph is empty then `None` is returned.
523 """
525 registry, collections, run = _ButlerFactory.makeRegistryAndCollections(args)
527 if args.qgraph:
528 # click passes empty tuple as default value for qgraph_node_id
529 nodes = args.qgraph_node_id or None
530 qgraph = QuantumGraph.loadUri(args.qgraph, registry.dimensions,
531 nodes=nodes, graphID=args.qgraph_id)
533 # pipeline can not be provided in this case
534 if pipeline:
535 raise ValueError("Pipeline must not be given when quantum graph is read from file.")
537 else:
539 # make execution plan (a.k.a. DAG) for pipeline
540 graphBuilder = GraphBuilder(registry,
541 skipExisting=args.skip_existing)
542 qgraph = graphBuilder.makeGraph(pipeline, collections, run, args.data_query)
544 # count quanta in graph and give a warning if it's empty and return None
545 nQuanta = len(qgraph)
546 if nQuanta == 0:
547 warnings.warn("QuantumGraph is empty", stacklevel=2)
548 return None
549 else:
550 _LOG.info("QuantumGraph contains %d quanta for %d tasks, graph ID: %r",
551 nQuanta, len(qgraph.taskGraph), qgraph.graphID)
553 if args.save_qgraph:
554 qgraph.saveUri(args.save_qgraph)
556 if args.save_single_quanta:
557 for quantumNode in qgraph:
558 sqgraph = qgraph.subset(quantumNode)
559 uri = args.save_single_quanta.format(quantumNode.nodeId.number)
560 sqgraph.saveUri(uri)
562 if args.qgraph_dot:
563 graph2dot(qgraph, args.qgraph_dot)
565 if args.execution_butler_location:
566 butler = Butler(args.butler_config)
567 newArgs = copy.deepcopy(args)
569 def builderShim(butler):
570 newArgs.butler_config = butler._config
571 # Calling makeWriteButler is done for the side effects of
572 # calling that method, maining parsing all the args into
573 # collection names, creating collections, etc.
574 newButler = _ButlerFactory.makeWriteButler(newArgs)
575 return newButler
577 buildExecutionButler(butler, qgraph, args.execution_butler_location, run,
578 butlerModifier=builderShim, collections=args.input,
579 clobber=args.clobber_execution_butler)
581 return qgraph
583 def runPipeline(self, graph, taskFactory, args, butler=None):
584 """Execute complete QuantumGraph.
586 Parameters
587 ----------
588 graph : `QuantumGraph`
589 Execution graph.
590 taskFactory : `~lsst.pipe.base.TaskFactory`
591 Task factory
592 args : `argparse.Namespace`
593 Parsed command line
594 butler : `~lsst.daf.butler.Butler`, optional
595 Data Butler instance, if not defined then new instance is made
596 using command line options.
597 """
598 # make butler instance
599 if butler is None:
600 butler = _ButlerFactory.makeWriteButler(args)
602 # Enable lsstDebug debugging. Note that this is done once in the
603 # main process before PreExecInit and it is also repeated before
604 # running each task in SingleQuantumExecutor (which may not be
605 # needed if `multipocessing` always uses fork start method).
606 if args.enableLsstDebug:
607 try:
608 _LOG.debug("Will try to import debug.py")
609 import debug # noqa:F401
610 except ImportError:
611 _LOG.warn("No 'debug' module found.")
613 # --skip-existing should have no effect unless --extend-run is passed
614 # so we make PreExecInit's skipExisting depend on the latter as well.
615 preExecInit = PreExecInit(butler, taskFactory, skipExisting=(args.skip_existing and args.extend_run))
616 preExecInit.initialize(graph,
617 saveInitOutputs=not args.skip_init_writes,
618 registerDatasetTypes=args.register_dataset_types,
619 saveVersions=not args.no_versions)
621 if not args.init_only:
622 graphFixup = self._importGraphFixup(args)
623 quantumExecutor = SingleQuantumExecutor(taskFactory,
624 skipExisting=args.skip_existing,
625 clobberPartialOutputs=args.clobber_partial_outputs,
626 enableLsstDebug=args.enableLsstDebug)
627 timeout = self.MP_TIMEOUT if args.timeout is None else args.timeout
628 executor = MPGraphExecutor(numProc=args.processes, timeout=timeout,
629 startMethod=args.start_method,
630 quantumExecutor=quantumExecutor,
631 failFast=args.fail_fast,
632 executionGraphFixup=graphFixup)
633 with util.profile(args.profile, _LOG):
634 executor.execute(graph, butler)
636 def showInfo(self, args, pipeline, graph=None):
637 """Display useful info about pipeline and environment.
639 Parameters
640 ----------
641 args : `argparse.Namespace`
642 Parsed command line
643 pipeline : `Pipeline`
644 Pipeline definition
645 graph : `QuantumGraph`, optional
646 Execution graph
647 """
648 showOpts = args.show
649 for what in showOpts:
650 showCommand, _, showArgs = what.partition("=")
652 if showCommand in ["pipeline", "config", "history", "tasks"]:
653 if not pipeline:
654 _LOG.warning("Pipeline is required for --show=%s", showCommand)
655 continue
657 if showCommand in ["graph", "workflow", "uri"]:
658 if not graph:
659 _LOG.warning("QuantumGraph is required for --show=%s", showCommand)
660 continue
662 if showCommand == "pipeline":
663 print(pipeline)
664 elif showCommand == "config":
665 self._showConfig(pipeline, showArgs, False)
666 elif showCommand == "dump-config":
667 self._showConfig(pipeline, showArgs, True)
668 elif showCommand == "history":
669 self._showConfigHistory(pipeline, showArgs)
670 elif showCommand == "tasks":
671 self._showTaskHierarchy(pipeline)
672 elif showCommand == "graph":
673 if graph:
674 self._showGraph(graph)
675 elif showCommand == "uri":
676 if graph:
677 self._showUri(graph, args)
678 elif showCommand == "workflow":
679 if graph:
680 self._showWorkflow(graph, args)
681 else:
682 print("Unknown value for show: %s (choose from '%s')" %
683 (what, "', '".join("pipeline config[=XXX] history=XXX tasks graph".split())),
684 file=sys.stderr)
685 sys.exit(1)
687 def _showConfig(self, pipeline, showArgs, dumpFullConfig):
688 """Show task configuration
690 Parameters
691 ----------
692 pipeline : `Pipeline`
693 Pipeline definition
694 showArgs : `str`
695 Defines what to show
696 dumpFullConfig : `bool`
697 If true then dump complete task configuration with all imports.
698 """
699 stream = sys.stdout
700 if dumpFullConfig:
701 # Task label can be given with this option
702 taskName = showArgs
703 else:
704 # The argument can have form [TaskLabel::][pattern:NOIGNORECASE]
705 matConfig = re.search(r"^(?:(\w+)::)?(?:config.)?(.+)?", showArgs)
706 taskName = matConfig.group(1)
707 pattern = matConfig.group(2)
708 if pattern:
709 stream = _FilteredStream(pattern)
711 tasks = util.filterTasks(pipeline, taskName)
712 if not tasks:
713 print("Pipeline has no tasks named {}".format(taskName), file=sys.stderr)
714 sys.exit(1)
716 for taskDef in tasks:
717 print("### Configuration for task `{}'".format(taskDef.label))
718 taskDef.config.saveToStream(stream, root="config", skipImports=not dumpFullConfig)
720 def _showConfigHistory(self, pipeline, showArgs):
721 """Show history for task configuration
723 Parameters
724 ----------
725 pipeline : `Pipeline`
726 Pipeline definition
727 showArgs : `str`
728 Defines what to show
729 """
731 taskName = None
732 pattern = None
733 matHistory = re.search(r"^(?:(\w+)::)?(?:config[.])?(.+)", showArgs)
734 if matHistory:
735 taskName = matHistory.group(1)
736 pattern = matHistory.group(2)
737 if not pattern:
738 print("Please provide a value with --show history (e.g. history=Task::param)", file=sys.stderr)
739 sys.exit(1)
741 tasks = util.filterTasks(pipeline, taskName)
742 if not tasks:
743 print(f"Pipeline has no tasks named {taskName}", file=sys.stderr)
744 sys.exit(1)
746 found = False
747 for taskDef in tasks:
749 config = taskDef.config
751 # Look for any matches in the config hierarchy for this name
752 for nmatch, thisName in enumerate(fnmatch.filter(config.names(), pattern)):
753 if nmatch > 0:
754 print("")
756 cpath, _, cname = thisName.rpartition(".")
757 try:
758 if not cpath:
759 # looking for top-level field
760 hconfig = taskDef.config
761 else:
762 hconfig = eval("config." + cpath, {}, {"config": config})
763 except AttributeError:
764 print(f"Error: Unable to extract attribute {cpath} from task {taskDef.label}",
765 file=sys.stderr)
766 hconfig = None
768 # Sometimes we end up with a non-Config so skip those
769 if isinstance(hconfig, (pexConfig.Config, pexConfig.ConfigurableInstance)) and \
770 hasattr(hconfig, cname):
771 print(f"### Configuration field for task `{taskDef.label}'")
772 print(pexConfig.history.format(hconfig, cname))
773 found = True
775 if not found:
776 print(f"None of the tasks has field matching {pattern}", file=sys.stderr)
777 sys.exit(1)
779 def _showTaskHierarchy(self, pipeline):
780 """Print task hierarchy to stdout
782 Parameters
783 ----------
784 pipeline: `Pipeline`
785 """
786 for taskDef in pipeline.toExpandedPipeline():
787 print("### Subtasks for task `{}'".format(taskDef.taskName))
789 for configName, taskName in util.subTaskIter(taskDef.config):
790 print("{}: {}".format(configName, taskName))
792 def _showGraph(self, graph):
793 """Print quanta information to stdout
795 Parameters
796 ----------
797 graph : `QuantumGraph`
798 Execution graph.
799 """
800 for taskNode in graph.taskGraph:
801 print(taskNode)
803 for iq, quantum in enumerate(graph.getQuantaForTask(taskNode)):
804 print(" Quantum {}:".format(iq))
805 print(" inputs:")
806 for key, refs in quantum.inputs.items():
807 dataIds = ["DataId({})".format(ref.dataId) for ref in refs]
808 print(" {}: [{}]".format(key, ", ".join(dataIds)))
809 print(" outputs:")
810 for key, refs in quantum.outputs.items():
811 dataIds = ["DataId({})".format(ref.dataId) for ref in refs]
812 print(" {}: [{}]".format(key, ", ".join(dataIds)))
814 def _showWorkflow(self, graph, args):
815 """Print quanta information and dependency to stdout
817 Parameters
818 ----------
819 graph : `QuantumGraph`
820 Execution graph.
821 args : `argparse.Namespace`
822 Parsed command line
823 """
824 for node in graph:
825 print(f"Quantum {node.nodeId.number}: {node.taskDef.taskName}")
826 for parent in graph.determineInputsToQuantumNode(node):
827 print(f"Parent Quantum {parent.nodeId.number} - Child Quantum {node.nodeId.number}")
829 def _showUri(self, graph, args):
830 """Print input and predicted output URIs to stdout
832 Parameters
833 ----------
834 graph : `QuantumGraph`
835 Execution graph
836 args : `argparse.Namespace`
837 Parsed command line
838 """
839 def dumpURIs(thisRef):
840 primary, components = butler.getURIs(thisRef, predict=True, run="TBD")
841 if primary:
842 print(f" {primary}")
843 else:
844 print(" (disassembled artifact)")
845 for compName, compUri in components.items():
846 print(f" {compName}: {compUri}")
848 butler = _ButlerFactory.makeReadButler(args)
849 for node in graph:
850 print(f"Quantum {node.nodeId.number}: {node.taskDef.taskName}")
851 print(" inputs:")
852 for key, refs in node.quantum.inputs.items():
853 for ref in refs:
854 dumpURIs(ref)
855 print(" outputs:")
856 for key, refs in node.quantum.outputs.items():
857 for ref in refs:
858 dumpURIs(ref)
860 def _importGraphFixup(self, args):
861 """Import/instantiate graph fixup object.
863 Parameters
864 ----------
865 args : `argparse.Namespace`
866 Parsed command line.
868 Returns
869 -------
870 fixup : `ExecutionGraphFixup` or `None`
872 Raises
873 ------
874 ValueError
875 Raised if import fails, method call raises exception, or returned
876 instance has unexpected type.
877 """
878 if args.graph_fixup:
879 try:
880 factory = doImport(args.graph_fixup)
881 except Exception as exc:
882 raise ValueError("Failed to import graph fixup class/method") from exc
883 try:
884 fixup = factory()
885 except Exception as exc:
886 raise ValueError("Failed to make instance of graph fixup") from exc
887 if not isinstance(fixup, ExecutionGraphFixup):
888 raise ValueError("Graph fixup is not an instance of ExecutionGraphFixup class")
889 return fixup