Coverage for python/lsst/ctrl/mpexec/cmdLineFwk.py : 12%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of ctrl_mpexec.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22"""Module defining CmdLineFwk class and related methods.
23"""
25__all__ = ['CmdLineFwk']
27# -------------------------------
28# Imports of standard modules --
29# -------------------------------
30import argparse
31import datetime
32import fnmatch
33import logging
34import os
35import re
36import sys
37from typing import List, Optional, Tuple
38import warnings
40# -----------------------------
41# Imports for other modules --
42# -----------------------------
43from lsst.daf.butler import (
44 Butler,
45 CollectionSearch,
46 CollectionType,
47 DatasetTypeRestriction,
48 Registry,
49)
50from lsst.daf.butler.registry import MissingCollectionError
51import lsst.log
52import lsst.pex.config as pexConfig
53from lsst.pipe.base import GraphBuilder, Pipeline, QuantumGraph
54from .cmdLineParser import makeParser
55from .dotTools import graph2dot, pipeline2dot
56from .executionGraphFixup import ExecutionGraphFixup
57from .mpGraphExecutor import MPGraphExecutor
58from .preExecInit import PreExecInit
59from .singleQuantumExecutor import SingleQuantumExecutor
60from .taskFactory import TaskFactory
61from . import util
62from lsst.utils import doImport
64# ----------------------------------
65# Local non-exported definitions --
66# ----------------------------------
68# logging properties
69_LOG_PROP = """\
70log4j.rootLogger=INFO, A1
71log4j.appender.A1=ConsoleAppender
72log4j.appender.A1.Target=System.err
73log4j.appender.A1.layout=PatternLayout
74log4j.appender.A1.layout.ConversionPattern={}
75"""
77_LOG = logging.getLogger(__name__.partition(".")[2])
80class _OutputChainedCollectionInfo:
81 """A helper class for handling command-line arguments related to an output
82 `~lsst.daf.butler.CollectionType.CHAINED` collection.
84 Parameters
85 ----------
86 registry : `lsst.daf.butler.Registry`
87 Butler registry that collections will be added to and/or queried from.
88 name : `str`
89 Name of the collection given on the command line.
90 """
91 def __init__(self, registry: Registry, name: str):
92 self.name = name
93 try:
94 self.chain = list(registry.getCollectionChain(name))
95 self.exists = True
96 except MissingCollectionError:
97 self.chain = []
98 self.exists = False
100 def __str__(self):
101 return self.name
103 name: str
104 """Name of the collection provided on the command line (`str`).
105 """
107 exists: bool
108 """Whether this collection already exists in the registry (`bool`).
109 """
111 chain: List[Tuple[str, DatasetTypeRestriction]]
112 """The definition of the collection, if it already exists (`list`).
114 Empty if the collection does not alredy exist.
115 """
118class _OutputRunCollectionInfo:
119 """A helper class for handling command-line arguments related to an output
120 `~lsst.daf.butler.CollectionType.RUN` collection.
122 Parameters
123 ----------
124 registry : `lsst.daf.butler.Registry`
125 Butler registry that collections will be added to and/or queried from.
126 name : `str`
127 Name of the collection given on the command line.
128 """
129 def __init__(self, registry: Registry, name: str):
130 self.name = name
131 try:
132 actualType = registry.getCollectionType(name)
133 if actualType is not CollectionType.RUN:
134 raise TypeError(f"Collection '{name}' exists but has type {actualType.name}, not RUN.")
135 self.exists = True
136 except MissingCollectionError:
137 self.exists = False
139 name: str
140 """Name of the collection provided on the command line (`str`).
141 """
143 exists: bool
144 """Whether this collection already exists in the registry (`bool`).
145 """
148class _ButlerFactory:
149 """A helper class for processing command-line arguments related to input
150 and output collections.
152 Parameters
153 ----------
154 registry : `lsst.daf.butler.Registry`
155 Butler registry that collections will be added to and/or queried from.
157 args : `argparse.Namespace`
158 Parsed command-line arguments. The following attributes are used,
159 either at construction or in later methods.
161 ``output``
162 The name of a `~lsst.daf.butler.CollectionType.CHAINED`
163 input/output collection.
165 ``output_run``
166 The name of a `~lsst.daf.butler.CollectionType.RUN` input/output
167 collection.
169 ``extend_run``
170 A boolean indicating whether ``output_run`` should already exist
171 and be extended.
173 ``replace_run``
174 A boolean indicating that (if `True`) ``output_run`` should already
175 exist but will be removed from the output chained collection and
176 replaced with a new one.
178 ``prune_replaced``
179 A boolean indicating whether to prune the replaced run (requires
180 ``replace_run``).
182 ``inputs``
183 Input collections of any type; may be any type handled by
184 `lsst.daf.butler.registry.CollectionSearch.fromExpression`.
186 ``butler_config``
187 Path to a data repository root or configuration file.
189 writeable : `bool`
190 If `True`, a `Butler` is being initialized in a context where actual
191 writes should happens, and hence no output run is necessary.
193 Raises
194 ------
195 ValueError
196 Raised if ``writeable is True`` but there are no output collections.
197 """
198 def __init__(self, registry: Registry, args: argparse.Namespace, writeable: bool):
199 if args.output is not None:
200 self.output = _OutputChainedCollectionInfo(registry, args.output)
201 else:
202 self.output = None
203 if args.output_run is not None:
204 self.outputRun = _OutputRunCollectionInfo(registry, args.output_run)
205 elif self.output is not None:
206 if args.extend_run:
207 runName, _ = self.output.chain[0]
208 else:
209 runName = "{}/{:%Y%m%dT%Hh%Mm%Ss}".format(self.output, datetime.datetime.now())
210 self.outputRun = _OutputRunCollectionInfo(registry, runName)
211 elif not writeable:
212 # If we're not writing yet, ok to have no output run.
213 self.outputRun = None
214 else:
215 raise ValueError("Cannot write without at least one of (--output, --output-run).")
216 self.inputs = list(CollectionSearch.fromExpression(args.input))
218 def check(self, args: argparse.Namespace):
219 """Check command-line options for consistency with each other and the
220 data repository.
222 Parameters
223 ----------
224 args : `argparse.Namespace`
225 Parsed command-line arguments. See class documentation for the
226 construction parameter of the same name.
227 """
228 assert not (args.extend_run and args.replace_run), "In mutually-exclusive group in ArgumentParser."
229 if self.inputs and self.output is not None and self.output.exists:
230 raise ValueError("Cannot use --output with existing collection with --inputs.")
231 if args.extend_run and self.outputRun is None:
232 raise ValueError("Cannot --extend-run when no output collection is given.")
233 if args.extend_run and not self.outputRun.exists:
234 raise ValueError(f"Cannot --extend-run; output collection "
235 f"'{self.outputRun.name}' does not exist.")
236 if not args.extend_run and self.outputRun is not None and self.outputRun.exists:
237 raise ValueError(f"Output run '{self.outputRun.name}' already exists, but "
238 f"--extend-run was not given.")
239 if args.prune_replaced and not args.replace_run:
240 raise ValueError(f"--prune-replaced requires --replace-run.")
241 if args.replace_run and (self.output is None or not self.output.exists):
242 raise ValueError(f"--output must point to an existing CHAINED collection for --replace-run.")
244 @classmethod
245 def _makeReadParts(cls, args: argparse.Namespace):
246 """Common implementation for `makeReadButler` and
247 `makeRegistryAndCollections`.
249 Parameters
250 ----------
251 args : `argparse.Namespace`
252 Parsed command-line arguments. See class documentation for the
253 construction parameter of the same name.
255 Returns
256 -------
257 butler : `lsst.daf.butler.Butler`
258 A read-only butler constructed from the repo at
259 ``args.butler_config``, but with no default collections.
260 inputs : `lsst.daf.butler.registry.CollectionSearch`
261 A collection search path constructed according to ``args``.
262 self : `_ButlerFactory`
263 A new `_ButlerFactory` instance representing the processed version
264 of ``args``.
265 """
266 butler = Butler(args.butler_config, writeable=False)
267 self = cls(butler.registry, args, writeable=False)
268 self.check(args)
269 if self.output and self.output.exists:
270 if args.replace_run:
271 replaced, _ = self.output.chain[0]
272 inputs = self.output.chain[1:]
273 _LOG.debug("Simulating collection search in '%s' after removing '%s'.",
274 self.output.name, replaced)
275 else:
276 inputs = [self.output.name]
277 else:
278 inputs = list(self.inputs)
279 if args.extend_run:
280 inputs.insert(0, self.outputRun.name)
281 inputs = CollectionSearch.fromExpression(inputs)
282 return butler, inputs, self
284 @classmethod
285 def makeReadButler(cls, args: argparse.Namespace) -> Butler:
286 """Construct a read-only butler according to the given command-line
287 arguments.
289 Parameters
290 ----------
291 args : `argparse.Namespace`
292 Parsed command-line arguments. See class documentation for the
293 construction parameter of the same name.
295 Returns
296 -------
297 butler : `lsst.daf.butler.Butler`
298 A read-only butler initialized with the collections specified by
299 ``args``.
300 """
301 butler, inputs, _ = cls._makeReadParts(args)
302 _LOG.debug("Preparing butler to read from %s.", inputs)
303 return Butler(butler=butler, collections=inputs)
305 @classmethod
306 def makeRegistryAndCollections(cls, args: argparse.Namespace) -> \
307 Tuple[Registry, CollectionSearch, Optional[str]]:
308 """Return a read-only registry, a collection search path, and the name
309 of the run to be used for future writes.
311 Parameters
312 ----------
313 args : `argparse.Namespace`
314 Parsed command-line arguments. See class documentation for the
315 construction parameter of the same name.
317 Returns
318 -------
319 registry : `lsst.daf.butler.Registry`
320 Butler registry that collections will be added to and/or queried
321 from.
322 inputs : `lsst.daf.butler.registry.CollectionSearch`
323 Collections to search for datasets.
324 run : `str` or `None`
325 Name of the output `~lsst.daf.butler.CollectionType.RUN` collection
326 if it already exists, or `None` if it does not.
327 """
328 butler, inputs, self = cls._makeReadParts(args)
329 run = self.outputRun.name if args.extend_run else None
330 _LOG.debug("Preparing registry to read from %s and expect future writes to '%s'.", inputs, run)
331 return butler.registry, inputs, run
333 @classmethod
334 def makeWriteButler(cls, args: argparse.Namespace) -> Butler:
335 """Return a read-write butler initialized to write to and read from
336 the collections specified by the given command-line arguments.
338 Parameters
339 ----------
340 args : `argparse.Namespace`
341 Parsed command-line arguments. See class documentation for the
342 construction parameter of the same name.
344 Returns
345 -------
346 butler : `lsst.daf.butler.Butler`
347 A read-write butler initialized according to the given arguments.
348 """
349 butler = Butler(args.butler_config, writeable=True)
350 self = cls(butler.registry, args, writeable=True)
351 self.check(args)
352 if self.output is not None:
353 chainDefinition = list(self.output.chain if self.output.exists else self.inputs)
354 if args.replace_run:
355 replaced, _ = chainDefinition.pop(0)
356 if args.prune_replaced:
357 # TODO: DM-23671: need a butler API for pruning an
358 # entire RUN collection, then apply it to 'replaced'
359 # here.
360 raise NotImplementedError("Support for --prune-replaced is not yet implemented.")
361 chainDefinition.insert(0, self.outputRun.name)
362 chainDefinition = CollectionSearch.fromExpression(chainDefinition)
363 _LOG.debug("Preparing butler to write to '%s' and read from '%s'=%s",
364 self.outputRun.name, self.output.name, chainDefinition)
365 return Butler(butler=butler, run=self.outputRun.name, collections=self.output.name,
366 chains={self.output.name: chainDefinition})
367 else:
368 inputs = CollectionSearch.fromExpression([self.outputRun.name] + self.inputs)
369 _LOG.debug("Preparing butler to write to '%s' and read from %s.", self.outputRun.name, inputs)
370 return Butler(butler=butler, run=self.outputRun.name, collections=inputs)
372 output: Optional[_OutputChainedCollectionInfo]
373 """Information about the output chained collection, if there is or will be
374 one (`_OutputChainedCollectionInfo` or `None`).
375 """
377 outputRun: Optional[_OutputRunCollectionInfo]
378 """Information about the output run collection, if there is or will be
379 one (`_OutputRunCollectionInfo` or `None`).
380 """
382 inputs: List[Tuple[str, DatasetTypeRestriction]]
383 """Input collections, including those also used for outputs and any
384 restrictions on dataset types (`list`).
385 """
388class _FilteredStream:
389 """A file-like object that filters some config fields.
391 Note
392 ----
393 This class depends on implementation details of ``Config.saveToStream``
394 methods, in particular that that method uses single call to write()
395 method to save information about single config field, and that call
396 combines comments string(s) for a field and field path and value.
397 This class will not work reliably on the "import" strings, so imports
398 should be disabled by passing ``skipImports=True`` to ``saveToStream()``.
399 """
400 def __init__(self, pattern):
401 # obey case if pattern isn't lowercase or requests NOIGNORECASE
402 mat = re.search(r"(.*):NOIGNORECASE$", pattern)
404 if mat:
405 pattern = mat.group(1)
406 self._pattern = re.compile(fnmatch.translate(pattern))
407 else:
408 if pattern != pattern.lower():
409 print(f"Matching \"{pattern}\" without regard to case "
410 "(append :NOIGNORECASE to prevent this)", file=sys.stdout)
411 self._pattern = re.compile(fnmatch.translate(pattern), re.IGNORECASE)
413 def write(self, showStr):
414 # Strip off doc string line(s) and cut off at "=" for string matching
415 matchStr = showStr.rstrip().split("\n")[-1].split("=")[0]
416 if self._pattern.search(matchStr):
417 sys.stdout.write(showStr)
419# ------------------------
420# Exported definitions --
421# ------------------------
424class CmdLineFwk:
425 """PipelineTask framework which executes tasks from command line.
427 In addition to executing tasks this activator provides additional methods
428 for task management like dumping configuration or execution chain.
429 """
431 MP_TIMEOUT = 9999 # Default timeout (sec) for multiprocessing
433 def __init__(self):
434 pass
436 def parseAndRun(self, argv=None):
437 """
438 This method is a main entry point for this class, it parses command
439 line and executes all commands.
441 Parameters
442 ----------
443 argv : `list` of `str`, optional
444 list of command line arguments, if not specified then
445 `sys.argv[1:]` is used
446 """
448 if argv is None:
449 argv = sys.argv[1:]
451 # start with parsing command line, only do partial parsing now as
452 # the tasks can add more arguments later
453 parser = makeParser()
454 args = parser.parse_args(argv)
456 # First thing to do is to setup logging.
457 self.configLog(args.longlog, args.loglevel)
459 taskFactory = TaskFactory()
461 # make pipeline out of command line arguments (can return empty pipeline)
462 try:
463 pipeline = self.makePipeline(args)
464 except Exception as exc:
465 print("Failed to build pipeline: {}".format(exc), file=sys.stderr)
466 raise
468 if args.subcommand == "build":
469 # stop here but process --show option first
470 self.showInfo(args, pipeline)
471 return 0
473 # make quantum graph
474 try:
475 qgraph = self.makeGraph(pipeline, args)
476 except Exception as exc:
477 print("Failed to build graph: {}".format(exc), file=sys.stderr)
478 raise
480 # optionally dump some info
481 self.showInfo(args, pipeline, qgraph)
483 if qgraph is None:
484 # No need to raise an exception here, code that makes graph
485 # should have printed warning message already.
486 return 2
488 if args.subcommand == "qgraph":
489 # stop here
490 return 0
492 # execute
493 if args.subcommand == "run":
494 return self.runPipeline(qgraph, taskFactory, args)
496 @staticmethod
497 def configLog(longlog, logLevels):
498 """Configure logging system.
500 Parameters
501 ----------
502 longlog : `bool`
503 If True then make log messages appear in "long format"
504 logLevels : `list` of `tuple`
505 per-component logging levels, each item in the list is a tuple
506 (component, level), `component` is a logger name or `None` for root
507 logger, `level` is a logging level name ('DEBUG', 'INFO', etc.)
508 """
509 if longlog:
510 message_fmt = "%-5p %d{yyyy-MM-ddTHH:mm:ss.SSSZ} %c (%X{LABEL})(%F:%L)- %m%n"
511 else:
512 message_fmt = "%c %p: %m%n"
514 # Initialize global logging config. Skip if the env var LSST_LOG_CONFIG exists.
515 # The file it points to would already configure lsst.log.
516 if not os.path.isfile(os.environ.get("LSST_LOG_CONFIG", "")):
517 lsst.log.configure_prop(_LOG_PROP.format(message_fmt))
519 # Forward all Python logging to lsst.log
520 lgr = logging.getLogger()
521 lgr.setLevel(logging.INFO) # same as in log4cxx config above
522 lgr.addHandler(lsst.log.LogHandler())
524 # also capture warnings and send them to logging
525 logging.captureWarnings(True)
527 # configure individual loggers
528 for component, level in logLevels:
529 level = getattr(lsst.log.Log, level.upper(), None)
530 if level is not None:
531 # set logging level for lsst.log
532 logger = lsst.log.Log.getLogger(component or "")
533 logger.setLevel(level)
534 # set logging level for Python logging
535 pyLevel = lsst.log.LevelTranslator.lsstLog2logging(level)
536 logging.getLogger(component).setLevel(pyLevel)
538 def makePipeline(self, args):
539 """Build a pipeline from command line arguments.
541 Parameters
542 ----------
543 args : `argparse.Namespace`
544 Parsed command line
546 Returns
547 -------
548 pipeline : `~lsst.pipe.base.Pipeline`
549 """
550 if args.pipeline:
551 pipeline = Pipeline.fromFile(args.pipeline)
552 else:
553 pipeline = Pipeline("anonymous")
555 # loop over all pipeline actions and apply them in order
556 for action in args.pipeline_actions:
557 if action.action == "add_instrument":
559 pipeline.addInstrument(action.value)
561 elif action.action == "new_task":
563 pipeline.addTask(action.value, action.label)
565 elif action.action == "delete_task":
567 pipeline.removeTask(action.label)
569 elif action.action == "config":
571 # action value string is "field=value", split it at '='
572 field, _, value = action.value.partition("=")
573 pipeline.addConfigOverride(action.label, field, value)
575 elif action.action == "configfile":
577 pipeline.addConfigFile(action.label, action.value)
579 else:
581 raise ValueError(f"Unexpected pipeline action: {action.action}")
583 if args.save_pipeline:
584 pipeline.toFile(args.save_pipeline)
586 if args.pipeline_dot:
587 pipeline2dot(pipeline, args.pipeline_dot)
589 return pipeline
591 def makeGraph(self, pipeline, args):
592 """Build a graph from command line arguments.
594 Parameters
595 ----------
596 pipeline : `~lsst.pipe.base.Pipeline`
597 Pipeline, can be empty or ``None`` if graph is read from a file.
598 args : `argparse.Namespace`
599 Parsed command line
601 Returns
602 -------
603 graph : `~lsst.pipe.base.QuantumGraph` or `None`
604 If resulting graph is empty then `None` is returned.
605 """
607 registry, collections, run = _ButlerFactory.makeRegistryAndCollections(args)
609 if args.qgraph:
611 with open(args.qgraph, 'rb') as pickleFile:
612 qgraph = QuantumGraph.load(pickleFile, registry.dimensions)
614 # pipeline can not be provided in this case
615 if pipeline:
616 raise ValueError("Pipeline must not be given when quantum graph is read from file.")
618 else:
620 # make execution plan (a.k.a. DAG) for pipeline
621 graphBuilder = GraphBuilder(registry,
622 skipExisting=args.skip_existing)
623 qgraph = graphBuilder.makeGraph(pipeline, collections, run, args.data_query)
625 # count quanta in graph and give a warning if it's empty and return None
626 nQuanta = qgraph.countQuanta()
627 if nQuanta == 0:
628 warnings.warn("QuantumGraph is empty", stacklevel=2)
629 return None
630 else:
631 _LOG.info("QuantumGraph contains %d quanta for %d tasks",
632 nQuanta, len(qgraph))
634 if args.save_qgraph:
635 with open(args.save_qgraph, "wb") as pickleFile:
636 qgraph.save(pickleFile)
638 if args.save_single_quanta:
639 for iq, sqgraph in enumerate(qgraph.quantaAsQgraph()):
640 filename = args.save_single_quanta.format(iq)
641 with open(filename, "wb") as pickleFile:
642 sqgraph.save(pickleFile)
644 if args.qgraph_dot:
645 graph2dot(qgraph, args.qgraph_dot)
647 return qgraph
649 def runPipeline(self, graph, taskFactory, args, butler=None):
650 """Execute complete QuantumGraph.
652 Parameters
653 ----------
654 graph : `QuantumGraph`
655 Execution graph.
656 taskFactory : `~lsst.pipe.base.TaskFactory`
657 Task factory
658 args : `argparse.Namespace`
659 Parsed command line
660 butler : `~lsst.daf.butler.Butler`, optional
661 Data Butler instance, if not defined then new instance is made
662 using command line options.
663 """
664 # make butler instance
665 if butler is None:
666 butler = _ButlerFactory.makeWriteButler(args)
668 # Enable lsstDebug debugging. Note that this is done once in the
669 # main process before PreExecInit and it is also repeated before
670 # running each task in SingleQuantumExecutor (which may not be
671 # needed if `multipocessing` always uses fork start method).
672 if args.enableLsstDebug:
673 try:
674 _LOG.debug("Will try to import debug.py")
675 import debug # noqa:F401
676 except ImportError:
677 _LOG.warn("No 'debug' module found.")
679 preExecInit = PreExecInit(butler, taskFactory, args.skip_existing)
680 preExecInit.initialize(graph,
681 saveInitOutputs=not args.skip_init_writes,
682 registerDatasetTypes=args.register_dataset_types,
683 saveVersions=not args.no_versions)
685 if not args.init_only:
686 graphFixup = self._importGraphFixup(args)
687 quantumExecutor = SingleQuantumExecutor(taskFactory,
688 skipExisting=args.skip_existing,
689 clobberPartialOutputs=args.clobber_partial_outputs,
690 enableLsstDebug=args.enableLsstDebug)
691 timeout = self.MP_TIMEOUT if args.timeout is None else args.timeout
692 executor = MPGraphExecutor(numProc=args.processes, timeout=timeout,
693 quantumExecutor=quantumExecutor,
694 failFast=args.fail_fast,
695 executionGraphFixup=graphFixup)
696 with util.profile(args.profile, _LOG):
697 executor.execute(graph, butler)
699 def showInfo(self, args, pipeline, graph=None):
700 """Display useful info about pipeline and environment.
702 Parameters
703 ----------
704 args : `argparse.Namespace`
705 Parsed command line
706 pipeline : `Pipeline`
707 Pipeline definition
708 graph : `QuantumGraph`, optional
709 Execution graph
710 """
711 showOpts = args.show
712 for what in showOpts:
713 showCommand, _, showArgs = what.partition("=")
715 if showCommand in ["pipeline", "config", "history", "tasks"]:
716 if not pipeline:
717 _LOG.warning("Pipeline is required for --show=%s", showCommand)
718 continue
720 if showCommand in ["graph", "workflow"]:
721 if not graph:
722 _LOG.warning("QuantumGraph is required for --show=%s", showCommand)
723 continue
725 if showCommand == "pipeline":
726 print(pipeline)
727 elif showCommand == "config":
728 self._showConfig(pipeline, showArgs, False)
729 elif showCommand == "dump-config":
730 self._showConfig(pipeline, showArgs, True)
731 elif showCommand == "history":
732 self._showConfigHistory(pipeline, showArgs)
733 elif showCommand == "tasks":
734 self._showTaskHierarchy(pipeline)
735 elif showCommand == "graph":
736 if graph:
737 self._showGraph(graph)
738 elif showCommand == "workflow":
739 if graph:
740 self._showWorkflow(graph, args)
741 else:
742 print("Unknown value for show: %s (choose from '%s')" %
743 (what, "', '".join("pipeline config[=XXX] history=XXX tasks graph".split())),
744 file=sys.stderr)
745 sys.exit(1)
747 def _showConfig(self, pipeline, showArgs, dumpFullConfig):
748 """Show task configuration
750 Parameters
751 ----------
752 pipeline : `Pipeline`
753 Pipeline definition
754 showArgs : `str`
755 Defines what to show
756 dumpFullConfig : `bool`
757 If true then dump complete task configuration with all imports.
758 """
759 stream = sys.stdout
760 if dumpFullConfig:
761 # Task label can be given with this option
762 taskName = showArgs
763 else:
764 # The argument can have form [TaskLabel::][pattern:NOIGNORECASE]
765 matConfig = re.search(r"^(?:(\w+)::)?(?:config.)?(.+)?", showArgs)
766 taskName = matConfig.group(1)
767 pattern = matConfig.group(2)
768 if pattern:
769 stream = _FilteredStream(pattern)
771 tasks = util.filterTasks(pipeline, taskName)
772 if not tasks:
773 print("Pipeline has no tasks named {}".format(taskName), file=sys.stderr)
774 sys.exit(1)
776 for taskDef in tasks:
777 print("### Configuration for task `{}'".format(taskDef.label))
778 taskDef.config.saveToStream(stream, root="config", skipImports=not dumpFullConfig)
780 def _showConfigHistory(self, pipeline, showArgs):
781 """Show history for task configuration
783 Parameters
784 ----------
785 pipeline : `Pipeline`
786 Pipeline definition
787 showArgs : `str`
788 Defines what to show
789 """
791 taskName = None
792 pattern = None
793 matHistory = re.search(r"^(?:(\w+)::)?(?:config[.])?(.+)", showArgs)
794 if matHistory:
795 taskName = matHistory.group(1)
796 pattern = matHistory.group(2)
797 if not pattern:
798 print("Please provide a value with --show history (e.g. history=Task::param)", file=sys.stderr)
799 sys.exit(1)
801 tasks = util.filterTasks(pipeline, taskName)
802 if not tasks:
803 print(f"Pipeline has no tasks named {taskName}", file=sys.stderr)
804 sys.exit(1)
806 cpath, _, cname = pattern.rpartition(".")
807 found = False
808 for taskDef in tasks:
809 try:
810 if not cpath:
811 # looking for top-level field
812 hconfig = taskDef.config
813 else:
814 hconfig = eval("config." + cpath, {}, {"config": taskDef.config})
815 except AttributeError:
816 # Means this config object has no such field, but maybe some other task has it.
817 continue
818 except Exception:
819 # Any other exception probably means some error in the expression.
820 print(f"ERROR: Failed to evaluate field expression `{pattern}'", file=sys.stderr)
821 sys.exit(1)
823 if hasattr(hconfig, cname):
824 print(f"### Configuration field for task `{taskDef.label}'")
825 print(pexConfig.history.format(hconfig, cname))
826 found = True
828 if not found:
829 print(f"None of the tasks has field named {pattern}", file=sys.stderr)
830 sys.exit(1)
832 def _showTaskHierarchy(self, pipeline):
833 """Print task hierarchy to stdout
835 Parameters
836 ----------
837 pipeline: `Pipeline`
838 """
839 for taskDef in pipeline.toExpandedPipeline():
840 print("### Subtasks for task `{}'".format(taskDef.taskName))
842 for configName, taskName in util.subTaskIter(taskDef.config):
843 print("{}: {}".format(configName, taskName))
845 def _showGraph(self, graph):
846 """Print quanta information to stdout
848 Parameters
849 ----------
850 graph : `QuantumGraph`
851 Execution graph.
852 """
853 for taskNodes in graph:
854 print(taskNodes.taskDef)
856 for iq, quantum in enumerate(taskNodes.quanta):
857 print(" Quantum {}:".format(iq))
858 print(" inputs:")
859 for key, refs in quantum.predictedInputs.items():
860 dataIds = ["DataId({})".format(ref.dataId) for ref in refs]
861 print(" {}: [{}]".format(key, ", ".join(dataIds)))
862 print(" outputs:")
863 for key, refs in quantum.outputs.items():
864 dataIds = ["DataId({})".format(ref.dataId) for ref in refs]
865 print(" {}: [{}]".format(key, ", ".join(dataIds)))
867 def _showWorkflow(self, graph, args):
868 """Print quanta information and dependency to stdout
870 The input and predicted output URIs based on the Butler repo are printed.
872 Parameters
873 ----------
874 graph : `QuantumGraph`
875 Execution graph.
876 args : `argparse.Namespace`
877 Parsed command line
878 """
879 def dumpURIs(thisRef):
880 primary, components = butler.getURIs(thisRef, predict=True, run="TBD")
881 if primary:
882 print(f" {primary}")
883 else:
884 print(f" (disassembled artifact)")
885 for compName, compUri in components.items():
886 print(f" {compName}: {compUri}")
888 butler = _ButlerFactory.makeReadButler(args)
889 for qdata in graph.traverse():
890 shortname = qdata.taskDef.taskName.split('.')[-1]
891 print("Quantum {}: {}".format(qdata.index, shortname))
892 print(" inputs:")
893 for key, refs in qdata.quantum.predictedInputs.items():
894 for ref in refs:
895 dumpURIs(ref)
896 print(" outputs:")
897 for key, refs in qdata.quantum.outputs.items():
898 for ref in refs:
899 dumpURIs(ref)
900 for parent in qdata.dependencies:
901 print("Parent Quantum {} - Child Quantum {}".format(parent, qdata.index))
903 def _importGraphFixup(self, args):
904 """Import/instantiate graph fixup object.
906 Parameters
907 ----------
908 args : `argparse.Namespace`
909 Parsed command line.
911 Returns
912 -------
913 fixup : `ExecutionGraphFixup` or `None`
915 Raises
916 ------
917 ValueError
918 Raised if import fails, method call raises exception, or returned
919 instance has unexpected type.
920 """
921 if args.graph_fixup:
922 try:
923 factory = doImport(args.graph_fixup)
924 except Exception as exc:
925 raise ValueError("Failed to import graph fixup class/method") from exc
926 try:
927 fixup = factory()
928 except Exception as exc:
929 raise ValueError("Failed to make instance of graph fixup") from exc
930 if not isinstance(fixup, ExecutionGraphFixup):
931 raise ValueError("Graph fixup is not an instance of ExecutionGraphFixup class")
932 return fixup