Coverage for python/lsst/ctrl/mpexec/cmdLineFwk.py : 12%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of ctrl_mpexec.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22"""Module defining CmdLineFwk class and related methods.
23"""
25__all__ = ['CmdLineFwk']
27# -------------------------------
28# Imports of standard modules --
29# -------------------------------
30import argparse
31import datetime
32import fnmatch
33import logging
34import os
35import re
36import sys
37from typing import List, Optional, Tuple
38import warnings
40# -----------------------------
41# Imports for other modules --
42# -----------------------------
43from lsst.daf.butler import (
44 Butler,
45 CollectionSearch,
46 CollectionType,
47 DatasetTypeRestriction,
48 Registry,
49)
50from lsst.daf.butler.registry import MissingCollectionError
51import lsst.log
52import lsst.pex.config as pexConfig
53from lsst.pipe.base import GraphBuilder, Pipeline, QuantumGraph
54from .dotTools import graph2dot, pipeline2dot
55from .executionGraphFixup import ExecutionGraphFixup
56from .mpGraphExecutor import MPGraphExecutor
57from .preExecInit import PreExecInit
58from .singleQuantumExecutor import SingleQuantumExecutor
59from . import util
60from lsst.utils import doImport
62# ----------------------------------
63# Local non-exported definitions --
64# ----------------------------------
66# logging properties
67_LOG_PROP = """\
68log4j.rootLogger=INFO, A1
69log4j.appender.A1=ConsoleAppender
70log4j.appender.A1.Target=System.err
71log4j.appender.A1.layout=PatternLayout
72log4j.appender.A1.layout.ConversionPattern={}
73"""
75_LOG = logging.getLogger(__name__.partition(".")[2])
78class _OutputChainedCollectionInfo:
79 """A helper class for handling command-line arguments related to an output
80 `~lsst.daf.butler.CollectionType.CHAINED` collection.
82 Parameters
83 ----------
84 registry : `lsst.daf.butler.Registry`
85 Butler registry that collections will be added to and/or queried from.
86 name : `str`
87 Name of the collection given on the command line.
88 """
89 def __init__(self, registry: Registry, name: str):
90 self.name = name
91 try:
92 self.chain = list(registry.getCollectionChain(name))
93 self.exists = True
94 except MissingCollectionError:
95 self.chain = []
96 self.exists = False
98 def __str__(self):
99 return self.name
101 name: str
102 """Name of the collection provided on the command line (`str`).
103 """
105 exists: bool
106 """Whether this collection already exists in the registry (`bool`).
107 """
109 chain: List[Tuple[str, DatasetTypeRestriction]]
110 """The definition of the collection, if it already exists (`list`).
112 Empty if the collection does not alredy exist.
113 """
116class _OutputRunCollectionInfo:
117 """A helper class for handling command-line arguments related to an output
118 `~lsst.daf.butler.CollectionType.RUN` collection.
120 Parameters
121 ----------
122 registry : `lsst.daf.butler.Registry`
123 Butler registry that collections will be added to and/or queried from.
124 name : `str`
125 Name of the collection given on the command line.
126 """
127 def __init__(self, registry: Registry, name: str):
128 self.name = name
129 try:
130 actualType = registry.getCollectionType(name)
131 if actualType is not CollectionType.RUN:
132 raise TypeError(f"Collection '{name}' exists but has type {actualType.name}, not RUN.")
133 self.exists = True
134 except MissingCollectionError:
135 self.exists = False
137 name: str
138 """Name of the collection provided on the command line (`str`).
139 """
141 exists: bool
142 """Whether this collection already exists in the registry (`bool`).
143 """
146class _ButlerFactory:
147 """A helper class for processing command-line arguments related to input
148 and output collections.
150 Parameters
151 ----------
152 registry : `lsst.daf.butler.Registry`
153 Butler registry that collections will be added to and/or queried from.
155 args : `argparse.Namespace`
156 Parsed command-line arguments. The following attributes are used,
157 either at construction or in later methods.
159 ``output``
160 The name of a `~lsst.daf.butler.CollectionType.CHAINED`
161 input/output collection.
163 ``output_run``
164 The name of a `~lsst.daf.butler.CollectionType.RUN` input/output
165 collection.
167 ``extend_run``
168 A boolean indicating whether ``output_run`` should already exist
169 and be extended.
171 ``replace_run``
172 A boolean indicating that (if `True`) ``output_run`` should already
173 exist but will be removed from the output chained collection and
174 replaced with a new one.
176 ``prune_replaced``
177 A boolean indicating whether to prune the replaced run (requires
178 ``replace_run``).
180 ``inputs``
181 Input collections of any type; may be any type handled by
182 `lsst.daf.butler.registry.CollectionSearch.fromExpression`.
184 ``butler_config``
185 Path to a data repository root or configuration file.
187 writeable : `bool`
188 If `True`, a `Butler` is being initialized in a context where actual
189 writes should happens, and hence no output run is necessary.
191 Raises
192 ------
193 ValueError
194 Raised if ``writeable is True`` but there are no output collections.
195 """
196 def __init__(self, registry: Registry, args: argparse.Namespace, writeable: bool):
197 if args.output is not None:
198 self.output = _OutputChainedCollectionInfo(registry, args.output)
199 else:
200 self.output = None
201 if args.output_run is not None:
202 self.outputRun = _OutputRunCollectionInfo(registry, args.output_run)
203 elif self.output is not None:
204 if args.extend_run:
205 runName, _ = self.output.chain[0]
206 else:
207 runName = "{}/{:%Y%m%dT%Hh%Mm%Ss}".format(self.output, datetime.datetime.now())
208 self.outputRun = _OutputRunCollectionInfo(registry, runName)
209 elif not writeable:
210 # If we're not writing yet, ok to have no output run.
211 self.outputRun = None
212 else:
213 raise ValueError("Cannot write without at least one of (--output, --output-run).")
214 self.inputs = list(CollectionSearch.fromExpression(args.input)) if args.input else []
216 def check(self, args: argparse.Namespace):
217 """Check command-line options for consistency with each other and the
218 data repository.
220 Parameters
221 ----------
222 args : `argparse.Namespace`
223 Parsed command-line arguments. See class documentation for the
224 construction parameter of the same name.
225 """
226 assert not (args.extend_run and args.replace_run), "In mutually-exclusive group in ArgumentParser."
227 if self.inputs and self.output is not None and self.output.exists:
228 raise ValueError("Cannot use --output with existing collection with --inputs.")
229 if args.extend_run and self.outputRun is None:
230 raise ValueError("Cannot --extend-run when no output collection is given.")
231 if args.extend_run and not self.outputRun.exists:
232 raise ValueError(f"Cannot --extend-run; output collection "
233 f"'{self.outputRun.name}' does not exist.")
234 if not args.extend_run and self.outputRun is not None and self.outputRun.exists:
235 raise ValueError(f"Output run '{self.outputRun.name}' already exists, but "
236 f"--extend-run was not given.")
237 if args.prune_replaced and not args.replace_run:
238 raise ValueError("--prune-replaced requires --replace-run.")
239 if args.replace_run and (self.output is None or not self.output.exists):
240 raise ValueError("--output must point to an existing CHAINED collection for --replace-run.")
242 @classmethod
243 def _makeReadParts(cls, args: argparse.Namespace):
244 """Common implementation for `makeReadButler` and
245 `makeRegistryAndCollections`.
247 Parameters
248 ----------
249 args : `argparse.Namespace`
250 Parsed command-line arguments. See class documentation for the
251 construction parameter of the same name.
253 Returns
254 -------
255 butler : `lsst.daf.butler.Butler`
256 A read-only butler constructed from the repo at
257 ``args.butler_config``, but with no default collections.
258 inputs : `lsst.daf.butler.registry.CollectionSearch`
259 A collection search path constructed according to ``args``.
260 self : `_ButlerFactory`
261 A new `_ButlerFactory` instance representing the processed version
262 of ``args``.
263 """
264 butler = Butler(args.butler_config, writeable=False)
265 self = cls(butler.registry, args, writeable=False)
266 self.check(args)
267 if self.output and self.output.exists:
268 if args.replace_run:
269 replaced, _ = self.output.chain[0]
270 inputs = self.output.chain[1:]
271 _LOG.debug("Simulating collection search in '%s' after removing '%s'.",
272 self.output.name, replaced)
273 else:
274 inputs = [self.output.name]
275 else:
276 inputs = list(self.inputs)
277 if args.extend_run:
278 inputs.insert(0, self.outputRun.name)
279 inputs = CollectionSearch.fromExpression(inputs)
280 return butler, inputs, self
282 @classmethod
283 def makeReadButler(cls, args: argparse.Namespace) -> Butler:
284 """Construct a read-only butler according to the given command-line
285 arguments.
287 Parameters
288 ----------
289 args : `argparse.Namespace`
290 Parsed command-line arguments. See class documentation for the
291 construction parameter of the same name.
293 Returns
294 -------
295 butler : `lsst.daf.butler.Butler`
296 A read-only butler initialized with the collections specified by
297 ``args``.
298 """
299 butler, inputs, _ = cls._makeReadParts(args)
300 _LOG.debug("Preparing butler to read from %s.", inputs)
301 return Butler(butler=butler, collections=inputs)
303 @classmethod
304 def makeRegistryAndCollections(cls, args: argparse.Namespace) -> \
305 Tuple[Registry, CollectionSearch, Optional[str]]:
306 """Return a read-only registry, a collection search path, and the name
307 of the run to be used for future writes.
309 Parameters
310 ----------
311 args : `argparse.Namespace`
312 Parsed command-line arguments. See class documentation for the
313 construction parameter of the same name.
315 Returns
316 -------
317 registry : `lsst.daf.butler.Registry`
318 Butler registry that collections will be added to and/or queried
319 from.
320 inputs : `lsst.daf.butler.registry.CollectionSearch`
321 Collections to search for datasets.
322 run : `str` or `None`
323 Name of the output `~lsst.daf.butler.CollectionType.RUN` collection
324 if it already exists, or `None` if it does not.
325 """
326 butler, inputs, self = cls._makeReadParts(args)
327 run = self.outputRun.name if args.extend_run else None
328 _LOG.debug("Preparing registry to read from %s and expect future writes to '%s'.", inputs, run)
329 return butler.registry, inputs, run
331 @classmethod
332 def makeWriteButler(cls, args: argparse.Namespace) -> Butler:
333 """Return a read-write butler initialized to write to and read from
334 the collections specified by the given command-line arguments.
336 Parameters
337 ----------
338 args : `argparse.Namespace`
339 Parsed command-line arguments. See class documentation for the
340 construction parameter of the same name.
342 Returns
343 -------
344 butler : `lsst.daf.butler.Butler`
345 A read-write butler initialized according to the given arguments.
346 """
347 butler = Butler(args.butler_config, writeable=True)
348 self = cls(butler.registry, args, writeable=True)
349 self.check(args)
350 if self.output is not None:
351 chainDefinition = list(self.output.chain if self.output.exists else self.inputs)
352 if args.replace_run:
353 replaced, _ = chainDefinition.pop(0)
354 if args.prune_replaced == "unstore":
355 # Remove datasets from datastore
356 with butler.transaction():
357 refs = butler.registry.queryDatasets(..., collections=replaced)
358 butler.pruneDatasets(refs, unstore=True, run=replaced, disassociate=False)
359 elif args.prune_replaced == "purge":
360 # Erase entire collection and all datasets, need to remove
361 # collection from its chain collection first.
362 with butler.transaction():
363 butler.registry.setCollectionChain(self.output.name, chainDefinition)
364 butler.pruneCollection(replaced, purge=True, unstore=True)
365 elif args.prune_replaced is not None:
366 raise NotImplementedError(
367 f"Unsupported --prune-replaced option '{args.prune_replaced}'."
368 )
369 chainDefinition.insert(0, self.outputRun.name)
370 chainDefinition = CollectionSearch.fromExpression(chainDefinition)
371 _LOG.debug("Preparing butler to write to '%s' and read from '%s'=%s",
372 self.outputRun.name, self.output.name, chainDefinition)
373 return Butler(butler=butler, run=self.outputRun.name, collections=self.output.name,
374 chains={self.output.name: chainDefinition})
375 else:
376 inputs = CollectionSearch.fromExpression([self.outputRun.name] + self.inputs)
377 _LOG.debug("Preparing butler to write to '%s' and read from %s.", self.outputRun.name, inputs)
378 return Butler(butler=butler, run=self.outputRun.name, collections=inputs)
380 output: Optional[_OutputChainedCollectionInfo]
381 """Information about the output chained collection, if there is or will be
382 one (`_OutputChainedCollectionInfo` or `None`).
383 """
385 outputRun: Optional[_OutputRunCollectionInfo]
386 """Information about the output run collection, if there is or will be
387 one (`_OutputRunCollectionInfo` or `None`).
388 """
390 inputs: List[Tuple[str, DatasetTypeRestriction]]
391 """Input collections, including those also used for outputs and any
392 restrictions on dataset types (`list`).
393 """
396class _FilteredStream:
397 """A file-like object that filters some config fields.
399 Note
400 ----
401 This class depends on implementation details of ``Config.saveToStream``
402 methods, in particular that that method uses single call to write()
403 method to save information about single config field, and that call
404 combines comments string(s) for a field and field path and value.
405 This class will not work reliably on the "import" strings, so imports
406 should be disabled by passing ``skipImports=True`` to ``saveToStream()``.
407 """
408 def __init__(self, pattern):
409 # obey case if pattern isn't lowercase or requests NOIGNORECASE
410 mat = re.search(r"(.*):NOIGNORECASE$", pattern)
412 if mat:
413 pattern = mat.group(1)
414 self._pattern = re.compile(fnmatch.translate(pattern))
415 else:
416 if pattern != pattern.lower():
417 print(f"Matching \"{pattern}\" without regard to case "
418 "(append :NOIGNORECASE to prevent this)", file=sys.stdout)
419 self._pattern = re.compile(fnmatch.translate(pattern), re.IGNORECASE)
421 def write(self, showStr):
422 # Strip off doc string line(s) and cut off at "=" for string matching
423 matchStr = showStr.rstrip().split("\n")[-1].split("=")[0]
424 if self._pattern.search(matchStr):
425 sys.stdout.write(showStr)
427# ------------------------
428# Exported definitions --
429# ------------------------
432class CmdLineFwk:
433 """PipelineTask framework which executes tasks from command line.
435 In addition to executing tasks this activator provides additional methods
436 for task management like dumping configuration or execution chain.
437 """
439 MP_TIMEOUT = 9999 # Default timeout (sec) for multiprocessing
441 def __init__(self):
442 pass
444 @staticmethod
445 def configLog(longlog, logLevels):
446 """Configure logging system.
448 Parameters
449 ----------
450 longlog : `bool`
451 If True then make log messages appear in "long format"
452 logLevels : `list` of `tuple`
453 per-component logging levels, each item in the list is a tuple
454 (component, level), `component` is a logger name or `None` for root
455 logger, `level` is a logging level name ('DEBUG', 'INFO', etc.)
456 """
457 if longlog:
458 message_fmt = "%-5p %d{yyyy-MM-ddTHH:mm:ss.SSSZ} %c (%X{LABEL})(%F:%L)- %m%n"
459 else:
460 message_fmt = "%c %p: %m%n"
462 # Initialize global logging config. Skip if the env var LSST_LOG_CONFIG exists.
463 # The file it points to would already configure lsst.log.
464 if not os.path.isfile(os.environ.get("LSST_LOG_CONFIG", "")):
465 lsst.log.configure_prop(_LOG_PROP.format(message_fmt))
467 # Forward all Python logging to lsst.log
468 lgr = logging.getLogger()
469 lgr.setLevel(logging.INFO) # same as in log4cxx config above
470 lgr.addHandler(lsst.log.LogHandler())
472 # also capture warnings and send them to logging
473 logging.captureWarnings(True)
475 # configure individual loggers
476 for component, level in logLevels:
477 level = getattr(lsst.log.Log, level.upper(), None)
478 if level is not None:
479 # set logging level for lsst.log
480 logger = lsst.log.Log.getLogger(component or "")
481 logger.setLevel(level)
482 # set logging level for Python logging
483 pyLevel = lsst.log.LevelTranslator.lsstLog2logging(level)
484 logging.getLogger(component).setLevel(pyLevel)
486 def makePipeline(self, args):
487 """Build a pipeline from command line arguments.
489 Parameters
490 ----------
491 args : `argparse.Namespace`
492 Parsed command line
494 Returns
495 -------
496 pipeline : `~lsst.pipe.base.Pipeline`
497 """
498 if args.pipeline:
499 pipeline = Pipeline.fromFile(args.pipeline)
500 else:
501 pipeline = Pipeline("anonymous")
503 # loop over all pipeline actions and apply them in order
504 for action in args.pipeline_actions:
505 if action.action == "add_instrument":
507 pipeline.addInstrument(action.value)
509 elif action.action == "new_task":
511 pipeline.addTask(action.value, action.label)
513 elif action.action == "delete_task":
515 pipeline.removeTask(action.label)
517 elif action.action == "config":
519 # action value string is "field=value", split it at '='
520 field, _, value = action.value.partition("=")
521 pipeline.addConfigOverride(action.label, field, value)
523 elif action.action == "configfile":
525 pipeline.addConfigFile(action.label, action.value)
527 else:
529 raise ValueError(f"Unexpected pipeline action: {action.action}")
531 if args.save_pipeline:
532 pipeline.toFile(args.save_pipeline)
534 if args.pipeline_dot:
535 pipeline2dot(pipeline, args.pipeline_dot)
537 return pipeline
539 def makeGraph(self, pipeline, args):
540 """Build a graph from command line arguments.
542 Parameters
543 ----------
544 pipeline : `~lsst.pipe.base.Pipeline`
545 Pipeline, can be empty or ``None`` if graph is read from a file.
546 args : `argparse.Namespace`
547 Parsed command line
549 Returns
550 -------
551 graph : `~lsst.pipe.base.QuantumGraph` or `None`
552 If resulting graph is empty then `None` is returned.
553 """
555 registry, collections, run = _ButlerFactory.makeRegistryAndCollections(args)
557 if args.qgraph:
559 with open(args.qgraph, 'rb') as pickleFile:
560 qgraph = QuantumGraph.load(pickleFile, registry.dimensions)
562 # pipeline can not be provided in this case
563 if pipeline:
564 raise ValueError("Pipeline must not be given when quantum graph is read from file.")
566 else:
568 # make execution plan (a.k.a. DAG) for pipeline
569 graphBuilder = GraphBuilder(registry,
570 skipExisting=args.skip_existing)
571 qgraph = graphBuilder.makeGraph(pipeline, collections, run, args.data_query)
573 # count quanta in graph and give a warning if it's empty and return None
574 nQuanta = len(qgraph)
575 if nQuanta == 0:
576 warnings.warn("QuantumGraph is empty", stacklevel=2)
577 return None
578 else:
579 _LOG.info("QuantumGraph contains %d quanta for %d tasks",
580 nQuanta, len(qgraph.taskGraph))
582 if args.save_qgraph:
583 with open(args.save_qgraph, "wb") as pickleFile:
584 qgraph.save(pickleFile)
586 if args.save_single_quanta:
587 for iq, quantumNode in enumerate(qgraph):
588 sqgraph = qgraph.subset(quantumNode)
589 filename = args.save_single_quanta.format(iq)
590 with open(filename, "wb") as pickleFile:
591 sqgraph.save(pickleFile)
593 if args.qgraph_dot:
594 graph2dot(qgraph, args.qgraph_dot)
596 return qgraph
598 def runPipeline(self, graph, taskFactory, args, butler=None):
599 """Execute complete QuantumGraph.
601 Parameters
602 ----------
603 graph : `QuantumGraph`
604 Execution graph.
605 taskFactory : `~lsst.pipe.base.TaskFactory`
606 Task factory
607 args : `argparse.Namespace`
608 Parsed command line
609 butler : `~lsst.daf.butler.Butler`, optional
610 Data Butler instance, if not defined then new instance is made
611 using command line options.
612 """
613 # make butler instance
614 if butler is None:
615 butler = _ButlerFactory.makeWriteButler(args)
617 # Enable lsstDebug debugging. Note that this is done once in the
618 # main process before PreExecInit and it is also repeated before
619 # running each task in SingleQuantumExecutor (which may not be
620 # needed if `multipocessing` always uses fork start method).
621 if args.enableLsstDebug:
622 try:
623 _LOG.debug("Will try to import debug.py")
624 import debug # noqa:F401
625 except ImportError:
626 _LOG.warn("No 'debug' module found.")
628 preExecInit = PreExecInit(butler, taskFactory, args.skip_existing)
629 preExecInit.initialize(graph,
630 saveInitOutputs=not args.skip_init_writes,
631 registerDatasetTypes=args.register_dataset_types,
632 saveVersions=not args.no_versions)
634 if not args.init_only:
635 graphFixup = self._importGraphFixup(args)
636 quantumExecutor = SingleQuantumExecutor(taskFactory,
637 skipExisting=args.skip_existing,
638 clobberPartialOutputs=args.clobber_partial_outputs,
639 enableLsstDebug=args.enableLsstDebug)
640 timeout = self.MP_TIMEOUT if args.timeout is None else args.timeout
641 executor = MPGraphExecutor(numProc=args.processes, timeout=timeout,
642 quantumExecutor=quantumExecutor,
643 failFast=args.fail_fast,
644 executionGraphFixup=graphFixup)
645 with util.profile(args.profile, _LOG):
646 executor.execute(graph, butler)
648 def showInfo(self, args, pipeline, graph=None):
649 """Display useful info about pipeline and environment.
651 Parameters
652 ----------
653 args : `argparse.Namespace`
654 Parsed command line
655 pipeline : `Pipeline`
656 Pipeline definition
657 graph : `QuantumGraph`, optional
658 Execution graph
659 """
660 showOpts = args.show
661 for what in showOpts:
662 showCommand, _, showArgs = what.partition("=")
664 if showCommand in ["pipeline", "config", "history", "tasks"]:
665 if not pipeline:
666 _LOG.warning("Pipeline is required for --show=%s", showCommand)
667 continue
669 if showCommand in ["graph", "workflow"]:
670 if not graph:
671 _LOG.warning("QuantumGraph is required for --show=%s", showCommand)
672 continue
674 if showCommand == "pipeline":
675 print(pipeline)
676 elif showCommand == "config":
677 self._showConfig(pipeline, showArgs, False)
678 elif showCommand == "dump-config":
679 self._showConfig(pipeline, showArgs, True)
680 elif showCommand == "history":
681 self._showConfigHistory(pipeline, showArgs)
682 elif showCommand == "tasks":
683 self._showTaskHierarchy(pipeline)
684 elif showCommand == "graph":
685 if graph:
686 self._showGraph(graph)
687 elif showCommand == "workflow":
688 if graph:
689 self._showWorkflow(graph, args)
690 else:
691 print("Unknown value for show: %s (choose from '%s')" %
692 (what, "', '".join("pipeline config[=XXX] history=XXX tasks graph".split())),
693 file=sys.stderr)
694 sys.exit(1)
696 def _showConfig(self, pipeline, showArgs, dumpFullConfig):
697 """Show task configuration
699 Parameters
700 ----------
701 pipeline : `Pipeline`
702 Pipeline definition
703 showArgs : `str`
704 Defines what to show
705 dumpFullConfig : `bool`
706 If true then dump complete task configuration with all imports.
707 """
708 stream = sys.stdout
709 if dumpFullConfig:
710 # Task label can be given with this option
711 taskName = showArgs
712 else:
713 # The argument can have form [TaskLabel::][pattern:NOIGNORECASE]
714 matConfig = re.search(r"^(?:(\w+)::)?(?:config.)?(.+)?", showArgs)
715 taskName = matConfig.group(1)
716 pattern = matConfig.group(2)
717 if pattern:
718 stream = _FilteredStream(pattern)
720 tasks = util.filterTasks(pipeline, taskName)
721 if not tasks:
722 print("Pipeline has no tasks named {}".format(taskName), file=sys.stderr)
723 sys.exit(1)
725 for taskDef in tasks:
726 print("### Configuration for task `{}'".format(taskDef.label))
727 taskDef.config.saveToStream(stream, root="config", skipImports=not dumpFullConfig)
729 def _showConfigHistory(self, pipeline, showArgs):
730 """Show history for task configuration
732 Parameters
733 ----------
734 pipeline : `Pipeline`
735 Pipeline definition
736 showArgs : `str`
737 Defines what to show
738 """
740 taskName = None
741 pattern = None
742 matHistory = re.search(r"^(?:(\w+)::)?(?:config[.])?(.+)", showArgs)
743 if matHistory:
744 taskName = matHistory.group(1)
745 pattern = matHistory.group(2)
746 if not pattern:
747 print("Please provide a value with --show history (e.g. history=Task::param)", file=sys.stderr)
748 sys.exit(1)
750 tasks = util.filterTasks(pipeline, taskName)
751 if not tasks:
752 print(f"Pipeline has no tasks named {taskName}", file=sys.stderr)
753 sys.exit(1)
755 cpath, _, cname = pattern.rpartition(".")
756 found = False
757 for taskDef in tasks:
758 try:
759 if not cpath:
760 # looking for top-level field
761 hconfig = taskDef.config
762 else:
763 hconfig = eval("config." + cpath, {}, {"config": taskDef.config})
764 except AttributeError:
765 # Means this config object has no such field, but maybe some other task has it.
766 continue
767 except Exception:
768 # Any other exception probably means some error in the expression.
769 print(f"ERROR: Failed to evaluate field expression `{pattern}'", file=sys.stderr)
770 sys.exit(1)
772 if hasattr(hconfig, cname):
773 print(f"### Configuration field for task `{taskDef.label}'")
774 print(pexConfig.history.format(hconfig, cname))
775 found = True
777 if not found:
778 print(f"None of the tasks has field named {pattern}", file=sys.stderr)
779 sys.exit(1)
781 def _showTaskHierarchy(self, pipeline):
782 """Print task hierarchy to stdout
784 Parameters
785 ----------
786 pipeline: `Pipeline`
787 """
788 for taskDef in pipeline.toExpandedPipeline():
789 print("### Subtasks for task `{}'".format(taskDef.taskName))
791 for configName, taskName in util.subTaskIter(taskDef.config):
792 print("{}: {}".format(configName, taskName))
794 def _showGraph(self, graph):
795 """Print quanta information to stdout
797 Parameters
798 ----------
799 graph : `QuantumGraph`
800 Execution graph.
801 """
802 for taskNode in graph.taskGraph:
803 print(taskNode)
805 for iq, quantum in enumerate(graph.getQuantaForTask(taskNode)):
806 print(" Quantum {}:".format(iq))
807 print(" inputs:")
808 for key, refs in quantum.inputs.items():
809 dataIds = ["DataId({})".format(ref.dataId) for ref in refs]
810 print(" {}: [{}]".format(key, ", ".join(dataIds)))
811 print(" outputs:")
812 for key, refs in quantum.outputs.items():
813 dataIds = ["DataId({})".format(ref.dataId) for ref in refs]
814 print(" {}: [{}]".format(key, ", ".join(dataIds)))
816 def _showWorkflow(self, graph, args):
817 """Print quanta information and dependency to stdout
819 The input and predicted output URIs based on the Butler repo are printed.
821 Parameters
822 ----------
823 graph : `QuantumGraph`
824 Execution graph.
825 args : `argparse.Namespace`
826 Parsed command line
827 """
828 def dumpURIs(thisRef):
829 primary, components = butler.getURIs(thisRef, predict=True, run="TBD")
830 if primary:
831 print(f" {primary}")
832 else:
833 print(" (disassembled artifact)")
834 for compName, compUri in components.items():
835 print(f" {compName}: {compUri}")
837 butler = _ButlerFactory.makeReadButler(args)
838 for qdata in graph.traverse():
839 shortname = qdata.taskDef.taskName.split('.')[-1]
840 print("Quantum {}: {}".format(qdata.index, shortname))
841 print(" inputs:")
842 for key, refs in qdata.quantum.inputs.items():
843 for ref in refs:
844 dumpURIs(ref)
845 print(" outputs:")
846 for key, refs in qdata.quantum.outputs.items():
847 for ref in refs:
848 dumpURIs(ref)
849 for parent in qdata.dependencies:
850 print("Parent Quantum {} - Child Quantum {}".format(parent, qdata.index))
852 def _importGraphFixup(self, args):
853 """Import/instantiate graph fixup object.
855 Parameters
856 ----------
857 args : `argparse.Namespace`
858 Parsed command line.
860 Returns
861 -------
862 fixup : `ExecutionGraphFixup` or `None`
864 Raises
865 ------
866 ValueError
867 Raised if import fails, method call raises exception, or returned
868 instance has unexpected type.
869 """
870 if args.graph_fixup:
871 try:
872 factory = doImport(args.graph_fixup)
873 except Exception as exc:
874 raise ValueError("Failed to import graph fixup class/method") from exc
875 try:
876 fixup = factory()
877 except Exception as exc:
878 raise ValueError("Failed to make instance of graph fixup") from exc
879 if not isinstance(fixup, ExecutionGraphFixup):
880 raise ValueError("Graph fixup is not an instance of ExecutionGraphFixup class")
881 return fixup