Coverage for python/lsst/ctrl/mpexec/cmdLineFwk.py: 15%
303 statements
« prev ^ index » next coverage.py v6.5.0, created at 2023-01-10 10:58 +0000
« prev ^ index » next coverage.py v6.5.0, created at 2023-01-10 10:58 +0000
1# This file is part of ctrl_mpexec.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22"""Module defining CmdLineFwk class and related methods.
23"""
25from __future__ import annotations
27__all__ = ["CmdLineFwk"]
29import atexit
30import copy
31import datetime
32import getpass
33import logging
34import shutil
35from collections.abc import Iterable, Sequence
36from types import SimpleNamespace
37from typing import Optional, Tuple
39from astropy.table import Table
40from lsst.daf.butler import Butler, CollectionType, DatasetRef, DatastoreCacheManager, Registry
41from lsst.daf.butler.registry import MissingCollectionError, RegistryDefaults
42from lsst.daf.butler.registry.wildcards import CollectionWildcard
43from lsst.pipe.base import (
44 GraphBuilder,
45 Instrument,
46 Pipeline,
47 PipelineDatasetTypes,
48 QuantumGraph,
49 TaskDef,
50 TaskFactory,
51 buildExecutionButler,
52)
53from lsst.utils import doImportType
54from lsst.utils.threads import disable_implicit_threading
56from . import util
57from .dotTools import graph2dot, pipeline2dot
58from .executionGraphFixup import ExecutionGraphFixup
59from .mpGraphExecutor import MPGraphExecutor
60from .preExecInit import PreExecInit
61from .singleQuantumExecutor import SingleQuantumExecutor
63# ----------------------------------
64# Local non-exported definitions --
65# ----------------------------------
67_LOG = logging.getLogger(__name__)
70class _OutputChainedCollectionInfo:
71 """A helper class for handling command-line arguments related to an output
72 `~lsst.daf.butler.CollectionType.CHAINED` collection.
74 Parameters
75 ----------
76 registry : `lsst.daf.butler.Registry`
77 Butler registry that collections will be added to and/or queried from.
78 name : `str`
79 Name of the collection given on the command line.
80 """
82 def __init__(self, registry: Registry, name: str):
83 self.name = name
84 try:
85 self.chain = tuple(registry.getCollectionChain(name))
86 self.exists = True
87 except MissingCollectionError:
88 self.chain = ()
89 self.exists = False
91 def __str__(self) -> str:
92 return self.name
94 name: str
95 """Name of the collection provided on the command line (`str`).
96 """
98 exists: bool
99 """Whether this collection already exists in the registry (`bool`).
100 """
102 chain: Tuple[str, ...]
103 """The definition of the collection, if it already exists (`tuple`[`str`]).
105 Empty if the collection does not already exist.
106 """
109class _OutputRunCollectionInfo:
110 """A helper class for handling command-line arguments related to an output
111 `~lsst.daf.butler.CollectionType.RUN` collection.
113 Parameters
114 ----------
115 registry : `lsst.daf.butler.Registry`
116 Butler registry that collections will be added to and/or queried from.
117 name : `str`
118 Name of the collection given on the command line.
119 """
121 def __init__(self, registry: Registry, name: str):
122 self.name = name
123 try:
124 actualType = registry.getCollectionType(name)
125 if actualType is not CollectionType.RUN:
126 raise TypeError(f"Collection '{name}' exists but has type {actualType.name}, not RUN.")
127 self.exists = True
128 except MissingCollectionError:
129 self.exists = False
131 name: str
132 """Name of the collection provided on the command line (`str`).
133 """
135 exists: bool
136 """Whether this collection already exists in the registry (`bool`).
137 """
140class _ButlerFactory:
141 """A helper class for processing command-line arguments related to input
142 and output collections.
144 Parameters
145 ----------
146 registry : `lsst.daf.butler.Registry`
147 Butler registry that collections will be added to and/or queried from.
149 args : `types.SimpleNamespace`
150 Parsed command-line arguments. The following attributes are used,
151 either at construction or in later methods.
153 ``output``
154 The name of a `~lsst.daf.butler.CollectionType.CHAINED`
155 input/output collection.
157 ``output_run``
158 The name of a `~lsst.daf.butler.CollectionType.RUN` input/output
159 collection.
161 ``extend_run``
162 A boolean indicating whether ``output_run`` should already exist
163 and be extended.
165 ``replace_run``
166 A boolean indicating that (if `True`) ``output_run`` should already
167 exist but will be removed from the output chained collection and
168 replaced with a new one.
170 ``prune_replaced``
171 A boolean indicating whether to prune the replaced run (requires
172 ``replace_run``).
174 ``inputs``
175 Input collections of any type; see
176 :ref:`daf_butler_ordered_collection_searches` for details.
178 ``butler_config``
179 Path to a data repository root or configuration file.
181 writeable : `bool`
182 If `True`, a `Butler` is being initialized in a context where actual
183 writes should happens, and hence no output run is necessary.
185 Raises
186 ------
187 ValueError
188 Raised if ``writeable is True`` but there are no output collections.
189 """
191 def __init__(self, registry: Registry, args: SimpleNamespace, writeable: bool):
192 if args.output is not None:
193 self.output = _OutputChainedCollectionInfo(registry, args.output)
194 else:
195 self.output = None
196 if args.output_run is not None:
197 self.outputRun = _OutputRunCollectionInfo(registry, args.output_run)
198 elif self.output is not None:
199 if args.extend_run:
200 if not self.output.chain:
201 raise ValueError("Cannot use --extend-run option with non-existing or empty output chain")
202 runName = self.output.chain[0]
203 else:
204 runName = "{}/{}".format(self.output, Instrument.makeCollectionTimestamp())
205 self.outputRun = _OutputRunCollectionInfo(registry, runName)
206 elif not writeable:
207 # If we're not writing yet, ok to have no output run.
208 self.outputRun = None
209 else:
210 raise ValueError("Cannot write without at least one of (--output, --output-run).")
211 # Recursively flatten any input CHAINED collections. We do this up
212 # front so we can tell if the user passes the same inputs on subsequent
213 # calls, even though we also flatten when we define the output CHAINED
214 # collection.
215 self.inputs = tuple(registry.queryCollections(args.input, flattenChains=True)) if args.input else ()
217 def check(self, args: SimpleNamespace) -> None:
218 """Check command-line options for consistency with each other and the
219 data repository.
221 Parameters
222 ----------
223 args : `types.SimpleNamespace`
224 Parsed command-line arguments. See class documentation for the
225 construction parameter of the same name.
226 """
227 assert not (args.extend_run and args.replace_run), "In mutually-exclusive group in ArgumentParser."
228 if self.inputs and self.output is not None and self.output.exists:
229 # Passing the same inputs that were used to initialize the output
230 # collection is allowed; this means they must _end_ with the same
231 # collections, because we push new runs to the front of the chain.
232 for c1, c2 in zip(self.inputs[::-1], self.output.chain[::-1]):
233 if c1 != c2:
234 raise ValueError(
235 f"Output CHAINED collection {self.output.name!r} exists, but it ends with "
236 "a different sequence of input collections than those given: "
237 f"{c1!r} != {c2!r} in inputs={self.inputs} vs "
238 f"{self.output.name}={self.output.chain}."
239 )
240 if len(self.inputs) > len(self.output.chain):
241 nNew = len(self.inputs) - len(self.output.chain)
242 raise ValueError(
243 f"Cannot add new input collections {self.inputs[:nNew]} after "
244 "output collection is first created."
245 )
246 if args.extend_run:
247 if self.outputRun is None:
248 raise ValueError("Cannot --extend-run when no output collection is given.")
249 elif not self.outputRun.exists:
250 raise ValueError(
251 f"Cannot --extend-run; output collection '{self.outputRun.name}' does not exist."
252 )
253 if not args.extend_run and self.outputRun is not None and self.outputRun.exists:
254 raise ValueError(
255 f"Output run '{self.outputRun.name}' already exists, but --extend-run was not given."
256 )
257 if args.prune_replaced and not args.replace_run:
258 raise ValueError("--prune-replaced requires --replace-run.")
259 if args.replace_run and (self.output is None or not self.output.exists):
260 raise ValueError("--output must point to an existing CHAINED collection for --replace-run.")
262 @classmethod
263 def _makeReadParts(cls, args: SimpleNamespace) -> tuple[Butler, Sequence[str], _ButlerFactory]:
264 """Common implementation for `makeReadButler` and
265 `makeButlerAndCollections`.
267 Parameters
268 ----------
269 args : `types.SimpleNamespace`
270 Parsed command-line arguments. See class documentation for the
271 construction parameter of the same name.
273 Returns
274 -------
275 butler : `lsst.daf.butler.Butler`
276 A read-only butler constructed from the repo at
277 ``args.butler_config``, but with no default collections.
278 inputs : `Sequence` [ `str` ]
279 A collection search path constructed according to ``args``.
280 self : `_ButlerFactory`
281 A new `_ButlerFactory` instance representing the processed version
282 of ``args``.
283 """
284 butler = Butler(args.butler_config, writeable=False)
285 self = cls(butler.registry, args, writeable=False)
286 self.check(args)
287 if self.output and self.output.exists:
288 if args.replace_run:
289 replaced = self.output.chain[0]
290 inputs = list(self.output.chain[1:])
291 _LOG.debug(
292 "Simulating collection search in '%s' after removing '%s'.", self.output.name, replaced
293 )
294 else:
295 inputs = [self.output.name]
296 else:
297 inputs = list(self.inputs)
298 if args.extend_run:
299 assert self.outputRun is not None, "Output collection has to be specified."
300 inputs.insert(0, self.outputRun.name)
301 collSearch = CollectionWildcard.from_expression(inputs).require_ordered()
302 return butler, collSearch, self
304 @classmethod
305 def makeReadButler(cls, args: SimpleNamespace) -> Butler:
306 """Construct a read-only butler according to the given command-line
307 arguments.
309 Parameters
310 ----------
311 args : `types.SimpleNamespace`
312 Parsed command-line arguments. See class documentation for the
313 construction parameter of the same name.
315 Returns
316 -------
317 butler : `lsst.daf.butler.Butler`
318 A read-only butler initialized with the collections specified by
319 ``args``.
320 """
321 cls.defineDatastoreCache() # Ensure that this butler can use a shared cache.
322 butler, inputs, _ = cls._makeReadParts(args)
323 _LOG.debug("Preparing butler to read from %s.", inputs)
324 return Butler(butler=butler, collections=inputs)
326 @classmethod
327 def makeButlerAndCollections(cls, args: SimpleNamespace) -> Tuple[Butler, Sequence[str], Optional[str]]:
328 """Return a read-only registry, a collection search path, and the name
329 of the run to be used for future writes.
331 Parameters
332 ----------
333 args : `types.SimpleNamespace`
334 Parsed command-line arguments. See class documentation for the
335 construction parameter of the same name.
337 Returns
338 -------
339 butler : `lsst.daf.butler.Butler`
340 A read-only butler that collections will be added to and/or queried
341 from.
342 inputs : `Sequence` [ `str` ]
343 Collections to search for datasets.
344 run : `str` or `None`
345 Name of the output `~lsst.daf.butler.CollectionType.RUN` collection
346 if it already exists, or `None` if it does not.
347 """
348 butler, inputs, self = cls._makeReadParts(args)
349 run: Optional[str] = None
350 if args.extend_run:
351 assert self.outputRun is not None, "Output collection has to be specified."
352 run = self.outputRun.name
353 _LOG.debug("Preparing registry to read from %s and expect future writes to '%s'.", inputs, run)
354 return butler, inputs, run
356 @staticmethod
357 def defineDatastoreCache() -> None:
358 """Define where datastore cache directories should be found.
360 Notes
361 -----
362 All the jobs should share a datastore cache if applicable. This
363 method asks for a shared fallback cache to be defined and then
364 configures an exit handler to clean it up.
365 """
366 defined, cache_dir = DatastoreCacheManager.set_fallback_cache_directory_if_unset()
367 if defined:
368 atexit.register(shutil.rmtree, cache_dir, ignore_errors=True)
369 _LOG.debug("Defining shared datastore cache directory to %s", cache_dir)
371 @classmethod
372 def makeWriteButler(cls, args: SimpleNamespace, taskDefs: Optional[Iterable[TaskDef]] = None) -> Butler:
373 """Return a read-write butler initialized to write to and read from
374 the collections specified by the given command-line arguments.
376 Parameters
377 ----------
378 args : `types.SimpleNamespace`
379 Parsed command-line arguments. See class documentation for the
380 construction parameter of the same name.
381 taskDefs : iterable of `TaskDef`, optional
382 Definitions for tasks in a pipeline. This argument is only needed
383 if ``args.replace_run`` is `True` and ``args.prune_replaced`` is
384 "unstore".
386 Returns
387 -------
388 butler : `lsst.daf.butler.Butler`
389 A read-write butler initialized according to the given arguments.
390 """
391 cls.defineDatastoreCache() # Ensure that this butler can use a shared cache.
392 butler = Butler(args.butler_config, writeable=True)
393 self = cls(butler.registry, args, writeable=True)
394 self.check(args)
395 assert self.outputRun is not None, "Output collection has to be specified." # for mypy
396 if self.output is not None:
397 chainDefinition = list(self.output.chain if self.output.exists else self.inputs)
398 if args.replace_run:
399 replaced = chainDefinition.pop(0)
400 if args.prune_replaced == "unstore":
401 # Remove datasets from datastore
402 with butler.transaction():
403 refs: Iterable[DatasetRef] = butler.registry.queryDatasets(..., collections=replaced)
404 # we want to remove regular outputs but keep
405 # initOutputs, configs, and versions.
406 if taskDefs is not None:
407 initDatasetNames = set(PipelineDatasetTypes.initOutputNames(taskDefs))
408 refs = [ref for ref in refs if ref.datasetType.name not in initDatasetNames]
409 butler.pruneDatasets(refs, unstore=True, disassociate=False)
410 elif args.prune_replaced == "purge":
411 # Erase entire collection and all datasets, need to remove
412 # collection from its chain collection first.
413 with butler.transaction():
414 butler.registry.setCollectionChain(self.output.name, chainDefinition, flatten=True)
415 butler.pruneCollection(replaced, purge=True, unstore=True)
416 elif args.prune_replaced is not None:
417 raise NotImplementedError(f"Unsupported --prune-replaced option '{args.prune_replaced}'.")
418 if not self.output.exists:
419 butler.registry.registerCollection(self.output.name, CollectionType.CHAINED)
420 if not args.extend_run:
421 butler.registry.registerCollection(self.outputRun.name, CollectionType.RUN)
422 chainDefinition.insert(0, self.outputRun.name)
423 butler.registry.setCollectionChain(self.output.name, chainDefinition, flatten=True)
424 _LOG.debug(
425 "Preparing butler to write to '%s' and read from '%s'=%s",
426 self.outputRun.name,
427 self.output.name,
428 chainDefinition,
429 )
430 butler.registry.defaults = RegistryDefaults(run=self.outputRun.name, collections=self.output.name)
431 else:
432 inputs = (self.outputRun.name,) + self.inputs
433 _LOG.debug("Preparing butler to write to '%s' and read from %s.", self.outputRun.name, inputs)
434 butler.registry.defaults = RegistryDefaults(run=self.outputRun.name, collections=inputs)
435 return butler
437 output: Optional[_OutputChainedCollectionInfo]
438 """Information about the output chained collection, if there is or will be
439 one (`_OutputChainedCollectionInfo` or `None`).
440 """
442 outputRun: Optional[_OutputRunCollectionInfo]
443 """Information about the output run collection, if there is or will be
444 one (`_OutputRunCollectionInfo` or `None`).
445 """
447 inputs: Tuple[str, ...]
448 """Input collections provided directly by the user (`tuple` [ `str` ]).
449 """
452# ------------------------
453# Exported definitions --
454# ------------------------
457class CmdLineFwk:
458 """PipelineTask framework which executes tasks from command line.
460 In addition to executing tasks this activator provides additional methods
461 for task management like dumping configuration or execution chain.
462 """
464 MP_TIMEOUT = 3600 * 24 * 30 # Default timeout (sec) for multiprocessing
466 def __init__(self) -> None:
467 pass
469 def makePipeline(self, args: SimpleNamespace) -> Pipeline:
470 """Build a pipeline from command line arguments.
472 Parameters
473 ----------
474 args : `types.SimpleNamespace`
475 Parsed command line
477 Returns
478 -------
479 pipeline : `~lsst.pipe.base.Pipeline`
480 """
481 if args.pipeline:
482 pipeline = Pipeline.from_uri(args.pipeline)
483 else:
484 pipeline = Pipeline("anonymous")
486 # loop over all pipeline actions and apply them in order
487 for action in args.pipeline_actions:
488 if action.action == "add_instrument":
490 pipeline.addInstrument(action.value)
492 elif action.action == "new_task":
494 pipeline.addTask(action.value, action.label)
496 elif action.action == "delete_task":
498 pipeline.removeTask(action.label)
500 elif action.action == "config":
502 # action value string is "field=value", split it at '='
503 field, _, value = action.value.partition("=")
504 pipeline.addConfigOverride(action.label, field, value)
506 elif action.action == "configfile":
508 pipeline.addConfigFile(action.label, action.value)
510 else:
512 raise ValueError(f"Unexpected pipeline action: {action.action}")
514 if args.save_pipeline:
515 pipeline.write_to_uri(args.save_pipeline)
517 if args.pipeline_dot:
518 pipeline2dot(pipeline, args.pipeline_dot)
520 return pipeline
522 def makeGraph(self, pipeline: Pipeline, args: SimpleNamespace) -> Optional[QuantumGraph]:
523 """Build a graph from command line arguments.
525 Parameters
526 ----------
527 pipeline : `~lsst.pipe.base.Pipeline`
528 Pipeline, can be empty or ``None`` if graph is read from a file.
529 args : `types.SimpleNamespace`
530 Parsed command line
532 Returns
533 -------
534 graph : `~lsst.pipe.base.QuantumGraph` or `None`
535 If resulting graph is empty then `None` is returned.
536 """
538 # make sure that --extend-run always enables --skip-existing
539 if args.extend_run:
540 args.skip_existing = True
542 butler, collections, run = _ButlerFactory.makeButlerAndCollections(args)
544 if args.skip_existing and run:
545 args.skip_existing_in += (run,)
547 if args.qgraph:
548 # click passes empty tuple as default value for qgraph_node_id
549 nodes = args.qgraph_node_id or None
550 qgraph = QuantumGraph.loadUri(
551 args.qgraph, butler.registry.dimensions, nodes=nodes, graphID=args.qgraph_id
552 )
554 # pipeline can not be provided in this case
555 if pipeline:
556 raise ValueError("Pipeline must not be given when quantum graph is read from file.")
557 if args.show_qgraph_header:
558 print(QuantumGraph.readHeader(args.qgraph))
559 else:
560 # make execution plan (a.k.a. DAG) for pipeline
561 graphBuilder = GraphBuilder(
562 butler.registry,
563 skipExistingIn=args.skip_existing_in,
564 clobberOutputs=args.clobber_outputs,
565 datastore=butler.datastore if args.qgraph_datastore_records else None,
566 )
567 # accumulate metadata
568 metadata = {
569 "input": args.input,
570 "output": args.output,
571 "butler_argument": args.butler_config,
572 "output_run": args.output_run,
573 "extend_run": args.extend_run,
574 "skip_existing_in": args.skip_existing_in,
575 "skip_existing": args.skip_existing,
576 "data_query": args.data_query,
577 "user": getpass.getuser(),
578 "time": f"{datetime.datetime.now()}",
579 }
580 qgraph = graphBuilder.makeGraph(
581 pipeline,
582 collections,
583 run,
584 args.data_query,
585 metadata=metadata,
586 datasetQueryConstraint=args.dataset_query_constraint,
587 )
588 if args.show_qgraph_header:
589 qgraph.buildAndPrintHeader()
591 # Count quanta in graph; give a warning if it's empty and return None.
592 nQuanta = len(qgraph)
593 if nQuanta == 0:
594 return None
595 else:
596 if _LOG.isEnabledFor(logging.INFO):
597 qg_task_table = self._generateTaskTable(qgraph)
598 qg_task_table_formatted = "\n".join(qg_task_table.pformat_all())
599 _LOG.info(
600 "QuantumGraph contains %d quanta for %d tasks, graph ID: %r\n%s",
601 nQuanta,
602 len(qgraph.taskGraph),
603 qgraph.graphID,
604 qg_task_table_formatted,
605 )
607 if args.save_qgraph:
608 qgraph.saveUri(args.save_qgraph)
610 if args.save_single_quanta:
611 for quantumNode in qgraph:
612 sqgraph = qgraph.subset(quantumNode)
613 uri = args.save_single_quanta.format(quantumNode)
614 sqgraph.saveUri(uri)
616 if args.qgraph_dot:
617 graph2dot(qgraph, args.qgraph_dot)
619 if args.execution_butler_location:
620 butler = Butler(args.butler_config)
621 newArgs = copy.deepcopy(args)
623 def builderShim(butler: Butler) -> Butler:
624 newArgs.butler_config = butler._config
625 # Calling makeWriteButler is done for the side effects of
626 # calling that method, maining parsing all the args into
627 # collection names, creating collections, etc.
628 newButler = _ButlerFactory.makeWriteButler(newArgs)
629 return newButler
631 # Include output collection in collections for input
632 # files if it exists in the repo.
633 all_inputs = args.input
634 if args.output is not None:
635 try:
636 all_inputs += (next(iter(butler.registry.queryCollections(args.output))),)
637 except MissingCollectionError:
638 pass
640 _LOG.debug("Calling buildExecutionButler with collections=%s", all_inputs)
641 buildExecutionButler(
642 butler,
643 qgraph,
644 args.execution_butler_location,
645 run,
646 butlerModifier=builderShim,
647 collections=all_inputs,
648 clobber=args.clobber_execution_butler,
649 datastoreRoot=args.target_datastore_root,
650 transfer=args.transfer,
651 )
653 return qgraph
655 def runPipeline(
656 self,
657 graph: QuantumGraph,
658 taskFactory: TaskFactory,
659 args: SimpleNamespace,
660 butler: Optional[Butler] = None,
661 ) -> None:
662 """Execute complete QuantumGraph.
664 Parameters
665 ----------
666 graph : `QuantumGraph`
667 Execution graph.
668 taskFactory : `~lsst.pipe.base.TaskFactory`
669 Task factory
670 args : `types.SimpleNamespace`
671 Parsed command line
672 butler : `~lsst.daf.butler.Butler`, optional
673 Data Butler instance, if not defined then new instance is made
674 using command line options.
675 """
676 # make sure that --extend-run always enables --skip-existing
677 if args.extend_run:
678 args.skip_existing = True
680 if not args.enable_implicit_threading:
681 disable_implicit_threading()
683 # make butler instance
684 if butler is None:
685 butler = _ButlerFactory.makeWriteButler(args, graph.iterTaskGraph())
687 if args.skip_existing:
688 args.skip_existing_in += (butler.run,)
690 # Enable lsstDebug debugging. Note that this is done once in the
691 # main process before PreExecInit and it is also repeated before
692 # running each task in SingleQuantumExecutor (which may not be
693 # needed if `multipocessing` always uses fork start method).
694 if args.enableLsstDebug:
695 try:
696 _LOG.debug("Will try to import debug.py")
697 import debug # type: ignore # noqa:F401
698 except ImportError:
699 _LOG.warn("No 'debug' module found.")
701 # Save all InitOutputs, configs, etc.
702 preExecInit = PreExecInit(butler, taskFactory, extendRun=args.extend_run, mock=args.mock)
703 preExecInit.initialize(
704 graph,
705 saveInitOutputs=not args.skip_init_writes,
706 registerDatasetTypes=args.register_dataset_types,
707 saveVersions=not args.no_versions,
708 )
710 if not args.init_only:
711 graphFixup = self._importGraphFixup(args)
712 quantumExecutor = SingleQuantumExecutor(
713 taskFactory,
714 skipExistingIn=args.skip_existing_in,
715 clobberOutputs=args.clobber_outputs,
716 enableLsstDebug=args.enableLsstDebug,
717 exitOnKnownError=args.fail_fast,
718 mock=args.mock,
719 mock_configs=args.mock_configs,
720 )
721 timeout = self.MP_TIMEOUT if args.timeout is None else args.timeout
722 executor = MPGraphExecutor(
723 numProc=args.processes,
724 timeout=timeout,
725 startMethod=args.start_method,
726 quantumExecutor=quantumExecutor,
727 failFast=args.fail_fast,
728 pdb=args.pdb,
729 executionGraphFixup=graphFixup,
730 )
731 try:
732 with util.profile(args.profile, _LOG):
733 executor.execute(graph, butler)
734 finally:
735 if args.summary:
736 report = executor.getReport()
737 if report:
738 with open(args.summary, "w") as out:
739 # Do not save fields that are not set.
740 out.write(report.json(exclude_none=True, indent=2))
742 def _generateTaskTable(self, qgraph: QuantumGraph) -> Table:
743 """Generate astropy table listing the number of quanta per task for a
744 given quantum graph.
746 Parameters
747 ----------
748 qgraph : `lsst.pipe.base.graph.graph.QuantumGraph`
749 A QuantumGraph object.
751 Returns
752 -------
753 qg_task_table : `astropy.table.table.Table`
754 An astropy table containing columns: Quanta and Tasks.
755 """
756 qg_quanta, qg_tasks = [], []
757 for task_def in qgraph.iterTaskGraph():
758 num_qnodes = qgraph.getNumberOfQuantaForTask(task_def)
759 qg_quanta.append(num_qnodes)
760 qg_tasks.append(task_def.label)
761 qg_task_table = Table(dict(Quanta=qg_quanta, Tasks=qg_tasks))
762 return qg_task_table
764 def _importGraphFixup(self, args: SimpleNamespace) -> Optional[ExecutionGraphFixup]:
765 """Import/instantiate graph fixup object.
767 Parameters
768 ----------
769 args : `types.SimpleNamespace`
770 Parsed command line.
772 Returns
773 -------
774 fixup : `ExecutionGraphFixup` or `None`
776 Raises
777 ------
778 ValueError
779 Raised if import fails, method call raises exception, or returned
780 instance has unexpected type.
781 """
782 if args.graph_fixup:
783 try:
784 factory = doImportType(args.graph_fixup)
785 except Exception as exc:
786 raise ValueError("Failed to import graph fixup class/method") from exc
787 try:
788 fixup = factory()
789 except Exception as exc:
790 raise ValueError("Failed to make instance of graph fixup") from exc
791 if not isinstance(fixup, ExecutionGraphFixup):
792 raise ValueError("Graph fixup is not an instance of ExecutionGraphFixup class")
793 return fixup
794 return None