Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of ctrl_mpexec. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22"""Module defining CmdLineFwk class and related methods. 

23""" 

24 

25__all__ = ['CmdLineFwk'] 

26 

27# ------------------------------- 

28# Imports of standard modules -- 

29# ------------------------------- 

30import argparse 

31import datetime 

32import fnmatch 

33import logging 

34import os 

35import re 

36import sys 

37from typing import List, Optional, Tuple 

38import warnings 

39 

40# ----------------------------- 

41# Imports for other modules -- 

42# ----------------------------- 

43from lsst.daf.butler import ( 

44 Butler, 

45 CollectionSearch, 

46 CollectionType, 

47 DatasetTypeRestriction, 

48 Registry, 

49) 

50from lsst.daf.butler.registry import MissingCollectionError 

51import lsst.log 

52import lsst.pex.config as pexConfig 

53from lsst.pipe.base import GraphBuilder, Pipeline, QuantumGraph 

54from .cmdLineParser import makeParser 

55from .dotTools import graph2dot, pipeline2dot 

56from .executionGraphFixup import ExecutionGraphFixup 

57from .mpGraphExecutor import MPGraphExecutor 

58from .preExecInit import PreExecInit 

59from .singleQuantumExecutor import SingleQuantumExecutor 

60from .taskFactory import TaskFactory 

61from . import util 

62from lsst.utils import doImport 

63 

64# ---------------------------------- 

65# Local non-exported definitions -- 

66# ---------------------------------- 

67 

68# logging properties 

69_LOG_PROP = """\ 

70log4j.rootLogger=INFO, A1 

71log4j.appender.A1=ConsoleAppender 

72log4j.appender.A1.Target=System.err 

73log4j.appender.A1.layout=PatternLayout 

74log4j.appender.A1.layout.ConversionPattern={} 

75""" 

76 

77_LOG = logging.getLogger(__name__.partition(".")[2]) 

78 

79 

80class _OutputChainedCollectionInfo: 

81 """A helper class for handling command-line arguments related to an output 

82 `~lsst.daf.butler.CollectionType.CHAINED` collection. 

83 

84 Parameters 

85 ---------- 

86 registry : `lsst.daf.butler.Registry` 

87 Butler registry that collections will be added to and/or queried from. 

88 name : `str` 

89 Name of the collection given on the command line. 

90 """ 

91 def __init__(self, registry: Registry, name: str): 

92 self.name = name 

93 try: 

94 self.chain = list(registry.getCollectionChain(name)) 

95 self.exists = True 

96 except MissingCollectionError: 

97 self.chain = [] 

98 self.exists = False 

99 

100 def __str__(self): 

101 return self.name 

102 

103 name: str 

104 """Name of the collection provided on the command line (`str`). 

105 """ 

106 

107 exists: bool 

108 """Whether this collection already exists in the registry (`bool`). 

109 """ 

110 

111 chain: List[Tuple[str, DatasetTypeRestriction]] 

112 """The definition of the collection, if it already exists (`list`). 

113 

114 Empty if the collection does not alredy exist. 

115 """ 

116 

117 

118class _OutputRunCollectionInfo: 

119 """A helper class for handling command-line arguments related to an output 

120 `~lsst.daf.butler.CollectionType.RUN` collection. 

121 

122 Parameters 

123 ---------- 

124 registry : `lsst.daf.butler.Registry` 

125 Butler registry that collections will be added to and/or queried from. 

126 name : `str` 

127 Name of the collection given on the command line. 

128 """ 

129 def __init__(self, registry: Registry, name: str): 

130 self.name = name 

131 try: 

132 actualType = registry.getCollectionType(name) 

133 if actualType is not CollectionType.RUN: 

134 raise TypeError(f"Collection '{name}' exists but has type {actualType.name}, not RUN.") 

135 self.exists = True 

136 except MissingCollectionError: 

137 self.exists = False 

138 

139 name: str 

140 """Name of the collection provided on the command line (`str`). 

141 """ 

142 

143 exists: bool 

144 """Whether this collection already exists in the registry (`bool`). 

145 """ 

146 

147 

148class _ButlerFactory: 

149 """A helper class for processing command-line arguments related to input 

150 and output collections. 

151 

152 Parameters 

153 ---------- 

154 registry : `lsst.daf.butler.Registry` 

155 Butler registry that collections will be added to and/or queried from. 

156 

157 args : `argparse.Namespace` 

158 Parsed command-line arguments. The following attributes are used, 

159 either at construction or in later methods. 

160 

161 ``output`` 

162 The name of a `~lsst.daf.butler.CollectionType.CHAINED` 

163 input/output collection. 

164 

165 ``output_run`` 

166 The name of a `~lsst.daf.butler.CollectionType.RUN` input/output 

167 collection. 

168 

169 ``extend_run`` 

170 A boolean indicating whether ``output_run`` should already exist 

171 and be extended. 

172 

173 ``replace_run`` 

174 A boolean indicating that (if `True`) ``output_run`` should already 

175 exist but will be removed from the output chained collection and 

176 replaced with a new one. 

177 

178 ``prune_replaced`` 

179 A boolean indicating whether to prune the replaced run (requires 

180 ``replace_run``). 

181 

182 ``inputs`` 

183 Input collections of any type; may be any type handled by 

184 `lsst.daf.butler.registry.CollectionSearch.fromExpression`. 

185 

186 ``butler_config`` 

187 Path to a data repository root or configuration file. 

188 

189 writeable : `bool` 

190 If `True`, a `Butler` is being initialized in a context where actual 

191 writes should happens, and hence no output run is necessary. 

192 

193 Raises 

194 ------ 

195 ValueError 

196 Raised if ``writeable is True`` but there are no output collections. 

197 """ 

198 def __init__(self, registry: Registry, args: argparse.Namespace, writeable: bool): 

199 if args.output is not None: 

200 self.output = _OutputChainedCollectionInfo(registry, args.output) 

201 else: 

202 self.output = None 

203 if args.output_run is not None: 

204 self.outputRun = _OutputRunCollectionInfo(registry, args.output_run) 

205 elif self.output is not None: 

206 if args.extend_run: 

207 runName, _ = self.output.chain[0] 

208 else: 

209 runName = "{}/{:%Y%m%dT%Hh%Mm%Ss}".format(self.output, datetime.datetime.now()) 

210 self.outputRun = _OutputRunCollectionInfo(registry, runName) 

211 elif not writeable: 

212 # If we're not writing yet, ok to have no output run. 

213 self.outputRun = None 

214 else: 

215 raise ValueError("Cannot write without at least one of (--output, --output-run).") 

216 self.inputs = list(CollectionSearch.fromExpression(args.input)) 

217 

218 def check(self, args: argparse.Namespace): 

219 """Check command-line options for consistency with each other and the 

220 data repository. 

221 

222 Parameters 

223 ---------- 

224 args : `argparse.Namespace` 

225 Parsed command-line arguments. See class documentation for the 

226 construction parameter of the same name. 

227 """ 

228 assert not (args.extend_run and args.replace_run), "In mutually-exclusive group in ArgumentParser." 

229 if self.inputs and self.output is not None and self.output.exists: 

230 raise ValueError("Cannot use --output with existing collection with --inputs.") 

231 if args.extend_run and self.outputRun is None: 

232 raise ValueError("Cannot --extend-run when no output collection is given.") 

233 if args.extend_run and not self.outputRun.exists: 

234 raise ValueError(f"Cannot --extend-run; output collection " 

235 f"'{self.outputRun.name}' does not exist.") 

236 if not args.extend_run and self.outputRun is not None and self.outputRun.exists: 

237 raise ValueError(f"Output run '{self.outputRun.name}' already exists, but " 

238 f"--extend-run was not given.") 

239 if args.prune_replaced and not args.replace_run: 

240 raise ValueError(f"--prune-replaced requires --replace-run.") 

241 if args.replace_run and (self.output is None or not self.output.exists): 

242 raise ValueError(f"--output must point to an existing CHAINED collection for --replace-run.") 

243 

244 @classmethod 

245 def _makeReadParts(cls, args: argparse.Namespace): 

246 """Common implementation for `makeReadButler` and 

247 `makeRegistryAndCollections`. 

248 

249 Parameters 

250 ---------- 

251 args : `argparse.Namespace` 

252 Parsed command-line arguments. See class documentation for the 

253 construction parameter of the same name. 

254 

255 Returns 

256 ------- 

257 butler : `lsst.daf.butler.Butler` 

258 A read-only butler constructed from the repo at 

259 ``args.butler_config``, but with no default collections. 

260 inputs : `lsst.daf.butler.registry.CollectionSearch` 

261 A collection search path constructed according to ``args``. 

262 self : `_ButlerFactory` 

263 A new `_ButlerFactory` instance representing the processed version 

264 of ``args``. 

265 """ 

266 butler = Butler(args.butler_config, writeable=False) 

267 self = cls(butler.registry, args, writeable=False) 

268 self.check(args) 

269 if self.output and self.output.exists: 

270 if args.replace_run: 

271 replaced, _ = self.output.chain[0] 

272 inputs = self.output.chain[1:] 

273 _LOG.debug("Simulating collection search in '%s' after removing '%s'.", 

274 self.output.name, replaced) 

275 else: 

276 inputs = [self.output.name] 

277 else: 

278 inputs = list(self.inputs) 

279 if args.extend_run: 

280 inputs.insert(0, self.outputRun.name) 

281 inputs = CollectionSearch.fromExpression(inputs) 

282 return butler, inputs, self 

283 

284 @classmethod 

285 def makeReadButler(cls, args: argparse.Namespace) -> Butler: 

286 """Construct a read-only butler according to the given command-line 

287 arguments. 

288 

289 Parameters 

290 ---------- 

291 args : `argparse.Namespace` 

292 Parsed command-line arguments. See class documentation for the 

293 construction parameter of the same name. 

294 

295 Returns 

296 ------- 

297 butler : `lsst.daf.butler.Butler` 

298 A read-only butler initialized with the collections specified by 

299 ``args``. 

300 """ 

301 butler, inputs, _ = cls._makeReadParts(args) 

302 _LOG.debug("Preparing butler to read from %s.", inputs) 

303 return Butler(butler=butler, collections=inputs) 

304 

305 @classmethod 

306 def makeRegistryAndCollections(cls, args: argparse.Namespace) -> \ 

307 Tuple[Registry, CollectionSearch, Optional[str]]: 

308 """Return a read-only registry, a collection search path, and the name 

309 of the run to be used for future writes. 

310 

311 Parameters 

312 ---------- 

313 args : `argparse.Namespace` 

314 Parsed command-line arguments. See class documentation for the 

315 construction parameter of the same name. 

316 

317 Returns 

318 ------- 

319 registry : `lsst.daf.butler.Registry` 

320 Butler registry that collections will be added to and/or queried 

321 from. 

322 inputs : `lsst.daf.butler.registry.CollectionSearch` 

323 Collections to search for datasets. 

324 run : `str` or `None` 

325 Name of the output `~lsst.daf.butler.CollectionType.RUN` collection 

326 if it already exists, or `None` if it does not. 

327 """ 

328 butler, inputs, self = cls._makeReadParts(args) 

329 run = self.outputRun.name if args.extend_run else None 

330 _LOG.debug("Preparing registry to read from %s and expect future writes to '%s'.", inputs, run) 

331 return butler.registry, inputs, run 

332 

333 @classmethod 

334 def makeWriteButler(cls, args: argparse.Namespace) -> Butler: 

335 """Return a read-write butler initialized to write to and read from 

336 the collections specified by the given command-line arguments. 

337 

338 Parameters 

339 ---------- 

340 args : `argparse.Namespace` 

341 Parsed command-line arguments. See class documentation for the 

342 construction parameter of the same name. 

343 

344 Returns 

345 ------- 

346 butler : `lsst.daf.butler.Butler` 

347 A read-write butler initialized according to the given arguments. 

348 """ 

349 butler = Butler(args.butler_config, writeable=True) 

350 self = cls(butler.registry, args, writeable=True) 

351 self.check(args) 

352 if self.output is not None: 

353 chainDefinition = list(self.output.chain if self.output.exists else self.inputs) 

354 if args.replace_run: 

355 replaced, _ = chainDefinition.pop(0) 

356 if args.prune_replaced: 

357 # TODO: DM-23671: need a butler API for pruning an 

358 # entire RUN collection, then apply it to 'replaced' 

359 # here. 

360 raise NotImplementedError("Support for --prune-replaced is not yet implemented.") 

361 chainDefinition.insert(0, self.outputRun.name) 

362 chainDefinition = CollectionSearch.fromExpression(chainDefinition) 

363 _LOG.debug("Preparing butler to write to '%s' and read from '%s'=%s", 

364 self.outputRun.name, self.output.name, chainDefinition) 

365 return Butler(butler=butler, run=self.outputRun.name, collections=self.output.name, 

366 chains={self.output.name: chainDefinition}) 

367 else: 

368 inputs = CollectionSearch.fromExpression([self.outputRun.name] + self.inputs) 

369 _LOG.debug("Preparing butler to write to '%s' and read from %s.", self.outputRun.name, inputs) 

370 return Butler(butler=butler, run=self.outputRun.name, collections=inputs) 

371 

372 output: Optional[_OutputChainedCollectionInfo] 

373 """Information about the output chained collection, if there is or will be 

374 one (`_OutputChainedCollectionInfo` or `None`). 

375 """ 

376 

377 outputRun: Optional[_OutputRunCollectionInfo] 

378 """Information about the output run collection, if there is or will be 

379 one (`_OutputRunCollectionInfo` or `None`). 

380 """ 

381 

382 inputs: List[Tuple[str, DatasetTypeRestriction]] 

383 """Input collections, including those also used for outputs and any 

384 restrictions on dataset types (`list`). 

385 """ 

386 

387 

388class _FilteredStream: 

389 """A file-like object that filters some config fields. 

390 

391 Note 

392 ---- 

393 This class depends on implementation details of ``Config.saveToStream`` 

394 methods, in particular that that method uses single call to write() 

395 method to save information about single config field, and that call 

396 combines comments string(s) for a field and field path and value. 

397 This class will not work reliably on the "import" strings, so imports 

398 should be disabled by passing ``skipImports=True`` to ``saveToStream()``. 

399 """ 

400 def __init__(self, pattern): 

401 # obey case if pattern isn't lowercase or requests NOIGNORECASE 

402 mat = re.search(r"(.*):NOIGNORECASE$", pattern) 

403 

404 if mat: 

405 pattern = mat.group(1) 

406 self._pattern = re.compile(fnmatch.translate(pattern)) 

407 else: 

408 if pattern != pattern.lower(): 

409 print(f"Matching \"{pattern}\" without regard to case " 

410 "(append :NOIGNORECASE to prevent this)", file=sys.stdout) 

411 self._pattern = re.compile(fnmatch.translate(pattern), re.IGNORECASE) 

412 

413 def write(self, showStr): 

414 # Strip off doc string line(s) and cut off at "=" for string matching 

415 matchStr = showStr.rstrip().split("\n")[-1].split("=")[0] 

416 if self._pattern.search(matchStr): 

417 sys.stdout.write(showStr) 

418 

419# ------------------------ 

420# Exported definitions -- 

421# ------------------------ 

422 

423 

424class CmdLineFwk: 

425 """PipelineTask framework which executes tasks from command line. 

426 

427 In addition to executing tasks this activator provides additional methods 

428 for task management like dumping configuration or execution chain. 

429 """ 

430 

431 MP_TIMEOUT = 9999 # Default timeout (sec) for multiprocessing 

432 

433 def __init__(self): 

434 pass 

435 

436 def parseAndRun(self, argv=None): 

437 """ 

438 This method is a main entry point for this class, it parses command 

439 line and executes all commands. 

440 

441 Parameters 

442 ---------- 

443 argv : `list` of `str`, optional 

444 list of command line arguments, if not specified then 

445 `sys.argv[1:]` is used 

446 """ 

447 

448 if argv is None: 

449 argv = sys.argv[1:] 

450 

451 # start with parsing command line, only do partial parsing now as 

452 # the tasks can add more arguments later 

453 parser = makeParser() 

454 args = parser.parse_args(argv) 

455 

456 # First thing to do is to setup logging. 

457 self.configLog(args.longlog, args.loglevel) 

458 

459 taskFactory = TaskFactory() 

460 

461 # make pipeline out of command line arguments (can return empty pipeline) 

462 try: 

463 pipeline = self.makePipeline(args) 

464 except Exception as exc: 

465 print("Failed to build pipeline: {}".format(exc), file=sys.stderr) 

466 raise 

467 

468 if args.subcommand == "build": 

469 # stop here but process --show option first 

470 self.showInfo(args, pipeline) 

471 return 0 

472 

473 # make quantum graph 

474 try: 

475 qgraph = self.makeGraph(pipeline, args) 

476 except Exception as exc: 

477 print("Failed to build graph: {}".format(exc), file=sys.stderr) 

478 raise 

479 

480 # optionally dump some info 

481 self.showInfo(args, pipeline, qgraph) 

482 

483 if qgraph is None: 

484 # No need to raise an exception here, code that makes graph 

485 # should have printed warning message already. 

486 return 2 

487 

488 if args.subcommand == "qgraph": 

489 # stop here 

490 return 0 

491 

492 # execute 

493 if args.subcommand == "run": 

494 return self.runPipeline(qgraph, taskFactory, args) 

495 

496 @staticmethod 

497 def configLog(longlog, logLevels): 

498 """Configure logging system. 

499 

500 Parameters 

501 ---------- 

502 longlog : `bool` 

503 If True then make log messages appear in "long format" 

504 logLevels : `list` of `tuple` 

505 per-component logging levels, each item in the list is a tuple 

506 (component, level), `component` is a logger name or `None` for root 

507 logger, `level` is a logging level name ('DEBUG', 'INFO', etc.) 

508 """ 

509 if longlog: 

510 message_fmt = "%-5p %d{yyyy-MM-ddTHH:mm:ss.SSSZ} %c (%X{LABEL})(%F:%L)- %m%n" 

511 else: 

512 message_fmt = "%c %p: %m%n" 

513 

514 # Initialize global logging config. Skip if the env var LSST_LOG_CONFIG exists. 

515 # The file it points to would already configure lsst.log. 

516 if not os.path.isfile(os.environ.get("LSST_LOG_CONFIG", "")): 

517 lsst.log.configure_prop(_LOG_PROP.format(message_fmt)) 

518 

519 # Forward all Python logging to lsst.log 

520 lgr = logging.getLogger() 

521 lgr.setLevel(logging.INFO) # same as in log4cxx config above 

522 lgr.addHandler(lsst.log.LogHandler()) 

523 

524 # also capture warnings and send them to logging 

525 logging.captureWarnings(True) 

526 

527 # configure individual loggers 

528 for component, level in logLevels: 

529 level = getattr(lsst.log.Log, level.upper(), None) 

530 if level is not None: 

531 # set logging level for lsst.log 

532 logger = lsst.log.Log.getLogger(component or "") 

533 logger.setLevel(level) 

534 # set logging level for Python logging 

535 pyLevel = lsst.log.LevelTranslator.lsstLog2logging(level) 

536 logging.getLogger(component).setLevel(pyLevel) 

537 

538 def makePipeline(self, args): 

539 """Build a pipeline from command line arguments. 

540 

541 Parameters 

542 ---------- 

543 args : `argparse.Namespace` 

544 Parsed command line 

545 

546 Returns 

547 ------- 

548 pipeline : `~lsst.pipe.base.Pipeline` 

549 """ 

550 if args.pipeline: 

551 pipeline = Pipeline.fromFile(args.pipeline) 

552 else: 

553 pipeline = Pipeline("anonymous") 

554 

555 # loop over all pipeline actions and apply them in order 

556 for action in args.pipeline_actions: 

557 if action.action == "add_instrument": 

558 

559 pipeline.addInstrument(action.value) 

560 

561 elif action.action == "new_task": 

562 

563 pipeline.addTask(action.value, action.label) 

564 

565 elif action.action == "delete_task": 

566 

567 pipeline.removeTask(action.label) 

568 

569 elif action.action == "config": 

570 

571 # action value string is "field=value", split it at '=' 

572 field, _, value = action.value.partition("=") 

573 pipeline.addConfigOverride(action.label, field, value) 

574 

575 elif action.action == "configfile": 

576 

577 pipeline.addConfigFile(action.label, action.value) 

578 

579 else: 

580 

581 raise ValueError(f"Unexpected pipeline action: {action.action}") 

582 

583 if args.save_pipeline: 

584 pipeline.toFile(args.save_pipeline) 

585 

586 if args.pipeline_dot: 

587 pipeline2dot(pipeline, args.pipeline_dot) 

588 

589 return pipeline 

590 

591 def makeGraph(self, pipeline, args): 

592 """Build a graph from command line arguments. 

593 

594 Parameters 

595 ---------- 

596 pipeline : `~lsst.pipe.base.Pipeline` 

597 Pipeline, can be empty or ``None`` if graph is read from a file. 

598 args : `argparse.Namespace` 

599 Parsed command line 

600 

601 Returns 

602 ------- 

603 graph : `~lsst.pipe.base.QuantumGraph` or `None` 

604 If resulting graph is empty then `None` is returned. 

605 """ 

606 

607 registry, collections, run = _ButlerFactory.makeRegistryAndCollections(args) 

608 

609 if args.qgraph: 

610 

611 with open(args.qgraph, 'rb') as pickleFile: 

612 qgraph = QuantumGraph.load(pickleFile, registry.dimensions) 

613 

614 # pipeline can not be provided in this case 

615 if pipeline: 

616 raise ValueError("Pipeline must not be given when quantum graph is read from file.") 

617 

618 else: 

619 

620 # make execution plan (a.k.a. DAG) for pipeline 

621 graphBuilder = GraphBuilder(registry, 

622 skipExisting=args.skip_existing) 

623 qgraph = graphBuilder.makeGraph(pipeline, collections, run, args.data_query) 

624 

625 # count quanta in graph and give a warning if it's empty and return None 

626 nQuanta = qgraph.countQuanta() 

627 if nQuanta == 0: 

628 warnings.warn("QuantumGraph is empty", stacklevel=2) 

629 return None 

630 else: 

631 _LOG.info("QuantumGraph contains %d quanta for %d tasks", 

632 nQuanta, len(qgraph)) 

633 

634 if args.save_qgraph: 

635 with open(args.save_qgraph, "wb") as pickleFile: 

636 qgraph.save(pickleFile) 

637 

638 if args.save_single_quanta: 

639 for iq, sqgraph in enumerate(qgraph.quantaAsQgraph()): 

640 filename = args.save_single_quanta.format(iq) 

641 with open(filename, "wb") as pickleFile: 

642 sqgraph.save(pickleFile) 

643 

644 if args.qgraph_dot: 

645 graph2dot(qgraph, args.qgraph_dot) 

646 

647 return qgraph 

648 

649 def runPipeline(self, graph, taskFactory, args, butler=None): 

650 """Execute complete QuantumGraph. 

651 

652 Parameters 

653 ---------- 

654 graph : `QuantumGraph` 

655 Execution graph. 

656 taskFactory : `~lsst.pipe.base.TaskFactory` 

657 Task factory 

658 args : `argparse.Namespace` 

659 Parsed command line 

660 butler : `~lsst.daf.butler.Butler`, optional 

661 Data Butler instance, if not defined then new instance is made 

662 using command line options. 

663 """ 

664 # make butler instance 

665 if butler is None: 

666 butler = _ButlerFactory.makeWriteButler(args) 

667 

668 # Enable lsstDebug debugging. Note that this is done once in the 

669 # main process before PreExecInit and it is also repeated before 

670 # running each task in SingleQuantumExecutor (which may not be 

671 # needed if `multipocessing` always uses fork start method). 

672 if args.enableLsstDebug: 

673 try: 

674 _LOG.debug("Will try to import debug.py") 

675 import debug # noqa:F401 

676 except ImportError: 

677 _LOG.warn("No 'debug' module found.") 

678 

679 preExecInit = PreExecInit(butler, taskFactory, args.skip_existing) 

680 preExecInit.initialize(graph, 

681 saveInitOutputs=not args.skip_init_writes, 

682 registerDatasetTypes=args.register_dataset_types, 

683 saveVersions=not args.no_versions) 

684 

685 if not args.init_only: 

686 graphFixup = self._importGraphFixup(args) 

687 quantumExecutor = SingleQuantumExecutor(taskFactory, 

688 skipExisting=args.skip_existing, 

689 enableLsstDebug=args.enableLsstDebug) 

690 timeout = self.MP_TIMEOUT if args.timeout is None else args.timeout 

691 executor = MPGraphExecutor(numProc=args.processes, timeout=timeout, 

692 quantumExecutor=quantumExecutor, 

693 executionGraphFixup=graphFixup) 

694 with util.profile(args.profile, _LOG): 

695 executor.execute(graph, butler) 

696 

697 def showInfo(self, args, pipeline, graph=None): 

698 """Display useful info about pipeline and environment. 

699 

700 Parameters 

701 ---------- 

702 args : `argparse.Namespace` 

703 Parsed command line 

704 pipeline : `Pipeline` 

705 Pipeline definition 

706 graph : `QuantumGraph`, optional 

707 Execution graph 

708 """ 

709 showOpts = args.show 

710 for what in showOpts: 

711 showCommand, _, showArgs = what.partition("=") 

712 

713 if showCommand in ["pipeline", "config", "history", "tasks"]: 

714 if not pipeline: 

715 _LOG.warning("Pipeline is required for --show=%s", showCommand) 

716 continue 

717 

718 if showCommand in ["graph", "workflow"]: 

719 if not graph: 

720 _LOG.warning("QuantumGraph is required for --show=%s", showCommand) 

721 continue 

722 

723 if showCommand == "pipeline": 

724 print(pipeline) 

725 elif showCommand == "config": 

726 self._showConfig(pipeline, showArgs, False) 

727 elif showCommand == "dump-config": 

728 self._showConfig(pipeline, showArgs, True) 

729 elif showCommand == "history": 

730 self._showConfigHistory(pipeline, showArgs) 

731 elif showCommand == "tasks": 

732 self._showTaskHierarchy(pipeline) 

733 elif showCommand == "graph": 

734 if graph: 

735 self._showGraph(graph) 

736 elif showCommand == "workflow": 

737 if graph: 

738 self._showWorkflow(graph, args) 

739 else: 

740 print("Unknown value for show: %s (choose from '%s')" % 

741 (what, "', '".join("pipeline config[=XXX] history=XXX tasks graph".split())), 

742 file=sys.stderr) 

743 sys.exit(1) 

744 

745 def _showConfig(self, pipeline, showArgs, dumpFullConfig): 

746 """Show task configuration 

747 

748 Parameters 

749 ---------- 

750 pipeline : `Pipeline` 

751 Pipeline definition 

752 showArgs : `str` 

753 Defines what to show 

754 dumpFullConfig : `bool` 

755 If true then dump complete task configuration with all imports. 

756 """ 

757 stream = sys.stdout 

758 if dumpFullConfig: 

759 # Task label can be given with this option 

760 taskName = showArgs 

761 else: 

762 # The argument can have form [TaskLabel::][pattern:NOIGNORECASE] 

763 matConfig = re.search(r"^(?:(\w+)::)?(?:config.)?(.+)?", showArgs) 

764 taskName = matConfig.group(1) 

765 pattern = matConfig.group(2) 

766 if pattern: 

767 stream = _FilteredStream(pattern) 

768 

769 tasks = util.filterTasks(pipeline, taskName) 

770 if not tasks: 

771 print("Pipeline has no tasks named {}".format(taskName), file=sys.stderr) 

772 sys.exit(1) 

773 

774 for taskDef in tasks: 

775 print("### Configuration for task `{}'".format(taskDef.label)) 

776 taskDef.config.saveToStream(stream, root="config", skipImports=not dumpFullConfig) 

777 

778 def _showConfigHistory(self, pipeline, showArgs): 

779 """Show history for task configuration 

780 

781 Parameters 

782 ---------- 

783 pipeline : `Pipeline` 

784 Pipeline definition 

785 showArgs : `str` 

786 Defines what to show 

787 """ 

788 

789 taskName = None 

790 pattern = None 

791 matHistory = re.search(r"^(?:(\w+)::)?(?:config[.])?(.+)", showArgs) 

792 if matHistory: 

793 taskName = matHistory.group(1) 

794 pattern = matHistory.group(2) 

795 if not pattern: 

796 print("Please provide a value with --show history (e.g. history=Task::param)", file=sys.stderr) 

797 sys.exit(1) 

798 

799 tasks = util.filterTasks(pipeline, taskName) 

800 if not tasks: 

801 print(f"Pipeline has no tasks named {taskName}", file=sys.stderr) 

802 sys.exit(1) 

803 

804 cpath, _, cname = pattern.rpartition(".") 

805 found = False 

806 for taskDef in tasks: 

807 try: 

808 if not cpath: 

809 # looking for top-level field 

810 hconfig = taskDef.config 

811 else: 

812 hconfig = eval("config." + cpath, {}, {"config": taskDef.config}) 

813 except AttributeError: 

814 # Means this config object has no such field, but maybe some other task has it. 

815 continue 

816 except Exception: 

817 # Any other exception probably means some error in the expression. 

818 print(f"ERROR: Failed to evaluate field expression `{pattern}'", file=sys.stderr) 

819 sys.exit(1) 

820 

821 if hasattr(hconfig, cname): 

822 print(f"### Configuration field for task `{taskDef.label}'") 

823 print(pexConfig.history.format(hconfig, cname)) 

824 found = True 

825 

826 if not found: 

827 print(f"None of the tasks has field named {pattern}", file=sys.stderr) 

828 sys.exit(1) 

829 

830 def _showTaskHierarchy(self, pipeline): 

831 """Print task hierarchy to stdout 

832 

833 Parameters 

834 ---------- 

835 pipeline: `Pipeline` 

836 """ 

837 for taskDef in pipeline.toExpandedPipeline(): 

838 print("### Subtasks for task `{}'".format(taskDef.taskName)) 

839 

840 for configName, taskName in util.subTaskIter(taskDef.config): 

841 print("{}: {}".format(configName, taskName)) 

842 

843 def _showGraph(self, graph): 

844 """Print quanta information to stdout 

845 

846 Parameters 

847 ---------- 

848 graph : `QuantumGraph` 

849 Execution graph. 

850 """ 

851 for taskNodes in graph: 

852 print(taskNodes.taskDef) 

853 

854 for iq, quantum in enumerate(taskNodes.quanta): 

855 print(" Quantum {}:".format(iq)) 

856 print(" inputs:") 

857 for key, refs in quantum.predictedInputs.items(): 

858 dataIds = ["DataId({})".format(ref.dataId) for ref in refs] 

859 print(" {}: [{}]".format(key, ", ".join(dataIds))) 

860 print(" outputs:") 

861 for key, refs in quantum.outputs.items(): 

862 dataIds = ["DataId({})".format(ref.dataId) for ref in refs] 

863 print(" {}: [{}]".format(key, ", ".join(dataIds))) 

864 

865 def _showWorkflow(self, graph, args): 

866 """Print quanta information and dependency to stdout 

867 

868 The input and predicted output URIs based on the Butler repo are printed. 

869 

870 Parameters 

871 ---------- 

872 graph : `QuantumGraph` 

873 Execution graph. 

874 args : `argparse.Namespace` 

875 Parsed command line 

876 """ 

877 def dumpURIs(thisRef): 

878 primary, components = butler.getURIs(thisRef, predict=True, run="TBD") 

879 if primary: 

880 print(f" {primary}") 

881 else: 

882 print(f" (disassembled artifact)") 

883 for compName, compUri in components.items(): 

884 print(f" {compName}: {compUri}") 

885 

886 butler = _ButlerFactory.makeReadButler(args) 

887 for qdata in graph.traverse(): 

888 shortname = qdata.taskDef.taskName.split('.')[-1] 

889 print("Quantum {}: {}".format(qdata.index, shortname)) 

890 print(" inputs:") 

891 for key, refs in qdata.quantum.predictedInputs.items(): 

892 for ref in refs: 

893 dumpURIs(ref) 

894 print(" outputs:") 

895 for key, refs in qdata.quantum.outputs.items(): 

896 for ref in refs: 

897 dumpURIs(ref) 

898 for parent in qdata.dependencies: 

899 print("Parent Quantum {} - Child Quantum {}".format(parent, qdata.index)) 

900 

901 def _importGraphFixup(self, args): 

902 """Import/instantiate graph fixup object. 

903 

904 Parameters 

905 ---------- 

906 args : `argparse.Namespace` 

907 Parsed command line. 

908 

909 Returns 

910 ------- 

911 fixup : `ExecutionGraphFixup` or `None` 

912 

913 Raises 

914 ------ 

915 ValueError 

916 Raised if import fails, method call raises exception, or returned 

917 instance has unexpected type. 

918 """ 

919 if args.graph_fixup: 

920 try: 

921 factory = doImport(args.graph_fixup) 

922 except Exception as exc: 

923 raise ValueError("Failed to import graph fixup class/method") from exc 

924 try: 

925 fixup = factory() 

926 except Exception as exc: 

927 raise ValueError("Failed to make instance of graph fixup") from exc 

928 if not isinstance(fixup, ExecutionGraphFixup): 

929 raise ValueError("Graph fixup is not an instance of ExecutionGraphFixup class") 

930 return fixup