Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of ctrl_mpexec. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22"""Module defining CmdLineFwk class and related methods. 

23""" 

24 

25__all__ = ['CmdLineFwk'] 

26 

27# ------------------------------- 

28# Imports of standard modules -- 

29# ------------------------------- 

30import argparse 

31import fnmatch 

32import logging 

33import re 

34import sys 

35from typing import Optional, Tuple 

36import warnings 

37 

38# ----------------------------- 

39# Imports for other modules -- 

40# ----------------------------- 

41from lsst.daf.butler import ( 

42 Butler, 

43 CollectionSearch, 

44 CollectionType, 

45 Registry, 

46) 

47from lsst.daf.butler.registry import MissingCollectionError, RegistryDefaults 

48import lsst.pex.config as pexConfig 

49from lsst.pipe.base import GraphBuilder, Pipeline, QuantumGraph 

50from lsst.obs.base import Instrument 

51from .dotTools import graph2dot, pipeline2dot 

52from .executionGraphFixup import ExecutionGraphFixup 

53from .mpGraphExecutor import MPGraphExecutor 

54from .preExecInit import PreExecInit 

55from .singleQuantumExecutor import SingleQuantumExecutor 

56from . import util 

57from lsst.utils import doImport 

58 

59# ---------------------------------- 

60# Local non-exported definitions -- 

61# ---------------------------------- 

62 

63_LOG = logging.getLogger(__name__.partition(".")[2]) 

64 

65 

66class _OutputChainedCollectionInfo: 

67 """A helper class for handling command-line arguments related to an output 

68 `~lsst.daf.butler.CollectionType.CHAINED` collection. 

69 

70 Parameters 

71 ---------- 

72 registry : `lsst.daf.butler.Registry` 

73 Butler registry that collections will be added to and/or queried from. 

74 name : `str` 

75 Name of the collection given on the command line. 

76 """ 

77 def __init__(self, registry: Registry, name: str): 

78 self.name = name 

79 try: 

80 self.chain = tuple(registry.getCollectionChain(name)) 

81 self.exists = True 

82 except MissingCollectionError: 

83 self.chain = () 

84 self.exists = False 

85 

86 def __str__(self): 

87 return self.name 

88 

89 name: str 

90 """Name of the collection provided on the command line (`str`). 

91 """ 

92 

93 exists: bool 

94 """Whether this collection already exists in the registry (`bool`). 

95 """ 

96 

97 chain: Tuple[str, ...] 

98 """The definition of the collection, if it already exists (`tuple` [`str`]). 

99 

100 Empty if the collection does not already exist. 

101 """ 

102 

103 

104class _OutputRunCollectionInfo: 

105 """A helper class for handling command-line arguments related to an output 

106 `~lsst.daf.butler.CollectionType.RUN` collection. 

107 

108 Parameters 

109 ---------- 

110 registry : `lsst.daf.butler.Registry` 

111 Butler registry that collections will be added to and/or queried from. 

112 name : `str` 

113 Name of the collection given on the command line. 

114 """ 

115 def __init__(self, registry: Registry, name: str): 

116 self.name = name 

117 try: 

118 actualType = registry.getCollectionType(name) 

119 if actualType is not CollectionType.RUN: 

120 raise TypeError(f"Collection '{name}' exists but has type {actualType.name}, not RUN.") 

121 self.exists = True 

122 except MissingCollectionError: 

123 self.exists = False 

124 

125 name: str 

126 """Name of the collection provided on the command line (`str`). 

127 """ 

128 

129 exists: bool 

130 """Whether this collection already exists in the registry (`bool`). 

131 """ 

132 

133 

134class _ButlerFactory: 

135 """A helper class for processing command-line arguments related to input 

136 and output collections. 

137 

138 Parameters 

139 ---------- 

140 registry : `lsst.daf.butler.Registry` 

141 Butler registry that collections will be added to and/or queried from. 

142 

143 args : `argparse.Namespace` 

144 Parsed command-line arguments. The following attributes are used, 

145 either at construction or in later methods. 

146 

147 ``output`` 

148 The name of a `~lsst.daf.butler.CollectionType.CHAINED` 

149 input/output collection. 

150 

151 ``output_run`` 

152 The name of a `~lsst.daf.butler.CollectionType.RUN` input/output 

153 collection. 

154 

155 ``extend_run`` 

156 A boolean indicating whether ``output_run`` should already exist 

157 and be extended. 

158 

159 ``replace_run`` 

160 A boolean indicating that (if `True`) ``output_run`` should already 

161 exist but will be removed from the output chained collection and 

162 replaced with a new one. 

163 

164 ``prune_replaced`` 

165 A boolean indicating whether to prune the replaced run (requires 

166 ``replace_run``). 

167 

168 ``inputs`` 

169 Input collections of any type; may be any type handled by 

170 `lsst.daf.butler.registry.CollectionSearch.fromExpression`. 

171 

172 ``butler_config`` 

173 Path to a data repository root or configuration file. 

174 

175 writeable : `bool` 

176 If `True`, a `Butler` is being initialized in a context where actual 

177 writes should happens, and hence no output run is necessary. 

178 

179 Raises 

180 ------ 

181 ValueError 

182 Raised if ``writeable is True`` but there are no output collections. 

183 """ 

184 def __init__(self, registry: Registry, args: argparse.Namespace, writeable: bool): 

185 if args.output is not None: 

186 self.output = _OutputChainedCollectionInfo(registry, args.output) 

187 else: 

188 self.output = None 

189 if args.output_run is not None: 

190 self.outputRun = _OutputRunCollectionInfo(registry, args.output_run) 

191 elif self.output is not None: 

192 if args.extend_run: 

193 runName = self.output.chain[0] 

194 else: 

195 runName = "{}/{}".format(self.output, Instrument.makeCollectionTimestamp()) 

196 self.outputRun = _OutputRunCollectionInfo(registry, runName) 

197 elif not writeable: 

198 # If we're not writing yet, ok to have no output run. 

199 self.outputRun = None 

200 else: 

201 raise ValueError("Cannot write without at least one of (--output, --output-run).") 

202 # Recursively flatten any input CHAINED collections. We do this up 

203 # front so we can tell if the user passes the same inputs on subsequent 

204 # calls, even though we also flatten when we define the output CHAINED 

205 # collection. 

206 self.inputs = tuple(registry.queryCollections(args.input, flattenChains=True)) if args.input else () 

207 

208 def check(self, args: argparse.Namespace): 

209 """Check command-line options for consistency with each other and the 

210 data repository. 

211 

212 Parameters 

213 ---------- 

214 args : `argparse.Namespace` 

215 Parsed command-line arguments. See class documentation for the 

216 construction parameter of the same name. 

217 """ 

218 assert not (args.extend_run and args.replace_run), "In mutually-exclusive group in ArgumentParser." 

219 if self.inputs and self.output is not None and self.output.exists: 

220 # Passing the same inputs that were used to initialize the output 

221 # collection is allowed; this means they must _end_ with the same 

222 # collections, because we push new runs to the front of the chain. 

223 for c1, c2 in zip(self.inputs[::-1], self.output.chain[::-1]): 

224 if c1 != c2: 

225 raise ValueError( 

226 f"Output CHAINED collection {self.output.name!r} exists, but it ends with " 

227 "a different sequence of input collections than those given: " 

228 f"{c1!r} != {c2!r} in inputs={self.inputs} vs " 

229 f"{self.output.name}={self.output.chain}." 

230 ) 

231 if len(self.inputs) > len(self.output.chain): 

232 nNew = len(self.inputs) - len(self.output.chain) 

233 raise ValueError( 

234 f"Cannot add new input collections {self.inputs[:nNew]} after " 

235 "output collection is first created." 

236 ) 

237 if args.extend_run and self.outputRun is None: 

238 raise ValueError("Cannot --extend-run when no output collection is given.") 

239 if args.extend_run and not self.outputRun.exists: 

240 raise ValueError(f"Cannot --extend-run; output collection " 

241 f"'{self.outputRun.name}' does not exist.") 

242 if not args.extend_run and self.outputRun is not None and self.outputRun.exists: 

243 raise ValueError(f"Output run '{self.outputRun.name}' already exists, but " 

244 f"--extend-run was not given.") 

245 if args.prune_replaced and not args.replace_run: 

246 raise ValueError("--prune-replaced requires --replace-run.") 

247 if args.replace_run and (self.output is None or not self.output.exists): 

248 raise ValueError("--output must point to an existing CHAINED collection for --replace-run.") 

249 

250 @classmethod 

251 def _makeReadParts(cls, args: argparse.Namespace): 

252 """Common implementation for `makeReadButler` and 

253 `makeRegistryAndCollections`. 

254 

255 Parameters 

256 ---------- 

257 args : `argparse.Namespace` 

258 Parsed command-line arguments. See class documentation for the 

259 construction parameter of the same name. 

260 

261 Returns 

262 ------- 

263 butler : `lsst.daf.butler.Butler` 

264 A read-only butler constructed from the repo at 

265 ``args.butler_config``, but with no default collections. 

266 inputs : `lsst.daf.butler.registry.CollectionSearch` 

267 A collection search path constructed according to ``args``. 

268 self : `_ButlerFactory` 

269 A new `_ButlerFactory` instance representing the processed version 

270 of ``args``. 

271 """ 

272 butler = Butler(args.butler_config, writeable=False) 

273 self = cls(butler.registry, args, writeable=False) 

274 self.check(args) 

275 if self.output and self.output.exists: 

276 if args.replace_run: 

277 replaced = self.output.chain[0] 

278 inputs = self.output.chain[1:] 

279 _LOG.debug("Simulating collection search in '%s' after removing '%s'.", 

280 self.output.name, replaced) 

281 else: 

282 inputs = [self.output.name] 

283 else: 

284 inputs = list(self.inputs) 

285 if args.extend_run: 

286 inputs.insert(0, self.outputRun.name) 

287 inputs = CollectionSearch.fromExpression(inputs) 

288 return butler, inputs, self 

289 

290 @classmethod 

291 def makeReadButler(cls, args: argparse.Namespace) -> Butler: 

292 """Construct a read-only butler according to the given command-line 

293 arguments. 

294 

295 Parameters 

296 ---------- 

297 args : `argparse.Namespace` 

298 Parsed command-line arguments. See class documentation for the 

299 construction parameter of the same name. 

300 

301 Returns 

302 ------- 

303 butler : `lsst.daf.butler.Butler` 

304 A read-only butler initialized with the collections specified by 

305 ``args``. 

306 """ 

307 butler, inputs, _ = cls._makeReadParts(args) 

308 _LOG.debug("Preparing butler to read from %s.", inputs) 

309 return Butler(butler=butler, collections=inputs) 

310 

311 @classmethod 

312 def makeRegistryAndCollections(cls, args: argparse.Namespace) -> \ 

313 Tuple[Registry, CollectionSearch, Optional[str]]: 

314 """Return a read-only registry, a collection search path, and the name 

315 of the run to be used for future writes. 

316 

317 Parameters 

318 ---------- 

319 args : `argparse.Namespace` 

320 Parsed command-line arguments. See class documentation for the 

321 construction parameter of the same name. 

322 

323 Returns 

324 ------- 

325 registry : `lsst.daf.butler.Registry` 

326 Butler registry that collections will be added to and/or queried 

327 from. 

328 inputs : `lsst.daf.butler.registry.CollectionSearch` 

329 Collections to search for datasets. 

330 run : `str` or `None` 

331 Name of the output `~lsst.daf.butler.CollectionType.RUN` collection 

332 if it already exists, or `None` if it does not. 

333 """ 

334 butler, inputs, self = cls._makeReadParts(args) 

335 run = self.outputRun.name if args.extend_run else None 

336 _LOG.debug("Preparing registry to read from %s and expect future writes to '%s'.", inputs, run) 

337 return butler.registry, inputs, run 

338 

339 @classmethod 

340 def makeWriteButler(cls, args: argparse.Namespace) -> Butler: 

341 """Return a read-write butler initialized to write to and read from 

342 the collections specified by the given command-line arguments. 

343 

344 Parameters 

345 ---------- 

346 args : `argparse.Namespace` 

347 Parsed command-line arguments. See class documentation for the 

348 construction parameter of the same name. 

349 

350 Returns 

351 ------- 

352 butler : `lsst.daf.butler.Butler` 

353 A read-write butler initialized according to the given arguments. 

354 """ 

355 butler = Butler(args.butler_config, writeable=True) 

356 self = cls(butler.registry, args, writeable=True) 

357 self.check(args) 

358 if self.output is not None: 

359 chainDefinition = list(self.output.chain if self.output.exists else self.inputs) 

360 if args.replace_run: 

361 replaced = chainDefinition.pop(0) 

362 if args.prune_replaced == "unstore": 

363 # Remove datasets from datastore 

364 with butler.transaction(): 

365 refs = butler.registry.queryDatasets(..., collections=replaced) 

366 butler.pruneDatasets(refs, unstore=True, run=replaced, disassociate=False) 

367 elif args.prune_replaced == "purge": 

368 # Erase entire collection and all datasets, need to remove 

369 # collection from its chain collection first. 

370 with butler.transaction(): 

371 butler.registry.setCollectionChain(self.output.name, chainDefinition, flatten=True) 

372 butler.pruneCollection(replaced, purge=True, unstore=True) 

373 elif args.prune_replaced is not None: 

374 raise NotImplementedError( 

375 f"Unsupported --prune-replaced option '{args.prune_replaced}'." 

376 ) 

377 if not self.output.exists: 

378 butler.registry.registerCollection(self.output.name, CollectionType.CHAINED) 

379 if not args.extend_run: 

380 butler.registry.registerCollection(self.outputRun.name, CollectionType.RUN) 

381 chainDefinition.insert(0, self.outputRun.name) 

382 butler.registry.setCollectionChain(self.output.name, chainDefinition, flatten=True) 

383 _LOG.debug("Preparing butler to write to '%s' and read from '%s'=%s", 

384 self.outputRun.name, self.output.name, chainDefinition) 

385 butler.registry.defaults = RegistryDefaults(run=self.outputRun.name, collections=self.output.name) 

386 else: 

387 inputs = CollectionSearch.fromExpression((self.outputRun.name,) + self.inputs) 

388 _LOG.debug("Preparing butler to write to '%s' and read from %s.", self.outputRun.name, inputs) 

389 butler.registry.defaults = RegistryDefaults(run=self.outputRun.name, collections=inputs) 

390 return butler 

391 

392 output: Optional[_OutputChainedCollectionInfo] 

393 """Information about the output chained collection, if there is or will be 

394 one (`_OutputChainedCollectionInfo` or `None`). 

395 """ 

396 

397 outputRun: Optional[_OutputRunCollectionInfo] 

398 """Information about the output run collection, if there is or will be 

399 one (`_OutputRunCollectionInfo` or `None`). 

400 """ 

401 

402 inputs: Tuple[str, ...] 

403 """Input collections provided directly by the user (`tuple` [ `str` ]). 

404 """ 

405 

406 

407class _FilteredStream: 

408 """A file-like object that filters some config fields. 

409 

410 Note 

411 ---- 

412 This class depends on implementation details of ``Config.saveToStream`` 

413 methods, in particular that that method uses single call to write() 

414 method to save information about single config field, and that call 

415 combines comments string(s) for a field and field path and value. 

416 This class will not work reliably on the "import" strings, so imports 

417 should be disabled by passing ``skipImports=True`` to ``saveToStream()``. 

418 """ 

419 def __init__(self, pattern): 

420 # obey case if pattern isn't lowercase or requests NOIGNORECASE 

421 mat = re.search(r"(.*):NOIGNORECASE$", pattern) 

422 

423 if mat: 

424 pattern = mat.group(1) 

425 self._pattern = re.compile(fnmatch.translate(pattern)) 

426 else: 

427 if pattern != pattern.lower(): 

428 print(f"Matching \"{pattern}\" without regard to case " 

429 "(append :NOIGNORECASE to prevent this)", file=sys.stdout) 

430 self._pattern = re.compile(fnmatch.translate(pattern), re.IGNORECASE) 

431 

432 def write(self, showStr): 

433 # Strip off doc string line(s) and cut off at "=" for string matching 

434 matchStr = showStr.rstrip().split("\n")[-1].split("=")[0] 

435 if self._pattern.search(matchStr): 

436 sys.stdout.write(showStr) 

437 

438# ------------------------ 

439# Exported definitions -- 

440# ------------------------ 

441 

442 

443class CmdLineFwk: 

444 """PipelineTask framework which executes tasks from command line. 

445 

446 In addition to executing tasks this activator provides additional methods 

447 for task management like dumping configuration or execution chain. 

448 """ 

449 

450 MP_TIMEOUT = 9999 # Default timeout (sec) for multiprocessing 

451 

452 def __init__(self): 

453 pass 

454 

455 def makePipeline(self, args): 

456 """Build a pipeline from command line arguments. 

457 

458 Parameters 

459 ---------- 

460 args : `argparse.Namespace` 

461 Parsed command line 

462 

463 Returns 

464 ------- 

465 pipeline : `~lsst.pipe.base.Pipeline` 

466 """ 

467 if args.pipeline: 

468 pipeline = Pipeline.from_uri(args.pipeline) 

469 else: 

470 pipeline = Pipeline("anonymous") 

471 

472 # loop over all pipeline actions and apply them in order 

473 for action in args.pipeline_actions: 

474 if action.action == "add_instrument": 

475 

476 pipeline.addInstrument(action.value) 

477 

478 elif action.action == "new_task": 

479 

480 pipeline.addTask(action.value, action.label) 

481 

482 elif action.action == "delete_task": 

483 

484 pipeline.removeTask(action.label) 

485 

486 elif action.action == "config": 

487 

488 # action value string is "field=value", split it at '=' 

489 field, _, value = action.value.partition("=") 

490 pipeline.addConfigOverride(action.label, field, value) 

491 

492 elif action.action == "configfile": 

493 

494 pipeline.addConfigFile(action.label, action.value) 

495 

496 else: 

497 

498 raise ValueError(f"Unexpected pipeline action: {action.action}") 

499 

500 if args.save_pipeline: 

501 pipeline.write_to_uri(args.save_pipeline) 

502 

503 if args.pipeline_dot: 

504 pipeline2dot(pipeline, args.pipeline_dot) 

505 

506 return pipeline 

507 

508 def makeGraph(self, pipeline, args): 

509 """Build a graph from command line arguments. 

510 

511 Parameters 

512 ---------- 

513 pipeline : `~lsst.pipe.base.Pipeline` 

514 Pipeline, can be empty or ``None`` if graph is read from a file. 

515 args : `argparse.Namespace` 

516 Parsed command line 

517 

518 Returns 

519 ------- 

520 graph : `~lsst.pipe.base.QuantumGraph` or `None` 

521 If resulting graph is empty then `None` is returned. 

522 """ 

523 

524 registry, collections, run = _ButlerFactory.makeRegistryAndCollections(args) 

525 

526 if args.qgraph: 

527 # click passes empty tuple as default value for qgraph_node_id 

528 nodes = args.qgraph_node_id or None 

529 qgraph = QuantumGraph.loadUri(args.qgraph, registry.dimensions, 

530 nodes=nodes, graphID=args.qgraph_id) 

531 

532 # pipeline can not be provided in this case 

533 if pipeline: 

534 raise ValueError("Pipeline must not be given when quantum graph is read from file.") 

535 

536 else: 

537 

538 # make execution plan (a.k.a. DAG) for pipeline 

539 graphBuilder = GraphBuilder(registry, 

540 skipExisting=args.skip_existing) 

541 qgraph = graphBuilder.makeGraph(pipeline, collections, run, args.data_query) 

542 

543 # count quanta in graph and give a warning if it's empty and return None 

544 nQuanta = len(qgraph) 

545 if nQuanta == 0: 

546 warnings.warn("QuantumGraph is empty", stacklevel=2) 

547 return None 

548 else: 

549 _LOG.info("QuantumGraph contains %d quanta for %d tasks, graph ID: %r", 

550 nQuanta, len(qgraph.taskGraph), qgraph.graphID) 

551 

552 if args.save_qgraph: 

553 qgraph.saveUri(args.save_qgraph) 

554 

555 if args.save_single_quanta: 

556 for quantumNode in qgraph: 

557 sqgraph = qgraph.subset(quantumNode) 

558 uri = args.save_single_quanta.format(quantumNode.nodeId.number) 

559 sqgraph.saveUri(uri) 

560 

561 if args.qgraph_dot: 

562 graph2dot(qgraph, args.qgraph_dot) 

563 

564 return qgraph 

565 

566 def runPipeline(self, graph, taskFactory, args, butler=None): 

567 """Execute complete QuantumGraph. 

568 

569 Parameters 

570 ---------- 

571 graph : `QuantumGraph` 

572 Execution graph. 

573 taskFactory : `~lsst.pipe.base.TaskFactory` 

574 Task factory 

575 args : `argparse.Namespace` 

576 Parsed command line 

577 butler : `~lsst.daf.butler.Butler`, optional 

578 Data Butler instance, if not defined then new instance is made 

579 using command line options. 

580 """ 

581 # make butler instance 

582 if butler is None: 

583 butler = _ButlerFactory.makeWriteButler(args) 

584 

585 # Enable lsstDebug debugging. Note that this is done once in the 

586 # main process before PreExecInit and it is also repeated before 

587 # running each task in SingleQuantumExecutor (which may not be 

588 # needed if `multipocessing` always uses fork start method). 

589 if args.enableLsstDebug: 

590 try: 

591 _LOG.debug("Will try to import debug.py") 

592 import debug # noqa:F401 

593 except ImportError: 

594 _LOG.warn("No 'debug' module found.") 

595 

596 # --skip-existing should have no effect unless --extend-run is passed 

597 # so we make PreExecInit's skipExisting depend on the latter as well. 

598 preExecInit = PreExecInit(butler, taskFactory, skipExisting=(args.skip_existing and args.extend_run)) 

599 preExecInit.initialize(graph, 

600 saveInitOutputs=not args.skip_init_writes, 

601 registerDatasetTypes=args.register_dataset_types, 

602 saveVersions=not args.no_versions) 

603 

604 if not args.init_only: 

605 graphFixup = self._importGraphFixup(args) 

606 quantumExecutor = SingleQuantumExecutor(taskFactory, 

607 skipExisting=args.skip_existing, 

608 clobberPartialOutputs=args.clobber_partial_outputs, 

609 enableLsstDebug=args.enableLsstDebug) 

610 timeout = self.MP_TIMEOUT if args.timeout is None else args.timeout 

611 executor = MPGraphExecutor(numProc=args.processes, timeout=timeout, 

612 startMethod=args.start_method, 

613 quantumExecutor=quantumExecutor, 

614 failFast=args.fail_fast, 

615 executionGraphFixup=graphFixup) 

616 with util.profile(args.profile, _LOG): 

617 executor.execute(graph, butler) 

618 

619 def showInfo(self, args, pipeline, graph=None): 

620 """Display useful info about pipeline and environment. 

621 

622 Parameters 

623 ---------- 

624 args : `argparse.Namespace` 

625 Parsed command line 

626 pipeline : `Pipeline` 

627 Pipeline definition 

628 graph : `QuantumGraph`, optional 

629 Execution graph 

630 """ 

631 showOpts = args.show 

632 for what in showOpts: 

633 showCommand, _, showArgs = what.partition("=") 

634 

635 if showCommand in ["pipeline", "config", "history", "tasks"]: 

636 if not pipeline: 

637 _LOG.warning("Pipeline is required for --show=%s", showCommand) 

638 continue 

639 

640 if showCommand in ["graph", "workflow", "uri"]: 

641 if not graph: 

642 _LOG.warning("QuantumGraph is required for --show=%s", showCommand) 

643 continue 

644 

645 if showCommand == "pipeline": 

646 print(pipeline) 

647 elif showCommand == "config": 

648 self._showConfig(pipeline, showArgs, False) 

649 elif showCommand == "dump-config": 

650 self._showConfig(pipeline, showArgs, True) 

651 elif showCommand == "history": 

652 self._showConfigHistory(pipeline, showArgs) 

653 elif showCommand == "tasks": 

654 self._showTaskHierarchy(pipeline) 

655 elif showCommand == "graph": 

656 if graph: 

657 self._showGraph(graph) 

658 elif showCommand == "uri": 

659 if graph: 

660 self._showUri(graph, args) 

661 elif showCommand == "workflow": 

662 if graph: 

663 self._showWorkflow(graph, args) 

664 else: 

665 print("Unknown value for show: %s (choose from '%s')" % 

666 (what, "', '".join("pipeline config[=XXX] history=XXX tasks graph".split())), 

667 file=sys.stderr) 

668 sys.exit(1) 

669 

670 def _showConfig(self, pipeline, showArgs, dumpFullConfig): 

671 """Show task configuration 

672 

673 Parameters 

674 ---------- 

675 pipeline : `Pipeline` 

676 Pipeline definition 

677 showArgs : `str` 

678 Defines what to show 

679 dumpFullConfig : `bool` 

680 If true then dump complete task configuration with all imports. 

681 """ 

682 stream = sys.stdout 

683 if dumpFullConfig: 

684 # Task label can be given with this option 

685 taskName = showArgs 

686 else: 

687 # The argument can have form [TaskLabel::][pattern:NOIGNORECASE] 

688 matConfig = re.search(r"^(?:(\w+)::)?(?:config.)?(.+)?", showArgs) 

689 taskName = matConfig.group(1) 

690 pattern = matConfig.group(2) 

691 if pattern: 

692 stream = _FilteredStream(pattern) 

693 

694 tasks = util.filterTasks(pipeline, taskName) 

695 if not tasks: 

696 print("Pipeline has no tasks named {}".format(taskName), file=sys.stderr) 

697 sys.exit(1) 

698 

699 for taskDef in tasks: 

700 print("### Configuration for task `{}'".format(taskDef.label)) 

701 taskDef.config.saveToStream(stream, root="config", skipImports=not dumpFullConfig) 

702 

703 def _showConfigHistory(self, pipeline, showArgs): 

704 """Show history for task configuration 

705 

706 Parameters 

707 ---------- 

708 pipeline : `Pipeline` 

709 Pipeline definition 

710 showArgs : `str` 

711 Defines what to show 

712 """ 

713 

714 taskName = None 

715 pattern = None 

716 matHistory = re.search(r"^(?:(\w+)::)?(?:config[.])?(.+)", showArgs) 

717 if matHistory: 

718 taskName = matHistory.group(1) 

719 pattern = matHistory.group(2) 

720 if not pattern: 

721 print("Please provide a value with --show history (e.g. history=Task::param)", file=sys.stderr) 

722 sys.exit(1) 

723 

724 tasks = util.filterTasks(pipeline, taskName) 

725 if not tasks: 

726 print(f"Pipeline has no tasks named {taskName}", file=sys.stderr) 

727 sys.exit(1) 

728 

729 found = False 

730 for taskDef in tasks: 

731 

732 config = taskDef.config 

733 

734 # Look for any matches in the config hierarchy for this name 

735 for nmatch, thisName in enumerate(fnmatch.filter(config.names(), pattern)): 

736 if nmatch > 0: 

737 print("") 

738 

739 cpath, _, cname = thisName.rpartition(".") 

740 try: 

741 if not cpath: 

742 # looking for top-level field 

743 hconfig = taskDef.config 

744 else: 

745 hconfig = eval("config." + cpath, {}, {"config": config}) 

746 except AttributeError: 

747 print(f"Error: Unable to extract attribute {cpath} from task {taskDef.label}", 

748 file=sys.stderr) 

749 hconfig = None 

750 

751 # Sometimes we end up with a non-Config so skip those 

752 if isinstance(hconfig, (pexConfig.Config, pexConfig.ConfigurableInstance)) and \ 

753 hasattr(hconfig, cname): 

754 print(f"### Configuration field for task `{taskDef.label}'") 

755 print(pexConfig.history.format(hconfig, cname)) 

756 found = True 

757 

758 if not found: 

759 print(f"None of the tasks has field matching {pattern}", file=sys.stderr) 

760 sys.exit(1) 

761 

762 def _showTaskHierarchy(self, pipeline): 

763 """Print task hierarchy to stdout 

764 

765 Parameters 

766 ---------- 

767 pipeline: `Pipeline` 

768 """ 

769 for taskDef in pipeline.toExpandedPipeline(): 

770 print("### Subtasks for task `{}'".format(taskDef.taskName)) 

771 

772 for configName, taskName in util.subTaskIter(taskDef.config): 

773 print("{}: {}".format(configName, taskName)) 

774 

775 def _showGraph(self, graph): 

776 """Print quanta information to stdout 

777 

778 Parameters 

779 ---------- 

780 graph : `QuantumGraph` 

781 Execution graph. 

782 """ 

783 for taskNode in graph.taskGraph: 

784 print(taskNode) 

785 

786 for iq, quantum in enumerate(graph.getQuantaForTask(taskNode)): 

787 print(" Quantum {}:".format(iq)) 

788 print(" inputs:") 

789 for key, refs in quantum.inputs.items(): 

790 dataIds = ["DataId({})".format(ref.dataId) for ref in refs] 

791 print(" {}: [{}]".format(key, ", ".join(dataIds))) 

792 print(" outputs:") 

793 for key, refs in quantum.outputs.items(): 

794 dataIds = ["DataId({})".format(ref.dataId) for ref in refs] 

795 print(" {}: [{}]".format(key, ", ".join(dataIds))) 

796 

797 def _showWorkflow(self, graph, args): 

798 """Print quanta information and dependency to stdout 

799 

800 Parameters 

801 ---------- 

802 graph : `QuantumGraph` 

803 Execution graph. 

804 args : `argparse.Namespace` 

805 Parsed command line 

806 """ 

807 for node in graph: 

808 print(f"Quantum {node.nodeId.number}: {node.taskDef.taskName}") 

809 for parent in graph.determineInputsToQuantumNode(node): 

810 print(f"Parent Quantum {parent.nodeId.number} - Child Quantum {node.nodeId.number}") 

811 

812 def _showUri(self, graph, args): 

813 """Print input and predicted output URIs to stdout 

814 

815 Parameters 

816 ---------- 

817 graph : `QuantumGraph` 

818 Execution graph 

819 args : `argparse.Namespace` 

820 Parsed command line 

821 """ 

822 def dumpURIs(thisRef): 

823 primary, components = butler.getURIs(thisRef, predict=True, run="TBD") 

824 if primary: 

825 print(f" {primary}") 

826 else: 

827 print(" (disassembled artifact)") 

828 for compName, compUri in components.items(): 

829 print(f" {compName}: {compUri}") 

830 

831 butler = _ButlerFactory.makeReadButler(args) 

832 for node in graph: 

833 print(f"Quantum {node.nodeId.number}: {node.taskDef.taskName}") 

834 print(" inputs:") 

835 for key, refs in node.quantum.inputs.items(): 

836 for ref in refs: 

837 dumpURIs(ref) 

838 print(" outputs:") 

839 for key, refs in node.quantum.outputs.items(): 

840 for ref in refs: 

841 dumpURIs(ref) 

842 

843 def _importGraphFixup(self, args): 

844 """Import/instantiate graph fixup object. 

845 

846 Parameters 

847 ---------- 

848 args : `argparse.Namespace` 

849 Parsed command line. 

850 

851 Returns 

852 ------- 

853 fixup : `ExecutionGraphFixup` or `None` 

854 

855 Raises 

856 ------ 

857 ValueError 

858 Raised if import fails, method call raises exception, or returned 

859 instance has unexpected type. 

860 """ 

861 if args.graph_fixup: 

862 try: 

863 factory = doImport(args.graph_fixup) 

864 except Exception as exc: 

865 raise ValueError("Failed to import graph fixup class/method") from exc 

866 try: 

867 fixup = factory() 

868 except Exception as exc: 

869 raise ValueError("Failed to make instance of graph fixup") from exc 

870 if not isinstance(fixup, ExecutionGraphFixup): 

871 raise ValueError("Graph fixup is not an instance of ExecutionGraphFixup class") 

872 return fixup