Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of ctrl_mpexec. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22"""Module defining CmdLineFwk class and related methods. 

23""" 

24 

25__all__ = ['CmdLineFwk'] 

26 

27# ------------------------------- 

28# Imports of standard modules -- 

29# ------------------------------- 

30import argparse 

31import fnmatch 

32import logging 

33import re 

34import sys 

35from typing import Optional, Tuple 

36import warnings 

37 

38# ----------------------------- 

39# Imports for other modules -- 

40# ----------------------------- 

41from lsst.daf.butler import ( 

42 Butler, 

43 CollectionSearch, 

44 CollectionType, 

45 Registry, 

46) 

47from lsst.daf.butler.registry import MissingCollectionError, RegistryDefaults 

48import lsst.pex.config as pexConfig 

49from lsst.pipe.base import GraphBuilder, Pipeline, QuantumGraph 

50from lsst.obs.base import Instrument 

51from .dotTools import graph2dot, pipeline2dot 

52from .executionGraphFixup import ExecutionGraphFixup 

53from .mpGraphExecutor import MPGraphExecutor 

54from .preExecInit import PreExecInit 

55from .singleQuantumExecutor import SingleQuantumExecutor 

56from . import util 

57from lsst.utils import doImport 

58 

59# ---------------------------------- 

60# Local non-exported definitions -- 

61# ---------------------------------- 

62 

63_LOG = logging.getLogger(__name__.partition(".")[2]) 

64 

65 

66class _OutputChainedCollectionInfo: 

67 """A helper class for handling command-line arguments related to an output 

68 `~lsst.daf.butler.CollectionType.CHAINED` collection. 

69 

70 Parameters 

71 ---------- 

72 registry : `lsst.daf.butler.Registry` 

73 Butler registry that collections will be added to and/or queried from. 

74 name : `str` 

75 Name of the collection given on the command line. 

76 """ 

77 def __init__(self, registry: Registry, name: str): 

78 self.name = name 

79 try: 

80 self.chain = tuple(registry.getCollectionChain(name)) 

81 self.exists = True 

82 except MissingCollectionError: 

83 self.chain = () 

84 self.exists = False 

85 

86 def __str__(self): 

87 return self.name 

88 

89 name: str 

90 """Name of the collection provided on the command line (`str`). 

91 """ 

92 

93 exists: bool 

94 """Whether this collection already exists in the registry (`bool`). 

95 """ 

96 

97 chain: Tuple[str, ...] 

98 """The definition of the collection, if it already exists (`tuple` [`str`]). 

99 

100 Empty if the collection does not already exist. 

101 """ 

102 

103 

104class _OutputRunCollectionInfo: 

105 """A helper class for handling command-line arguments related to an output 

106 `~lsst.daf.butler.CollectionType.RUN` collection. 

107 

108 Parameters 

109 ---------- 

110 registry : `lsst.daf.butler.Registry` 

111 Butler registry that collections will be added to and/or queried from. 

112 name : `str` 

113 Name of the collection given on the command line. 

114 """ 

115 def __init__(self, registry: Registry, name: str): 

116 self.name = name 

117 try: 

118 actualType = registry.getCollectionType(name) 

119 if actualType is not CollectionType.RUN: 

120 raise TypeError(f"Collection '{name}' exists but has type {actualType.name}, not RUN.") 

121 self.exists = True 

122 except MissingCollectionError: 

123 self.exists = False 

124 

125 name: str 

126 """Name of the collection provided on the command line (`str`). 

127 """ 

128 

129 exists: bool 

130 """Whether this collection already exists in the registry (`bool`). 

131 """ 

132 

133 

134class _ButlerFactory: 

135 """A helper class for processing command-line arguments related to input 

136 and output collections. 

137 

138 Parameters 

139 ---------- 

140 registry : `lsst.daf.butler.Registry` 

141 Butler registry that collections will be added to and/or queried from. 

142 

143 args : `argparse.Namespace` 

144 Parsed command-line arguments. The following attributes are used, 

145 either at construction or in later methods. 

146 

147 ``output`` 

148 The name of a `~lsst.daf.butler.CollectionType.CHAINED` 

149 input/output collection. 

150 

151 ``output_run`` 

152 The name of a `~lsst.daf.butler.CollectionType.RUN` input/output 

153 collection. 

154 

155 ``extend_run`` 

156 A boolean indicating whether ``output_run`` should already exist 

157 and be extended. 

158 

159 ``replace_run`` 

160 A boolean indicating that (if `True`) ``output_run`` should already 

161 exist but will be removed from the output chained collection and 

162 replaced with a new one. 

163 

164 ``prune_replaced`` 

165 A boolean indicating whether to prune the replaced run (requires 

166 ``replace_run``). 

167 

168 ``inputs`` 

169 Input collections of any type; may be any type handled by 

170 `lsst.daf.butler.registry.CollectionSearch.fromExpression`. 

171 

172 ``butler_config`` 

173 Path to a data repository root or configuration file. 

174 

175 writeable : `bool` 

176 If `True`, a `Butler` is being initialized in a context where actual 

177 writes should happens, and hence no output run is necessary. 

178 

179 Raises 

180 ------ 

181 ValueError 

182 Raised if ``writeable is True`` but there are no output collections. 

183 """ 

184 def __init__(self, registry: Registry, args: argparse.Namespace, writeable: bool): 

185 if args.output is not None: 

186 self.output = _OutputChainedCollectionInfo(registry, args.output) 

187 else: 

188 self.output = None 

189 if args.output_run is not None: 

190 self.outputRun = _OutputRunCollectionInfo(registry, args.output_run) 

191 elif self.output is not None: 

192 if args.extend_run: 

193 runName = self.output.chain[0] 

194 else: 

195 runName = "{}/{}".format(self.output, Instrument.makeCollectionTimestamp()) 

196 self.outputRun = _OutputRunCollectionInfo(registry, runName) 

197 elif not writeable: 

198 # If we're not writing yet, ok to have no output run. 

199 self.outputRun = None 

200 else: 

201 raise ValueError("Cannot write without at least one of (--output, --output-run).") 

202 self.inputs = tuple(CollectionSearch.fromExpression(args.input)) if args.input else () 

203 

204 def check(self, args: argparse.Namespace): 

205 """Check command-line options for consistency with each other and the 

206 data repository. 

207 

208 Parameters 

209 ---------- 

210 args : `argparse.Namespace` 

211 Parsed command-line arguments. See class documentation for the 

212 construction parameter of the same name. 

213 """ 

214 assert not (args.extend_run and args.replace_run), "In mutually-exclusive group in ArgumentParser." 

215 if self.inputs and self.output is not None and self.output.exists: 

216 # Passing the same inputs that were used to initialize the output 

217 # collection is allowed; this means they must _end_ with the same 

218 # collections, because we push new runs to the front of the chain. 

219 for c1, c2 in zip(self.inputs[::-1], self.output.chain[::-1]): 

220 if c1 != c2: 

221 raise ValueError( 

222 f"Output CHAINED collection {self.output.name!r} exists, but it ends with " 

223 "a different sequence of input collections than those given: " 

224 f"{c1!r} != {c2!r} in inputs={self.inputs} vs " 

225 f"{self.output.name}={self.output.chain}." 

226 ) 

227 if len(self.inputs) > len(self.output.chain): 

228 nNew = len(self.inputs) - len(self.output.chain) 

229 raise ValueError( 

230 f"Cannot add new input collections {self.inputs[:nNew]} after " 

231 "output collection is first created." 

232 ) 

233 if args.extend_run and self.outputRun is None: 

234 raise ValueError("Cannot --extend-run when no output collection is given.") 

235 if args.extend_run and not self.outputRun.exists: 

236 raise ValueError(f"Cannot --extend-run; output collection " 

237 f"'{self.outputRun.name}' does not exist.") 

238 if not args.extend_run and self.outputRun is not None and self.outputRun.exists: 

239 raise ValueError(f"Output run '{self.outputRun.name}' already exists, but " 

240 f"--extend-run was not given.") 

241 if args.prune_replaced and not args.replace_run: 

242 raise ValueError("--prune-replaced requires --replace-run.") 

243 if args.replace_run and (self.output is None or not self.output.exists): 

244 raise ValueError("--output must point to an existing CHAINED collection for --replace-run.") 

245 

246 @classmethod 

247 def _makeReadParts(cls, args: argparse.Namespace): 

248 """Common implementation for `makeReadButler` and 

249 `makeRegistryAndCollections`. 

250 

251 Parameters 

252 ---------- 

253 args : `argparse.Namespace` 

254 Parsed command-line arguments. See class documentation for the 

255 construction parameter of the same name. 

256 

257 Returns 

258 ------- 

259 butler : `lsst.daf.butler.Butler` 

260 A read-only butler constructed from the repo at 

261 ``args.butler_config``, but with no default collections. 

262 inputs : `lsst.daf.butler.registry.CollectionSearch` 

263 A collection search path constructed according to ``args``. 

264 self : `_ButlerFactory` 

265 A new `_ButlerFactory` instance representing the processed version 

266 of ``args``. 

267 """ 

268 butler = Butler(args.butler_config, writeable=False) 

269 self = cls(butler.registry, args, writeable=False) 

270 self.check(args) 

271 if self.output and self.output.exists: 

272 if args.replace_run: 

273 replaced = self.output.chain[0] 

274 inputs = self.output.chain[1:] 

275 _LOG.debug("Simulating collection search in '%s' after removing '%s'.", 

276 self.output.name, replaced) 

277 else: 

278 inputs = [self.output.name] 

279 else: 

280 inputs = list(self.inputs) 

281 if args.extend_run: 

282 inputs.insert(0, self.outputRun.name) 

283 inputs = CollectionSearch.fromExpression(inputs) 

284 return butler, inputs, self 

285 

286 @classmethod 

287 def makeReadButler(cls, args: argparse.Namespace) -> Butler: 

288 """Construct a read-only butler according to the given command-line 

289 arguments. 

290 

291 Parameters 

292 ---------- 

293 args : `argparse.Namespace` 

294 Parsed command-line arguments. See class documentation for the 

295 construction parameter of the same name. 

296 

297 Returns 

298 ------- 

299 butler : `lsst.daf.butler.Butler` 

300 A read-only butler initialized with the collections specified by 

301 ``args``. 

302 """ 

303 butler, inputs, _ = cls._makeReadParts(args) 

304 _LOG.debug("Preparing butler to read from %s.", inputs) 

305 return Butler(butler=butler, collections=inputs) 

306 

307 @classmethod 

308 def makeRegistryAndCollections(cls, args: argparse.Namespace) -> \ 

309 Tuple[Registry, CollectionSearch, Optional[str]]: 

310 """Return a read-only registry, a collection search path, and the name 

311 of the run to be used for future writes. 

312 

313 Parameters 

314 ---------- 

315 args : `argparse.Namespace` 

316 Parsed command-line arguments. See class documentation for the 

317 construction parameter of the same name. 

318 

319 Returns 

320 ------- 

321 registry : `lsst.daf.butler.Registry` 

322 Butler registry that collections will be added to and/or queried 

323 from. 

324 inputs : `lsst.daf.butler.registry.CollectionSearch` 

325 Collections to search for datasets. 

326 run : `str` or `None` 

327 Name of the output `~lsst.daf.butler.CollectionType.RUN` collection 

328 if it already exists, or `None` if it does not. 

329 """ 

330 butler, inputs, self = cls._makeReadParts(args) 

331 run = self.outputRun.name if args.extend_run else None 

332 _LOG.debug("Preparing registry to read from %s and expect future writes to '%s'.", inputs, run) 

333 return butler.registry, inputs, run 

334 

335 @classmethod 

336 def makeWriteButler(cls, args: argparse.Namespace) -> Butler: 

337 """Return a read-write butler initialized to write to and read from 

338 the collections specified by the given command-line arguments. 

339 

340 Parameters 

341 ---------- 

342 args : `argparse.Namespace` 

343 Parsed command-line arguments. See class documentation for the 

344 construction parameter of the same name. 

345 

346 Returns 

347 ------- 

348 butler : `lsst.daf.butler.Butler` 

349 A read-write butler initialized according to the given arguments. 

350 """ 

351 butler = Butler(args.butler_config, writeable=True) 

352 self = cls(butler.registry, args, writeable=True) 

353 self.check(args) 

354 if self.output is not None: 

355 chainDefinition = list(self.output.chain if self.output.exists else self.inputs) 

356 if args.replace_run: 

357 replaced = chainDefinition.pop(0) 

358 if args.prune_replaced == "unstore": 

359 # Remove datasets from datastore 

360 with butler.transaction(): 

361 refs = butler.registry.queryDatasets(..., collections=replaced) 

362 butler.pruneDatasets(refs, unstore=True, run=replaced, disassociate=False) 

363 elif args.prune_replaced == "purge": 

364 # Erase entire collection and all datasets, need to remove 

365 # collection from its chain collection first. 

366 with butler.transaction(): 

367 butler.registry.setCollectionChain(self.output.name, chainDefinition) 

368 butler.pruneCollection(replaced, purge=True, unstore=True) 

369 elif args.prune_replaced is not None: 

370 raise NotImplementedError( 

371 f"Unsupported --prune-replaced option '{args.prune_replaced}'." 

372 ) 

373 if not self.output.exists: 

374 butler.registry.registerCollection(self.output.name, CollectionType.CHAINED) 

375 if not args.extend_run: 

376 butler.registry.registerCollection(self.outputRun.name, CollectionType.RUN) 

377 chainDefinition.insert(0, self.outputRun.name) 

378 butler.registry.setCollectionChain(self.output.name, chainDefinition) 

379 _LOG.debug("Preparing butler to write to '%s' and read from '%s'=%s", 

380 self.outputRun.name, self.output.name, chainDefinition) 

381 butler.registry.defaults = RegistryDefaults(run=self.outputRun.name, collections=self.output.name) 

382 else: 

383 inputs = CollectionSearch.fromExpression((self.outputRun.name,) + self.inputs) 

384 _LOG.debug("Preparing butler to write to '%s' and read from %s.", self.outputRun.name, inputs) 

385 butler.registry.defaults = RegistryDefaults(run=self.outputRun.name, collections=inputs) 

386 return butler 

387 

388 output: Optional[_OutputChainedCollectionInfo] 

389 """Information about the output chained collection, if there is or will be 

390 one (`_OutputChainedCollectionInfo` or `None`). 

391 """ 

392 

393 outputRun: Optional[_OutputRunCollectionInfo] 

394 """Information about the output run collection, if there is or will be 

395 one (`_OutputRunCollectionInfo` or `None`). 

396 """ 

397 

398 inputs: Tuple[str, ...] 

399 """Input collections provided directly by the user (`tuple` [ `str` ]). 

400 """ 

401 

402 

403class _FilteredStream: 

404 """A file-like object that filters some config fields. 

405 

406 Note 

407 ---- 

408 This class depends on implementation details of ``Config.saveToStream`` 

409 methods, in particular that that method uses single call to write() 

410 method to save information about single config field, and that call 

411 combines comments string(s) for a field and field path and value. 

412 This class will not work reliably on the "import" strings, so imports 

413 should be disabled by passing ``skipImports=True`` to ``saveToStream()``. 

414 """ 

415 def __init__(self, pattern): 

416 # obey case if pattern isn't lowercase or requests NOIGNORECASE 

417 mat = re.search(r"(.*):NOIGNORECASE$", pattern) 

418 

419 if mat: 

420 pattern = mat.group(1) 

421 self._pattern = re.compile(fnmatch.translate(pattern)) 

422 else: 

423 if pattern != pattern.lower(): 

424 print(f"Matching \"{pattern}\" without regard to case " 

425 "(append :NOIGNORECASE to prevent this)", file=sys.stdout) 

426 self._pattern = re.compile(fnmatch.translate(pattern), re.IGNORECASE) 

427 

428 def write(self, showStr): 

429 # Strip off doc string line(s) and cut off at "=" for string matching 

430 matchStr = showStr.rstrip().split("\n")[-1].split("=")[0] 

431 if self._pattern.search(matchStr): 

432 sys.stdout.write(showStr) 

433 

434# ------------------------ 

435# Exported definitions -- 

436# ------------------------ 

437 

438 

439class CmdLineFwk: 

440 """PipelineTask framework which executes tasks from command line. 

441 

442 In addition to executing tasks this activator provides additional methods 

443 for task management like dumping configuration or execution chain. 

444 """ 

445 

446 MP_TIMEOUT = 9999 # Default timeout (sec) for multiprocessing 

447 

448 def __init__(self): 

449 pass 

450 

451 def makePipeline(self, args): 

452 """Build a pipeline from command line arguments. 

453 

454 Parameters 

455 ---------- 

456 args : `argparse.Namespace` 

457 Parsed command line 

458 

459 Returns 

460 ------- 

461 pipeline : `~lsst.pipe.base.Pipeline` 

462 """ 

463 if args.pipeline: 

464 pipeline = Pipeline.from_uri(args.pipeline) 

465 else: 

466 pipeline = Pipeline("anonymous") 

467 

468 # loop over all pipeline actions and apply them in order 

469 for action in args.pipeline_actions: 

470 if action.action == "add_instrument": 

471 

472 pipeline.addInstrument(action.value) 

473 

474 elif action.action == "new_task": 

475 

476 pipeline.addTask(action.value, action.label) 

477 

478 elif action.action == "delete_task": 

479 

480 pipeline.removeTask(action.label) 

481 

482 elif action.action == "config": 

483 

484 # action value string is "field=value", split it at '=' 

485 field, _, value = action.value.partition("=") 

486 pipeline.addConfigOverride(action.label, field, value) 

487 

488 elif action.action == "configfile": 

489 

490 pipeline.addConfigFile(action.label, action.value) 

491 

492 else: 

493 

494 raise ValueError(f"Unexpected pipeline action: {action.action}") 

495 

496 if args.save_pipeline: 

497 pipeline.write_to_uri(args.save_pipeline) 

498 

499 if args.pipeline_dot: 

500 pipeline2dot(pipeline, args.pipeline_dot) 

501 

502 return pipeline 

503 

504 def makeGraph(self, pipeline, args): 

505 """Build a graph from command line arguments. 

506 

507 Parameters 

508 ---------- 

509 pipeline : `~lsst.pipe.base.Pipeline` 

510 Pipeline, can be empty or ``None`` if graph is read from a file. 

511 args : `argparse.Namespace` 

512 Parsed command line 

513 

514 Returns 

515 ------- 

516 graph : `~lsst.pipe.base.QuantumGraph` or `None` 

517 If resulting graph is empty then `None` is returned. 

518 """ 

519 

520 registry, collections, run = _ButlerFactory.makeRegistryAndCollections(args) 

521 

522 if args.qgraph: 

523 # click passes empty tuple as default value for qgraph_node_id 

524 nodes = args.qgraph_node_id or None 

525 qgraph = QuantumGraph.loadUri(args.qgraph, registry.dimensions, 

526 nodes=nodes, graphID=args.qgraph_id) 

527 

528 # pipeline can not be provided in this case 

529 if pipeline: 

530 raise ValueError("Pipeline must not be given when quantum graph is read from file.") 

531 

532 else: 

533 

534 # make execution plan (a.k.a. DAG) for pipeline 

535 graphBuilder = GraphBuilder(registry, 

536 skipExisting=args.skip_existing) 

537 qgraph = graphBuilder.makeGraph(pipeline, collections, run, args.data_query) 

538 

539 # count quanta in graph and give a warning if it's empty and return None 

540 nQuanta = len(qgraph) 

541 if nQuanta == 0: 

542 warnings.warn("QuantumGraph is empty", stacklevel=2) 

543 return None 

544 else: 

545 _LOG.info("QuantumGraph contains %d quanta for %d tasks, graph ID: %r", 

546 nQuanta, len(qgraph.taskGraph), qgraph.graphID) 

547 

548 if args.save_qgraph: 

549 qgraph.saveUri(args.save_qgraph) 

550 

551 if args.save_single_quanta: 

552 for quantumNode in qgraph: 

553 sqgraph = qgraph.subset(quantumNode) 

554 uri = args.save_single_quanta.format(quantumNode.nodeId.number) 

555 sqgraph.saveUri(uri) 

556 

557 if args.qgraph_dot: 

558 graph2dot(qgraph, args.qgraph_dot) 

559 

560 return qgraph 

561 

562 def runPipeline(self, graph, taskFactory, args, butler=None): 

563 """Execute complete QuantumGraph. 

564 

565 Parameters 

566 ---------- 

567 graph : `QuantumGraph` 

568 Execution graph. 

569 taskFactory : `~lsst.pipe.base.TaskFactory` 

570 Task factory 

571 args : `argparse.Namespace` 

572 Parsed command line 

573 butler : `~lsst.daf.butler.Butler`, optional 

574 Data Butler instance, if not defined then new instance is made 

575 using command line options. 

576 """ 

577 # make butler instance 

578 if butler is None: 

579 butler = _ButlerFactory.makeWriteButler(args) 

580 

581 # Enable lsstDebug debugging. Note that this is done once in the 

582 # main process before PreExecInit and it is also repeated before 

583 # running each task in SingleQuantumExecutor (which may not be 

584 # needed if `multipocessing` always uses fork start method). 

585 if args.enableLsstDebug: 

586 try: 

587 _LOG.debug("Will try to import debug.py") 

588 import debug # noqa:F401 

589 except ImportError: 

590 _LOG.warn("No 'debug' module found.") 

591 

592 preExecInit = PreExecInit(butler, taskFactory, args.skip_existing) 

593 preExecInit.initialize(graph, 

594 saveInitOutputs=not args.skip_init_writes, 

595 registerDatasetTypes=args.register_dataset_types, 

596 saveVersions=not args.no_versions) 

597 

598 if not args.init_only: 

599 graphFixup = self._importGraphFixup(args) 

600 quantumExecutor = SingleQuantumExecutor(taskFactory, 

601 skipExisting=args.skip_existing, 

602 clobberPartialOutputs=args.clobber_partial_outputs, 

603 enableLsstDebug=args.enableLsstDebug) 

604 timeout = self.MP_TIMEOUT if args.timeout is None else args.timeout 

605 executor = MPGraphExecutor(numProc=args.processes, timeout=timeout, 

606 startMethod=args.start_method, 

607 quantumExecutor=quantumExecutor, 

608 failFast=args.fail_fast, 

609 executionGraphFixup=graphFixup) 

610 with util.profile(args.profile, _LOG): 

611 executor.execute(graph, butler) 

612 

613 def showInfo(self, args, pipeline, graph=None): 

614 """Display useful info about pipeline and environment. 

615 

616 Parameters 

617 ---------- 

618 args : `argparse.Namespace` 

619 Parsed command line 

620 pipeline : `Pipeline` 

621 Pipeline definition 

622 graph : `QuantumGraph`, optional 

623 Execution graph 

624 """ 

625 showOpts = args.show 

626 for what in showOpts: 

627 showCommand, _, showArgs = what.partition("=") 

628 

629 if showCommand in ["pipeline", "config", "history", "tasks"]: 

630 if not pipeline: 

631 _LOG.warning("Pipeline is required for --show=%s", showCommand) 

632 continue 

633 

634 if showCommand in ["graph", "workflow", "uri"]: 

635 if not graph: 

636 _LOG.warning("QuantumGraph is required for --show=%s", showCommand) 

637 continue 

638 

639 if showCommand == "pipeline": 

640 print(pipeline) 

641 elif showCommand == "config": 

642 self._showConfig(pipeline, showArgs, False) 

643 elif showCommand == "dump-config": 

644 self._showConfig(pipeline, showArgs, True) 

645 elif showCommand == "history": 

646 self._showConfigHistory(pipeline, showArgs) 

647 elif showCommand == "tasks": 

648 self._showTaskHierarchy(pipeline) 

649 elif showCommand == "graph": 

650 if graph: 

651 self._showGraph(graph) 

652 elif showCommand == "uri": 

653 if graph: 

654 self._showUri(graph, args) 

655 elif showCommand == "workflow": 

656 if graph: 

657 self._showWorkflow(graph, args) 

658 else: 

659 print("Unknown value for show: %s (choose from '%s')" % 

660 (what, "', '".join("pipeline config[=XXX] history=XXX tasks graph".split())), 

661 file=sys.stderr) 

662 sys.exit(1) 

663 

664 def _showConfig(self, pipeline, showArgs, dumpFullConfig): 

665 """Show task configuration 

666 

667 Parameters 

668 ---------- 

669 pipeline : `Pipeline` 

670 Pipeline definition 

671 showArgs : `str` 

672 Defines what to show 

673 dumpFullConfig : `bool` 

674 If true then dump complete task configuration with all imports. 

675 """ 

676 stream = sys.stdout 

677 if dumpFullConfig: 

678 # Task label can be given with this option 

679 taskName = showArgs 

680 else: 

681 # The argument can have form [TaskLabel::][pattern:NOIGNORECASE] 

682 matConfig = re.search(r"^(?:(\w+)::)?(?:config.)?(.+)?", showArgs) 

683 taskName = matConfig.group(1) 

684 pattern = matConfig.group(2) 

685 if pattern: 

686 stream = _FilteredStream(pattern) 

687 

688 tasks = util.filterTasks(pipeline, taskName) 

689 if not tasks: 

690 print("Pipeline has no tasks named {}".format(taskName), file=sys.stderr) 

691 sys.exit(1) 

692 

693 for taskDef in tasks: 

694 print("### Configuration for task `{}'".format(taskDef.label)) 

695 taskDef.config.saveToStream(stream, root="config", skipImports=not dumpFullConfig) 

696 

697 def _showConfigHistory(self, pipeline, showArgs): 

698 """Show history for task configuration 

699 

700 Parameters 

701 ---------- 

702 pipeline : `Pipeline` 

703 Pipeline definition 

704 showArgs : `str` 

705 Defines what to show 

706 """ 

707 

708 taskName = None 

709 pattern = None 

710 matHistory = re.search(r"^(?:(\w+)::)?(?:config[.])?(.+)", showArgs) 

711 if matHistory: 

712 taskName = matHistory.group(1) 

713 pattern = matHistory.group(2) 

714 if not pattern: 

715 print("Please provide a value with --show history (e.g. history=Task::param)", file=sys.stderr) 

716 sys.exit(1) 

717 

718 tasks = util.filterTasks(pipeline, taskName) 

719 if not tasks: 

720 print(f"Pipeline has no tasks named {taskName}", file=sys.stderr) 

721 sys.exit(1) 

722 

723 found = False 

724 for taskDef in tasks: 

725 

726 config = taskDef.config 

727 

728 # Look for any matches in the config hierarchy for this name 

729 for nmatch, thisName in enumerate(fnmatch.filter(config.names(), pattern)): 

730 if nmatch > 0: 

731 print("") 

732 

733 cpath, _, cname = thisName.rpartition(".") 

734 try: 

735 if not cpath: 

736 # looking for top-level field 

737 hconfig = taskDef.config 

738 else: 

739 hconfig = eval("config." + cpath, {}, {"config": config}) 

740 except AttributeError: 

741 print(f"Error: Unable to extract attribute {cpath} from task {taskDef.label}", 

742 file=sys.stderr) 

743 hconfig = None 

744 

745 # Sometimes we end up with a non-Config so skip those 

746 if isinstance(hconfig, (pexConfig.Config, pexConfig.ConfigurableInstance)) and \ 

747 hasattr(hconfig, cname): 

748 print(f"### Configuration field for task `{taskDef.label}'") 

749 print(pexConfig.history.format(hconfig, cname)) 

750 found = True 

751 

752 if not found: 

753 print(f"None of the tasks has field matching {pattern}", file=sys.stderr) 

754 sys.exit(1) 

755 

756 def _showTaskHierarchy(self, pipeline): 

757 """Print task hierarchy to stdout 

758 

759 Parameters 

760 ---------- 

761 pipeline: `Pipeline` 

762 """ 

763 for taskDef in pipeline.toExpandedPipeline(): 

764 print("### Subtasks for task `{}'".format(taskDef.taskName)) 

765 

766 for configName, taskName in util.subTaskIter(taskDef.config): 

767 print("{}: {}".format(configName, taskName)) 

768 

769 def _showGraph(self, graph): 

770 """Print quanta information to stdout 

771 

772 Parameters 

773 ---------- 

774 graph : `QuantumGraph` 

775 Execution graph. 

776 """ 

777 for taskNode in graph.taskGraph: 

778 print(taskNode) 

779 

780 for iq, quantum in enumerate(graph.getQuantaForTask(taskNode)): 

781 print(" Quantum {}:".format(iq)) 

782 print(" inputs:") 

783 for key, refs in quantum.inputs.items(): 

784 dataIds = ["DataId({})".format(ref.dataId) for ref in refs] 

785 print(" {}: [{}]".format(key, ", ".join(dataIds))) 

786 print(" outputs:") 

787 for key, refs in quantum.outputs.items(): 

788 dataIds = ["DataId({})".format(ref.dataId) for ref in refs] 

789 print(" {}: [{}]".format(key, ", ".join(dataIds))) 

790 

791 def _showWorkflow(self, graph, args): 

792 """Print quanta information and dependency to stdout 

793 

794 Parameters 

795 ---------- 

796 graph : `QuantumGraph` 

797 Execution graph. 

798 args : `argparse.Namespace` 

799 Parsed command line 

800 """ 

801 for node in graph: 

802 print(f"Quantum {node.nodeId.number}: {node.taskDef.taskName}") 

803 for parent in graph.determineInputsToQuantumNode(node): 

804 print(f"Parent Quantum {parent.nodeId.number} - Child Quantum {node.nodeId.number}") 

805 

806 def _showUri(self, graph, args): 

807 """Print input and predicted output URIs to stdout 

808 

809 Parameters 

810 ---------- 

811 graph : `QuantumGraph` 

812 Execution graph 

813 args : `argparse.Namespace` 

814 Parsed command line 

815 """ 

816 def dumpURIs(thisRef): 

817 primary, components = butler.getURIs(thisRef, predict=True, run="TBD") 

818 if primary: 

819 print(f" {primary}") 

820 else: 

821 print(" (disassembled artifact)") 

822 for compName, compUri in components.items(): 

823 print(f" {compName}: {compUri}") 

824 

825 butler = _ButlerFactory.makeReadButler(args) 

826 for node in graph: 

827 print(f"Quantum {node.nodeId.number}: {node.taskDef.taskName}") 

828 print(" inputs:") 

829 for key, refs in node.quantum.inputs.items(): 

830 for ref in refs: 

831 dumpURIs(ref) 

832 print(" outputs:") 

833 for key, refs in node.quantum.outputs.items(): 

834 for ref in refs: 

835 dumpURIs(ref) 

836 

837 def _importGraphFixup(self, args): 

838 """Import/instantiate graph fixup object. 

839 

840 Parameters 

841 ---------- 

842 args : `argparse.Namespace` 

843 Parsed command line. 

844 

845 Returns 

846 ------- 

847 fixup : `ExecutionGraphFixup` or `None` 

848 

849 Raises 

850 ------ 

851 ValueError 

852 Raised if import fails, method call raises exception, or returned 

853 instance has unexpected type. 

854 """ 

855 if args.graph_fixup: 

856 try: 

857 factory = doImport(args.graph_fixup) 

858 except Exception as exc: 

859 raise ValueError("Failed to import graph fixup class/method") from exc 

860 try: 

861 fixup = factory() 

862 except Exception as exc: 

863 raise ValueError("Failed to make instance of graph fixup") from exc 

864 if not isinstance(fixup, ExecutionGraphFixup): 

865 raise ValueError("Graph fixup is not an instance of ExecutionGraphFixup class") 

866 return fixup