Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of ctrl_mpexec. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22"""Module defining CmdLineFwk class and related methods. 

23""" 

24 

25__all__ = ['CmdLineFwk'] 

26 

27# ------------------------------- 

28# Imports of standard modules -- 

29# ------------------------------- 

30import argparse 

31import fnmatch 

32import logging 

33import re 

34import sys 

35from typing import List, Optional, Tuple 

36import warnings 

37 

38# ----------------------------- 

39# Imports for other modules -- 

40# ----------------------------- 

41from lsst.daf.butler import ( 

42 Butler, 

43 CollectionSearch, 

44 CollectionType, 

45 DatasetTypeRestriction, 

46 Registry, 

47) 

48from lsst.daf.butler.registry import MissingCollectionError 

49import lsst.pex.config as pexConfig 

50from lsst.pipe.base import GraphBuilder, Pipeline, QuantumGraph 

51from lsst.obs.base import Instrument 

52from .dotTools import graph2dot, pipeline2dot 

53from .executionGraphFixup import ExecutionGraphFixup 

54from .mpGraphExecutor import MPGraphExecutor 

55from .preExecInit import PreExecInit 

56from .singleQuantumExecutor import SingleQuantumExecutor 

57from . import util 

58from lsst.utils import doImport 

59 

60# ---------------------------------- 

61# Local non-exported definitions -- 

62# ---------------------------------- 

63 

64_LOG = logging.getLogger(__name__.partition(".")[2]) 

65 

66 

67class _OutputChainedCollectionInfo: 

68 """A helper class for handling command-line arguments related to an output 

69 `~lsst.daf.butler.CollectionType.CHAINED` collection. 

70 

71 Parameters 

72 ---------- 

73 registry : `lsst.daf.butler.Registry` 

74 Butler registry that collections will be added to and/or queried from. 

75 name : `str` 

76 Name of the collection given on the command line. 

77 """ 

78 def __init__(self, registry: Registry, name: str): 

79 self.name = name 

80 try: 

81 self.chain = list(registry.getCollectionChain(name)) 

82 self.exists = True 

83 except MissingCollectionError: 

84 self.chain = [] 

85 self.exists = False 

86 

87 def __str__(self): 

88 return self.name 

89 

90 name: str 

91 """Name of the collection provided on the command line (`str`). 

92 """ 

93 

94 exists: bool 

95 """Whether this collection already exists in the registry (`bool`). 

96 """ 

97 

98 chain: List[str] 

99 """The definition of the collection, if it already exists (`list`). 

100 

101 Empty if the collection does not alredy exist. 

102 """ 

103 

104 

105class _OutputRunCollectionInfo: 

106 """A helper class for handling command-line arguments related to an output 

107 `~lsst.daf.butler.CollectionType.RUN` collection. 

108 

109 Parameters 

110 ---------- 

111 registry : `lsst.daf.butler.Registry` 

112 Butler registry that collections will be added to and/or queried from. 

113 name : `str` 

114 Name of the collection given on the command line. 

115 """ 

116 def __init__(self, registry: Registry, name: str): 

117 self.name = name 

118 try: 

119 actualType = registry.getCollectionType(name) 

120 if actualType is not CollectionType.RUN: 

121 raise TypeError(f"Collection '{name}' exists but has type {actualType.name}, not RUN.") 

122 self.exists = True 

123 except MissingCollectionError: 

124 self.exists = False 

125 

126 name: str 

127 """Name of the collection provided on the command line (`str`). 

128 """ 

129 

130 exists: bool 

131 """Whether this collection already exists in the registry (`bool`). 

132 """ 

133 

134 

135class _ButlerFactory: 

136 """A helper class for processing command-line arguments related to input 

137 and output collections. 

138 

139 Parameters 

140 ---------- 

141 registry : `lsst.daf.butler.Registry` 

142 Butler registry that collections will be added to and/or queried from. 

143 

144 args : `argparse.Namespace` 

145 Parsed command-line arguments. The following attributes are used, 

146 either at construction or in later methods. 

147 

148 ``output`` 

149 The name of a `~lsst.daf.butler.CollectionType.CHAINED` 

150 input/output collection. 

151 

152 ``output_run`` 

153 The name of a `~lsst.daf.butler.CollectionType.RUN` input/output 

154 collection. 

155 

156 ``extend_run`` 

157 A boolean indicating whether ``output_run`` should already exist 

158 and be extended. 

159 

160 ``replace_run`` 

161 A boolean indicating that (if `True`) ``output_run`` should already 

162 exist but will be removed from the output chained collection and 

163 replaced with a new one. 

164 

165 ``prune_replaced`` 

166 A boolean indicating whether to prune the replaced run (requires 

167 ``replace_run``). 

168 

169 ``inputs`` 

170 Input collections of any type; may be any type handled by 

171 `lsst.daf.butler.registry.CollectionSearch.fromExpression`. 

172 

173 ``butler_config`` 

174 Path to a data repository root or configuration file. 

175 

176 writeable : `bool` 

177 If `True`, a `Butler` is being initialized in a context where actual 

178 writes should happens, and hence no output run is necessary. 

179 

180 Raises 

181 ------ 

182 ValueError 

183 Raised if ``writeable is True`` but there are no output collections. 

184 """ 

185 def __init__(self, registry: Registry, args: argparse.Namespace, writeable: bool): 

186 if args.output is not None: 

187 self.output = _OutputChainedCollectionInfo(registry, args.output) 

188 else: 

189 self.output = None 

190 if args.output_run is not None: 

191 self.outputRun = _OutputRunCollectionInfo(registry, args.output_run) 

192 elif self.output is not None: 

193 if args.extend_run: 

194 runName = self.output.chain[0] 

195 else: 

196 runName = "{}/{}".format(self.output, Instrument.makeCollectionTimestamp()) 

197 self.outputRun = _OutputRunCollectionInfo(registry, runName) 

198 elif not writeable: 

199 # If we're not writing yet, ok to have no output run. 

200 self.outputRun = None 

201 else: 

202 raise ValueError("Cannot write without at least one of (--output, --output-run).") 

203 self.inputs = list(CollectionSearch.fromExpression(args.input)) if args.input else [] 

204 

205 def check(self, args: argparse.Namespace): 

206 """Check command-line options for consistency with each other and the 

207 data repository. 

208 

209 Parameters 

210 ---------- 

211 args : `argparse.Namespace` 

212 Parsed command-line arguments. See class documentation for the 

213 construction parameter of the same name. 

214 """ 

215 assert not (args.extend_run and args.replace_run), "In mutually-exclusive group in ArgumentParser." 

216 if self.inputs and self.output is not None and self.output.exists: 

217 raise ValueError("Cannot use --output with existing collection with --inputs.") 

218 if args.extend_run and self.outputRun is None: 

219 raise ValueError("Cannot --extend-run when no output collection is given.") 

220 if args.extend_run and not self.outputRun.exists: 

221 raise ValueError(f"Cannot --extend-run; output collection " 

222 f"'{self.outputRun.name}' does not exist.") 

223 if not args.extend_run and self.outputRun is not None and self.outputRun.exists: 

224 raise ValueError(f"Output run '{self.outputRun.name}' already exists, but " 

225 f"--extend-run was not given.") 

226 if args.prune_replaced and not args.replace_run: 

227 raise ValueError("--prune-replaced requires --replace-run.") 

228 if args.replace_run and (self.output is None or not self.output.exists): 

229 raise ValueError("--output must point to an existing CHAINED collection for --replace-run.") 

230 

231 @classmethod 

232 def _makeReadParts(cls, args: argparse.Namespace): 

233 """Common implementation for `makeReadButler` and 

234 `makeRegistryAndCollections`. 

235 

236 Parameters 

237 ---------- 

238 args : `argparse.Namespace` 

239 Parsed command-line arguments. See class documentation for the 

240 construction parameter of the same name. 

241 

242 Returns 

243 ------- 

244 butler : `lsst.daf.butler.Butler` 

245 A read-only butler constructed from the repo at 

246 ``args.butler_config``, but with no default collections. 

247 inputs : `lsst.daf.butler.registry.CollectionSearch` 

248 A collection search path constructed according to ``args``. 

249 self : `_ButlerFactory` 

250 A new `_ButlerFactory` instance representing the processed version 

251 of ``args``. 

252 """ 

253 butler = Butler(args.butler_config, writeable=False) 

254 self = cls(butler.registry, args, writeable=False) 

255 self.check(args) 

256 if self.output and self.output.exists: 

257 if args.replace_run: 

258 replaced = self.output.chain[0] 

259 inputs = self.output.chain[1:] 

260 _LOG.debug("Simulating collection search in '%s' after removing '%s'.", 

261 self.output.name, replaced) 

262 else: 

263 inputs = [self.output.name] 

264 else: 

265 inputs = list(self.inputs) 

266 if args.extend_run: 

267 inputs.insert(0, self.outputRun.name) 

268 inputs = CollectionSearch.fromExpression(inputs) 

269 return butler, inputs, self 

270 

271 @classmethod 

272 def makeReadButler(cls, args: argparse.Namespace) -> Butler: 

273 """Construct a read-only butler according to the given command-line 

274 arguments. 

275 

276 Parameters 

277 ---------- 

278 args : `argparse.Namespace` 

279 Parsed command-line arguments. See class documentation for the 

280 construction parameter of the same name. 

281 

282 Returns 

283 ------- 

284 butler : `lsst.daf.butler.Butler` 

285 A read-only butler initialized with the collections specified by 

286 ``args``. 

287 """ 

288 butler, inputs, _ = cls._makeReadParts(args) 

289 _LOG.debug("Preparing butler to read from %s.", inputs) 

290 return Butler(butler=butler, collections=inputs) 

291 

292 @classmethod 

293 def makeRegistryAndCollections(cls, args: argparse.Namespace) -> \ 

294 Tuple[Registry, CollectionSearch, Optional[str]]: 

295 """Return a read-only registry, a collection search path, and the name 

296 of the run to be used for future writes. 

297 

298 Parameters 

299 ---------- 

300 args : `argparse.Namespace` 

301 Parsed command-line arguments. See class documentation for the 

302 construction parameter of the same name. 

303 

304 Returns 

305 ------- 

306 registry : `lsst.daf.butler.Registry` 

307 Butler registry that collections will be added to and/or queried 

308 from. 

309 inputs : `lsst.daf.butler.registry.CollectionSearch` 

310 Collections to search for datasets. 

311 run : `str` or `None` 

312 Name of the output `~lsst.daf.butler.CollectionType.RUN` collection 

313 if it already exists, or `None` if it does not. 

314 """ 

315 butler, inputs, self = cls._makeReadParts(args) 

316 run = self.outputRun.name if args.extend_run else None 

317 _LOG.debug("Preparing registry to read from %s and expect future writes to '%s'.", inputs, run) 

318 return butler.registry, inputs, run 

319 

320 @classmethod 

321 def makeWriteButler(cls, args: argparse.Namespace) -> Butler: 

322 """Return a read-write butler initialized to write to and read from 

323 the collections specified by the given command-line arguments. 

324 

325 Parameters 

326 ---------- 

327 args : `argparse.Namespace` 

328 Parsed command-line arguments. See class documentation for the 

329 construction parameter of the same name. 

330 

331 Returns 

332 ------- 

333 butler : `lsst.daf.butler.Butler` 

334 A read-write butler initialized according to the given arguments. 

335 """ 

336 butler = Butler(args.butler_config, writeable=True) 

337 self = cls(butler.registry, args, writeable=True) 

338 self.check(args) 

339 if self.output is not None: 

340 chainDefinition = list(self.output.chain if self.output.exists else self.inputs) 

341 if args.replace_run: 

342 replaced = chainDefinition.pop(0) 

343 if args.prune_replaced == "unstore": 

344 # Remove datasets from datastore 

345 with butler.transaction(): 

346 refs = butler.registry.queryDatasets(..., collections=replaced) 

347 butler.pruneDatasets(refs, unstore=True, run=replaced, disassociate=False) 

348 elif args.prune_replaced == "purge": 

349 # Erase entire collection and all datasets, need to remove 

350 # collection from its chain collection first. 

351 with butler.transaction(): 

352 butler.registry.setCollectionChain(self.output.name, chainDefinition) 

353 butler.pruneCollection(replaced, purge=True, unstore=True) 

354 elif args.prune_replaced is not None: 

355 raise NotImplementedError( 

356 f"Unsupported --prune-replaced option '{args.prune_replaced}'." 

357 ) 

358 chainDefinition.insert(0, self.outputRun.name) 

359 chainDefinition = CollectionSearch.fromExpression(chainDefinition) 

360 _LOG.debug("Preparing butler to write to '%s' and read from '%s'=%s", 

361 self.outputRun.name, self.output.name, chainDefinition) 

362 return Butler(butler=butler, run=self.outputRun.name, collections=self.output.name, 

363 chains={self.output.name: chainDefinition}) 

364 else: 

365 inputs = CollectionSearch.fromExpression([self.outputRun.name] + self.inputs) 

366 _LOG.debug("Preparing butler to write to '%s' and read from %s.", self.outputRun.name, inputs) 

367 return Butler(butler=butler, run=self.outputRun.name, collections=inputs) 

368 

369 output: Optional[_OutputChainedCollectionInfo] 

370 """Information about the output chained collection, if there is or will be 

371 one (`_OutputChainedCollectionInfo` or `None`). 

372 """ 

373 

374 outputRun: Optional[_OutputRunCollectionInfo] 

375 """Information about the output run collection, if there is or will be 

376 one (`_OutputRunCollectionInfo` or `None`). 

377 """ 

378 

379 inputs: List[Tuple[str, DatasetTypeRestriction]] 

380 """Input collections, including those also used for outputs and any 

381 restrictions on dataset types (`list`). 

382 """ 

383 

384 

385class _FilteredStream: 

386 """A file-like object that filters some config fields. 

387 

388 Note 

389 ---- 

390 This class depends on implementation details of ``Config.saveToStream`` 

391 methods, in particular that that method uses single call to write() 

392 method to save information about single config field, and that call 

393 combines comments string(s) for a field and field path and value. 

394 This class will not work reliably on the "import" strings, so imports 

395 should be disabled by passing ``skipImports=True`` to ``saveToStream()``. 

396 """ 

397 def __init__(self, pattern): 

398 # obey case if pattern isn't lowercase or requests NOIGNORECASE 

399 mat = re.search(r"(.*):NOIGNORECASE$", pattern) 

400 

401 if mat: 

402 pattern = mat.group(1) 

403 self._pattern = re.compile(fnmatch.translate(pattern)) 

404 else: 

405 if pattern != pattern.lower(): 

406 print(f"Matching \"{pattern}\" without regard to case " 

407 "(append :NOIGNORECASE to prevent this)", file=sys.stdout) 

408 self._pattern = re.compile(fnmatch.translate(pattern), re.IGNORECASE) 

409 

410 def write(self, showStr): 

411 # Strip off doc string line(s) and cut off at "=" for string matching 

412 matchStr = showStr.rstrip().split("\n")[-1].split("=")[0] 

413 if self._pattern.search(matchStr): 

414 sys.stdout.write(showStr) 

415 

416# ------------------------ 

417# Exported definitions -- 

418# ------------------------ 

419 

420 

421class CmdLineFwk: 

422 """PipelineTask framework which executes tasks from command line. 

423 

424 In addition to executing tasks this activator provides additional methods 

425 for task management like dumping configuration or execution chain. 

426 """ 

427 

428 MP_TIMEOUT = 9999 # Default timeout (sec) for multiprocessing 

429 

430 def __init__(self): 

431 pass 

432 

433 def makePipeline(self, args): 

434 """Build a pipeline from command line arguments. 

435 

436 Parameters 

437 ---------- 

438 args : `argparse.Namespace` 

439 Parsed command line 

440 

441 Returns 

442 ------- 

443 pipeline : `~lsst.pipe.base.Pipeline` 

444 """ 

445 if args.pipeline: 

446 pipeline = Pipeline.fromFile(args.pipeline) 

447 else: 

448 pipeline = Pipeline("anonymous") 

449 

450 # loop over all pipeline actions and apply them in order 

451 for action in args.pipeline_actions: 

452 if action.action == "add_instrument": 

453 

454 pipeline.addInstrument(action.value) 

455 

456 elif action.action == "new_task": 

457 

458 pipeline.addTask(action.value, action.label) 

459 

460 elif action.action == "delete_task": 

461 

462 pipeline.removeTask(action.label) 

463 

464 elif action.action == "config": 

465 

466 # action value string is "field=value", split it at '=' 

467 field, _, value = action.value.partition("=") 

468 pipeline.addConfigOverride(action.label, field, value) 

469 

470 elif action.action == "configfile": 

471 

472 pipeline.addConfigFile(action.label, action.value) 

473 

474 else: 

475 

476 raise ValueError(f"Unexpected pipeline action: {action.action}") 

477 

478 if args.save_pipeline: 

479 pipeline.toFile(args.save_pipeline) 

480 

481 if args.pipeline_dot: 

482 pipeline2dot(pipeline, args.pipeline_dot) 

483 

484 return pipeline 

485 

486 def makeGraph(self, pipeline, args): 

487 """Build a graph from command line arguments. 

488 

489 Parameters 

490 ---------- 

491 pipeline : `~lsst.pipe.base.Pipeline` 

492 Pipeline, can be empty or ``None`` if graph is read from a file. 

493 args : `argparse.Namespace` 

494 Parsed command line 

495 

496 Returns 

497 ------- 

498 graph : `~lsst.pipe.base.QuantumGraph` or `None` 

499 If resulting graph is empty then `None` is returned. 

500 """ 

501 

502 registry, collections, run = _ButlerFactory.makeRegistryAndCollections(args) 

503 

504 if args.qgraph: 

505 # click passes empty tuple as default value for qgraph_node_id 

506 nodes = args.qgraph_node_id or None 

507 qgraph = QuantumGraph.loadUri(args.qgraph, registry.dimensions, 

508 nodes=nodes, graphID=args.qgraph_id) 

509 

510 # pipeline can not be provided in this case 

511 if pipeline: 

512 raise ValueError("Pipeline must not be given when quantum graph is read from file.") 

513 

514 else: 

515 

516 # make execution plan (a.k.a. DAG) for pipeline 

517 graphBuilder = GraphBuilder(registry, 

518 skipExisting=args.skip_existing) 

519 qgraph = graphBuilder.makeGraph(pipeline, collections, run, args.data_query) 

520 

521 # count quanta in graph and give a warning if it's empty and return None 

522 nQuanta = len(qgraph) 

523 if nQuanta == 0: 

524 warnings.warn("QuantumGraph is empty", stacklevel=2) 

525 return None 

526 else: 

527 _LOG.info("QuantumGraph contains %d quanta for %d tasks, graph ID: %r", 

528 nQuanta, len(qgraph.taskGraph), qgraph.graphID) 

529 

530 if args.save_qgraph: 

531 qgraph.saveUri(args.save_qgraph) 

532 

533 if args.save_single_quanta: 

534 for quantumNode in qgraph: 

535 sqgraph = qgraph.subset(quantumNode) 

536 uri = args.save_single_quanta.format(quantumNode.nodeId.number) 

537 sqgraph.saveUri(uri) 

538 

539 if args.qgraph_dot: 

540 graph2dot(qgraph, args.qgraph_dot) 

541 

542 return qgraph 

543 

544 def runPipeline(self, graph, taskFactory, args, butler=None): 

545 """Execute complete QuantumGraph. 

546 

547 Parameters 

548 ---------- 

549 graph : `QuantumGraph` 

550 Execution graph. 

551 taskFactory : `~lsst.pipe.base.TaskFactory` 

552 Task factory 

553 args : `argparse.Namespace` 

554 Parsed command line 

555 butler : `~lsst.daf.butler.Butler`, optional 

556 Data Butler instance, if not defined then new instance is made 

557 using command line options. 

558 """ 

559 # make butler instance 

560 if butler is None: 

561 butler = _ButlerFactory.makeWriteButler(args) 

562 

563 # Enable lsstDebug debugging. Note that this is done once in the 

564 # main process before PreExecInit and it is also repeated before 

565 # running each task in SingleQuantumExecutor (which may not be 

566 # needed if `multipocessing` always uses fork start method). 

567 if args.enableLsstDebug: 

568 try: 

569 _LOG.debug("Will try to import debug.py") 

570 import debug # noqa:F401 

571 except ImportError: 

572 _LOG.warn("No 'debug' module found.") 

573 

574 preExecInit = PreExecInit(butler, taskFactory, args.skip_existing) 

575 preExecInit.initialize(graph, 

576 saveInitOutputs=not args.skip_init_writes, 

577 registerDatasetTypes=args.register_dataset_types, 

578 saveVersions=not args.no_versions) 

579 

580 if not args.init_only: 

581 graphFixup = self._importGraphFixup(args) 

582 quantumExecutor = SingleQuantumExecutor(taskFactory, 

583 skipExisting=args.skip_existing, 

584 clobberPartialOutputs=args.clobber_partial_outputs, 

585 enableLsstDebug=args.enableLsstDebug) 

586 timeout = self.MP_TIMEOUT if args.timeout is None else args.timeout 

587 executor = MPGraphExecutor(numProc=args.processes, timeout=timeout, 

588 startMethod=args.start_method, 

589 quantumExecutor=quantumExecutor, 

590 failFast=args.fail_fast, 

591 executionGraphFixup=graphFixup) 

592 with util.profile(args.profile, _LOG): 

593 executor.execute(graph, butler) 

594 

595 def showInfo(self, args, pipeline, graph=None): 

596 """Display useful info about pipeline and environment. 

597 

598 Parameters 

599 ---------- 

600 args : `argparse.Namespace` 

601 Parsed command line 

602 pipeline : `Pipeline` 

603 Pipeline definition 

604 graph : `QuantumGraph`, optional 

605 Execution graph 

606 """ 

607 showOpts = args.show 

608 for what in showOpts: 

609 showCommand, _, showArgs = what.partition("=") 

610 

611 if showCommand in ["pipeline", "config", "history", "tasks"]: 

612 if not pipeline: 

613 _LOG.warning("Pipeline is required for --show=%s", showCommand) 

614 continue 

615 

616 if showCommand in ["graph", "workflow", "uri"]: 

617 if not graph: 

618 _LOG.warning("QuantumGraph is required for --show=%s", showCommand) 

619 continue 

620 

621 if showCommand == "pipeline": 

622 print(pipeline) 

623 elif showCommand == "config": 

624 self._showConfig(pipeline, showArgs, False) 

625 elif showCommand == "dump-config": 

626 self._showConfig(pipeline, showArgs, True) 

627 elif showCommand == "history": 

628 self._showConfigHistory(pipeline, showArgs) 

629 elif showCommand == "tasks": 

630 self._showTaskHierarchy(pipeline) 

631 elif showCommand == "graph": 

632 if graph: 

633 self._showGraph(graph) 

634 elif showCommand == "uri": 

635 if graph: 

636 self._showUri(graph, args) 

637 elif showCommand == "workflow": 

638 if graph: 

639 self._showWorkflow(graph, args) 

640 else: 

641 print("Unknown value for show: %s (choose from '%s')" % 

642 (what, "', '".join("pipeline config[=XXX] history=XXX tasks graph".split())), 

643 file=sys.stderr) 

644 sys.exit(1) 

645 

646 def _showConfig(self, pipeline, showArgs, dumpFullConfig): 

647 """Show task configuration 

648 

649 Parameters 

650 ---------- 

651 pipeline : `Pipeline` 

652 Pipeline definition 

653 showArgs : `str` 

654 Defines what to show 

655 dumpFullConfig : `bool` 

656 If true then dump complete task configuration with all imports. 

657 """ 

658 stream = sys.stdout 

659 if dumpFullConfig: 

660 # Task label can be given with this option 

661 taskName = showArgs 

662 else: 

663 # The argument can have form [TaskLabel::][pattern:NOIGNORECASE] 

664 matConfig = re.search(r"^(?:(\w+)::)?(?:config.)?(.+)?", showArgs) 

665 taskName = matConfig.group(1) 

666 pattern = matConfig.group(2) 

667 if pattern: 

668 stream = _FilteredStream(pattern) 

669 

670 tasks = util.filterTasks(pipeline, taskName) 

671 if not tasks: 

672 print("Pipeline has no tasks named {}".format(taskName), file=sys.stderr) 

673 sys.exit(1) 

674 

675 for taskDef in tasks: 

676 print("### Configuration for task `{}'".format(taskDef.label)) 

677 taskDef.config.saveToStream(stream, root="config", skipImports=not dumpFullConfig) 

678 

679 def _showConfigHistory(self, pipeline, showArgs): 

680 """Show history for task configuration 

681 

682 Parameters 

683 ---------- 

684 pipeline : `Pipeline` 

685 Pipeline definition 

686 showArgs : `str` 

687 Defines what to show 

688 """ 

689 

690 taskName = None 

691 pattern = None 

692 matHistory = re.search(r"^(?:(\w+)::)?(?:config[.])?(.+)", showArgs) 

693 if matHistory: 

694 taskName = matHistory.group(1) 

695 pattern = matHistory.group(2) 

696 if not pattern: 

697 print("Please provide a value with --show history (e.g. history=Task::param)", file=sys.stderr) 

698 sys.exit(1) 

699 

700 tasks = util.filterTasks(pipeline, taskName) 

701 if not tasks: 

702 print(f"Pipeline has no tasks named {taskName}", file=sys.stderr) 

703 sys.exit(1) 

704 

705 found = False 

706 for taskDef in tasks: 

707 

708 config = taskDef.config 

709 

710 # Look for any matches in the config hierarchy for this name 

711 for nmatch, thisName in enumerate(fnmatch.filter(config.names(), pattern)): 

712 if nmatch > 0: 

713 print("") 

714 

715 cpath, _, cname = thisName.rpartition(".") 

716 try: 

717 if not cpath: 

718 # looking for top-level field 

719 hconfig = taskDef.config 

720 else: 

721 hconfig = eval("config." + cpath, {}, {"config": config}) 

722 except AttributeError: 

723 print(f"Error: Unable to extract attribute {cpath} from task {taskDef.label}", 

724 file=sys.stderr) 

725 hconfig = None 

726 

727 # Sometimes we end up with a non-Config so skip those 

728 if isinstance(hconfig, (pexConfig.Config, pexConfig.ConfigurableInstance)) and \ 

729 hasattr(hconfig, cname): 

730 print(f"### Configuration field for task `{taskDef.label}'") 

731 print(pexConfig.history.format(hconfig, cname)) 

732 found = True 

733 

734 if not found: 

735 print(f"None of the tasks has field matching {pattern}", file=sys.stderr) 

736 sys.exit(1) 

737 

738 def _showTaskHierarchy(self, pipeline): 

739 """Print task hierarchy to stdout 

740 

741 Parameters 

742 ---------- 

743 pipeline: `Pipeline` 

744 """ 

745 for taskDef in pipeline.toExpandedPipeline(): 

746 print("### Subtasks for task `{}'".format(taskDef.taskName)) 

747 

748 for configName, taskName in util.subTaskIter(taskDef.config): 

749 print("{}: {}".format(configName, taskName)) 

750 

751 def _showGraph(self, graph): 

752 """Print quanta information to stdout 

753 

754 Parameters 

755 ---------- 

756 graph : `QuantumGraph` 

757 Execution graph. 

758 """ 

759 for taskNode in graph.taskGraph: 

760 print(taskNode) 

761 

762 for iq, quantum in enumerate(graph.getQuantaForTask(taskNode)): 

763 print(" Quantum {}:".format(iq)) 

764 print(" inputs:") 

765 for key, refs in quantum.inputs.items(): 

766 dataIds = ["DataId({})".format(ref.dataId) for ref in refs] 

767 print(" {}: [{}]".format(key, ", ".join(dataIds))) 

768 print(" outputs:") 

769 for key, refs in quantum.outputs.items(): 

770 dataIds = ["DataId({})".format(ref.dataId) for ref in refs] 

771 print(" {}: [{}]".format(key, ", ".join(dataIds))) 

772 

773 def _showWorkflow(self, graph, args): 

774 """Print quanta information and dependency to stdout 

775 

776 Parameters 

777 ---------- 

778 graph : `QuantumGraph` 

779 Execution graph. 

780 args : `argparse.Namespace` 

781 Parsed command line 

782 """ 

783 for node in graph: 

784 print(f"Quantum {node.nodeId.number}: {node.taskDef.taskName}") 

785 for parent in graph.determineInputsToQuantumNode(node): 

786 print(f"Parent Quantum {parent.nodeId.number} - Child Quantum {node.nodeId.number}") 

787 

788 def _showUri(self, graph, args): 

789 """Print input and predicted output URIs to stdout 

790 

791 Parameters 

792 ---------- 

793 graph : `QuantumGraph` 

794 Execution graph 

795 args : `argparse.Namespace` 

796 Parsed command line 

797 """ 

798 def dumpURIs(thisRef): 

799 primary, components = butler.getURIs(thisRef, predict=True, run="TBD") 

800 if primary: 

801 print(f" {primary}") 

802 else: 

803 print(" (disassembled artifact)") 

804 for compName, compUri in components.items(): 

805 print(f" {compName}: {compUri}") 

806 

807 butler = _ButlerFactory.makeReadButler(args) 

808 for node in graph: 

809 print(f"Quantum {node.nodeId.number}: {node.taskDef.taskName}") 

810 print(" inputs:") 

811 for key, refs in node.quantum.inputs.items(): 

812 for ref in refs: 

813 dumpURIs(ref) 

814 print(" outputs:") 

815 for key, refs in node.quantum.outputs.items(): 

816 for ref in refs: 

817 dumpURIs(ref) 

818 

819 def _importGraphFixup(self, args): 

820 """Import/instantiate graph fixup object. 

821 

822 Parameters 

823 ---------- 

824 args : `argparse.Namespace` 

825 Parsed command line. 

826 

827 Returns 

828 ------- 

829 fixup : `ExecutionGraphFixup` or `None` 

830 

831 Raises 

832 ------ 

833 ValueError 

834 Raised if import fails, method call raises exception, or returned 

835 instance has unexpected type. 

836 """ 

837 if args.graph_fixup: 

838 try: 

839 factory = doImport(args.graph_fixup) 

840 except Exception as exc: 

841 raise ValueError("Failed to import graph fixup class/method") from exc 

842 try: 

843 fixup = factory() 

844 except Exception as exc: 

845 raise ValueError("Failed to make instance of graph fixup") from exc 

846 if not isinstance(fixup, ExecutionGraphFixup): 

847 raise ValueError("Graph fixup is not an instance of ExecutionGraphFixup class") 

848 return fixup