Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of ctrl_mpexec. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22"""Module defining CmdLineFwk class and related methods. 

23""" 

24 

25__all__ = ['CmdLineFwk'] 

26 

27# ------------------------------- 

28# Imports of standard modules -- 

29# ------------------------------- 

30import argparse 

31import datetime 

32import fnmatch 

33import logging 

34import re 

35import sys 

36from typing import List, Optional, Tuple 

37import warnings 

38 

39# ----------------------------- 

40# Imports for other modules -- 

41# ----------------------------- 

42from lsst.daf.butler import ( 

43 Butler, 

44 CollectionSearch, 

45 CollectionType, 

46 DatasetTypeRestriction, 

47 Registry, 

48) 

49from lsst.daf.butler.registry import MissingCollectionError 

50import lsst.pex.config as pexConfig 

51from lsst.pipe.base import GraphBuilder, Pipeline, QuantumGraph 

52from .dotTools import graph2dot, pipeline2dot 

53from .executionGraphFixup import ExecutionGraphFixup 

54from .mpGraphExecutor import MPGraphExecutor 

55from .preExecInit import PreExecInit 

56from .singleQuantumExecutor import SingleQuantumExecutor 

57from . import util 

58from lsst.utils import doImport 

59 

60# ---------------------------------- 

61# Local non-exported definitions -- 

62# ---------------------------------- 

63 

64_LOG = logging.getLogger(__name__.partition(".")[2]) 

65 

66 

67class _OutputChainedCollectionInfo: 

68 """A helper class for handling command-line arguments related to an output 

69 `~lsst.daf.butler.CollectionType.CHAINED` collection. 

70 

71 Parameters 

72 ---------- 

73 registry : `lsst.daf.butler.Registry` 

74 Butler registry that collections will be added to and/or queried from. 

75 name : `str` 

76 Name of the collection given on the command line. 

77 """ 

78 def __init__(self, registry: Registry, name: str): 

79 self.name = name 

80 try: 

81 self.chain = list(registry.getCollectionChain(name)) 

82 self.exists = True 

83 except MissingCollectionError: 

84 self.chain = [] 

85 self.exists = False 

86 

87 def __str__(self): 

88 return self.name 

89 

90 name: str 

91 """Name of the collection provided on the command line (`str`). 

92 """ 

93 

94 exists: bool 

95 """Whether this collection already exists in the registry (`bool`). 

96 """ 

97 

98 chain: List[str] 

99 """The definition of the collection, if it already exists (`list`). 

100 

101 Empty if the collection does not alredy exist. 

102 """ 

103 

104 

105class _OutputRunCollectionInfo: 

106 """A helper class for handling command-line arguments related to an output 

107 `~lsst.daf.butler.CollectionType.RUN` collection. 

108 

109 Parameters 

110 ---------- 

111 registry : `lsst.daf.butler.Registry` 

112 Butler registry that collections will be added to and/or queried from. 

113 name : `str` 

114 Name of the collection given on the command line. 

115 """ 

116 def __init__(self, registry: Registry, name: str): 

117 self.name = name 

118 try: 

119 actualType = registry.getCollectionType(name) 

120 if actualType is not CollectionType.RUN: 

121 raise TypeError(f"Collection '{name}' exists but has type {actualType.name}, not RUN.") 

122 self.exists = True 

123 except MissingCollectionError: 

124 self.exists = False 

125 

126 name: str 

127 """Name of the collection provided on the command line (`str`). 

128 """ 

129 

130 exists: bool 

131 """Whether this collection already exists in the registry (`bool`). 

132 """ 

133 

134 

135class _ButlerFactory: 

136 """A helper class for processing command-line arguments related to input 

137 and output collections. 

138 

139 Parameters 

140 ---------- 

141 registry : `lsst.daf.butler.Registry` 

142 Butler registry that collections will be added to and/or queried from. 

143 

144 args : `argparse.Namespace` 

145 Parsed command-line arguments. The following attributes are used, 

146 either at construction or in later methods. 

147 

148 ``output`` 

149 The name of a `~lsst.daf.butler.CollectionType.CHAINED` 

150 input/output collection. 

151 

152 ``output_run`` 

153 The name of a `~lsst.daf.butler.CollectionType.RUN` input/output 

154 collection. 

155 

156 ``extend_run`` 

157 A boolean indicating whether ``output_run`` should already exist 

158 and be extended. 

159 

160 ``replace_run`` 

161 A boolean indicating that (if `True`) ``output_run`` should already 

162 exist but will be removed from the output chained collection and 

163 replaced with a new one. 

164 

165 ``prune_replaced`` 

166 A boolean indicating whether to prune the replaced run (requires 

167 ``replace_run``). 

168 

169 ``inputs`` 

170 Input collections of any type; may be any type handled by 

171 `lsst.daf.butler.registry.CollectionSearch.fromExpression`. 

172 

173 ``butler_config`` 

174 Path to a data repository root or configuration file. 

175 

176 writeable : `bool` 

177 If `True`, a `Butler` is being initialized in a context where actual 

178 writes should happens, and hence no output run is necessary. 

179 

180 Raises 

181 ------ 

182 ValueError 

183 Raised if ``writeable is True`` but there are no output collections. 

184 """ 

185 def __init__(self, registry: Registry, args: argparse.Namespace, writeable: bool): 

186 if args.output is not None: 

187 self.output = _OutputChainedCollectionInfo(registry, args.output) 

188 else: 

189 self.output = None 

190 if args.output_run is not None: 

191 self.outputRun = _OutputRunCollectionInfo(registry, args.output_run) 

192 elif self.output is not None: 

193 if args.extend_run: 

194 runName, _ = self.output.chain[0] 

195 else: 

196 runName = "{}/{:%Y%m%dT%Hh%Mm%Ss}".format(self.output, datetime.datetime.now()) 

197 self.outputRun = _OutputRunCollectionInfo(registry, runName) 

198 elif not writeable: 

199 # If we're not writing yet, ok to have no output run. 

200 self.outputRun = None 

201 else: 

202 raise ValueError("Cannot write without at least one of (--output, --output-run).") 

203 self.inputs = list(CollectionSearch.fromExpression(args.input)) if args.input else [] 

204 

205 def check(self, args: argparse.Namespace): 

206 """Check command-line options for consistency with each other and the 

207 data repository. 

208 

209 Parameters 

210 ---------- 

211 args : `argparse.Namespace` 

212 Parsed command-line arguments. See class documentation for the 

213 construction parameter of the same name. 

214 """ 

215 assert not (args.extend_run and args.replace_run), "In mutually-exclusive group in ArgumentParser." 

216 if self.inputs and self.output is not None and self.output.exists: 

217 raise ValueError("Cannot use --output with existing collection with --inputs.") 

218 if args.extend_run and self.outputRun is None: 

219 raise ValueError("Cannot --extend-run when no output collection is given.") 

220 if args.extend_run and not self.outputRun.exists: 

221 raise ValueError(f"Cannot --extend-run; output collection " 

222 f"'{self.outputRun.name}' does not exist.") 

223 if not args.extend_run and self.outputRun is not None and self.outputRun.exists: 

224 raise ValueError(f"Output run '{self.outputRun.name}' already exists, but " 

225 f"--extend-run was not given.") 

226 if args.prune_replaced and not args.replace_run: 

227 raise ValueError("--prune-replaced requires --replace-run.") 

228 if args.replace_run and (self.output is None or not self.output.exists): 

229 raise ValueError("--output must point to an existing CHAINED collection for --replace-run.") 

230 

231 @classmethod 

232 def _makeReadParts(cls, args: argparse.Namespace): 

233 """Common implementation for `makeReadButler` and 

234 `makeRegistryAndCollections`. 

235 

236 Parameters 

237 ---------- 

238 args : `argparse.Namespace` 

239 Parsed command-line arguments. See class documentation for the 

240 construction parameter of the same name. 

241 

242 Returns 

243 ------- 

244 butler : `lsst.daf.butler.Butler` 

245 A read-only butler constructed from the repo at 

246 ``args.butler_config``, but with no default collections. 

247 inputs : `lsst.daf.butler.registry.CollectionSearch` 

248 A collection search path constructed according to ``args``. 

249 self : `_ButlerFactory` 

250 A new `_ButlerFactory` instance representing the processed version 

251 of ``args``. 

252 """ 

253 butler = Butler(args.butler_config, writeable=False) 

254 self = cls(butler.registry, args, writeable=False) 

255 self.check(args) 

256 if self.output and self.output.exists: 

257 if args.replace_run: 

258 replaced = self.output.chain[0] 

259 inputs = self.output.chain[1:] 

260 _LOG.debug("Simulating collection search in '%s' after removing '%s'.", 

261 self.output.name, replaced) 

262 else: 

263 inputs = [self.output.name] 

264 else: 

265 inputs = list(self.inputs) 

266 if args.extend_run: 

267 inputs.insert(0, self.outputRun.name) 

268 inputs = CollectionSearch.fromExpression(inputs) 

269 return butler, inputs, self 

270 

271 @classmethod 

272 def makeReadButler(cls, args: argparse.Namespace) -> Butler: 

273 """Construct a read-only butler according to the given command-line 

274 arguments. 

275 

276 Parameters 

277 ---------- 

278 args : `argparse.Namespace` 

279 Parsed command-line arguments. See class documentation for the 

280 construction parameter of the same name. 

281 

282 Returns 

283 ------- 

284 butler : `lsst.daf.butler.Butler` 

285 A read-only butler initialized with the collections specified by 

286 ``args``. 

287 """ 

288 butler, inputs, _ = cls._makeReadParts(args) 

289 _LOG.debug("Preparing butler to read from %s.", inputs) 

290 return Butler(butler=butler, collections=inputs) 

291 

292 @classmethod 

293 def makeRegistryAndCollections(cls, args: argparse.Namespace) -> \ 

294 Tuple[Registry, CollectionSearch, Optional[str]]: 

295 """Return a read-only registry, a collection search path, and the name 

296 of the run to be used for future writes. 

297 

298 Parameters 

299 ---------- 

300 args : `argparse.Namespace` 

301 Parsed command-line arguments. See class documentation for the 

302 construction parameter of the same name. 

303 

304 Returns 

305 ------- 

306 registry : `lsst.daf.butler.Registry` 

307 Butler registry that collections will be added to and/or queried 

308 from. 

309 inputs : `lsst.daf.butler.registry.CollectionSearch` 

310 Collections to search for datasets. 

311 run : `str` or `None` 

312 Name of the output `~lsst.daf.butler.CollectionType.RUN` collection 

313 if it already exists, or `None` if it does not. 

314 """ 

315 butler, inputs, self = cls._makeReadParts(args) 

316 run = self.outputRun.name if args.extend_run else None 

317 _LOG.debug("Preparing registry to read from %s and expect future writes to '%s'.", inputs, run) 

318 return butler.registry, inputs, run 

319 

320 @classmethod 

321 def makeWriteButler(cls, args: argparse.Namespace) -> Butler: 

322 """Return a read-write butler initialized to write to and read from 

323 the collections specified by the given command-line arguments. 

324 

325 Parameters 

326 ---------- 

327 args : `argparse.Namespace` 

328 Parsed command-line arguments. See class documentation for the 

329 construction parameter of the same name. 

330 

331 Returns 

332 ------- 

333 butler : `lsst.daf.butler.Butler` 

334 A read-write butler initialized according to the given arguments. 

335 """ 

336 butler = Butler(args.butler_config, writeable=True) 

337 self = cls(butler.registry, args, writeable=True) 

338 self.check(args) 

339 if self.output is not None: 

340 chainDefinition = list(self.output.chain if self.output.exists else self.inputs) 

341 if args.replace_run: 

342 replaced = chainDefinition.pop(0) 

343 if args.prune_replaced == "unstore": 

344 # Remove datasets from datastore 

345 with butler.transaction(): 

346 refs = butler.registry.queryDatasets(..., collections=replaced) 

347 butler.pruneDatasets(refs, unstore=True, run=replaced, disassociate=False) 

348 elif args.prune_replaced == "purge": 

349 # Erase entire collection and all datasets, need to remove 

350 # collection from its chain collection first. 

351 with butler.transaction(): 

352 butler.registry.setCollectionChain(self.output.name, chainDefinition) 

353 butler.pruneCollection(replaced, purge=True, unstore=True) 

354 elif args.prune_replaced is not None: 

355 raise NotImplementedError( 

356 f"Unsupported --prune-replaced option '{args.prune_replaced}'." 

357 ) 

358 chainDefinition.insert(0, self.outputRun.name) 

359 chainDefinition = CollectionSearch.fromExpression(chainDefinition) 

360 _LOG.debug("Preparing butler to write to '%s' and read from '%s'=%s", 

361 self.outputRun.name, self.output.name, chainDefinition) 

362 return Butler(butler=butler, run=self.outputRun.name, collections=self.output.name, 

363 chains={self.output.name: chainDefinition}) 

364 else: 

365 inputs = CollectionSearch.fromExpression([self.outputRun.name] + self.inputs) 

366 _LOG.debug("Preparing butler to write to '%s' and read from %s.", self.outputRun.name, inputs) 

367 return Butler(butler=butler, run=self.outputRun.name, collections=inputs) 

368 

369 output: Optional[_OutputChainedCollectionInfo] 

370 """Information about the output chained collection, if there is or will be 

371 one (`_OutputChainedCollectionInfo` or `None`). 

372 """ 

373 

374 outputRun: Optional[_OutputRunCollectionInfo] 

375 """Information about the output run collection, if there is or will be 

376 one (`_OutputRunCollectionInfo` or `None`). 

377 """ 

378 

379 inputs: List[Tuple[str, DatasetTypeRestriction]] 

380 """Input collections, including those also used for outputs and any 

381 restrictions on dataset types (`list`). 

382 """ 

383 

384 

385class _FilteredStream: 

386 """A file-like object that filters some config fields. 

387 

388 Note 

389 ---- 

390 This class depends on implementation details of ``Config.saveToStream`` 

391 methods, in particular that that method uses single call to write() 

392 method to save information about single config field, and that call 

393 combines comments string(s) for a field and field path and value. 

394 This class will not work reliably on the "import" strings, so imports 

395 should be disabled by passing ``skipImports=True`` to ``saveToStream()``. 

396 """ 

397 def __init__(self, pattern): 

398 # obey case if pattern isn't lowercase or requests NOIGNORECASE 

399 mat = re.search(r"(.*):NOIGNORECASE$", pattern) 

400 

401 if mat: 

402 pattern = mat.group(1) 

403 self._pattern = re.compile(fnmatch.translate(pattern)) 

404 else: 

405 if pattern != pattern.lower(): 

406 print(f"Matching \"{pattern}\" without regard to case " 

407 "(append :NOIGNORECASE to prevent this)", file=sys.stdout) 

408 self._pattern = re.compile(fnmatch.translate(pattern), re.IGNORECASE) 

409 

410 def write(self, showStr): 

411 # Strip off doc string line(s) and cut off at "=" for string matching 

412 matchStr = showStr.rstrip().split("\n")[-1].split("=")[0] 

413 if self._pattern.search(matchStr): 

414 sys.stdout.write(showStr) 

415 

416# ------------------------ 

417# Exported definitions -- 

418# ------------------------ 

419 

420 

421class CmdLineFwk: 

422 """PipelineTask framework which executes tasks from command line. 

423 

424 In addition to executing tasks this activator provides additional methods 

425 for task management like dumping configuration or execution chain. 

426 """ 

427 

428 MP_TIMEOUT = 9999 # Default timeout (sec) for multiprocessing 

429 

430 def __init__(self): 

431 pass 

432 

433 def makePipeline(self, args): 

434 """Build a pipeline from command line arguments. 

435 

436 Parameters 

437 ---------- 

438 args : `argparse.Namespace` 

439 Parsed command line 

440 

441 Returns 

442 ------- 

443 pipeline : `~lsst.pipe.base.Pipeline` 

444 """ 

445 if args.pipeline: 

446 pipeline = Pipeline.fromFile(args.pipeline) 

447 else: 

448 pipeline = Pipeline("anonymous") 

449 

450 # loop over all pipeline actions and apply them in order 

451 for action in args.pipeline_actions: 

452 if action.action == "add_instrument": 

453 

454 pipeline.addInstrument(action.value) 

455 

456 elif action.action == "new_task": 

457 

458 pipeline.addTask(action.value, action.label) 

459 

460 elif action.action == "delete_task": 

461 

462 pipeline.removeTask(action.label) 

463 

464 elif action.action == "config": 

465 

466 # action value string is "field=value", split it at '=' 

467 field, _, value = action.value.partition("=") 

468 pipeline.addConfigOverride(action.label, field, value) 

469 

470 elif action.action == "configfile": 

471 

472 pipeline.addConfigFile(action.label, action.value) 

473 

474 else: 

475 

476 raise ValueError(f"Unexpected pipeline action: {action.action}") 

477 

478 if args.save_pipeline: 

479 pipeline.toFile(args.save_pipeline) 

480 

481 if args.pipeline_dot: 

482 pipeline2dot(pipeline, args.pipeline_dot) 

483 

484 return pipeline 

485 

486 def makeGraph(self, pipeline, args): 

487 """Build a graph from command line arguments. 

488 

489 Parameters 

490 ---------- 

491 pipeline : `~lsst.pipe.base.Pipeline` 

492 Pipeline, can be empty or ``None`` if graph is read from a file. 

493 args : `argparse.Namespace` 

494 Parsed command line 

495 

496 Returns 

497 ------- 

498 graph : `~lsst.pipe.base.QuantumGraph` or `None` 

499 If resulting graph is empty then `None` is returned. 

500 """ 

501 

502 registry, collections, run = _ButlerFactory.makeRegistryAndCollections(args) 

503 

504 if args.qgraph: 

505 

506 with open(args.qgraph, 'rb') as pickleFile: 

507 qgraph = QuantumGraph.load(pickleFile, registry.dimensions) 

508 

509 # pipeline can not be provided in this case 

510 if pipeline: 

511 raise ValueError("Pipeline must not be given when quantum graph is read from file.") 

512 

513 else: 

514 

515 # make execution plan (a.k.a. DAG) for pipeline 

516 graphBuilder = GraphBuilder(registry, 

517 skipExisting=args.skip_existing) 

518 qgraph = graphBuilder.makeGraph(pipeline, collections, run, args.data_query) 

519 

520 # count quanta in graph and give a warning if it's empty and return None 

521 nQuanta = len(qgraph) 

522 if nQuanta == 0: 

523 warnings.warn("QuantumGraph is empty", stacklevel=2) 

524 return None 

525 else: 

526 _LOG.info("QuantumGraph contains %d quanta for %d tasks", 

527 nQuanta, len(qgraph.taskGraph)) 

528 

529 if args.save_qgraph: 

530 with open(args.save_qgraph, "wb") as pickleFile: 

531 qgraph.save(pickleFile) 

532 

533 if args.save_single_quanta: 

534 for quantumNode in qgraph: 

535 sqgraph = qgraph.subset(quantumNode) 

536 filename = args.save_single_quanta.format(quantumNode.nodeId.number) 

537 with open(filename, "wb") as pickleFile: 

538 sqgraph.save(pickleFile) 

539 

540 if args.qgraph_dot: 

541 graph2dot(qgraph, args.qgraph_dot) 

542 

543 return qgraph 

544 

545 def runPipeline(self, graph, taskFactory, args, butler=None): 

546 """Execute complete QuantumGraph. 

547 

548 Parameters 

549 ---------- 

550 graph : `QuantumGraph` 

551 Execution graph. 

552 taskFactory : `~lsst.pipe.base.TaskFactory` 

553 Task factory 

554 args : `argparse.Namespace` 

555 Parsed command line 

556 butler : `~lsst.daf.butler.Butler`, optional 

557 Data Butler instance, if not defined then new instance is made 

558 using command line options. 

559 """ 

560 # make butler instance 

561 if butler is None: 

562 butler = _ButlerFactory.makeWriteButler(args) 

563 

564 # Enable lsstDebug debugging. Note that this is done once in the 

565 # main process before PreExecInit and it is also repeated before 

566 # running each task in SingleQuantumExecutor (which may not be 

567 # needed if `multipocessing` always uses fork start method). 

568 if args.enableLsstDebug: 

569 try: 

570 _LOG.debug("Will try to import debug.py") 

571 import debug # noqa:F401 

572 except ImportError: 

573 _LOG.warn("No 'debug' module found.") 

574 

575 preExecInit = PreExecInit(butler, taskFactory, args.skip_existing) 

576 preExecInit.initialize(graph, 

577 saveInitOutputs=not args.skip_init_writes, 

578 registerDatasetTypes=args.register_dataset_types, 

579 saveVersions=not args.no_versions) 

580 

581 if not args.init_only: 

582 graphFixup = self._importGraphFixup(args) 

583 quantumExecutor = SingleQuantumExecutor(taskFactory, 

584 skipExisting=args.skip_existing, 

585 clobberPartialOutputs=args.clobber_partial_outputs, 

586 enableLsstDebug=args.enableLsstDebug) 

587 timeout = self.MP_TIMEOUT if args.timeout is None else args.timeout 

588 executor = MPGraphExecutor(numProc=args.processes, timeout=timeout, 

589 startMethod=args.start_method, 

590 quantumExecutor=quantumExecutor, 

591 failFast=args.fail_fast, 

592 executionGraphFixup=graphFixup) 

593 with util.profile(args.profile, _LOG): 

594 executor.execute(graph, butler) 

595 

596 def showInfo(self, args, pipeline, graph=None): 

597 """Display useful info about pipeline and environment. 

598 

599 Parameters 

600 ---------- 

601 args : `argparse.Namespace` 

602 Parsed command line 

603 pipeline : `Pipeline` 

604 Pipeline definition 

605 graph : `QuantumGraph`, optional 

606 Execution graph 

607 """ 

608 showOpts = args.show 

609 for what in showOpts: 

610 showCommand, _, showArgs = what.partition("=") 

611 

612 if showCommand in ["pipeline", "config", "history", "tasks"]: 

613 if not pipeline: 

614 _LOG.warning("Pipeline is required for --show=%s", showCommand) 

615 continue 

616 

617 if showCommand in ["graph", "workflow", "uri"]: 

618 if not graph: 

619 _LOG.warning("QuantumGraph is required for --show=%s", showCommand) 

620 continue 

621 

622 if showCommand == "pipeline": 

623 print(pipeline) 

624 elif showCommand == "config": 

625 self._showConfig(pipeline, showArgs, False) 

626 elif showCommand == "dump-config": 

627 self._showConfig(pipeline, showArgs, True) 

628 elif showCommand == "history": 

629 self._showConfigHistory(pipeline, showArgs) 

630 elif showCommand == "tasks": 

631 self._showTaskHierarchy(pipeline) 

632 elif showCommand == "graph": 

633 if graph: 

634 self._showGraph(graph) 

635 elif showCommand == "uri": 

636 if graph: 

637 self._showUri(graph, args) 

638 elif showCommand == "workflow": 

639 if graph: 

640 self._showWorkflow(graph, args) 

641 else: 

642 print("Unknown value for show: %s (choose from '%s')" % 

643 (what, "', '".join("pipeline config[=XXX] history=XXX tasks graph".split())), 

644 file=sys.stderr) 

645 sys.exit(1) 

646 

647 def _showConfig(self, pipeline, showArgs, dumpFullConfig): 

648 """Show task configuration 

649 

650 Parameters 

651 ---------- 

652 pipeline : `Pipeline` 

653 Pipeline definition 

654 showArgs : `str` 

655 Defines what to show 

656 dumpFullConfig : `bool` 

657 If true then dump complete task configuration with all imports. 

658 """ 

659 stream = sys.stdout 

660 if dumpFullConfig: 

661 # Task label can be given with this option 

662 taskName = showArgs 

663 else: 

664 # The argument can have form [TaskLabel::][pattern:NOIGNORECASE] 

665 matConfig = re.search(r"^(?:(\w+)::)?(?:config.)?(.+)?", showArgs) 

666 taskName = matConfig.group(1) 

667 pattern = matConfig.group(2) 

668 if pattern: 

669 stream = _FilteredStream(pattern) 

670 

671 tasks = util.filterTasks(pipeline, taskName) 

672 if not tasks: 

673 print("Pipeline has no tasks named {}".format(taskName), file=sys.stderr) 

674 sys.exit(1) 

675 

676 for taskDef in tasks: 

677 print("### Configuration for task `{}'".format(taskDef.label)) 

678 taskDef.config.saveToStream(stream, root="config", skipImports=not dumpFullConfig) 

679 

680 def _showConfigHistory(self, pipeline, showArgs): 

681 """Show history for task configuration 

682 

683 Parameters 

684 ---------- 

685 pipeline : `Pipeline` 

686 Pipeline definition 

687 showArgs : `str` 

688 Defines what to show 

689 """ 

690 

691 taskName = None 

692 pattern = None 

693 matHistory = re.search(r"^(?:(\w+)::)?(?:config[.])?(.+)", showArgs) 

694 if matHistory: 

695 taskName = matHistory.group(1) 

696 pattern = matHistory.group(2) 

697 if not pattern: 

698 print("Please provide a value with --show history (e.g. history=Task::param)", file=sys.stderr) 

699 sys.exit(1) 

700 

701 tasks = util.filterTasks(pipeline, taskName) 

702 if not tasks: 

703 print(f"Pipeline has no tasks named {taskName}", file=sys.stderr) 

704 sys.exit(1) 

705 

706 found = False 

707 for taskDef in tasks: 

708 

709 config = taskDef.config 

710 

711 # Look for any matches in the config hierarchy for this name 

712 for nmatch, thisName in enumerate(fnmatch.filter(config.names(), pattern)): 

713 if nmatch > 0: 

714 print("") 

715 

716 cpath, _, cname = thisName.rpartition(".") 

717 try: 

718 if not cpath: 

719 # looking for top-level field 

720 hconfig = taskDef.config 

721 else: 

722 hconfig = eval("config." + cpath, {}, {"config": config}) 

723 except AttributeError: 

724 print(f"Error: Unable to extract attribute {cpath} from task {taskDef.label}", 

725 file=sys.stderr) 

726 hconfig = None 

727 

728 # Sometimes we end up with a non-Config so skip those 

729 if isinstance(hconfig, (pexConfig.Config, pexConfig.ConfigurableInstance)) and \ 

730 hasattr(hconfig, cname): 

731 print(f"### Configuration field for task `{taskDef.label}'") 

732 print(pexConfig.history.format(hconfig, cname)) 

733 found = True 

734 

735 if not found: 

736 print(f"None of the tasks has field matching {pattern}", file=sys.stderr) 

737 sys.exit(1) 

738 

739 def _showTaskHierarchy(self, pipeline): 

740 """Print task hierarchy to stdout 

741 

742 Parameters 

743 ---------- 

744 pipeline: `Pipeline` 

745 """ 

746 for taskDef in pipeline.toExpandedPipeline(): 

747 print("### Subtasks for task `{}'".format(taskDef.taskName)) 

748 

749 for configName, taskName in util.subTaskIter(taskDef.config): 

750 print("{}: {}".format(configName, taskName)) 

751 

752 def _showGraph(self, graph): 

753 """Print quanta information to stdout 

754 

755 Parameters 

756 ---------- 

757 graph : `QuantumGraph` 

758 Execution graph. 

759 """ 

760 for taskNode in graph.taskGraph: 

761 print(taskNode) 

762 

763 for iq, quantum in enumerate(graph.getQuantaForTask(taskNode)): 

764 print(" Quantum {}:".format(iq)) 

765 print(" inputs:") 

766 for key, refs in quantum.inputs.items(): 

767 dataIds = ["DataId({})".format(ref.dataId) for ref in refs] 

768 print(" {}: [{}]".format(key, ", ".join(dataIds))) 

769 print(" outputs:") 

770 for key, refs in quantum.outputs.items(): 

771 dataIds = ["DataId({})".format(ref.dataId) for ref in refs] 

772 print(" {}: [{}]".format(key, ", ".join(dataIds))) 

773 

774 def _showWorkflow(self, graph, args): 

775 """Print quanta information and dependency to stdout 

776 

777 Parameters 

778 ---------- 

779 graph : `QuantumGraph` 

780 Execution graph. 

781 args : `argparse.Namespace` 

782 Parsed command line 

783 """ 

784 for node in graph: 

785 print(f"Quantum {node.nodeId.number}: {node.taskDef.taskName}") 

786 for parent in graph.determineInputsToQuantumNode(node): 

787 print(f"Parent Quantum {parent.nodeId.number} - Child Quantum {node.nodeId.number}") 

788 

789 def _showUri(self, graph, args): 

790 """Print input and predicted output URIs to stdout 

791 

792 Parameters 

793 ---------- 

794 graph : `QuantumGraph` 

795 Execution graph 

796 args : `argparse.Namespace` 

797 Parsed command line 

798 """ 

799 def dumpURIs(thisRef): 

800 primary, components = butler.getURIs(thisRef, predict=True, run="TBD") 

801 if primary: 

802 print(f" {primary}") 

803 else: 

804 print(" (disassembled artifact)") 

805 for compName, compUri in components.items(): 

806 print(f" {compName}: {compUri}") 

807 

808 butler = _ButlerFactory.makeReadButler(args) 

809 for node in graph: 

810 print(f"Quantum {node.nodeId.number}: {node.taskDef.taskName}") 

811 print(" inputs:") 

812 for key, refs in node.quantum.inputs.items(): 

813 for ref in refs: 

814 dumpURIs(ref) 

815 print(" outputs:") 

816 for key, refs in node.quantum.outputs.items(): 

817 for ref in refs: 

818 dumpURIs(ref) 

819 

820 def _importGraphFixup(self, args): 

821 """Import/instantiate graph fixup object. 

822 

823 Parameters 

824 ---------- 

825 args : `argparse.Namespace` 

826 Parsed command line. 

827 

828 Returns 

829 ------- 

830 fixup : `ExecutionGraphFixup` or `None` 

831 

832 Raises 

833 ------ 

834 ValueError 

835 Raised if import fails, method call raises exception, or returned 

836 instance has unexpected type. 

837 """ 

838 if args.graph_fixup: 

839 try: 

840 factory = doImport(args.graph_fixup) 

841 except Exception as exc: 

842 raise ValueError("Failed to import graph fixup class/method") from exc 

843 try: 

844 fixup = factory() 

845 except Exception as exc: 

846 raise ValueError("Failed to make instance of graph fixup") from exc 

847 if not isinstance(fixup, ExecutionGraphFixup): 

848 raise ValueError("Graph fixup is not an instance of ExecutionGraphFixup class") 

849 return fixup