Coverage for python/lsst/ctrl/mpexec/cli/opt/options.py: 80%

72 statements  

« prev     ^ index     » next       coverage.py v7.4.1, created at 2024-02-07 12:18 +0000

1# This file is part of ctrl_mpexec. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This software is dual licensed under the GNU General Public License and also 

10# under a 3-clause BSD license. Recipients may choose which of these licenses 

11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, 

12# respectively. If you choose the GPL option then the following text applies 

13# (but note that there is still no warranty even if you opt for BSD instead): 

14# 

15# This program is free software: you can redistribute it and/or modify 

16# it under the terms of the GNU General Public License as published by 

17# the Free Software Foundation, either version 3 of the License, or 

18# (at your option) any later version. 

19# 

20# This program is distributed in the hope that it will be useful, 

21# but WITHOUT ANY WARRANTY; without even the implied warranty of 

22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

23# GNU General Public License for more details. 

24# 

25# You should have received a copy of the GNU General Public License 

26# along with this program. If not, see <http://www.gnu.org/licenses/>. 

27 

28from __future__ import annotations 

29 

30from collections.abc import Iterable, Mapping 

31 

32import click 

33from lsst.daf.butler.cli.utils import MWOptionDecorator, MWPath, split_commas, unwrap 

34from lsst.utils.doImport import doImportType 

35 

36butler_config_option = MWOptionDecorator( 

37 "-b", "--butler-config", help="Location of the gen3 butler/registry config file." 

38) 

39 

40 

41data_query_option = MWOptionDecorator( 

42 "-d", "--data-query", help="User data selection expression.", metavar="QUERY" 

43) 

44 

45 

46debug_option = MWOptionDecorator( 

47 "--debug", help="Enable debugging output using lsstDebug facility (imports debug.py).", is_flag=True 

48) 

49 

50coverage_option = MWOptionDecorator("--coverage", help="Enable coverage output.", is_flag=True) 

51 

52coverage_report_option = MWOptionDecorator( 

53 "--cov-report/--no-cov-report", 

54 help="If coverage is enabled, controls whether to produce an HTML coverage report.", 

55 default=True, 

56) 

57 

58coverage_packages_option = MWOptionDecorator( 

59 "--cov-packages", 

60 help=unwrap( 

61 """Python packages to restrict coverage to. If none are provided, runs coverage on all packages.""" 

62 ), 

63 multiple=True, 

64 callback=split_commas, 

65) 

66 

67delete_option = MWOptionDecorator( 

68 "--delete", callback=split_commas, help="Delete task with given label from pipeline.", multiple=True 

69) 

70 

71 

72pdb_option = MWOptionDecorator( 

73 "--pdb", 

74 help="Post-mortem debugger to launch for exceptions (defaults to pdb if unspecified; requires a tty).", 

75 is_flag=False, 

76 flag_value="pdb", 

77 default=None, 

78) 

79 

80 

81extend_run_option = MWOptionDecorator( 

82 "--extend-run", 

83 help=( 

84 "Instead of creating a new RUN collection, insert datasets into either the one given by " 

85 "--output-run (if provided) or the first child collection of --output (which must be of type RUN). " 

86 "This also enables --skip-existing option when building a graph. " 

87 "When executing a graph this option skips quanta with all existing outputs." 

88 ), 

89 is_flag=True, 

90) 

91 

92 

93graph_fixup_option = MWOptionDecorator( 

94 "--graph-fixup", 

95 help="Name of the class or factory method which makes an instance used for execution graph fixup.", 

96) 

97 

98 

99init_only_option = MWOptionDecorator( 

100 "--init-only", 

101 help="Do not actually run; just register dataset types and/or save init outputs.", 

102 is_flag=True, 

103) 

104 

105 

106input_option = MWOptionDecorator( 

107 "-i", 

108 "--input", 

109 callback=split_commas, 

110 default=[], 

111 help="Comma-separated names of the input collection(s).", 

112 metavar="COLLECTION", 

113 multiple=True, 

114) 

115 

116 

117rebase_option = MWOptionDecorator( 

118 "--rebase", 

119 help=unwrap("""Reset output collection chain if it is inconsistent with --inputs"""), 

120 is_flag=True, 

121) 

122 

123 

124no_versions_option = MWOptionDecorator( 

125 "--no-versions", help="Do not save or check package versions.", is_flag=True 

126) 

127 

128 

129order_pipeline_option = MWOptionDecorator( 

130 "--order-pipeline", 

131 help=unwrap( 

132 """Order tasks in pipeline based on their data 

133 dependencies, ordering is performed as last step before saving or 

134 executing pipeline.""" 

135 ), 

136 is_flag=True, 

137) 

138 

139 

140output_option = MWOptionDecorator( 

141 "-o", 

142 "--output", 

143 help=unwrap( 

144 """Name of the output CHAINED collection. This may either be an 

145 existing CHAINED collection to use as both input and output 

146 (incompatible with --input), or a new CHAINED collection created 

147 to include all inputs (requires --input). In both cases, the 

148 collection's children will start with an output RUN collection 

149 that directly holds all new datasets (see --output-run).""" 

150 ), 

151 metavar="COLL", 

152) 

153 

154 

155output_run_option = MWOptionDecorator( 

156 "--output-run", 

157 help=unwrap( 

158 """Name of the new output RUN collection. If not provided 

159 then --output must be provided and a new RUN collection will 

160 be created by appending a timestamp to the value passed with 

161 --output. If this collection already exists then 

162 --extend-run must be passed.""" 

163 ), 

164 metavar="COLL", 

165) 

166 

167 

168pipeline_option = MWOptionDecorator( 

169 "-p", 

170 "--pipeline", 

171 help="Location of a pipeline definition file in YAML format.", 

172 type=MWPath(file_okay=True, dir_okay=False, readable=True), 

173) 

174 

175 

176pipeline_dot_option = MWOptionDecorator( 

177 "--pipeline-dot", 

178 help="Location for storing GraphViz DOT representation of a pipeline.", 

179 type=MWPath(writable=True, file_okay=True, dir_okay=False), 

180) 

181 

182 

183profile_option = MWOptionDecorator( 

184 "--profile", help="Dump cProfile statistics to file name.", type=MWPath(file_okay=True, dir_okay=False) 

185) 

186 

187 

188prune_replaced_option = MWOptionDecorator( 

189 "--prune-replaced", 

190 help=unwrap( 

191 """Delete the datasets in the collection replaced by 

192 --replace-run, either just from the datastore 

193 ('unstore') or by removing them and the RUN completely 

194 ('purge'). Requires --replace-run.""" 

195 ), 

196 type=click.Choice(choices=("unstore", "purge"), case_sensitive=False), 

197) 

198 

199 

200qgraph_option = MWOptionDecorator( 

201 "-g", 

202 "--qgraph", 

203 help=unwrap( 

204 """Location for a serialized quantum graph definition (pickle 

205 file). If this option is given then all input data options and 

206 pipeline-building options cannot be used. Can be a URI.""" 

207 ), 

208) 

209 

210 

211qgraph_id_option = MWOptionDecorator( 

212 "--qgraph-id", 

213 help=unwrap( 

214 """Quantum graph identifier, if specified must match the 

215 identifier of the graph loaded from a file. Ignored if graph 

216 is not loaded from a file.""" 

217 ), 

218) 

219 

220 

221qgraph_datastore_records_option = MWOptionDecorator( 

222 "--qgraph-datastore-records", 

223 help=unwrap( 

224 """Include datastore records into generated quantum graph, these records are used by a 

225 quantum-backed butler. 

226 """ 

227 ), 

228 is_flag=True, 

229) 

230 

231 

232# I wanted to use default=None here to match Python API but click silently 

233# replaces None with an empty tuple when multiple=True. 

234qgraph_node_id_option = MWOptionDecorator( 

235 "--qgraph-node-id", 

236 callback=split_commas, 

237 multiple=True, 

238 help=unwrap( 

239 """Only load a specified set of nodes when graph is 

240 loaded from a file, nodes are identified by UUID 

241 values. One or more comma-separated integers are 

242 accepted. By default all nodes are loaded. Ignored if 

243 graph is not loaded from a file.""" 

244 ), 

245) 

246 

247qgraph_header_data_option = MWOptionDecorator( 

248 "--show-qgraph-header", 

249 is_flag=True, 

250 default=False, 

251 help="Print the headerData for Quantum Graph to the console", 

252) 

253 

254qgraph_dot_option = MWOptionDecorator( 

255 "--qgraph-dot", 

256 help="Location for storing GraphViz DOT representation of a quantum graph.", 

257 type=MWPath(writable=True, file_okay=True, dir_okay=False), 

258) 

259 

260 

261replace_run_option = MWOptionDecorator( 

262 "--replace-run", 

263 help=unwrap( 

264 """Before creating a new RUN collection in an existing 

265 CHAINED collection, remove the first child collection 

266 (which must be of type RUN). This can be used to repeatedly 

267 write to the same (parent) collection during development, 

268 but it does not delete the datasets associated with the 

269 replaced run unless --prune-replaced is also passed. 

270 Requires --output, and incompatible with --extend-run.""" 

271 ), 

272 is_flag=True, 

273) 

274 

275 

276save_pipeline_option = MWOptionDecorator( 

277 "-s", 

278 "--save-pipeline", 

279 help="Location for storing resulting pipeline definition in YAML format.", 

280 type=MWPath(dir_okay=False, file_okay=True, writable=True), 

281) 

282 

283save_qgraph_option = MWOptionDecorator( 

284 "-q", 

285 "--save-qgraph", 

286 help="URI location for storing a serialized quantum graph definition (pickle file).", 

287) 

288 

289 

290save_single_quanta_option = MWOptionDecorator( 

291 "--save-single-quanta", 

292 help=unwrap( 

293 """Format string of locations for storing individual 

294 quantum graph definition (pickle files). The curly 

295 brace {} in the input string will be replaced by a 

296 quantum number. Can be a URI.""" 

297 ), 

298) 

299 

300 

301show_option = MWOptionDecorator( 

302 "--show", 

303 callback=split_commas, 

304 help=unwrap( 

305 """Dump various info to standard output. Possible items are: 

306 ``config``, ``config=[Task::]<PATTERN>`` or 

307 ``config=[Task::]<PATTERN>:NOIGNORECASE`` to dump configuration 

308 fields possibly matching given pattern and/or task label; 

309 ``history=<FIELD>`` to dump configuration history for a field, 

310 field name is specified as ``[Task::]<PATTERN>``; ``dump-config``, 

311 ``dump-config=Task`` to dump complete configuration for a task 

312 given its label or all tasks; ``pipeline`` to show pipeline 

313 composition; ``graph`` to show information about quanta; 

314 ``workflow`` to show information about quanta and their 

315 dependency; ``tasks`` to show task composition; ``uri`` to show 

316 predicted dataset URIs of quanta; ``pipeline-graph`` for a 

317 text-based visualization of the pipeline (tasks and dataset types); 

318 ``task-graph`` for a text-based visualization of just the tasks. 

319 With -b, pipeline-graph and task-graph include additional information. 

320 """ 

321 ), 

322 metavar="ITEM|ITEM=VALUE", 

323 multiple=True, 

324) 

325 

326 

327skip_existing_in_option = MWOptionDecorator( 

328 "--skip-existing-in", 

329 callback=split_commas, 

330 default=None, 

331 metavar="COLLECTION", 

332 multiple=True, 

333 help=unwrap( 

334 """If all Quantum outputs already exist in the specified list of 

335 collections then that Quantum will be excluded from the QuantumGraph. 

336 """ 

337 ), 

338) 

339 

340 

341skip_existing_option = MWOptionDecorator( 

342 "--skip-existing", 

343 is_flag=True, 

344 help=unwrap( 

345 """This option is equivalent to --skip-existing-in with the name of 

346 the output RUN collection. If both --skip-existing-in and 

347 --skip-existing are given then output RUN collection is appended to 

348 the list of collections.""" 

349 ), 

350) 

351 

352 

353clobber_outputs_option = MWOptionDecorator( 

354 "--clobber-outputs", 

355 help=( 

356 "Remove outputs of failed quanta from the output run when they would block the execution of new " 

357 "quanta with the same data ID (or assume that this will be done, if just building a QuantumGraph). " 

358 "Does nothing if --extend-run is not passed." 

359 ), 

360 is_flag=True, 

361) 

362 

363 

364skip_init_writes_option = MWOptionDecorator( 

365 "--skip-init-writes", 

366 help=unwrap( 

367 """Do not write collection-wide 'init output' datasets 

368 (e.g.schemas).""" 

369 ), 

370 is_flag=True, 

371) 

372 

373 

374enable_implicit_threading_option = MWOptionDecorator( 

375 "--enable-implicit-threading", 

376 help=unwrap( 

377 """Do not disable implicit threading use by third-party libraries (e.g. OpenBLAS). 

378 Implicit threading is always disabled during execution with multiprocessing.""" 

379 ), 

380 is_flag=True, 

381) 

382 

383cores_per_quantum_option = MWOptionDecorator( 

384 "-n", 

385 "--cores-per-quantum", 

386 default=1, 

387 help=unwrap( 

388 """Number of cores available to each quantum when executing. 

389 If '-j' is used each subprocess will be allowed to use this number of cores.""" 

390 ), 

391 type=click.IntRange(min=1), 

392) 

393 

394memory_per_quantum_option = MWOptionDecorator( 

395 "--memory-per-quantum", 

396 default="", 

397 help=unwrap( 

398 """Memory allocated for each quantum to use when executing. 

399 This memory allocation is not enforced by the execution system and is purely advisory. 

400 If '-j' used each subprocess will be allowed to use this amount of memory. 

401 Units are allowed and the default units for a plain integer are MB. 

402 For example: '3GB', '3000MB' and '3000' would all result in the same 

403 memory limit. Default is for no limit.""" 

404 ), 

405 type=str, 

406) 

407 

408task_option = MWOptionDecorator( 

409 "-t", 

410 "--task", 

411 callback=split_commas, 

412 help=unwrap( 

413 """Task name to add to pipeline, must be a fully qualified task 

414 name. Task name can be followed by colon and label name, if label 

415 is not given then task base name (class name) is used as 

416 label.""" 

417 ), 

418 metavar="TASK[:LABEL]", 

419 multiple=True, 

420) 

421 

422 

423timeout_option = MWOptionDecorator( 

424 "--timeout", type=click.IntRange(min=0), help="Timeout for multiprocessing; maximum wall time (sec)." 

425) 

426 

427 

428start_method_option = MWOptionDecorator( 

429 "--start-method", 

430 default=None, 

431 type=click.Choice(choices=["spawn", "fork", "forkserver"]), 

432 help=( 

433 "Multiprocessing start method, default is platform-specific. " 

434 "Fork method is no longer supported, spawn is used instead if fork is selected." 

435 ), 

436) 

437 

438 

439fail_fast_option = MWOptionDecorator( 

440 "--fail-fast", 

441 help="Stop processing at first error, default is to process as many tasks as possible.", 

442 is_flag=True, 

443) 

444 

445save_execution_butler_option = MWOptionDecorator( 

446 "--save-execution-butler", 

447 help="Export location for an execution-specific butler after making QuantumGraph", 

448) 

449 

450mock_option = MWOptionDecorator( 

451 "--mock", 

452 help="Mock pipeline execution.", 

453 is_flag=True, 

454) 

455 

456unmocked_dataset_types_option = MWOptionDecorator( 

457 "--unmocked-dataset-types", 

458 callback=split_commas, 

459 default=None, 

460 metavar="COLLECTION", 

461 multiple=True, 

462 help="Names of input dataset types that should not be mocked.", 

463) 

464 

465 

466def parse_mock_failure( 

467 ctx: click.Context, param: click.Option, value: Iterable[str] | None 

468) -> Mapping[str, tuple[str, type[Exception] | None]]: 

469 """Parse the --mock-failure option values into the mapping accepted by 

470 `~lsst.pipe.base.tests.mocks.mock_task_defs`. 

471 

472 Parameters 

473 ---------- 

474 ctx : `click.Context` 

475 Context provided by Click. 

476 param : `click.Option` 

477 Click option. 

478 value : `~collections.abc.Iterable` [`str`] or `None` 

479 Value from option. 

480 """ 

481 result: dict[str, tuple[str, type[Exception] | None]] = {} 

482 if value is None: 

483 return result 

484 for entry in value: 

485 try: 

486 task_label, error_type_name, where = entry.split(":", 2) 

487 except ValueError: 

488 raise click.UsageError( 

489 f"Invalid value for --mock-failure option: {entry!r}; " 

490 "expected a string of the form 'task:error:where'." 

491 ) from None 

492 error_type = doImportType(error_type_name) if error_type_name else None 

493 result[task_label] = (where, error_type) 

494 return result 

495 

496 

497mock_failure_option = MWOptionDecorator( 

498 "--mock-failure", 

499 callback=parse_mock_failure, 

500 metavar="LABEL:EXCEPTION:WHERE", 

501 default=None, 

502 multiple=True, 

503 help=unwrap( 

504 """Specifications for tasks that should be configured to fail 

505 when mocking execution. This is a colon-separated 3-tuple, where the 

506 first entry the task label, the second the fully-qualified exception 

507 type (empty for ValueError, and the third a string (which typically 

508 needs to be quoted to be passed as one argument value by the shell) of 

509 the form passed to --where, indicating which data IDs should fail.""" 

510 ), 

511) 

512 

513 

514clobber_execution_butler_option = MWOptionDecorator( 

515 "--clobber-execution-butler", 

516 help=unwrap( 

517 """When creating execution butler overwrite 

518 any existing products""" 

519 ), 

520 is_flag=True, 

521) 

522 

523target_datastore_root_option = MWOptionDecorator( 

524 "--target-datastore-root", 

525 help=unwrap( 

526 """Root directory for datastore of execution butler. 

527 Default is to use the original datastore. 

528 """ 

529 ), 

530) 

531 

532dataset_query_constraint = MWOptionDecorator( 

533 "--dataset-query-constraint", 

534 help=unwrap( 

535 """When constructing a quantum graph constrain by 

536 pre-existence of specified dataset types. Valid 

537 values are `all` for all inputs dataset types in 

538 pipeline, ``off`` to not consider dataset type 

539 existence as a constraint, single or comma 

540 separated list of dataset type names.""" 

541 ), 

542 default="all", 

543) 

544 

545summary_option = MWOptionDecorator( 

546 "--summary", 

547 help=( 

548 "Location for storing job summary (JSON file). Note that the" 

549 " structure of this file may not be stable." 

550 ), 

551 type=MWPath(dir_okay=False, file_okay=True, writable=True), 

552) 

553 

554 

555recursive_option = MWOptionDecorator( 

556 "--recursive", 

557 is_flag=True, 

558) 

559 

560config_search_path_option = MWOptionDecorator( 

561 "--config-search-path", 

562 callback=split_commas, 

563 default=[], 

564 help="Additional search paths for butler configuration.", 

565 metavar="PATH", 

566 multiple=True, 

567) 

568 

569update_graph_id_option = MWOptionDecorator( 

570 "--update-graph-id", 

571 help=unwrap("Update graph ID with new unique value."), 

572 is_flag=True, 

573) 

574 

575metadata_run_key_option = MWOptionDecorator( 

576 "--metadata-run-key", 

577 help=( 

578 "Quantum graph metadata key for the name of the output run. " 

579 "Empty string disables update of the metadata. " 

580 "Default value: output_run." 

581 ), 

582 default="output_run", 

583)