Coverage for python/lsst/ctrl/mpexec/cli/opt/options.py: 75%

77 statements  

« prev     ^ index     » next       coverage.py v7.5.1, created at 2024-05-07 09:59 +0000

1# This file is part of ctrl_mpexec. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This software is dual licensed under the GNU General Public License and also 

10# under a 3-clause BSD license. Recipients may choose which of these licenses 

11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, 

12# respectively. If you choose the GPL option then the following text applies 

13# (but note that there is still no warranty even if you opt for BSD instead): 

14# 

15# This program is free software: you can redistribute it and/or modify 

16# it under the terms of the GNU General Public License as published by 

17# the Free Software Foundation, either version 3 of the License, or 

18# (at your option) any later version. 

19# 

20# This program is distributed in the hope that it will be useful, 

21# but WITHOUT ANY WARRANTY; without even the implied warranty of 

22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

23# GNU General Public License for more details. 

24# 

25# You should have received a copy of the GNU General Public License 

26# along with this program. If not, see <http://www.gnu.org/licenses/>. 

27 

28from __future__ import annotations 

29 

30from collections.abc import Iterable, Mapping 

31from typing import TYPE_CHECKING 

32 

33import click 

34from lsst.daf.butler.cli.utils import MWOptionDecorator, MWPath, split_commas, unwrap 

35from lsst.utils.doImport import doImportType 

36 

37if TYPE_CHECKING: 

38 # Avoid regular module-scope import of test-only code that tinkers with the 

39 # storage class singleton. 

40 from lsst.pipe.base.tests.mocks import ForcedFailure 

41 

42 

43butler_config_option = MWOptionDecorator( 

44 "-b", "--butler-config", help="Location of the gen3 butler/registry config file." 

45) 

46 

47 

48data_query_option = MWOptionDecorator( 

49 "-d", "--data-query", help="User data selection expression.", metavar="QUERY" 

50) 

51 

52 

53debug_option = MWOptionDecorator( 

54 "--debug", help="Enable debugging output using lsstDebug facility (imports debug.py).", is_flag=True 

55) 

56 

57coverage_option = MWOptionDecorator("--coverage", help="Enable coverage output.", is_flag=True) 

58 

59coverage_report_option = MWOptionDecorator( 

60 "--cov-report/--no-cov-report", 

61 help="If coverage is enabled, controls whether to produce an HTML coverage report.", 

62 default=True, 

63) 

64 

65coverage_packages_option = MWOptionDecorator( 

66 "--cov-packages", 

67 help=unwrap( 

68 """Python packages to restrict coverage to. If none are provided, runs coverage on all packages.""" 

69 ), 

70 multiple=True, 

71 callback=split_commas, 

72) 

73 

74delete_option = MWOptionDecorator( 

75 "--delete", callback=split_commas, help="Delete task with given label from pipeline.", multiple=True 

76) 

77 

78 

79pdb_option = MWOptionDecorator( 

80 "--pdb", 

81 help="Post-mortem debugger to launch for exceptions (defaults to pdb if unspecified; requires a tty).", 

82 is_flag=False, 

83 flag_value="pdb", 

84 default=None, 

85) 

86 

87 

88extend_run_option = MWOptionDecorator( 

89 "--extend-run", 

90 help=( 

91 "Instead of creating a new RUN collection, insert datasets into either the one given by " 

92 "--output-run (if provided) or the first child collection of --output (which must be of type RUN). " 

93 "This also enables --skip-existing option when building a graph. " 

94 "When executing a graph this option skips quanta with all existing outputs." 

95 ), 

96 is_flag=True, 

97) 

98 

99 

100graph_fixup_option = MWOptionDecorator( 

101 "--graph-fixup", 

102 help="Name of the class or factory method which makes an instance used for execution graph fixup.", 

103) 

104 

105 

106init_only_option = MWOptionDecorator( 

107 "--init-only", 

108 help="Do not actually run; just register dataset types and/or save init outputs.", 

109 is_flag=True, 

110) 

111 

112 

113input_option = MWOptionDecorator( 

114 "-i", 

115 "--input", 

116 callback=split_commas, 

117 default=[], 

118 help="Comma-separated names of the input collection(s).", 

119 metavar="COLLECTION", 

120 multiple=True, 

121) 

122 

123 

124rebase_option = MWOptionDecorator( 

125 "--rebase", 

126 help=unwrap("""Reset output collection chain if it is inconsistent with --inputs"""), 

127 is_flag=True, 

128) 

129 

130 

131no_versions_option = MWOptionDecorator( 

132 "--no-versions", help="Do not save or check package versions.", is_flag=True 

133) 

134 

135 

136order_pipeline_option = MWOptionDecorator( 

137 "--order-pipeline", 

138 help=unwrap( 

139 """Order tasks in pipeline based on their data 

140 dependencies, ordering is performed as last step before saving or 

141 executing pipeline.""" 

142 ), 

143 is_flag=True, 

144) 

145 

146 

147output_option = MWOptionDecorator( 

148 "-o", 

149 "--output", 

150 help=unwrap( 

151 """Name of the output CHAINED collection. This may either be an 

152 existing CHAINED collection to use as both input and output 

153 (incompatible with --input), or a new CHAINED collection created 

154 to include all inputs (requires --input). In both cases, the 

155 collection's children will start with an output RUN collection 

156 that directly holds all new datasets (see --output-run).""" 

157 ), 

158 metavar="COLL", 

159) 

160 

161 

162output_run_option = MWOptionDecorator( 

163 "--output-run", 

164 help=unwrap( 

165 """Name of the new output RUN collection. If not provided 

166 then --output must be provided and a new RUN collection will 

167 be created by appending a timestamp to the value passed with 

168 --output. If this collection already exists then 

169 --extend-run must be passed.""" 

170 ), 

171 metavar="COLL", 

172) 

173 

174 

175pipeline_option = MWOptionDecorator( 

176 "-p", 

177 "--pipeline", 

178 help="Location of a pipeline definition file in YAML format.", 

179 type=MWPath(file_okay=True, dir_okay=False, readable=True), 

180) 

181 

182 

183pipeline_dot_option = MWOptionDecorator( 

184 "--pipeline-dot", 

185 help="Location for storing GraphViz DOT representation of a pipeline.", 

186 type=MWPath(writable=True, file_okay=True, dir_okay=False), 

187) 

188 

189 

190profile_option = MWOptionDecorator( 

191 "--profile", help="Dump cProfile statistics to file name.", type=MWPath(file_okay=True, dir_okay=False) 

192) 

193 

194 

195prune_replaced_option = MWOptionDecorator( 

196 "--prune-replaced", 

197 help=unwrap( 

198 """Delete the datasets in the collection replaced by 

199 --replace-run, either just from the datastore 

200 ('unstore') or by removing them and the RUN completely 

201 ('purge'). Requires --replace-run.""" 

202 ), 

203 type=click.Choice(choices=("unstore", "purge"), case_sensitive=False), 

204) 

205 

206 

207qgraph_option = MWOptionDecorator( 

208 "-g", 

209 "--qgraph", 

210 help=unwrap( 

211 """Location for a serialized quantum graph definition (pickle 

212 file). If this option is given then all input data options and 

213 pipeline-building options cannot be used. Can be a URI.""" 

214 ), 

215) 

216 

217 

218qgraph_id_option = MWOptionDecorator( 

219 "--qgraph-id", 

220 help=unwrap( 

221 """Quantum graph identifier, if specified must match the 

222 identifier of the graph loaded from a file. Ignored if graph 

223 is not loaded from a file.""" 

224 ), 

225) 

226 

227 

228qgraph_datastore_records_option = MWOptionDecorator( 

229 "--qgraph-datastore-records", 

230 help=unwrap( 

231 """Include datastore records into generated quantum graph, these records are used by a 

232 quantum-backed butler. 

233 """ 

234 ), 

235 is_flag=True, 

236) 

237 

238 

239# I wanted to use default=None here to match Python API but click silently 

240# replaces None with an empty tuple when multiple=True. 

241qgraph_node_id_option = MWOptionDecorator( 

242 "--qgraph-node-id", 

243 callback=split_commas, 

244 multiple=True, 

245 help=unwrap( 

246 """Only load a specified set of nodes when graph is 

247 loaded from a file, nodes are identified by UUID 

248 values. One or more comma-separated integers are 

249 accepted. By default all nodes are loaded. Ignored if 

250 graph is not loaded from a file.""" 

251 ), 

252) 

253 

254qgraph_header_data_option = MWOptionDecorator( 

255 "--show-qgraph-header", 

256 is_flag=True, 

257 default=False, 

258 help="Print the headerData for Quantum Graph to the console", 

259) 

260 

261qgraph_dot_option = MWOptionDecorator( 

262 "--qgraph-dot", 

263 help="Location for storing GraphViz DOT representation of a quantum graph.", 

264 type=MWPath(writable=True, file_okay=True, dir_okay=False), 

265) 

266 

267 

268replace_run_option = MWOptionDecorator( 

269 "--replace-run", 

270 help=unwrap( 

271 """Before creating a new RUN collection in an existing 

272 CHAINED collection, remove the first child collection 

273 (which must be of type RUN). This can be used to repeatedly 

274 write to the same (parent) collection during development, 

275 but it does not delete the datasets associated with the 

276 replaced run unless --prune-replaced is also passed. 

277 Requires --output, and incompatible with --extend-run.""" 

278 ), 

279 is_flag=True, 

280) 

281 

282 

283save_pipeline_option = MWOptionDecorator( 

284 "-s", 

285 "--save-pipeline", 

286 help="Location for storing resulting pipeline definition in YAML format.", 

287 type=MWPath(dir_okay=False, file_okay=True, writable=True), 

288) 

289 

290save_qgraph_option = MWOptionDecorator( 

291 "-q", 

292 "--save-qgraph", 

293 help="URI location for storing a serialized quantum graph definition (pickle file).", 

294) 

295 

296 

297save_single_quanta_option = MWOptionDecorator( 

298 "--save-single-quanta", 

299 help=unwrap( 

300 """Format string of locations for storing individual 

301 quantum graph definition (pickle files). The curly 

302 brace {} in the input string will be replaced by a 

303 quantum number. Can be a URI.""" 

304 ), 

305) 

306 

307 

308show_option = MWOptionDecorator( 

309 "--show", 

310 callback=split_commas, 

311 help=unwrap( 

312 """Dump various info to standard output. Possible items are: 

313 ``config``, ``config=[Task::]<PATTERN>`` or 

314 ``config=[Task::]<PATTERN>:NOIGNORECASE`` to dump configuration 

315 fields possibly matching given pattern and/or task label; 

316 ``history=<FIELD>`` to dump configuration history for a field, 

317 field name is specified as ``[Task::]<PATTERN>``; ``dump-config``, 

318 ``dump-config=Task`` to dump complete configuration for a task 

319 given its label or all tasks; ``pipeline`` to show pipeline 

320 composition; ``graph`` to show information about quanta; 

321 ``workflow`` to show information about quanta and their 

322 dependency; ``tasks`` to show task composition; ``uri`` to show 

323 predicted dataset URIs of quanta; ``pipeline-graph`` for a 

324 text-based visualization of the pipeline (tasks and dataset types); 

325 ``task-graph`` for a text-based visualization of just the tasks. 

326 With -b, pipeline-graph and task-graph include additional information. 

327 """ 

328 ), 

329 metavar="ITEM|ITEM=VALUE", 

330 multiple=True, 

331) 

332 

333 

334skip_existing_in_option = MWOptionDecorator( 

335 "--skip-existing-in", 

336 callback=split_commas, 

337 default=None, 

338 metavar="COLLECTION", 

339 multiple=True, 

340 help=unwrap( 

341 """If all Quantum outputs already exist in the specified list of 

342 collections then that Quantum will be excluded from the QuantumGraph. 

343 """ 

344 ), 

345) 

346 

347 

348skip_existing_option = MWOptionDecorator( 

349 "--skip-existing", 

350 is_flag=True, 

351 help=unwrap( 

352 """This option is equivalent to --skip-existing-in with the name of 

353 the output RUN collection. If both --skip-existing-in and 

354 --skip-existing are given then output RUN collection is appended to 

355 the list of collections.""" 

356 ), 

357) 

358 

359 

360clobber_outputs_option = MWOptionDecorator( 

361 "--clobber-outputs", 

362 help=( 

363 "Remove outputs of failed quanta from the output run when they would block the execution of new " 

364 "quanta with the same data ID (or assume that this will be done, if just building a QuantumGraph). " 

365 "Does nothing if --extend-run is not passed." 

366 ), 

367 is_flag=True, 

368) 

369 

370 

371skip_init_writes_option = MWOptionDecorator( 

372 "--skip-init-writes", 

373 help=unwrap( 

374 """Do not write collection-wide 'init output' datasets 

375 (e.g.schemas).""" 

376 ), 

377 is_flag=True, 

378) 

379 

380 

381enable_implicit_threading_option = MWOptionDecorator( 

382 "--enable-implicit-threading", 

383 help=unwrap( 

384 """Do not disable implicit threading use by third-party libraries (e.g. OpenBLAS). 

385 Implicit threading is always disabled during execution with multiprocessing.""" 

386 ), 

387 is_flag=True, 

388) 

389 

390cores_per_quantum_option = MWOptionDecorator( 

391 "-n", 

392 "--cores-per-quantum", 

393 default=1, 

394 help=unwrap( 

395 """Number of cores available to each quantum when executing. 

396 If '-j' is used each subprocess will be allowed to use this number of cores.""" 

397 ), 

398 type=click.IntRange(min=1), 

399) 

400 

401memory_per_quantum_option = MWOptionDecorator( 

402 "--memory-per-quantum", 

403 default="", 

404 help=unwrap( 

405 """Memory allocated for each quantum to use when executing. 

406 This memory allocation is not enforced by the execution system and is purely advisory. 

407 If '-j' used each subprocess will be allowed to use this amount of memory. 

408 Units are allowed and the default units for a plain integer are MB. 

409 For example: '3GB', '3000MB' and '3000' would all result in the same 

410 memory limit. Default is for no limit.""" 

411 ), 

412 type=str, 

413) 

414 

415task_option = MWOptionDecorator( 

416 "-t", 

417 "--task", 

418 callback=split_commas, 

419 help=unwrap( 

420 """Task name to add to pipeline, must be a fully qualified task 

421 name. Task name can be followed by colon and label name, if label 

422 is not given then task base name (class name) is used as 

423 label.""" 

424 ), 

425 metavar="TASK[:LABEL]", 

426 multiple=True, 

427) 

428 

429 

430timeout_option = MWOptionDecorator( 

431 "--timeout", type=click.IntRange(min=0), help="Timeout for multiprocessing; maximum wall time (sec)." 

432) 

433 

434 

435start_method_option = MWOptionDecorator( 

436 "--start-method", 

437 default=None, 

438 type=click.Choice(choices=["spawn", "fork", "forkserver"]), 

439 help=( 

440 "Multiprocessing start method, default is platform-specific. " 

441 "Fork method is no longer supported, spawn is used instead if fork is selected." 

442 ), 

443) 

444 

445 

446fail_fast_option = MWOptionDecorator( 

447 "--fail-fast", 

448 help="Stop processing at first error, default is to process as many tasks as possible.", 

449 is_flag=True, 

450) 

451 

452save_execution_butler_option = MWOptionDecorator( 

453 "--save-execution-butler", 

454 help="Export location for an execution-specific butler after making QuantumGraph", 

455) 

456 

457mock_option = MWOptionDecorator( 

458 "--mock", 

459 help="Mock pipeline execution.", 

460 is_flag=True, 

461) 

462 

463unmocked_dataset_types_option = MWOptionDecorator( 

464 "--unmocked-dataset-types", 

465 callback=split_commas, 

466 default=None, 

467 metavar="COLLECTION", 

468 multiple=True, 

469 help="Names of input dataset types that should not be mocked.", 

470) 

471 

472 

473def parse_mock_failure( 

474 ctx: click.Context, param: click.Option, value: Iterable[str] | None 

475) -> Mapping[str, ForcedFailure]: 

476 """Parse the --mock-failure option values into the mapping accepted by 

477 `~lsst.pipe.base.tests.mocks.mock_task_defs`. 

478 

479 Parameters 

480 ---------- 

481 ctx : `click.Context` 

482 Context provided by Click. 

483 param : `click.Option` 

484 Click option. 

485 value : `~collections.abc.Iterable` [`str`] or `None` 

486 Value from option. 

487 """ 

488 # Avoid regular module-scope import of test-only code that tinkers with the 

489 # storage class singleton. 

490 from lsst.pipe.base.tests.mocks import ForcedFailure 

491 

492 result: dict[str, ForcedFailure] = {} 

493 if value is None: 

494 return result 

495 for entry in value: 

496 try: 

497 task_label, error_type_name, where, *rest = entry.split(":") 

498 if rest: 

499 (memory_required,) = rest 

500 else: 

501 memory_required = None 

502 except ValueError: 

503 raise click.UsageError( 

504 f"Invalid value for --mock-failure option: {entry!r}; " 

505 "expected a string of the form 'task:error:where[:mem]'." 

506 ) from None 

507 error_type = doImportType(error_type_name) if error_type_name else None 

508 result[task_label] = ForcedFailure(where, error_type, memory_required) 

509 return result 

510 

511 

512mock_failure_option = MWOptionDecorator( 

513 "--mock-failure", 

514 callback=parse_mock_failure, 

515 metavar="LABEL:EXCEPTION:WHERE", 

516 default=None, 

517 multiple=True, 

518 help=unwrap( 

519 """Specifications for tasks that should be configured to fail 

520 when mocking execution. This is a colon-separated 3-tuple or 4-tuple, 

521 where the first entry the task label, the second the fully-qualified 

522 exception type (empty for ValueError, and the third a string (which 

523 typically needs to be quoted to be passed as one argument value by the 

524 shell) of the form passed to --where, indicating which data IDs should 

525 fail. The final optional term is the memory "required" by the task 

526 (with units recognized by astropy), which will cause the error to only 

527 occur if the "available" memory (according to 

528 ExecutionResources.max_mem) is less than this value. Note that actual 

529 memory usage is irrelevant here; this is all mock behavior.""" 

530 ), 

531) 

532 

533 

534clobber_execution_butler_option = MWOptionDecorator( 

535 "--clobber-execution-butler", 

536 help=unwrap( 

537 """When creating execution butler overwrite 

538 any existing products""" 

539 ), 

540 is_flag=True, 

541) 

542 

543target_datastore_root_option = MWOptionDecorator( 

544 "--target-datastore-root", 

545 help=unwrap( 

546 """Root directory for datastore of execution butler. 

547 Default is to use the original datastore. 

548 """ 

549 ), 

550) 

551 

552dataset_query_constraint = MWOptionDecorator( 

553 "--dataset-query-constraint", 

554 help=unwrap( 

555 """When constructing a quantum graph constrain by 

556 pre-existence of specified dataset types. Valid 

557 values are `all` for all inputs dataset types in 

558 pipeline, ``off`` to not consider dataset type 

559 existence as a constraint, single or comma 

560 separated list of dataset type names.""" 

561 ), 

562 default="all", 

563) 

564 

565summary_option = MWOptionDecorator( 

566 "--summary", 

567 help=( 

568 "Location for storing job summary (JSON file). Note that the" 

569 " structure of this file may not be stable." 

570 ), 

571 type=MWPath(dir_okay=False, file_okay=True, writable=True), 

572) 

573 

574 

575recursive_option = MWOptionDecorator( 

576 "--recursive", 

577 is_flag=True, 

578) 

579 

580config_search_path_option = MWOptionDecorator( 

581 "--config-search-path", 

582 callback=split_commas, 

583 default=[], 

584 help="Additional search paths for butler configuration.", 

585 metavar="PATH", 

586 multiple=True, 

587) 

588 

589update_graph_id_option = MWOptionDecorator( 

590 "--update-graph-id", 

591 help=unwrap("Update graph ID with new unique value."), 

592 is_flag=True, 

593) 

594 

595metadata_run_key_option = MWOptionDecorator( 

596 "--metadata-run-key", 

597 help=( 

598 "Quantum graph metadata key for the name of the output run. " 

599 "Empty string disables update of the metadata. " 

600 "Default value: output_run." 

601 ), 

602 default="output_run", 

603)