Coverage for python/lsst/ctrl/mpexec/cli/opt/options.py: 80%

71 statements  

« prev     ^ index     » next       coverage.py v7.2.7, created at 2023-07-05 09:15 +0000

1# This file is part of ctrl_mpexec. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from __future__ import annotations 

23 

24from collections.abc import Iterable, Mapping 

25 

26import click 

27from lsst.daf.butler.cli.utils import MWOptionDecorator, MWPath, split_commas, unwrap 

28from lsst.utils.doImport import doImportType 

29 

30butler_config_option = MWOptionDecorator( 

31 "-b", "--butler-config", help="Location of the gen3 butler/registry config file." 

32) 

33 

34 

35data_query_option = MWOptionDecorator( 

36 "-d", "--data-query", help="User data selection expression.", metavar="QUERY" 

37) 

38 

39 

40debug_option = MWOptionDecorator( 

41 "--debug", help="Enable debugging output using lsstDebug facility (imports debug.py).", is_flag=True 

42) 

43 

44coverage_option = MWOptionDecorator("--coverage", help="Enable coverage output.", is_flag=True) 

45 

46coverage_report_option = MWOptionDecorator( 

47 "--cov-report/--no-cov-report", 

48 help="If coverage is enabled, controls whether to produce an HTML coverage report.", 

49 default=True, 

50) 

51 

52coverage_packages_option = MWOptionDecorator( 

53 "--cov-packages", 

54 help=unwrap( 

55 """Python packages to restrict coverage to. If none are provided, runs coverage on all packages.""" 

56 ), 

57 multiple=True, 

58 callback=split_commas, 

59) 

60 

61delete_option = MWOptionDecorator( 

62 "--delete", callback=split_commas, help="Delete task with given label from pipeline.", multiple=True 

63) 

64 

65 

66pdb_option = MWOptionDecorator( 

67 "--pdb", 

68 help="Post-mortem debugger to launch for exceptions (defaults to pdb if unspecified; requires a tty).", 

69 is_flag=False, 

70 flag_value="pdb", 

71 default=None, 

72) 

73 

74 

75extend_run_option = MWOptionDecorator( 

76 "--extend-run", 

77 help=( 

78 "Instead of creating a new RUN collection, insert datasets into either the one given by " 

79 "--output-run (if provided) or the first child collection of --output (which must be of type RUN). " 

80 "This also enables --skip-existing option when building a graph. " 

81 "When executing a graph this option skips quanta with all existing outputs." 

82 ), 

83 is_flag=True, 

84) 

85 

86 

87graph_fixup_option = MWOptionDecorator( 

88 "--graph-fixup", 

89 help=unwrap( 

90 """Name of the class or factory method which makes an 

91 instance used for execution graph fixup.""" 

92 ), 

93) 

94 

95 

96init_only_option = MWOptionDecorator( 

97 "--init-only", 

98 help=unwrap( 

99 """Do not actually run; just register dataset types and/or 

100 save init outputs. """ 

101 ), 

102 is_flag=True, 

103) 

104 

105 

106input_option = MWOptionDecorator( 

107 "-i", 

108 "--input", 

109 callback=split_commas, 

110 default=list(), 

111 help=unwrap("""Comma-separated names of the input collection(s)."""), 

112 metavar="COLLECTION", 

113 multiple=True, 

114) 

115 

116no_versions_option = MWOptionDecorator( 

117 "--no-versions", help="Do not save or check package versions.", is_flag=True 

118) 

119 

120 

121order_pipeline_option = MWOptionDecorator( 

122 "--order-pipeline", 

123 help=unwrap( 

124 """Order tasks in pipeline based on their data 

125 dependencies, ordering is performed as last step before saving or 

126 executing pipeline.""" 

127 ), 

128 is_flag=True, 

129) 

130 

131 

132output_option = MWOptionDecorator( 

133 "-o", 

134 "--output", 

135 help=unwrap( 

136 """Name of the output CHAINED collection. This may either be an 

137 existing CHAINED collection to use as both input and output 

138 (incompatible with --input), or a new CHAINED collection created 

139 to include all inputs (requires --input). In both cases, the 

140 collection's children will start with an output RUN collection 

141 that directly holds all new datasets (see --output-run).""" 

142 ), 

143 metavar="COLL", 

144) 

145 

146 

147output_run_option = MWOptionDecorator( 

148 "--output-run", 

149 help=unwrap( 

150 """Name of the new output RUN collection. If not provided 

151 then --output must be provided and a new RUN collection will 

152 be created by appending a timestamp to the value passed with 

153 --output. If this collection already exists then 

154 --extend-run must be passed.""" 

155 ), 

156 metavar="COLL", 

157) 

158 

159 

160pipeline_option = MWOptionDecorator( 

161 "-p", 

162 "--pipeline", 

163 help="Location of a pipeline definition file in YAML format.", 

164 type=MWPath(file_okay=True, dir_okay=False, readable=True), 

165) 

166 

167 

168pipeline_dot_option = MWOptionDecorator( 

169 "--pipeline-dot", 

170 help=unwrap( 

171 """"Location for storing GraphViz DOT representation of a 

172 pipeline.""" 

173 ), 

174 type=MWPath(writable=True, file_okay=True, dir_okay=False), 

175) 

176 

177 

178profile_option = MWOptionDecorator( 

179 "--profile", help="Dump cProfile statistics to file name.", type=MWPath(file_okay=True, dir_okay=False) 

180) 

181 

182 

183prune_replaced_option = MWOptionDecorator( 

184 "--prune-replaced", 

185 help=unwrap( 

186 """Delete the datasets in the collection replaced by 

187 --replace-run, either just from the datastore 

188 ('unstore') or by removing them and the RUN completely 

189 ('purge'). Requires --replace-run.""" 

190 ), 

191 type=click.Choice(choices=("unstore", "purge"), case_sensitive=False), 

192) 

193 

194 

195qgraph_option = MWOptionDecorator( 

196 "-g", 

197 "--qgraph", 

198 help=unwrap( 

199 """Location for a serialized quantum graph definition (pickle 

200 file). If this option is given then all input data options and 

201 pipeline-building options cannot be used. Can be a URI.""" 

202 ), 

203) 

204 

205 

206qgraph_id_option = MWOptionDecorator( 

207 "--qgraph-id", 

208 help=unwrap( 

209 """Quantum graph identifier, if specified must match the 

210 identifier of the graph loaded from a file. Ignored if graph 

211 is not loaded from a file.""" 

212 ), 

213) 

214 

215 

216qgraph_datastore_records_option = MWOptionDecorator( 

217 "--qgraph-datastore-records", 

218 help=unwrap( 

219 """Include datastore records into generated quantum graph, these records are used by a 

220 quantum-backed butler. 

221 """ 

222 ), 

223 is_flag=True, 

224) 

225 

226 

227# I wanted to use default=None here to match Python API but click silently 

228# replaces None with an empty tuple when multiple=True. 

229qgraph_node_id_option = MWOptionDecorator( 

230 "--qgraph-node-id", 

231 callback=split_commas, 

232 multiple=True, 

233 help=unwrap( 

234 """Only load a specified set of nodes when graph is 

235 loaded from a file, nodes are identified by UUID 

236 values. One or more comma-separated integers are 

237 accepted. By default all nodes are loaded. Ignored if 

238 graph is not loaded from a file.""" 

239 ), 

240) 

241 

242qgraph_header_data_option = MWOptionDecorator( 

243 "--show-qgraph-header", 

244 is_flag=True, 

245 default=False, 

246 help=unwrap( 

247 """Print the headerData for Quantum Graph to the 

248 console""" 

249 ), 

250) 

251 

252qgraph_dot_option = MWOptionDecorator( 

253 "--qgraph-dot", 

254 help=unwrap( 

255 """Location for storing GraphViz DOT representation of a 

256 quantum graph.""" 

257 ), 

258 type=MWPath(writable=True, file_okay=True, dir_okay=False), 

259) 

260 

261 

262replace_run_option = MWOptionDecorator( 

263 "--replace-run", 

264 help=unwrap( 

265 """Before creating a new RUN collection in an existing 

266 CHAINED collection, remove the first child collection 

267 (which must be of type RUN). This can be used to repeatedly 

268 write to the same (parent) collection during development, 

269 but it does not delete the datasets associated with the 

270 replaced run unless --prune-replaced is also passed. 

271 Requires --output, and incompatible with --extend-run.""" 

272 ), 

273 is_flag=True, 

274) 

275 

276 

277save_pipeline_option = MWOptionDecorator( 

278 "-s", 

279 "--save-pipeline", 

280 help=unwrap( 

281 """Location for storing resulting pipeline definition in 

282 YAML format.""" 

283 ), 

284 type=MWPath(dir_okay=False, file_okay=True, writable=True), 

285) 

286 

287save_qgraph_option = MWOptionDecorator( 

288 "-q", 

289 "--save-qgraph", 

290 help=unwrap( 

291 """URI location for storing a serialized quantum graph 

292 definition (pickle file).""" 

293 ), 

294) 

295 

296 

297save_single_quanta_option = MWOptionDecorator( 

298 "--save-single-quanta", 

299 help=unwrap( 

300 """Format string of locations for storing individual 

301 quantum graph definition (pickle files). The curly 

302 brace {} in the input string will be replaced by a 

303 quantum number. Can be a URI.""" 

304 ), 

305) 

306 

307 

308show_option = MWOptionDecorator( 

309 "--show", 

310 callback=split_commas, 

311 help=unwrap( 

312 """Dump various info to standard output. Possible items are: 

313 `config`, `config=[Task::]<PATTERN>` or 

314 `config=[Task::]<PATTERN>:NOIGNORECASE` to dump configuration 

315 fields possibly matching given pattern and/or task label; 

316 `history=<FIELD>` to dump configuration history for a field, field 

317 name is specified as [Task::]<PATTERN>; `dump-config`, 

318 `dump-config=Task` to dump complete configuration for a task given 

319 its label or all tasks; `pipeline` to show pipeline composition; 

320 `graph` to show information about quanta; `workflow` to show 

321 information about quanta and their dependency; `tasks` to show 

322 task composition; `uri` to show predicted dataset URIs of 

323 quanta""" 

324 ), 

325 metavar="ITEM|ITEM=VALUE", 

326 multiple=True, 

327) 

328 

329 

330skip_existing_in_option = MWOptionDecorator( 

331 "--skip-existing-in", 

332 callback=split_commas, 

333 default=None, 

334 metavar="COLLECTION", 

335 multiple=True, 

336 help=unwrap( 

337 """If all Quantum outputs already exist in the specified list of 

338 collections then that Quantum will be excluded from the QuantumGraph. 

339 """ 

340 ), 

341) 

342 

343 

344skip_existing_option = MWOptionDecorator( 

345 "--skip-existing", 

346 is_flag=True, 

347 help=unwrap( 

348 """This option is equivalent to --skip-existing-in with the name of 

349 the output RUN collection. If both --skip-existing-in and 

350 --skip-existing are given then output RUN collection is appended to 

351 the list of collections.""" 

352 ), 

353) 

354 

355 

356clobber_outputs_option = MWOptionDecorator( 

357 "--clobber-outputs", 

358 help=( 

359 "Remove outputs of failed quanta from the output run when they would block the execution of new " 

360 "quanta with the same data ID (or assume that this will be done, if just building a QuantumGraph). " 

361 "Does nothing if --extend-run is not passed." 

362 ), 

363 is_flag=True, 

364) 

365 

366 

367skip_init_writes_option = MWOptionDecorator( 

368 "--skip-init-writes", 

369 help=unwrap( 

370 """Do not write collection-wide 'init output' datasets 

371 (e.g.schemas).""" 

372 ), 

373 is_flag=True, 

374) 

375 

376 

377enable_implicit_threading_option = MWOptionDecorator( 

378 "--enable-implicit-threading", 

379 help=unwrap( 

380 """Do not disable implicit threading use by third-party libraries (e.g. OpenBLAS). 

381 Implicit threading is always disabled during execution with multiprocessing.""" 

382 ), 

383 is_flag=True, 

384) 

385 

386cores_per_quantum_option = MWOptionDecorator( 

387 "-n", 

388 "--cores-per-quantum", 

389 default=1, 

390 help=unwrap( 

391 """Number of cores available to each quantum when executing. 

392 If '-j' is used each subprocess will be allowed to use this number of cores.""" 

393 ), 

394 type=click.IntRange(min=1), 

395) 

396 

397memory_per_quantum_option = MWOptionDecorator( 

398 "--memory-per-quantum", 

399 default="", 

400 help=unwrap( 

401 """Memory allocated for each quantum to use when executing. 

402 This memory allocation is not enforced by the execution system and is purely advisory. 

403 If '-j' used each subprocess will be allowed to use this amount of memory. 

404 Units are allowed and the default units for a plain integer are MB. 

405 For example: '3GB', '3000MB' and '3000' would all result in the same 

406 memory limit. Default is for no limit.""" 

407 ), 

408 type=str, 

409) 

410 

411task_option = MWOptionDecorator( 

412 "-t", 

413 "--task", 

414 callback=split_commas, 

415 help=unwrap( 

416 """Task name to add to pipeline, must be a fully qualified task 

417 name. Task name can be followed by colon and label name, if label 

418 is not given then task base name (class name) is used as 

419 label.""" 

420 ), 

421 metavar="TASK[:LABEL]", 

422 multiple=True, 

423) 

424 

425 

426timeout_option = MWOptionDecorator( 

427 "--timeout", type=click.IntRange(min=0), help="Timeout for multiprocessing; maximum wall time (sec)." 

428) 

429 

430 

431start_method_option = MWOptionDecorator( 

432 "--start-method", 

433 default=None, 

434 type=click.Choice(choices=["spawn", "fork", "forkserver"]), 

435 help="Multiprocessing start method, default is platform-specific.", 

436) 

437 

438 

439fail_fast_option = MWOptionDecorator( 

440 "--fail-fast", 

441 help=unwrap( 

442 """Stop processing at first error, default is to process 

443 as many tasks as possible.""" 

444 ), 

445 is_flag=True, 

446) 

447 

448save_execution_butler_option = MWOptionDecorator( 

449 "--save-execution-butler", 

450 help=unwrap( 

451 """Export location for an 

452 execution-specific butler after making 

453 QuantumGraph""" 

454 ), 

455) 

456 

457mock_option = MWOptionDecorator( 

458 "--mock", 

459 help=unwrap("""Mock pipeline execution."""), 

460 is_flag=True, 

461) 

462 

463unmocked_dataset_types_option = MWOptionDecorator( 

464 "--unmocked-dataset-types", 

465 callback=split_commas, 

466 default=None, 

467 metavar="COLLECTION", 

468 multiple=True, 

469 help=unwrap("""Names of input dataset types that should not be mocked."""), 

470) 

471 

472 

473def parse_mock_failure( 

474 ctx: click.Context, param: click.Option, value: Iterable[str] | None 

475) -> Mapping[str, tuple[str, type[Exception] | None]]: 

476 """Parse the --mock-failure option values into the mapping accepted by 

477 `lsst.pipe.base.tests.mocks.mock_task_defs`. 

478 """ 

479 result: dict[str, tuple[str, type[Exception] | None]] = {} 

480 if value is None: 

481 return result 

482 for entry in value: 

483 try: 

484 task_label, error_type_name, where = entry.split(":", 2) 

485 except ValueError: 

486 raise click.UsageError( 

487 f"Invalid value for --mock-failure option: {entry!r}; " 

488 "expected a string of the form 'task:error:where'." 

489 ) from None 

490 error_type = doImportType(error_type_name) if error_type_name else None 

491 result[task_label] = (where, error_type) 

492 return result 

493 

494 

495mock_failure_option = MWOptionDecorator( 

496 "--mock-failure", 

497 callback=parse_mock_failure, 

498 metavar="LABEL:EXCEPTION:WHERE", 

499 default=None, 

500 multiple=True, 

501 help=unwrap( 

502 """Specifications for tasks that should be configured to fail 

503 when mocking execution. This is a colon-separated 3-tuple, where the 

504 first entry the task label, the second the fully-qualified exception 

505 type (empty for ValueError, and the third a string (which typically 

506 needs to be quoted to be passed as one argument value by the shell) of 

507 the form passed to --where, indicating which data IDs should fail.""" 

508 ), 

509) 

510 

511 

512clobber_execution_butler_option = MWOptionDecorator( 

513 "--clobber-execution-butler", 

514 help=unwrap( 

515 """When creating execution butler overwrite 

516 any existing products""" 

517 ), 

518 is_flag=True, 

519) 

520 

521target_datastore_root_option = MWOptionDecorator( 

522 "--target-datastore-root", 

523 help=unwrap( 

524 """Root directory for datastore of execution butler. 

525 Default is to use the original datastore. 

526 """ 

527 ), 

528) 

529 

530dataset_query_constraint = MWOptionDecorator( 

531 "--dataset-query-constraint", 

532 help=unwrap( 

533 """When constructing a quantum graph constrain by 

534 pre-existence of specified dataset types. Valid 

535 values are `all` for all inputs dataset types in 

536 pipeline, `off` to not consider dataset type 

537 existance as a constraint, single or comma 

538 separated list of dataset type names""" 

539 ), 

540 default="all", 

541) 

542 

543summary_option = MWOptionDecorator( 

544 "--summary", 

545 help=( 

546 "Location for storing job summary (JSON file). Note that the" 

547 " structure of this file may not be stable." 

548 ), 

549 type=MWPath(dir_okay=False, file_okay=True, writable=True), 

550) 

551 

552 

553recursive_option = MWOptionDecorator( 

554 "--recursive", 

555 is_flag=True, 

556) 

557 

558config_search_path_option = MWOptionDecorator( 

559 "--config-search-path", 

560 callback=split_commas, 

561 default=list(), 

562 help="Additional search paths for butler configuration.", 

563 metavar="PATH", 

564 multiple=True, 

565) 

566 

567update_graph_id_option = MWOptionDecorator( 

568 "--update-graph-id", 

569 help=unwrap("Update graph ID with new unique value."), 

570 is_flag=True, 

571) 

572 

573metadata_run_key_option = MWOptionDecorator( 

574 "--metadata-run-key", 

575 help=( 

576 "Quantum graph metadata key for the name of the output run. " 

577 "Empty string disables update of the metadata. " 

578 "Default value: output_run." 

579 ), 

580 default="output_run", 

581)