Coverage for python/lsst/ctrl/mpexec/cli/opt/options.py: 80%

72 statements  

« prev     ^ index     » next       coverage.py v7.3.0, created at 2023-09-01 09:30 +0000

1# This file is part of ctrl_mpexec. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from __future__ import annotations 

23 

24from collections.abc import Iterable, Mapping 

25 

26import click 

27from lsst.daf.butler.cli.utils import MWOptionDecorator, MWPath, split_commas, unwrap 

28from lsst.utils.doImport import doImportType 

29 

30butler_config_option = MWOptionDecorator( 

31 "-b", "--butler-config", help="Location of the gen3 butler/registry config file." 

32) 

33 

34 

35data_query_option = MWOptionDecorator( 

36 "-d", "--data-query", help="User data selection expression.", metavar="QUERY" 

37) 

38 

39 

40debug_option = MWOptionDecorator( 

41 "--debug", help="Enable debugging output using lsstDebug facility (imports debug.py).", is_flag=True 

42) 

43 

44coverage_option = MWOptionDecorator("--coverage", help="Enable coverage output.", is_flag=True) 

45 

46coverage_report_option = MWOptionDecorator( 

47 "--cov-report/--no-cov-report", 

48 help="If coverage is enabled, controls whether to produce an HTML coverage report.", 

49 default=True, 

50) 

51 

52coverage_packages_option = MWOptionDecorator( 

53 "--cov-packages", 

54 help=unwrap( 

55 """Python packages to restrict coverage to. If none are provided, runs coverage on all packages.""" 

56 ), 

57 multiple=True, 

58 callback=split_commas, 

59) 

60 

61delete_option = MWOptionDecorator( 

62 "--delete", callback=split_commas, help="Delete task with given label from pipeline.", multiple=True 

63) 

64 

65 

66pdb_option = MWOptionDecorator( 

67 "--pdb", 

68 help="Post-mortem debugger to launch for exceptions (defaults to pdb if unspecified; requires a tty).", 

69 is_flag=False, 

70 flag_value="pdb", 

71 default=None, 

72) 

73 

74 

75extend_run_option = MWOptionDecorator( 

76 "--extend-run", 

77 help=( 

78 "Instead of creating a new RUN collection, insert datasets into either the one given by " 

79 "--output-run (if provided) or the first child collection of --output (which must be of type RUN). " 

80 "This also enables --skip-existing option when building a graph. " 

81 "When executing a graph this option skips quanta with all existing outputs." 

82 ), 

83 is_flag=True, 

84) 

85 

86 

87graph_fixup_option = MWOptionDecorator( 

88 "--graph-fixup", 

89 help="Name of the class or factory method which makes an instance used for execution graph fixup.", 

90) 

91 

92 

93init_only_option = MWOptionDecorator( 

94 "--init-only", 

95 help="Do not actually run; just register dataset types and/or save init outputs.", 

96 is_flag=True, 

97) 

98 

99 

100input_option = MWOptionDecorator( 

101 "-i", 

102 "--input", 

103 callback=split_commas, 

104 default=[], 

105 help="Comma-separated names of the input collection(s).", 

106 metavar="COLLECTION", 

107 multiple=True, 

108) 

109 

110 

111rebase_option = MWOptionDecorator( 

112 "--rebase", 

113 help=unwrap("""Reset output collection chain if it is inconsistent with --inputs"""), 

114 is_flag=True, 

115) 

116 

117 

118no_versions_option = MWOptionDecorator( 

119 "--no-versions", help="Do not save or check package versions.", is_flag=True 

120) 

121 

122 

123order_pipeline_option = MWOptionDecorator( 

124 "--order-pipeline", 

125 help=unwrap( 

126 """Order tasks in pipeline based on their data 

127 dependencies, ordering is performed as last step before saving or 

128 executing pipeline.""" 

129 ), 

130 is_flag=True, 

131) 

132 

133 

134output_option = MWOptionDecorator( 

135 "-o", 

136 "--output", 

137 help=unwrap( 

138 """Name of the output CHAINED collection. This may either be an 

139 existing CHAINED collection to use as both input and output 

140 (incompatible with --input), or a new CHAINED collection created 

141 to include all inputs (requires --input). In both cases, the 

142 collection's children will start with an output RUN collection 

143 that directly holds all new datasets (see --output-run).""" 

144 ), 

145 metavar="COLL", 

146) 

147 

148 

149output_run_option = MWOptionDecorator( 

150 "--output-run", 

151 help=unwrap( 

152 """Name of the new output RUN collection. If not provided 

153 then --output must be provided and a new RUN collection will 

154 be created by appending a timestamp to the value passed with 

155 --output. If this collection already exists then 

156 --extend-run must be passed.""" 

157 ), 

158 metavar="COLL", 

159) 

160 

161 

162pipeline_option = MWOptionDecorator( 

163 "-p", 

164 "--pipeline", 

165 help="Location of a pipeline definition file in YAML format.", 

166 type=MWPath(file_okay=True, dir_okay=False, readable=True), 

167) 

168 

169 

170pipeline_dot_option = MWOptionDecorator( 

171 "--pipeline-dot", 

172 help="Location for storing GraphViz DOT representation of a pipeline.", 

173 type=MWPath(writable=True, file_okay=True, dir_okay=False), 

174) 

175 

176 

177profile_option = MWOptionDecorator( 

178 "--profile", help="Dump cProfile statistics to file name.", type=MWPath(file_okay=True, dir_okay=False) 

179) 

180 

181 

182prune_replaced_option = MWOptionDecorator( 

183 "--prune-replaced", 

184 help=unwrap( 

185 """Delete the datasets in the collection replaced by 

186 --replace-run, either just from the datastore 

187 ('unstore') or by removing them and the RUN completely 

188 ('purge'). Requires --replace-run.""" 

189 ), 

190 type=click.Choice(choices=("unstore", "purge"), case_sensitive=False), 

191) 

192 

193 

194qgraph_option = MWOptionDecorator( 

195 "-g", 

196 "--qgraph", 

197 help=unwrap( 

198 """Location for a serialized quantum graph definition (pickle 

199 file). If this option is given then all input data options and 

200 pipeline-building options cannot be used. Can be a URI.""" 

201 ), 

202) 

203 

204 

205qgraph_id_option = MWOptionDecorator( 

206 "--qgraph-id", 

207 help=unwrap( 

208 """Quantum graph identifier, if specified must match the 

209 identifier of the graph loaded from a file. Ignored if graph 

210 is not loaded from a file.""" 

211 ), 

212) 

213 

214 

215qgraph_datastore_records_option = MWOptionDecorator( 

216 "--qgraph-datastore-records", 

217 help=unwrap( 

218 """Include datastore records into generated quantum graph, these records are used by a 

219 quantum-backed butler. 

220 """ 

221 ), 

222 is_flag=True, 

223) 

224 

225 

226# I wanted to use default=None here to match Python API but click silently 

227# replaces None with an empty tuple when multiple=True. 

228qgraph_node_id_option = MWOptionDecorator( 

229 "--qgraph-node-id", 

230 callback=split_commas, 

231 multiple=True, 

232 help=unwrap( 

233 """Only load a specified set of nodes when graph is 

234 loaded from a file, nodes are identified by UUID 

235 values. One or more comma-separated integers are 

236 accepted. By default all nodes are loaded. Ignored if 

237 graph is not loaded from a file.""" 

238 ), 

239) 

240 

241qgraph_header_data_option = MWOptionDecorator( 

242 "--show-qgraph-header", 

243 is_flag=True, 

244 default=False, 

245 help="Print the headerData for Quantum Graph to the console", 

246) 

247 

248qgraph_dot_option = MWOptionDecorator( 

249 "--qgraph-dot", 

250 help="Location for storing GraphViz DOT representation of a quantum graph.", 

251 type=MWPath(writable=True, file_okay=True, dir_okay=False), 

252) 

253 

254 

255replace_run_option = MWOptionDecorator( 

256 "--replace-run", 

257 help=unwrap( 

258 """Before creating a new RUN collection in an existing 

259 CHAINED collection, remove the first child collection 

260 (which must be of type RUN). This can be used to repeatedly 

261 write to the same (parent) collection during development, 

262 but it does not delete the datasets associated with the 

263 replaced run unless --prune-replaced is also passed. 

264 Requires --output, and incompatible with --extend-run.""" 

265 ), 

266 is_flag=True, 

267) 

268 

269 

270save_pipeline_option = MWOptionDecorator( 

271 "-s", 

272 "--save-pipeline", 

273 help="Location for storing resulting pipeline definition in YAML format.", 

274 type=MWPath(dir_okay=False, file_okay=True, writable=True), 

275) 

276 

277save_qgraph_option = MWOptionDecorator( 

278 "-q", 

279 "--save-qgraph", 

280 help="URI location for storing a serialized quantum graph definition (pickle file).", 

281) 

282 

283 

284save_single_quanta_option = MWOptionDecorator( 

285 "--save-single-quanta", 

286 help=unwrap( 

287 """Format string of locations for storing individual 

288 quantum graph definition (pickle files). The curly 

289 brace {} in the input string will be replaced by a 

290 quantum number. Can be a URI.""" 

291 ), 

292) 

293 

294 

295show_option = MWOptionDecorator( 

296 "--show", 

297 callback=split_commas, 

298 help=unwrap( 

299 """Dump various info to standard output. Possible items are: 

300 ``config``, ``config=[Task::]<PATTERN>`` or 

301 ``config=[Task::]<PATTERN>:NOIGNORECASE`` to dump configuration 

302 fields possibly matching given pattern and/or task label; 

303 ``history=<FIELD>`` to dump configuration history for a field, 

304 field name is specified as ``[Task::]<PATTERN>``; ``dump-config``, 

305 ``dump-config=Task`` to dump complete configuration for a task 

306 given its label or all tasks; ``pipeline`` to show pipeline 

307 composition; ``graph`` to show information about quanta; 

308 ``workflow`` to show information about quanta and their 

309 dependency; ``tasks`` to show task composition; ``uri`` to show 

310 predicted dataset URIs of quanta.""" 

311 ), 

312 metavar="ITEM|ITEM=VALUE", 

313 multiple=True, 

314) 

315 

316 

317skip_existing_in_option = MWOptionDecorator( 

318 "--skip-existing-in", 

319 callback=split_commas, 

320 default=None, 

321 metavar="COLLECTION", 

322 multiple=True, 

323 help=unwrap( 

324 """If all Quantum outputs already exist in the specified list of 

325 collections then that Quantum will be excluded from the QuantumGraph. 

326 """ 

327 ), 

328) 

329 

330 

331skip_existing_option = MWOptionDecorator( 

332 "--skip-existing", 

333 is_flag=True, 

334 help=unwrap( 

335 """This option is equivalent to --skip-existing-in with the name of 

336 the output RUN collection. If both --skip-existing-in and 

337 --skip-existing are given then output RUN collection is appended to 

338 the list of collections.""" 

339 ), 

340) 

341 

342 

343clobber_outputs_option = MWOptionDecorator( 

344 "--clobber-outputs", 

345 help=( 

346 "Remove outputs of failed quanta from the output run when they would block the execution of new " 

347 "quanta with the same data ID (or assume that this will be done, if just building a QuantumGraph). " 

348 "Does nothing if --extend-run is not passed." 

349 ), 

350 is_flag=True, 

351) 

352 

353 

354skip_init_writes_option = MWOptionDecorator( 

355 "--skip-init-writes", 

356 help=unwrap( 

357 """Do not write collection-wide 'init output' datasets 

358 (e.g.schemas).""" 

359 ), 

360 is_flag=True, 

361) 

362 

363 

364enable_implicit_threading_option = MWOptionDecorator( 

365 "--enable-implicit-threading", 

366 help=unwrap( 

367 """Do not disable implicit threading use by third-party libraries (e.g. OpenBLAS). 

368 Implicit threading is always disabled during execution with multiprocessing.""" 

369 ), 

370 is_flag=True, 

371) 

372 

373cores_per_quantum_option = MWOptionDecorator( 

374 "-n", 

375 "--cores-per-quantum", 

376 default=1, 

377 help=unwrap( 

378 """Number of cores available to each quantum when executing. 

379 If '-j' is used each subprocess will be allowed to use this number of cores.""" 

380 ), 

381 type=click.IntRange(min=1), 

382) 

383 

384memory_per_quantum_option = MWOptionDecorator( 

385 "--memory-per-quantum", 

386 default="", 

387 help=unwrap( 

388 """Memory allocated for each quantum to use when executing. 

389 This memory allocation is not enforced by the execution system and is purely advisory. 

390 If '-j' used each subprocess will be allowed to use this amount of memory. 

391 Units are allowed and the default units for a plain integer are MB. 

392 For example: '3GB', '3000MB' and '3000' would all result in the same 

393 memory limit. Default is for no limit.""" 

394 ), 

395 type=str, 

396) 

397 

398task_option = MWOptionDecorator( 

399 "-t", 

400 "--task", 

401 callback=split_commas, 

402 help=unwrap( 

403 """Task name to add to pipeline, must be a fully qualified task 

404 name. Task name can be followed by colon and label name, if label 

405 is not given then task base name (class name) is used as 

406 label.""" 

407 ), 

408 metavar="TASK[:LABEL]", 

409 multiple=True, 

410) 

411 

412 

413timeout_option = MWOptionDecorator( 

414 "--timeout", type=click.IntRange(min=0), help="Timeout for multiprocessing; maximum wall time (sec)." 

415) 

416 

417 

418start_method_option = MWOptionDecorator( 

419 "--start-method", 

420 default=None, 

421 type=click.Choice(choices=["spawn", "fork", "forkserver"]), 

422 help="Multiprocessing start method, default is platform-specific.", 

423) 

424 

425 

426fail_fast_option = MWOptionDecorator( 

427 "--fail-fast", 

428 help="Stop processing at first error, default is to process as many tasks as possible.", 

429 is_flag=True, 

430) 

431 

432save_execution_butler_option = MWOptionDecorator( 

433 "--save-execution-butler", 

434 help="Export location for an execution-specific butler after making QuantumGraph", 

435) 

436 

437mock_option = MWOptionDecorator( 

438 "--mock", 

439 help="Mock pipeline execution.", 

440 is_flag=True, 

441) 

442 

443unmocked_dataset_types_option = MWOptionDecorator( 

444 "--unmocked-dataset-types", 

445 callback=split_commas, 

446 default=None, 

447 metavar="COLLECTION", 

448 multiple=True, 

449 help="Names of input dataset types that should not be mocked.", 

450) 

451 

452 

453def parse_mock_failure( 

454 ctx: click.Context, param: click.Option, value: Iterable[str] | None 

455) -> Mapping[str, tuple[str, type[Exception] | None]]: 

456 """Parse the --mock-failure option values into the mapping accepted by 

457 `lsst.pipe.base.tests.mocks.mock_task_defs`. 

458 """ 

459 result: dict[str, tuple[str, type[Exception] | None]] = {} 

460 if value is None: 

461 return result 

462 for entry in value: 

463 try: 

464 task_label, error_type_name, where = entry.split(":", 2) 

465 except ValueError: 

466 raise click.UsageError( 

467 f"Invalid value for --mock-failure option: {entry!r}; " 

468 "expected a string of the form 'task:error:where'." 

469 ) from None 

470 error_type = doImportType(error_type_name) if error_type_name else None 

471 result[task_label] = (where, error_type) 

472 return result 

473 

474 

475mock_failure_option = MWOptionDecorator( 

476 "--mock-failure", 

477 callback=parse_mock_failure, 

478 metavar="LABEL:EXCEPTION:WHERE", 

479 default=None, 

480 multiple=True, 

481 help=unwrap( 

482 """Specifications for tasks that should be configured to fail 

483 when mocking execution. This is a colon-separated 3-tuple, where the 

484 first entry the task label, the second the fully-qualified exception 

485 type (empty for ValueError, and the third a string (which typically 

486 needs to be quoted to be passed as one argument value by the shell) of 

487 the form passed to --where, indicating which data IDs should fail.""" 

488 ), 

489) 

490 

491 

492clobber_execution_butler_option = MWOptionDecorator( 

493 "--clobber-execution-butler", 

494 help=unwrap( 

495 """When creating execution butler overwrite 

496 any existing products""" 

497 ), 

498 is_flag=True, 

499) 

500 

501target_datastore_root_option = MWOptionDecorator( 

502 "--target-datastore-root", 

503 help=unwrap( 

504 """Root directory for datastore of execution butler. 

505 Default is to use the original datastore. 

506 """ 

507 ), 

508) 

509 

510dataset_query_constraint = MWOptionDecorator( 

511 "--dataset-query-constraint", 

512 help=unwrap( 

513 """When constructing a quantum graph constrain by 

514 pre-existence of specified dataset types. Valid 

515 values are `all` for all inputs dataset types in 

516 pipeline, ``off`` to not consider dataset type 

517 existence as a constraint, single or comma 

518 separated list of dataset type names.""" 

519 ), 

520 default="all", 

521) 

522 

523summary_option = MWOptionDecorator( 

524 "--summary", 

525 help=( 

526 "Location for storing job summary (JSON file). Note that the" 

527 " structure of this file may not be stable." 

528 ), 

529 type=MWPath(dir_okay=False, file_okay=True, writable=True), 

530) 

531 

532 

533recursive_option = MWOptionDecorator( 

534 "--recursive", 

535 is_flag=True, 

536) 

537 

538config_search_path_option = MWOptionDecorator( 

539 "--config-search-path", 

540 callback=split_commas, 

541 default=[], 

542 help="Additional search paths for butler configuration.", 

543 metavar="PATH", 

544 multiple=True, 

545) 

546 

547update_graph_id_option = MWOptionDecorator( 

548 "--update-graph-id", 

549 help=unwrap("Update graph ID with new unique value."), 

550 is_flag=True, 

551) 

552 

553metadata_run_key_option = MWOptionDecorator( 

554 "--metadata-run-key", 

555 help=( 

556 "Quantum graph metadata key for the name of the output run. " 

557 "Empty string disables update of the metadata. " 

558 "Default value: output_run." 

559 ), 

560 default="output_run", 

561)