Coverage for python/lsst/ctrl/mpexec/cli/opt/options.py: 80%

71 statements  

« prev     ^ index     » next       coverage.py v7.2.7, created at 2023-07-14 19:56 +0000

1# This file is part of ctrl_mpexec. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from __future__ import annotations 

23 

24from collections.abc import Iterable, Mapping 

25 

26import click 

27from lsst.daf.butler.cli.utils import MWOptionDecorator, MWPath, split_commas, unwrap 

28from lsst.utils.doImport import doImportType 

29 

30butler_config_option = MWOptionDecorator( 

31 "-b", "--butler-config", help="Location of the gen3 butler/registry config file." 

32) 

33 

34 

35data_query_option = MWOptionDecorator( 

36 "-d", "--data-query", help="User data selection expression.", metavar="QUERY" 

37) 

38 

39 

40debug_option = MWOptionDecorator( 

41 "--debug", help="Enable debugging output using lsstDebug facility (imports debug.py).", is_flag=True 

42) 

43 

44coverage_option = MWOptionDecorator("--coverage", help="Enable coverage output.", is_flag=True) 

45 

46coverage_report_option = MWOptionDecorator( 

47 "--cov-report/--no-cov-report", 

48 help="If coverage is enabled, controls whether to produce an HTML coverage report.", 

49 default=True, 

50) 

51 

52coverage_packages_option = MWOptionDecorator( 

53 "--cov-packages", 

54 help=unwrap( 

55 """Python packages to restrict coverage to. If none are provided, runs coverage on all packages.""" 

56 ), 

57 multiple=True, 

58 callback=split_commas, 

59) 

60 

61delete_option = MWOptionDecorator( 

62 "--delete", callback=split_commas, help="Delete task with given label from pipeline.", multiple=True 

63) 

64 

65 

66pdb_option = MWOptionDecorator( 

67 "--pdb", 

68 help="Post-mortem debugger to launch for exceptions (defaults to pdb if unspecified; requires a tty).", 

69 is_flag=False, 

70 flag_value="pdb", 

71 default=None, 

72) 

73 

74 

75extend_run_option = MWOptionDecorator( 

76 "--extend-run", 

77 help=( 

78 "Instead of creating a new RUN collection, insert datasets into either the one given by " 

79 "--output-run (if provided) or the first child collection of --output (which must be of type RUN). " 

80 "This also enables --skip-existing option when building a graph. " 

81 "When executing a graph this option skips quanta with all existing outputs." 

82 ), 

83 is_flag=True, 

84) 

85 

86 

87graph_fixup_option = MWOptionDecorator( 

88 "--graph-fixup", 

89 help="Name of the class or factory method which makes an instance used for execution graph fixup.", 

90) 

91 

92 

93init_only_option = MWOptionDecorator( 

94 "--init-only", 

95 help="Do not actually run; just register dataset types and/or save init outputs.", 

96 is_flag=True, 

97) 

98 

99 

100input_option = MWOptionDecorator( 

101 "-i", 

102 "--input", 

103 callback=split_commas, 

104 default=list(), 

105 help="Comma-separated names of the input collection(s).", 

106 metavar="COLLECTION", 

107 multiple=True, 

108) 

109 

110no_versions_option = MWOptionDecorator( 

111 "--no-versions", help="Do not save or check package versions.", is_flag=True 

112) 

113 

114 

115order_pipeline_option = MWOptionDecorator( 

116 "--order-pipeline", 

117 help=unwrap( 

118 """Order tasks in pipeline based on their data 

119 dependencies, ordering is performed as last step before saving or 

120 executing pipeline.""" 

121 ), 

122 is_flag=True, 

123) 

124 

125 

126output_option = MWOptionDecorator( 

127 "-o", 

128 "--output", 

129 help=unwrap( 

130 """Name of the output CHAINED collection. This may either be an 

131 existing CHAINED collection to use as both input and output 

132 (incompatible with --input), or a new CHAINED collection created 

133 to include all inputs (requires --input). In both cases, the 

134 collection's children will start with an output RUN collection 

135 that directly holds all new datasets (see --output-run).""" 

136 ), 

137 metavar="COLL", 

138) 

139 

140 

141output_run_option = MWOptionDecorator( 

142 "--output-run", 

143 help=unwrap( 

144 """Name of the new output RUN collection. If not provided 

145 then --output must be provided and a new RUN collection will 

146 be created by appending a timestamp to the value passed with 

147 --output. If this collection already exists then 

148 --extend-run must be passed.""" 

149 ), 

150 metavar="COLL", 

151) 

152 

153 

154pipeline_option = MWOptionDecorator( 

155 "-p", 

156 "--pipeline", 

157 help="Location of a pipeline definition file in YAML format.", 

158 type=MWPath(file_okay=True, dir_okay=False, readable=True), 

159) 

160 

161 

162pipeline_dot_option = MWOptionDecorator( 

163 "--pipeline-dot", 

164 help="Location for storing GraphViz DOT representation of a pipeline.", 

165 type=MWPath(writable=True, file_okay=True, dir_okay=False), 

166) 

167 

168 

169profile_option = MWOptionDecorator( 

170 "--profile", help="Dump cProfile statistics to file name.", type=MWPath(file_okay=True, dir_okay=False) 

171) 

172 

173 

174prune_replaced_option = MWOptionDecorator( 

175 "--prune-replaced", 

176 help=unwrap( 

177 """Delete the datasets in the collection replaced by 

178 --replace-run, either just from the datastore 

179 ('unstore') or by removing them and the RUN completely 

180 ('purge'). Requires --replace-run.""" 

181 ), 

182 type=click.Choice(choices=("unstore", "purge"), case_sensitive=False), 

183) 

184 

185 

186qgraph_option = MWOptionDecorator( 

187 "-g", 

188 "--qgraph", 

189 help=unwrap( 

190 """Location for a serialized quantum graph definition (pickle 

191 file). If this option is given then all input data options and 

192 pipeline-building options cannot be used. Can be a URI.""" 

193 ), 

194) 

195 

196 

197qgraph_id_option = MWOptionDecorator( 

198 "--qgraph-id", 

199 help=unwrap( 

200 """Quantum graph identifier, if specified must match the 

201 identifier of the graph loaded from a file. Ignored if graph 

202 is not loaded from a file.""" 

203 ), 

204) 

205 

206 

207qgraph_datastore_records_option = MWOptionDecorator( 

208 "--qgraph-datastore-records", 

209 help=unwrap( 

210 """Include datastore records into generated quantum graph, these records are used by a 

211 quantum-backed butler. 

212 """ 

213 ), 

214 is_flag=True, 

215) 

216 

217 

218# I wanted to use default=None here to match Python API but click silently 

219# replaces None with an empty tuple when multiple=True. 

220qgraph_node_id_option = MWOptionDecorator( 

221 "--qgraph-node-id", 

222 callback=split_commas, 

223 multiple=True, 

224 help=unwrap( 

225 """Only load a specified set of nodes when graph is 

226 loaded from a file, nodes are identified by UUID 

227 values. One or more comma-separated integers are 

228 accepted. By default all nodes are loaded. Ignored if 

229 graph is not loaded from a file.""" 

230 ), 

231) 

232 

233qgraph_header_data_option = MWOptionDecorator( 

234 "--show-qgraph-header", 

235 is_flag=True, 

236 default=False, 

237 help="Print the headerData for Quantum Graph to the console", 

238) 

239 

240qgraph_dot_option = MWOptionDecorator( 

241 "--qgraph-dot", 

242 help="Location for storing GraphViz DOT representation of a quantum graph.", 

243 type=MWPath(writable=True, file_okay=True, dir_okay=False), 

244) 

245 

246 

247replace_run_option = MWOptionDecorator( 

248 "--replace-run", 

249 help=unwrap( 

250 """Before creating a new RUN collection in an existing 

251 CHAINED collection, remove the first child collection 

252 (which must be of type RUN). This can be used to repeatedly 

253 write to the same (parent) collection during development, 

254 but it does not delete the datasets associated with the 

255 replaced run unless --prune-replaced is also passed. 

256 Requires --output, and incompatible with --extend-run.""" 

257 ), 

258 is_flag=True, 

259) 

260 

261 

262save_pipeline_option = MWOptionDecorator( 

263 "-s", 

264 "--save-pipeline", 

265 help="Location for storing resulting pipeline definition in YAML format.", 

266 type=MWPath(dir_okay=False, file_okay=True, writable=True), 

267) 

268 

269save_qgraph_option = MWOptionDecorator( 

270 "-q", 

271 "--save-qgraph", 

272 help="URI location for storing a serialized quantum graph definition (pickle file).", 

273) 

274 

275 

276save_single_quanta_option = MWOptionDecorator( 

277 "--save-single-quanta", 

278 help=unwrap( 

279 """Format string of locations for storing individual 

280 quantum graph definition (pickle files). The curly 

281 brace {} in the input string will be replaced by a 

282 quantum number. Can be a URI.""" 

283 ), 

284) 

285 

286 

287show_option = MWOptionDecorator( 

288 "--show", 

289 callback=split_commas, 

290 help=unwrap( 

291 """Dump various info to standard output. Possible items are: 

292 ``config``, ``config=[Task::]<PATTERN>`` or 

293 ``config=[Task::]<PATTERN>:NOIGNORECASE`` to dump configuration 

294 fields possibly matching given pattern and/or task label; 

295 ``history=<FIELD>`` to dump configuration history for a field, 

296 field name is specified as ``[Task::]<PATTERN>``; ``dump-config``, 

297 ``dump-config=Task`` to dump complete configuration for a task 

298 given its label or all tasks; ``pipeline`` to show pipeline 

299 composition; ``graph`` to show information about quanta; 

300 ``workflow`` to show information about quanta and their 

301 dependency; ``tasks`` to show task composition; ``uri`` to show 

302 predicted dataset URIs of quanta.""" 

303 ), 

304 metavar="ITEM|ITEM=VALUE", 

305 multiple=True, 

306) 

307 

308 

309skip_existing_in_option = MWOptionDecorator( 

310 "--skip-existing-in", 

311 callback=split_commas, 

312 default=None, 

313 metavar="COLLECTION", 

314 multiple=True, 

315 help=unwrap( 

316 """If all Quantum outputs already exist in the specified list of 

317 collections then that Quantum will be excluded from the QuantumGraph. 

318 """ 

319 ), 

320) 

321 

322 

323skip_existing_option = MWOptionDecorator( 

324 "--skip-existing", 

325 is_flag=True, 

326 help=unwrap( 

327 """This option is equivalent to --skip-existing-in with the name of 

328 the output RUN collection. If both --skip-existing-in and 

329 --skip-existing are given then output RUN collection is appended to 

330 the list of collections.""" 

331 ), 

332) 

333 

334 

335clobber_outputs_option = MWOptionDecorator( 

336 "--clobber-outputs", 

337 help=( 

338 "Remove outputs of failed quanta from the output run when they would block the execution of new " 

339 "quanta with the same data ID (or assume that this will be done, if just building a QuantumGraph). " 

340 "Does nothing if --extend-run is not passed." 

341 ), 

342 is_flag=True, 

343) 

344 

345 

346skip_init_writes_option = MWOptionDecorator( 

347 "--skip-init-writes", 

348 help=unwrap( 

349 """Do not write collection-wide 'init output' datasets 

350 (e.g.schemas).""" 

351 ), 

352 is_flag=True, 

353) 

354 

355 

356enable_implicit_threading_option = MWOptionDecorator( 

357 "--enable-implicit-threading", 

358 help=unwrap( 

359 """Do not disable implicit threading use by third-party libraries (e.g. OpenBLAS). 

360 Implicit threading is always disabled during execution with multiprocessing.""" 

361 ), 

362 is_flag=True, 

363) 

364 

365cores_per_quantum_option = MWOptionDecorator( 

366 "-n", 

367 "--cores-per-quantum", 

368 default=1, 

369 help=unwrap( 

370 """Number of cores available to each quantum when executing. 

371 If '-j' is used each subprocess will be allowed to use this number of cores.""" 

372 ), 

373 type=click.IntRange(min=1), 

374) 

375 

376memory_per_quantum_option = MWOptionDecorator( 

377 "--memory-per-quantum", 

378 default="", 

379 help=unwrap( 

380 """Memory allocated for each quantum to use when executing. 

381 This memory allocation is not enforced by the execution system and is purely advisory. 

382 If '-j' used each subprocess will be allowed to use this amount of memory. 

383 Units are allowed and the default units for a plain integer are MB. 

384 For example: '3GB', '3000MB' and '3000' would all result in the same 

385 memory limit. Default is for no limit.""" 

386 ), 

387 type=str, 

388) 

389 

390task_option = MWOptionDecorator( 

391 "-t", 

392 "--task", 

393 callback=split_commas, 

394 help=unwrap( 

395 """Task name to add to pipeline, must be a fully qualified task 

396 name. Task name can be followed by colon and label name, if label 

397 is not given then task base name (class name) is used as 

398 label.""" 

399 ), 

400 metavar="TASK[:LABEL]", 

401 multiple=True, 

402) 

403 

404 

405timeout_option = MWOptionDecorator( 

406 "--timeout", type=click.IntRange(min=0), help="Timeout for multiprocessing; maximum wall time (sec)." 

407) 

408 

409 

410start_method_option = MWOptionDecorator( 

411 "--start-method", 

412 default=None, 

413 type=click.Choice(choices=["spawn", "fork", "forkserver"]), 

414 help="Multiprocessing start method, default is platform-specific.", 

415) 

416 

417 

418fail_fast_option = MWOptionDecorator( 

419 "--fail-fast", 

420 help="Stop processing at first error, default is to process as many tasks as possible.", 

421 is_flag=True, 

422) 

423 

424save_execution_butler_option = MWOptionDecorator( 

425 "--save-execution-butler", 

426 help="Export location for an execution-specific butler after making QuantumGraph", 

427) 

428 

429mock_option = MWOptionDecorator( 

430 "--mock", 

431 help="Mock pipeline execution.", 

432 is_flag=True, 

433) 

434 

435unmocked_dataset_types_option = MWOptionDecorator( 

436 "--unmocked-dataset-types", 

437 callback=split_commas, 

438 default=None, 

439 metavar="COLLECTION", 

440 multiple=True, 

441 help="Names of input dataset types that should not be mocked.", 

442) 

443 

444 

445def parse_mock_failure( 

446 ctx: click.Context, param: click.Option, value: Iterable[str] | None 

447) -> Mapping[str, tuple[str, type[Exception] | None]]: 

448 """Parse the --mock-failure option values into the mapping accepted by 

449 `lsst.pipe.base.tests.mocks.mock_task_defs`. 

450 """ 

451 result: dict[str, tuple[str, type[Exception] | None]] = {} 

452 if value is None: 

453 return result 

454 for entry in value: 

455 try: 

456 task_label, error_type_name, where = entry.split(":", 2) 

457 except ValueError: 

458 raise click.UsageError( 

459 f"Invalid value for --mock-failure option: {entry!r}; " 

460 "expected a string of the form 'task:error:where'." 

461 ) from None 

462 error_type = doImportType(error_type_name) if error_type_name else None 

463 result[task_label] = (where, error_type) 

464 return result 

465 

466 

467mock_failure_option = MWOptionDecorator( 

468 "--mock-failure", 

469 callback=parse_mock_failure, 

470 metavar="LABEL:EXCEPTION:WHERE", 

471 default=None, 

472 multiple=True, 

473 help=unwrap( 

474 """Specifications for tasks that should be configured to fail 

475 when mocking execution. This is a colon-separated 3-tuple, where the 

476 first entry the task label, the second the fully-qualified exception 

477 type (empty for ValueError, and the third a string (which typically 

478 needs to be quoted to be passed as one argument value by the shell) of 

479 the form passed to --where, indicating which data IDs should fail.""" 

480 ), 

481) 

482 

483 

484clobber_execution_butler_option = MWOptionDecorator( 

485 "--clobber-execution-butler", 

486 help=unwrap( 

487 """When creating execution butler overwrite 

488 any existing products""" 

489 ), 

490 is_flag=True, 

491) 

492 

493target_datastore_root_option = MWOptionDecorator( 

494 "--target-datastore-root", 

495 help=unwrap( 

496 """Root directory for datastore of execution butler. 

497 Default is to use the original datastore. 

498 """ 

499 ), 

500) 

501 

502dataset_query_constraint = MWOptionDecorator( 

503 "--dataset-query-constraint", 

504 help=unwrap( 

505 """When constructing a quantum graph constrain by 

506 pre-existence of specified dataset types. Valid 

507 values are `all` for all inputs dataset types in 

508 pipeline, ``off`` to not consider dataset type 

509 existence as a constraint, single or comma 

510 separated list of dataset type names.""" 

511 ), 

512 default="all", 

513) 

514 

515summary_option = MWOptionDecorator( 

516 "--summary", 

517 help=( 

518 "Location for storing job summary (JSON file). Note that the" 

519 " structure of this file may not be stable." 

520 ), 

521 type=MWPath(dir_okay=False, file_okay=True, writable=True), 

522) 

523 

524 

525recursive_option = MWOptionDecorator( 

526 "--recursive", 

527 is_flag=True, 

528) 

529 

530config_search_path_option = MWOptionDecorator( 

531 "--config-search-path", 

532 callback=split_commas, 

533 default=list(), 

534 help="Additional search paths for butler configuration.", 

535 metavar="PATH", 

536 multiple=True, 

537) 

538 

539update_graph_id_option = MWOptionDecorator( 

540 "--update-graph-id", 

541 help=unwrap("Update graph ID with new unique value."), 

542 is_flag=True, 

543) 

544 

545metadata_run_key_option = MWOptionDecorator( 

546 "--metadata-run-key", 

547 help=( 

548 "Quantum graph metadata key for the name of the output run. " 

549 "Empty string disables update of the metadata. " 

550 "Default value: output_run." 

551 ), 

552 default="output_run", 

553)