Coverage for python/lsst/ctrl/mpexec/cli/opt/options.py: 100%

53 statements  

« prev     ^ index     » next       coverage.py v7.2.7, created at 2023-06-09 02:48 -0700

1# This file is part of ctrl_mpexec. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22import click 

23from lsst.daf.butler.cli.utils import MWOptionDecorator, MWPath, split_commas, unwrap 

24 

25butler_config_option = MWOptionDecorator( 

26 "-b", "--butler-config", help="Location of the gen3 butler/registry config file." 

27) 

28 

29 

30data_query_option = MWOptionDecorator( 

31 "-d", "--data-query", help="User data selection expression.", metavar="QUERY" 

32) 

33 

34 

35debug_option = MWOptionDecorator( 

36 "--debug", help="Enable debugging output using lsstDebug facility (imports debug.py).", is_flag=True 

37) 

38 

39coverage_option = MWOptionDecorator("--coverage", help="Enable coverage output.", is_flag=True) 

40 

41coverage_report_option = MWOptionDecorator( 

42 "--cov-report/--no-cov-report", 

43 help="If coverage is enabled, controls whether to produce an HTML coverage report.", 

44 default=True, 

45) 

46 

47coverage_packages_option = MWOptionDecorator( 

48 "--cov-packages", 

49 help=unwrap( 

50 """Python packages to restrict coverage to. If none are provided, runs coverage on all packages.""" 

51 ), 

52 multiple=True, 

53 callback=split_commas, 

54) 

55 

56delete_option = MWOptionDecorator( 

57 "--delete", callback=split_commas, help="Delete task with given label from pipeline.", multiple=True 

58) 

59 

60 

61pdb_option = MWOptionDecorator( 

62 "--pdb", 

63 help="Post-mortem debugger to launch for exceptions (defaults to pdb if unspecified; requires a tty).", 

64 is_flag=False, 

65 flag_value="pdb", 

66 default=None, 

67) 

68 

69 

70extend_run_option = MWOptionDecorator( 

71 "--extend-run", 

72 help=( 

73 "Instead of creating a new RUN collection, insert datasets into either the one given by " 

74 "--output-run (if provided) or the first child collection of --output (which must be of type RUN). " 

75 "This also enables --skip-existing option when building a graph. " 

76 "When executing a graph this option skips quanta with all existing outputs." 

77 ), 

78 is_flag=True, 

79) 

80 

81 

82graph_fixup_option = MWOptionDecorator( 

83 "--graph-fixup", 

84 help=unwrap( 

85 """Name of the class or factory method which makes an 

86 instance used for execution graph fixup.""" 

87 ), 

88) 

89 

90 

91init_only_option = MWOptionDecorator( 

92 "--init-only", 

93 help=unwrap( 

94 """Do not actually run; just register dataset types and/or 

95 save init outputs. """ 

96 ), 

97 is_flag=True, 

98) 

99 

100 

101input_option = MWOptionDecorator( 

102 "-i", 

103 "--input", 

104 callback=split_commas, 

105 default=list(), 

106 help=unwrap("""Comma-separated names of the input collection(s)."""), 

107 metavar="COLLECTION", 

108 multiple=True, 

109) 

110 

111no_versions_option = MWOptionDecorator( 

112 "--no-versions", help="Do not save or check package versions.", is_flag=True 

113) 

114 

115 

116order_pipeline_option = MWOptionDecorator( 

117 "--order-pipeline", 

118 help=unwrap( 

119 """Order tasks in pipeline based on their data 

120 dependencies, ordering is performed as last step before saving or 

121 executing pipeline.""" 

122 ), 

123 is_flag=True, 

124) 

125 

126 

127output_option = MWOptionDecorator( 

128 "-o", 

129 "--output", 

130 help=unwrap( 

131 """Name of the output CHAINED collection. This may either be an 

132 existing CHAINED collection to use as both input and output 

133 (incompatible with --input), or a new CHAINED collection created 

134 to include all inputs (requires --input). In both cases, the 

135 collection's children will start with an output RUN collection 

136 that directly holds all new datasets (see --output-run).""" 

137 ), 

138 metavar="COLL", 

139) 

140 

141 

142output_run_option = MWOptionDecorator( 

143 "--output-run", 

144 help=unwrap( 

145 """Name of the new output RUN collection. If not provided 

146 then --output must be provided and a new RUN collection will 

147 be created by appending a timestamp to the value passed with 

148 --output. If this collection already exists then 

149 --extend-run must be passed.""" 

150 ), 

151 metavar="COLL", 

152) 

153 

154 

155pipeline_option = MWOptionDecorator( 

156 "-p", 

157 "--pipeline", 

158 help="Location of a pipeline definition file in YAML format.", 

159 type=MWPath(file_okay=True, dir_okay=False, readable=True), 

160) 

161 

162 

163pipeline_dot_option = MWOptionDecorator( 

164 "--pipeline-dot", 

165 help=unwrap( 

166 """"Location for storing GraphViz DOT representation of a 

167 pipeline.""" 

168 ), 

169 type=MWPath(writable=True, file_okay=True, dir_okay=False), 

170) 

171 

172 

173profile_option = MWOptionDecorator( 

174 "--profile", help="Dump cProfile statistics to file name.", type=MWPath(file_okay=True, dir_okay=False) 

175) 

176 

177 

178prune_replaced_option = MWOptionDecorator( 

179 "--prune-replaced", 

180 help=unwrap( 

181 """Delete the datasets in the collection replaced by 

182 --replace-run, either just from the datastore 

183 ('unstore') or by removing them and the RUN completely 

184 ('purge'). Requires --replace-run.""" 

185 ), 

186 type=click.Choice(choices=("unstore", "purge"), case_sensitive=False), 

187) 

188 

189 

190qgraph_option = MWOptionDecorator( 

191 "-g", 

192 "--qgraph", 

193 help=unwrap( 

194 """Location for a serialized quantum graph definition (pickle 

195 file). If this option is given then all input data options and 

196 pipeline-building options cannot be used. Can be a URI.""" 

197 ), 

198) 

199 

200 

201qgraph_id_option = MWOptionDecorator( 

202 "--qgraph-id", 

203 help=unwrap( 

204 """Quantum graph identifier, if specified must match the 

205 identifier of the graph loaded from a file. Ignored if graph 

206 is not loaded from a file.""" 

207 ), 

208) 

209 

210 

211qgraph_datastore_records_option = MWOptionDecorator( 

212 "--qgraph-datastore-records", 

213 help=unwrap( 

214 """Include datastore records into generated quantum graph, these records are used by a 

215 quantum-backed butler. 

216 """ 

217 ), 

218 is_flag=True, 

219) 

220 

221 

222# I wanted to use default=None here to match Python API but click silently 

223# replaces None with an empty tuple when multiple=True. 

224qgraph_node_id_option = MWOptionDecorator( 

225 "--qgraph-node-id", 

226 callback=split_commas, 

227 multiple=True, 

228 help=unwrap( 

229 """Only load a specified set of nodes when graph is 

230 loaded from a file, nodes are identified by UUID 

231 values. One or more comma-separated integers are 

232 accepted. By default all nodes are loaded. Ignored if 

233 graph is not loaded from a file.""" 

234 ), 

235) 

236 

237qgraph_header_data_option = MWOptionDecorator( 

238 "--show-qgraph-header", 

239 is_flag=True, 

240 default=False, 

241 help=unwrap( 

242 """Print the headerData for Quantum Graph to the 

243 console""" 

244 ), 

245) 

246 

247qgraph_dot_option = MWOptionDecorator( 

248 "--qgraph-dot", 

249 help=unwrap( 

250 """Location for storing GraphViz DOT representation of a 

251 quantum graph.""" 

252 ), 

253 type=MWPath(writable=True, file_okay=True, dir_okay=False), 

254) 

255 

256 

257replace_run_option = MWOptionDecorator( 

258 "--replace-run", 

259 help=unwrap( 

260 """Before creating a new RUN collection in an existing 

261 CHAINED collection, remove the first child collection 

262 (which must be of type RUN). This can be used to repeatedly 

263 write to the same (parent) collection during development, 

264 but it does not delete the datasets associated with the 

265 replaced run unless --prune-replaced is also passed. 

266 Requires --output, and incompatible with --extend-run.""" 

267 ), 

268 is_flag=True, 

269) 

270 

271 

272save_pipeline_option = MWOptionDecorator( 

273 "-s", 

274 "--save-pipeline", 

275 help=unwrap( 

276 """Location for storing resulting pipeline definition in 

277 YAML format.""" 

278 ), 

279 type=MWPath(dir_okay=False, file_okay=True, writable=True), 

280) 

281 

282save_qgraph_option = MWOptionDecorator( 

283 "-q", 

284 "--save-qgraph", 

285 help=unwrap( 

286 """URI location for storing a serialized quantum graph 

287 definition (pickle file).""" 

288 ), 

289) 

290 

291 

292save_single_quanta_option = MWOptionDecorator( 

293 "--save-single-quanta", 

294 help=unwrap( 

295 """Format string of locations for storing individual 

296 quantum graph definition (pickle files). The curly 

297 brace {} in the input string will be replaced by a 

298 quantum number. Can be a URI.""" 

299 ), 

300) 

301 

302 

303show_option = MWOptionDecorator( 

304 "--show", 

305 callback=split_commas, 

306 help=unwrap( 

307 """Dump various info to standard output. Possible items are: 

308 `config`, `config=[Task::]<PATTERN>` or 

309 `config=[Task::]<PATTERN>:NOIGNORECASE` to dump configuration 

310 fields possibly matching given pattern and/or task label; 

311 `history=<FIELD>` to dump configuration history for a field, field 

312 name is specified as [Task::]<PATTERN>; `dump-config`, 

313 `dump-config=Task` to dump complete configuration for a task given 

314 its label or all tasks; `pipeline` to show pipeline composition; 

315 `graph` to show information about quanta; `workflow` to show 

316 information about quanta and their dependency; `tasks` to show 

317 task composition; `uri` to show predicted dataset URIs of 

318 quanta""" 

319 ), 

320 metavar="ITEM|ITEM=VALUE", 

321 multiple=True, 

322) 

323 

324 

325skip_existing_in_option = MWOptionDecorator( 

326 "--skip-existing-in", 

327 callback=split_commas, 

328 default=None, 

329 metavar="COLLECTION", 

330 multiple=True, 

331 help=unwrap( 

332 """If all Quantum outputs already exist in the specified list of 

333 collections then that Quantum will be excluded from the QuantumGraph. 

334 """ 

335 ), 

336) 

337 

338 

339skip_existing_option = MWOptionDecorator( 

340 "--skip-existing", 

341 is_flag=True, 

342 help=unwrap( 

343 """This option is equivalent to --skip-existing-in with the name of 

344 the output RUN collection. If both --skip-existing-in and 

345 --skip-existing are given then output RUN collection is appended to 

346 the list of collections.""" 

347 ), 

348) 

349 

350 

351clobber_outputs_option = MWOptionDecorator( 

352 "--clobber-outputs", 

353 help=( 

354 "Remove outputs of failed quanta from the output run when they would block the execution of new " 

355 "quanta with the same data ID (or assume that this will be done, if just building a QuantumGraph). " 

356 "Does nothing if --extend-run is not passed." 

357 ), 

358 is_flag=True, 

359) 

360 

361 

362skip_init_writes_option = MWOptionDecorator( 

363 "--skip-init-writes", 

364 help=unwrap( 

365 """Do not write collection-wide 'init output' datasets 

366 (e.g.schemas).""" 

367 ), 

368 is_flag=True, 

369) 

370 

371 

372enable_implicit_threading_option = MWOptionDecorator( 

373 "--enable-implicit-threading", 

374 help=unwrap( 

375 """Do not disable implicit threading use by third-party libraries (e.g. OpenBLAS). 

376 Implicit threading is always disabled during execution with multiprocessing.""" 

377 ), 

378 is_flag=True, 

379) 

380 

381 

382task_option = MWOptionDecorator( 

383 "-t", 

384 "--task", 

385 callback=split_commas, 

386 help=unwrap( 

387 """Task name to add to pipeline, must be a fully qualified task 

388 name. Task name can be followed by colon and label name, if label 

389 is not given then task base name (class name) is used as 

390 label.""" 

391 ), 

392 metavar="TASK[:LABEL]", 

393 multiple=True, 

394) 

395 

396 

397timeout_option = MWOptionDecorator( 

398 "--timeout", type=click.IntRange(min=0), help="Timeout for multiprocessing; maximum wall time (sec)." 

399) 

400 

401 

402start_method_option = MWOptionDecorator( 

403 "--start-method", 

404 default=None, 

405 type=click.Choice(choices=["spawn", "fork", "forkserver"]), 

406 help="Multiprocessing start method, default is platform-specific.", 

407) 

408 

409 

410fail_fast_option = MWOptionDecorator( 

411 "--fail-fast", 

412 help=unwrap( 

413 """Stop processing at first error, default is to process 

414 as many tasks as possible.""" 

415 ), 

416 is_flag=True, 

417) 

418 

419save_execution_butler_option = MWOptionDecorator( 

420 "--save-execution-butler", 

421 help=unwrap( 

422 """Export location for an 

423 execution-specific butler after making 

424 QuantumGraph""" 

425 ), 

426) 

427 

428mock_option = MWOptionDecorator( 

429 "--mock", 

430 help=unwrap("""Mock pipeline execution."""), 

431 is_flag=True, 

432) 

433 

434unmocked_dataset_types_option = MWOptionDecorator( 

435 "--unmocked-dataset-types", 

436 callback=split_commas, 

437 default=None, 

438 metavar="COLLECTION", 

439 multiple=True, 

440 help=unwrap("""Names of input dataset types that should not be mocked."""), 

441) 

442 

443clobber_execution_butler_option = MWOptionDecorator( 

444 "--clobber-execution-butler", 

445 help=unwrap( 

446 """When creating execution butler overwrite 

447 any existing products""" 

448 ), 

449 is_flag=True, 

450) 

451 

452target_datastore_root_option = MWOptionDecorator( 

453 "--target-datastore-root", 

454 help=unwrap( 

455 """Root directory for datastore of execution butler. 

456 Default is to use the original datastore. 

457 """ 

458 ), 

459) 

460 

461dataset_query_constraint = MWOptionDecorator( 

462 "--dataset-query-constraint", 

463 help=unwrap( 

464 """When constructing a quantum graph constrain by 

465 pre-existence of specified dataset types. Valid 

466 values are `all` for all inputs dataset types in 

467 pipeline, `off` to not consider dataset type 

468 existance as a constraint, single or comma 

469 separated list of dataset type names""" 

470 ), 

471 default="all", 

472) 

473 

474summary_option = MWOptionDecorator( 

475 "--summary", 

476 help=( 

477 "Location for storing job summary (JSON file). Note that the" 

478 " structure of this file may not be stable." 

479 ), 

480 type=MWPath(dir_okay=False, file_okay=True, writable=True), 

481) 

482 

483 

484recursive_option = MWOptionDecorator( 

485 "--recursive", 

486 is_flag=True, 

487) 

488 

489config_search_path_option = MWOptionDecorator( 

490 "--config-search-path", 

491 callback=split_commas, 

492 default=list(), 

493 help="Additional search paths for butler configuration.", 

494 metavar="PATH", 

495 multiple=True, 

496) 

497 

498update_graph_id_option = MWOptionDecorator( 

499 "--update-graph-id", 

500 help=unwrap("Update graph ID with new unique value."), 

501 is_flag=True, 

502) 

503 

504metadata_run_key_option = MWOptionDecorator( 

505 "--metadata-run-key", 

506 help=( 

507 "Quantum graph metadata key for the name of the output run. " 

508 "Empty string disables update of the metadata. " 

509 "Default value: output_run." 

510 ), 

511 default="output_run", 

512)