Coverage for python/lsst/ctrl/mpexec/cli/opt/options.py: 100%

51 statements  

« prev     ^ index     » next       coverage.py v7.2.5, created at 2023-05-05 10:47 +0000

1# This file is part of ctrl_mpexec. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22import click 

23from lsst.daf.butler.cli.utils import MWOptionDecorator, MWPath, split_commas, unwrap 

24 

25butler_config_option = MWOptionDecorator( 

26 "-b", "--butler-config", help="Location of the gen3 butler/registry config file." 

27) 

28 

29 

30data_query_option = MWOptionDecorator( 

31 "-d", "--data-query", help="User data selection expression.", metavar="QUERY" 

32) 

33 

34 

35debug_option = MWOptionDecorator( 

36 "--debug", help="Enable debugging output using lsstDebug facility (imports debug.py).", is_flag=True 

37) 

38 

39coverage_option = MWOptionDecorator("--coverage", help="Enable coverage output.", is_flag=True) 

40 

41coverage_packages_option = MWOptionDecorator( 

42 "--cov-packages", 

43 help=unwrap( 

44 """Python packages to restrict coverage to. If none are provided, runs coverage on all packages.""" 

45 ), 

46 multiple=True, 

47 callback=split_commas, 

48) 

49 

50delete_option = MWOptionDecorator( 

51 "--delete", callback=split_commas, help="Delete task with given label from pipeline.", multiple=True 

52) 

53 

54 

55pdb_option = MWOptionDecorator( 

56 "--pdb", 

57 help="Post-mortem debugger to launch for exceptions (defaults to pdb if unspecified; requires a tty).", 

58 is_flag=False, 

59 flag_value="pdb", 

60 default=None, 

61) 

62 

63 

64extend_run_option = MWOptionDecorator( 

65 "--extend-run", 

66 help=unwrap( 

67 """Instead of creating a new RUN collection, insert datasets 

68 into either the one given by --output-run (if provided) or 

69 the first child collection of --output (which must be of 

70 type RUN). This also enables --skip-existing option.""" 

71 ), 

72 is_flag=True, 

73) 

74 

75 

76graph_fixup_option = MWOptionDecorator( 

77 "--graph-fixup", 

78 help=unwrap( 

79 """Name of the class or factory method which makes an 

80 instance used for execution graph fixup.""" 

81 ), 

82) 

83 

84 

85init_only_option = MWOptionDecorator( 

86 "--init-only", 

87 help=unwrap( 

88 """Do not actually run; just register dataset types and/or 

89 save init outputs. """ 

90 ), 

91 is_flag=True, 

92) 

93 

94 

95input_option = MWOptionDecorator( 

96 "-i", 

97 "--input", 

98 callback=split_commas, 

99 default=list(), 

100 help=unwrap("""Comma-separated names of the input collection(s)."""), 

101 metavar="COLLECTION", 

102 multiple=True, 

103) 

104 

105no_versions_option = MWOptionDecorator( 

106 "--no-versions", help="Do not save or check package versions.", is_flag=True 

107) 

108 

109 

110order_pipeline_option = MWOptionDecorator( 

111 "--order-pipeline", 

112 help=unwrap( 

113 """Order tasks in pipeline based on their data 

114 dependencies, ordering is performed as last step before saving or 

115 executing pipeline.""" 

116 ), 

117 is_flag=True, 

118) 

119 

120 

121output_option = MWOptionDecorator( 

122 "-o", 

123 "--output", 

124 help=unwrap( 

125 """Name of the output CHAINED collection. This may either be an 

126 existing CHAINED collection to use as both input and output 

127 (incompatible with --input), or a new CHAINED collection created 

128 to include all inputs (requires --input). In both cases, the 

129 collection's children will start with an output RUN collection 

130 that directly holds all new datasets (see --output-run).""" 

131 ), 

132 metavar="COLL", 

133) 

134 

135 

136output_run_option = MWOptionDecorator( 

137 "--output-run", 

138 help=unwrap( 

139 """Name of the new output RUN collection. If not provided 

140 then --output must be provided and a new RUN collection will 

141 be created by appending a timestamp to the value passed with 

142 --output. If this collection already exists then 

143 --extend-run must be passed.""" 

144 ), 

145 metavar="COLL", 

146) 

147 

148 

149pipeline_option = MWOptionDecorator( 

150 "-p", 

151 "--pipeline", 

152 help="Location of a pipeline definition file in YAML format.", 

153 type=MWPath(file_okay=True, dir_okay=False, readable=True), 

154) 

155 

156 

157pipeline_dot_option = MWOptionDecorator( 

158 "--pipeline-dot", 

159 help=unwrap( 

160 """"Location for storing GraphViz DOT representation of a 

161 pipeline.""" 

162 ), 

163 type=MWPath(writable=True, file_okay=True, dir_okay=False), 

164) 

165 

166 

167profile_option = MWOptionDecorator( 

168 "--profile", help="Dump cProfile statistics to file name.", type=MWPath(file_okay=True, dir_okay=False) 

169) 

170 

171 

172prune_replaced_option = MWOptionDecorator( 

173 "--prune-replaced", 

174 help=unwrap( 

175 """Delete the datasets in the collection replaced by 

176 --replace-run, either just from the datastore 

177 ('unstore') or by removing them and the RUN completely 

178 ('purge'). Requires --replace-run.""" 

179 ), 

180 type=click.Choice(choices=("unstore", "purge"), case_sensitive=False), 

181) 

182 

183 

184qgraph_option = MWOptionDecorator( 

185 "-g", 

186 "--qgraph", 

187 help=unwrap( 

188 """Location for a serialized quantum graph definition (pickle 

189 file). If this option is given then all input data options and 

190 pipeline-building options cannot be used. Can be a URI.""" 

191 ), 

192) 

193 

194 

195qgraph_id_option = MWOptionDecorator( 

196 "--qgraph-id", 

197 help=unwrap( 

198 """Quantum graph identifier, if specified must match the 

199 identifier of the graph loaded from a file. Ignored if graph 

200 is not loaded from a file.""" 

201 ), 

202) 

203 

204 

205qgraph_datastore_records_option = MWOptionDecorator( 

206 "--qgraph-datastore-records", 

207 help=unwrap( 

208 """Include datastore records into generated quantum graph, these records are used by a 

209 quantum-backed butler. 

210 """ 

211 ), 

212 is_flag=True, 

213) 

214 

215 

216# I wanted to use default=None here to match Python API but click silently 

217# replaces None with an empty tuple when multiple=True. 

218qgraph_node_id_option = MWOptionDecorator( 

219 "--qgraph-node-id", 

220 callback=split_commas, 

221 multiple=True, 

222 help=unwrap( 

223 """Only load a specified set of nodes when graph is 

224 loaded from a file, nodes are identified by UUID 

225 values. One or more comma-separated integers are 

226 accepted. By default all nodes are loaded. Ignored if 

227 graph is not loaded from a file.""" 

228 ), 

229) 

230 

231qgraph_header_data_option = MWOptionDecorator( 

232 "--show-qgraph-header", 

233 is_flag=True, 

234 default=False, 

235 help=unwrap( 

236 """Print the headerData for Quantum Graph to the 

237 console""" 

238 ), 

239) 

240 

241qgraph_dot_option = MWOptionDecorator( 

242 "--qgraph-dot", 

243 help=unwrap( 

244 """Location for storing GraphViz DOT representation of a 

245 quantum graph.""" 

246 ), 

247 type=MWPath(writable=True, file_okay=True, dir_okay=False), 

248) 

249 

250 

251replace_run_option = MWOptionDecorator( 

252 "--replace-run", 

253 help=unwrap( 

254 """Before creating a new RUN collection in an existing 

255 CHAINED collection, remove the first child collection 

256 (which must be of type RUN). This can be used to repeatedly 

257 write to the same (parent) collection during development, 

258 but it does not delete the datasets associated with the 

259 replaced run unless --prune-replaced is also passed. 

260 Requires --output, and incompatible with --extend-run.""" 

261 ), 

262 is_flag=True, 

263) 

264 

265 

266save_pipeline_option = MWOptionDecorator( 

267 "-s", 

268 "--save-pipeline", 

269 help=unwrap( 

270 """Location for storing resulting pipeline definition in 

271 YAML format.""" 

272 ), 

273 type=MWPath(dir_okay=False, file_okay=True, writable=True), 

274) 

275 

276save_qgraph_option = MWOptionDecorator( 

277 "-q", 

278 "--save-qgraph", 

279 help=unwrap( 

280 """URI location for storing a serialized quantum graph 

281 definition (pickle file).""" 

282 ), 

283) 

284 

285 

286save_single_quanta_option = MWOptionDecorator( 

287 "--save-single-quanta", 

288 help=unwrap( 

289 """Format string of locations for storing individual 

290 quantum graph definition (pickle files). The curly 

291 brace {} in the input string will be replaced by a 

292 quantum number. Can be a URI.""" 

293 ), 

294) 

295 

296 

297show_option = MWOptionDecorator( 

298 "--show", 

299 callback=split_commas, 

300 help=unwrap( 

301 """Dump various info to standard output. Possible items are: 

302 `config`, `config=[Task::]<PATTERN>` or 

303 `config=[Task::]<PATTERN>:NOIGNORECASE` to dump configuration 

304 fields possibly matching given pattern and/or task label; 

305 `history=<FIELD>` to dump configuration history for a field, field 

306 name is specified as [Task::]<PATTERN>; `dump-config`, 

307 `dump-config=Task` to dump complete configuration for a task given 

308 its label or all tasks; `pipeline` to show pipeline composition; 

309 `graph` to show information about quanta; `workflow` to show 

310 information about quanta and their dependency; `tasks` to show 

311 task composition; `uri` to show predicted dataset URIs of 

312 quanta""" 

313 ), 

314 metavar="ITEM|ITEM=VALUE", 

315 multiple=True, 

316) 

317 

318 

319skip_existing_in_option = MWOptionDecorator( 

320 "--skip-existing-in", 

321 callback=split_commas, 

322 default=None, 

323 metavar="COLLECTION", 

324 multiple=True, 

325 help=unwrap( 

326 """If all Quantum outputs already exist in the specified list of 

327 collections then that Quantum will be excluded from the QuantumGraph. 

328 """ 

329 ), 

330) 

331 

332 

333skip_existing_option = MWOptionDecorator( 

334 "--skip-existing", 

335 is_flag=True, 

336 help=unwrap( 

337 """This option is equivalent to --skip-existing-in with the name of 

338 the output RUN collection. If both --skip-existing-in and 

339 --skip-existing are given then output RUN collection is appended to 

340 the list of collections.""" 

341 ), 

342) 

343 

344 

345clobber_outputs_option = MWOptionDecorator( 

346 "--clobber-outputs", 

347 help=unwrap( 

348 """Remove outputs from previous execution of the same 

349 quantum before new execution. If --skip-existing 

350 is also passed, then only failed quanta will be 

351 clobbered. Requires the 'run' command's --extend-run 

352 flag to be set.""" 

353 ), 

354 is_flag=True, 

355) 

356 

357 

358skip_init_writes_option = MWOptionDecorator( 

359 "--skip-init-writes", 

360 help=unwrap( 

361 """Do not write collection-wide 'init output' datasets 

362 (e.g.schemas).""" 

363 ), 

364 is_flag=True, 

365) 

366 

367 

368enable_implicit_threading_option = MWOptionDecorator( 

369 "--enable-implicit-threading", 

370 help=unwrap( 

371 """Do not disable implicit threading use by third-party libraries (e.g. OpenBLAS). 

372 Implicit threading is always disabled during execution with multiprocessing.""" 

373 ), 

374 is_flag=True, 

375) 

376 

377 

378task_option = MWOptionDecorator( 

379 "-t", 

380 "--task", 

381 callback=split_commas, 

382 help=unwrap( 

383 """Task name to add to pipeline, must be a fully qualified task 

384 name. Task name can be followed by colon and label name, if label 

385 is not given then task base name (class name) is used as 

386 label.""" 

387 ), 

388 metavar="TASK[:LABEL]", 

389 multiple=True, 

390) 

391 

392 

393timeout_option = MWOptionDecorator( 

394 "--timeout", type=click.IntRange(min=0), help="Timeout for multiprocessing; maximum wall time (sec)." 

395) 

396 

397 

398start_method_option = MWOptionDecorator( 

399 "--start-method", 

400 default=None, 

401 type=click.Choice(choices=["spawn", "fork", "forkserver"]), 

402 help="Multiprocessing start method, default is platform-specific.", 

403) 

404 

405 

406fail_fast_option = MWOptionDecorator( 

407 "--fail-fast", 

408 help=unwrap( 

409 """Stop processing at first error, default is to process 

410 as many tasks as possible.""" 

411 ), 

412 is_flag=True, 

413) 

414 

415save_execution_butler_option = MWOptionDecorator( 

416 "--save-execution-butler", 

417 help=unwrap( 

418 """Export location for an 

419 execution-specific butler after making 

420 QuantumGraph""" 

421 ), 

422) 

423 

424mock_option = MWOptionDecorator( 

425 "--mock", 

426 help=unwrap("""Mock pipeline execution."""), 

427 is_flag=True, 

428) 

429 

430clobber_execution_butler_option = MWOptionDecorator( 

431 "--clobber-execution-butler", 

432 help=unwrap( 

433 """When creating execution butler overwrite 

434 any existing products""" 

435 ), 

436 is_flag=True, 

437) 

438 

439target_datastore_root_option = MWOptionDecorator( 

440 "--target-datastore-root", 

441 help=unwrap( 

442 """Root directory for datastore of execution butler. 

443 Default is to use the original datastore. 

444 """ 

445 ), 

446) 

447 

448dataset_query_constraint = MWOptionDecorator( 

449 "--dataset-query-constraint", 

450 help=unwrap( 

451 """When constructing a quantum graph constrain by 

452 pre-existence of specified dataset types. Valid 

453 values are `all` for all inputs dataset types in 

454 pipeline, `off` to not consider dataset type 

455 existance as a constraint, single or comma 

456 separated list of dataset type names""" 

457 ), 

458 default="all", 

459) 

460 

461summary_option = MWOptionDecorator( 

462 "--summary", 

463 help=( 

464 "Location for storing job summary (JSON file). Note that the" 

465 " structure of this file may not be stable." 

466 ), 

467 type=MWPath(dir_okay=False, file_okay=True, writable=True), 

468) 

469 

470 

471recursive_option = MWOptionDecorator( 

472 "--recursive", 

473 is_flag=True, 

474) 

475 

476config_search_path_option = MWOptionDecorator( 

477 "--config-search-path", 

478 callback=split_commas, 

479 default=list(), 

480 help="Additional search paths for butler configuration.", 

481 metavar="PATH", 

482 multiple=True, 

483) 

484 

485update_graph_id_option = MWOptionDecorator( 

486 "--update-graph-id", 

487 help=unwrap("Update graph ID with new unique value."), 

488 is_flag=True, 

489) 

490 

491metadata_run_key_option = MWOptionDecorator( 

492 "--metadata-run-key", 

493 help=( 

494 "Quantum graph metadata key for the name of the output run. " 

495 "Empty string disables update of the metadata. " 

496 "Default value: output_run." 

497 ), 

498 default="output_run", 

499)