Coverage for python/lsst/ctrl/mpexec/cli/opt/options.py: 100%

51 statements  

« prev     ^ index     » next       coverage.py v7.2.6, created at 2023-05-27 02:12 -0700

1# This file is part of ctrl_mpexec. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22import click 

23from lsst.daf.butler.cli.utils import MWOptionDecorator, MWPath, split_commas, unwrap 

24 

25butler_config_option = MWOptionDecorator( 

26 "-b", "--butler-config", help="Location of the gen3 butler/registry config file." 

27) 

28 

29 

30data_query_option = MWOptionDecorator( 

31 "-d", "--data-query", help="User data selection expression.", metavar="QUERY" 

32) 

33 

34 

35debug_option = MWOptionDecorator( 

36 "--debug", help="Enable debugging output using lsstDebug facility (imports debug.py).", is_flag=True 

37) 

38 

39coverage_option = MWOptionDecorator("--coverage", help="Enable coverage output.", is_flag=True) 

40 

41coverage_packages_option = MWOptionDecorator( 

42 "--cov-packages", 

43 help=unwrap( 

44 """Python packages to restrict coverage to. If none are provided, runs coverage on all packages.""" 

45 ), 

46 multiple=True, 

47 callback=split_commas, 

48) 

49 

50delete_option = MWOptionDecorator( 

51 "--delete", callback=split_commas, help="Delete task with given label from pipeline.", multiple=True 

52) 

53 

54 

55pdb_option = MWOptionDecorator( 

56 "--pdb", 

57 help="Post-mortem debugger to launch for exceptions (defaults to pdb if unspecified; requires a tty).", 

58 is_flag=False, 

59 flag_value="pdb", 

60 default=None, 

61) 

62 

63 

64extend_run_option = MWOptionDecorator( 

65 "--extend-run", 

66 help=( 

67 "Instead of creating a new RUN collection, insert datasets into either the one given by " 

68 "--output-run (if provided) or the first child collection of --output (which must be of type RUN). " 

69 "This also enables --skip-existing option when building a graph. " 

70 "When executing a graph this option skips quanta with all existing outputs." 

71 ), 

72 is_flag=True, 

73) 

74 

75 

76graph_fixup_option = MWOptionDecorator( 

77 "--graph-fixup", 

78 help=unwrap( 

79 """Name of the class or factory method which makes an 

80 instance used for execution graph fixup.""" 

81 ), 

82) 

83 

84 

85init_only_option = MWOptionDecorator( 

86 "--init-only", 

87 help=unwrap( 

88 """Do not actually run; just register dataset types and/or 

89 save init outputs. """ 

90 ), 

91 is_flag=True, 

92) 

93 

94 

95input_option = MWOptionDecorator( 

96 "-i", 

97 "--input", 

98 callback=split_commas, 

99 default=list(), 

100 help=unwrap("""Comma-separated names of the input collection(s)."""), 

101 metavar="COLLECTION", 

102 multiple=True, 

103) 

104 

105no_versions_option = MWOptionDecorator( 

106 "--no-versions", help="Do not save or check package versions.", is_flag=True 

107) 

108 

109 

110order_pipeline_option = MWOptionDecorator( 

111 "--order-pipeline", 

112 help=unwrap( 

113 """Order tasks in pipeline based on their data 

114 dependencies, ordering is performed as last step before saving or 

115 executing pipeline.""" 

116 ), 

117 is_flag=True, 

118) 

119 

120 

121output_option = MWOptionDecorator( 

122 "-o", 

123 "--output", 

124 help=unwrap( 

125 """Name of the output CHAINED collection. This may either be an 

126 existing CHAINED collection to use as both input and output 

127 (incompatible with --input), or a new CHAINED collection created 

128 to include all inputs (requires --input). In both cases, the 

129 collection's children will start with an output RUN collection 

130 that directly holds all new datasets (see --output-run).""" 

131 ), 

132 metavar="COLL", 

133) 

134 

135 

136output_run_option = MWOptionDecorator( 

137 "--output-run", 

138 help=unwrap( 

139 """Name of the new output RUN collection. If not provided 

140 then --output must be provided and a new RUN collection will 

141 be created by appending a timestamp to the value passed with 

142 --output. If this collection already exists then 

143 --extend-run must be passed.""" 

144 ), 

145 metavar="COLL", 

146) 

147 

148 

149pipeline_option = MWOptionDecorator( 

150 "-p", 

151 "--pipeline", 

152 help="Location of a pipeline definition file in YAML format.", 

153 type=MWPath(file_okay=True, dir_okay=False, readable=True), 

154) 

155 

156 

157pipeline_dot_option = MWOptionDecorator( 

158 "--pipeline-dot", 

159 help=unwrap( 

160 """"Location for storing GraphViz DOT representation of a 

161 pipeline.""" 

162 ), 

163 type=MWPath(writable=True, file_okay=True, dir_okay=False), 

164) 

165 

166 

167profile_option = MWOptionDecorator( 

168 "--profile", help="Dump cProfile statistics to file name.", type=MWPath(file_okay=True, dir_okay=False) 

169) 

170 

171 

172prune_replaced_option = MWOptionDecorator( 

173 "--prune-replaced", 

174 help=unwrap( 

175 """Delete the datasets in the collection replaced by 

176 --replace-run, either just from the datastore 

177 ('unstore') or by removing them and the RUN completely 

178 ('purge'). Requires --replace-run.""" 

179 ), 

180 type=click.Choice(choices=("unstore", "purge"), case_sensitive=False), 

181) 

182 

183 

184qgraph_option = MWOptionDecorator( 

185 "-g", 

186 "--qgraph", 

187 help=unwrap( 

188 """Location for a serialized quantum graph definition (pickle 

189 file). If this option is given then all input data options and 

190 pipeline-building options cannot be used. Can be a URI.""" 

191 ), 

192) 

193 

194 

195qgraph_id_option = MWOptionDecorator( 

196 "--qgraph-id", 

197 help=unwrap( 

198 """Quantum graph identifier, if specified must match the 

199 identifier of the graph loaded from a file. Ignored if graph 

200 is not loaded from a file.""" 

201 ), 

202) 

203 

204 

205qgraph_datastore_records_option = MWOptionDecorator( 

206 "--qgraph-datastore-records", 

207 help=unwrap( 

208 """Include datastore records into generated quantum graph, these records are used by a 

209 quantum-backed butler. 

210 """ 

211 ), 

212 is_flag=True, 

213) 

214 

215 

216# I wanted to use default=None here to match Python API but click silently 

217# replaces None with an empty tuple when multiple=True. 

218qgraph_node_id_option = MWOptionDecorator( 

219 "--qgraph-node-id", 

220 callback=split_commas, 

221 multiple=True, 

222 help=unwrap( 

223 """Only load a specified set of nodes when graph is 

224 loaded from a file, nodes are identified by UUID 

225 values. One or more comma-separated integers are 

226 accepted. By default all nodes are loaded. Ignored if 

227 graph is not loaded from a file.""" 

228 ), 

229) 

230 

231qgraph_header_data_option = MWOptionDecorator( 

232 "--show-qgraph-header", 

233 is_flag=True, 

234 default=False, 

235 help=unwrap( 

236 """Print the headerData for Quantum Graph to the 

237 console""" 

238 ), 

239) 

240 

241qgraph_dot_option = MWOptionDecorator( 

242 "--qgraph-dot", 

243 help=unwrap( 

244 """Location for storing GraphViz DOT representation of a 

245 quantum graph.""" 

246 ), 

247 type=MWPath(writable=True, file_okay=True, dir_okay=False), 

248) 

249 

250 

251replace_run_option = MWOptionDecorator( 

252 "--replace-run", 

253 help=unwrap( 

254 """Before creating a new RUN collection in an existing 

255 CHAINED collection, remove the first child collection 

256 (which must be of type RUN). This can be used to repeatedly 

257 write to the same (parent) collection during development, 

258 but it does not delete the datasets associated with the 

259 replaced run unless --prune-replaced is also passed. 

260 Requires --output, and incompatible with --extend-run.""" 

261 ), 

262 is_flag=True, 

263) 

264 

265 

266save_pipeline_option = MWOptionDecorator( 

267 "-s", 

268 "--save-pipeline", 

269 help=unwrap( 

270 """Location for storing resulting pipeline definition in 

271 YAML format.""" 

272 ), 

273 type=MWPath(dir_okay=False, file_okay=True, writable=True), 

274) 

275 

276save_qgraph_option = MWOptionDecorator( 

277 "-q", 

278 "--save-qgraph", 

279 help=unwrap( 

280 """URI location for storing a serialized quantum graph 

281 definition (pickle file).""" 

282 ), 

283) 

284 

285 

286save_single_quanta_option = MWOptionDecorator( 

287 "--save-single-quanta", 

288 help=unwrap( 

289 """Format string of locations for storing individual 

290 quantum graph definition (pickle files). The curly 

291 brace {} in the input string will be replaced by a 

292 quantum number. Can be a URI.""" 

293 ), 

294) 

295 

296 

297show_option = MWOptionDecorator( 

298 "--show", 

299 callback=split_commas, 

300 help=unwrap( 

301 """Dump various info to standard output. Possible items are: 

302 `config`, `config=[Task::]<PATTERN>` or 

303 `config=[Task::]<PATTERN>:NOIGNORECASE` to dump configuration 

304 fields possibly matching given pattern and/or task label; 

305 `history=<FIELD>` to dump configuration history for a field, field 

306 name is specified as [Task::]<PATTERN>; `dump-config`, 

307 `dump-config=Task` to dump complete configuration for a task given 

308 its label or all tasks; `pipeline` to show pipeline composition; 

309 `graph` to show information about quanta; `workflow` to show 

310 information about quanta and their dependency; `tasks` to show 

311 task composition; `uri` to show predicted dataset URIs of 

312 quanta""" 

313 ), 

314 metavar="ITEM|ITEM=VALUE", 

315 multiple=True, 

316) 

317 

318 

319skip_existing_in_option = MWOptionDecorator( 

320 "--skip-existing-in", 

321 callback=split_commas, 

322 default=None, 

323 metavar="COLLECTION", 

324 multiple=True, 

325 help=unwrap( 

326 """If all Quantum outputs already exist in the specified list of 

327 collections then that Quantum will be excluded from the QuantumGraph. 

328 """ 

329 ), 

330) 

331 

332 

333skip_existing_option = MWOptionDecorator( 

334 "--skip-existing", 

335 is_flag=True, 

336 help=unwrap( 

337 """This option is equivalent to --skip-existing-in with the name of 

338 the output RUN collection. If both --skip-existing-in and 

339 --skip-existing are given then output RUN collection is appended to 

340 the list of collections.""" 

341 ), 

342) 

343 

344 

345clobber_outputs_option = MWOptionDecorator( 

346 "--clobber-outputs", 

347 help=( 

348 "Remove outputs of failed quanta from the output run when they would block the execution of new " 

349 "quanta with the same data ID (or assume that this will be done, if just building a QuantumGraph). " 

350 "Does nothing if --extend-run is not passed." 

351 ), 

352 is_flag=True, 

353) 

354 

355 

356skip_init_writes_option = MWOptionDecorator( 

357 "--skip-init-writes", 

358 help=unwrap( 

359 """Do not write collection-wide 'init output' datasets 

360 (e.g.schemas).""" 

361 ), 

362 is_flag=True, 

363) 

364 

365 

366enable_implicit_threading_option = MWOptionDecorator( 

367 "--enable-implicit-threading", 

368 help=unwrap( 

369 """Do not disable implicit threading use by third-party libraries (e.g. OpenBLAS). 

370 Implicit threading is always disabled during execution with multiprocessing.""" 

371 ), 

372 is_flag=True, 

373) 

374 

375 

376task_option = MWOptionDecorator( 

377 "-t", 

378 "--task", 

379 callback=split_commas, 

380 help=unwrap( 

381 """Task name to add to pipeline, must be a fully qualified task 

382 name. Task name can be followed by colon and label name, if label 

383 is not given then task base name (class name) is used as 

384 label.""" 

385 ), 

386 metavar="TASK[:LABEL]", 

387 multiple=True, 

388) 

389 

390 

391timeout_option = MWOptionDecorator( 

392 "--timeout", type=click.IntRange(min=0), help="Timeout for multiprocessing; maximum wall time (sec)." 

393) 

394 

395 

396start_method_option = MWOptionDecorator( 

397 "--start-method", 

398 default=None, 

399 type=click.Choice(choices=["spawn", "fork", "forkserver"]), 

400 help="Multiprocessing start method, default is platform-specific.", 

401) 

402 

403 

404fail_fast_option = MWOptionDecorator( 

405 "--fail-fast", 

406 help=unwrap( 

407 """Stop processing at first error, default is to process 

408 as many tasks as possible.""" 

409 ), 

410 is_flag=True, 

411) 

412 

413save_execution_butler_option = MWOptionDecorator( 

414 "--save-execution-butler", 

415 help=unwrap( 

416 """Export location for an 

417 execution-specific butler after making 

418 QuantumGraph""" 

419 ), 

420) 

421 

422mock_option = MWOptionDecorator( 

423 "--mock", 

424 help=unwrap("""Mock pipeline execution."""), 

425 is_flag=True, 

426) 

427 

428clobber_execution_butler_option = MWOptionDecorator( 

429 "--clobber-execution-butler", 

430 help=unwrap( 

431 """When creating execution butler overwrite 

432 any existing products""" 

433 ), 

434 is_flag=True, 

435) 

436 

437target_datastore_root_option = MWOptionDecorator( 

438 "--target-datastore-root", 

439 help=unwrap( 

440 """Root directory for datastore of execution butler. 

441 Default is to use the original datastore. 

442 """ 

443 ), 

444) 

445 

446dataset_query_constraint = MWOptionDecorator( 

447 "--dataset-query-constraint", 

448 help=unwrap( 

449 """When constructing a quantum graph constrain by 

450 pre-existence of specified dataset types. Valid 

451 values are `all` for all inputs dataset types in 

452 pipeline, `off` to not consider dataset type 

453 existance as a constraint, single or comma 

454 separated list of dataset type names""" 

455 ), 

456 default="all", 

457) 

458 

459summary_option = MWOptionDecorator( 

460 "--summary", 

461 help=( 

462 "Location for storing job summary (JSON file). Note that the" 

463 " structure of this file may not be stable." 

464 ), 

465 type=MWPath(dir_okay=False, file_okay=True, writable=True), 

466) 

467 

468 

469recursive_option = MWOptionDecorator( 

470 "--recursive", 

471 is_flag=True, 

472) 

473 

474config_search_path_option = MWOptionDecorator( 

475 "--config-search-path", 

476 callback=split_commas, 

477 default=list(), 

478 help="Additional search paths for butler configuration.", 

479 metavar="PATH", 

480 multiple=True, 

481) 

482 

483update_graph_id_option = MWOptionDecorator( 

484 "--update-graph-id", 

485 help=unwrap("Update graph ID with new unique value."), 

486 is_flag=True, 

487) 

488 

489metadata_run_key_option = MWOptionDecorator( 

490 "--metadata-run-key", 

491 help=( 

492 "Quantum graph metadata key for the name of the output run. " 

493 "Empty string disables update of the metadata. " 

494 "Default value: output_run." 

495 ), 

496 default="output_run", 

497)