Coverage for python/lsst/ctrl/mpexec/cli/opt/options.py: 77%

Shortcuts on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

52 statements  

1# This file is part of ctrl_mpexec. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22import click 

23from lsst.daf.butler.cli.utils import MWOptionDecorator, MWPath, split_commas, unwrap 

24 

25 

26def _split_commas_int(context, param, values): 

27 """Special callback that handles comma-separated list of integers. 

28 

29 Parameters 

30 ---------- 

31 context : `click.Context` or `None` 

32 The current execution context. 

33 param : `click.core.Option` or `None` 

34 The parameter being handled. 

35 values : `list` [ `str` ] 

36 All the values passed for this option. Strings may contain commas, 

37 which will be treated as delimiters for separate values. 

38 

39 Returns 

40 ------- 

41 numbers : `tuple` [ `int` ] 

42 The passed in values separated by commas and combined into a single 

43 list. 

44 """ 

45 

46 def _to_int(value): 

47 """Convert string to integer, handle errors.""" 

48 try: 

49 return int(value) 

50 except ValueError: 

51 raise click.BadParameter(f"'{value}' is not a valid integer", context, param) 

52 

53 values = split_commas(context, param, values) 

54 if values is None: 

55 return values 

56 return tuple(_to_int(value) for value in values) 

57 

58 

59butler_config_option = MWOptionDecorator( 

60 "-b", "--butler-config", help="Location of the gen3 butler/registry config file." 

61) 

62 

63 

64data_query_option = MWOptionDecorator( 

65 "-d", "--data-query", help="User data selection expression.", metavar="QUERY" 

66) 

67 

68 

69debug_option = MWOptionDecorator( 

70 "--debug", help="Enable debugging output using lsstDebug facility (imports debug.py).", is_flag=True 

71) 

72 

73 

74delete_option = MWOptionDecorator( 

75 "--delete", callback=split_commas, help="Delete task with given label from pipeline.", multiple=True 

76) 

77 

78 

79do_raise_option = MWOptionDecorator( 

80 "--do-raise", help="Raise an exception on error. (else log a message and continue?)", is_flag=True 

81) 

82 

83 

84extend_run_option = MWOptionDecorator( 

85 "--extend-run", 

86 help=unwrap( 

87 """Instead of creating a new RUN collection, insert datasets 

88 into either the one given by --output-run (if provided) or 

89 the first child collection of --output (which must be of 

90 type RUN). This also enables --skip-existing option.""" 

91 ), 

92 is_flag=True, 

93) 

94 

95 

96graph_fixup_option = MWOptionDecorator( 

97 "--graph-fixup", 

98 help=unwrap( 

99 """Name of the class or factory method which makes an 

100 instance used for execution graph fixup.""" 

101 ), 

102) 

103 

104 

105init_only_option = MWOptionDecorator( 

106 "--init-only", 

107 help=unwrap( 

108 """Do not actually run; just register dataset types and/or 

109 save init outputs. """ 

110 ), 

111 is_flag=True, 

112) 

113 

114 

115input_option = MWOptionDecorator( 

116 "-i", 

117 "--input", 

118 callback=split_commas, 

119 default=list(), 

120 help=unwrap("""Comma-separated names of the input collection(s)."""), 

121 metavar="COLLECTION", 

122 multiple=True, 

123) 

124 

125no_versions_option = MWOptionDecorator( 

126 "--no-versions", help="Do not save or check package versions.", is_flag=True 

127) 

128 

129 

130order_pipeline_option = MWOptionDecorator( 

131 "--order-pipeline", 

132 help=unwrap( 

133 """Order tasks in pipeline based on their data 

134 dependencies, ordering is performed as last step before saving or 

135 executing pipeline.""" 

136 ), 

137 is_flag=True, 

138) 

139 

140 

141output_option = MWOptionDecorator( 

142 "-o", 

143 "--output", 

144 help=unwrap( 

145 """Name of the output CHAINED collection. This may either be an 

146 existing CHAINED collection to use as both input and output 

147 (incompatible with --input), or a new CHAINED collection created 

148 to include all inputs (requires --input). In both cases, the 

149 collection's children will start with an output RUN collection 

150 that directly holds all new datasets (see --output-run).""" 

151 ), 

152 metavar="COLL", 

153) 

154 

155 

156output_run_option = MWOptionDecorator( 

157 "--output-run", 

158 help=unwrap( 

159 """Name of the new output RUN collection. If not provided 

160 then --output must be provided and a new RUN collection will 

161 be created by appending a timestamp to the value passed with 

162 --output. If this collection already exists then 

163 --extend-run must be passed.""" 

164 ), 

165 metavar="COLL", 

166) 

167 

168 

169pipeline_option = MWOptionDecorator( 

170 "-p", 

171 "--pipeline", 

172 help="Location of a pipeline definition file in YAML format.", 

173 type=MWPath(file_okay=True, dir_okay=False, readable=True), 

174) 

175 

176 

177pipeline_dot_option = MWOptionDecorator( 

178 "--pipeline-dot", 

179 help=unwrap( 

180 """"Location for storing GraphViz DOT representation of a 

181 pipeline.""" 

182 ), 

183 type=MWPath(writable=True, file_okay=True, dir_okay=False), 

184) 

185 

186 

187profile_option = MWOptionDecorator( 

188 "--profile", help="Dump cProfile statistics to file name.", type=MWPath(file_okay=True, dir_okay=False) 

189) 

190 

191 

192prune_replaced_option = MWOptionDecorator( 

193 "--prune-replaced", 

194 help=unwrap( 

195 """Delete the datasets in the collection replaced by 

196 --replace-run, either just from the datastore 

197 ('unstore') or by removing them and the RUN completely 

198 ('purge'). Requires --replace-run.""" 

199 ), 

200 type=click.Choice(("unstore", "purge"), case_sensitive=False), 

201) 

202 

203 

204qgraph_option = MWOptionDecorator( 

205 "-g", 

206 "--qgraph", 

207 help=unwrap( 

208 """Location for a serialized quantum graph definition (pickle 

209 file). If this option is given then all input data options and 

210 pipeline-building options cannot be used. Can be a URI.""" 

211 ), 

212) 

213 

214 

215qgraph_id_option = MWOptionDecorator( 

216 "--qgraph-id", 

217 help=unwrap( 

218 """Quantum graph identifier, if specified must match the 

219 identifier of the graph loaded from a file. Ignored if graph 

220 is not loaded from a file.""" 

221 ), 

222) 

223 

224 

225# I wanted to use default=None here to match Python API but click silently 

226# replaces None with an empty tuple when multiple=True. 

227qgraph_node_id_option = MWOptionDecorator( 

228 "--qgraph-node-id", 

229 callback=split_commas, 

230 multiple=True, 

231 help=unwrap( 

232 """Only load a specified set of nodes when graph is 

233 loaded from a file, nodes are identified by UUID 

234 values. One or more comma-separated integers are 

235 accepted. By default all nodes are loaded. Ignored if 

236 graph is not loaded from a file.""" 

237 ), 

238) 

239 

240qgraph_header_data_option = MWOptionDecorator( 

241 "--show-qgraph-header", 

242 is_flag=True, 

243 default=False, 

244 help=unwrap( 

245 """Print the headerData for Quantum Graph to the 

246 console""" 

247 ), 

248) 

249 

250qgraph_dot_option = MWOptionDecorator( 

251 "--qgraph-dot", 

252 help=unwrap( 

253 """Location for storing GraphViz DOT representation of a 

254 quantum graph.""" 

255 ), 

256 type=MWPath(writable=True, file_okay=True, dir_okay=False), 

257) 

258 

259 

260replace_run_option = MWOptionDecorator( 

261 "--replace-run", 

262 help=unwrap( 

263 """Before creating a new RUN collection in an existing 

264 CHAINED collection, remove the first child collection 

265 (which must be of type RUN). This can be used to repeatedly 

266 write to the same (parent) collection during development, 

267 but it does not delete the datasets associated with the 

268 replaced run unless --prune-replaced is also passed. 

269 Requires --output, and incompatible with --extend-run.""" 

270 ), 

271 is_flag=True, 

272) 

273 

274 

275save_pipeline_option = MWOptionDecorator( 

276 "-s", 

277 "--save-pipeline", 

278 help=unwrap( 

279 """Location for storing resulting pipeline definition in 

280 YAML format.""" 

281 ), 

282 type=MWPath(dir_okay=False, file_okay=True, writable=True), 

283) 

284 

285save_qgraph_option = MWOptionDecorator( 

286 "-q", 

287 "--save-qgraph", 

288 help=unwrap( 

289 """URI location for storing a serialized quantum graph 

290 definition (pickle file).""" 

291 ), 

292) 

293 

294 

295save_single_quanta_option = MWOptionDecorator( 

296 "--save-single-quanta", 

297 help=unwrap( 

298 """Format string of locations for storing individual 

299 quantum graph definition (pickle files). The curly 

300 brace {} in the input string will be replaced by a 

301 quantum number. Can be a URI.""" 

302 ), 

303) 

304 

305 

306show_option = MWOptionDecorator( 

307 "--show", 

308 callback=split_commas, 

309 help=unwrap( 

310 """Dump various info to standard output. Possible items are: 

311 `config`, `config=[Task::]<PATTERN>` or 

312 `config=[Task::]<PATTERN>:NOIGNORECASE` to dump configuration 

313 fields possibly matching given pattern and/or task label; 

314 `history=<FIELD>` to dump configuration history for a field, field 

315 name is specified as [Task::]<PATTERN>; `dump-config`, 

316 `dump-config=Task` to dump complete configuration for a task given 

317 its label or all tasks; `pipeline` to show pipeline composition; 

318 `graph` to show information about quanta; `workflow` to show 

319 information about quanta and their dependency; `tasks` to show 

320 task composition; `uri` to show predicted dataset URIs of 

321 quanta""" 

322 ), 

323 metavar="ITEM|ITEM=VALUE", 

324 multiple=True, 

325) 

326 

327 

328skip_existing_in_option = MWOptionDecorator( 

329 "--skip-existing-in", 

330 callback=split_commas, 

331 default=None, 

332 metavar="COLLECTION", 

333 multiple=True, 

334 help=unwrap( 

335 """If all Quantum outputs already exist in the specified list of 

336 collections then that Quantum will be excluded from the QuantumGraph. 

337 """ 

338 ), 

339) 

340 

341 

342skip_existing_option = MWOptionDecorator( 

343 "--skip-existing", 

344 is_flag=True, 

345 help=unwrap( 

346 """This option is equivalent to --skip-existing-in with the name of 

347 the output RUN collection. If both --skip-existing-in and 

348 --skip-existing are given then output RUN collection is appended to 

349 the list of collections.""" 

350 ), 

351) 

352 

353 

354clobber_outputs_option = MWOptionDecorator( 

355 "--clobber-outputs", 

356 help=unwrap( 

357 """Remove outputs from previous execution of the same 

358 quantum before new execution. If --skip-existing 

359 is also passed, then only failed quanta will be 

360 clobbered. Requires the 'run' command's --extend-run 

361 flag to be set.""" 

362 ), 

363 is_flag=True, 

364) 

365 

366 

367skip_init_writes_option = MWOptionDecorator( 

368 "--skip-init-writes", 

369 help=unwrap( 

370 """Do not write collection-wide 'init output' datasets 

371 (e.g.schemas).""" 

372 ), 

373 is_flag=True, 

374) 

375 

376 

377task_option = MWOptionDecorator( 

378 "-t", 

379 "--task", 

380 callback=split_commas, 

381 help=unwrap( 

382 """Task name to add to pipeline, must be a fully qualified task 

383 name. Task name can be followed by colon and label name, if label 

384 is not given then task base name (class name) is used as 

385 label.""" 

386 ), 

387 metavar="TASK[:LABEL]", 

388 multiple=True, 

389) 

390 

391 

392timeout_option = MWOptionDecorator( 

393 "--timeout", type=click.IntRange(min=0), help="Timeout for multiprocessing; maximum wall time (sec)." 

394) 

395 

396 

397start_method_option = MWOptionDecorator( 

398 "--start-method", 

399 default=None, 

400 type=click.Choice(["spawn", "fork", "forkserver"]), 

401 help="Multiprocessing start method, default is platform-specific.", 

402) 

403 

404 

405fail_fast_option = MWOptionDecorator( 

406 "--fail-fast", 

407 help=unwrap( 

408 """Stop processing at first error, default is to process 

409 as many tasks as possible.""" 

410 ), 

411 is_flag=True, 

412) 

413 

414save_execution_butler_option = MWOptionDecorator( 

415 "--save-execution-butler", 

416 help=unwrap( 

417 """Export location for an 

418 execution-specific butler after making 

419 QuantumGraph""" 

420 ), 

421) 

422 

423mock_option = MWOptionDecorator( 

424 "--mock", 

425 help=unwrap("""Mock pipeline execution."""), 

426 is_flag=True, 

427) 

428 

429clobber_execution_butler_option = MWOptionDecorator( 

430 "--clobber-execution-butler", 

431 help=unwrap( 

432 """When creating execution butler overwrite 

433 any existing products""" 

434 ), 

435 is_flag=True, 

436) 

437dataset_query_constraint = MWOptionDecorator( 

438 "--dataset-query-constraint", 

439 help=unwrap( 

440 """When constructing a quantum graph constrain by 

441 pre-existence of specified dataset types. Valid 

442 values are `all` for all inputs dataset types in 

443 pipeline, `off` to not consider dataset type 

444 existance as a constraint, single or comma 

445 separated list of dataset type names""" 

446 ), 

447 default="all", 

448) 

449 

450summary_option = MWOptionDecorator( 

451 "--summary", 

452 help=( 

453 "Location for storing job summary (JSON file). Note that the" 

454 " structure of this file may not be stable." 

455 ), 

456 type=MWPath(dir_okay=False, file_okay=True, writable=True), 

457)