Coverage for python/lsst/ctrl/mpexec/cli/opt/options.py: 100%

Shortcuts on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

42 statements  

1# This file is part of ctrl_mpexec. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22import click 

23from lsst.daf.butler.cli.utils import MWOptionDecorator, MWPath, split_commas, unwrap 

24 

25butler_config_option = MWOptionDecorator( 

26 "-b", "--butler-config", help="Location of the gen3 butler/registry config file." 

27) 

28 

29 

30data_query_option = MWOptionDecorator( 

31 "-d", "--data-query", help="User data selection expression.", metavar="QUERY" 

32) 

33 

34 

35debug_option = MWOptionDecorator( 

36 "--debug", help="Enable debugging output using lsstDebug facility (imports debug.py).", is_flag=True 

37) 

38 

39 

40delete_option = MWOptionDecorator( 

41 "--delete", callback=split_commas, help="Delete task with given label from pipeline.", multiple=True 

42) 

43 

44 

45pdb_option = MWOptionDecorator( 

46 "--pdb", 

47 help="Post-mortem debugger to launch for exceptions (defaults to pdb if unspecified; requires a tty).", 

48 is_flag=False, 

49 flag_value="pdb", 

50 default=None, 

51) 

52 

53 

54extend_run_option = MWOptionDecorator( 

55 "--extend-run", 

56 help=unwrap( 

57 """Instead of creating a new RUN collection, insert datasets 

58 into either the one given by --output-run (if provided) or 

59 the first child collection of --output (which must be of 

60 type RUN). This also enables --skip-existing option.""" 

61 ), 

62 is_flag=True, 

63) 

64 

65 

66graph_fixup_option = MWOptionDecorator( 

67 "--graph-fixup", 

68 help=unwrap( 

69 """Name of the class or factory method which makes an 

70 instance used for execution graph fixup.""" 

71 ), 

72) 

73 

74 

75init_only_option = MWOptionDecorator( 

76 "--init-only", 

77 help=unwrap( 

78 """Do not actually run; just register dataset types and/or 

79 save init outputs. """ 

80 ), 

81 is_flag=True, 

82) 

83 

84 

85input_option = MWOptionDecorator( 

86 "-i", 

87 "--input", 

88 callback=split_commas, 

89 default=list(), 

90 help=unwrap("""Comma-separated names of the input collection(s)."""), 

91 metavar="COLLECTION", 

92 multiple=True, 

93) 

94 

95no_versions_option = MWOptionDecorator( 

96 "--no-versions", help="Do not save or check package versions.", is_flag=True 

97) 

98 

99 

100order_pipeline_option = MWOptionDecorator( 

101 "--order-pipeline", 

102 help=unwrap( 

103 """Order tasks in pipeline based on their data 

104 dependencies, ordering is performed as last step before saving or 

105 executing pipeline.""" 

106 ), 

107 is_flag=True, 

108) 

109 

110 

111output_option = MWOptionDecorator( 

112 "-o", 

113 "--output", 

114 help=unwrap( 

115 """Name of the output CHAINED collection. This may either be an 

116 existing CHAINED collection to use as both input and output 

117 (incompatible with --input), or a new CHAINED collection created 

118 to include all inputs (requires --input). In both cases, the 

119 collection's children will start with an output RUN collection 

120 that directly holds all new datasets (see --output-run).""" 

121 ), 

122 metavar="COLL", 

123) 

124 

125 

126output_run_option = MWOptionDecorator( 

127 "--output-run", 

128 help=unwrap( 

129 """Name of the new output RUN collection. If not provided 

130 then --output must be provided and a new RUN collection will 

131 be created by appending a timestamp to the value passed with 

132 --output. If this collection already exists then 

133 --extend-run must be passed.""" 

134 ), 

135 metavar="COLL", 

136) 

137 

138 

139pipeline_option = MWOptionDecorator( 

140 "-p", 

141 "--pipeline", 

142 help="Location of a pipeline definition file in YAML format.", 

143 type=MWPath(file_okay=True, dir_okay=False, readable=True), 

144) 

145 

146 

147pipeline_dot_option = MWOptionDecorator( 

148 "--pipeline-dot", 

149 help=unwrap( 

150 """"Location for storing GraphViz DOT representation of a 

151 pipeline.""" 

152 ), 

153 type=MWPath(writable=True, file_okay=True, dir_okay=False), 

154) 

155 

156 

157profile_option = MWOptionDecorator( 

158 "--profile", help="Dump cProfile statistics to file name.", type=MWPath(file_okay=True, dir_okay=False) 

159) 

160 

161 

162prune_replaced_option = MWOptionDecorator( 

163 "--prune-replaced", 

164 help=unwrap( 

165 """Delete the datasets in the collection replaced by 

166 --replace-run, either just from the datastore 

167 ('unstore') or by removing them and the RUN completely 

168 ('purge'). Requires --replace-run.""" 

169 ), 

170 type=click.Choice(choices=("unstore", "purge"), case_sensitive=False), 

171) 

172 

173 

174qgraph_option = MWOptionDecorator( 

175 "-g", 

176 "--qgraph", 

177 help=unwrap( 

178 """Location for a serialized quantum graph definition (pickle 

179 file). If this option is given then all input data options and 

180 pipeline-building options cannot be used. Can be a URI.""" 

181 ), 

182) 

183 

184 

185qgraph_id_option = MWOptionDecorator( 

186 "--qgraph-id", 

187 help=unwrap( 

188 """Quantum graph identifier, if specified must match the 

189 identifier of the graph loaded from a file. Ignored if graph 

190 is not loaded from a file.""" 

191 ), 

192) 

193 

194 

195# I wanted to use default=None here to match Python API but click silently 

196# replaces None with an empty tuple when multiple=True. 

197qgraph_node_id_option = MWOptionDecorator( 

198 "--qgraph-node-id", 

199 callback=split_commas, 

200 multiple=True, 

201 help=unwrap( 

202 """Only load a specified set of nodes when graph is 

203 loaded from a file, nodes are identified by UUID 

204 values. One or more comma-separated integers are 

205 accepted. By default all nodes are loaded. Ignored if 

206 graph is not loaded from a file.""" 

207 ), 

208) 

209 

210qgraph_header_data_option = MWOptionDecorator( 

211 "--show-qgraph-header", 

212 is_flag=True, 

213 default=False, 

214 help=unwrap( 

215 """Print the headerData for Quantum Graph to the 

216 console""" 

217 ), 

218) 

219 

220qgraph_dot_option = MWOptionDecorator( 

221 "--qgraph-dot", 

222 help=unwrap( 

223 """Location for storing GraphViz DOT representation of a 

224 quantum graph.""" 

225 ), 

226 type=MWPath(writable=True, file_okay=True, dir_okay=False), 

227) 

228 

229 

230replace_run_option = MWOptionDecorator( 

231 "--replace-run", 

232 help=unwrap( 

233 """Before creating a new RUN collection in an existing 

234 CHAINED collection, remove the first child collection 

235 (which must be of type RUN). This can be used to repeatedly 

236 write to the same (parent) collection during development, 

237 but it does not delete the datasets associated with the 

238 replaced run unless --prune-replaced is also passed. 

239 Requires --output, and incompatible with --extend-run.""" 

240 ), 

241 is_flag=True, 

242) 

243 

244 

245save_pipeline_option = MWOptionDecorator( 

246 "-s", 

247 "--save-pipeline", 

248 help=unwrap( 

249 """Location for storing resulting pipeline definition in 

250 YAML format.""" 

251 ), 

252 type=MWPath(dir_okay=False, file_okay=True, writable=True), 

253) 

254 

255save_qgraph_option = MWOptionDecorator( 

256 "-q", 

257 "--save-qgraph", 

258 help=unwrap( 

259 """URI location for storing a serialized quantum graph 

260 definition (pickle file).""" 

261 ), 

262) 

263 

264 

265save_single_quanta_option = MWOptionDecorator( 

266 "--save-single-quanta", 

267 help=unwrap( 

268 """Format string of locations for storing individual 

269 quantum graph definition (pickle files). The curly 

270 brace {} in the input string will be replaced by a 

271 quantum number. Can be a URI.""" 

272 ), 

273) 

274 

275 

276show_option = MWOptionDecorator( 

277 "--show", 

278 callback=split_commas, 

279 help=unwrap( 

280 """Dump various info to standard output. Possible items are: 

281 `config`, `config=[Task::]<PATTERN>` or 

282 `config=[Task::]<PATTERN>:NOIGNORECASE` to dump configuration 

283 fields possibly matching given pattern and/or task label; 

284 `history=<FIELD>` to dump configuration history for a field, field 

285 name is specified as [Task::]<PATTERN>; `dump-config`, 

286 `dump-config=Task` to dump complete configuration for a task given 

287 its label or all tasks; `pipeline` to show pipeline composition; 

288 `graph` to show information about quanta; `workflow` to show 

289 information about quanta and their dependency; `tasks` to show 

290 task composition; `uri` to show predicted dataset URIs of 

291 quanta""" 

292 ), 

293 metavar="ITEM|ITEM=VALUE", 

294 multiple=True, 

295) 

296 

297 

298skip_existing_in_option = MWOptionDecorator( 

299 "--skip-existing-in", 

300 callback=split_commas, 

301 default=None, 

302 metavar="COLLECTION", 

303 multiple=True, 

304 help=unwrap( 

305 """If all Quantum outputs already exist in the specified list of 

306 collections then that Quantum will be excluded from the QuantumGraph. 

307 """ 

308 ), 

309) 

310 

311 

312skip_existing_option = MWOptionDecorator( 

313 "--skip-existing", 

314 is_flag=True, 

315 help=unwrap( 

316 """This option is equivalent to --skip-existing-in with the name of 

317 the output RUN collection. If both --skip-existing-in and 

318 --skip-existing are given then output RUN collection is appended to 

319 the list of collections.""" 

320 ), 

321) 

322 

323 

324clobber_outputs_option = MWOptionDecorator( 

325 "--clobber-outputs", 

326 help=unwrap( 

327 """Remove outputs from previous execution of the same 

328 quantum before new execution. If --skip-existing 

329 is also passed, then only failed quanta will be 

330 clobbered. Requires the 'run' command's --extend-run 

331 flag to be set.""" 

332 ), 

333 is_flag=True, 

334) 

335 

336 

337skip_init_writes_option = MWOptionDecorator( 

338 "--skip-init-writes", 

339 help=unwrap( 

340 """Do not write collection-wide 'init output' datasets 

341 (e.g.schemas).""" 

342 ), 

343 is_flag=True, 

344) 

345 

346 

347task_option = MWOptionDecorator( 

348 "-t", 

349 "--task", 

350 callback=split_commas, 

351 help=unwrap( 

352 """Task name to add to pipeline, must be a fully qualified task 

353 name. Task name can be followed by colon and label name, if label 

354 is not given then task base name (class name) is used as 

355 label.""" 

356 ), 

357 metavar="TASK[:LABEL]", 

358 multiple=True, 

359) 

360 

361 

362timeout_option = MWOptionDecorator( 

363 "--timeout", type=click.IntRange(min=0), help="Timeout for multiprocessing; maximum wall time (sec)." 

364) 

365 

366 

367start_method_option = MWOptionDecorator( 

368 "--start-method", 

369 default=None, 

370 type=click.Choice(choices=["spawn", "fork", "forkserver"]), 

371 help="Multiprocessing start method, default is platform-specific.", 

372) 

373 

374 

375fail_fast_option = MWOptionDecorator( 

376 "--fail-fast", 

377 help=unwrap( 

378 """Stop processing at first error, default is to process 

379 as many tasks as possible.""" 

380 ), 

381 is_flag=True, 

382) 

383 

384save_execution_butler_option = MWOptionDecorator( 

385 "--save-execution-butler", 

386 help=unwrap( 

387 """Export location for an 

388 execution-specific butler after making 

389 QuantumGraph""" 

390 ), 

391) 

392 

393mock_option = MWOptionDecorator( 

394 "--mock", 

395 help=unwrap("""Mock pipeline execution."""), 

396 is_flag=True, 

397) 

398 

399clobber_execution_butler_option = MWOptionDecorator( 

400 "--clobber-execution-butler", 

401 help=unwrap( 

402 """When creating execution butler overwrite 

403 any existing products""" 

404 ), 

405 is_flag=True, 

406) 

407dataset_query_constraint = MWOptionDecorator( 

408 "--dataset-query-constraint", 

409 help=unwrap( 

410 """When constructing a quantum graph constrain by 

411 pre-existence of specified dataset types. Valid 

412 values are `all` for all inputs dataset types in 

413 pipeline, `off` to not consider dataset type 

414 existance as a constraint, single or comma 

415 separated list of dataset type names""" 

416 ), 

417 default="all", 

418) 

419 

420summary_option = MWOptionDecorator( 

421 "--summary", 

422 help=( 

423 "Location for storing job summary (JSON file). Note that the" 

424 " structure of this file may not be stable." 

425 ), 

426 type=MWPath(dir_okay=False, file_okay=True, writable=True), 

427)