Coverage for python/lsst/ctrl/mpexec/cli/opt/options.py: 76%

Shortcuts on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

50 statements  

1# This file is part of ctrl_mpexec. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22import click 

23 

24from lsst.daf.butler.cli.utils import MWOptionDecorator, MWPath, split_commas, unwrap 

25 

26 

27def _split_commas_int(context, param, values): 

28 """Special callback that handles comma-separated list of integers. 

29 

30 Parameters 

31 ---------- 

32 context : `click.Context` or `None` 

33 The current execution context. 

34 param : `click.core.Option` or `None` 

35 The parameter being handled. 

36 values : `list` [ `str` ] 

37 All the values passed for this option. Strings may contain commas, 

38 which will be treated as delimiters for separate values. 

39 

40 Returns 

41 ------- 

42 numbers : `tuple` [ `int` ] 

43 The passed in values separated by commas and combined into a single 

44 list. 

45 """ 

46 def _to_int(value): 

47 """Convert string to integer, handle errors.""" 

48 try: 

49 return int(value) 

50 except ValueError: 

51 raise click.BadParameter(f"'{value}' is not a valid integer", context, param) 

52 

53 values = split_commas(context, param, values) 

54 if values is None: 

55 return values 

56 return tuple(_to_int(value) for value in values) 

57 

58 

59butler_config_option = MWOptionDecorator("-b", "--butler-config", 

60 help="Location of the gen3 butler/registry config file.") 

61 

62 

63data_query_option = MWOptionDecorator("-d", "--data-query", 

64 help="User data selection expression.", 

65 metavar="QUERY") 

66 

67 

68debug_option = MWOptionDecorator("--debug", 

69 help="Enable debugging output using lsstDebug facility (imports debug.py).", 

70 is_flag=True) 

71 

72 

73delete_option = MWOptionDecorator("--delete", 

74 callback=split_commas, 

75 help="Delete task with given label from pipeline.", 

76 multiple=True) 

77 

78 

79do_raise_option = MWOptionDecorator("--do-raise", 

80 help="Raise an exception on error. (else log a message and continue?)", 

81 is_flag=True) 

82 

83 

84extend_run_option = MWOptionDecorator("--extend-run", 

85 help=unwrap("""Instead of creating a new RUN collection, insert datasets 

86 into either the one given by --output-run (if provided) or 

87 the first child collection of --output (which must be of 

88 type RUN). This also enables --skip-existing option."""), 

89 is_flag=True) 

90 

91 

92graph_fixup_option = MWOptionDecorator("--graph-fixup", 

93 help=unwrap("""Name of the class or factory method which makes an 

94 instance used for execution graph fixup.""")) 

95 

96 

97init_only_option = MWOptionDecorator("--init-only", 

98 help=unwrap("""Do not actually run; just register dataset types and/or 

99 save init outputs. """), 

100 is_flag=True) 

101 

102 

103input_option = MWOptionDecorator("-i", "--input", 

104 callback=split_commas, 

105 default=list(), 

106 help=unwrap("""Comma-separated names of the input collection(s)."""), 

107 metavar="COLLECTION", 

108 multiple=True) 

109 

110no_versions_option = MWOptionDecorator("--no-versions", 

111 help="Do not save or check package versions.", 

112 is_flag=True) 

113 

114 

115order_pipeline_option = MWOptionDecorator("--order-pipeline", 

116 help=unwrap("""Order tasks in pipeline based on their data 

117 dependencies, ordering is performed as last step before saving or 

118 executing pipeline."""), 

119 is_flag=True) 

120 

121 

122output_option = MWOptionDecorator("-o", "--output", 

123 help=unwrap("""Name of the output CHAINED collection. This may either be an 

124 existing CHAINED collection to use as both input and output 

125 (incompatible with --input), or a new CHAINED collection created 

126 to include all inputs (requires --input). In both cases, the 

127 collection's children will start with an output RUN collection 

128 that directly holds all new datasets (see --output-run)."""), 

129 metavar="COLL") 

130 

131 

132output_run_option = MWOptionDecorator("--output-run", 

133 help=unwrap("""Name of the new output RUN collection. If not provided 

134 then --output must be provided and a new RUN collection will 

135 be created by appending a timestamp to the value passed with 

136 --output. If this collection already exists then 

137 --extend-run must be passed."""), 

138 metavar="COLL") 

139 

140 

141pipeline_option = MWOptionDecorator("-p", "--pipeline", 

142 help="Location of a pipeline definition file in YAML format.", 

143 type=MWPath(file_okay=True, dir_okay=False, readable=True)) 

144 

145 

146pipeline_dot_option = MWOptionDecorator("--pipeline-dot", 

147 help=unwrap(""""Location for storing GraphViz DOT representation of a 

148 pipeline."""), 

149 type=MWPath(writable=True, file_okay=True, dir_okay=False)) 

150 

151 

152profile_option = MWOptionDecorator("--profile", 

153 help="Dump cProfile statistics to file name.", 

154 type=MWPath(file_okay=True, dir_okay=False)) 

155 

156 

157prune_replaced_option = MWOptionDecorator("--prune-replaced", 

158 help=unwrap("""Delete the datasets in the collection replaced by 

159 --replace-run, either just from the datastore 

160 ('unstore') or by removing them and the RUN completely 

161 ('purge'). Requires --replace-run."""), 

162 type=click.Choice(("unstore", "purge"), case_sensitive=False)) 

163 

164 

165qgraph_option = MWOptionDecorator("-g", "--qgraph", 

166 help=unwrap("""Location for a serialized quantum graph definition (pickle 

167 file). If this option is given then all input data options and 

168 pipeline-building options cannot be used. Can be a URI.""")) 

169 

170 

171qgraph_id_option = MWOptionDecorator("--qgraph-id", 

172 help=unwrap("""Quantum graph identifier, if specified must match the 

173 identifier of the graph loaded from a file. Ignored if graph 

174 is not loaded from a file.""")) 

175 

176 

177# I wanted to use default=None here to match Python API but click silently 

178# replaces None with an empty tuple when multiple=True. 

179qgraph_node_id_option = MWOptionDecorator("--qgraph-node-id", 

180 callback=split_commas, 

181 multiple=True, 

182 help=unwrap("""Only load a specified set of nodes when graph is 

183 loaded from a file, nodes are identified by UUID 

184 values. One or more comma-separated integers are 

185 accepted. By default all nodes are loaded. Ignored if 

186 graph is not loaded from a file.""")) 

187 

188qgraph_header_data_option = MWOptionDecorator("--show-qgraph-header", 

189 is_flag=True, 

190 default=False, 

191 help=unwrap("""Print the headerData for Quantum Graph to the 

192 console""")) 

193 

194qgraph_dot_option = MWOptionDecorator("--qgraph-dot", 

195 help=unwrap("""Location for storing GraphViz DOT representation of a 

196 quantum graph."""), 

197 type=MWPath(writable=True, file_okay=True, dir_okay=False)) 

198 

199 

200replace_run_option = MWOptionDecorator("--replace-run", 

201 help=unwrap("""Before creating a new RUN collection in an existing 

202 CHAINED collection, remove the first child collection 

203 (which must be of type RUN). This can be used to repeatedly 

204 write to the same (parent) collection during development, 

205 but it does not delete the datasets associated with the 

206 replaced run unless --prune-replaced is also passed. 

207 Requires --output, and incompatible with --extend-run."""), 

208 is_flag=True) 

209 

210 

211save_pipeline_option = MWOptionDecorator("-s", "--save-pipeline", 

212 help=unwrap("""Location for storing resulting pipeline definition in 

213 YAML format."""), 

214 type=MWPath(dir_okay=False, file_okay=True, writable=True)) 

215 

216save_qgraph_option = MWOptionDecorator("-q", "--save-qgraph", 

217 help=unwrap("""URI location for storing a serialized quantum graph 

218 definition (pickle file).""")) 

219 

220 

221save_single_quanta_option = MWOptionDecorator("--save-single-quanta", 

222 help=unwrap("""Format string of locations for storing individual 

223 quantum graph definition (pickle files). The curly 

224 brace {} in the input string will be replaced by a 

225 quantum number. Can be a URI.""")) 

226 

227 

228show_option = MWOptionDecorator("--show", 

229 callback=split_commas, 

230 help=unwrap("""Dump various info to standard output. Possible items are: 

231 `config`, `config=[Task::]<PATTERN>` or 

232 `config=[Task::]<PATTERN>:NOIGNORECASE` to dump configuration 

233 fields possibly matching given pattern and/or task label; 

234 `history=<FIELD>` to dump configuration history for a field, field 

235 name is specified as [Task::]<PATTERN>; `dump-config`, 

236 `dump-config=Task` to dump complete configuration for a task given 

237 its label or all tasks; `pipeline` to show pipeline composition; 

238 `graph` to show information about quanta; `workflow` to show 

239 information about quanta and their dependency; `tasks` to show 

240 task composition; `uri` to show predicted dataset URIs of 

241 quanta"""), 

242 metavar="ITEM|ITEM=VALUE", 

243 multiple=True) 

244 

245 

246skip_existing_in_option = MWOptionDecorator( 

247 "--skip-existing-in", 

248 callback=split_commas, 

249 default=None, 

250 metavar="COLLECTION", 

251 multiple=True, 

252 help=unwrap( 

253 """If all Quantum outputs already exist in the specified list of 

254 collections then that Quantum will be excluded from the QuantumGraph. 

255 """ 

256 ) 

257) 

258 

259 

260skip_existing_option = MWOptionDecorator( 

261 "--skip-existing", 

262 is_flag=True, 

263 help=unwrap( 

264 """This option is equivalent to --skip-existing-in with the name of 

265 the output RUN collection. If both --skip-existing-in and 

266 --skip-existing are given then output RUN collection is appended to 

267 the list of collections.""" 

268 ) 

269) 

270 

271 

272clobber_outputs_option = MWOptionDecorator("--clobber-outputs", 

273 help=unwrap("""Remove outputs from previous execution of the same 

274 quantum before new execution. If --skip-existing 

275 is also passed, then only failed quanta will be 

276 clobbered. Requires the 'run' command's --extend-run 

277 flag to be set."""), 

278 is_flag=True) 

279 

280 

281skip_init_writes_option = MWOptionDecorator("--skip-init-writes", 

282 help=unwrap("""Do not write collection-wide 'init output' datasets 

283 (e.g.schemas)."""), 

284 is_flag=True) 

285 

286 

287task_option = MWOptionDecorator("-t", "--task", 

288 callback=split_commas, 

289 help=unwrap("""Task name to add to pipeline, must be a fully qualified task 

290 name. Task name can be followed by colon and label name, if label 

291 is not given then task base name (class name) is used as 

292 label."""), 

293 metavar="TASK[:LABEL]", 

294 multiple=True) 

295 

296 

297timeout_option = MWOptionDecorator("--timeout", 

298 type=click.IntRange(min=0), 

299 help="Timeout for multiprocessing; maximum wall time (sec).") 

300 

301 

302start_method_option = MWOptionDecorator("--start-method", 

303 default=None, 

304 type=click.Choice(["spawn", "fork", "forkserver"]), 

305 help="Multiprocessing start method, default is platform-specific.") 

306 

307 

308fail_fast_option = MWOptionDecorator("--fail-fast", 

309 help=unwrap("""Stop processing at first error, default is to process 

310 as many tasks as possible."""), 

311 is_flag=True) 

312 

313save_execution_butler_option = MWOptionDecorator("--save-execution-butler", 

314 help=unwrap("""Export location for an 

315 execution-specific butler after making 

316 QuantumGraph""")) 

317 

318clobber_execution_butler_option = MWOptionDecorator("--clobber-execution-butler", 

319 help=unwrap("""When creating execution butler overwrite 

320 any existing products"""), 

321 is_flag=True) 

322dataset_query_constraint = MWOptionDecorator("--dataset-query-constraint", 

323 help=unwrap("""When constructing a quantum graph constrain by 

324 pre-existence of specified dataset types. Valid 

325 values are `all` for all inputs dataset types in 

326 pipeline, `off` to not consider dataset type 

327 existance as a constraint, single or comma 

328 separated list of dataset type names"""), 

329 default='all')