Coverage for python/lsst/ctrl/mpexec/cli/opt/options.py: 75%

48 statements  

« prev     ^ index     » next       coverage.py v7.2.1, created at 2023-03-12 01:56 -0800

1# This file is part of ctrl_mpexec. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22import click 

23 

24from lsst.daf.butler.cli.utils import MWOptionDecorator, MWPath, split_commas, unwrap 

25 

26 

27def _split_commas_int(context, param, values): 

28 """Special callback that handles comma-separated list of integers. 

29 

30 Parameters 

31 ---------- 

32 context : `click.Context` or `None` 

33 The current execution context. 

34 param : `click.core.Option` or `None` 

35 The parameter being handled. 

36 values : `list` [ `str` ] 

37 All the values passed for this option. Strings may contain commas, 

38 which will be treated as delimiters for separate values. 

39 

40 Returns 

41 ------- 

42 numbers : `tuple` [ `int` ] 

43 The passed in values separated by commas and combined into a single 

44 list. 

45 """ 

46 def _to_int(value): 

47 """Convert string to integer, handle errors.""" 

48 try: 

49 return int(value) 

50 except ValueError: 

51 raise click.BadParameter(f"'{value}' is not a valid integer", context, param) 

52 

53 values = split_commas(context, param, values) 

54 if values is None: 

55 return values 

56 return tuple(_to_int(value) for value in values) 

57 

58 

59butler_config_option = MWOptionDecorator("-b", "--butler-config", 

60 help="Location of the gen3 butler/registry config file.") 

61 

62 

63data_query_option = MWOptionDecorator("-d", "--data-query", 

64 help="User data selection expression.", 

65 metavar="QUERY") 

66 

67 

68debug_option = MWOptionDecorator("--debug", 

69 help="Enable debugging output using lsstDebug facility (imports debug.py).", 

70 is_flag=True) 

71 

72 

73delete_option = MWOptionDecorator("--delete", 

74 callback=split_commas, 

75 help="Delete task with given label from pipeline.", 

76 multiple=True) 

77 

78 

79do_raise_option = MWOptionDecorator("--do-raise", 

80 help="Raise an exception on error. (else log a message and continue?)", 

81 is_flag=True) 

82 

83 

84extend_run_option = MWOptionDecorator("--extend-run", 

85 help=unwrap("""Instead of creating a new RUN collection, insert datasets 

86 into either the one given by --output-run (if provided) or 

87 the first child collection of --output (which must be of 

88 type RUN). This also enables --skip-existing option."""), 

89 is_flag=True) 

90 

91 

92graph_fixup_option = MWOptionDecorator("--graph-fixup", 

93 help=unwrap("""Name of the class or factory method which makes an 

94 instance used for execution graph fixup.""")) 

95 

96 

97init_only_option = MWOptionDecorator("--init-only", 

98 help=unwrap("""Do not actually run; just register dataset types and/or 

99 save init outputs. """), 

100 is_flag=True) 

101 

102 

103input_option = MWOptionDecorator("-i", "--input", 

104 callback=split_commas, 

105 default=list(), 

106 help=unwrap("""Comma-separated names of the input collection(s)."""), 

107 metavar="COLLECTION", 

108 multiple=True) 

109 

110no_versions_option = MWOptionDecorator("--no-versions", 

111 help="Do not save or check package versions.", 

112 is_flag=True) 

113 

114 

115order_pipeline_option = MWOptionDecorator("--order-pipeline", 

116 help=unwrap("""Order tasks in pipeline based on their data 

117 dependencies, ordering is performed as last step before saving or 

118 executing pipeline."""), 

119 is_flag=True) 

120 

121 

122output_option = MWOptionDecorator("-o", "--output", 

123 help=unwrap("""Name of the output CHAINED collection. This may either be an 

124 existing CHAINED collection to use as both input and output 

125 (incompatible with --input), or a new CHAINED collection created 

126 to include all inputs (requires --input). In both cases, the 

127 collection's children will start with an output RUN collection 

128 that directly holds all new datasets (see --output-run)."""), 

129 metavar="COLL") 

130 

131 

132output_run_option = MWOptionDecorator("--output-run", 

133 help=unwrap("""Name of the new output RUN collection. If not provided 

134 then --output must be provided and a new RUN collection will 

135 be created by appending a timestamp to the value passed with 

136 --output. If this collection already exists then 

137 --extend-run must be passed."""), 

138 metavar="COLL") 

139 

140 

141pipeline_option = MWOptionDecorator("-p", "--pipeline", 

142 help="Location of a pipeline definition file in YAML format.", 

143 type=MWPath(file_okay=True, dir_okay=False, readable=True)) 

144 

145 

146pipeline_dot_option = MWOptionDecorator("--pipeline-dot", 

147 help=unwrap(""""Location for storing GraphViz DOT representation of a 

148 pipeline."""), 

149 type=MWPath(writable=True, file_okay=True, dir_okay=False)) 

150 

151 

152profile_option = MWOptionDecorator("--profile", 

153 help="Dump cProfile statistics to file name.", 

154 type=MWPath(file_okay=True, dir_okay=False)) 

155 

156 

157prune_replaced_option = MWOptionDecorator("--prune-replaced", 

158 help=unwrap("""Delete the datasets in the collection replaced by 

159 --replace-run, either just from the datastore 

160 ('unstore') or by removing them and the RUN completely 

161 ('purge'). Requires --replace-run."""), 

162 type=click.Choice(("unstore", "purge"), case_sensitive=False)) 

163 

164 

165qgraph_option = MWOptionDecorator("-g", "--qgraph", 

166 help=unwrap("""Location for a serialized quantum graph definition (pickle 

167 file). If this option is given then all input data options and 

168 pipeline-building options cannot be used. Can be a URI.""")) 

169 

170 

171qgraph_id_option = MWOptionDecorator("--qgraph-id", 

172 help=unwrap("""Quantum graph identifier, if specified must match the 

173 identifier of the graph loaded from a file. Ignored if graph 

174 is not loaded from a file.""")) 

175 

176 

177# I wanted to use default=None here to match Python API but click silently 

178# replaces None with an empty tuple when multiple=True. 

179qgraph_node_id_option = MWOptionDecorator("--qgraph-node-id", 

180 callback=_split_commas_int, 

181 multiple=True, 

182 help=unwrap("""Only load a specified set of nodes when graph is 

183 loaded from a file, nodes are identified by integer 

184 IDs. One or more comma-separated integers are accepted. 

185 By default all nodes are loaded. Ignored if graph is 

186 not loaded from a file.""")) 

187 

188 

189qgraph_dot_option = MWOptionDecorator("--qgraph-dot", 

190 help=unwrap("""Location for storing GraphViz DOT representation of a 

191 quantum graph."""), 

192 type=MWPath(writable=True, file_okay=True, dir_okay=False)) 

193 

194 

195replace_run_option = MWOptionDecorator("--replace-run", 

196 help=unwrap("""Before creating a new RUN collection in an existing 

197 CHAINED collection, remove the first child collection 

198 (which must be of type RUN). This can be used to repeatedly 

199 write to the same (parent) collection during development, 

200 but it does not delete the datasets associated with the 

201 replaced run unless --prune-replaced is also passed. 

202 Requires --output, and incompatible with --extend-run."""), 

203 is_flag=True) 

204 

205 

206save_pipeline_option = MWOptionDecorator("-s", "--save-pipeline", 

207 help=unwrap("""Location for storing resulting pipeline definition in 

208 YAML format."""), 

209 type=MWPath(dir_okay=False, file_okay=True, writable=True)) 

210 

211save_qgraph_option = MWOptionDecorator("-q", "--save-qgraph", 

212 help=unwrap("""URI location for storing a serialized quantum graph 

213 definition (pickle file).""")) 

214 

215 

216save_single_quanta_option = MWOptionDecorator("--save-single-quanta", 

217 help=unwrap("""Format string of locations for storing individual 

218 quantum graph definition (pickle files). The curly 

219 brace {} in the input string will be replaced by a 

220 quantum number. Can be a URI.""")) 

221 

222 

223show_option = MWOptionDecorator("--show", 

224 callback=split_commas, 

225 help=unwrap("""Dump various info to standard output. Possible items are: 

226 `config`, `config=[Task::]<PATTERN>` or 

227 `config=[Task::]<PATTERN>:NOIGNORECASE` to dump configuration 

228 fields possibly matching given pattern and/or task label; 

229 `history=<FIELD>` to dump configuration history for a field, field 

230 name is specified as [Task::]<PATTERN>; `dump-config`, 

231 `dump-config=Task` to dump complete configuration for a task given 

232 its label or all tasks; `pipeline` to show pipeline composition; 

233 `graph` to show information about quanta; `workflow` to show 

234 information about quanta and their dependency; `tasks` to show 

235 task composition; `uri` to show predicted dataset URIs of 

236 quanta"""), 

237 metavar="ITEM|ITEM=VALUE", 

238 multiple=True) 

239 

240 

241skip_existing_in_option = MWOptionDecorator( 

242 "--skip-existing-in", 

243 callback=split_commas, 

244 default=None, 

245 metavar="COLLECTION", 

246 multiple=True, 

247 help=unwrap( 

248 """If all Quantum outputs already exist in the specified list of 

249 collections then that Quantum will be excluded from the QuantumGraph. 

250 """ 

251 ) 

252) 

253 

254 

255skip_existing_option = MWOptionDecorator( 

256 "--skip-existing", 

257 is_flag=True, 

258 help=unwrap( 

259 """This option is equivalent to --skip-existing-in with the name of 

260 the output RUN collection. If both --skip-existing-in and 

261 --skip-existing are given then output RUN collection is appended to 

262 the list of collections.""" 

263 ) 

264) 

265 

266 

267clobber_outputs_option = MWOptionDecorator("--clobber-outputs", 

268 help=unwrap("""Remove outputs from previous execution of the same 

269 quantum before new execution. If --skip-existing 

270 is also passed, then only failed quanta will be 

271 clobbered. Requires the 'run' command's --extend-run 

272 flag to be set."""), 

273 is_flag=True) 

274 

275 

276skip_init_writes_option = MWOptionDecorator("--skip-init-writes", 

277 help=unwrap("""Do not write collection-wide 'init output' datasets 

278 (e.g.schemas)."""), 

279 is_flag=True) 

280 

281 

282task_option = MWOptionDecorator("-t", "--task", 

283 callback=split_commas, 

284 help=unwrap("""Task name to add to pipeline, must be a fully qualified task 

285 name. Task name can be followed by colon and label name, if label 

286 is not given then task base name (class name) is used as 

287 label."""), 

288 metavar="TASK[:LABEL]", 

289 multiple=True) 

290 

291 

292timeout_option = MWOptionDecorator("--timeout", 

293 type=click.IntRange(min=0), 

294 help="Timeout for multiprocessing; maximum wall time (sec).") 

295 

296 

297start_method_option = MWOptionDecorator("--start-method", 

298 default=None, 

299 type=click.Choice(["spawn", "fork", "forkserver"]), 

300 help="Multiprocessing start method, default is platform-specific.") 

301 

302 

303fail_fast_option = MWOptionDecorator("--fail-fast", 

304 help=unwrap("""Stop processing at first error, default is to process 

305 as many tasks as possible."""), 

306 is_flag=True) 

307 

308save_execution_butler_option = MWOptionDecorator("--save-execution-butler", 

309 help=unwrap("""Export location for an 

310 execution-specific butler after making 

311 QuantumGraph""")) 

312 

313clobber_execution_butler_option = MWOptionDecorator("--clobber-execution-butler", 

314 help=unwrap("""When creating execution butler overwrite 

315 any existing products"""), 

316 is_flag=True)