Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of ctrl_mpexec. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22import click 

23 

24from lsst.daf.butler.cli.utils import MWOptionDecorator, MWPath, split_commas, unwrap 

25 

26 

27def _split_commas_int(context, param, values): 

28 """Special callback that handles comma-separated list of integers. 

29 

30 Parameters 

31 ---------- 

32 context : `click.Context` or `None` 

33 The current execution context. 

34 param : `click.core.Option` or `None` 

35 The parameter being handled. 

36 values : `list` [ `str` ] 

37 All the values passed for this option. Strings may contain commas, 

38 which will be treated as delimiters for separate values. 

39 

40 Returns 

41 ------- 

42 numbers : `tuple` [ `int` ] 

43 The passed in values separated by commas and combined into a single 

44 list. 

45 """ 

46 def _to_int(value): 

47 """Convert string to integer, handle errors.""" 

48 try: 

49 return int(value) 

50 except ValueError: 

51 raise click.BadParameter(f"'{value}' is not a valid integer", context, param) 

52 

53 values = split_commas(context, param, values) 

54 if values is None: 

55 return values 

56 return tuple(_to_int(value) for value in values) 

57 

58 

59butler_config_option = MWOptionDecorator("-b", "--butler-config", 

60 help="Location of the gen3 butler/registry config file.") 

61 

62 

63data_query_option = MWOptionDecorator("-d", "--data-query", 

64 help="User data selection expression.", 

65 metavar="QUERY") 

66 

67 

68debug_option = MWOptionDecorator("--debug", 

69 help="Enable debugging output using lsstDebug facility (imports debug.py).", 

70 is_flag=True) 

71 

72 

73delete_option = MWOptionDecorator("--delete", 

74 callback=split_commas, 

75 help="Delete task with given label from pipeline.", 

76 multiple=True) 

77 

78 

79do_raise_option = MWOptionDecorator("--do-raise", 

80 help="Raise an exception on error. (else log a message and continue?)", 

81 is_flag=True) 

82 

83 

84extend_run_option = MWOptionDecorator("--extend-run", 

85 help=unwrap("""Instead of creating a new RUN collection, insert datasets 

86 into either the one given by --output-run (if provided) or 

87 the first child collection of - -output(which must be of 

88 type RUN)."""), 

89 is_flag=True) 

90 

91 

92graph_fixup_option = MWOptionDecorator("--graph-fixup", 

93 help=unwrap("""Name of the class or factory method which makes an 

94 instance used for execution graph fixup.""")) 

95 

96 

97init_only_option = MWOptionDecorator("--init-only", 

98 help=unwrap("""Do not actually run; just register dataset types and/or 

99 save init outputs. """), 

100 is_flag=True) 

101 

102 

103input_option = MWOptionDecorator("-i", "--input", 

104 callback=split_commas, 

105 default=list(), 

106 help=unwrap("""Comma-separated names of the input collection(s)."""), 

107 metavar="COLLECTION", 

108 multiple=True) 

109 

110no_versions_option = MWOptionDecorator("--no-versions", 

111 help="Do not save or check package versions.", 

112 is_flag=True) 

113 

114 

115order_pipeline_option = MWOptionDecorator("--order-pipeline", 

116 help=unwrap("""Order tasks in pipeline based on their data 

117 dependencies, ordering is performed as last step before saving or 

118 executing pipeline."""), 

119 is_flag=True) 

120 

121 

122output_option = MWOptionDecorator("-o", "--output", 

123 help=unwrap("""Name of the output CHAINED collection. This may either be an 

124 existing CHAINED collection to use as both input and output 

125 (incompatible with --input), or a new CHAINED collection created 

126 to include all inputs (requires --input). In both cases, the 

127 collection's children will start with an output RUN collection 

128 that directly holds all new datasets (see --output-run)."""), 

129 metavar="COLL") 

130 

131 

132output_run_option = MWOptionDecorator("--output-run", 

133 help=unwrap("""Name of the new output RUN collection. If not provided 

134 then --output must be provided and a new RUN collection will 

135 be created by appending a timestamp to the value passed with 

136 --output. If this collection already exists then 

137 --extend-run must be passed."""), 

138 metavar="COLL") 

139 

140 

141pipeline_option = MWOptionDecorator("-p", "--pipeline", 

142 help="Location of a pipeline definition file in YAML format.", 

143 type=MWPath(file_okay=True, dir_okay=False, readable=True)) 

144 

145 

146pipeline_dot_option = MWOptionDecorator("--pipeline-dot", 

147 help=unwrap(""""Location for storing GraphViz DOT representation of a 

148 pipeline."""), 

149 type=MWPath(writable=True, file_okay=True, dir_okay=False)) 

150 

151 

152profile_option = MWOptionDecorator("--profile", 

153 help="Dump cProfile statistics to file name.", 

154 type=MWPath(file_okay=True, dir_okay=False)) 

155 

156 

157prune_replaced_option = MWOptionDecorator("--prune-replaced", 

158 help=unwrap("""Delete the datasets in the collection replaced by 

159 --replace-run, either just from the datastore 

160 ('unstore') or by removing them and the RUN completely 

161 ('purge'). Requires --replace-run."""), 

162 type=click.Choice(("unstore", "purge"), case_sensitive=False)) 

163 

164 

165qgraph_option = MWOptionDecorator("-g", "--qgraph", 

166 help=unwrap("""Location for a serialized quantum graph definition (pickle 

167 file). If this option is given then all input data options and 

168 pipeline-building options cannot be used. Can be a URI.""")) 

169 

170 

171qgraph_id_option = MWOptionDecorator("--qgraph-id", 

172 help=unwrap("""Quantum graph identifier, if specified must match the 

173 identifier of the graph loaded from a file. Ignored if graph 

174 is not loaded from a file.""")) 

175 

176 

177# I wanted to use default=None here to match Python API but click silently 

178# replaces None with an empty tuple when multiple=True. 

179qgraph_node_id_option = MWOptionDecorator("--qgraph-node-id", 

180 callback=_split_commas_int, 

181 multiple=True, 

182 help=unwrap("""Only load a specified set of nodes when graph is 

183 loaded from a file, nodes are identified by integer 

184 IDs. One or more comma-separated integers are accepted. 

185 By default all nodes are loaded. Ignored if graph is 

186 not loaded from a file.""")) 

187 

188 

189qgraph_dot_option = MWOptionDecorator("--qgraph-dot", 

190 help=unwrap("""Location for storing GraphViz DOT representation of a 

191 quantum graph."""), 

192 type=MWPath(writable=True, file_okay=True, dir_okay=False)) 

193 

194 

195register_dataset_types_option = MWOptionDecorator("--register-dataset-types", 

196 help=unwrap("""Register DatasetTypes that do not already 

197 exist in the Registry."""), 

198 is_flag=True) 

199 

200 

201replace_run_option = MWOptionDecorator("--replace-run", 

202 help=unwrap("""Before creating a new RUN collection in an existing 

203 CHAINED collection, remove the first child collection 

204 (which must be of type RUN). This can be used to repeatedly 

205 write to the same (parent) collection during development, 

206 but it does not delete the datasets associated with the 

207 replaced run unless --prune-replaced is also passed. 

208 Requires --output, and incompatible with --extend-run."""), 

209 is_flag=True) 

210 

211 

212save_pipeline_option = MWOptionDecorator("-s", "--save-pipeline", 

213 help=unwrap("""Location for storing resulting pipeline definition in 

214 YAML format."""), 

215 type=MWPath(dir_okay=False, file_okay=True, writable=True)) 

216 

217save_qgraph_option = MWOptionDecorator("-q", "--save-qgraph", 

218 help=unwrap("""URI location for storing a serialized quantum graph 

219 definition (pickle file).""")) 

220 

221 

222save_single_quanta_option = MWOptionDecorator("--save-single-quanta", 

223 help=unwrap("""Format string of locations for storing individual 

224 quantum graph definition (pickle files). The curly 

225 brace {} in the input string will be replaced by a 

226 quantum number. Can be a URI.""")) 

227 

228 

229show_option = MWOptionDecorator("--show", 

230 callback=split_commas, 

231 help=unwrap("""Dump various info to standard output. Possible items are: 

232 `config`, `config=[Task::]<PATTERN>` or 

233 `config=[Task::]<PATTERN>:NOIGNORECASE` to dump configuration 

234 fields possibly matching given pattern and/or task label; 

235 `history=<FIELD>` to dump configuration history for a field, field 

236 name is specified as [Task::]<PATTERN>; `dump-config`, 

237 `dump-config=Task` to dump complete configuration for a task given 

238 its label or all tasks; `pipeline` to show pipeline composition; 

239 `graph` to show information about quanta; `workflow` to show 

240 information about quanta and their dependency; `tasks` to show 

241 task composition; `uri` to show predicted dataset URIs of 

242 quanta"""), 

243 metavar="ITEM|ITEM=VALUE", 

244 multiple=True) 

245 

246 

247skip_existing_option = MWOptionDecorator("--skip-existing", 

248 help=unwrap("""If all Quantum outputs already exist in the output RUN 

249 collection then that Quantum will be excluded from the 

250 QuantumGraph. Requires the 'run` command's `--extend-run` 

251 flag to be set."""), 

252 is_flag=True) 

253 

254 

255clobber_outputs_option = MWOptionDecorator("--clobber-outputs", 

256 help=unwrap("""Remove outputs from previous execution of the same 

257 quantum before new execution. If `--skip-existing` 

258 is also passed, then only failed quanta will be 

259 clobbered. Requires the 'run` command's `--extend-run` 

260 flag to be set."""), 

261 is_flag=True) 

262 

263 

264skip_init_writes_option = MWOptionDecorator("--skip-init-writes", 

265 help=unwrap("""Do not write collection-wide 'init output' datasets 

266 (e.g.schemas)."""), 

267 is_flag=True) 

268 

269 

270task_option = MWOptionDecorator("-t", "--task", 

271 callback=split_commas, 

272 help=unwrap("""Task name to add to pipeline, must be a fully qualified task 

273 name. Task name can be followed by colon and label name, if label 

274 is not given then task base name (class name) is used as 

275 label."""), 

276 metavar="TASK[:LABEL]", 

277 multiple=True) 

278 

279 

280timeout_option = MWOptionDecorator("--timeout", 

281 type=click.IntRange(min=0), 

282 help="Timeout for multiprocessing; maximum wall time (sec).") 

283 

284 

285start_method_option = MWOptionDecorator("--start-method", 

286 default=None, 

287 type=click.Choice(["spawn", "fork", "forkserver"]), 

288 help="Multiprocessing start method, default is platform-specific.") 

289 

290 

291fail_fast_option = MWOptionDecorator("--fail-fast", 

292 help=unwrap("""Stop processing at first error, default is to process 

293 as many tasks as possible."""), 

294 is_flag=True) 

295 

296save_execution_butler_option = MWOptionDecorator("--save-execution-butler", 

297 help=unwrap("""Export location for an 

298 execution-specific butler after making 

299 QuantumGraph""")) 

300 

301clobber_execution_butler_option = MWOptionDecorator("--clobber-execution-butler", 

302 help=unwrap("""When creating execution butler overwrite 

303 any existing products"""), 

304 is_flag=True)