Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of ctrl_mpexec. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22import click 

23 

24from lsst.daf.butler.cli.utils import MWOptionDecorator, MWPath, split_commas, unwrap 

25 

26 

27butler_config_option = MWOptionDecorator("-b", "--butler-config", 

28 help="Location of the gen3 butler/registry config file.") 

29 

30 

31data_query_option = MWOptionDecorator("-d", "--data-query", 

32 help="User data selection expression.", 

33 metavar="QUERY") 

34 

35 

36debug_option = MWOptionDecorator("--debug", 

37 help="Enable debugging output using lsstDebug facility (imports debug.py).", 

38 is_flag=True) 

39 

40 

41delete_option = MWOptionDecorator("--delete", 

42 callback=split_commas, 

43 help="Delete task with given label from pipeline.", 

44 multiple=True) 

45 

46 

47do_raise_option = MWOptionDecorator("--do-raise", 

48 help="Raise an exception on error. (else log a message and continue?)", 

49 is_flag=True) 

50 

51 

52extend_run_option = MWOptionDecorator("--extend-run", 

53 help=unwrap("""Instead of creating a new RUN collection, insert datasets 

54 into either the one given by --output-run (if provided) or 

55 the first child collection of - -output(which must be of 

56 type RUN)."""), 

57 is_flag=True) 

58 

59 

60graph_fixup_option = MWOptionDecorator("--graph-fixup", 

61 help=unwrap("""Name of the class or factory method which makes an 

62 instance used for execution graph fixup.""")) 

63 

64 

65init_only_option = MWOptionDecorator("--init-only", 

66 help=unwrap("""Do not actually run; just register dataset types and/or 

67 save init outputs. """), 

68 is_flag=True) 

69 

70 

71# TODO defaultMetavar and defaultHelp both match with the handling 

72# specified by the input_option callback defined in commands.py. Should 

73# that callback definition get moved here? Should these defaults be made 

74# less use-case specific and moved to commands.py? Is it ok as is? 

75input_option = MWOptionDecorator("-i", "--input", 

76 callback=split_commas, 

77 default=list(), 

78 help=unwrap("""Comma-separated names of the input collection(s). Entries may 

79 include a colon (:), the first string is a dataset type name that 

80 restricts the search in that collection."""), 

81 metavar="COLL,DSTYPE:COLL", 

82 multiple=True) 

83 

84no_versions_option = MWOptionDecorator("--no-versions", 

85 help="Do not save or check package versions.", 

86 is_flag=True) 

87 

88 

89order_pipeline_option = MWOptionDecorator("--order-pipeline", 

90 help=unwrap("""Order tasks in pipeline based on their data 

91 dependencies, ordering is performed as last step before saving or 

92 executing pipeline."""), 

93 is_flag=True) 

94 

95 

96output_option = MWOptionDecorator("-o", "--output", 

97 help=unwrap("""Name of the output CHAINED collection. This may either be an 

98 existing CHAINED collection to use as both input and output 

99 (incompatible with --input), or a new CHAINED collection created 

100 to include all inputs (requires --input). In both cases, the 

101 collection's children will start with an output RUN collection 

102 that directly holds all new datasets (see --output-run)."""), 

103 metavar="COLL") 

104 

105 

106output_run_option = MWOptionDecorator("--output-run", 

107 help=unwrap("""Name of the new output RUN collection. If not provided 

108 then --output must be provided and a new RUN collection will 

109 be created by appending a timestamp to the value passed with 

110 --output. If this collection already exists then 

111 --extend-run must be passed."""), 

112 metavar="COLL") 

113 

114 

115pipeline_option = MWOptionDecorator("-p", "--pipeline", 

116 help="Location of a pipeline definition file in YAML format.", 

117 type=MWPath(file_okay=True, dir_okay=False, readable=True)) 

118 

119 

120pipeline_dot_option = MWOptionDecorator("--pipeline-dot", 

121 help=unwrap(""""Location for storing GraphViz DOT representation of a 

122 pipeline."""), 

123 type=MWPath(writable=True, file_okay=True, dir_okay=False)) 

124 

125 

126profile_option = MWOptionDecorator("--profile", 

127 help="Dump cProfile statistics to file name.", 

128 type=MWPath(file_okay=True, dir_okay=False)) 

129 

130 

131prune_replaced_option = MWOptionDecorator("--prune-replaced", 

132 help=unwrap("""Delete the datasets in the collection replaced by 

133 --replace-run, either just from the datastore 

134 ('unstore') or by removing them and the RUN completely 

135 ('purge'). Requires --replace-run."""), 

136 type=click.Choice(("unstore", "purge"), case_sensitive=False)) 

137 

138 

139qgraph_option = MWOptionDecorator("-g", "--qgraph", 

140 help=unwrap("""Location for a serialized quantum graph definition (pickle 

141 file). If this option is given then all input data options and 

142 pipeline-building options cannot be used."""), 

143 type=MWPath(exists=True, file_okay=True, dir_okay=False, readable=True)) 

144 

145 

146qgraph_dot_option = MWOptionDecorator("--qgraph-dot", 

147 help=unwrap("""Location for storing GraphViz DOT representation of a 

148 quantum graph."""), 

149 type=MWPath(writable=True, file_okay=True, dir_okay=False)) 

150 

151 

152register_dataset_types_option = MWOptionDecorator("--register-dataset-types", 

153 help=unwrap("""Register DatasetTypes that do not already 

154 exist in the Registry."""), 

155 is_flag=True) 

156 

157 

158replace_run_option = MWOptionDecorator("--replace-run", 

159 help=unwrap("""Before creating a new RUN collection in an existing 

160 CHAINED collection, remove the first child collection 

161 (which must be of type RUN). This can be used to repeatedly 

162 write to the same (parent) collection during development, 

163 but it does not delete the datasets associated with the 

164 replaced run unless --prune-replaced is also passed. 

165 Requires --output, and incompatible with --extend-run."""), 

166 is_flag=True) 

167 

168 

169save_pipeline_option = MWOptionDecorator("-s", "--save-pipeline", 

170 help=unwrap("""Location for storing resulting pipeline definition in 

171 YAML format."""), 

172 type=MWPath(dir_okay=False, file_okay=True, writable=True)) 

173 

174save_qgraph_option = MWOptionDecorator("-q", "--save-qgraph", 

175 help=unwrap("""Location for storing a serialized quantum graph 

176 definition (pickle file)."""), 

177 type=MWPath(file_okay=True, dir_okay=False, readable=True)) 

178 

179 

180save_single_quanta_option = MWOptionDecorator("--save-single-quanta", 

181 help=unwrap("""Format string of locations for storing individual 

182 quantum graph definition (pickle files). The curly 

183 brace {} in the input string will be replaced by a 

184 quantum number.""")) 

185 

186 

187show_option = MWOptionDecorator("--show", 

188 callback=split_commas, 

189 help=unwrap("""Dump various info to standard output. Possible items are: 

190 `config`, `config=[Task::]<PATTERN>` or 

191 `config=[Task::]<PATTERN>:NOIGNORECASE` to dump configuration 

192 fields possibly matching given pattern and/or task label; 

193 `history=<FIELD>` to dump configuration history for a field, field 

194 name is specified as [Task::][SubTask.]Field; `dump-config`, 

195 `dump-config=Task` to dump complete configuration for a task given 

196 its label or all tasks; `pipeline` to show pipeline composition; 

197 `graph` to show information about quanta; `workflow` to show 

198 information about quanta and their dependency; `tasks` to show 

199 task composition; `uri` to show predicted dataset URIs of 

200 quanta"""), 

201 metavar="ITEM|ITEM=VALUE", 

202 multiple=True) 

203 

204 

205skip_existing_option = MWOptionDecorator("--skip-existing", 

206 help=unwrap("""If all Quantum outputs already exist in the output RUN 

207 collection then that Quantum will be excluded from the 

208 QuantumGraph. Requires the 'run` command's `--extend-run` 

209 flag to be set."""), 

210 is_flag=True) 

211 

212 

213clobber_partial_outputs_option = MWOptionDecorator("--clobber-partial-outputs", 

214 help=unwrap("""Remove incomplete outputs from previous 

215 execution of the same quantum before new 

216 execution."""), 

217 is_flag=True) 

218 

219 

220skip_init_writes_option = MWOptionDecorator("--skip-init-writes", 

221 help=unwrap("""Do not write collection-wide 'init output' datasets 

222 (e.g.schemas)."""), 

223 is_flag=True) 

224 

225 

226task_option = MWOptionDecorator("-t", "--task", 

227 callback=split_commas, 

228 help=unwrap("""Task name to add to pipeline, must be a fully qualified task 

229 name. Task name can be followed by colon and label name, if label 

230 is not given then task base name (class name) is used as 

231 label."""), 

232 metavar="TASK[:LABEL]", 

233 multiple=True) 

234 

235 

236timeout_option = MWOptionDecorator("--timeout", 

237 help="Timeout for multiprocessing; maximum wall time (sec).") 

238 

239fail_fast_option = MWOptionDecorator("--fail-fast", 

240 help=unwrap("""Stop processing at first error, default is to process 

241 as many tasks as possible."""), 

242 is_flag=True)