Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of ctrl_mpexec. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22import click 

23 

24from lsst.daf.butler.cli.utils import MWOptionDecorator, MWPath, split_commas, unwrap 

25 

26 

27butler_config_option = MWOptionDecorator("-b", "--butler-config", 

28 help="Location of the gen3 butler/registry config file.") 

29 

30 

31data_query_option = MWOptionDecorator("-d", "--data-query", 

32 help="User data selection expression.", 

33 metavar="QUERY") 

34 

35 

36debug_option = MWOptionDecorator("--debug", 

37 help="Enable debugging output using lsstDebug facility (imports debug.py).", 

38 is_flag=True) 

39 

40 

41delete_option = MWOptionDecorator("--delete", 

42 callback=split_commas, 

43 help="Delete task with given label from pipeline.", 

44 multiple=True) 

45 

46 

47do_raise_option = MWOptionDecorator("--do-raise", 

48 help="Raise an exception on error. (else log a message and continue?)", 

49 is_flag=True) 

50 

51 

52extend_run_option = MWOptionDecorator("--extend-run", 

53 help=unwrap("""Instead of creating a new RUN collection, insert datasets 

54 into either the one given by --output-run (if provided) or 

55 the first child collection of - -output(which must be of 

56 type RUN)."""), 

57 is_flag=True) 

58 

59 

60graph_fixup_option = MWOptionDecorator("--graph-fixup", 

61 help=unwrap("""Name of the class or factory method which makes an 

62 instance used for execution graph fixup.""")) 

63 

64 

65init_only_option = MWOptionDecorator("--init-only", 

66 help=unwrap("""Do not actually run; just register dataset types and/or 

67 save init outputs. """), 

68 is_flag=True) 

69 

70 

71input_option = MWOptionDecorator("-i", "--input", 

72 callback=split_commas, 

73 default=list(), 

74 help=unwrap("""Comma-separated names of the input collection(s)."""), 

75 metavar="COLLECTION", 

76 multiple=True) 

77 

78no_versions_option = MWOptionDecorator("--no-versions", 

79 help="Do not save or check package versions.", 

80 is_flag=True) 

81 

82 

83order_pipeline_option = MWOptionDecorator("--order-pipeline", 

84 help=unwrap("""Order tasks in pipeline based on their data 

85 dependencies, ordering is performed as last step before saving or 

86 executing pipeline."""), 

87 is_flag=True) 

88 

89 

90output_option = MWOptionDecorator("-o", "--output", 

91 help=unwrap("""Name of the output CHAINED collection. This may either be an 

92 existing CHAINED collection to use as both input and output 

93 (incompatible with --input), or a new CHAINED collection created 

94 to include all inputs (requires --input). In both cases, the 

95 collection's children will start with an output RUN collection 

96 that directly holds all new datasets (see --output-run)."""), 

97 metavar="COLL") 

98 

99 

100output_run_option = MWOptionDecorator("--output-run", 

101 help=unwrap("""Name of the new output RUN collection. If not provided 

102 then --output must be provided and a new RUN collection will 

103 be created by appending a timestamp to the value passed with 

104 --output. If this collection already exists then 

105 --extend-run must be passed."""), 

106 metavar="COLL") 

107 

108 

109pipeline_option = MWOptionDecorator("-p", "--pipeline", 

110 help="Location of a pipeline definition file in YAML format.", 

111 type=MWPath(file_okay=True, dir_okay=False, readable=True)) 

112 

113 

114pipeline_dot_option = MWOptionDecorator("--pipeline-dot", 

115 help=unwrap(""""Location for storing GraphViz DOT representation of a 

116 pipeline."""), 

117 type=MWPath(writable=True, file_okay=True, dir_okay=False)) 

118 

119 

120profile_option = MWOptionDecorator("--profile", 

121 help="Dump cProfile statistics to file name.", 

122 type=MWPath(file_okay=True, dir_okay=False)) 

123 

124 

125prune_replaced_option = MWOptionDecorator("--prune-replaced", 

126 help=unwrap("""Delete the datasets in the collection replaced by 

127 --replace-run, either just from the datastore 

128 ('unstore') or by removing them and the RUN completely 

129 ('purge'). Requires --replace-run."""), 

130 type=click.Choice(("unstore", "purge"), case_sensitive=False)) 

131 

132 

133qgraph_option = MWOptionDecorator("-g", "--qgraph", 

134 help=unwrap("""Location for a serialized quantum graph definition (pickle 

135 file). If this option is given then all input data options and 

136 pipeline-building options cannot be used."""), 

137 type=MWPath(exists=True, file_okay=True, dir_okay=False, readable=True)) 

138 

139 

140qgraph_dot_option = MWOptionDecorator("--qgraph-dot", 

141 help=unwrap("""Location for storing GraphViz DOT representation of a 

142 quantum graph."""), 

143 type=MWPath(writable=True, file_okay=True, dir_okay=False)) 

144 

145 

146register_dataset_types_option = MWOptionDecorator("--register-dataset-types", 

147 help=unwrap("""Register DatasetTypes that do not already 

148 exist in the Registry."""), 

149 is_flag=True) 

150 

151 

152replace_run_option = MWOptionDecorator("--replace-run", 

153 help=unwrap("""Before creating a new RUN collection in an existing 

154 CHAINED collection, remove the first child collection 

155 (which must be of type RUN). This can be used to repeatedly 

156 write to the same (parent) collection during development, 

157 but it does not delete the datasets associated with the 

158 replaced run unless --prune-replaced is also passed. 

159 Requires --output, and incompatible with --extend-run."""), 

160 is_flag=True) 

161 

162 

163save_pipeline_option = MWOptionDecorator("-s", "--save-pipeline", 

164 help=unwrap("""Location for storing resulting pipeline definition in 

165 YAML format."""), 

166 type=MWPath(dir_okay=False, file_okay=True, writable=True)) 

167 

168save_qgraph_option = MWOptionDecorator("-q", "--save-qgraph", 

169 help=unwrap("""Location for storing a serialized quantum graph 

170 definition (pickle file)."""), 

171 type=MWPath(file_okay=True, dir_okay=False, readable=True)) 

172 

173 

174save_single_quanta_option = MWOptionDecorator("--save-single-quanta", 

175 help=unwrap("""Format string of locations for storing individual 

176 quantum graph definition (pickle files). The curly 

177 brace {} in the input string will be replaced by a 

178 quantum number.""")) 

179 

180 

181show_option = MWOptionDecorator("--show", 

182 callback=split_commas, 

183 help=unwrap("""Dump various info to standard output. Possible items are: 

184 `config`, `config=[Task::]<PATTERN>` or 

185 `config=[Task::]<PATTERN>:NOIGNORECASE` to dump configuration 

186 fields possibly matching given pattern and/or task label; 

187 `history=<FIELD>` to dump configuration history for a field, field 

188 name is specified as [Task::]<PATTERN>; `dump-config`, 

189 `dump-config=Task` to dump complete configuration for a task given 

190 its label or all tasks; `pipeline` to show pipeline composition; 

191 `graph` to show information about quanta; `workflow` to show 

192 information about quanta and their dependency; `tasks` to show 

193 task composition; `uri` to show predicted dataset URIs of 

194 quanta"""), 

195 metavar="ITEM|ITEM=VALUE", 

196 multiple=True) 

197 

198 

199skip_existing_option = MWOptionDecorator("--skip-existing", 

200 help=unwrap("""If all Quantum outputs already exist in the output RUN 

201 collection then that Quantum will be excluded from the 

202 QuantumGraph. Requires the 'run` command's `--extend-run` 

203 flag to be set."""), 

204 is_flag=True) 

205 

206 

207clobber_partial_outputs_option = MWOptionDecorator("--clobber-partial-outputs", 

208 help=unwrap("""Remove incomplete outputs from previous 

209 execution of the same quantum before new 

210 execution."""), 

211 is_flag=True) 

212 

213 

214skip_init_writes_option = MWOptionDecorator("--skip-init-writes", 

215 help=unwrap("""Do not write collection-wide 'init output' datasets 

216 (e.g.schemas)."""), 

217 is_flag=True) 

218 

219 

220task_option = MWOptionDecorator("-t", "--task", 

221 callback=split_commas, 

222 help=unwrap("""Task name to add to pipeline, must be a fully qualified task 

223 name. Task name can be followed by colon and label name, if label 

224 is not given then task base name (class name) is used as 

225 label."""), 

226 metavar="TASK[:LABEL]", 

227 multiple=True) 

228 

229 

230timeout_option = MWOptionDecorator("--timeout", 

231 help="Timeout for multiprocessing; maximum wall time (sec).") 

232 

233fail_fast_option = MWOptionDecorator("--fail-fast", 

234 help=unwrap("""Stop processing at first error, default is to process 

235 as many tasks as possible."""), 

236 is_flag=True)