Coverage for python/lsst/ctrl/mpexec/cli/cmd/commands.py: 69%

116 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2023-02-09 11:21 +0000

1# This file is part of ctrl_mpexec. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22import sys 

23from functools import partial 

24from typing import Any 

25 

26import click 

27import lsst.pipe.base.cli.opt as pipeBaseOpts 

28from lsst.ctrl.mpexec.showInfo import ShowInfo 

29from lsst.daf.butler.cli.opt import ( 

30 config_file_option, 

31 config_option, 

32 confirm_option, 

33 options_file_option, 

34 processes_option, 

35 repo_argument, 

36) 

37from lsst.daf.butler.cli.utils import MWCtxObj, catch_and_exit, option_section, unwrap 

38 

39from .. import opt as ctrlMpExecOpts 

40from .. import script 

41from ..script import confirmable 

42from ..utils import _ACTION_CONFIG, _ACTION_CONFIG_FILE, PipetaskCommand, makePipelineActions 

43 

44epilog = unwrap( 

45 """Notes: 

46 

47--task, --delete, --config, --config-file, and --instrument action options can 

48appear multiple times; all values are used, in order left to right. 

49 

50FILE reads command-line options from the specified file. Data may be 

51distributed among multiple lines (e.g. one option per line). Data after # is 

52treated as a comment and ignored. Blank lines and lines starting with # are 

53ignored.) 

54""" 

55) 

56 

57 

58def _collectActions(ctx: click.Context, **kwargs: Any) -> dict[str, Any]: 

59 """Extract pipeline building options, replace them with PipelineActions, 

60 return updated `kwargs`. 

61 

62 Notes 

63 ----- 

64 The pipeline actions (task, delete, config, config_file, and instrument) 

65 must be handled in the order they appear on the command line, but the CLI 

66 specification gives them all different option names. So, instead of using 

67 the individual action options as they appear in kwargs (because 

68 invocation order can't be known), we capture the CLI arguments by 

69 overriding `click.Command.parse_args` and save them in the Context's 

70 `obj` parameter. We use `makePipelineActions` to create a list of 

71 pipeline actions from the CLI arguments and pass that list to the script 

72 function using the `pipeline_actions` kwarg name, and remove the action 

73 options from kwargs. 

74 """ 

75 for pipelineAction in ( 

76 ctrlMpExecOpts.task_option.name(), 

77 ctrlMpExecOpts.delete_option.name(), 

78 config_option.name(), 

79 config_file_option.name(), 

80 pipeBaseOpts.instrument_option.name(), 

81 ): 

82 kwargs.pop(pipelineAction) 

83 

84 actions = makePipelineActions(MWCtxObj.getFrom(ctx).args) 

85 mock_configs = [] 

86 pipeline_actions = [] 

87 for action in actions: 

88 if action.label and action.label.endswith("-mock"): 88 ↛ 89line 88 didn't jump to line 89, because the condition on line 88 was never true

89 if action.action not in (_ACTION_CONFIG.action, _ACTION_CONFIG_FILE.action): 

90 raise ValueError(f"Unexpected option for mock task config overrides: {action}") 

91 mock_configs.append(action) 

92 else: 

93 pipeline_actions.append(action) 

94 

95 kwargs["mock_configs"] = mock_configs 

96 kwargs["pipeline_actions"] = pipeline_actions 

97 return kwargs 

98 

99 

100def _unhandledShow(show: ShowInfo, cmd: str) -> None: 

101 if show.unhandled: 101 ↛ 102line 101 didn't jump to line 102, because the condition on line 101 was never true

102 print( 

103 f"The following '--show' options were not known to the {cmd} command: " 

104 f"{', '.join(show.unhandled)}", 

105 file=sys.stderr, 

106 ) 

107 

108 

109@click.command(cls=PipetaskCommand, epilog=epilog, short_help="Build pipeline definition.") 

110@click.pass_context 

111@ctrlMpExecOpts.show_option() 

112@ctrlMpExecOpts.pipeline_build_options() 

113@option_section(sectionText="") 

114@options_file_option() 

115@catch_and_exit 

116def build(ctx: click.Context, **kwargs: Any) -> None: 

117 """Build and optionally save pipeline definition. 

118 

119 This does not require input data to be specified. 

120 """ 

121 kwargs = _collectActions(ctx, **kwargs) 

122 show = ShowInfo(kwargs.pop("show", [])) 

123 script.build(**kwargs, show=show) 

124 _unhandledShow(show, "build") 

125 

126 

127@click.command(cls=PipetaskCommand, epilog=epilog) 

128@click.pass_context 

129@ctrlMpExecOpts.show_option() 

130@ctrlMpExecOpts.pipeline_build_options() 

131@ctrlMpExecOpts.qgraph_options() 

132@ctrlMpExecOpts.butler_options() 

133@option_section(sectionText="") 

134@options_file_option() 

135@catch_and_exit 

136def qgraph(ctx: click.Context, **kwargs: Any) -> None: 

137 """Build and optionally save quantum graph.""" 

138 kwargs = _collectActions(ctx, **kwargs) 

139 show = ShowInfo(kwargs.pop("show", [])) 

140 pipeline = script.build(**kwargs, show=show) 

141 if show.handled and not show.unhandled: 

142 print( 

143 "No quantum graph generated. The --show option was given and all options were processed.", 

144 file=sys.stderr, 

145 ) 

146 return 

147 if script.qgraph(pipelineObj=pipeline, **kwargs, show=show) is None: 

148 raise click.ClickException("QuantumGraph was empty; CRITICAL logs above should provide details.") 

149 _unhandledShow(show, "qgraph") 

150 

151 

152@click.command(cls=PipetaskCommand, epilog=epilog) 

153@ctrlMpExecOpts.run_options() 

154@catch_and_exit 

155def run(ctx: click.Context, **kwargs: Any) -> None: 

156 """Build and execute pipeline and quantum graph.""" 

157 kwargs = _collectActions(ctx, **kwargs) 

158 show = ShowInfo(kwargs.pop("show", [])) 

159 pipeline = script.build(**kwargs, show=show) 

160 if show.handled and not show.unhandled: 

161 print( 

162 "No quantum graph generated or pipeline executed. " 

163 "The --show option was given and all options were processed.", 

164 file=sys.stderr, 

165 ) 

166 return 

167 if (qgraph := script.qgraph(pipelineObj=pipeline, **kwargs, show=show)) is None: 

168 raise click.ClickException("QuantumGraph was empty; CRITICAL logs above should provide details.") 

169 _unhandledShow(show, "run") 

170 if show.handled: 

171 print( 

172 "No pipeline executed. The --show option was given and all options were processed.", 

173 file=sys.stderr, 

174 ) 

175 return 

176 script.run(qgraphObj=qgraph, **kwargs) 

177 

178 

179@click.command(cls=PipetaskCommand) 

180@ctrlMpExecOpts.butler_config_option() 

181@ctrlMpExecOpts.collection_argument() 

182@confirm_option() 

183@ctrlMpExecOpts.recursive_option( 

184 help="""If the parent CHAINED collection has child CHAINED collections, 

185 search the children until nested chains that start with the parent's name 

186 are removed.""" 

187) 

188def purge(confirm: bool, **kwargs: Any) -> None: 

189 """Remove a CHAINED collection and its contained collections. 

190 

191 COLLECTION is the name of the chained collection to purge. it must not be a 

192 child of any other CHAINED collections 

193 

194 Child collections must be members of exactly one collection. 

195 

196 The collections that will be removed will be printed, there will be an 

197 option to continue or abort (unless using --no-confirm). 

198 """ 

199 confirmable.confirm(partial(script.purge, **kwargs), confirm) 

200 

201 

202@click.command(cls=PipetaskCommand) 

203@ctrlMpExecOpts.butler_config_option() 

204@ctrlMpExecOpts.collection_argument() 

205@confirm_option() 

206def cleanup(confirm: bool, **kwargs: Any) -> None: 

207 """Remove non-members of CHAINED collections. 

208 

209 Removes collections that start with the same name as a CHAINED 

210 collection but are not members of that collection. 

211 """ 

212 confirmable.confirm(partial(script.cleanup, **kwargs), confirm) 

213 

214 

215@click.command(cls=PipetaskCommand) 

216@repo_argument() 

217@ctrlMpExecOpts.qgraph_argument() 

218@ctrlMpExecOpts.config_search_path_option() 

219@ctrlMpExecOpts.qgraph_id_option() 

220def pre_exec_init_qbb(repo: str, qgraph: str, **kwargs: Any) -> None: 

221 """Execute pre-exec-init on Quantum-Backed Butler. 

222 

223 REPO is the location of the butler/registry config file. 

224 

225 QGRAPH is the path to a serialized Quantum Graph file. 

226 """ 

227 script.pre_exec_init_qbb(repo, qgraph, **kwargs) 

228 

229 

230@click.command(cls=PipetaskCommand) 

231@repo_argument() 

232@ctrlMpExecOpts.qgraph_argument() 

233@ctrlMpExecOpts.config_search_path_option() 

234@ctrlMpExecOpts.qgraph_id_option() 

235@ctrlMpExecOpts.qgraph_node_id_option() 

236@processes_option() 

237@ctrlMpExecOpts.pdb_option() 

238@ctrlMpExecOpts.profile_option() 

239@ctrlMpExecOpts.debug_option() 

240@ctrlMpExecOpts.start_method_option() 

241@ctrlMpExecOpts.timeout_option() 

242@ctrlMpExecOpts.fail_fast_option() 

243@ctrlMpExecOpts.summary_option() 

244@ctrlMpExecOpts.enable_implicit_threading_option() 

245def run_qbb(repo: str, qgraph: str, **kwargs: Any) -> None: 

246 """Execute pipeline using Quantum-Backed Butler. 

247 

248 REPO is the location of the butler/registry config file. 

249 

250 QGRAPH is the path to a serialized Quantum Graph file. 

251 """ 

252 script.run_qbb(repo, qgraph, **kwargs)