Coverage for python/lsst/ctrl/mpexec/cli/cmd/commands.py: 58%

151 statements  

« prev     ^ index     » next       coverage.py v7.2.7, created at 2023-06-14 09:14 +0000

1# This file is part of ctrl_mpexec. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22import sys 

23from collections.abc import Iterator 

24from contextlib import contextmanager 

25from functools import partial 

26from tempfile import NamedTemporaryFile 

27from typing import Any 

28 

29import click 

30import coverage 

31import lsst.pipe.base.cli.opt as pipeBaseOpts 

32from lsst.ctrl.mpexec.showInfo import ShowInfo 

33from lsst.daf.butler.cli.opt import ( 

34 config_file_option, 

35 config_option, 

36 confirm_option, 

37 options_file_option, 

38 processes_option, 

39 repo_argument, 

40) 

41from lsst.daf.butler.cli.utils import MWCtxObj, catch_and_exit, option_section, unwrap 

42 

43from .. import opt as ctrlMpExecOpts 

44from .. import script 

45from ..script import confirmable 

46from ..utils import PipetaskCommand, makePipelineActions 

47 

48epilog = unwrap( 

49 """Notes: 

50 

51--task, --delete, --config, --config-file, and --instrument action options can 

52appear multiple times; all values are used, in order left to right. 

53 

54FILE reads command-line options from the specified file. Data may be 

55distributed among multiple lines (e.g. one option per line). Data after # is 

56treated as a comment and ignored. Blank lines and lines starting with # are 

57ignored.) 

58""" 

59) 

60 

61 

62def _collectActions(ctx: click.Context, **kwargs: Any) -> dict[str, Any]: 

63 """Extract pipeline building options, replace them with PipelineActions, 

64 return updated `kwargs`. 

65 

66 Notes 

67 ----- 

68 The pipeline actions (task, delete, config, config_file, and instrument) 

69 must be handled in the order they appear on the command line, but the CLI 

70 specification gives them all different option names. So, instead of using 

71 the individual action options as they appear in kwargs (because 

72 invocation order can't be known), we capture the CLI arguments by 

73 overriding `click.Command.parse_args` and save them in the Context's 

74 `obj` parameter. We use `makePipelineActions` to create a list of 

75 pipeline actions from the CLI arguments and pass that list to the script 

76 function using the `pipeline_actions` kwarg name, and remove the action 

77 options from kwargs. 

78 """ 

79 for pipelineAction in ( 

80 ctrlMpExecOpts.task_option.name(), 

81 ctrlMpExecOpts.delete_option.name(), 

82 config_option.name(), 

83 config_file_option.name(), 

84 pipeBaseOpts.instrument_option.name(), 

85 ): 

86 kwargs.pop(pipelineAction) 

87 

88 actions = makePipelineActions(MWCtxObj.getFrom(ctx).args) 

89 pipeline_actions = [] 

90 for action in actions: 

91 pipeline_actions.append(action) 

92 

93 kwargs["pipeline_actions"] = pipeline_actions 

94 return kwargs 

95 

96 

97def _unhandledShow(show: ShowInfo, cmd: str) -> None: 

98 if show.unhandled: 98 ↛ 99line 98 didn't jump to line 99, because the condition on line 98 was never true

99 print( 

100 f"The following '--show' options were not known to the {cmd} command: " 

101 f"{', '.join(show.unhandled)}", 

102 file=sys.stderr, 

103 ) 

104 

105 

106@click.command(cls=PipetaskCommand, epilog=epilog, short_help="Build pipeline definition.") 

107@click.pass_context 

108@ctrlMpExecOpts.show_option() 

109@ctrlMpExecOpts.pipeline_build_options() 

110@option_section(sectionText="") 

111@options_file_option() 

112@catch_and_exit 

113def build(ctx: click.Context, **kwargs: Any) -> None: 

114 """Build and optionally save pipeline definition. 

115 

116 This does not require input data to be specified. 

117 """ 

118 kwargs = _collectActions(ctx, **kwargs) 

119 show = ShowInfo(kwargs.pop("show", [])) 

120 script.build(**kwargs, show=show) 

121 _unhandledShow(show, "build") 

122 

123 

124@contextmanager 

125def coverage_context(kwargs: dict[str, Any]) -> Iterator[None]: 

126 """Enable coverage recording.""" 

127 packages = kwargs.pop("cov_packages", ()) 

128 report = kwargs.pop("cov_report", True) 

129 if not kwargs.pop("coverage", False): 

130 yield 

131 return 

132 with NamedTemporaryFile("w") as rcfile: 

133 rcfile.write( 

134 """ 

135[run] 

136branch = True 

137concurrency = multiprocessing 

138""" 

139 ) 

140 if packages: 

141 packages_str = ",".join(packages) 

142 rcfile.write(f"source_pkgs = {packages_str}\n") 

143 rcfile.flush() 

144 cov = coverage.Coverage(config_file=rcfile.name) 

145 cov.start() 

146 try: 

147 yield 

148 finally: 

149 cov.stop() 

150 cov.save() 

151 if report: 

152 outdir = "./covhtml" 

153 cov.html_report(directory=outdir) 

154 click.echo(f"Coverage report written to {outdir}.") 

155 

156 

157@click.command(cls=PipetaskCommand, epilog=epilog) 

158@click.pass_context 

159@ctrlMpExecOpts.show_option() 

160@ctrlMpExecOpts.pipeline_build_options() 

161@ctrlMpExecOpts.qgraph_options() 

162@ctrlMpExecOpts.butler_options() 

163@option_section(sectionText="") 

164@options_file_option() 

165@catch_and_exit 

166def qgraph(ctx: click.Context, **kwargs: Any) -> None: 

167 """Build and optionally save quantum graph.""" 

168 kwargs = _collectActions(ctx, **kwargs) 

169 with coverage_context(kwargs): 

170 show = ShowInfo(kwargs.pop("show", [])) 

171 pipeline = script.build(**kwargs, show=show) 

172 if show.handled and not show.unhandled: 

173 print( 

174 "No quantum graph generated. The --show option was given and all options were processed.", 

175 file=sys.stderr, 

176 ) 

177 return 

178 if script.qgraph(pipelineObj=pipeline, **kwargs, show=show) is None: 

179 raise click.ClickException("QuantumGraph was empty; CRITICAL logs above should provide details.") 

180 _unhandledShow(show, "qgraph") 

181 

182 

183@click.command(cls=PipetaskCommand, epilog=epilog) 

184@ctrlMpExecOpts.run_options() 

185@catch_and_exit 

186def run(ctx: click.Context, **kwargs: Any) -> None: 

187 """Build and execute pipeline and quantum graph.""" 

188 kwargs = _collectActions(ctx, **kwargs) 

189 with coverage_context(kwargs): 

190 show = ShowInfo(kwargs.pop("show", [])) 

191 pipeline = script.build(**kwargs, show=show) 

192 if show.handled and not show.unhandled: 

193 print( 

194 "No quantum graph generated or pipeline executed. " 

195 "The --show option was given and all options were processed.", 

196 file=sys.stderr, 

197 ) 

198 return 

199 if (qgraph := script.qgraph(pipelineObj=pipeline, **kwargs, show=show)) is None: 

200 raise click.ClickException("QuantumGraph was empty; CRITICAL logs above should provide details.") 

201 _unhandledShow(show, "run") 

202 if show.handled: 

203 print( 

204 "No pipeline executed. The --show option was given and all options were processed.", 

205 file=sys.stderr, 

206 ) 

207 return 

208 script.run(qgraphObj=qgraph, **kwargs) 

209 

210 

211@click.command(cls=PipetaskCommand) 

212@ctrlMpExecOpts.butler_config_option() 

213@ctrlMpExecOpts.collection_argument() 

214@confirm_option() 

215@ctrlMpExecOpts.recursive_option( 

216 help="""If the parent CHAINED collection has child CHAINED collections, 

217 search the children until nested chains that start with the parent's name 

218 are removed.""" 

219) 

220def purge(confirm: bool, **kwargs: Any) -> None: 

221 """Remove a CHAINED collection and its contained collections. 

222 

223 COLLECTION is the name of the chained collection to purge. it must not be a 

224 child of any other CHAINED collections 

225 

226 Child collections must be members of exactly one collection. 

227 

228 The collections that will be removed will be printed, there will be an 

229 option to continue or abort (unless using --no-confirm). 

230 """ 

231 confirmable.confirm(partial(script.purge, **kwargs), confirm) 

232 

233 

234@click.command(cls=PipetaskCommand) 

235@ctrlMpExecOpts.butler_config_option() 

236@ctrlMpExecOpts.collection_argument() 

237@confirm_option() 

238def cleanup(confirm: bool, **kwargs: Any) -> None: 

239 """Remove non-members of CHAINED collections. 

240 

241 Removes collections that start with the same name as a CHAINED 

242 collection but are not members of that collection. 

243 """ 

244 confirmable.confirm(partial(script.cleanup, **kwargs), confirm) 

245 

246 

247@click.command(cls=PipetaskCommand) 

248@repo_argument() 

249@ctrlMpExecOpts.qgraph_argument() 

250@ctrlMpExecOpts.config_search_path_option() 

251@ctrlMpExecOpts.qgraph_id_option() 

252@ctrlMpExecOpts.coverage_options() 

253def pre_exec_init_qbb(repo: str, qgraph: str, **kwargs: Any) -> None: 

254 """Execute pre-exec-init on Quantum-Backed Butler. 

255 

256 REPO is the location of the butler/registry config file. 

257 

258 QGRAPH is the path to a serialized Quantum Graph file. 

259 """ 

260 with coverage_context(kwargs): 

261 script.pre_exec_init_qbb(repo, qgraph, **kwargs) 

262 

263 

264@click.command(cls=PipetaskCommand) 

265@repo_argument() 

266@ctrlMpExecOpts.qgraph_argument() 

267@ctrlMpExecOpts.config_search_path_option() 

268@ctrlMpExecOpts.qgraph_id_option() 

269@ctrlMpExecOpts.qgraph_node_id_option() 

270@processes_option() 

271@ctrlMpExecOpts.pdb_option() 

272@ctrlMpExecOpts.profile_option() 

273@ctrlMpExecOpts.coverage_options() 

274@ctrlMpExecOpts.debug_option() 

275@ctrlMpExecOpts.start_method_option() 

276@ctrlMpExecOpts.timeout_option() 

277@ctrlMpExecOpts.fail_fast_option() 

278@ctrlMpExecOpts.summary_option() 

279@ctrlMpExecOpts.enable_implicit_threading_option() 

280def run_qbb(repo: str, qgraph: str, **kwargs: Any) -> None: 

281 """Execute pipeline using Quantum-Backed Butler. 

282 

283 REPO is the location of the butler/registry config file. 

284 

285 QGRAPH is the path to a serialized Quantum Graph file. 

286 """ 

287 with coverage_context(kwargs): 

288 script.run_qbb(repo, qgraph, **kwargs) 

289 

290 

291@click.command(cls=PipetaskCommand) 

292@ctrlMpExecOpts.qgraph_argument() 

293@ctrlMpExecOpts.run_argument() 

294@ctrlMpExecOpts.output_qgraph_argument() 

295@ctrlMpExecOpts.metadata_run_key_option() 

296@ctrlMpExecOpts.update_graph_id_option() 

297def update_graph_run( 

298 qgraph: str, 

299 run: str, 

300 output_qgraph: str, 

301 metadata_run_key: str, 

302 update_graph_id: bool, 

303) -> None: 

304 """Update existing quantum graph with new output run name and re-generate 

305 output dataset IDs. 

306 

307 QGRAPH is the URL to a serialized Quantum Graph file. 

308 

309 RUN is the new RUN collection name for output graph. 

310 

311 OUTPUT_QGRAPH is the URL to store the updated Quantum Graph. 

312 """ 

313 script.update_graph_run(qgraph, run, output_qgraph, metadata_run_key, update_graph_id)