Coverage for python/lsst/ctrl/mpexec/cli/cmd/commands.py: 76%

153 statements  

« prev     ^ index     » next       coverage.py v7.3.1, created at 2023-09-13 09:53 +0000

1# This file is part of ctrl_mpexec. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This software is dual licensed under the GNU General Public License and also 

10# under a 3-clause BSD license. Recipients may choose which of these licenses 

11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, 

12# respectively. If you choose the GPL option then the following text applies 

13# (but note that there is still no warranty even if you opt for BSD instead): 

14# 

15# This program is free software: you can redistribute it and/or modify 

16# it under the terms of the GNU General Public License as published by 

17# the Free Software Foundation, either version 3 of the License, or 

18# (at your option) any later version. 

19# 

20# This program is distributed in the hope that it will be useful, 

21# but WITHOUT ANY WARRANTY; without even the implied warranty of 

22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

23# GNU General Public License for more details. 

24# 

25# You should have received a copy of the GNU General Public License 

26# along with this program. If not, see <http://www.gnu.org/licenses/>. 

27 

28import sys 

29from collections.abc import Iterator 

30from contextlib import contextmanager 

31from functools import partial 

32from tempfile import NamedTemporaryFile 

33from typing import Any 

34 

35import click 

36import coverage 

37import lsst.pipe.base.cli.opt as pipeBaseOpts 

38from lsst.ctrl.mpexec.showInfo import ShowInfo 

39from lsst.daf.butler.cli.opt import ( 

40 config_file_option, 

41 config_option, 

42 confirm_option, 

43 options_file_option, 

44 processes_option, 

45 repo_argument, 

46) 

47from lsst.daf.butler.cli.utils import MWCtxObj, catch_and_exit, option_section, unwrap 

48 

49from .. import opt as ctrlMpExecOpts 

50from .. import script 

51from ..script import confirmable 

52from ..utils import PipetaskCommand, makePipelineActions 

53 

54epilog = unwrap( 

55 """Notes: 

56 

57--task, --delete, --config, --config-file, and --instrument action options can 

58appear multiple times; all values are used, in order left to right. 

59 

60FILE reads command-line options from the specified file. Data may be 

61distributed among multiple lines (e.g. one option per line). Data after # is 

62treated as a comment and ignored. Blank lines and lines starting with # are 

63ignored.) 

64""" 

65) 

66 

67 

68def _collectActions(ctx: click.Context, **kwargs: Any) -> dict[str, Any]: 

69 """Extract pipeline building options, replace them with PipelineActions, 

70 return updated `kwargs`. 

71 

72 Notes 

73 ----- 

74 The pipeline actions (task, delete, config, config_file, and instrument) 

75 must be handled in the order they appear on the command line, but the CLI 

76 specification gives them all different option names. So, instead of using 

77 the individual action options as they appear in kwargs (because 

78 invocation order can't be known), we capture the CLI arguments by 

79 overriding `click.Command.parse_args` and save them in the Context's 

80 `obj` parameter. We use `makePipelineActions` to create a list of 

81 pipeline actions from the CLI arguments and pass that list to the script 

82 function using the `pipeline_actions` kwarg name, and remove the action 

83 options from kwargs. 

84 """ 

85 for pipelineAction in ( 

86 ctrlMpExecOpts.task_option.name(), 

87 ctrlMpExecOpts.delete_option.name(), 

88 config_option.name(), 

89 config_file_option.name(), 

90 pipeBaseOpts.instrument_option.name(), 

91 ): 

92 kwargs.pop(pipelineAction) 

93 

94 actions = makePipelineActions(MWCtxObj.getFrom(ctx).args) 

95 pipeline_actions = [] 

96 for action in actions: 

97 pipeline_actions.append(action) 

98 

99 kwargs["pipeline_actions"] = pipeline_actions 

100 return kwargs 

101 

102 

103def _unhandledShow(show: ShowInfo, cmd: str) -> None: 

104 if show.unhandled: 104 ↛ 105line 104 didn't jump to line 105, because the condition on line 104 was never true

105 print( 

106 f"The following '--show' options were not known to the {cmd} command: " 

107 f"{', '.join(show.unhandled)}", 

108 file=sys.stderr, 

109 ) 

110 

111 

112@click.command(cls=PipetaskCommand, epilog=epilog, short_help="Build pipeline definition.") 

113@click.pass_context 

114@ctrlMpExecOpts.show_option() 

115@ctrlMpExecOpts.pipeline_build_options() 

116@option_section(sectionText="") 

117@options_file_option() 

118@catch_and_exit 

119def build(ctx: click.Context, **kwargs: Any) -> None: 

120 """Build and optionally save pipeline definition. 

121 

122 This does not require input data to be specified. 

123 """ 

124 kwargs = _collectActions(ctx, **kwargs) 

125 show = ShowInfo(kwargs.pop("show", [])) 

126 script.build(**kwargs, show=show) 

127 _unhandledShow(show, "build") 

128 

129 

130@contextmanager 

131def coverage_context(kwargs: dict[str, Any]) -> Iterator[None]: 

132 """Enable coverage recording.""" 

133 packages = kwargs.pop("cov_packages", ()) 

134 report = kwargs.pop("cov_report", True) 

135 if not kwargs.pop("coverage", False): 

136 yield 

137 return 

138 with NamedTemporaryFile("w") as rcfile: 

139 rcfile.write( 

140 """ 

141[run] 

142branch = True 

143concurrency = multiprocessing 

144""" 

145 ) 

146 if packages: 

147 packages_str = ",".join(packages) 

148 rcfile.write(f"source_pkgs = {packages_str}\n") 

149 rcfile.flush() 

150 cov = coverage.Coverage(config_file=rcfile.name) 

151 cov.start() 

152 try: 

153 yield 

154 finally: 

155 cov.stop() 

156 cov.save() 

157 if report: 

158 outdir = "./covhtml" 

159 cov.html_report(directory=outdir) 

160 click.echo(f"Coverage report written to {outdir}.") 

161 

162 

163@click.command(cls=PipetaskCommand, epilog=epilog) 

164@click.pass_context 

165@ctrlMpExecOpts.show_option() 

166@ctrlMpExecOpts.pipeline_build_options() 

167@ctrlMpExecOpts.qgraph_options() 

168@ctrlMpExecOpts.butler_options() 

169@option_section(sectionText="") 

170@options_file_option() 

171@catch_and_exit 

172def qgraph(ctx: click.Context, **kwargs: Any) -> None: 

173 """Build and optionally save quantum graph.""" 

174 kwargs = _collectActions(ctx, **kwargs) 

175 with coverage_context(kwargs): 

176 show = ShowInfo(kwargs.pop("show", [])) 

177 pipeline = script.build(**kwargs, show=show) 

178 if show.handled and not show.unhandled: 

179 print( 

180 "No quantum graph generated. The --show option was given and all options were processed.", 

181 file=sys.stderr, 

182 ) 

183 return 

184 if script.qgraph(pipelineObj=pipeline, **kwargs, show=show) is None: 

185 raise click.ClickException("QuantumGraph was empty; CRITICAL logs above should provide details.") 

186 _unhandledShow(show, "qgraph") 

187 

188 

189@click.command(cls=PipetaskCommand, epilog=epilog) 

190@ctrlMpExecOpts.run_options() 

191@catch_and_exit 

192def run(ctx: click.Context, **kwargs: Any) -> None: 

193 """Build and execute pipeline and quantum graph.""" 

194 kwargs = _collectActions(ctx, **kwargs) 

195 with coverage_context(kwargs): 

196 show = ShowInfo(kwargs.pop("show", [])) 

197 pipeline = script.build(**kwargs, show=show) 

198 if show.handled and not show.unhandled: 

199 print( 

200 "No quantum graph generated or pipeline executed. " 

201 "The --show option was given and all options were processed.", 

202 file=sys.stderr, 

203 ) 

204 return 

205 if (qgraph := script.qgraph(pipelineObj=pipeline, **kwargs, show=show)) is None: 

206 raise click.ClickException("QuantumGraph was empty; CRITICAL logs above should provide details.") 

207 _unhandledShow(show, "run") 

208 if show.handled: 

209 print( 

210 "No pipeline executed. The --show option was given and all options were processed.", 

211 file=sys.stderr, 

212 ) 

213 return 

214 script.run(qgraphObj=qgraph, **kwargs) 

215 

216 

217@click.command(cls=PipetaskCommand) 

218@ctrlMpExecOpts.butler_config_option() 

219@ctrlMpExecOpts.collection_argument() 

220@confirm_option() 

221@ctrlMpExecOpts.recursive_option( 

222 help="""If the parent CHAINED collection has child CHAINED collections, 

223 search the children until nested chains that start with the parent's name 

224 are removed.""" 

225) 

226def purge(confirm: bool, **kwargs: Any) -> None: 

227 """Remove a CHAINED collection and its contained collections. 

228 

229 COLLECTION is the name of the chained collection to purge. it must not be a 

230 child of any other CHAINED collections 

231 

232 Child collections must be members of exactly one collection. 

233 

234 The collections that will be removed will be printed, there will be an 

235 option to continue or abort (unless using --no-confirm). 

236 """ 

237 confirmable.confirm(partial(script.purge, **kwargs), confirm) 

238 

239 

240@click.command(cls=PipetaskCommand) 

241@ctrlMpExecOpts.butler_config_option() 

242@ctrlMpExecOpts.collection_argument() 

243@confirm_option() 

244def cleanup(confirm: bool, **kwargs: Any) -> None: 

245 """Remove non-members of CHAINED collections. 

246 

247 Removes collections that start with the same name as a CHAINED 

248 collection but are not members of that collection. 

249 """ 

250 confirmable.confirm(partial(script.cleanup, **kwargs), confirm) 

251 

252 

253@click.command(cls=PipetaskCommand) 

254@repo_argument() 

255@ctrlMpExecOpts.qgraph_argument() 

256@ctrlMpExecOpts.config_search_path_option() 

257@ctrlMpExecOpts.qgraph_id_option() 

258@ctrlMpExecOpts.coverage_options() 

259def pre_exec_init_qbb(repo: str, qgraph: str, **kwargs: Any) -> None: 

260 """Execute pre-exec-init on Quantum-Backed Butler. 

261 

262 REPO is the location of the butler/registry config file. 

263 

264 QGRAPH is the path to a serialized Quantum Graph file. 

265 """ 

266 with coverage_context(kwargs): 

267 script.pre_exec_init_qbb(repo, qgraph, **kwargs) 

268 

269 

270@click.command(cls=PipetaskCommand) 

271@repo_argument() 

272@ctrlMpExecOpts.qgraph_argument() 

273@ctrlMpExecOpts.config_search_path_option() 

274@ctrlMpExecOpts.qgraph_id_option() 

275@ctrlMpExecOpts.qgraph_node_id_option() 

276@processes_option() 

277@ctrlMpExecOpts.pdb_option() 

278@ctrlMpExecOpts.profile_option() 

279@ctrlMpExecOpts.coverage_options() 

280@ctrlMpExecOpts.debug_option() 

281@ctrlMpExecOpts.start_method_option() 

282@ctrlMpExecOpts.timeout_option() 

283@ctrlMpExecOpts.fail_fast_option() 

284@ctrlMpExecOpts.summary_option() 

285@ctrlMpExecOpts.enable_implicit_threading_option() 

286@ctrlMpExecOpts.cores_per_quantum_option() 

287@ctrlMpExecOpts.memory_per_quantum_option() 

288def run_qbb(repo: str, qgraph: str, **kwargs: Any) -> None: 

289 """Execute pipeline using Quantum-Backed Butler. 

290 

291 REPO is the location of the butler/registry config file. 

292 

293 QGRAPH is the path to a serialized Quantum Graph file. 

294 """ 

295 with coverage_context(kwargs): 

296 script.run_qbb(repo, qgraph, **kwargs) 

297 

298 

299@click.command(cls=PipetaskCommand) 

300@ctrlMpExecOpts.qgraph_argument() 

301@ctrlMpExecOpts.run_argument() 

302@ctrlMpExecOpts.output_qgraph_argument() 

303@ctrlMpExecOpts.metadata_run_key_option() 

304@ctrlMpExecOpts.update_graph_id_option() 

305def update_graph_run( 

306 qgraph: str, 

307 run: str, 

308 output_qgraph: str, 

309 metadata_run_key: str, 

310 update_graph_id: bool, 

311) -> None: 

312 """Update existing quantum graph with new output run name and re-generate 

313 output dataset IDs. 

314 

315 QGRAPH is the URL to a serialized Quantum Graph file. 

316 

317 RUN is the new RUN collection name for output graph. 

318 

319 OUTPUT_QGRAPH is the URL to store the updated Quantum Graph. 

320 """ 

321 script.update_graph_run(qgraph, run, output_qgraph, metadata_run_key, update_graph_id)