Coverage for python/lsst/ctrl/mpexec/cli/cmd/commands.py: 77%

163 statements  

« prev     ^ index     » next       coverage.py v7.4.4, created at 2024-04-05 02:56 -0700

1# This file is part of ctrl_mpexec. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This software is dual licensed under the GNU General Public License and also 

10# under a 3-clause BSD license. Recipients may choose which of these licenses 

11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, 

12# respectively. If you choose the GPL option then the following text applies 

13# (but note that there is still no warranty even if you opt for BSD instead): 

14# 

15# This program is free software: you can redistribute it and/or modify 

16# it under the terms of the GNU General Public License as published by 

17# the Free Software Foundation, either version 3 of the License, or 

18# (at your option) any later version. 

19# 

20# This program is distributed in the hope that it will be useful, 

21# but WITHOUT ANY WARRANTY; without even the implied warranty of 

22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

23# GNU General Public License for more details. 

24# 

25# You should have received a copy of the GNU General Public License 

26# along with this program. If not, see <http://www.gnu.org/licenses/>. 

27 

28import sys 

29from collections.abc import Iterator 

30from contextlib import contextmanager 

31from functools import partial 

32from tempfile import NamedTemporaryFile 

33from typing import Any 

34 

35import click 

36import coverage 

37import lsst.pipe.base.cli.opt as pipeBaseOpts 

38from lsst.ctrl.mpexec.showInfo import ShowInfo 

39from lsst.daf.butler.cli.opt import ( 

40 config_file_option, 

41 config_option, 

42 confirm_option, 

43 options_file_option, 

44 processes_option, 

45 repo_argument, 

46) 

47from lsst.daf.butler.cli.utils import MWCtxObj, catch_and_exit, option_section, unwrap 

48 

49from .. import opt as ctrlMpExecOpts 

50from .. import script 

51from ..script import confirmable 

52from ..utils import PipetaskCommand, makePipelineActions 

53 

54epilog = unwrap( 

55 """Notes: 

56 

57--task, --delete, --config, --config-file, and --instrument action options can 

58appear multiple times; all values are used, in order left to right. 

59 

60FILE reads command-line options from the specified file. Data may be 

61distributed among multiple lines (e.g. one option per line). Data after # is 

62treated as a comment and ignored. Blank lines and lines starting with # are 

63ignored.) 

64""" 

65) 

66 

67 

68def _collectActions(ctx: click.Context, **kwargs: Any) -> dict[str, Any]: 

69 """Extract pipeline building options, replace them with PipelineActions, 

70 return updated `kwargs`. 

71 

72 Notes 

73 ----- 

74 The pipeline actions (task, delete, config, config_file, and instrument) 

75 must be handled in the order they appear on the command line, but the CLI 

76 specification gives them all different option names. So, instead of using 

77 the individual action options as they appear in kwargs (because 

78 invocation order can't be known), we capture the CLI arguments by 

79 overriding `click.Command.parse_args` and save them in the Context's 

80 `obj` parameter. We use `makePipelineActions` to create a list of 

81 pipeline actions from the CLI arguments and pass that list to the script 

82 function using the `pipeline_actions` kwarg name, and remove the action 

83 options from kwargs. 

84 """ 

85 for pipelineAction in ( 

86 ctrlMpExecOpts.task_option.name(), 

87 ctrlMpExecOpts.delete_option.name(), 

88 config_option.name(), 

89 config_file_option.name(), 

90 pipeBaseOpts.instrument_option.name(), 

91 ): 

92 kwargs.pop(pipelineAction) 

93 

94 actions = makePipelineActions(MWCtxObj.getFrom(ctx).args) 

95 pipeline_actions = [] 

96 for action in actions: 

97 pipeline_actions.append(action) 

98 

99 kwargs["pipeline_actions"] = pipeline_actions 

100 return kwargs 

101 

102 

103def _unhandledShow(show: ShowInfo, cmd: str) -> None: 

104 if show.unhandled: 104 ↛ 105line 104 didn't jump to line 105, because the condition on line 104 was never true

105 print( 

106 f"The following '--show' options were not known to the {cmd} command: " 

107 f"{', '.join(show.unhandled)}", 

108 file=sys.stderr, 

109 ) 

110 

111 

112@click.command(cls=PipetaskCommand, epilog=epilog, short_help="Build pipeline definition.") 

113@click.pass_context 

114@ctrlMpExecOpts.show_option() 

115@ctrlMpExecOpts.pipeline_build_options() 

116@option_section(sectionText="") 

117@options_file_option() 

118@catch_and_exit 

119def build(ctx: click.Context, **kwargs: Any) -> None: 

120 """Build and optionally save pipeline definition. 

121 

122 This does not require input data to be specified. 

123 """ 

124 kwargs = _collectActions(ctx, **kwargs) 

125 show = ShowInfo(kwargs.pop("show", [])) 

126 if kwargs.get("butler_config") is not None and {"pipeline-graph", "task-graph"}.isdisjoint(show.commands): 126 ↛ 127line 126 didn't jump to line 127, because the condition on line 126 was never true

127 raise click.ClickException( 

128 "--butler-config was provided but nothing uses it " 

129 "(only --show pipeline-graph and --show task-graph do)." 

130 ) 

131 script.build(**kwargs, show=show) 

132 _unhandledShow(show, "build") 

133 

134 

135@contextmanager 

136def coverage_context(kwargs: dict[str, Any]) -> Iterator[None]: 

137 """Enable coverage recording.""" 

138 packages = kwargs.pop("cov_packages", ()) 

139 report = kwargs.pop("cov_report", True) 

140 if not kwargs.pop("coverage", False): 

141 yield 

142 return 

143 with NamedTemporaryFile("w") as rcfile: 

144 rcfile.write( 

145 """ 

146[run] 

147branch = True 

148concurrency = multiprocessing 

149""" 

150 ) 

151 if packages: 

152 packages_str = ",".join(packages) 

153 rcfile.write(f"source_pkgs = {packages_str}\n") 

154 rcfile.flush() 

155 cov = coverage.Coverage(config_file=rcfile.name) 

156 cov.start() 

157 try: 

158 yield 

159 finally: 

160 cov.stop() 

161 cov.save() 

162 if report: 

163 outdir = "./covhtml" 

164 cov.html_report(directory=outdir) 

165 click.echo(f"Coverage report written to {outdir}.") 

166 

167 

168@click.command(cls=PipetaskCommand, epilog=epilog) 

169@click.pass_context 

170@ctrlMpExecOpts.show_option() 

171@ctrlMpExecOpts.pipeline_build_options() 

172@ctrlMpExecOpts.qgraph_options() 

173@ctrlMpExecOpts.butler_options() 

174@option_section(sectionText="") 

175@options_file_option() 

176@catch_and_exit 

177def qgraph(ctx: click.Context, **kwargs: Any) -> None: 

178 """Build and optionally save quantum graph.""" 

179 kwargs = _collectActions(ctx, **kwargs) 

180 with coverage_context(kwargs): 

181 show = ShowInfo(kwargs.pop("show", [])) 

182 pipeline = script.build(**kwargs, show=show) 

183 if show.handled and not show.unhandled: 

184 print( 

185 "No quantum graph generated. The --show option was given and all options were processed.", 

186 file=sys.stderr, 

187 ) 

188 return 

189 if script.qgraph(pipelineObj=pipeline, **kwargs, show=show) is None: 

190 raise click.ClickException("QuantumGraph was empty; CRITICAL logs above should provide details.") 

191 _unhandledShow(show, "qgraph") 

192 

193 

194@click.command(cls=PipetaskCommand, epilog=epilog) 

195@ctrlMpExecOpts.run_options() 

196@catch_and_exit 

197def run(ctx: click.Context, **kwargs: Any) -> None: 

198 """Build and execute pipeline and quantum graph.""" 

199 kwargs = _collectActions(ctx, **kwargs) 

200 with coverage_context(kwargs): 

201 show = ShowInfo(kwargs.pop("show", [])) 

202 pipeline = script.build(**kwargs, show=show) 

203 if show.handled and not show.unhandled: 

204 print( 

205 "No quantum graph generated or pipeline executed. " 

206 "The --show option was given and all options were processed.", 

207 file=sys.stderr, 

208 ) 

209 return 

210 if (qgraph := script.qgraph(pipelineObj=pipeline, **kwargs, show=show)) is None: 

211 raise click.ClickException("QuantumGraph was empty; CRITICAL logs above should provide details.") 

212 _unhandledShow(show, "run") 

213 if show.handled: 

214 print( 

215 "No pipeline executed. The --show option was given and all options were processed.", 

216 file=sys.stderr, 

217 ) 

218 return 

219 script.run(qgraphObj=qgraph, **kwargs) 

220 

221 

222@click.command(cls=PipetaskCommand) 

223@ctrlMpExecOpts.butler_config_option() 

224@ctrlMpExecOpts.collection_argument() 

225@confirm_option() 

226@ctrlMpExecOpts.recursive_option( 

227 help="""If the parent CHAINED collection has child CHAINED collections, 

228 search the children until nested chains that start with the parent's name 

229 are removed.""" 

230) 

231def purge(confirm: bool, **kwargs: Any) -> None: 

232 """Remove a CHAINED collection and its contained collections. 

233 

234 COLLECTION is the name of the chained collection to purge. it must not be a 

235 child of any other CHAINED collections 

236 

237 Child collections must be members of exactly one collection. 

238 

239 The collections that will be removed will be printed, there will be an 

240 option to continue or abort (unless using --no-confirm). 

241 """ 

242 confirmable.confirm(partial(script.purge, **kwargs), confirm) 

243 

244 

245@click.command(cls=PipetaskCommand) 

246@ctrlMpExecOpts.butler_config_option() 

247@ctrlMpExecOpts.collection_argument() 

248@confirm_option() 

249def cleanup(confirm: bool, **kwargs: Any) -> None: 

250 """Remove non-members of CHAINED collections. 

251 

252 Removes collections that start with the same name as a CHAINED 

253 collection but are not members of that collection. 

254 """ 

255 confirmable.confirm(partial(script.cleanup, **kwargs), confirm) 

256 

257 

258@click.command(cls=PipetaskCommand) 

259@repo_argument() 

260@ctrlMpExecOpts.qgraph_argument() 

261@ctrlMpExecOpts.config_search_path_option() 

262@ctrlMpExecOpts.qgraph_id_option() 

263@ctrlMpExecOpts.coverage_options() 

264def pre_exec_init_qbb(repo: str, qgraph: str, **kwargs: Any) -> None: 

265 """Execute pre-exec-init on Quantum-Backed Butler. 

266 

267 REPO is the location of the butler/registry config file. 

268 

269 QGRAPH is the path to a serialized Quantum Graph file. 

270 """ 

271 with coverage_context(kwargs): 

272 script.pre_exec_init_qbb(repo, qgraph, **kwargs) 

273 

274 

275@click.command(cls=PipetaskCommand) 

276@repo_argument() 

277@ctrlMpExecOpts.qgraph_argument() 

278@ctrlMpExecOpts.config_search_path_option() 

279@ctrlMpExecOpts.qgraph_id_option() 

280@ctrlMpExecOpts.qgraph_node_id_option() 

281@processes_option() 

282@ctrlMpExecOpts.pdb_option() 

283@ctrlMpExecOpts.profile_option() 

284@ctrlMpExecOpts.coverage_options() 

285@ctrlMpExecOpts.debug_option() 

286@ctrlMpExecOpts.start_method_option() 

287@ctrlMpExecOpts.timeout_option() 

288@ctrlMpExecOpts.fail_fast_option() 

289@ctrlMpExecOpts.summary_option() 

290@ctrlMpExecOpts.enable_implicit_threading_option() 

291@ctrlMpExecOpts.cores_per_quantum_option() 

292@ctrlMpExecOpts.memory_per_quantum_option() 

293def run_qbb(repo: str, qgraph: str, **kwargs: Any) -> None: 

294 """Execute pipeline using Quantum-Backed Butler. 

295 

296 REPO is the location of the butler/registry config file. 

297 

298 QGRAPH is the path to a serialized Quantum Graph file. 

299 """ 

300 with coverage_context(kwargs): 

301 script.run_qbb(repo, qgraph, **kwargs) 

302 

303 

304@click.command(cls=PipetaskCommand) 

305@ctrlMpExecOpts.qgraph_argument() 

306@ctrlMpExecOpts.run_argument() 

307@ctrlMpExecOpts.output_qgraph_argument() 

308@ctrlMpExecOpts.metadata_run_key_option() 

309@ctrlMpExecOpts.update_graph_id_option() 

310def update_graph_run( 

311 qgraph: str, 

312 run: str, 

313 output_qgraph: str, 

314 metadata_run_key: str, 

315 update_graph_id: bool, 

316) -> None: 

317 """Update existing quantum graph with new output run name and re-generate 

318 output dataset IDs. 

319 

320 QGRAPH is the URL to a serialized Quantum Graph file. 

321 

322 RUN is the new RUN collection name for output graph. 

323 

324 OUTPUT_QGRAPH is the URL to store the updated Quantum Graph. 

325 """ 

326 script.update_graph_run(qgraph, run, output_qgraph, metadata_run_key, update_graph_id) 

327 

328 

329@click.command(cls=PipetaskCommand) 

330@repo_argument() 

331@ctrlMpExecOpts.qgraph_argument() 

332@click.option("--full-output-filename", default="", help="Summarize report in a yaml file") 

333@click.option("--logs/--no-logs", default=True, help="Get butler log datasets for extra information.") 

334@click.option( 

335 "--show-errors", 

336 is_flag=True, 

337 default=False, 

338 help="Pretty-print a dict of errors from failed" 

339 " quanta to the screen. Note: the default is to output a yaml file with error information" 

340 " (data_ids and associated messages) to the current working directory instead.", 

341) 

342def report( 

343 repo: str, qgraph: str, full_output_filename: str = "", logs: bool = True, show_errors: bool = False 

344) -> None: 

345 """Write a yaml file summarizing the produced and missing expected datasets 

346 in a quantum graph. 

347 

348 REPO is the location of the butler/registry config file. 

349 

350 QGRAPH is the URL to a serialized Quantum Graph file. 

351 """ 

352 script.report(repo, qgraph, full_output_filename, logs, show_errors)