Coverage for python/lsst/ctrl/mpexec/cli/cmd/commands.py: 82%

169 statements  

« prev     ^ index     » next       coverage.py v7.5.1, created at 2024-05-11 03:35 -0700

1# This file is part of ctrl_mpexec. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This software is dual licensed under the GNU General Public License and also 

10# under a 3-clause BSD license. Recipients may choose which of these licenses 

11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, 

12# respectively. If you choose the GPL option then the following text applies 

13# (but note that there is still no warranty even if you opt for BSD instead): 

14# 

15# This program is free software: you can redistribute it and/or modify 

16# it under the terms of the GNU General Public License as published by 

17# the Free Software Foundation, either version 3 of the License, or 

18# (at your option) any later version. 

19# 

20# This program is distributed in the hope that it will be useful, 

21# but WITHOUT ANY WARRANTY; without even the implied warranty of 

22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

23# GNU General Public License for more details. 

24# 

25# You should have received a copy of the GNU General Public License 

26# along with this program. If not, see <http://www.gnu.org/licenses/>. 

27 

28import sys 

29from collections.abc import Iterator 

30from contextlib import contextmanager 

31from functools import partial 

32from tempfile import NamedTemporaryFile 

33from typing import Any 

34 

35import click 

36import coverage 

37import lsst.pipe.base.cli.opt as pipeBaseOpts 

38from lsst.ctrl.mpexec import Report 

39from lsst.ctrl.mpexec.showInfo import ShowInfo 

40from lsst.daf.butler.cli.opt import ( 

41 config_file_option, 

42 config_option, 

43 confirm_option, 

44 options_file_option, 

45 processes_option, 

46 repo_argument, 

47) 

48from lsst.daf.butler.cli.utils import MWCtxObj, catch_and_exit, option_section, unwrap 

49 

50from .. import opt as ctrlMpExecOpts 

51from .. import script 

52from ..script import confirmable 

53from ..utils import PipetaskCommand, makePipelineActions 

54 

55epilog = unwrap( 

56 """Notes: 

57 

58--task, --delete, --config, --config-file, and --instrument action options can 

59appear multiple times; all values are used, in order left to right. 

60 

61FILE reads command-line options from the specified file. Data may be 

62distributed among multiple lines (e.g. one option per line). Data after # is 

63treated as a comment and ignored. Blank lines and lines starting with # are 

64ignored.) 

65""" 

66) 

67 

68 

69def _collectActions(ctx: click.Context, **kwargs: Any) -> dict[str, Any]: 

70 """Extract pipeline building options, replace them with PipelineActions, 

71 return updated `kwargs`. 

72 

73 Notes 

74 ----- 

75 The pipeline actions (task, delete, config, config_file, and instrument) 

76 must be handled in the order they appear on the command line, but the CLI 

77 specification gives them all different option names. So, instead of using 

78 the individual action options as they appear in kwargs (because 

79 invocation order can't be known), we capture the CLI arguments by 

80 overriding `click.Command.parse_args` and save them in the Context's 

81 `obj` parameter. We use `makePipelineActions` to create a list of 

82 pipeline actions from the CLI arguments and pass that list to the script 

83 function using the `pipeline_actions` kwarg name, and remove the action 

84 options from kwargs. 

85 """ 

86 for pipelineAction in ( 

87 ctrlMpExecOpts.task_option.name(), 

88 ctrlMpExecOpts.delete_option.name(), 

89 config_option.name(), 

90 config_file_option.name(), 

91 pipeBaseOpts.instrument_option.name(), 

92 ): 

93 kwargs.pop(pipelineAction) 

94 

95 actions = makePipelineActions(MWCtxObj.getFrom(ctx).args) 

96 pipeline_actions = [] 

97 for action in actions: 

98 pipeline_actions.append(action) 

99 

100 kwargs["pipeline_actions"] = pipeline_actions 

101 return kwargs 

102 

103 

104def _unhandledShow(show: ShowInfo, cmd: str) -> None: 

105 if show.unhandled: 105 ↛ 106line 105 didn't jump to line 106, because the condition on line 105 was never true

106 print( 

107 f"The following '--show' options were not known to the {cmd} command: " 

108 f"{', '.join(show.unhandled)}", 

109 file=sys.stderr, 

110 ) 

111 

112 

113@click.command(cls=PipetaskCommand, epilog=epilog, short_help="Build pipeline definition.") 

114@click.pass_context 

115@ctrlMpExecOpts.show_option() 

116@ctrlMpExecOpts.pipeline_build_options() 

117@option_section(sectionText="") 

118@options_file_option() 

119@catch_and_exit 

120def build(ctx: click.Context, **kwargs: Any) -> None: 

121 """Build and optionally save pipeline definition. 

122 

123 This does not require input data to be specified. 

124 """ 

125 kwargs = _collectActions(ctx, **kwargs) 

126 show = ShowInfo(kwargs.pop("show", [])) 

127 if kwargs.get("butler_config") is not None and {"pipeline-graph", "task-graph"}.isdisjoint(show.commands): 127 ↛ 128line 127 didn't jump to line 128, because the condition on line 127 was never true

128 raise click.ClickException( 

129 "--butler-config was provided but nothing uses it " 

130 "(only --show pipeline-graph and --show task-graph do)." 

131 ) 

132 script.build(**kwargs, show=show) 

133 _unhandledShow(show, "build") 

134 

135 

136@contextmanager 

137def coverage_context(kwargs: dict[str, Any]) -> Iterator[None]: 

138 """Enable coverage recording.""" 

139 packages = kwargs.pop("cov_packages", ()) 

140 report = kwargs.pop("cov_report", True) 

141 if not kwargs.pop("coverage", False): 141 ↛ 144line 141 didn't jump to line 144, because the condition on line 141 was always true

142 yield 

143 return 

144 with NamedTemporaryFile("w") as rcfile: 

145 rcfile.write( 

146 """ 

147[run] 

148branch = True 

149concurrency = multiprocessing 

150""" 

151 ) 

152 if packages: 

153 packages_str = ",".join(packages) 

154 rcfile.write(f"source_pkgs = {packages_str}\n") 

155 rcfile.flush() 

156 cov = coverage.Coverage(config_file=rcfile.name) 

157 cov.start() 

158 try: 

159 yield 

160 finally: 

161 cov.stop() 

162 cov.save() 

163 if report: 

164 outdir = "./covhtml" 

165 cov.html_report(directory=outdir) 

166 click.echo(f"Coverage report written to {outdir}.") 

167 

168 

169@click.command(cls=PipetaskCommand, epilog=epilog) 

170@click.pass_context 

171@ctrlMpExecOpts.show_option() 

172@ctrlMpExecOpts.pipeline_build_options() 

173@ctrlMpExecOpts.qgraph_options() 

174@ctrlMpExecOpts.butler_options() 

175@option_section(sectionText="") 

176@options_file_option() 

177@catch_and_exit 

178def qgraph(ctx: click.Context, **kwargs: Any) -> None: 

179 """Build and optionally save quantum graph.""" 

180 kwargs = _collectActions(ctx, **kwargs) 

181 summary = kwargs.pop("summary", None) 

182 with coverage_context(kwargs): 

183 show = ShowInfo(kwargs.pop("show", [])) 

184 pipeline = script.build(**kwargs, show=show) 

185 if show.handled and not show.unhandled: 185 ↛ 186line 185 didn't jump to line 186, because the condition on line 185 was never true

186 print( 

187 "No quantum graph generated. The --show option was given and all options were processed.", 

188 file=sys.stderr, 

189 ) 

190 return 

191 if (qgraph := script.qgraph(pipelineObj=pipeline, **kwargs, show=show)) is None: 191 ↛ 192line 191 didn't jump to line 192, because the condition on line 191 was never true

192 raise click.ClickException("QuantumGraph was empty; CRITICAL logs above should provide details.") 

193 # QuantumGraph-only summary call here since script.qgraph also called 

194 # by run methods. 

195 if summary: 195 ↛ 201line 195 didn't jump to line 201, because the condition on line 195 was always true

196 report = Report(qgraphSummary=qgraph.getSummary()) 

197 with open(summary, "w") as out: 

198 # Do not save fields that are not set. 

199 out.write(report.model_dump_json(exclude_none=True, indent=2)) 

200 

201 _unhandledShow(show, "qgraph") 

202 

203 

204@click.command(cls=PipetaskCommand, epilog=epilog) 

205@ctrlMpExecOpts.run_options() 

206@catch_and_exit 

207def run(ctx: click.Context, **kwargs: Any) -> None: 

208 """Build and execute pipeline and quantum graph.""" 

209 kwargs = _collectActions(ctx, **kwargs) 

210 with coverage_context(kwargs): 

211 show = ShowInfo(kwargs.pop("show", [])) 

212 pipeline = script.build(**kwargs, show=show) 

213 if show.handled and not show.unhandled: 

214 print( 

215 "No quantum graph generated or pipeline executed. " 

216 "The --show option was given and all options were processed.", 

217 file=sys.stderr, 

218 ) 

219 return 

220 if (qgraph := script.qgraph(pipelineObj=pipeline, **kwargs, show=show)) is None: 

221 raise click.ClickException("QuantumGraph was empty; CRITICAL logs above should provide details.") 

222 _unhandledShow(show, "run") 

223 if show.handled: 

224 print( 

225 "No pipeline executed. The --show option was given and all options were processed.", 

226 file=sys.stderr, 

227 ) 

228 return 

229 script.run(qgraphObj=qgraph, **kwargs) 

230 

231 

232@click.command(cls=PipetaskCommand) 

233@ctrlMpExecOpts.butler_config_option() 

234@ctrlMpExecOpts.collection_argument() 

235@confirm_option() 

236@ctrlMpExecOpts.recursive_option( 

237 help="""If the parent CHAINED collection has child CHAINED collections, 

238 search the children until nested chains that start with the parent's name 

239 are removed.""" 

240) 

241def purge(confirm: bool, **kwargs: Any) -> None: 

242 """Remove a CHAINED collection and its contained collections. 

243 

244 COLLECTION is the name of the chained collection to purge. it must not be a 

245 child of any other CHAINED collections 

246 

247 Child collections must be members of exactly one collection. 

248 

249 The collections that will be removed will be printed, there will be an 

250 option to continue or abort (unless using --no-confirm). 

251 """ 

252 confirmable.confirm(partial(script.purge, **kwargs), confirm) 

253 

254 

255@click.command(cls=PipetaskCommand) 

256@ctrlMpExecOpts.butler_config_option() 

257@ctrlMpExecOpts.collection_argument() 

258@confirm_option() 

259def cleanup(confirm: bool, **kwargs: Any) -> None: 

260 """Remove non-members of CHAINED collections. 

261 

262 Removes collections that start with the same name as a CHAINED 

263 collection but are not members of that collection. 

264 """ 

265 confirmable.confirm(partial(script.cleanup, **kwargs), confirm) 

266 

267 

268@click.command(cls=PipetaskCommand) 

269@repo_argument() 

270@ctrlMpExecOpts.qgraph_argument() 

271@ctrlMpExecOpts.config_search_path_option() 

272@ctrlMpExecOpts.qgraph_id_option() 

273@ctrlMpExecOpts.coverage_options() 

274def pre_exec_init_qbb(repo: str, qgraph: str, **kwargs: Any) -> None: 

275 """Execute pre-exec-init on Quantum-Backed Butler. 

276 

277 REPO is the location of the butler/registry config file. 

278 

279 QGRAPH is the path to a serialized Quantum Graph file. 

280 """ 

281 with coverage_context(kwargs): 

282 script.pre_exec_init_qbb(repo, qgraph, **kwargs) 

283 

284 

285@click.command(cls=PipetaskCommand) 

286@repo_argument() 

287@ctrlMpExecOpts.qgraph_argument() 

288@ctrlMpExecOpts.config_search_path_option() 

289@ctrlMpExecOpts.qgraph_id_option() 

290@ctrlMpExecOpts.qgraph_node_id_option() 

291@processes_option() 

292@ctrlMpExecOpts.pdb_option() 

293@ctrlMpExecOpts.profile_option() 

294@ctrlMpExecOpts.coverage_options() 

295@ctrlMpExecOpts.debug_option() 

296@ctrlMpExecOpts.start_method_option() 

297@ctrlMpExecOpts.timeout_option() 

298@ctrlMpExecOpts.fail_fast_option() 

299@ctrlMpExecOpts.summary_option() 

300@ctrlMpExecOpts.enable_implicit_threading_option() 

301@ctrlMpExecOpts.cores_per_quantum_option() 

302@ctrlMpExecOpts.memory_per_quantum_option() 

303def run_qbb(repo: str, qgraph: str, **kwargs: Any) -> None: 

304 """Execute pipeline using Quantum-Backed Butler. 

305 

306 REPO is the location of the butler/registry config file. 

307 

308 QGRAPH is the path to a serialized Quantum Graph file. 

309 """ 

310 with coverage_context(kwargs): 

311 script.run_qbb(repo, qgraph, **kwargs) 

312 

313 

314@click.command(cls=PipetaskCommand) 

315@ctrlMpExecOpts.qgraph_argument() 

316@ctrlMpExecOpts.run_argument() 

317@ctrlMpExecOpts.output_qgraph_argument() 

318@ctrlMpExecOpts.metadata_run_key_option() 

319@ctrlMpExecOpts.update_graph_id_option() 

320def update_graph_run( 

321 qgraph: str, 

322 run: str, 

323 output_qgraph: str, 

324 metadata_run_key: str, 

325 update_graph_id: bool, 

326) -> None: 

327 """Update existing quantum graph with new output run name and re-generate 

328 output dataset IDs. 

329 

330 QGRAPH is the URL to a serialized Quantum Graph file. 

331 

332 RUN is the new RUN collection name for output graph. 

333 

334 OUTPUT_QGRAPH is the URL to store the updated Quantum Graph. 

335 """ 

336 script.update_graph_run(qgraph, run, output_qgraph, metadata_run_key, update_graph_id) 

337 

338 

339@click.command(cls=PipetaskCommand) 

340@repo_argument() 

341@ctrlMpExecOpts.qgraph_argument() 

342@click.option("--full-output-filename", default="", help="Summarize report in a yaml file") 

343@click.option("--logs/--no-logs", default=True, help="Get butler log datasets for extra information.") 

344@click.option( 

345 "--show-errors", 

346 is_flag=True, 

347 default=False, 

348 help="Pretty-print a dict of errors from failed" 

349 " quanta to the screen. Note: the default is to output a yaml file with error information" 

350 " (data_ids and associated messages) to the current working directory instead.", 

351) 

352def report( 

353 repo: str, qgraph: str, full_output_filename: str = "", logs: bool = True, show_errors: bool = False 

354) -> None: 

355 """Write a yaml file summarizing the produced and missing expected datasets 

356 in a quantum graph. 

357 

358 REPO is the location of the butler/registry config file. 

359 

360 QGRAPH is the URL to a serialized Quantum Graph file. 

361 """ 

362 script.report(repo, qgraph, full_output_filename, logs, show_errors)