Coverage for python/lsst/ctrl/mpexec/cli/cmd/commands.py: 53%

170 statements  

« prev     ^ index     » next       coverage.py v7.2.6, created at 2023-05-26 02:14 -0700

1# This file is part of ctrl_mpexec. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22import sys 

23import tempfile 

24from functools import partial 

25from typing import Any 

26 

27import click 

28import coverage 

29import lsst.pipe.base.cli.opt as pipeBaseOpts 

30from lsst.ctrl.mpexec.showInfo import ShowInfo 

31from lsst.daf.butler.cli.opt import ( 

32 config_file_option, 

33 config_option, 

34 confirm_option, 

35 options_file_option, 

36 processes_option, 

37 repo_argument, 

38) 

39from lsst.daf.butler.cli.utils import MWCtxObj, catch_and_exit, option_section, unwrap 

40 

41from .. import opt as ctrlMpExecOpts 

42from .. import script 

43from ..script import confirmable 

44from ..utils import _ACTION_CONFIG, _ACTION_CONFIG_FILE, PipetaskCommand, makePipelineActions 

45 

46epilog = unwrap( 

47 """Notes: 

48 

49--task, --delete, --config, --config-file, and --instrument action options can 

50appear multiple times; all values are used, in order left to right. 

51 

52FILE reads command-line options from the specified file. Data may be 

53distributed among multiple lines (e.g. one option per line). Data after # is 

54treated as a comment and ignored. Blank lines and lines starting with # are 

55ignored.) 

56""" 

57) 

58 

59 

60def _collectActions(ctx: click.Context, **kwargs: Any) -> dict[str, Any]: 

61 """Extract pipeline building options, replace them with PipelineActions, 

62 return updated `kwargs`. 

63 

64 Notes 

65 ----- 

66 The pipeline actions (task, delete, config, config_file, and instrument) 

67 must be handled in the order they appear on the command line, but the CLI 

68 specification gives them all different option names. So, instead of using 

69 the individual action options as they appear in kwargs (because 

70 invocation order can't be known), we capture the CLI arguments by 

71 overriding `click.Command.parse_args` and save them in the Context's 

72 `obj` parameter. We use `makePipelineActions` to create a list of 

73 pipeline actions from the CLI arguments and pass that list to the script 

74 function using the `pipeline_actions` kwarg name, and remove the action 

75 options from kwargs. 

76 """ 

77 for pipelineAction in ( 

78 ctrlMpExecOpts.task_option.name(), 

79 ctrlMpExecOpts.delete_option.name(), 

80 config_option.name(), 

81 config_file_option.name(), 

82 pipeBaseOpts.instrument_option.name(), 

83 ): 

84 kwargs.pop(pipelineAction) 

85 

86 actions = makePipelineActions(MWCtxObj.getFrom(ctx).args) 

87 mock_configs = [] 

88 pipeline_actions = [] 

89 for action in actions: 

90 if action.label and action.label.endswith("-mock"): 90 ↛ 91line 90 didn't jump to line 91, because the condition on line 90 was never true

91 if action.action not in (_ACTION_CONFIG.action, _ACTION_CONFIG_FILE.action): 

92 raise ValueError(f"Unexpected option for mock task config overrides: {action}") 

93 mock_configs.append(action) 

94 else: 

95 pipeline_actions.append(action) 

96 

97 kwargs["mock_configs"] = mock_configs 

98 kwargs["pipeline_actions"] = pipeline_actions 

99 return kwargs 

100 

101 

102def _unhandledShow(show: ShowInfo, cmd: str) -> None: 

103 if show.unhandled: 103 ↛ 104line 103 didn't jump to line 104, because the condition on line 103 was never true

104 print( 

105 f"The following '--show' options were not known to the {cmd} command: " 

106 f"{', '.join(show.unhandled)}", 

107 file=sys.stderr, 

108 ) 

109 

110 

111@click.command(cls=PipetaskCommand, epilog=epilog, short_help="Build pipeline definition.") 

112@click.pass_context 

113@ctrlMpExecOpts.show_option() 

114@ctrlMpExecOpts.pipeline_build_options() 

115@option_section(sectionText="") 

116@options_file_option() 

117@catch_and_exit 

118def build(ctx: click.Context, **kwargs: Any) -> None: 

119 """Build and optionally save pipeline definition. 

120 

121 This does not require input data to be specified. 

122 """ 

123 kwargs = _collectActions(ctx, **kwargs) 

124 show = ShowInfo(kwargs.pop("show", [])) 

125 script.build(**kwargs, show=show) 

126 _unhandledShow(show, "build") 

127 

128 

129def _start_coverage(coverage_packages: tuple) -> coverage.Coverage: 

130 coveragerc = """ 

131[html] 

132directory = covhtml 

133 

134[run] 

135branch = True 

136concurrency = multiprocessing 

137""" 

138 

139 if coverage_packages: 

140 pkgs = ",".join(coverage_packages) 

141 click.echo(f"Coverage enabled of packages: {pkgs}") 

142 coveragerc += f"source_pkgs={pkgs}" 

143 else: 

144 click.echo("Coverage enabled") 

145 

146 with tempfile.NamedTemporaryFile(mode="w") as cov_file: 

147 cov_file.write(coveragerc) 

148 cov_file.flush() 

149 cov = coverage.Coverage(config_file=cov_file.name) 

150 

151 cov.start() 

152 return cov 

153 

154 

155def _stop_coverage(cov: coverage.Coverage) -> None: 

156 cov.stop() 

157 outdir = "./covhtml" 

158 cov.html_report(directory=outdir) 

159 cov.report() 

160 click.echo(f"Coverage data written to {outdir}") 

161 

162 

163@click.command(cls=PipetaskCommand, epilog=epilog) 

164@click.pass_context 

165@ctrlMpExecOpts.coverage_option() 

166@ctrlMpExecOpts.coverage_packages_option() 

167@ctrlMpExecOpts.show_option() 

168@ctrlMpExecOpts.pipeline_build_options() 

169@ctrlMpExecOpts.qgraph_options() 

170@ctrlMpExecOpts.butler_options() 

171@option_section(sectionText="") 

172@options_file_option() 

173@catch_and_exit 

174def qgraph(ctx: click.Context, **kwargs: Any) -> None: 

175 """Build and optionally save quantum graph.""" 

176 kwargs = _collectActions(ctx, **kwargs) 

177 coverage = kwargs.pop("coverage", False) 

178 coverage_packages = kwargs.pop("cov_packages", ()) 

179 if coverage: 

180 cov = _start_coverage(coverage_packages) 

181 

182 try: 

183 show = ShowInfo(kwargs.pop("show", [])) 

184 pipeline = script.build(**kwargs, show=show) 

185 if show.handled and not show.unhandled: 

186 print( 

187 "No quantum graph generated. The --show option was given and all options were processed.", 

188 file=sys.stderr, 

189 ) 

190 return 

191 if script.qgraph(pipelineObj=pipeline, **kwargs, show=show) is None: 

192 raise click.ClickException("QuantumGraph was empty; CRITICAL logs above should provide details.") 

193 _unhandledShow(show, "qgraph") 

194 finally: 

195 if coverage: 

196 _stop_coverage(cov) 

197 

198 

199@click.command(cls=PipetaskCommand, epilog=epilog) 

200@ctrlMpExecOpts.run_options() 

201@catch_and_exit 

202def run(ctx: click.Context, **kwargs: Any) -> None: 

203 """Build and execute pipeline and quantum graph.""" 

204 kwargs = _collectActions(ctx, **kwargs) 

205 coverage = kwargs.pop("coverage", False) 

206 if coverage: 

207 coverage_packages = kwargs.pop("cov_packages", ()) 

208 cov = _start_coverage(coverage_packages) 

209 

210 try: 

211 show = ShowInfo(kwargs.pop("show", [])) 

212 pipeline = script.build(**kwargs, show=show) 

213 if show.handled and not show.unhandled: 

214 print( 

215 "No quantum graph generated or pipeline executed. " 

216 "The --show option was given and all options were processed.", 

217 file=sys.stderr, 

218 ) 

219 return 

220 if (qgraph := script.qgraph(pipelineObj=pipeline, **kwargs, show=show)) is None: 

221 raise click.ClickException("QuantumGraph was empty; CRITICAL logs above should provide details.") 

222 _unhandledShow(show, "run") 

223 if show.handled: 

224 print( 

225 "No pipeline executed. The --show option was given and all options were processed.", 

226 file=sys.stderr, 

227 ) 

228 return 

229 script.run(qgraphObj=qgraph, **kwargs) 

230 finally: 

231 if coverage: 

232 _stop_coverage(cov) 

233 

234 

235@click.command(cls=PipetaskCommand) 

236@ctrlMpExecOpts.butler_config_option() 

237@ctrlMpExecOpts.collection_argument() 

238@confirm_option() 

239@ctrlMpExecOpts.recursive_option( 

240 help="""If the parent CHAINED collection has child CHAINED collections, 

241 search the children until nested chains that start with the parent's name 

242 are removed.""" 

243) 

244def purge(confirm: bool, **kwargs: Any) -> None: 

245 """Remove a CHAINED collection and its contained collections. 

246 

247 COLLECTION is the name of the chained collection to purge. it must not be a 

248 child of any other CHAINED collections 

249 

250 Child collections must be members of exactly one collection. 

251 

252 The collections that will be removed will be printed, there will be an 

253 option to continue or abort (unless using --no-confirm). 

254 """ 

255 confirmable.confirm(partial(script.purge, **kwargs), confirm) 

256 

257 

258@click.command(cls=PipetaskCommand) 

259@ctrlMpExecOpts.butler_config_option() 

260@ctrlMpExecOpts.collection_argument() 

261@confirm_option() 

262def cleanup(confirm: bool, **kwargs: Any) -> None: 

263 """Remove non-members of CHAINED collections. 

264 

265 Removes collections that start with the same name as a CHAINED 

266 collection but are not members of that collection. 

267 """ 

268 confirmable.confirm(partial(script.cleanup, **kwargs), confirm) 

269 

270 

271@click.command(cls=PipetaskCommand) 

272@repo_argument() 

273@ctrlMpExecOpts.qgraph_argument() 

274@ctrlMpExecOpts.config_search_path_option() 

275@ctrlMpExecOpts.qgraph_id_option() 

276def pre_exec_init_qbb(repo: str, qgraph: str, **kwargs: Any) -> None: 

277 """Execute pre-exec-init on Quantum-Backed Butler. 

278 

279 REPO is the location of the butler/registry config file. 

280 

281 QGRAPH is the path to a serialized Quantum Graph file. 

282 """ 

283 script.pre_exec_init_qbb(repo, qgraph, **kwargs) 

284 

285 

286@click.command(cls=PipetaskCommand) 

287@repo_argument() 

288@ctrlMpExecOpts.qgraph_argument() 

289@ctrlMpExecOpts.config_search_path_option() 

290@ctrlMpExecOpts.qgraph_id_option() 

291@ctrlMpExecOpts.qgraph_node_id_option() 

292@processes_option() 

293@ctrlMpExecOpts.pdb_option() 

294@ctrlMpExecOpts.profile_option() 

295@ctrlMpExecOpts.coverage_option() 

296@ctrlMpExecOpts.coverage_packages_option() 

297@ctrlMpExecOpts.debug_option() 

298@ctrlMpExecOpts.start_method_option() 

299@ctrlMpExecOpts.timeout_option() 

300@ctrlMpExecOpts.fail_fast_option() 

301@ctrlMpExecOpts.summary_option() 

302@ctrlMpExecOpts.enable_implicit_threading_option() 

303def run_qbb(repo: str, qgraph: str, **kwargs: Any) -> None: 

304 """Execute pipeline using Quantum-Backed Butler. 

305 

306 REPO is the location of the butler/registry config file. 

307 

308 QGRAPH is the path to a serialized Quantum Graph file. 

309 """ 

310 coverage = kwargs.pop("coverage", False) 

311 coverage_packages = kwargs.pop("cov_packages", ()) 

312 if coverage: 

313 cov = _start_coverage(coverage_packages) 

314 

315 try: 

316 script.run_qbb(repo, qgraph, **kwargs) 

317 finally: 

318 if coverage: 

319 _stop_coverage(cov) 

320 

321 

322@click.command(cls=PipetaskCommand) 

323@ctrlMpExecOpts.qgraph_argument() 

324@ctrlMpExecOpts.run_argument() 

325@ctrlMpExecOpts.output_qgraph_argument() 

326@ctrlMpExecOpts.metadata_run_key_option() 

327@ctrlMpExecOpts.update_graph_id_option() 

328def update_graph_run( 

329 qgraph: str, 

330 run: str, 

331 output_qgraph: str, 

332 metadata_run_key: str, 

333 update_graph_id: bool, 

334) -> None: 

335 """Update existing quantum graph with new output run name and re-generate 

336 output dataset IDs. 

337 

338 QGRAPH is the URL to a serialized Quantum Graph file. 

339 

340 RUN is the new RUN collection name for output graph. 

341 

342 OUTPUT_QGRAPH is the URL to store the updated Quantum Graph. 

343 """ 

344 script.update_graph_run(qgraph, run, output_qgraph, metadata_run_key, update_graph_id)