Coverage for python/lsst/ctrl/mpexec/cli/cmd/commands.py: 76%
153 statements
« prev ^ index » next coverage.py v7.3.1, created at 2023-09-13 09:53 +0000
« prev ^ index » next coverage.py v7.3.1, created at 2023-09-13 09:53 +0000
1# This file is part of ctrl_mpexec.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This software is dual licensed under the GNU General Public License and also
10# under a 3-clause BSD license. Recipients may choose which of these licenses
11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
12# respectively. If you choose the GPL option then the following text applies
13# (but note that there is still no warranty even if you opt for BSD instead):
14#
15# This program is free software: you can redistribute it and/or modify
16# it under the terms of the GNU General Public License as published by
17# the Free Software Foundation, either version 3 of the License, or
18# (at your option) any later version.
19#
20# This program is distributed in the hope that it will be useful,
21# but WITHOUT ANY WARRANTY; without even the implied warranty of
22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
23# GNU General Public License for more details.
24#
25# You should have received a copy of the GNU General Public License
26# along with this program. If not, see <http://www.gnu.org/licenses/>.
28import sys
29from collections.abc import Iterator
30from contextlib import contextmanager
31from functools import partial
32from tempfile import NamedTemporaryFile
33from typing import Any
35import click
36import coverage
37import lsst.pipe.base.cli.opt as pipeBaseOpts
38from lsst.ctrl.mpexec.showInfo import ShowInfo
39from lsst.daf.butler.cli.opt import (
40 config_file_option,
41 config_option,
42 confirm_option,
43 options_file_option,
44 processes_option,
45 repo_argument,
46)
47from lsst.daf.butler.cli.utils import MWCtxObj, catch_and_exit, option_section, unwrap
49from .. import opt as ctrlMpExecOpts
50from .. import script
51from ..script import confirmable
52from ..utils import PipetaskCommand, makePipelineActions
54epilog = unwrap(
55 """Notes:
57--task, --delete, --config, --config-file, and --instrument action options can
58appear multiple times; all values are used, in order left to right.
60FILE reads command-line options from the specified file. Data may be
61distributed among multiple lines (e.g. one option per line). Data after # is
62treated as a comment and ignored. Blank lines and lines starting with # are
63ignored.)
64"""
65)
68def _collectActions(ctx: click.Context, **kwargs: Any) -> dict[str, Any]:
69 """Extract pipeline building options, replace them with PipelineActions,
70 return updated `kwargs`.
72 Notes
73 -----
74 The pipeline actions (task, delete, config, config_file, and instrument)
75 must be handled in the order they appear on the command line, but the CLI
76 specification gives them all different option names. So, instead of using
77 the individual action options as they appear in kwargs (because
78 invocation order can't be known), we capture the CLI arguments by
79 overriding `click.Command.parse_args` and save them in the Context's
80 `obj` parameter. We use `makePipelineActions` to create a list of
81 pipeline actions from the CLI arguments and pass that list to the script
82 function using the `pipeline_actions` kwarg name, and remove the action
83 options from kwargs.
84 """
85 for pipelineAction in (
86 ctrlMpExecOpts.task_option.name(),
87 ctrlMpExecOpts.delete_option.name(),
88 config_option.name(),
89 config_file_option.name(),
90 pipeBaseOpts.instrument_option.name(),
91 ):
92 kwargs.pop(pipelineAction)
94 actions = makePipelineActions(MWCtxObj.getFrom(ctx).args)
95 pipeline_actions = []
96 for action in actions:
97 pipeline_actions.append(action)
99 kwargs["pipeline_actions"] = pipeline_actions
100 return kwargs
103def _unhandledShow(show: ShowInfo, cmd: str) -> None:
104 if show.unhandled: 104 ↛ 105line 104 didn't jump to line 105, because the condition on line 104 was never true
105 print(
106 f"The following '--show' options were not known to the {cmd} command: "
107 f"{', '.join(show.unhandled)}",
108 file=sys.stderr,
109 )
112@click.command(cls=PipetaskCommand, epilog=epilog, short_help="Build pipeline definition.")
113@click.pass_context
114@ctrlMpExecOpts.show_option()
115@ctrlMpExecOpts.pipeline_build_options()
116@option_section(sectionText="")
117@options_file_option()
118@catch_and_exit
119def build(ctx: click.Context, **kwargs: Any) -> None:
120 """Build and optionally save pipeline definition.
122 This does not require input data to be specified.
123 """
124 kwargs = _collectActions(ctx, **kwargs)
125 show = ShowInfo(kwargs.pop("show", []))
126 script.build(**kwargs, show=show)
127 _unhandledShow(show, "build")
130@contextmanager
131def coverage_context(kwargs: dict[str, Any]) -> Iterator[None]:
132 """Enable coverage recording."""
133 packages = kwargs.pop("cov_packages", ())
134 report = kwargs.pop("cov_report", True)
135 if not kwargs.pop("coverage", False):
136 yield
137 return
138 with NamedTemporaryFile("w") as rcfile:
139 rcfile.write(
140 """
141[run]
142branch = True
143concurrency = multiprocessing
144"""
145 )
146 if packages:
147 packages_str = ",".join(packages)
148 rcfile.write(f"source_pkgs = {packages_str}\n")
149 rcfile.flush()
150 cov = coverage.Coverage(config_file=rcfile.name)
151 cov.start()
152 try:
153 yield
154 finally:
155 cov.stop()
156 cov.save()
157 if report:
158 outdir = "./covhtml"
159 cov.html_report(directory=outdir)
160 click.echo(f"Coverage report written to {outdir}.")
163@click.command(cls=PipetaskCommand, epilog=epilog)
164@click.pass_context
165@ctrlMpExecOpts.show_option()
166@ctrlMpExecOpts.pipeline_build_options()
167@ctrlMpExecOpts.qgraph_options()
168@ctrlMpExecOpts.butler_options()
169@option_section(sectionText="")
170@options_file_option()
171@catch_and_exit
172def qgraph(ctx: click.Context, **kwargs: Any) -> None:
173 """Build and optionally save quantum graph."""
174 kwargs = _collectActions(ctx, **kwargs)
175 with coverage_context(kwargs):
176 show = ShowInfo(kwargs.pop("show", []))
177 pipeline = script.build(**kwargs, show=show)
178 if show.handled and not show.unhandled:
179 print(
180 "No quantum graph generated. The --show option was given and all options were processed.",
181 file=sys.stderr,
182 )
183 return
184 if script.qgraph(pipelineObj=pipeline, **kwargs, show=show) is None:
185 raise click.ClickException("QuantumGraph was empty; CRITICAL logs above should provide details.")
186 _unhandledShow(show, "qgraph")
189@click.command(cls=PipetaskCommand, epilog=epilog)
190@ctrlMpExecOpts.run_options()
191@catch_and_exit
192def run(ctx: click.Context, **kwargs: Any) -> None:
193 """Build and execute pipeline and quantum graph."""
194 kwargs = _collectActions(ctx, **kwargs)
195 with coverage_context(kwargs):
196 show = ShowInfo(kwargs.pop("show", []))
197 pipeline = script.build(**kwargs, show=show)
198 if show.handled and not show.unhandled:
199 print(
200 "No quantum graph generated or pipeline executed. "
201 "The --show option was given and all options were processed.",
202 file=sys.stderr,
203 )
204 return
205 if (qgraph := script.qgraph(pipelineObj=pipeline, **kwargs, show=show)) is None:
206 raise click.ClickException("QuantumGraph was empty; CRITICAL logs above should provide details.")
207 _unhandledShow(show, "run")
208 if show.handled:
209 print(
210 "No pipeline executed. The --show option was given and all options were processed.",
211 file=sys.stderr,
212 )
213 return
214 script.run(qgraphObj=qgraph, **kwargs)
217@click.command(cls=PipetaskCommand)
218@ctrlMpExecOpts.butler_config_option()
219@ctrlMpExecOpts.collection_argument()
220@confirm_option()
221@ctrlMpExecOpts.recursive_option(
222 help="""If the parent CHAINED collection has child CHAINED collections,
223 search the children until nested chains that start with the parent's name
224 are removed."""
225)
226def purge(confirm: bool, **kwargs: Any) -> None:
227 """Remove a CHAINED collection and its contained collections.
229 COLLECTION is the name of the chained collection to purge. it must not be a
230 child of any other CHAINED collections
232 Child collections must be members of exactly one collection.
234 The collections that will be removed will be printed, there will be an
235 option to continue or abort (unless using --no-confirm).
236 """
237 confirmable.confirm(partial(script.purge, **kwargs), confirm)
240@click.command(cls=PipetaskCommand)
241@ctrlMpExecOpts.butler_config_option()
242@ctrlMpExecOpts.collection_argument()
243@confirm_option()
244def cleanup(confirm: bool, **kwargs: Any) -> None:
245 """Remove non-members of CHAINED collections.
247 Removes collections that start with the same name as a CHAINED
248 collection but are not members of that collection.
249 """
250 confirmable.confirm(partial(script.cleanup, **kwargs), confirm)
253@click.command(cls=PipetaskCommand)
254@repo_argument()
255@ctrlMpExecOpts.qgraph_argument()
256@ctrlMpExecOpts.config_search_path_option()
257@ctrlMpExecOpts.qgraph_id_option()
258@ctrlMpExecOpts.coverage_options()
259def pre_exec_init_qbb(repo: str, qgraph: str, **kwargs: Any) -> None:
260 """Execute pre-exec-init on Quantum-Backed Butler.
262 REPO is the location of the butler/registry config file.
264 QGRAPH is the path to a serialized Quantum Graph file.
265 """
266 with coverage_context(kwargs):
267 script.pre_exec_init_qbb(repo, qgraph, **kwargs)
270@click.command(cls=PipetaskCommand)
271@repo_argument()
272@ctrlMpExecOpts.qgraph_argument()
273@ctrlMpExecOpts.config_search_path_option()
274@ctrlMpExecOpts.qgraph_id_option()
275@ctrlMpExecOpts.qgraph_node_id_option()
276@processes_option()
277@ctrlMpExecOpts.pdb_option()
278@ctrlMpExecOpts.profile_option()
279@ctrlMpExecOpts.coverage_options()
280@ctrlMpExecOpts.debug_option()
281@ctrlMpExecOpts.start_method_option()
282@ctrlMpExecOpts.timeout_option()
283@ctrlMpExecOpts.fail_fast_option()
284@ctrlMpExecOpts.summary_option()
285@ctrlMpExecOpts.enable_implicit_threading_option()
286@ctrlMpExecOpts.cores_per_quantum_option()
287@ctrlMpExecOpts.memory_per_quantum_option()
288def run_qbb(repo: str, qgraph: str, **kwargs: Any) -> None:
289 """Execute pipeline using Quantum-Backed Butler.
291 REPO is the location of the butler/registry config file.
293 QGRAPH is the path to a serialized Quantum Graph file.
294 """
295 with coverage_context(kwargs):
296 script.run_qbb(repo, qgraph, **kwargs)
299@click.command(cls=PipetaskCommand)
300@ctrlMpExecOpts.qgraph_argument()
301@ctrlMpExecOpts.run_argument()
302@ctrlMpExecOpts.output_qgraph_argument()
303@ctrlMpExecOpts.metadata_run_key_option()
304@ctrlMpExecOpts.update_graph_id_option()
305def update_graph_run(
306 qgraph: str,
307 run: str,
308 output_qgraph: str,
309 metadata_run_key: str,
310 update_graph_id: bool,
311) -> None:
312 """Update existing quantum graph with new output run name and re-generate
313 output dataset IDs.
315 QGRAPH is the URL to a serialized Quantum Graph file.
317 RUN is the new RUN collection name for output graph.
319 OUTPUT_QGRAPH is the URL to store the updated Quantum Graph.
320 """
321 script.update_graph_run(qgraph, run, output_qgraph, metadata_run_key, update_graph_id)