Coverage for python/lsst/ctrl/mpexec/cli/cmd/commands.py: 58%
151 statements
« prev ^ index » next coverage.py v7.2.7, created at 2023-06-09 02:48 -0700
« prev ^ index » next coverage.py v7.2.7, created at 2023-06-09 02:48 -0700
1# This file is part of ctrl_mpexec.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22import sys
23from collections.abc import Iterator
24from contextlib import contextmanager
25from functools import partial
26from tempfile import NamedTemporaryFile
27from typing import Any
29import click
30import coverage
31import lsst.pipe.base.cli.opt as pipeBaseOpts
32from lsst.ctrl.mpexec.showInfo import ShowInfo
33from lsst.daf.butler.cli.opt import (
34 config_file_option,
35 config_option,
36 confirm_option,
37 options_file_option,
38 processes_option,
39 repo_argument,
40)
41from lsst.daf.butler.cli.utils import MWCtxObj, catch_and_exit, option_section, unwrap
43from .. import opt as ctrlMpExecOpts
44from .. import script
45from ..script import confirmable
46from ..utils import PipetaskCommand, makePipelineActions
48epilog = unwrap(
49 """Notes:
51--task, --delete, --config, --config-file, and --instrument action options can
52appear multiple times; all values are used, in order left to right.
54FILE reads command-line options from the specified file. Data may be
55distributed among multiple lines (e.g. one option per line). Data after # is
56treated as a comment and ignored. Blank lines and lines starting with # are
57ignored.)
58"""
59)
62def _collectActions(ctx: click.Context, **kwargs: Any) -> dict[str, Any]:
63 """Extract pipeline building options, replace them with PipelineActions,
64 return updated `kwargs`.
66 Notes
67 -----
68 The pipeline actions (task, delete, config, config_file, and instrument)
69 must be handled in the order they appear on the command line, but the CLI
70 specification gives them all different option names. So, instead of using
71 the individual action options as they appear in kwargs (because
72 invocation order can't be known), we capture the CLI arguments by
73 overriding `click.Command.parse_args` and save them in the Context's
74 `obj` parameter. We use `makePipelineActions` to create a list of
75 pipeline actions from the CLI arguments and pass that list to the script
76 function using the `pipeline_actions` kwarg name, and remove the action
77 options from kwargs.
78 """
79 for pipelineAction in (
80 ctrlMpExecOpts.task_option.name(),
81 ctrlMpExecOpts.delete_option.name(),
82 config_option.name(),
83 config_file_option.name(),
84 pipeBaseOpts.instrument_option.name(),
85 ):
86 kwargs.pop(pipelineAction)
88 actions = makePipelineActions(MWCtxObj.getFrom(ctx).args)
89 pipeline_actions = []
90 for action in actions:
91 pipeline_actions.append(action)
93 kwargs["pipeline_actions"] = pipeline_actions
94 return kwargs
97def _unhandledShow(show: ShowInfo, cmd: str) -> None:
98 if show.unhandled: 98 ↛ 99line 98 didn't jump to line 99, because the condition on line 98 was never true
99 print(
100 f"The following '--show' options were not known to the {cmd} command: "
101 f"{', '.join(show.unhandled)}",
102 file=sys.stderr,
103 )
106@click.command(cls=PipetaskCommand, epilog=epilog, short_help="Build pipeline definition.")
107@click.pass_context
108@ctrlMpExecOpts.show_option()
109@ctrlMpExecOpts.pipeline_build_options()
110@option_section(sectionText="")
111@options_file_option()
112@catch_and_exit
113def build(ctx: click.Context, **kwargs: Any) -> None:
114 """Build and optionally save pipeline definition.
116 This does not require input data to be specified.
117 """
118 kwargs = _collectActions(ctx, **kwargs)
119 show = ShowInfo(kwargs.pop("show", []))
120 script.build(**kwargs, show=show)
121 _unhandledShow(show, "build")
124@contextmanager
125def coverage_context(kwargs: dict[str, Any]) -> Iterator[None]:
126 packages = kwargs.pop("cov_packages", ())
127 report = kwargs.pop("cov_report", True)
128 if not kwargs.pop("coverage", False):
129 yield
130 return
131 with NamedTemporaryFile("w") as rcfile:
132 rcfile.write(
133 """
134[run]
135branch = True
136concurrency = multiprocessing
137"""
138 )
139 if packages:
140 packages_str = ",".join(packages)
141 rcfile.write(f"source_pkgs = {packages_str}\n")
142 rcfile.flush()
143 cov = coverage.Coverage(config_file=rcfile.name)
144 cov.start()
145 try:
146 yield
147 finally:
148 cov.stop()
149 cov.save()
150 if report:
151 outdir = "./covhtml"
152 cov.html_report(directory=outdir)
153 click.echo(f"Coverage report written to {outdir}.")
156@click.command(cls=PipetaskCommand, epilog=epilog)
157@click.pass_context
158@ctrlMpExecOpts.show_option()
159@ctrlMpExecOpts.pipeline_build_options()
160@ctrlMpExecOpts.qgraph_options()
161@ctrlMpExecOpts.butler_options()
162@option_section(sectionText="")
163@options_file_option()
164@catch_and_exit
165def qgraph(ctx: click.Context, **kwargs: Any) -> None:
166 """Build and optionally save quantum graph."""
167 kwargs = _collectActions(ctx, **kwargs)
168 with coverage_context(kwargs):
169 show = ShowInfo(kwargs.pop("show", []))
170 pipeline = script.build(**kwargs, show=show)
171 if show.handled and not show.unhandled:
172 print(
173 "No quantum graph generated. The --show option was given and all options were processed.",
174 file=sys.stderr,
175 )
176 return
177 if script.qgraph(pipelineObj=pipeline, **kwargs, show=show) is None:
178 raise click.ClickException("QuantumGraph was empty; CRITICAL logs above should provide details.")
179 _unhandledShow(show, "qgraph")
182@click.command(cls=PipetaskCommand, epilog=epilog)
183@ctrlMpExecOpts.run_options()
184@catch_and_exit
185def run(ctx: click.Context, **kwargs: Any) -> None:
186 """Build and execute pipeline and quantum graph."""
187 kwargs = _collectActions(ctx, **kwargs)
188 with coverage_context(kwargs):
189 show = ShowInfo(kwargs.pop("show", []))
190 pipeline = script.build(**kwargs, show=show)
191 if show.handled and not show.unhandled:
192 print(
193 "No quantum graph generated or pipeline executed. "
194 "The --show option was given and all options were processed.",
195 file=sys.stderr,
196 )
197 return
198 if (qgraph := script.qgraph(pipelineObj=pipeline, **kwargs, show=show)) is None:
199 raise click.ClickException("QuantumGraph was empty; CRITICAL logs above should provide details.")
200 _unhandledShow(show, "run")
201 if show.handled:
202 print(
203 "No pipeline executed. The --show option was given and all options were processed.",
204 file=sys.stderr,
205 )
206 return
207 script.run(qgraphObj=qgraph, **kwargs)
210@click.command(cls=PipetaskCommand)
211@ctrlMpExecOpts.butler_config_option()
212@ctrlMpExecOpts.collection_argument()
213@confirm_option()
214@ctrlMpExecOpts.recursive_option(
215 help="""If the parent CHAINED collection has child CHAINED collections,
216 search the children until nested chains that start with the parent's name
217 are removed."""
218)
219def purge(confirm: bool, **kwargs: Any) -> None:
220 """Remove a CHAINED collection and its contained collections.
222 COLLECTION is the name of the chained collection to purge. it must not be a
223 child of any other CHAINED collections
225 Child collections must be members of exactly one collection.
227 The collections that will be removed will be printed, there will be an
228 option to continue or abort (unless using --no-confirm).
229 """
230 confirmable.confirm(partial(script.purge, **kwargs), confirm)
233@click.command(cls=PipetaskCommand)
234@ctrlMpExecOpts.butler_config_option()
235@ctrlMpExecOpts.collection_argument()
236@confirm_option()
237def cleanup(confirm: bool, **kwargs: Any) -> None:
238 """Remove non-members of CHAINED collections.
240 Removes collections that start with the same name as a CHAINED
241 collection but are not members of that collection.
242 """
243 confirmable.confirm(partial(script.cleanup, **kwargs), confirm)
246@click.command(cls=PipetaskCommand)
247@repo_argument()
248@ctrlMpExecOpts.qgraph_argument()
249@ctrlMpExecOpts.config_search_path_option()
250@ctrlMpExecOpts.qgraph_id_option()
251@ctrlMpExecOpts.coverage_options()
252def pre_exec_init_qbb(repo: str, qgraph: str, **kwargs: Any) -> None:
253 """Execute pre-exec-init on Quantum-Backed Butler.
255 REPO is the location of the butler/registry config file.
257 QGRAPH is the path to a serialized Quantum Graph file.
258 """
259 with coverage_context(kwargs):
260 script.pre_exec_init_qbb(repo, qgraph, **kwargs)
263@click.command(cls=PipetaskCommand)
264@repo_argument()
265@ctrlMpExecOpts.qgraph_argument()
266@ctrlMpExecOpts.config_search_path_option()
267@ctrlMpExecOpts.qgraph_id_option()
268@ctrlMpExecOpts.qgraph_node_id_option()
269@processes_option()
270@ctrlMpExecOpts.pdb_option()
271@ctrlMpExecOpts.profile_option()
272@ctrlMpExecOpts.coverage_options()
273@ctrlMpExecOpts.debug_option()
274@ctrlMpExecOpts.start_method_option()
275@ctrlMpExecOpts.timeout_option()
276@ctrlMpExecOpts.fail_fast_option()
277@ctrlMpExecOpts.summary_option()
278@ctrlMpExecOpts.enable_implicit_threading_option()
279def run_qbb(repo: str, qgraph: str, **kwargs: Any) -> None:
280 """Execute pipeline using Quantum-Backed Butler.
282 REPO is the location of the butler/registry config file.
284 QGRAPH is the path to a serialized Quantum Graph file.
285 """
286 with coverage_context(kwargs):
287 script.run_qbb(repo, qgraph, **kwargs)
290@click.command(cls=PipetaskCommand)
291@ctrlMpExecOpts.qgraph_argument()
292@ctrlMpExecOpts.run_argument()
293@ctrlMpExecOpts.output_qgraph_argument()
294@ctrlMpExecOpts.metadata_run_key_option()
295@ctrlMpExecOpts.update_graph_id_option()
296def update_graph_run(
297 qgraph: str,
298 run: str,
299 output_qgraph: str,
300 metadata_run_key: str,
301 update_graph_id: bool,
302) -> None:
303 """Update existing quantum graph with new output run name and re-generate
304 output dataset IDs.
306 QGRAPH is the URL to a serialized Quantum Graph file.
308 RUN is the new RUN collection name for output graph.
310 OUTPUT_QGRAPH is the URL to store the updated Quantum Graph.
311 """
312 script.update_graph_run(qgraph, run, output_qgraph, metadata_run_key, update_graph_id)