Coverage for python/lsst/ctrl/mpexec/cli/cmd/commands.py: 77%
163 statements
« prev ^ index » next coverage.py v7.4.4, created at 2024-03-30 02:54 -0700
« prev ^ index » next coverage.py v7.4.4, created at 2024-03-30 02:54 -0700
1# This file is part of ctrl_mpexec.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This software is dual licensed under the GNU General Public License and also
10# under a 3-clause BSD license. Recipients may choose which of these licenses
11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
12# respectively. If you choose the GPL option then the following text applies
13# (but note that there is still no warranty even if you opt for BSD instead):
14#
15# This program is free software: you can redistribute it and/or modify
16# it under the terms of the GNU General Public License as published by
17# the Free Software Foundation, either version 3 of the License, or
18# (at your option) any later version.
19#
20# This program is distributed in the hope that it will be useful,
21# but WITHOUT ANY WARRANTY; without even the implied warranty of
22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
23# GNU General Public License for more details.
24#
25# You should have received a copy of the GNU General Public License
26# along with this program. If not, see <http://www.gnu.org/licenses/>.
28import sys
29from collections.abc import Iterator
30from contextlib import contextmanager
31from functools import partial
32from tempfile import NamedTemporaryFile
33from typing import Any
35import click
36import coverage
37import lsst.pipe.base.cli.opt as pipeBaseOpts
38from lsst.ctrl.mpexec.showInfo import ShowInfo
39from lsst.daf.butler.cli.opt import (
40 config_file_option,
41 config_option,
42 confirm_option,
43 options_file_option,
44 processes_option,
45 repo_argument,
46)
47from lsst.daf.butler.cli.utils import MWCtxObj, catch_and_exit, option_section, unwrap
49from .. import opt as ctrlMpExecOpts
50from .. import script
51from ..script import confirmable
52from ..utils import PipetaskCommand, makePipelineActions
54epilog = unwrap(
55 """Notes:
57--task, --delete, --config, --config-file, and --instrument action options can
58appear multiple times; all values are used, in order left to right.
60FILE reads command-line options from the specified file. Data may be
61distributed among multiple lines (e.g. one option per line). Data after # is
62treated as a comment and ignored. Blank lines and lines starting with # are
63ignored.)
64"""
65)
68def _collectActions(ctx: click.Context, **kwargs: Any) -> dict[str, Any]:
69 """Extract pipeline building options, replace them with PipelineActions,
70 return updated `kwargs`.
72 Notes
73 -----
74 The pipeline actions (task, delete, config, config_file, and instrument)
75 must be handled in the order they appear on the command line, but the CLI
76 specification gives them all different option names. So, instead of using
77 the individual action options as they appear in kwargs (because
78 invocation order can't be known), we capture the CLI arguments by
79 overriding `click.Command.parse_args` and save them in the Context's
80 `obj` parameter. We use `makePipelineActions` to create a list of
81 pipeline actions from the CLI arguments and pass that list to the script
82 function using the `pipeline_actions` kwarg name, and remove the action
83 options from kwargs.
84 """
85 for pipelineAction in (
86 ctrlMpExecOpts.task_option.name(),
87 ctrlMpExecOpts.delete_option.name(),
88 config_option.name(),
89 config_file_option.name(),
90 pipeBaseOpts.instrument_option.name(),
91 ):
92 kwargs.pop(pipelineAction)
94 actions = makePipelineActions(MWCtxObj.getFrom(ctx).args)
95 pipeline_actions = []
96 for action in actions:
97 pipeline_actions.append(action)
99 kwargs["pipeline_actions"] = pipeline_actions
100 return kwargs
103def _unhandledShow(show: ShowInfo, cmd: str) -> None:
104 if show.unhandled: 104 ↛ 105line 104 didn't jump to line 105, because the condition on line 104 was never true
105 print(
106 f"The following '--show' options were not known to the {cmd} command: "
107 f"{', '.join(show.unhandled)}",
108 file=sys.stderr,
109 )
112@click.command(cls=PipetaskCommand, epilog=epilog, short_help="Build pipeline definition.")
113@click.pass_context
114@ctrlMpExecOpts.show_option()
115@ctrlMpExecOpts.pipeline_build_options()
116@option_section(sectionText="")
117@options_file_option()
118@catch_and_exit
119def build(ctx: click.Context, **kwargs: Any) -> None:
120 """Build and optionally save pipeline definition.
122 This does not require input data to be specified.
123 """
124 kwargs = _collectActions(ctx, **kwargs)
125 show = ShowInfo(kwargs.pop("show", []))
126 if kwargs.get("butler_config") is not None and {"pipeline-graph", "task-graph"}.isdisjoint(show.commands): 126 ↛ 127line 126 didn't jump to line 127, because the condition on line 126 was never true
127 raise click.ClickException(
128 "--butler-config was provided but nothing uses it "
129 "(only --show pipeline-graph and --show task-graph do)."
130 )
131 script.build(**kwargs, show=show)
132 _unhandledShow(show, "build")
135@contextmanager
136def coverage_context(kwargs: dict[str, Any]) -> Iterator[None]:
137 """Enable coverage recording."""
138 packages = kwargs.pop("cov_packages", ())
139 report = kwargs.pop("cov_report", True)
140 if not kwargs.pop("coverage", False):
141 yield
142 return
143 with NamedTemporaryFile("w") as rcfile:
144 rcfile.write(
145 """
146[run]
147branch = True
148concurrency = multiprocessing
149"""
150 )
151 if packages:
152 packages_str = ",".join(packages)
153 rcfile.write(f"source_pkgs = {packages_str}\n")
154 rcfile.flush()
155 cov = coverage.Coverage(config_file=rcfile.name)
156 cov.start()
157 try:
158 yield
159 finally:
160 cov.stop()
161 cov.save()
162 if report:
163 outdir = "./covhtml"
164 cov.html_report(directory=outdir)
165 click.echo(f"Coverage report written to {outdir}.")
168@click.command(cls=PipetaskCommand, epilog=epilog)
169@click.pass_context
170@ctrlMpExecOpts.show_option()
171@ctrlMpExecOpts.pipeline_build_options()
172@ctrlMpExecOpts.qgraph_options()
173@ctrlMpExecOpts.butler_options()
174@option_section(sectionText="")
175@options_file_option()
176@catch_and_exit
177def qgraph(ctx: click.Context, **kwargs: Any) -> None:
178 """Build and optionally save quantum graph."""
179 kwargs = _collectActions(ctx, **kwargs)
180 with coverage_context(kwargs):
181 show = ShowInfo(kwargs.pop("show", []))
182 pipeline = script.build(**kwargs, show=show)
183 if show.handled and not show.unhandled:
184 print(
185 "No quantum graph generated. The --show option was given and all options were processed.",
186 file=sys.stderr,
187 )
188 return
189 if script.qgraph(pipelineObj=pipeline, **kwargs, show=show) is None:
190 raise click.ClickException("QuantumGraph was empty; CRITICAL logs above should provide details.")
191 _unhandledShow(show, "qgraph")
194@click.command(cls=PipetaskCommand, epilog=epilog)
195@ctrlMpExecOpts.run_options()
196@catch_and_exit
197def run(ctx: click.Context, **kwargs: Any) -> None:
198 """Build and execute pipeline and quantum graph."""
199 kwargs = _collectActions(ctx, **kwargs)
200 with coverage_context(kwargs):
201 show = ShowInfo(kwargs.pop("show", []))
202 pipeline = script.build(**kwargs, show=show)
203 if show.handled and not show.unhandled:
204 print(
205 "No quantum graph generated or pipeline executed. "
206 "The --show option was given and all options were processed.",
207 file=sys.stderr,
208 )
209 return
210 if (qgraph := script.qgraph(pipelineObj=pipeline, **kwargs, show=show)) is None:
211 raise click.ClickException("QuantumGraph was empty; CRITICAL logs above should provide details.")
212 _unhandledShow(show, "run")
213 if show.handled:
214 print(
215 "No pipeline executed. The --show option was given and all options were processed.",
216 file=sys.stderr,
217 )
218 return
219 script.run(qgraphObj=qgraph, **kwargs)
222@click.command(cls=PipetaskCommand)
223@ctrlMpExecOpts.butler_config_option()
224@ctrlMpExecOpts.collection_argument()
225@confirm_option()
226@ctrlMpExecOpts.recursive_option(
227 help="""If the parent CHAINED collection has child CHAINED collections,
228 search the children until nested chains that start with the parent's name
229 are removed."""
230)
231def purge(confirm: bool, **kwargs: Any) -> None:
232 """Remove a CHAINED collection and its contained collections.
234 COLLECTION is the name of the chained collection to purge. it must not be a
235 child of any other CHAINED collections
237 Child collections must be members of exactly one collection.
239 The collections that will be removed will be printed, there will be an
240 option to continue or abort (unless using --no-confirm).
241 """
242 confirmable.confirm(partial(script.purge, **kwargs), confirm)
245@click.command(cls=PipetaskCommand)
246@ctrlMpExecOpts.butler_config_option()
247@ctrlMpExecOpts.collection_argument()
248@confirm_option()
249def cleanup(confirm: bool, **kwargs: Any) -> None:
250 """Remove non-members of CHAINED collections.
252 Removes collections that start with the same name as a CHAINED
253 collection but are not members of that collection.
254 """
255 confirmable.confirm(partial(script.cleanup, **kwargs), confirm)
258@click.command(cls=PipetaskCommand)
259@repo_argument()
260@ctrlMpExecOpts.qgraph_argument()
261@ctrlMpExecOpts.config_search_path_option()
262@ctrlMpExecOpts.qgraph_id_option()
263@ctrlMpExecOpts.coverage_options()
264def pre_exec_init_qbb(repo: str, qgraph: str, **kwargs: Any) -> None:
265 """Execute pre-exec-init on Quantum-Backed Butler.
267 REPO is the location of the butler/registry config file.
269 QGRAPH is the path to a serialized Quantum Graph file.
270 """
271 with coverage_context(kwargs):
272 script.pre_exec_init_qbb(repo, qgraph, **kwargs)
275@click.command(cls=PipetaskCommand)
276@repo_argument()
277@ctrlMpExecOpts.qgraph_argument()
278@ctrlMpExecOpts.config_search_path_option()
279@ctrlMpExecOpts.qgraph_id_option()
280@ctrlMpExecOpts.qgraph_node_id_option()
281@processes_option()
282@ctrlMpExecOpts.pdb_option()
283@ctrlMpExecOpts.profile_option()
284@ctrlMpExecOpts.coverage_options()
285@ctrlMpExecOpts.debug_option()
286@ctrlMpExecOpts.start_method_option()
287@ctrlMpExecOpts.timeout_option()
288@ctrlMpExecOpts.fail_fast_option()
289@ctrlMpExecOpts.summary_option()
290@ctrlMpExecOpts.enable_implicit_threading_option()
291@ctrlMpExecOpts.cores_per_quantum_option()
292@ctrlMpExecOpts.memory_per_quantum_option()
293def run_qbb(repo: str, qgraph: str, **kwargs: Any) -> None:
294 """Execute pipeline using Quantum-Backed Butler.
296 REPO is the location of the butler/registry config file.
298 QGRAPH is the path to a serialized Quantum Graph file.
299 """
300 with coverage_context(kwargs):
301 script.run_qbb(repo, qgraph, **kwargs)
304@click.command(cls=PipetaskCommand)
305@ctrlMpExecOpts.qgraph_argument()
306@ctrlMpExecOpts.run_argument()
307@ctrlMpExecOpts.output_qgraph_argument()
308@ctrlMpExecOpts.metadata_run_key_option()
309@ctrlMpExecOpts.update_graph_id_option()
310def update_graph_run(
311 qgraph: str,
312 run: str,
313 output_qgraph: str,
314 metadata_run_key: str,
315 update_graph_id: bool,
316) -> None:
317 """Update existing quantum graph with new output run name and re-generate
318 output dataset IDs.
320 QGRAPH is the URL to a serialized Quantum Graph file.
322 RUN is the new RUN collection name for output graph.
324 OUTPUT_QGRAPH is the URL to store the updated Quantum Graph.
325 """
326 script.update_graph_run(qgraph, run, output_qgraph, metadata_run_key, update_graph_id)
329@click.command(cls=PipetaskCommand)
330@repo_argument()
331@ctrlMpExecOpts.qgraph_argument()
332@click.option("--full-output-filename", default="", help="Summarize report in a yaml file")
333@click.option("--logs/--no-logs", default=True, help="Get butler log datasets for extra information.")
334@click.option(
335 "--show-errors",
336 is_flag=True,
337 default=False,
338 help="Pretty-print a dict of errors from failed"
339 " quanta to the screen. Note: the default is to output a yaml file with error information"
340 " (data_ids and associated messages) to the current working directory instead.",
341)
342def report(
343 repo: str, qgraph: str, full_output_filename: str = "", logs: bool = True, show_errors: bool = False
344) -> None:
345 """Write a yaml file summarizing the produced and missing expected datasets
346 in a quantum graph.
348 REPO is the location of the butler/registry config file.
350 QGRAPH is the URL to a serialized Quantum Graph file.
351 """
352 script.report(repo, qgraph, full_output_filename, logs, show_errors)