Coverage for python/lsst/ctrl/mpexec/cli/cmd/commands.py: 82%
169 statements
« prev ^ index » next coverage.py v7.5.1, created at 2024-05-07 02:50 -0700
« prev ^ index » next coverage.py v7.5.1, created at 2024-05-07 02:50 -0700
1# This file is part of ctrl_mpexec.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This software is dual licensed under the GNU General Public License and also
10# under a 3-clause BSD license. Recipients may choose which of these licenses
11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
12# respectively. If you choose the GPL option then the following text applies
13# (but note that there is still no warranty even if you opt for BSD instead):
14#
15# This program is free software: you can redistribute it and/or modify
16# it under the terms of the GNU General Public License as published by
17# the Free Software Foundation, either version 3 of the License, or
18# (at your option) any later version.
19#
20# This program is distributed in the hope that it will be useful,
21# but WITHOUT ANY WARRANTY; without even the implied warranty of
22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
23# GNU General Public License for more details.
24#
25# You should have received a copy of the GNU General Public License
26# along with this program. If not, see <http://www.gnu.org/licenses/>.
28import sys
29from collections.abc import Iterator
30from contextlib import contextmanager
31from functools import partial
32from tempfile import NamedTemporaryFile
33from typing import Any
35import click
36import coverage
37import lsst.pipe.base.cli.opt as pipeBaseOpts
38from lsst.ctrl.mpexec import Report
39from lsst.ctrl.mpexec.showInfo import ShowInfo
40from lsst.daf.butler.cli.opt import (
41 config_file_option,
42 config_option,
43 confirm_option,
44 options_file_option,
45 processes_option,
46 repo_argument,
47)
48from lsst.daf.butler.cli.utils import MWCtxObj, catch_and_exit, option_section, unwrap
50from .. import opt as ctrlMpExecOpts
51from .. import script
52from ..script import confirmable
53from ..utils import PipetaskCommand, makePipelineActions
55epilog = unwrap(
56 """Notes:
58--task, --delete, --config, --config-file, and --instrument action options can
59appear multiple times; all values are used, in order left to right.
61FILE reads command-line options from the specified file. Data may be
62distributed among multiple lines (e.g. one option per line). Data after # is
63treated as a comment and ignored. Blank lines and lines starting with # are
64ignored.)
65"""
66)
69def _collectActions(ctx: click.Context, **kwargs: Any) -> dict[str, Any]:
70 """Extract pipeline building options, replace them with PipelineActions,
71 return updated `kwargs`.
73 Notes
74 -----
75 The pipeline actions (task, delete, config, config_file, and instrument)
76 must be handled in the order they appear on the command line, but the CLI
77 specification gives them all different option names. So, instead of using
78 the individual action options as they appear in kwargs (because
79 invocation order can't be known), we capture the CLI arguments by
80 overriding `click.Command.parse_args` and save them in the Context's
81 `obj` parameter. We use `makePipelineActions` to create a list of
82 pipeline actions from the CLI arguments and pass that list to the script
83 function using the `pipeline_actions` kwarg name, and remove the action
84 options from kwargs.
85 """
86 for pipelineAction in (
87 ctrlMpExecOpts.task_option.name(),
88 ctrlMpExecOpts.delete_option.name(),
89 config_option.name(),
90 config_file_option.name(),
91 pipeBaseOpts.instrument_option.name(),
92 ):
93 kwargs.pop(pipelineAction)
95 actions = makePipelineActions(MWCtxObj.getFrom(ctx).args)
96 pipeline_actions = []
97 for action in actions:
98 pipeline_actions.append(action)
100 kwargs["pipeline_actions"] = pipeline_actions
101 return kwargs
104def _unhandledShow(show: ShowInfo, cmd: str) -> None:
105 if show.unhandled: 105 ↛ 106line 105 didn't jump to line 106, because the condition on line 105 was never true
106 print(
107 f"The following '--show' options were not known to the {cmd} command: "
108 f"{', '.join(show.unhandled)}",
109 file=sys.stderr,
110 )
113@click.command(cls=PipetaskCommand, epilog=epilog, short_help="Build pipeline definition.")
114@click.pass_context
115@ctrlMpExecOpts.show_option()
116@ctrlMpExecOpts.pipeline_build_options()
117@option_section(sectionText="")
118@options_file_option()
119@catch_and_exit
120def build(ctx: click.Context, **kwargs: Any) -> None:
121 """Build and optionally save pipeline definition.
123 This does not require input data to be specified.
124 """
125 kwargs = _collectActions(ctx, **kwargs)
126 show = ShowInfo(kwargs.pop("show", []))
127 if kwargs.get("butler_config") is not None and {"pipeline-graph", "task-graph"}.isdisjoint(show.commands): 127 ↛ 128line 127 didn't jump to line 128, because the condition on line 127 was never true
128 raise click.ClickException(
129 "--butler-config was provided but nothing uses it "
130 "(only --show pipeline-graph and --show task-graph do)."
131 )
132 script.build(**kwargs, show=show)
133 _unhandledShow(show, "build")
136@contextmanager
137def coverage_context(kwargs: dict[str, Any]) -> Iterator[None]:
138 """Enable coverage recording."""
139 packages = kwargs.pop("cov_packages", ())
140 report = kwargs.pop("cov_report", True)
141 if not kwargs.pop("coverage", False): 141 ↛ 144line 141 didn't jump to line 144, because the condition on line 141 was always true
142 yield
143 return
144 with NamedTemporaryFile("w") as rcfile:
145 rcfile.write(
146 """
147[run]
148branch = True
149concurrency = multiprocessing
150"""
151 )
152 if packages:
153 packages_str = ",".join(packages)
154 rcfile.write(f"source_pkgs = {packages_str}\n")
155 rcfile.flush()
156 cov = coverage.Coverage(config_file=rcfile.name)
157 cov.start()
158 try:
159 yield
160 finally:
161 cov.stop()
162 cov.save()
163 if report:
164 outdir = "./covhtml"
165 cov.html_report(directory=outdir)
166 click.echo(f"Coverage report written to {outdir}.")
169@click.command(cls=PipetaskCommand, epilog=epilog)
170@click.pass_context
171@ctrlMpExecOpts.show_option()
172@ctrlMpExecOpts.pipeline_build_options()
173@ctrlMpExecOpts.qgraph_options()
174@ctrlMpExecOpts.butler_options()
175@option_section(sectionText="")
176@options_file_option()
177@catch_and_exit
178def qgraph(ctx: click.Context, **kwargs: Any) -> None:
179 """Build and optionally save quantum graph."""
180 kwargs = _collectActions(ctx, **kwargs)
181 summary = kwargs.pop("summary", None)
182 with coverage_context(kwargs):
183 show = ShowInfo(kwargs.pop("show", []))
184 pipeline = script.build(**kwargs, show=show)
185 if show.handled and not show.unhandled: 185 ↛ 186line 185 didn't jump to line 186, because the condition on line 185 was never true
186 print(
187 "No quantum graph generated. The --show option was given and all options were processed.",
188 file=sys.stderr,
189 )
190 return
191 if (qgraph := script.qgraph(pipelineObj=pipeline, **kwargs, show=show)) is None: 191 ↛ 192line 191 didn't jump to line 192, because the condition on line 191 was never true
192 raise click.ClickException("QuantumGraph was empty; CRITICAL logs above should provide details.")
193 # QuantumGraph-only summary call here since script.qgraph also called
194 # by run methods.
195 if summary: 195 ↛ 201line 195 didn't jump to line 201, because the condition on line 195 was always true
196 report = Report(qgraphSummary=qgraph.getSummary())
197 with open(summary, "w") as out:
198 # Do not save fields that are not set.
199 out.write(report.model_dump_json(exclude_none=True, indent=2))
201 _unhandledShow(show, "qgraph")
204@click.command(cls=PipetaskCommand, epilog=epilog)
205@ctrlMpExecOpts.run_options()
206@catch_and_exit
207def run(ctx: click.Context, **kwargs: Any) -> None:
208 """Build and execute pipeline and quantum graph."""
209 kwargs = _collectActions(ctx, **kwargs)
210 with coverage_context(kwargs):
211 show = ShowInfo(kwargs.pop("show", []))
212 pipeline = script.build(**kwargs, show=show)
213 if show.handled and not show.unhandled:
214 print(
215 "No quantum graph generated or pipeline executed. "
216 "The --show option was given and all options were processed.",
217 file=sys.stderr,
218 )
219 return
220 if (qgraph := script.qgraph(pipelineObj=pipeline, **kwargs, show=show)) is None:
221 raise click.ClickException("QuantumGraph was empty; CRITICAL logs above should provide details.")
222 _unhandledShow(show, "run")
223 if show.handled:
224 print(
225 "No pipeline executed. The --show option was given and all options were processed.",
226 file=sys.stderr,
227 )
228 return
229 script.run(qgraphObj=qgraph, **kwargs)
232@click.command(cls=PipetaskCommand)
233@ctrlMpExecOpts.butler_config_option()
234@ctrlMpExecOpts.collection_argument()
235@confirm_option()
236@ctrlMpExecOpts.recursive_option(
237 help="""If the parent CHAINED collection has child CHAINED collections,
238 search the children until nested chains that start with the parent's name
239 are removed."""
240)
241def purge(confirm: bool, **kwargs: Any) -> None:
242 """Remove a CHAINED collection and its contained collections.
244 COLLECTION is the name of the chained collection to purge. it must not be a
245 child of any other CHAINED collections
247 Child collections must be members of exactly one collection.
249 The collections that will be removed will be printed, there will be an
250 option to continue or abort (unless using --no-confirm).
251 """
252 confirmable.confirm(partial(script.purge, **kwargs), confirm)
255@click.command(cls=PipetaskCommand)
256@ctrlMpExecOpts.butler_config_option()
257@ctrlMpExecOpts.collection_argument()
258@confirm_option()
259def cleanup(confirm: bool, **kwargs: Any) -> None:
260 """Remove non-members of CHAINED collections.
262 Removes collections that start with the same name as a CHAINED
263 collection but are not members of that collection.
264 """
265 confirmable.confirm(partial(script.cleanup, **kwargs), confirm)
268@click.command(cls=PipetaskCommand)
269@repo_argument()
270@ctrlMpExecOpts.qgraph_argument()
271@ctrlMpExecOpts.config_search_path_option()
272@ctrlMpExecOpts.qgraph_id_option()
273@ctrlMpExecOpts.coverage_options()
274def pre_exec_init_qbb(repo: str, qgraph: str, **kwargs: Any) -> None:
275 """Execute pre-exec-init on Quantum-Backed Butler.
277 REPO is the location of the butler/registry config file.
279 QGRAPH is the path to a serialized Quantum Graph file.
280 """
281 with coverage_context(kwargs):
282 script.pre_exec_init_qbb(repo, qgraph, **kwargs)
285@click.command(cls=PipetaskCommand)
286@repo_argument()
287@ctrlMpExecOpts.qgraph_argument()
288@ctrlMpExecOpts.config_search_path_option()
289@ctrlMpExecOpts.qgraph_id_option()
290@ctrlMpExecOpts.qgraph_node_id_option()
291@processes_option()
292@ctrlMpExecOpts.pdb_option()
293@ctrlMpExecOpts.profile_option()
294@ctrlMpExecOpts.coverage_options()
295@ctrlMpExecOpts.debug_option()
296@ctrlMpExecOpts.start_method_option()
297@ctrlMpExecOpts.timeout_option()
298@ctrlMpExecOpts.fail_fast_option()
299@ctrlMpExecOpts.summary_option()
300@ctrlMpExecOpts.enable_implicit_threading_option()
301@ctrlMpExecOpts.cores_per_quantum_option()
302@ctrlMpExecOpts.memory_per_quantum_option()
303def run_qbb(repo: str, qgraph: str, **kwargs: Any) -> None:
304 """Execute pipeline using Quantum-Backed Butler.
306 REPO is the location of the butler/registry config file.
308 QGRAPH is the path to a serialized Quantum Graph file.
309 """
310 with coverage_context(kwargs):
311 script.run_qbb(repo, qgraph, **kwargs)
314@click.command(cls=PipetaskCommand)
315@ctrlMpExecOpts.qgraph_argument()
316@ctrlMpExecOpts.run_argument()
317@ctrlMpExecOpts.output_qgraph_argument()
318@ctrlMpExecOpts.metadata_run_key_option()
319@ctrlMpExecOpts.update_graph_id_option()
320def update_graph_run(
321 qgraph: str,
322 run: str,
323 output_qgraph: str,
324 metadata_run_key: str,
325 update_graph_id: bool,
326) -> None:
327 """Update existing quantum graph with new output run name and re-generate
328 output dataset IDs.
330 QGRAPH is the URL to a serialized Quantum Graph file.
332 RUN is the new RUN collection name for output graph.
334 OUTPUT_QGRAPH is the URL to store the updated Quantum Graph.
335 """
336 script.update_graph_run(qgraph, run, output_qgraph, metadata_run_key, update_graph_id)
339@click.command(cls=PipetaskCommand)
340@repo_argument()
341@ctrlMpExecOpts.qgraph_argument()
342@click.option("--full-output-filename", default="", help="Summarize report in a yaml file")
343@click.option("--logs/--no-logs", default=True, help="Get butler log datasets for extra information.")
344@click.option(
345 "--show-errors",
346 is_flag=True,
347 default=False,
348 help="Pretty-print a dict of errors from failed"
349 " quanta to the screen. Note: the default is to output a yaml file with error information"
350 " (data_ids and associated messages) to the current working directory instead.",
351)
352def report(
353 repo: str, qgraph: str, full_output_filename: str = "", logs: bool = True, show_errors: bool = False
354) -> None:
355 """Write a yaml file summarizing the produced and missing expected datasets
356 in a quantum graph.
358 REPO is the location of the butler/registry config file.
360 QGRAPH is the URL to a serialized Quantum Graph file.
361 """
362 script.report(repo, qgraph, full_output_filename, logs, show_errors)