Coverage for python/lsst/ctrl/mpexec/cli/opt/options.py: 79%
69 statements
« prev ^ index » next coverage.py v7.2.7, created at 2023-06-22 09:52 +0000
« prev ^ index » next coverage.py v7.2.7, created at 2023-06-22 09:52 +0000
1# This file is part of ctrl_mpexec.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22from __future__ import annotations
24from collections.abc import Iterable, Mapping
26import click
27from lsst.daf.butler.cli.utils import MWOptionDecorator, MWPath, split_commas, unwrap
28from lsst.utils.doImport import doImportType
30butler_config_option = MWOptionDecorator(
31 "-b", "--butler-config", help="Location of the gen3 butler/registry config file."
32)
35data_query_option = MWOptionDecorator(
36 "-d", "--data-query", help="User data selection expression.", metavar="QUERY"
37)
40debug_option = MWOptionDecorator(
41 "--debug", help="Enable debugging output using lsstDebug facility (imports debug.py).", is_flag=True
42)
44coverage_option = MWOptionDecorator("--coverage", help="Enable coverage output.", is_flag=True)
46coverage_report_option = MWOptionDecorator(
47 "--cov-report/--no-cov-report",
48 help="If coverage is enabled, controls whether to produce an HTML coverage report.",
49 default=True,
50)
52coverage_packages_option = MWOptionDecorator(
53 "--cov-packages",
54 help=unwrap(
55 """Python packages to restrict coverage to. If none are provided, runs coverage on all packages."""
56 ),
57 multiple=True,
58 callback=split_commas,
59)
61delete_option = MWOptionDecorator(
62 "--delete", callback=split_commas, help="Delete task with given label from pipeline.", multiple=True
63)
66pdb_option = MWOptionDecorator(
67 "--pdb",
68 help="Post-mortem debugger to launch for exceptions (defaults to pdb if unspecified; requires a tty).",
69 is_flag=False,
70 flag_value="pdb",
71 default=None,
72)
75extend_run_option = MWOptionDecorator(
76 "--extend-run",
77 help=(
78 "Instead of creating a new RUN collection, insert datasets into either the one given by "
79 "--output-run (if provided) or the first child collection of --output (which must be of type RUN). "
80 "This also enables --skip-existing option when building a graph. "
81 "When executing a graph this option skips quanta with all existing outputs."
82 ),
83 is_flag=True,
84)
87graph_fixup_option = MWOptionDecorator(
88 "--graph-fixup",
89 help=unwrap(
90 """Name of the class or factory method which makes an
91 instance used for execution graph fixup."""
92 ),
93)
96init_only_option = MWOptionDecorator(
97 "--init-only",
98 help=unwrap(
99 """Do not actually run; just register dataset types and/or
100 save init outputs. """
101 ),
102 is_flag=True,
103)
106input_option = MWOptionDecorator(
107 "-i",
108 "--input",
109 callback=split_commas,
110 default=list(),
111 help=unwrap("""Comma-separated names of the input collection(s)."""),
112 metavar="COLLECTION",
113 multiple=True,
114)
116no_versions_option = MWOptionDecorator(
117 "--no-versions", help="Do not save or check package versions.", is_flag=True
118)
121order_pipeline_option = MWOptionDecorator(
122 "--order-pipeline",
123 help=unwrap(
124 """Order tasks in pipeline based on their data
125 dependencies, ordering is performed as last step before saving or
126 executing pipeline."""
127 ),
128 is_flag=True,
129)
132output_option = MWOptionDecorator(
133 "-o",
134 "--output",
135 help=unwrap(
136 """Name of the output CHAINED collection. This may either be an
137 existing CHAINED collection to use as both input and output
138 (incompatible with --input), or a new CHAINED collection created
139 to include all inputs (requires --input). In both cases, the
140 collection's children will start with an output RUN collection
141 that directly holds all new datasets (see --output-run)."""
142 ),
143 metavar="COLL",
144)
147output_run_option = MWOptionDecorator(
148 "--output-run",
149 help=unwrap(
150 """Name of the new output RUN collection. If not provided
151 then --output must be provided and a new RUN collection will
152 be created by appending a timestamp to the value passed with
153 --output. If this collection already exists then
154 --extend-run must be passed."""
155 ),
156 metavar="COLL",
157)
160pipeline_option = MWOptionDecorator(
161 "-p",
162 "--pipeline",
163 help="Location of a pipeline definition file in YAML format.",
164 type=MWPath(file_okay=True, dir_okay=False, readable=True),
165)
168pipeline_dot_option = MWOptionDecorator(
169 "--pipeline-dot",
170 help=unwrap(
171 """"Location for storing GraphViz DOT representation of a
172 pipeline."""
173 ),
174 type=MWPath(writable=True, file_okay=True, dir_okay=False),
175)
178profile_option = MWOptionDecorator(
179 "--profile", help="Dump cProfile statistics to file name.", type=MWPath(file_okay=True, dir_okay=False)
180)
183prune_replaced_option = MWOptionDecorator(
184 "--prune-replaced",
185 help=unwrap(
186 """Delete the datasets in the collection replaced by
187 --replace-run, either just from the datastore
188 ('unstore') or by removing them and the RUN completely
189 ('purge'). Requires --replace-run."""
190 ),
191 type=click.Choice(choices=("unstore", "purge"), case_sensitive=False),
192)
195qgraph_option = MWOptionDecorator(
196 "-g",
197 "--qgraph",
198 help=unwrap(
199 """Location for a serialized quantum graph definition (pickle
200 file). If this option is given then all input data options and
201 pipeline-building options cannot be used. Can be a URI."""
202 ),
203)
206qgraph_id_option = MWOptionDecorator(
207 "--qgraph-id",
208 help=unwrap(
209 """Quantum graph identifier, if specified must match the
210 identifier of the graph loaded from a file. Ignored if graph
211 is not loaded from a file."""
212 ),
213)
216qgraph_datastore_records_option = MWOptionDecorator(
217 "--qgraph-datastore-records",
218 help=unwrap(
219 """Include datastore records into generated quantum graph, these records are used by a
220 quantum-backed butler.
221 """
222 ),
223 is_flag=True,
224)
227# I wanted to use default=None here to match Python API but click silently
228# replaces None with an empty tuple when multiple=True.
229qgraph_node_id_option = MWOptionDecorator(
230 "--qgraph-node-id",
231 callback=split_commas,
232 multiple=True,
233 help=unwrap(
234 """Only load a specified set of nodes when graph is
235 loaded from a file, nodes are identified by UUID
236 values. One or more comma-separated integers are
237 accepted. By default all nodes are loaded. Ignored if
238 graph is not loaded from a file."""
239 ),
240)
242qgraph_header_data_option = MWOptionDecorator(
243 "--show-qgraph-header",
244 is_flag=True,
245 default=False,
246 help=unwrap(
247 """Print the headerData for Quantum Graph to the
248 console"""
249 ),
250)
252qgraph_dot_option = MWOptionDecorator(
253 "--qgraph-dot",
254 help=unwrap(
255 """Location for storing GraphViz DOT representation of a
256 quantum graph."""
257 ),
258 type=MWPath(writable=True, file_okay=True, dir_okay=False),
259)
262replace_run_option = MWOptionDecorator(
263 "--replace-run",
264 help=unwrap(
265 """Before creating a new RUN collection in an existing
266 CHAINED collection, remove the first child collection
267 (which must be of type RUN). This can be used to repeatedly
268 write to the same (parent) collection during development,
269 but it does not delete the datasets associated with the
270 replaced run unless --prune-replaced is also passed.
271 Requires --output, and incompatible with --extend-run."""
272 ),
273 is_flag=True,
274)
277save_pipeline_option = MWOptionDecorator(
278 "-s",
279 "--save-pipeline",
280 help=unwrap(
281 """Location for storing resulting pipeline definition in
282 YAML format."""
283 ),
284 type=MWPath(dir_okay=False, file_okay=True, writable=True),
285)
287save_qgraph_option = MWOptionDecorator(
288 "-q",
289 "--save-qgraph",
290 help=unwrap(
291 """URI location for storing a serialized quantum graph
292 definition (pickle file)."""
293 ),
294)
297save_single_quanta_option = MWOptionDecorator(
298 "--save-single-quanta",
299 help=unwrap(
300 """Format string of locations for storing individual
301 quantum graph definition (pickle files). The curly
302 brace {} in the input string will be replaced by a
303 quantum number. Can be a URI."""
304 ),
305)
308show_option = MWOptionDecorator(
309 "--show",
310 callback=split_commas,
311 help=unwrap(
312 """Dump various info to standard output. Possible items are:
313 `config`, `config=[Task::]<PATTERN>` or
314 `config=[Task::]<PATTERN>:NOIGNORECASE` to dump configuration
315 fields possibly matching given pattern and/or task label;
316 `history=<FIELD>` to dump configuration history for a field, field
317 name is specified as [Task::]<PATTERN>; `dump-config`,
318 `dump-config=Task` to dump complete configuration for a task given
319 its label or all tasks; `pipeline` to show pipeline composition;
320 `graph` to show information about quanta; `workflow` to show
321 information about quanta and their dependency; `tasks` to show
322 task composition; `uri` to show predicted dataset URIs of
323 quanta"""
324 ),
325 metavar="ITEM|ITEM=VALUE",
326 multiple=True,
327)
330skip_existing_in_option = MWOptionDecorator(
331 "--skip-existing-in",
332 callback=split_commas,
333 default=None,
334 metavar="COLLECTION",
335 multiple=True,
336 help=unwrap(
337 """If all Quantum outputs already exist in the specified list of
338 collections then that Quantum will be excluded from the QuantumGraph.
339 """
340 ),
341)
344skip_existing_option = MWOptionDecorator(
345 "--skip-existing",
346 is_flag=True,
347 help=unwrap(
348 """This option is equivalent to --skip-existing-in with the name of
349 the output RUN collection. If both --skip-existing-in and
350 --skip-existing are given then output RUN collection is appended to
351 the list of collections."""
352 ),
353)
356clobber_outputs_option = MWOptionDecorator(
357 "--clobber-outputs",
358 help=(
359 "Remove outputs of failed quanta from the output run when they would block the execution of new "
360 "quanta with the same data ID (or assume that this will be done, if just building a QuantumGraph). "
361 "Does nothing if --extend-run is not passed."
362 ),
363 is_flag=True,
364)
367skip_init_writes_option = MWOptionDecorator(
368 "--skip-init-writes",
369 help=unwrap(
370 """Do not write collection-wide 'init output' datasets
371 (e.g.schemas)."""
372 ),
373 is_flag=True,
374)
377enable_implicit_threading_option = MWOptionDecorator(
378 "--enable-implicit-threading",
379 help=unwrap(
380 """Do not disable implicit threading use by third-party libraries (e.g. OpenBLAS).
381 Implicit threading is always disabled during execution with multiprocessing."""
382 ),
383 is_flag=True,
384)
387task_option = MWOptionDecorator(
388 "-t",
389 "--task",
390 callback=split_commas,
391 help=unwrap(
392 """Task name to add to pipeline, must be a fully qualified task
393 name. Task name can be followed by colon and label name, if label
394 is not given then task base name (class name) is used as
395 label."""
396 ),
397 metavar="TASK[:LABEL]",
398 multiple=True,
399)
402timeout_option = MWOptionDecorator(
403 "--timeout", type=click.IntRange(min=0), help="Timeout for multiprocessing; maximum wall time (sec)."
404)
407start_method_option = MWOptionDecorator(
408 "--start-method",
409 default=None,
410 type=click.Choice(choices=["spawn", "fork", "forkserver"]),
411 help="Multiprocessing start method, default is platform-specific.",
412)
415fail_fast_option = MWOptionDecorator(
416 "--fail-fast",
417 help=unwrap(
418 """Stop processing at first error, default is to process
419 as many tasks as possible."""
420 ),
421 is_flag=True,
422)
424save_execution_butler_option = MWOptionDecorator(
425 "--save-execution-butler",
426 help=unwrap(
427 """Export location for an
428 execution-specific butler after making
429 QuantumGraph"""
430 ),
431)
433mock_option = MWOptionDecorator(
434 "--mock",
435 help=unwrap("""Mock pipeline execution."""),
436 is_flag=True,
437)
439unmocked_dataset_types_option = MWOptionDecorator(
440 "--unmocked-dataset-types",
441 callback=split_commas,
442 default=None,
443 metavar="COLLECTION",
444 multiple=True,
445 help=unwrap("""Names of input dataset types that should not be mocked."""),
446)
449def parse_mock_failure(
450 ctx: click.Context, param: click.Option, value: Iterable[str] | None
451) -> Mapping[str, tuple[str, type[Exception] | None]]:
452 """Parse the --mock-failure option values into the mapping accepted by
453 `lsst.pipe.base.tests.mocks.mock_task_defs`.
454 """
455 result: dict[str, tuple[str, type[Exception] | None]] = {}
456 if value is None:
457 return result
458 for entry in value:
459 try:
460 task_label, error_type_name, where = entry.split(":", 2)
461 except ValueError:
462 raise click.UsageError(
463 f"Invalid value for --mock-failure option: {entry!r}; "
464 "expected a string of the form 'task:error:where'."
465 ) from None
466 error_type = doImportType(error_type_name) if error_type_name else None
467 result[task_label] = (where, error_type)
468 return result
471mock_failure_option = MWOptionDecorator(
472 "--mock-failure",
473 callback=parse_mock_failure,
474 metavar="LABEL:EXCEPTION:WHERE",
475 default=None,
476 multiple=True,
477 help=unwrap(
478 """Specifications for tasks that should be configured to fail
479 when mocking execution. This is a colon-separated 3-tuple, where the
480 first entry the task label, the second the fully-qualified exception
481 type (empty for ValueError, and the third a string (which typically
482 needs to be quoted to be passed as one argument value by the shell) of
483 the form passed to --where, indicating which data IDs should fail."""
484 ),
485)
488clobber_execution_butler_option = MWOptionDecorator(
489 "--clobber-execution-butler",
490 help=unwrap(
491 """When creating execution butler overwrite
492 any existing products"""
493 ),
494 is_flag=True,
495)
497target_datastore_root_option = MWOptionDecorator(
498 "--target-datastore-root",
499 help=unwrap(
500 """Root directory for datastore of execution butler.
501 Default is to use the original datastore.
502 """
503 ),
504)
506dataset_query_constraint = MWOptionDecorator(
507 "--dataset-query-constraint",
508 help=unwrap(
509 """When constructing a quantum graph constrain by
510 pre-existence of specified dataset types. Valid
511 values are `all` for all inputs dataset types in
512 pipeline, `off` to not consider dataset type
513 existance as a constraint, single or comma
514 separated list of dataset type names"""
515 ),
516 default="all",
517)
519summary_option = MWOptionDecorator(
520 "--summary",
521 help=(
522 "Location for storing job summary (JSON file). Note that the"
523 " structure of this file may not be stable."
524 ),
525 type=MWPath(dir_okay=False, file_okay=True, writable=True),
526)
529recursive_option = MWOptionDecorator(
530 "--recursive",
531 is_flag=True,
532)
534config_search_path_option = MWOptionDecorator(
535 "--config-search-path",
536 callback=split_commas,
537 default=list(),
538 help="Additional search paths for butler configuration.",
539 metavar="PATH",
540 multiple=True,
541)
543update_graph_id_option = MWOptionDecorator(
544 "--update-graph-id",
545 help=unwrap("Update graph ID with new unique value."),
546 is_flag=True,
547)
549metadata_run_key_option = MWOptionDecorator(
550 "--metadata-run-key",
551 help=(
552 "Quantum graph metadata key for the name of the output run. "
553 "Empty string disables update of the metadata. "
554 "Default value: output_run."
555 ),
556 default="output_run",
557)