Coverage for python/lsst/ctrl/mpexec/cli/opt/options.py: 80%
72 statements
« prev ^ index » next coverage.py v7.3.2, created at 2023-11-04 09:50 +0000
« prev ^ index » next coverage.py v7.3.2, created at 2023-11-04 09:50 +0000
1# This file is part of ctrl_mpexec.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This software is dual licensed under the GNU General Public License and also
10# under a 3-clause BSD license. Recipients may choose which of these licenses
11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
12# respectively. If you choose the GPL option then the following text applies
13# (but note that there is still no warranty even if you opt for BSD instead):
14#
15# This program is free software: you can redistribute it and/or modify
16# it under the terms of the GNU General Public License as published by
17# the Free Software Foundation, either version 3 of the License, or
18# (at your option) any later version.
19#
20# This program is distributed in the hope that it will be useful,
21# but WITHOUT ANY WARRANTY; without even the implied warranty of
22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
23# GNU General Public License for more details.
24#
25# You should have received a copy of the GNU General Public License
26# along with this program. If not, see <http://www.gnu.org/licenses/>.
28from __future__ import annotations
30from collections.abc import Iterable, Mapping
32import click
33from lsst.daf.butler.cli.utils import MWOptionDecorator, MWPath, split_commas, unwrap
34from lsst.utils.doImport import doImportType
36butler_config_option = MWOptionDecorator(
37 "-b", "--butler-config", help="Location of the gen3 butler/registry config file."
38)
41data_query_option = MWOptionDecorator(
42 "-d", "--data-query", help="User data selection expression.", metavar="QUERY"
43)
46debug_option = MWOptionDecorator(
47 "--debug", help="Enable debugging output using lsstDebug facility (imports debug.py).", is_flag=True
48)
50coverage_option = MWOptionDecorator("--coverage", help="Enable coverage output.", is_flag=True)
52coverage_report_option = MWOptionDecorator(
53 "--cov-report/--no-cov-report",
54 help="If coverage is enabled, controls whether to produce an HTML coverage report.",
55 default=True,
56)
58coverage_packages_option = MWOptionDecorator(
59 "--cov-packages",
60 help=unwrap(
61 """Python packages to restrict coverage to. If none are provided, runs coverage on all packages."""
62 ),
63 multiple=True,
64 callback=split_commas,
65)
67delete_option = MWOptionDecorator(
68 "--delete", callback=split_commas, help="Delete task with given label from pipeline.", multiple=True
69)
72pdb_option = MWOptionDecorator(
73 "--pdb",
74 help="Post-mortem debugger to launch for exceptions (defaults to pdb if unspecified; requires a tty).",
75 is_flag=False,
76 flag_value="pdb",
77 default=None,
78)
81extend_run_option = MWOptionDecorator(
82 "--extend-run",
83 help=(
84 "Instead of creating a new RUN collection, insert datasets into either the one given by "
85 "--output-run (if provided) or the first child collection of --output (which must be of type RUN). "
86 "This also enables --skip-existing option when building a graph. "
87 "When executing a graph this option skips quanta with all existing outputs."
88 ),
89 is_flag=True,
90)
93graph_fixup_option = MWOptionDecorator(
94 "--graph-fixup",
95 help="Name of the class or factory method which makes an instance used for execution graph fixup.",
96)
99init_only_option = MWOptionDecorator(
100 "--init-only",
101 help="Do not actually run; just register dataset types and/or save init outputs.",
102 is_flag=True,
103)
106input_option = MWOptionDecorator(
107 "-i",
108 "--input",
109 callback=split_commas,
110 default=[],
111 help="Comma-separated names of the input collection(s).",
112 metavar="COLLECTION",
113 multiple=True,
114)
117rebase_option = MWOptionDecorator(
118 "--rebase",
119 help=unwrap("""Reset output collection chain if it is inconsistent with --inputs"""),
120 is_flag=True,
121)
124no_versions_option = MWOptionDecorator(
125 "--no-versions", help="Do not save or check package versions.", is_flag=True
126)
129order_pipeline_option = MWOptionDecorator(
130 "--order-pipeline",
131 help=unwrap(
132 """Order tasks in pipeline based on their data
133 dependencies, ordering is performed as last step before saving or
134 executing pipeline."""
135 ),
136 is_flag=True,
137)
140output_option = MWOptionDecorator(
141 "-o",
142 "--output",
143 help=unwrap(
144 """Name of the output CHAINED collection. This may either be an
145 existing CHAINED collection to use as both input and output
146 (incompatible with --input), or a new CHAINED collection created
147 to include all inputs (requires --input). In both cases, the
148 collection's children will start with an output RUN collection
149 that directly holds all new datasets (see --output-run)."""
150 ),
151 metavar="COLL",
152)
155output_run_option = MWOptionDecorator(
156 "--output-run",
157 help=unwrap(
158 """Name of the new output RUN collection. If not provided
159 then --output must be provided and a new RUN collection will
160 be created by appending a timestamp to the value passed with
161 --output. If this collection already exists then
162 --extend-run must be passed."""
163 ),
164 metavar="COLL",
165)
168pipeline_option = MWOptionDecorator(
169 "-p",
170 "--pipeline",
171 help="Location of a pipeline definition file in YAML format.",
172 type=MWPath(file_okay=True, dir_okay=False, readable=True),
173)
176pipeline_dot_option = MWOptionDecorator(
177 "--pipeline-dot",
178 help="Location for storing GraphViz DOT representation of a pipeline.",
179 type=MWPath(writable=True, file_okay=True, dir_okay=False),
180)
183profile_option = MWOptionDecorator(
184 "--profile", help="Dump cProfile statistics to file name.", type=MWPath(file_okay=True, dir_okay=False)
185)
188prune_replaced_option = MWOptionDecorator(
189 "--prune-replaced",
190 help=unwrap(
191 """Delete the datasets in the collection replaced by
192 --replace-run, either just from the datastore
193 ('unstore') or by removing them and the RUN completely
194 ('purge'). Requires --replace-run."""
195 ),
196 type=click.Choice(choices=("unstore", "purge"), case_sensitive=False),
197)
200qgraph_option = MWOptionDecorator(
201 "-g",
202 "--qgraph",
203 help=unwrap(
204 """Location for a serialized quantum graph definition (pickle
205 file). If this option is given then all input data options and
206 pipeline-building options cannot be used. Can be a URI."""
207 ),
208)
211qgraph_id_option = MWOptionDecorator(
212 "--qgraph-id",
213 help=unwrap(
214 """Quantum graph identifier, if specified must match the
215 identifier of the graph loaded from a file. Ignored if graph
216 is not loaded from a file."""
217 ),
218)
221qgraph_datastore_records_option = MWOptionDecorator(
222 "--qgraph-datastore-records",
223 help=unwrap(
224 """Include datastore records into generated quantum graph, these records are used by a
225 quantum-backed butler.
226 """
227 ),
228 is_flag=True,
229)
232# I wanted to use default=None here to match Python API but click silently
233# replaces None with an empty tuple when multiple=True.
234qgraph_node_id_option = MWOptionDecorator(
235 "--qgraph-node-id",
236 callback=split_commas,
237 multiple=True,
238 help=unwrap(
239 """Only load a specified set of nodes when graph is
240 loaded from a file, nodes are identified by UUID
241 values. One or more comma-separated integers are
242 accepted. By default all nodes are loaded. Ignored if
243 graph is not loaded from a file."""
244 ),
245)
247qgraph_header_data_option = MWOptionDecorator(
248 "--show-qgraph-header",
249 is_flag=True,
250 default=False,
251 help="Print the headerData for Quantum Graph to the console",
252)
254qgraph_dot_option = MWOptionDecorator(
255 "--qgraph-dot",
256 help="Location for storing GraphViz DOT representation of a quantum graph.",
257 type=MWPath(writable=True, file_okay=True, dir_okay=False),
258)
261replace_run_option = MWOptionDecorator(
262 "--replace-run",
263 help=unwrap(
264 """Before creating a new RUN collection in an existing
265 CHAINED collection, remove the first child collection
266 (which must be of type RUN). This can be used to repeatedly
267 write to the same (parent) collection during development,
268 but it does not delete the datasets associated with the
269 replaced run unless --prune-replaced is also passed.
270 Requires --output, and incompatible with --extend-run."""
271 ),
272 is_flag=True,
273)
276save_pipeline_option = MWOptionDecorator(
277 "-s",
278 "--save-pipeline",
279 help="Location for storing resulting pipeline definition in YAML format.",
280 type=MWPath(dir_okay=False, file_okay=True, writable=True),
281)
283save_qgraph_option = MWOptionDecorator(
284 "-q",
285 "--save-qgraph",
286 help="URI location for storing a serialized quantum graph definition (pickle file).",
287)
290save_single_quanta_option = MWOptionDecorator(
291 "--save-single-quanta",
292 help=unwrap(
293 """Format string of locations for storing individual
294 quantum graph definition (pickle files). The curly
295 brace {} in the input string will be replaced by a
296 quantum number. Can be a URI."""
297 ),
298)
301show_option = MWOptionDecorator(
302 "--show",
303 callback=split_commas,
304 help=unwrap(
305 """Dump various info to standard output. Possible items are:
306 ``config``, ``config=[Task::]<PATTERN>`` or
307 ``config=[Task::]<PATTERN>:NOIGNORECASE`` to dump configuration
308 fields possibly matching given pattern and/or task label;
309 ``history=<FIELD>`` to dump configuration history for a field,
310 field name is specified as ``[Task::]<PATTERN>``; ``dump-config``,
311 ``dump-config=Task`` to dump complete configuration for a task
312 given its label or all tasks; ``pipeline`` to show pipeline
313 composition; ``graph`` to show information about quanta;
314 ``workflow`` to show information about quanta and their
315 dependency; ``tasks`` to show task composition; ``uri`` to show
316 predicted dataset URIs of quanta; ``pipeline-graph`` for a
317 text-based visualization of the pipeline (tasks and dataset types);
318 ``task-graph`` for a text-based visualization of just the tasks.
319 With -b, pipeline-graph and task-graph include additional information.
320 """
321 ),
322 metavar="ITEM|ITEM=VALUE",
323 multiple=True,
324)
327skip_existing_in_option = MWOptionDecorator(
328 "--skip-existing-in",
329 callback=split_commas,
330 default=None,
331 metavar="COLLECTION",
332 multiple=True,
333 help=unwrap(
334 """If all Quantum outputs already exist in the specified list of
335 collections then that Quantum will be excluded from the QuantumGraph.
336 """
337 ),
338)
341skip_existing_option = MWOptionDecorator(
342 "--skip-existing",
343 is_flag=True,
344 help=unwrap(
345 """This option is equivalent to --skip-existing-in with the name of
346 the output RUN collection. If both --skip-existing-in and
347 --skip-existing are given then output RUN collection is appended to
348 the list of collections."""
349 ),
350)
353clobber_outputs_option = MWOptionDecorator(
354 "--clobber-outputs",
355 help=(
356 "Remove outputs of failed quanta from the output run when they would block the execution of new "
357 "quanta with the same data ID (or assume that this will be done, if just building a QuantumGraph). "
358 "Does nothing if --extend-run is not passed."
359 ),
360 is_flag=True,
361)
364skip_init_writes_option = MWOptionDecorator(
365 "--skip-init-writes",
366 help=unwrap(
367 """Do not write collection-wide 'init output' datasets
368 (e.g.schemas)."""
369 ),
370 is_flag=True,
371)
374enable_implicit_threading_option = MWOptionDecorator(
375 "--enable-implicit-threading",
376 help=unwrap(
377 """Do not disable implicit threading use by third-party libraries (e.g. OpenBLAS).
378 Implicit threading is always disabled during execution with multiprocessing."""
379 ),
380 is_flag=True,
381)
383cores_per_quantum_option = MWOptionDecorator(
384 "-n",
385 "--cores-per-quantum",
386 default=1,
387 help=unwrap(
388 """Number of cores available to each quantum when executing.
389 If '-j' is used each subprocess will be allowed to use this number of cores."""
390 ),
391 type=click.IntRange(min=1),
392)
394memory_per_quantum_option = MWOptionDecorator(
395 "--memory-per-quantum",
396 default="",
397 help=unwrap(
398 """Memory allocated for each quantum to use when executing.
399 This memory allocation is not enforced by the execution system and is purely advisory.
400 If '-j' used each subprocess will be allowed to use this amount of memory.
401 Units are allowed and the default units for a plain integer are MB.
402 For example: '3GB', '3000MB' and '3000' would all result in the same
403 memory limit. Default is for no limit."""
404 ),
405 type=str,
406)
408task_option = MWOptionDecorator(
409 "-t",
410 "--task",
411 callback=split_commas,
412 help=unwrap(
413 """Task name to add to pipeline, must be a fully qualified task
414 name. Task name can be followed by colon and label name, if label
415 is not given then task base name (class name) is used as
416 label."""
417 ),
418 metavar="TASK[:LABEL]",
419 multiple=True,
420)
423timeout_option = MWOptionDecorator(
424 "--timeout", type=click.IntRange(min=0), help="Timeout for multiprocessing; maximum wall time (sec)."
425)
428start_method_option = MWOptionDecorator(
429 "--start-method",
430 default=None,
431 type=click.Choice(choices=["spawn", "fork", "forkserver"]),
432 help="Multiprocessing start method, default is platform-specific.",
433)
436fail_fast_option = MWOptionDecorator(
437 "--fail-fast",
438 help="Stop processing at first error, default is to process as many tasks as possible.",
439 is_flag=True,
440)
442save_execution_butler_option = MWOptionDecorator(
443 "--save-execution-butler",
444 help="Export location for an execution-specific butler after making QuantumGraph",
445)
447mock_option = MWOptionDecorator(
448 "--mock",
449 help="Mock pipeline execution.",
450 is_flag=True,
451)
453unmocked_dataset_types_option = MWOptionDecorator(
454 "--unmocked-dataset-types",
455 callback=split_commas,
456 default=None,
457 metavar="COLLECTION",
458 multiple=True,
459 help="Names of input dataset types that should not be mocked.",
460)
463def parse_mock_failure(
464 ctx: click.Context, param: click.Option, value: Iterable[str] | None
465) -> Mapping[str, tuple[str, type[Exception] | None]]:
466 """Parse the --mock-failure option values into the mapping accepted by
467 `lsst.pipe.base.tests.mocks.mock_task_defs`.
468 """
469 result: dict[str, tuple[str, type[Exception] | None]] = {}
470 if value is None:
471 return result
472 for entry in value:
473 try:
474 task_label, error_type_name, where = entry.split(":", 2)
475 except ValueError:
476 raise click.UsageError(
477 f"Invalid value for --mock-failure option: {entry!r}; "
478 "expected a string of the form 'task:error:where'."
479 ) from None
480 error_type = doImportType(error_type_name) if error_type_name else None
481 result[task_label] = (where, error_type)
482 return result
485mock_failure_option = MWOptionDecorator(
486 "--mock-failure",
487 callback=parse_mock_failure,
488 metavar="LABEL:EXCEPTION:WHERE",
489 default=None,
490 multiple=True,
491 help=unwrap(
492 """Specifications for tasks that should be configured to fail
493 when mocking execution. This is a colon-separated 3-tuple, where the
494 first entry the task label, the second the fully-qualified exception
495 type (empty for ValueError, and the third a string (which typically
496 needs to be quoted to be passed as one argument value by the shell) of
497 the form passed to --where, indicating which data IDs should fail."""
498 ),
499)
502clobber_execution_butler_option = MWOptionDecorator(
503 "--clobber-execution-butler",
504 help=unwrap(
505 """When creating execution butler overwrite
506 any existing products"""
507 ),
508 is_flag=True,
509)
511target_datastore_root_option = MWOptionDecorator(
512 "--target-datastore-root",
513 help=unwrap(
514 """Root directory for datastore of execution butler.
515 Default is to use the original datastore.
516 """
517 ),
518)
520dataset_query_constraint = MWOptionDecorator(
521 "--dataset-query-constraint",
522 help=unwrap(
523 """When constructing a quantum graph constrain by
524 pre-existence of specified dataset types. Valid
525 values are `all` for all inputs dataset types in
526 pipeline, ``off`` to not consider dataset type
527 existence as a constraint, single or comma
528 separated list of dataset type names."""
529 ),
530 default="all",
531)
533summary_option = MWOptionDecorator(
534 "--summary",
535 help=(
536 "Location for storing job summary (JSON file). Note that the"
537 " structure of this file may not be stable."
538 ),
539 type=MWPath(dir_okay=False, file_okay=True, writable=True),
540)
543recursive_option = MWOptionDecorator(
544 "--recursive",
545 is_flag=True,
546)
548config_search_path_option = MWOptionDecorator(
549 "--config-search-path",
550 callback=split_commas,
551 default=[],
552 help="Additional search paths for butler configuration.",
553 metavar="PATH",
554 multiple=True,
555)
557update_graph_id_option = MWOptionDecorator(
558 "--update-graph-id",
559 help=unwrap("Update graph ID with new unique value."),
560 is_flag=True,
561)
563metadata_run_key_option = MWOptionDecorator(
564 "--metadata-run-key",
565 help=(
566 "Quantum graph metadata key for the name of the output run. "
567 "Empty string disables update of the metadata. "
568 "Default value: output_run."
569 ),
570 default="output_run",
571)