Coverage for python/lsst/ctrl/mpexec/cli/opt/options.py: 100%
53 statements
« prev ^ index » next coverage.py v7.2.7, created at 2023-06-14 02:12 -0700
« prev ^ index » next coverage.py v7.2.7, created at 2023-06-14 02:12 -0700
1# This file is part of ctrl_mpexec.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22import click
23from lsst.daf.butler.cli.utils import MWOptionDecorator, MWPath, split_commas, unwrap
25butler_config_option = MWOptionDecorator(
26 "-b", "--butler-config", help="Location of the gen3 butler/registry config file."
27)
30data_query_option = MWOptionDecorator(
31 "-d", "--data-query", help="User data selection expression.", metavar="QUERY"
32)
35debug_option = MWOptionDecorator(
36 "--debug", help="Enable debugging output using lsstDebug facility (imports debug.py).", is_flag=True
37)
39coverage_option = MWOptionDecorator("--coverage", help="Enable coverage output.", is_flag=True)
41coverage_report_option = MWOptionDecorator(
42 "--cov-report/--no-cov-report",
43 help="If coverage is enabled, controls whether to produce an HTML coverage report.",
44 default=True,
45)
47coverage_packages_option = MWOptionDecorator(
48 "--cov-packages",
49 help=unwrap(
50 """Python packages to restrict coverage to. If none are provided, runs coverage on all packages."""
51 ),
52 multiple=True,
53 callback=split_commas,
54)
56delete_option = MWOptionDecorator(
57 "--delete", callback=split_commas, help="Delete task with given label from pipeline.", multiple=True
58)
61pdb_option = MWOptionDecorator(
62 "--pdb",
63 help="Post-mortem debugger to launch for exceptions (defaults to pdb if unspecified; requires a tty).",
64 is_flag=False,
65 flag_value="pdb",
66 default=None,
67)
70extend_run_option = MWOptionDecorator(
71 "--extend-run",
72 help=(
73 "Instead of creating a new RUN collection, insert datasets into either the one given by "
74 "--output-run (if provided) or the first child collection of --output (which must be of type RUN). "
75 "This also enables --skip-existing option when building a graph. "
76 "When executing a graph this option skips quanta with all existing outputs."
77 ),
78 is_flag=True,
79)
82graph_fixup_option = MWOptionDecorator(
83 "--graph-fixup",
84 help=unwrap(
85 """Name of the class or factory method which makes an
86 instance used for execution graph fixup."""
87 ),
88)
91init_only_option = MWOptionDecorator(
92 "--init-only",
93 help=unwrap(
94 """Do not actually run; just register dataset types and/or
95 save init outputs. """
96 ),
97 is_flag=True,
98)
101input_option = MWOptionDecorator(
102 "-i",
103 "--input",
104 callback=split_commas,
105 default=list(),
106 help=unwrap("""Comma-separated names of the input collection(s)."""),
107 metavar="COLLECTION",
108 multiple=True,
109)
111no_versions_option = MWOptionDecorator(
112 "--no-versions", help="Do not save or check package versions.", is_flag=True
113)
116order_pipeline_option = MWOptionDecorator(
117 "--order-pipeline",
118 help=unwrap(
119 """Order tasks in pipeline based on their data
120 dependencies, ordering is performed as last step before saving or
121 executing pipeline."""
122 ),
123 is_flag=True,
124)
127output_option = MWOptionDecorator(
128 "-o",
129 "--output",
130 help=unwrap(
131 """Name of the output CHAINED collection. This may either be an
132 existing CHAINED collection to use as both input and output
133 (incompatible with --input), or a new CHAINED collection created
134 to include all inputs (requires --input). In both cases, the
135 collection's children will start with an output RUN collection
136 that directly holds all new datasets (see --output-run)."""
137 ),
138 metavar="COLL",
139)
142output_run_option = MWOptionDecorator(
143 "--output-run",
144 help=unwrap(
145 """Name of the new output RUN collection. If not provided
146 then --output must be provided and a new RUN collection will
147 be created by appending a timestamp to the value passed with
148 --output. If this collection already exists then
149 --extend-run must be passed."""
150 ),
151 metavar="COLL",
152)
155pipeline_option = MWOptionDecorator(
156 "-p",
157 "--pipeline",
158 help="Location of a pipeline definition file in YAML format.",
159 type=MWPath(file_okay=True, dir_okay=False, readable=True),
160)
163pipeline_dot_option = MWOptionDecorator(
164 "--pipeline-dot",
165 help=unwrap(
166 """"Location for storing GraphViz DOT representation of a
167 pipeline."""
168 ),
169 type=MWPath(writable=True, file_okay=True, dir_okay=False),
170)
173profile_option = MWOptionDecorator(
174 "--profile", help="Dump cProfile statistics to file name.", type=MWPath(file_okay=True, dir_okay=False)
175)
178prune_replaced_option = MWOptionDecorator(
179 "--prune-replaced",
180 help=unwrap(
181 """Delete the datasets in the collection replaced by
182 --replace-run, either just from the datastore
183 ('unstore') or by removing them and the RUN completely
184 ('purge'). Requires --replace-run."""
185 ),
186 type=click.Choice(choices=("unstore", "purge"), case_sensitive=False),
187)
190qgraph_option = MWOptionDecorator(
191 "-g",
192 "--qgraph",
193 help=unwrap(
194 """Location for a serialized quantum graph definition (pickle
195 file). If this option is given then all input data options and
196 pipeline-building options cannot be used. Can be a URI."""
197 ),
198)
201qgraph_id_option = MWOptionDecorator(
202 "--qgraph-id",
203 help=unwrap(
204 """Quantum graph identifier, if specified must match the
205 identifier of the graph loaded from a file. Ignored if graph
206 is not loaded from a file."""
207 ),
208)
211qgraph_datastore_records_option = MWOptionDecorator(
212 "--qgraph-datastore-records",
213 help=unwrap(
214 """Include datastore records into generated quantum graph, these records are used by a
215 quantum-backed butler.
216 """
217 ),
218 is_flag=True,
219)
222# I wanted to use default=None here to match Python API but click silently
223# replaces None with an empty tuple when multiple=True.
224qgraph_node_id_option = MWOptionDecorator(
225 "--qgraph-node-id",
226 callback=split_commas,
227 multiple=True,
228 help=unwrap(
229 """Only load a specified set of nodes when graph is
230 loaded from a file, nodes are identified by UUID
231 values. One or more comma-separated integers are
232 accepted. By default all nodes are loaded. Ignored if
233 graph is not loaded from a file."""
234 ),
235)
237qgraph_header_data_option = MWOptionDecorator(
238 "--show-qgraph-header",
239 is_flag=True,
240 default=False,
241 help=unwrap(
242 """Print the headerData for Quantum Graph to the
243 console"""
244 ),
245)
247qgraph_dot_option = MWOptionDecorator(
248 "--qgraph-dot",
249 help=unwrap(
250 """Location for storing GraphViz DOT representation of a
251 quantum graph."""
252 ),
253 type=MWPath(writable=True, file_okay=True, dir_okay=False),
254)
257replace_run_option = MWOptionDecorator(
258 "--replace-run",
259 help=unwrap(
260 """Before creating a new RUN collection in an existing
261 CHAINED collection, remove the first child collection
262 (which must be of type RUN). This can be used to repeatedly
263 write to the same (parent) collection during development,
264 but it does not delete the datasets associated with the
265 replaced run unless --prune-replaced is also passed.
266 Requires --output, and incompatible with --extend-run."""
267 ),
268 is_flag=True,
269)
272save_pipeline_option = MWOptionDecorator(
273 "-s",
274 "--save-pipeline",
275 help=unwrap(
276 """Location for storing resulting pipeline definition in
277 YAML format."""
278 ),
279 type=MWPath(dir_okay=False, file_okay=True, writable=True),
280)
282save_qgraph_option = MWOptionDecorator(
283 "-q",
284 "--save-qgraph",
285 help=unwrap(
286 """URI location for storing a serialized quantum graph
287 definition (pickle file)."""
288 ),
289)
292save_single_quanta_option = MWOptionDecorator(
293 "--save-single-quanta",
294 help=unwrap(
295 """Format string of locations for storing individual
296 quantum graph definition (pickle files). The curly
297 brace {} in the input string will be replaced by a
298 quantum number. Can be a URI."""
299 ),
300)
303show_option = MWOptionDecorator(
304 "--show",
305 callback=split_commas,
306 help=unwrap(
307 """Dump various info to standard output. Possible items are:
308 `config`, `config=[Task::]<PATTERN>` or
309 `config=[Task::]<PATTERN>:NOIGNORECASE` to dump configuration
310 fields possibly matching given pattern and/or task label;
311 `history=<FIELD>` to dump configuration history for a field, field
312 name is specified as [Task::]<PATTERN>; `dump-config`,
313 `dump-config=Task` to dump complete configuration for a task given
314 its label or all tasks; `pipeline` to show pipeline composition;
315 `graph` to show information about quanta; `workflow` to show
316 information about quanta and their dependency; `tasks` to show
317 task composition; `uri` to show predicted dataset URIs of
318 quanta"""
319 ),
320 metavar="ITEM|ITEM=VALUE",
321 multiple=True,
322)
325skip_existing_in_option = MWOptionDecorator(
326 "--skip-existing-in",
327 callback=split_commas,
328 default=None,
329 metavar="COLLECTION",
330 multiple=True,
331 help=unwrap(
332 """If all Quantum outputs already exist in the specified list of
333 collections then that Quantum will be excluded from the QuantumGraph.
334 """
335 ),
336)
339skip_existing_option = MWOptionDecorator(
340 "--skip-existing",
341 is_flag=True,
342 help=unwrap(
343 """This option is equivalent to --skip-existing-in with the name of
344 the output RUN collection. If both --skip-existing-in and
345 --skip-existing are given then output RUN collection is appended to
346 the list of collections."""
347 ),
348)
351clobber_outputs_option = MWOptionDecorator(
352 "--clobber-outputs",
353 help=(
354 "Remove outputs of failed quanta from the output run when they would block the execution of new "
355 "quanta with the same data ID (or assume that this will be done, if just building a QuantumGraph). "
356 "Does nothing if --extend-run is not passed."
357 ),
358 is_flag=True,
359)
362skip_init_writes_option = MWOptionDecorator(
363 "--skip-init-writes",
364 help=unwrap(
365 """Do not write collection-wide 'init output' datasets
366 (e.g.schemas)."""
367 ),
368 is_flag=True,
369)
372enable_implicit_threading_option = MWOptionDecorator(
373 "--enable-implicit-threading",
374 help=unwrap(
375 """Do not disable implicit threading use by third-party libraries (e.g. OpenBLAS).
376 Implicit threading is always disabled during execution with multiprocessing."""
377 ),
378 is_flag=True,
379)
382task_option = MWOptionDecorator(
383 "-t",
384 "--task",
385 callback=split_commas,
386 help=unwrap(
387 """Task name to add to pipeline, must be a fully qualified task
388 name. Task name can be followed by colon and label name, if label
389 is not given then task base name (class name) is used as
390 label."""
391 ),
392 metavar="TASK[:LABEL]",
393 multiple=True,
394)
397timeout_option = MWOptionDecorator(
398 "--timeout", type=click.IntRange(min=0), help="Timeout for multiprocessing; maximum wall time (sec)."
399)
402start_method_option = MWOptionDecorator(
403 "--start-method",
404 default=None,
405 type=click.Choice(choices=["spawn", "fork", "forkserver"]),
406 help="Multiprocessing start method, default is platform-specific.",
407)
410fail_fast_option = MWOptionDecorator(
411 "--fail-fast",
412 help=unwrap(
413 """Stop processing at first error, default is to process
414 as many tasks as possible."""
415 ),
416 is_flag=True,
417)
419save_execution_butler_option = MWOptionDecorator(
420 "--save-execution-butler",
421 help=unwrap(
422 """Export location for an
423 execution-specific butler after making
424 QuantumGraph"""
425 ),
426)
428mock_option = MWOptionDecorator(
429 "--mock",
430 help=unwrap("""Mock pipeline execution."""),
431 is_flag=True,
432)
434unmocked_dataset_types_option = MWOptionDecorator(
435 "--unmocked-dataset-types",
436 callback=split_commas,
437 default=None,
438 metavar="COLLECTION",
439 multiple=True,
440 help=unwrap("""Names of input dataset types that should not be mocked."""),
441)
443clobber_execution_butler_option = MWOptionDecorator(
444 "--clobber-execution-butler",
445 help=unwrap(
446 """When creating execution butler overwrite
447 any existing products"""
448 ),
449 is_flag=True,
450)
452target_datastore_root_option = MWOptionDecorator(
453 "--target-datastore-root",
454 help=unwrap(
455 """Root directory for datastore of execution butler.
456 Default is to use the original datastore.
457 """
458 ),
459)
461dataset_query_constraint = MWOptionDecorator(
462 "--dataset-query-constraint",
463 help=unwrap(
464 """When constructing a quantum graph constrain by
465 pre-existence of specified dataset types. Valid
466 values are `all` for all inputs dataset types in
467 pipeline, `off` to not consider dataset type
468 existance as a constraint, single or comma
469 separated list of dataset type names"""
470 ),
471 default="all",
472)
474summary_option = MWOptionDecorator(
475 "--summary",
476 help=(
477 "Location for storing job summary (JSON file). Note that the"
478 " structure of this file may not be stable."
479 ),
480 type=MWPath(dir_okay=False, file_okay=True, writable=True),
481)
484recursive_option = MWOptionDecorator(
485 "--recursive",
486 is_flag=True,
487)
489config_search_path_option = MWOptionDecorator(
490 "--config-search-path",
491 callback=split_commas,
492 default=list(),
493 help="Additional search paths for butler configuration.",
494 metavar="PATH",
495 multiple=True,
496)
498update_graph_id_option = MWOptionDecorator(
499 "--update-graph-id",
500 help=unwrap("Update graph ID with new unique value."),
501 is_flag=True,
502)
504metadata_run_key_option = MWOptionDecorator(
505 "--metadata-run-key",
506 help=(
507 "Quantum graph metadata key for the name of the output run. "
508 "Empty string disables update of the metadata. "
509 "Default value: output_run."
510 ),
511 default="output_run",
512)