Coverage for python/lsst/ctrl/mpexec/cli/opt/options.py: 100%
49 statements
« prev ^ index » next coverage.py v6.5.0, created at 2023-04-13 10:12 +0000
« prev ^ index » next coverage.py v6.5.0, created at 2023-04-13 10:12 +0000
1# This file is part of ctrl_mpexec.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22import click
23from lsst.daf.butler.cli.utils import MWOptionDecorator, MWPath, split_commas, unwrap
25butler_config_option = MWOptionDecorator(
26 "-b", "--butler-config", help="Location of the gen3 butler/registry config file."
27)
30data_query_option = MWOptionDecorator(
31 "-d", "--data-query", help="User data selection expression.", metavar="QUERY"
32)
35debug_option = MWOptionDecorator(
36 "--debug", help="Enable debugging output using lsstDebug facility (imports debug.py).", is_flag=True
37)
39coverage_option = MWOptionDecorator("--coverage", help="Enable coverage output.", is_flag=True)
41coverage_packages_option = MWOptionDecorator(
42 "--cov-packages",
43 help=unwrap(
44 """Python packages to restrict coverage to. If none are provided, runs coverage on all packages."""
45 ),
46 multiple=True,
47 callback=split_commas,
48)
50delete_option = MWOptionDecorator(
51 "--delete", callback=split_commas, help="Delete task with given label from pipeline.", multiple=True
52)
55pdb_option = MWOptionDecorator(
56 "--pdb",
57 help="Post-mortem debugger to launch for exceptions (defaults to pdb if unspecified; requires a tty).",
58 is_flag=False,
59 flag_value="pdb",
60 default=None,
61)
64extend_run_option = MWOptionDecorator(
65 "--extend-run",
66 help=unwrap(
67 """Instead of creating a new RUN collection, insert datasets
68 into either the one given by --output-run (if provided) or
69 the first child collection of --output (which must be of
70 type RUN). This also enables --skip-existing option."""
71 ),
72 is_flag=True,
73)
76graph_fixup_option = MWOptionDecorator(
77 "--graph-fixup",
78 help=unwrap(
79 """Name of the class or factory method which makes an
80 instance used for execution graph fixup."""
81 ),
82)
85init_only_option = MWOptionDecorator(
86 "--init-only",
87 help=unwrap(
88 """Do not actually run; just register dataset types and/or
89 save init outputs. """
90 ),
91 is_flag=True,
92)
95input_option = MWOptionDecorator(
96 "-i",
97 "--input",
98 callback=split_commas,
99 default=list(),
100 help=unwrap("""Comma-separated names of the input collection(s)."""),
101 metavar="COLLECTION",
102 multiple=True,
103)
105no_versions_option = MWOptionDecorator(
106 "--no-versions", help="Do not save or check package versions.", is_flag=True
107)
110order_pipeline_option = MWOptionDecorator(
111 "--order-pipeline",
112 help=unwrap(
113 """Order tasks in pipeline based on their data
114 dependencies, ordering is performed as last step before saving or
115 executing pipeline."""
116 ),
117 is_flag=True,
118)
121output_option = MWOptionDecorator(
122 "-o",
123 "--output",
124 help=unwrap(
125 """Name of the output CHAINED collection. This may either be an
126 existing CHAINED collection to use as both input and output
127 (incompatible with --input), or a new CHAINED collection created
128 to include all inputs (requires --input). In both cases, the
129 collection's children will start with an output RUN collection
130 that directly holds all new datasets (see --output-run)."""
131 ),
132 metavar="COLL",
133)
136output_run_option = MWOptionDecorator(
137 "--output-run",
138 help=unwrap(
139 """Name of the new output RUN collection. If not provided
140 then --output must be provided and a new RUN collection will
141 be created by appending a timestamp to the value passed with
142 --output. If this collection already exists then
143 --extend-run must be passed."""
144 ),
145 metavar="COLL",
146)
149pipeline_option = MWOptionDecorator(
150 "-p",
151 "--pipeline",
152 help="Location of a pipeline definition file in YAML format.",
153 type=MWPath(file_okay=True, dir_okay=False, readable=True),
154)
157pipeline_dot_option = MWOptionDecorator(
158 "--pipeline-dot",
159 help=unwrap(
160 """"Location for storing GraphViz DOT representation of a
161 pipeline."""
162 ),
163 type=MWPath(writable=True, file_okay=True, dir_okay=False),
164)
167profile_option = MWOptionDecorator(
168 "--profile", help="Dump cProfile statistics to file name.", type=MWPath(file_okay=True, dir_okay=False)
169)
172prune_replaced_option = MWOptionDecorator(
173 "--prune-replaced",
174 help=unwrap(
175 """Delete the datasets in the collection replaced by
176 --replace-run, either just from the datastore
177 ('unstore') or by removing them and the RUN completely
178 ('purge'). Requires --replace-run."""
179 ),
180 type=click.Choice(choices=("unstore", "purge"), case_sensitive=False),
181)
184qgraph_option = MWOptionDecorator(
185 "-g",
186 "--qgraph",
187 help=unwrap(
188 """Location for a serialized quantum graph definition (pickle
189 file). If this option is given then all input data options and
190 pipeline-building options cannot be used. Can be a URI."""
191 ),
192)
195qgraph_id_option = MWOptionDecorator(
196 "--qgraph-id",
197 help=unwrap(
198 """Quantum graph identifier, if specified must match the
199 identifier of the graph loaded from a file. Ignored if graph
200 is not loaded from a file."""
201 ),
202)
205qgraph_datastore_records_option = MWOptionDecorator(
206 "--qgraph-datastore-records",
207 help=unwrap(
208 """Include datastore records into generated quantum graph, these records are used by a
209 quantum-backed butler.
210 """
211 ),
212 is_flag=True,
213)
216# I wanted to use default=None here to match Python API but click silently
217# replaces None with an empty tuple when multiple=True.
218qgraph_node_id_option = MWOptionDecorator(
219 "--qgraph-node-id",
220 callback=split_commas,
221 multiple=True,
222 help=unwrap(
223 """Only load a specified set of nodes when graph is
224 loaded from a file, nodes are identified by UUID
225 values. One or more comma-separated integers are
226 accepted. By default all nodes are loaded. Ignored if
227 graph is not loaded from a file."""
228 ),
229)
231qgraph_header_data_option = MWOptionDecorator(
232 "--show-qgraph-header",
233 is_flag=True,
234 default=False,
235 help=unwrap(
236 """Print the headerData for Quantum Graph to the
237 console"""
238 ),
239)
241qgraph_dot_option = MWOptionDecorator(
242 "--qgraph-dot",
243 help=unwrap(
244 """Location for storing GraphViz DOT representation of a
245 quantum graph."""
246 ),
247 type=MWPath(writable=True, file_okay=True, dir_okay=False),
248)
251replace_run_option = MWOptionDecorator(
252 "--replace-run",
253 help=unwrap(
254 """Before creating a new RUN collection in an existing
255 CHAINED collection, remove the first child collection
256 (which must be of type RUN). This can be used to repeatedly
257 write to the same (parent) collection during development,
258 but it does not delete the datasets associated with the
259 replaced run unless --prune-replaced is also passed.
260 Requires --output, and incompatible with --extend-run."""
261 ),
262 is_flag=True,
263)
266save_pipeline_option = MWOptionDecorator(
267 "-s",
268 "--save-pipeline",
269 help=unwrap(
270 """Location for storing resulting pipeline definition in
271 YAML format."""
272 ),
273 type=MWPath(dir_okay=False, file_okay=True, writable=True),
274)
276save_qgraph_option = MWOptionDecorator(
277 "-q",
278 "--save-qgraph",
279 help=unwrap(
280 """URI location for storing a serialized quantum graph
281 definition (pickle file)."""
282 ),
283)
286save_single_quanta_option = MWOptionDecorator(
287 "--save-single-quanta",
288 help=unwrap(
289 """Format string of locations for storing individual
290 quantum graph definition (pickle files). The curly
291 brace {} in the input string will be replaced by a
292 quantum number. Can be a URI."""
293 ),
294)
297show_option = MWOptionDecorator(
298 "--show",
299 callback=split_commas,
300 help=unwrap(
301 """Dump various info to standard output. Possible items are:
302 `config`, `config=[Task::]<PATTERN>` or
303 `config=[Task::]<PATTERN>:NOIGNORECASE` to dump configuration
304 fields possibly matching given pattern and/or task label;
305 `history=<FIELD>` to dump configuration history for a field, field
306 name is specified as [Task::]<PATTERN>; `dump-config`,
307 `dump-config=Task` to dump complete configuration for a task given
308 its label or all tasks; `pipeline` to show pipeline composition;
309 `graph` to show information about quanta; `workflow` to show
310 information about quanta and their dependency; `tasks` to show
311 task composition; `uri` to show predicted dataset URIs of
312 quanta"""
313 ),
314 metavar="ITEM|ITEM=VALUE",
315 multiple=True,
316)
319skip_existing_in_option = MWOptionDecorator(
320 "--skip-existing-in",
321 callback=split_commas,
322 default=None,
323 metavar="COLLECTION",
324 multiple=True,
325 help=unwrap(
326 """If all Quantum outputs already exist in the specified list of
327 collections then that Quantum will be excluded from the QuantumGraph.
328 """
329 ),
330)
333skip_existing_option = MWOptionDecorator(
334 "--skip-existing",
335 is_flag=True,
336 help=unwrap(
337 """This option is equivalent to --skip-existing-in with the name of
338 the output RUN collection. If both --skip-existing-in and
339 --skip-existing are given then output RUN collection is appended to
340 the list of collections."""
341 ),
342)
345clobber_outputs_option = MWOptionDecorator(
346 "--clobber-outputs",
347 help=unwrap(
348 """Remove outputs from previous execution of the same
349 quantum before new execution. If --skip-existing
350 is also passed, then only failed quanta will be
351 clobbered. Requires the 'run' command's --extend-run
352 flag to be set."""
353 ),
354 is_flag=True,
355)
358skip_init_writes_option = MWOptionDecorator(
359 "--skip-init-writes",
360 help=unwrap(
361 """Do not write collection-wide 'init output' datasets
362 (e.g.schemas)."""
363 ),
364 is_flag=True,
365)
368enable_implicit_threading_option = MWOptionDecorator(
369 "--enable-implicit-threading",
370 help=unwrap(
371 """Do not disable implicit threading use by third-party libraries (e.g. OpenBLAS).
372 Implicit threading is always disabled during execution with multiprocessing."""
373 ),
374 is_flag=True,
375)
378task_option = MWOptionDecorator(
379 "-t",
380 "--task",
381 callback=split_commas,
382 help=unwrap(
383 """Task name to add to pipeline, must be a fully qualified task
384 name. Task name can be followed by colon and label name, if label
385 is not given then task base name (class name) is used as
386 label."""
387 ),
388 metavar="TASK[:LABEL]",
389 multiple=True,
390)
393timeout_option = MWOptionDecorator(
394 "--timeout", type=click.IntRange(min=0), help="Timeout for multiprocessing; maximum wall time (sec)."
395)
398start_method_option = MWOptionDecorator(
399 "--start-method",
400 default=None,
401 type=click.Choice(choices=["spawn", "fork", "forkserver"]),
402 help="Multiprocessing start method, default is platform-specific.",
403)
406fail_fast_option = MWOptionDecorator(
407 "--fail-fast",
408 help=unwrap(
409 """Stop processing at first error, default is to process
410 as many tasks as possible."""
411 ),
412 is_flag=True,
413)
415save_execution_butler_option = MWOptionDecorator(
416 "--save-execution-butler",
417 help=unwrap(
418 """Export location for an
419 execution-specific butler after making
420 QuantumGraph"""
421 ),
422)
424mock_option = MWOptionDecorator(
425 "--mock",
426 help=unwrap("""Mock pipeline execution."""),
427 is_flag=True,
428)
430clobber_execution_butler_option = MWOptionDecorator(
431 "--clobber-execution-butler",
432 help=unwrap(
433 """When creating execution butler overwrite
434 any existing products"""
435 ),
436 is_flag=True,
437)
439target_datastore_root_option = MWOptionDecorator(
440 "--target-datastore-root",
441 help=unwrap(
442 """Root directory for datastore of execution butler.
443 Default is to use the original datastore.
444 """
445 ),
446)
448dataset_query_constraint = MWOptionDecorator(
449 "--dataset-query-constraint",
450 help=unwrap(
451 """When constructing a quantum graph constrain by
452 pre-existence of specified dataset types. Valid
453 values are `all` for all inputs dataset types in
454 pipeline, `off` to not consider dataset type
455 existance as a constraint, single or comma
456 separated list of dataset type names"""
457 ),
458 default="all",
459)
461summary_option = MWOptionDecorator(
462 "--summary",
463 help=(
464 "Location for storing job summary (JSON file). Note that the"
465 " structure of this file may not be stable."
466 ),
467 type=MWPath(dir_okay=False, file_okay=True, writable=True),
468)
471recursive_option = MWOptionDecorator(
472 "--recursive",
473 is_flag=True,
474)
476config_search_path_option = MWOptionDecorator(
477 "--config-search-path",
478 callback=split_commas,
479 default=list(),
480 help="Additional search paths for butler configuration.",
481 metavar="PATH",
482 multiple=True,
483)