Coverage for python/lsst/ctrl/mpexec/cli/opt/options.py: 76%
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of ctrl_mpexec.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22import click
24from lsst.daf.butler.cli.utils import MWOptionDecorator, MWPath, split_commas, unwrap
27def _split_commas_int(context, param, values):
28 """Special callback that handles comma-separated list of integers.
30 Parameters
31 ----------
32 context : `click.Context` or `None`
33 The current execution context.
34 param : `click.core.Option` or `None`
35 The parameter being handled.
36 values : `list` [ `str` ]
37 All the values passed for this option. Strings may contain commas,
38 which will be treated as delimiters for separate values.
40 Returns
41 -------
42 numbers : `tuple` [ `int` ]
43 The passed in values separated by commas and combined into a single
44 list.
45 """
46 def _to_int(value):
47 """Convert string to integer, handle errors."""
48 try:
49 return int(value)
50 except ValueError:
51 raise click.BadParameter(f"'{value}' is not a valid integer", context, param)
53 values = split_commas(context, param, values)
54 if values is None:
55 return values
56 return tuple(_to_int(value) for value in values)
59butler_config_option = MWOptionDecorator("-b", "--butler-config",
60 help="Location of the gen3 butler/registry config file.")
63data_query_option = MWOptionDecorator("-d", "--data-query",
64 help="User data selection expression.",
65 metavar="QUERY")
68debug_option = MWOptionDecorator("--debug",
69 help="Enable debugging output using lsstDebug facility (imports debug.py).",
70 is_flag=True)
73delete_option = MWOptionDecorator("--delete",
74 callback=split_commas,
75 help="Delete task with given label from pipeline.",
76 multiple=True)
79do_raise_option = MWOptionDecorator("--do-raise",
80 help="Raise an exception on error. (else log a message and continue?)",
81 is_flag=True)
84extend_run_option = MWOptionDecorator("--extend-run",
85 help=unwrap("""Instead of creating a new RUN collection, insert datasets
86 into either the one given by --output-run (if provided) or
87 the first child collection of --output (which must be of
88 type RUN). This also enables --skip-existing option."""),
89 is_flag=True)
92graph_fixup_option = MWOptionDecorator("--graph-fixup",
93 help=unwrap("""Name of the class or factory method which makes an
94 instance used for execution graph fixup."""))
97init_only_option = MWOptionDecorator("--init-only",
98 help=unwrap("""Do not actually run; just register dataset types and/or
99 save init outputs. """),
100 is_flag=True)
103input_option = MWOptionDecorator("-i", "--input",
104 callback=split_commas,
105 default=list(),
106 help=unwrap("""Comma-separated names of the input collection(s)."""),
107 metavar="COLLECTION",
108 multiple=True)
110no_versions_option = MWOptionDecorator("--no-versions",
111 help="Do not save or check package versions.",
112 is_flag=True)
115order_pipeline_option = MWOptionDecorator("--order-pipeline",
116 help=unwrap("""Order tasks in pipeline based on their data
117 dependencies, ordering is performed as last step before saving or
118 executing pipeline."""),
119 is_flag=True)
122output_option = MWOptionDecorator("-o", "--output",
123 help=unwrap("""Name of the output CHAINED collection. This may either be an
124 existing CHAINED collection to use as both input and output
125 (incompatible with --input), or a new CHAINED collection created
126 to include all inputs (requires --input). In both cases, the
127 collection's children will start with an output RUN collection
128 that directly holds all new datasets (see --output-run)."""),
129 metavar="COLL")
132output_run_option = MWOptionDecorator("--output-run",
133 help=unwrap("""Name of the new output RUN collection. If not provided
134 then --output must be provided and a new RUN collection will
135 be created by appending a timestamp to the value passed with
136 --output. If this collection already exists then
137 --extend-run must be passed."""),
138 metavar="COLL")
141pipeline_option = MWOptionDecorator("-p", "--pipeline",
142 help="Location of a pipeline definition file in YAML format.",
143 type=MWPath(file_okay=True, dir_okay=False, readable=True))
146pipeline_dot_option = MWOptionDecorator("--pipeline-dot",
147 help=unwrap(""""Location for storing GraphViz DOT representation of a
148 pipeline."""),
149 type=MWPath(writable=True, file_okay=True, dir_okay=False))
152profile_option = MWOptionDecorator("--profile",
153 help="Dump cProfile statistics to file name.",
154 type=MWPath(file_okay=True, dir_okay=False))
157prune_replaced_option = MWOptionDecorator("--prune-replaced",
158 help=unwrap("""Delete the datasets in the collection replaced by
159 --replace-run, either just from the datastore
160 ('unstore') or by removing them and the RUN completely
161 ('purge'). Requires --replace-run."""),
162 type=click.Choice(("unstore", "purge"), case_sensitive=False))
165qgraph_option = MWOptionDecorator("-g", "--qgraph",
166 help=unwrap("""Location for a serialized quantum graph definition (pickle
167 file). If this option is given then all input data options and
168 pipeline-building options cannot be used. Can be a URI."""))
171qgraph_id_option = MWOptionDecorator("--qgraph-id",
172 help=unwrap("""Quantum graph identifier, if specified must match the
173 identifier of the graph loaded from a file. Ignored if graph
174 is not loaded from a file."""))
177# I wanted to use default=None here to match Python API but click silently
178# replaces None with an empty tuple when multiple=True.
179qgraph_node_id_option = MWOptionDecorator("--qgraph-node-id",
180 callback=split_commas,
181 multiple=True,
182 help=unwrap("""Only load a specified set of nodes when graph is
183 loaded from a file, nodes are identified by UUID
184 values. One or more comma-separated integers are
185 accepted. By default all nodes are loaded. Ignored if
186 graph is not loaded from a file."""))
188qgraph_header_data_option = MWOptionDecorator("--show-qgraph-header",
189 is_flag=True,
190 default=False,
191 help=unwrap("""Print the headerData for Quantum Graph to the
192 console"""))
194qgraph_dot_option = MWOptionDecorator("--qgraph-dot",
195 help=unwrap("""Location for storing GraphViz DOT representation of a
196 quantum graph."""),
197 type=MWPath(writable=True, file_okay=True, dir_okay=False))
200replace_run_option = MWOptionDecorator("--replace-run",
201 help=unwrap("""Before creating a new RUN collection in an existing
202 CHAINED collection, remove the first child collection
203 (which must be of type RUN). This can be used to repeatedly
204 write to the same (parent) collection during development,
205 but it does not delete the datasets associated with the
206 replaced run unless --prune-replaced is also passed.
207 Requires --output, and incompatible with --extend-run."""),
208 is_flag=True)
211save_pipeline_option = MWOptionDecorator("-s", "--save-pipeline",
212 help=unwrap("""Location for storing resulting pipeline definition in
213 YAML format."""),
214 type=MWPath(dir_okay=False, file_okay=True, writable=True))
216save_qgraph_option = MWOptionDecorator("-q", "--save-qgraph",
217 help=unwrap("""URI location for storing a serialized quantum graph
218 definition (pickle file)."""))
221save_single_quanta_option = MWOptionDecorator("--save-single-quanta",
222 help=unwrap("""Format string of locations for storing individual
223 quantum graph definition (pickle files). The curly
224 brace {} in the input string will be replaced by a
225 quantum number. Can be a URI."""))
228show_option = MWOptionDecorator("--show",
229 callback=split_commas,
230 help=unwrap("""Dump various info to standard output. Possible items are:
231 `config`, `config=[Task::]<PATTERN>` or
232 `config=[Task::]<PATTERN>:NOIGNORECASE` to dump configuration
233 fields possibly matching given pattern and/or task label;
234 `history=<FIELD>` to dump configuration history for a field, field
235 name is specified as [Task::]<PATTERN>; `dump-config`,
236 `dump-config=Task` to dump complete configuration for a task given
237 its label or all tasks; `pipeline` to show pipeline composition;
238 `graph` to show information about quanta; `workflow` to show
239 information about quanta and their dependency; `tasks` to show
240 task composition; `uri` to show predicted dataset URIs of
241 quanta"""),
242 metavar="ITEM|ITEM=VALUE",
243 multiple=True)
246skip_existing_in_option = MWOptionDecorator(
247 "--skip-existing-in",
248 callback=split_commas,
249 default=None,
250 metavar="COLLECTION",
251 multiple=True,
252 help=unwrap(
253 """If all Quantum outputs already exist in the specified list of
254 collections then that Quantum will be excluded from the QuantumGraph.
255 """
256 )
257)
260skip_existing_option = MWOptionDecorator(
261 "--skip-existing",
262 is_flag=True,
263 help=unwrap(
264 """This option is equivalent to --skip-existing-in with the name of
265 the output RUN collection. If both --skip-existing-in and
266 --skip-existing are given then output RUN collection is appended to
267 the list of collections."""
268 )
269)
272clobber_outputs_option = MWOptionDecorator("--clobber-outputs",
273 help=unwrap("""Remove outputs from previous execution of the same
274 quantum before new execution. If --skip-existing
275 is also passed, then only failed quanta will be
276 clobbered. Requires the 'run' command's --extend-run
277 flag to be set."""),
278 is_flag=True)
281skip_init_writes_option = MWOptionDecorator("--skip-init-writes",
282 help=unwrap("""Do not write collection-wide 'init output' datasets
283 (e.g.schemas)."""),
284 is_flag=True)
287task_option = MWOptionDecorator("-t", "--task",
288 callback=split_commas,
289 help=unwrap("""Task name to add to pipeline, must be a fully qualified task
290 name. Task name can be followed by colon and label name, if label
291 is not given then task base name (class name) is used as
292 label."""),
293 metavar="TASK[:LABEL]",
294 multiple=True)
297timeout_option = MWOptionDecorator("--timeout",
298 type=click.IntRange(min=0),
299 help="Timeout for multiprocessing; maximum wall time (sec).")
302start_method_option = MWOptionDecorator("--start-method",
303 default=None,
304 type=click.Choice(["spawn", "fork", "forkserver"]),
305 help="Multiprocessing start method, default is platform-specific.")
308fail_fast_option = MWOptionDecorator("--fail-fast",
309 help=unwrap("""Stop processing at first error, default is to process
310 as many tasks as possible."""),
311 is_flag=True)
313save_execution_butler_option = MWOptionDecorator("--save-execution-butler",
314 help=unwrap("""Export location for an
315 execution-specific butler after making
316 QuantumGraph"""))
318clobber_execution_butler_option = MWOptionDecorator("--clobber-execution-butler",
319 help=unwrap("""When creating execution butler overwrite
320 any existing products"""),
321 is_flag=True)
322dataset_query_constraint = MWOptionDecorator("--dataset-query-constraint",
323 help=unwrap("""When constructing a quantum graph constrain by
324 pre-existence of specified dataset types. Valid
325 values are `all` for all inputs dataset types in
326 pipeline, `off` to not consider dataset type
327 existance as a constraint, single or comma
328 separated list of dataset type names"""),
329 default='all')