Coverage for python/lsst/ctrl/mpexec/cli/opt/options.py: 76%
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of ctrl_mpexec.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22import click
23from lsst.daf.butler.cli.utils import MWOptionDecorator, MWPath, split_commas, unwrap
26def _split_commas_int(context, param, values):
27 """Special callback that handles comma-separated list of integers.
29 Parameters
30 ----------
31 context : `click.Context` or `None`
32 The current execution context.
33 param : `click.core.Option` or `None`
34 The parameter being handled.
35 values : `list` [ `str` ]
36 All the values passed for this option. Strings may contain commas,
37 which will be treated as delimiters for separate values.
39 Returns
40 -------
41 numbers : `tuple` [ `int` ]
42 The passed in values separated by commas and combined into a single
43 list.
44 """
46 def _to_int(value):
47 """Convert string to integer, handle errors."""
48 try:
49 return int(value)
50 except ValueError:
51 raise click.BadParameter(f"'{value}' is not a valid integer", context, param)
53 values = split_commas(context, param, values)
54 if values is None:
55 return values
56 return tuple(_to_int(value) for value in values)
59butler_config_option = MWOptionDecorator(
60 "-b", "--butler-config", help="Location of the gen3 butler/registry config file."
61)
64data_query_option = MWOptionDecorator(
65 "-d", "--data-query", help="User data selection expression.", metavar="QUERY"
66)
69debug_option = MWOptionDecorator(
70 "--debug", help="Enable debugging output using lsstDebug facility (imports debug.py).", is_flag=True
71)
74delete_option = MWOptionDecorator(
75 "--delete", callback=split_commas, help="Delete task with given label from pipeline.", multiple=True
76)
79do_raise_option = MWOptionDecorator(
80 "--do-raise", help="Raise an exception on error. (else log a message and continue?)", is_flag=True
81)
84extend_run_option = MWOptionDecorator(
85 "--extend-run",
86 help=unwrap(
87 """Instead of creating a new RUN collection, insert datasets
88 into either the one given by --output-run (if provided) or
89 the first child collection of --output (which must be of
90 type RUN). This also enables --skip-existing option."""
91 ),
92 is_flag=True,
93)
96graph_fixup_option = MWOptionDecorator(
97 "--graph-fixup",
98 help=unwrap(
99 """Name of the class or factory method which makes an
100 instance used for execution graph fixup."""
101 ),
102)
105init_only_option = MWOptionDecorator(
106 "--init-only",
107 help=unwrap(
108 """Do not actually run; just register dataset types and/or
109 save init outputs. """
110 ),
111 is_flag=True,
112)
115input_option = MWOptionDecorator(
116 "-i",
117 "--input",
118 callback=split_commas,
119 default=list(),
120 help=unwrap("""Comma-separated names of the input collection(s)."""),
121 metavar="COLLECTION",
122 multiple=True,
123)
125no_versions_option = MWOptionDecorator(
126 "--no-versions", help="Do not save or check package versions.", is_flag=True
127)
130order_pipeline_option = MWOptionDecorator(
131 "--order-pipeline",
132 help=unwrap(
133 """Order tasks in pipeline based on their data
134 dependencies, ordering is performed as last step before saving or
135 executing pipeline."""
136 ),
137 is_flag=True,
138)
141output_option = MWOptionDecorator(
142 "-o",
143 "--output",
144 help=unwrap(
145 """Name of the output CHAINED collection. This may either be an
146 existing CHAINED collection to use as both input and output
147 (incompatible with --input), or a new CHAINED collection created
148 to include all inputs (requires --input). In both cases, the
149 collection's children will start with an output RUN collection
150 that directly holds all new datasets (see --output-run)."""
151 ),
152 metavar="COLL",
153)
156output_run_option = MWOptionDecorator(
157 "--output-run",
158 help=unwrap(
159 """Name of the new output RUN collection. If not provided
160 then --output must be provided and a new RUN collection will
161 be created by appending a timestamp to the value passed with
162 --output. If this collection already exists then
163 --extend-run must be passed."""
164 ),
165 metavar="COLL",
166)
169pipeline_option = MWOptionDecorator(
170 "-p",
171 "--pipeline",
172 help="Location of a pipeline definition file in YAML format.",
173 type=MWPath(file_okay=True, dir_okay=False, readable=True),
174)
177pipeline_dot_option = MWOptionDecorator(
178 "--pipeline-dot",
179 help=unwrap(
180 """"Location for storing GraphViz DOT representation of a
181 pipeline."""
182 ),
183 type=MWPath(writable=True, file_okay=True, dir_okay=False),
184)
187profile_option = MWOptionDecorator(
188 "--profile", help="Dump cProfile statistics to file name.", type=MWPath(file_okay=True, dir_okay=False)
189)
192prune_replaced_option = MWOptionDecorator(
193 "--prune-replaced",
194 help=unwrap(
195 """Delete the datasets in the collection replaced by
196 --replace-run, either just from the datastore
197 ('unstore') or by removing them and the RUN completely
198 ('purge'). Requires --replace-run."""
199 ),
200 type=click.Choice(("unstore", "purge"), case_sensitive=False),
201)
204qgraph_option = MWOptionDecorator(
205 "-g",
206 "--qgraph",
207 help=unwrap(
208 """Location for a serialized quantum graph definition (pickle
209 file). If this option is given then all input data options and
210 pipeline-building options cannot be used. Can be a URI."""
211 ),
212)
215qgraph_id_option = MWOptionDecorator(
216 "--qgraph-id",
217 help=unwrap(
218 """Quantum graph identifier, if specified must match the
219 identifier of the graph loaded from a file. Ignored if graph
220 is not loaded from a file."""
221 ),
222)
225# I wanted to use default=None here to match Python API but click silently
226# replaces None with an empty tuple when multiple=True.
227qgraph_node_id_option = MWOptionDecorator(
228 "--qgraph-node-id",
229 callback=split_commas,
230 multiple=True,
231 help=unwrap(
232 """Only load a specified set of nodes when graph is
233 loaded from a file, nodes are identified by UUID
234 values. One or more comma-separated integers are
235 accepted. By default all nodes are loaded. Ignored if
236 graph is not loaded from a file."""
237 ),
238)
240qgraph_header_data_option = MWOptionDecorator(
241 "--show-qgraph-header",
242 is_flag=True,
243 default=False,
244 help=unwrap(
245 """Print the headerData for Quantum Graph to the
246 console"""
247 ),
248)
250qgraph_dot_option = MWOptionDecorator(
251 "--qgraph-dot",
252 help=unwrap(
253 """Location for storing GraphViz DOT representation of a
254 quantum graph."""
255 ),
256 type=MWPath(writable=True, file_okay=True, dir_okay=False),
257)
260replace_run_option = MWOptionDecorator(
261 "--replace-run",
262 help=unwrap(
263 """Before creating a new RUN collection in an existing
264 CHAINED collection, remove the first child collection
265 (which must be of type RUN). This can be used to repeatedly
266 write to the same (parent) collection during development,
267 but it does not delete the datasets associated with the
268 replaced run unless --prune-replaced is also passed.
269 Requires --output, and incompatible with --extend-run."""
270 ),
271 is_flag=True,
272)
275save_pipeline_option = MWOptionDecorator(
276 "-s",
277 "--save-pipeline",
278 help=unwrap(
279 """Location for storing resulting pipeline definition in
280 YAML format."""
281 ),
282 type=MWPath(dir_okay=False, file_okay=True, writable=True),
283)
285save_qgraph_option = MWOptionDecorator(
286 "-q",
287 "--save-qgraph",
288 help=unwrap(
289 """URI location for storing a serialized quantum graph
290 definition (pickle file)."""
291 ),
292)
295save_single_quanta_option = MWOptionDecorator(
296 "--save-single-quanta",
297 help=unwrap(
298 """Format string of locations for storing individual
299 quantum graph definition (pickle files). The curly
300 brace {} in the input string will be replaced by a
301 quantum number. Can be a URI."""
302 ),
303)
306show_option = MWOptionDecorator(
307 "--show",
308 callback=split_commas,
309 help=unwrap(
310 """Dump various info to standard output. Possible items are:
311 `config`, `config=[Task::]<PATTERN>` or
312 `config=[Task::]<PATTERN>:NOIGNORECASE` to dump configuration
313 fields possibly matching given pattern and/or task label;
314 `history=<FIELD>` to dump configuration history for a field, field
315 name is specified as [Task::]<PATTERN>; `dump-config`,
316 `dump-config=Task` to dump complete configuration for a task given
317 its label or all tasks; `pipeline` to show pipeline composition;
318 `graph` to show information about quanta; `workflow` to show
319 information about quanta and their dependency; `tasks` to show
320 task composition; `uri` to show predicted dataset URIs of
321 quanta"""
322 ),
323 metavar="ITEM|ITEM=VALUE",
324 multiple=True,
325)
328skip_existing_in_option = MWOptionDecorator(
329 "--skip-existing-in",
330 callback=split_commas,
331 default=None,
332 metavar="COLLECTION",
333 multiple=True,
334 help=unwrap(
335 """If all Quantum outputs already exist in the specified list of
336 collections then that Quantum will be excluded from the QuantumGraph.
337 """
338 ),
339)
342skip_existing_option = MWOptionDecorator(
343 "--skip-existing",
344 is_flag=True,
345 help=unwrap(
346 """This option is equivalent to --skip-existing-in with the name of
347 the output RUN collection. If both --skip-existing-in and
348 --skip-existing are given then output RUN collection is appended to
349 the list of collections."""
350 ),
351)
354clobber_outputs_option = MWOptionDecorator(
355 "--clobber-outputs",
356 help=unwrap(
357 """Remove outputs from previous execution of the same
358 quantum before new execution. If --skip-existing
359 is also passed, then only failed quanta will be
360 clobbered. Requires the 'run' command's --extend-run
361 flag to be set."""
362 ),
363 is_flag=True,
364)
367skip_init_writes_option = MWOptionDecorator(
368 "--skip-init-writes",
369 help=unwrap(
370 """Do not write collection-wide 'init output' datasets
371 (e.g.schemas)."""
372 ),
373 is_flag=True,
374)
377task_option = MWOptionDecorator(
378 "-t",
379 "--task",
380 callback=split_commas,
381 help=unwrap(
382 """Task name to add to pipeline, must be a fully qualified task
383 name. Task name can be followed by colon and label name, if label
384 is not given then task base name (class name) is used as
385 label."""
386 ),
387 metavar="TASK[:LABEL]",
388 multiple=True,
389)
392timeout_option = MWOptionDecorator(
393 "--timeout", type=click.IntRange(min=0), help="Timeout for multiprocessing; maximum wall time (sec)."
394)
397start_method_option = MWOptionDecorator(
398 "--start-method",
399 default=None,
400 type=click.Choice(["spawn", "fork", "forkserver"]),
401 help="Multiprocessing start method, default is platform-specific.",
402)
405fail_fast_option = MWOptionDecorator(
406 "--fail-fast",
407 help=unwrap(
408 """Stop processing at first error, default is to process
409 as many tasks as possible."""
410 ),
411 is_flag=True,
412)
414save_execution_butler_option = MWOptionDecorator(
415 "--save-execution-butler",
416 help=unwrap(
417 """Export location for an
418 execution-specific butler after making
419 QuantumGraph"""
420 ),
421)
423mock_option = MWOptionDecorator(
424 "--mock",
425 help=unwrap("""Mock pipeline execution."""),
426 is_flag=True,
427)
429clobber_execution_butler_option = MWOptionDecorator(
430 "--clobber-execution-butler",
431 help=unwrap(
432 """When creating execution butler overwrite
433 any existing products"""
434 ),
435 is_flag=True,
436)
437dataset_query_constraint = MWOptionDecorator(
438 "--dataset-query-constraint",
439 help=unwrap(
440 """When constructing a quantum graph constrain by
441 pre-existence of specified dataset types. Valid
442 values are `all` for all inputs dataset types in
443 pipeline, `off` to not consider dataset type
444 existance as a constraint, single or comma
445 separated list of dataset type names"""
446 ),
447 default="all",
448)