Coverage for python/lsst/ctrl/mpexec/cli/opt/options.py: 100%
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of ctrl_mpexec.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22import click
23from lsst.daf.butler.cli.utils import MWOptionDecorator, MWPath, split_commas, unwrap
25butler_config_option = MWOptionDecorator(
26 "-b", "--butler-config", help="Location of the gen3 butler/registry config file."
27)
30data_query_option = MWOptionDecorator(
31 "-d", "--data-query", help="User data selection expression.", metavar="QUERY"
32)
35debug_option = MWOptionDecorator(
36 "--debug", help="Enable debugging output using lsstDebug facility (imports debug.py).", is_flag=True
37)
40delete_option = MWOptionDecorator(
41 "--delete", callback=split_commas, help="Delete task with given label from pipeline.", multiple=True
42)
45pdb_option = MWOptionDecorator(
46 "--pdb",
47 help="Post-mortem debugger to launch for exceptions (defaults to pdb if unspecified; requires a tty).",
48 is_flag=False,
49 flag_value="pdb",
50 default=None,
51)
54extend_run_option = MWOptionDecorator(
55 "--extend-run",
56 help=unwrap(
57 """Instead of creating a new RUN collection, insert datasets
58 into either the one given by --output-run (if provided) or
59 the first child collection of --output (which must be of
60 type RUN). This also enables --skip-existing option."""
61 ),
62 is_flag=True,
63)
66graph_fixup_option = MWOptionDecorator(
67 "--graph-fixup",
68 help=unwrap(
69 """Name of the class or factory method which makes an
70 instance used for execution graph fixup."""
71 ),
72)
75init_only_option = MWOptionDecorator(
76 "--init-only",
77 help=unwrap(
78 """Do not actually run; just register dataset types and/or
79 save init outputs. """
80 ),
81 is_flag=True,
82)
85input_option = MWOptionDecorator(
86 "-i",
87 "--input",
88 callback=split_commas,
89 default=list(),
90 help=unwrap("""Comma-separated names of the input collection(s)."""),
91 metavar="COLLECTION",
92 multiple=True,
93)
95no_versions_option = MWOptionDecorator(
96 "--no-versions", help="Do not save or check package versions.", is_flag=True
97)
100order_pipeline_option = MWOptionDecorator(
101 "--order-pipeline",
102 help=unwrap(
103 """Order tasks in pipeline based on their data
104 dependencies, ordering is performed as last step before saving or
105 executing pipeline."""
106 ),
107 is_flag=True,
108)
111output_option = MWOptionDecorator(
112 "-o",
113 "--output",
114 help=unwrap(
115 """Name of the output CHAINED collection. This may either be an
116 existing CHAINED collection to use as both input and output
117 (incompatible with --input), or a new CHAINED collection created
118 to include all inputs (requires --input). In both cases, the
119 collection's children will start with an output RUN collection
120 that directly holds all new datasets (see --output-run)."""
121 ),
122 metavar="COLL",
123)
126output_run_option = MWOptionDecorator(
127 "--output-run",
128 help=unwrap(
129 """Name of the new output RUN collection. If not provided
130 then --output must be provided and a new RUN collection will
131 be created by appending a timestamp to the value passed with
132 --output. If this collection already exists then
133 --extend-run must be passed."""
134 ),
135 metavar="COLL",
136)
139pipeline_option = MWOptionDecorator(
140 "-p",
141 "--pipeline",
142 help="Location of a pipeline definition file in YAML format.",
143 type=MWPath(file_okay=True, dir_okay=False, readable=True),
144)
147pipeline_dot_option = MWOptionDecorator(
148 "--pipeline-dot",
149 help=unwrap(
150 """"Location for storing GraphViz DOT representation of a
151 pipeline."""
152 ),
153 type=MWPath(writable=True, file_okay=True, dir_okay=False),
154)
157profile_option = MWOptionDecorator(
158 "--profile", help="Dump cProfile statistics to file name.", type=MWPath(file_okay=True, dir_okay=False)
159)
162prune_replaced_option = MWOptionDecorator(
163 "--prune-replaced",
164 help=unwrap(
165 """Delete the datasets in the collection replaced by
166 --replace-run, either just from the datastore
167 ('unstore') or by removing them and the RUN completely
168 ('purge'). Requires --replace-run."""
169 ),
170 type=click.Choice(choices=("unstore", "purge"), case_sensitive=False),
171)
174qgraph_option = MWOptionDecorator(
175 "-g",
176 "--qgraph",
177 help=unwrap(
178 """Location for a serialized quantum graph definition (pickle
179 file). If this option is given then all input data options and
180 pipeline-building options cannot be used. Can be a URI."""
181 ),
182)
185qgraph_id_option = MWOptionDecorator(
186 "--qgraph-id",
187 help=unwrap(
188 """Quantum graph identifier, if specified must match the
189 identifier of the graph loaded from a file. Ignored if graph
190 is not loaded from a file."""
191 ),
192)
195# I wanted to use default=None here to match Python API but click silently
196# replaces None with an empty tuple when multiple=True.
197qgraph_node_id_option = MWOptionDecorator(
198 "--qgraph-node-id",
199 callback=split_commas,
200 multiple=True,
201 help=unwrap(
202 """Only load a specified set of nodes when graph is
203 loaded from a file, nodes are identified by UUID
204 values. One or more comma-separated integers are
205 accepted. By default all nodes are loaded. Ignored if
206 graph is not loaded from a file."""
207 ),
208)
210qgraph_header_data_option = MWOptionDecorator(
211 "--show-qgraph-header",
212 is_flag=True,
213 default=False,
214 help=unwrap(
215 """Print the headerData for Quantum Graph to the
216 console"""
217 ),
218)
220qgraph_dot_option = MWOptionDecorator(
221 "--qgraph-dot",
222 help=unwrap(
223 """Location for storing GraphViz DOT representation of a
224 quantum graph."""
225 ),
226 type=MWPath(writable=True, file_okay=True, dir_okay=False),
227)
230replace_run_option = MWOptionDecorator(
231 "--replace-run",
232 help=unwrap(
233 """Before creating a new RUN collection in an existing
234 CHAINED collection, remove the first child collection
235 (which must be of type RUN). This can be used to repeatedly
236 write to the same (parent) collection during development,
237 but it does not delete the datasets associated with the
238 replaced run unless --prune-replaced is also passed.
239 Requires --output, and incompatible with --extend-run."""
240 ),
241 is_flag=True,
242)
245save_pipeline_option = MWOptionDecorator(
246 "-s",
247 "--save-pipeline",
248 help=unwrap(
249 """Location for storing resulting pipeline definition in
250 YAML format."""
251 ),
252 type=MWPath(dir_okay=False, file_okay=True, writable=True),
253)
255save_qgraph_option = MWOptionDecorator(
256 "-q",
257 "--save-qgraph",
258 help=unwrap(
259 """URI location for storing a serialized quantum graph
260 definition (pickle file)."""
261 ),
262)
265save_single_quanta_option = MWOptionDecorator(
266 "--save-single-quanta",
267 help=unwrap(
268 """Format string of locations for storing individual
269 quantum graph definition (pickle files). The curly
270 brace {} in the input string will be replaced by a
271 quantum number. Can be a URI."""
272 ),
273)
276show_option = MWOptionDecorator(
277 "--show",
278 callback=split_commas,
279 help=unwrap(
280 """Dump various info to standard output. Possible items are:
281 `config`, `config=[Task::]<PATTERN>` or
282 `config=[Task::]<PATTERN>:NOIGNORECASE` to dump configuration
283 fields possibly matching given pattern and/or task label;
284 `history=<FIELD>` to dump configuration history for a field, field
285 name is specified as [Task::]<PATTERN>; `dump-config`,
286 `dump-config=Task` to dump complete configuration for a task given
287 its label or all tasks; `pipeline` to show pipeline composition;
288 `graph` to show information about quanta; `workflow` to show
289 information about quanta and their dependency; `tasks` to show
290 task composition; `uri` to show predicted dataset URIs of
291 quanta"""
292 ),
293 metavar="ITEM|ITEM=VALUE",
294 multiple=True,
295)
298skip_existing_in_option = MWOptionDecorator(
299 "--skip-existing-in",
300 callback=split_commas,
301 default=None,
302 metavar="COLLECTION",
303 multiple=True,
304 help=unwrap(
305 """If all Quantum outputs already exist in the specified list of
306 collections then that Quantum will be excluded from the QuantumGraph.
307 """
308 ),
309)
312skip_existing_option = MWOptionDecorator(
313 "--skip-existing",
314 is_flag=True,
315 help=unwrap(
316 """This option is equivalent to --skip-existing-in with the name of
317 the output RUN collection. If both --skip-existing-in and
318 --skip-existing are given then output RUN collection is appended to
319 the list of collections."""
320 ),
321)
324clobber_outputs_option = MWOptionDecorator(
325 "--clobber-outputs",
326 help=unwrap(
327 """Remove outputs from previous execution of the same
328 quantum before new execution. If --skip-existing
329 is also passed, then only failed quanta will be
330 clobbered. Requires the 'run' command's --extend-run
331 flag to be set."""
332 ),
333 is_flag=True,
334)
337skip_init_writes_option = MWOptionDecorator(
338 "--skip-init-writes",
339 help=unwrap(
340 """Do not write collection-wide 'init output' datasets
341 (e.g.schemas)."""
342 ),
343 is_flag=True,
344)
347task_option = MWOptionDecorator(
348 "-t",
349 "--task",
350 callback=split_commas,
351 help=unwrap(
352 """Task name to add to pipeline, must be a fully qualified task
353 name. Task name can be followed by colon and label name, if label
354 is not given then task base name (class name) is used as
355 label."""
356 ),
357 metavar="TASK[:LABEL]",
358 multiple=True,
359)
362timeout_option = MWOptionDecorator(
363 "--timeout", type=click.IntRange(min=0), help="Timeout for multiprocessing; maximum wall time (sec)."
364)
367start_method_option = MWOptionDecorator(
368 "--start-method",
369 default=None,
370 type=click.Choice(choices=["spawn", "fork", "forkserver"]),
371 help="Multiprocessing start method, default is platform-specific.",
372)
375fail_fast_option = MWOptionDecorator(
376 "--fail-fast",
377 help=unwrap(
378 """Stop processing at first error, default is to process
379 as many tasks as possible."""
380 ),
381 is_flag=True,
382)
384save_execution_butler_option = MWOptionDecorator(
385 "--save-execution-butler",
386 help=unwrap(
387 """Export location for an
388 execution-specific butler after making
389 QuantumGraph"""
390 ),
391)
393mock_option = MWOptionDecorator(
394 "--mock",
395 help=unwrap("""Mock pipeline execution."""),
396 is_flag=True,
397)
399clobber_execution_butler_option = MWOptionDecorator(
400 "--clobber-execution-butler",
401 help=unwrap(
402 """When creating execution butler overwrite
403 any existing products"""
404 ),
405 is_flag=True,
406)
407dataset_query_constraint = MWOptionDecorator(
408 "--dataset-query-constraint",
409 help=unwrap(
410 """When constructing a quantum graph constrain by
411 pre-existence of specified dataset types. Valid
412 values are `all` for all inputs dataset types in
413 pipeline, `off` to not consider dataset type
414 existance as a constraint, single or comma
415 separated list of dataset type names"""
416 ),
417 default="all",
418)
420summary_option = MWOptionDecorator(
421 "--summary",
422 help=(
423 "Location for storing job summary (JSON file). Note that the"
424 " structure of this file may not be stable."
425 ),
426 type=MWPath(dir_okay=False, file_okay=True, writable=True),
427)