Coverage for python/lsst/ctrl/mpexec/cli/script/run.py : 43%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of ctrl_mpexec.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22import logging
23from types import SimpleNamespace
25from lsst.daf.butler.cli.cliLog import CliLog
26from ... import CmdLineFwk, TaskFactory
28_log = logging.getLogger(__name__.partition(".")[2])
31def run(do_raise,
32 graph_fixup,
33 init_only,
34 log_level,
35 no_versions,
36 processes,
37 profile,
38 qgraphObj,
39 register_dataset_types,
40 skip_init_writes,
41 timeout,
42 butler_config,
43 input,
44 output,
45 output_run,
46 extend_run,
47 replace_run,
48 prune_replaced,
49 data_query,
50 skip_existing,
51 debug,
52 fail_fast,
53 clobber_partial_outputs,
54 **kwargs):
55 """Implements the command line interface `pipetask run` subcommand, should
56 only be called by command line tools and unit test code that test this
57 function.
59 Parameters
60 ----------
61 do_raise : `bool`
62 Raise an exception in the case of an error.
63 graph_fixup : `str`
64 The name of the class or factory method which makes an instance used
65 for execution graph fixup.
66 init_only : `bool`
67 If true, do not actually run; just register dataset types and/or save
68 init outputs.
69 log_level : `list` of `tuple`
70 per-component logging levels, each item in the list is a tuple
71 (component, level), `component` is a logger name or an empty string
72 or `None` for root logger, `level` is a logging level name, one of
73 CRITICAL, ERROR, WARNING, INFO, DEBUG (case insensitive).
74 no_versions : `bool`
75 If true, do not save or check package versions.
76 processes : `int`
77 The number of processes to use.
78 profile : `str`
79 File name to dump cProfile information to.
80 qgraphObj : `lsst.pipe.base.QuantumGraph`
81 A QuantumGraph generated by a previous subcommand.
82 register_dataset_types : `bool`
83 If true, register DatasetTypes that do not already exist in the Registry.
84 skip_init_writes : `bool`
85 If true, do not write collection-wide 'init output' datasets (e.g.
86 schemas).
87 timeout : `int`
88 Timeout for multiprocessing; maximum wall time (sec).
89 butler_config : `str`, `dict`, or `lsst.daf.butler.Config`
90 If `str`, `butler_config` is the path location of the gen3
91 butler/registry config file. If `dict`, `butler_config` is key value
92 pairs used to init or update the `lsst.daf.butler.Config` instance. If
93 `Config`, it is the object used to configure a Butler.
94 input : `str`
95 Comma-separated names of the input collection(s). Entries may include a
96 colon (:), the first string is a dataset type name that restricts the
97 search in that collection.
98 output : `str`
99 Name of the output CHAINED collection. This may either be an existing
100 CHAINED collection to use as both input and output (if `input` is
101 `None`), or a new CHAINED collection created to include all inputs
102 (if `input` is not `None`). In both cases, the collection's children
103 will start with an output RUN collection that directly holds all new
104 datasets (see `output_run`).
105 output_run : `str`
106 Name of the new output RUN collection. If not provided then `output`
107 must be provided and a new RUN collection will be created by appending
108 a timestamp to the value passed with `output`. If this collection
109 already exists then `extend_run` must be passed.
110 extend_run : `bool`
111 Instead of creating a new RUN collection, insert datasets into either
112 the one given by `output_run` (if provided) or the first child
113 collection of `output` (which must be of type RUN).
114 replace_run : `bool`
115 Before creating a new RUN collection in an existing CHAINED collection,
116 remove the first child collection (which must be of type RUN). This can
117 be used to repeatedly write to the same (parent) collection during
118 development, but it does not delete the datasets associated with the
119 replaced run unless `prune-replaced` is also True. Requires `output`,
120 and `extend_run` must be `None`.
121 prune_replaced : "unstore", "purge", or `None`.
122 If not `None`, delete the datasets in the collection replaced by
123 `replace_run`, either just from the datastore ("unstore") or by
124 removing them and the RUN completely ("purge"). Requires `replace_run`.
125 data_query : `str`
126 User query selection expression.
127 skip_existing : `bool`
128 If all Quantum outputs already exist in the output RUN collection then
129 that Quantum will be excluded from the QuantumGraph. Requires the 'run`
130 command's `--extend-run` flag to be set.
131 debug : `bool`
132 If true, enable debugging output using lsstDebug facility (imports
133 debug.py).
134 fail_fast : `bool`
135 If true then stop processing at first error, otherwise process as many
136 tasks as possible.
137 clobber_partial_outputs : `bool`
138 Remove incomplete outputs from previous execution of the same quantum
139 before new execution.
140 kwargs : `dict` [`str`, `str`]
141 Ignored; click commands may accept options for more than one script
142 function and pass all the option kwargs to each of the script functions
143 which ingore these unused kwargs.
144 """
146 if log_level is not None:
147 CliLog.setLogLevels(log_level)
149 args = SimpleNamespace(do_raise=do_raise,
150 graph_fixup=graph_fixup,
151 init_only=init_only,
152 no_versions=no_versions,
153 processes=processes,
154 profile=profile,
155 skip_init_writes=skip_init_writes,
156 timeout=timeout,
157 register_dataset_types=register_dataset_types,
158 butler_config=butler_config,
159 input=input,
160 output=output,
161 output_run=output_run,
162 extend_run=extend_run,
163 replace_run=replace_run,
164 prune_replaced=prune_replaced,
165 data_query=data_query,
166 skip_existing=skip_existing,
167 enableLsstDebug=debug,
168 fail_fast=fail_fast,
169 clobber_partial_outputs=clobber_partial_outputs)
171 f = CmdLineFwk()
172 taskFactory = TaskFactory()
173 f.runPipeline(qgraphObj, taskFactory, args)