Coverage for python/lsst/ctrl/mpexec/cli/script/run.py : 43%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of ctrl_mpexec.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22import logging
23from types import SimpleNamespace
25from lsst.daf.butler.cli.cliLog import CliLog
26from ... import CmdLineFwk, TaskFactory
28_log = logging.getLogger(__name__.partition(".")[2])
31def run(do_raise,
32 graph_fixup,
33 init_only,
34 log_level,
35 no_versions,
36 processes,
37 profile,
38 qgraphObj,
39 register_dataset_types,
40 skip_init_writes,
41 timeout,
42 butler_config,
43 input,
44 output,
45 output_run,
46 extend_run,
47 replace_run,
48 prune_replaced,
49 data_query,
50 skip_existing,
51 debug,
52 fail_fast,
53 **kwargs):
54 """Implements the command line interface `pipetask run` subcommand, should
55 only be called by command line tools and unit test code that test this
56 function.
58 Parameters
59 ----------
60 do_raise : `bool`
61 Raise an exception in the case of an error.
62 graph_fixup : `str`
63 The name of the class or factory method which makes an instance used
64 for execution graph fixup.
65 init_only : `bool`
66 If true, do not actually run; just register dataset types and/or save
67 init outputs.
68 log_level : `list` of `tuple`
69 per-component logging levels, each item in the list is a tuple
70 (component, level), `component` is a logger name or an empty string
71 or `None` for root logger, `level` is a logging level name, one of
72 CRITICAL, ERROR, WARNING, INFO, DEBUG (case insensitive).
73 no_versions : `bool`
74 If true, do not save or check package versions.
75 processes : `int`
76 The number of processes to use.
77 profile : `str`
78 File name to dump cProfile information to.
79 qgraphObj : `lsst.pipe.base.QuantumGraph`
80 A QuantumGraph generated by a previous subcommand.
81 register_dataset_types : `bool`
82 If true, register DatasetTypes that do not already exist in the Registry.
83 skip_init_writes : `bool`
84 If true, do not write collection-wide 'init output' datasets (e.g.
85 schemas).
86 timeout : `int`
87 Timeout for multiprocessing; maximum wall time (sec).
88 butler_config : `str`, `dict`, or `lsst.daf.butler.Config`
89 If `str`, `butler_config` is the path location of the gen3
90 butler/registry config file. If `dict`, `butler_config` is key value
91 pairs used to init or update the `lsst.daf.butler.Config` instance. If
92 `Config`, it is the object used to configure a Butler.
93 input : `str`
94 Comma-separated names of the input collection(s). Entries may include a
95 colon (:), the first string is a dataset type name that restricts the
96 search in that collection.
97 output : `str`
98 Name of the output CHAINED collection. This may either be an existing
99 CHAINED collection to use as both input and output (if `input` is
100 `None`), or a new CHAINED collection created to include all inputs
101 (if `input` is not `None`). In both cases, the collection's children
102 will start with an output RUN collection that directly holds all new
103 datasets (see `output_run`).
104 output_run : `str`
105 Name of the new output RUN collection. If not provided then `output`
106 must be provided and a new RUN collection will be created by appending
107 a timestamp to the value passed with `output`. If this collection
108 already exists then `extend_run` must be passed.
109 extend_run : `bool`
110 Instead of creating a new RUN collection, insert datasets into either
111 the one given by `output_run` (if provided) or the first child
112 collection of `output` (which must be of type RUN).
113 replace_run : `bool`
114 Before creating a new RUN collection in an existing CHAINED collection,
115 remove the first child collection (which must be of type RUN). This can
116 be used to repeatedly write to the same (parent) collection during
117 development, but it does not delete the datasets associated with the
118 replaced run unless `prune-replaced` is also True. Requires `output`,
119 and `extend_run` must be `None`.
120 prune_replaced : "unstore", "purge", or `None`.
121 If not `None`, delete the datasets in the collection replaced by
122 `replace_run`, either just from the datastore ("unstore") or by
123 removing them and the RUN completely ("purge"). Requires `replace_run`.
124 data_query : `str`
125 User query selection expression.
126 skip_existing : `bool`
127 If all Quantum outputs already exist in the output RUN collection then
128 that Quantum will be excluded from the QuantumGraph. Requires the 'run`
129 command's `--extend-run` flag to be set.
130 debug : `bool`
131 If true, enable debugging output using lsstDebug facility (imports
132 debug.py).
133 fail_fast : `bool`
134 If true then stop processing at first error, otherwise process as many
135 tasks as possible.
136 kwargs : `dict` [`str`, `str`]
137 Ignored; click commands may accept options for more than one script
138 function and pass all the option kwargs to each of the script functions
139 which ingore these unused kwargs.
140 """
142 if log_level is not None:
143 CliLog.setLogLevels(log_level)
145 args = SimpleNamespace(do_raise=do_raise,
146 graph_fixup=graph_fixup,
147 init_only=init_only,
148 no_versions=no_versions,
149 processes=processes,
150 profile=profile,
151 skip_init_writes=skip_init_writes,
152 timeout=timeout,
153 register_dataset_types=register_dataset_types,
154 butler_config=butler_config,
155 input=input,
156 output=output,
157 output_run=output_run,
158 extend_run=extend_run,
159 replace_run=replace_run,
160 prune_replaced=prune_replaced,
161 data_query=data_query,
162 skip_existing=skip_existing,
163 enableLsstDebug=debug,
164 fail_fast=fail_fast)
166 f = CmdLineFwk()
167 taskFactory = TaskFactory()
168 f.runPipeline(qgraphObj, taskFactory, args)