Coverage for python/lsst/ctrl/mpexec/cli/script/run.py: 38%
11 statements
« prev ^ index » next coverage.py v7.2.7, created at 2023-06-28 10:40 +0000
« prev ^ index » next coverage.py v7.2.7, created at 2023-06-28 10:40 +0000
1# This file is part of ctrl_mpexec.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22import logging
23from types import SimpleNamespace
25from ... import CmdLineFwk, TaskFactory
27_log = logging.getLogger(__name__)
30def run( # type: ignore
31 pdb,
32 graph_fixup,
33 init_only,
34 no_versions,
35 processes,
36 start_method,
37 profile,
38 qgraphObj,
39 register_dataset_types,
40 skip_init_writes,
41 timeout,
42 butler_config,
43 input,
44 output,
45 output_run,
46 extend_run,
47 replace_run,
48 prune_replaced,
49 data_query,
50 skip_existing_in,
51 skip_existing,
52 debug,
53 fail_fast,
54 clobber_outputs,
55 summary,
56 mock,
57 unmocked_dataset_types,
58 mock_failure,
59 enable_implicit_threading,
60 cores_per_quantum: int,
61 memory_per_quantum: str,
62 **kwargs,
63):
64 """Implement the command line interface `pipetask run` subcommand.
66 Should only be called by command line tools and unit test code that test
67 this function.
69 Parameters
70 ----------
71 pdb : `bool`
72 Drop into pdb on exception?
73 graph_fixup : `str`
74 The name of the class or factory method which makes an instance used
75 for execution graph fixup.
76 init_only : `bool`
77 If true, do not actually run; just register dataset types and/or save
78 init outputs.
79 no_versions : `bool`
80 If true, do not save or check package versions.
81 processes : `int`
82 The number of processes to use.
83 start_method : `str` or `None`
84 Start method from `multiprocessing` module, `None` selects the best
85 one for current platform.
86 profile : `str`
87 File name to dump cProfile information to.
88 qgraphObj : `lsst.pipe.base.QuantumGraph`
89 A QuantumGraph generated by a previous subcommand.
90 register_dataset_types : `bool`
91 If true, register DatasetTypes that do not already exist in the
92 Registry.
93 skip_init_writes : `bool`
94 If true, do not write collection-wide 'init output' datasets (e.g.
95 schemas).
96 timeout : `int`
97 Timeout for multiprocessing; maximum wall time (sec).
98 butler_config : `str`, `dict`, or `lsst.daf.butler.Config`
99 If `str`, `butler_config` is the path location of the gen3
100 butler/registry config file. If `dict`, `butler_config` is key value
101 pairs used to init or update the `lsst.daf.butler.Config` instance. If
102 `Config`, it is the object used to configure a Butler.
103 input : `list` [ `str` ]
104 List of names of the input collection(s).
105 output : `str`
106 Name of the output CHAINED collection. This may either be an existing
107 CHAINED collection to use as both input and output (if `input` is
108 `None`), or a new CHAINED collection created to include all inputs
109 (if `input` is not `None`). In both cases, the collection's children
110 will start with an output RUN collection that directly holds all new
111 datasets (see `output_run`).
112 output_run : `str`
113 Name of the new output RUN collection. If not provided then `output`
114 must be provided and a new RUN collection will be created by appending
115 a timestamp to the value passed with `output`. If this collection
116 already exists then `extend_run` must be passed.
117 extend_run : `bool`
118 Instead of creating a new RUN collection, insert datasets into either
119 the one given by `output_run` (if provided) or the first child
120 collection of `output` (which must be of type RUN).
121 replace_run : `bool`
122 Before creating a new RUN collection in an existing CHAINED collection,
123 remove the first child collection (which must be of type RUN). This can
124 be used to repeatedly write to the same (parent) collection during
125 development, but it does not delete the datasets associated with the
126 replaced run unless `prune-replaced` is also True. Requires `output`,
127 and `extend_run` must be `None`.
128 prune_replaced : "unstore", "purge", or `None`.
129 If not `None`, delete the datasets in the collection replaced by
130 `replace_run`, either just from the datastore ("unstore") or by
131 removing them and the RUN completely ("purge"). Requires `replace_run`.
132 data_query : `str`
133 User query selection expression.
134 skip_existing_in : `list` [ `str` ]
135 Accepts list of collections, if all Quantum outputs already exist in
136 the specified list of collections then that Quantum will be excluded
137 from the QuantumGraph.
138 skip_existing : `bool`
139 Appends output RUN collection to the ``skip_existing_in`` list.
140 debug : `bool`
141 If true, enable debugging output using lsstDebug facility (imports
142 debug.py).
143 fail_fast : `bool`
144 If true then stop processing at first error, otherwise process as many
145 tasks as possible.
146 clobber_outputs : `bool`
147 Remove outputs from previous execution of the same quantum before new
148 execution. Only applies to failed quanta if skip_existing is also
149 given.
150 summary : `str`
151 File path to store job report in JSON format.
152 mock : `bool`, optional
153 If `True` then run mock pipeline instead of real one. Ignored if an
154 existing QuantumGraph is provided.
155 unmocked_dataset_types : `collections.abc.Sequence` [ `str` ]
156 List of overall-input dataset types that should not be mocked.
157 Ignored if an existing QuantumGraph is provided.
158 mock_failure : `~collections.abc.Sequence`, optional
159 List of quanta that should raise exceptions.
160 enable_implicit_threading : `bool`, optional
161 If `True`, do not disable implicit threading by third-party libraries.
162 Implicit threading is always disabled during actual quantum execution
163 if ``processes > 1``.
164 cores_per_quantum : `int`
165 Number of cores that can be used by each quantum.
166 memory_per_quantum : `str`
167 Amount of memory that each quantum can be allowed to use. Empty string
168 implies no limit. The string can be either a single integer (implying
169 units of MB) or a combination of number and unit.
170 kwargs : `dict` [`str`, `str`]
171 Ignored; click commands may accept options for more than one script
172 function and pass all the option kwargs to each of the script functions
173 which ignore these unused kwargs.
174 """
175 args = SimpleNamespace(
176 pdb=pdb,
177 graph_fixup=graph_fixup,
178 init_only=init_only,
179 no_versions=no_versions,
180 processes=processes,
181 start_method=start_method,
182 profile=profile,
183 skip_init_writes=skip_init_writes,
184 timeout=timeout,
185 register_dataset_types=register_dataset_types,
186 butler_config=butler_config,
187 input=input,
188 output=output,
189 output_run=output_run,
190 extend_run=extend_run,
191 replace_run=replace_run,
192 prune_replaced=prune_replaced,
193 data_query=data_query,
194 skip_existing_in=skip_existing_in,
195 skip_existing=skip_existing,
196 enableLsstDebug=debug,
197 fail_fast=fail_fast,
198 clobber_outputs=clobber_outputs,
199 summary=summary,
200 # Mock options only used by qgraph.
201 enable_implicit_threading=enable_implicit_threading,
202 cores_per_quantum=cores_per_quantum,
203 memory_per_quantum=memory_per_quantum,
204 )
206 f = CmdLineFwk()
207 taskFactory = TaskFactory()
209 # If we have no output run specified, use the one from the graph rather
210 # than letting a new timestamped run be created.
211 if not args.output_run and qgraphObj.metadata and (output_run := qgraphObj.metadata.get("output_run")):
212 args.output_run = output_run
214 f.runPipeline(qgraphObj, taskFactory, args)