Coverage for python/lsst/ctrl/mpexec/cli/script/run.py: 38%

11 statements  

« prev     ^ index     » next       coverage.py v7.2.7, created at 2023-07-14 19:55 +0000

1# This file is part of ctrl_mpexec. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22import logging 

23from types import SimpleNamespace 

24 

25from ... import CmdLineFwk, TaskFactory 

26 

27_log = logging.getLogger(__name__) 

28 

29 

30def run( # type: ignore 

31 pdb, 

32 graph_fixup, 

33 init_only, 

34 no_versions, 

35 processes, 

36 start_method, 

37 profile, 

38 qgraphObj, 

39 register_dataset_types, 

40 skip_init_writes, 

41 timeout, 

42 butler_config, 

43 input, 

44 output, 

45 output_run, 

46 extend_run, 

47 replace_run, 

48 prune_replaced, 

49 data_query, 

50 skip_existing_in, 

51 skip_existing, 

52 debug, 

53 fail_fast, 

54 clobber_outputs, 

55 summary, 

56 mock, 

57 unmocked_dataset_types, 

58 mock_failure, 

59 enable_implicit_threading, 

60 cores_per_quantum: int, 

61 memory_per_quantum: str, 

62 **kwargs, 

63): 

64 """Implement the command line interface `pipetask run` subcommand. 

65 

66 Should only be called by command line tools and unit test code that test 

67 this function. 

68 

69 Parameters 

70 ---------- 

71 pdb : `bool` 

72 Drop into pdb on exception? 

73 graph_fixup : `str` 

74 The name of the class or factory method which makes an instance used 

75 for execution graph fixup. 

76 init_only : `bool` 

77 If true, do not actually run; just register dataset types and/or save 

78 init outputs. 

79 no_versions : `bool` 

80 If true, do not save or check package versions. 

81 processes : `int` 

82 The number of processes to use. 

83 start_method : `str` or `None` 

84 Start method from `multiprocessing` module, `None` selects the best 

85 one for current platform. 

86 profile : `str` 

87 File name to dump cProfile information to. 

88 qgraphObj : `lsst.pipe.base.QuantumGraph` 

89 A QuantumGraph generated by a previous subcommand. 

90 register_dataset_types : `bool` 

91 If true, register DatasetTypes that do not already exist in the 

92 Registry. 

93 skip_init_writes : `bool` 

94 If true, do not write collection-wide 'init output' datasets (e.g. 

95 schemas). 

96 timeout : `int` 

97 Timeout for multiprocessing; maximum wall time (sec). 

98 butler_config : `str`, `dict`, or `lsst.daf.butler.Config` 

99 If `str`, `butler_config` is the path location of the gen3 

100 butler/registry config file. If `dict`, `butler_config` is key value 

101 pairs used to init or update the `lsst.daf.butler.Config` instance. If 

102 `Config`, it is the object used to configure a Butler. 

103 input : `list` [ `str` ] 

104 List of names of the input collection(s). 

105 output : `str` 

106 Name of the output CHAINED collection. This may either be an existing 

107 CHAINED collection to use as both input and output (if `input` is 

108 `None`), or a new CHAINED collection created to include all inputs 

109 (if `input` is not `None`). In both cases, the collection's children 

110 will start with an output RUN collection that directly holds all new 

111 datasets (see `output_run`). 

112 output_run : `str` 

113 Name of the new output RUN collection. If not provided then `output` 

114 must be provided and a new RUN collection will be created by appending 

115 a timestamp to the value passed with `output`. If this collection 

116 already exists then `extend_run` must be passed. 

117 extend_run : `bool` 

118 Instead of creating a new RUN collection, insert datasets into either 

119 the one given by `output_run` (if provided) or the first child 

120 collection of `output` (which must be of type RUN). 

121 replace_run : `bool` 

122 Before creating a new RUN collection in an existing CHAINED collection, 

123 remove the first child collection (which must be of type RUN). This can 

124 be used to repeatedly write to the same (parent) collection during 

125 development, but it does not delete the datasets associated with the 

126 replaced run unless `prune-replaced` is also True. Requires `output`, 

127 and `extend_run` must be `None`. 

128 prune_replaced : "unstore", "purge", or `None`. 

129 If not `None`, delete the datasets in the collection replaced by 

130 `replace_run`, either just from the datastore ("unstore") or by 

131 removing them and the RUN completely ("purge"). Requires `replace_run`. 

132 data_query : `str` 

133 User query selection expression. 

134 skip_existing_in : `list` [ `str` ] 

135 Accepts list of collections, if all Quantum outputs already exist in 

136 the specified list of collections then that Quantum will be excluded 

137 from the QuantumGraph. 

138 skip_existing : `bool` 

139 Appends output RUN collection to the ``skip_existing_in`` list. 

140 debug : `bool` 

141 If true, enable debugging output using lsstDebug facility (imports 

142 debug.py). 

143 fail_fast : `bool` 

144 If true then stop processing at first error, otherwise process as many 

145 tasks as possible. 

146 clobber_outputs : `bool` 

147 Remove outputs from previous execution of the same quantum before new 

148 execution. Only applies to failed quanta if skip_existing is also 

149 given. 

150 summary : `str` 

151 File path to store job report in JSON format. 

152 mock : `bool`, optional 

153 If `True` then run mock pipeline instead of real one. Ignored if an 

154 existing QuantumGraph is provided. 

155 unmocked_dataset_types : `collections.abc.Sequence` [ `str` ] 

156 List of overall-input dataset types that should not be mocked. 

157 Ignored if an existing QuantumGraph is provided. 

158 mock_failure : `~collections.abc.Sequence`, optional 

159 List of quanta that should raise exceptions. 

160 enable_implicit_threading : `bool`, optional 

161 If `True`, do not disable implicit threading by third-party libraries. 

162 Implicit threading is always disabled during actual quantum execution 

163 if ``processes > 1``. 

164 cores_per_quantum : `int` 

165 Number of cores that can be used by each quantum. 

166 memory_per_quantum : `str` 

167 Amount of memory that each quantum can be allowed to use. Empty string 

168 implies no limit. The string can be either a single integer (implying 

169 units of MB) or a combination of number and unit. 

170 kwargs : `dict` [`str`, `str`] 

171 Ignored; click commands may accept options for more than one script 

172 function and pass all the option kwargs to each of the script functions 

173 which ignore these unused kwargs. 

174 """ 

175 args = SimpleNamespace( 

176 pdb=pdb, 

177 graph_fixup=graph_fixup, 

178 init_only=init_only, 

179 no_versions=no_versions, 

180 processes=processes, 

181 start_method=start_method, 

182 profile=profile, 

183 skip_init_writes=skip_init_writes, 

184 timeout=timeout, 

185 register_dataset_types=register_dataset_types, 

186 butler_config=butler_config, 

187 input=input, 

188 output=output, 

189 output_run=output_run, 

190 extend_run=extend_run, 

191 replace_run=replace_run, 

192 prune_replaced=prune_replaced, 

193 data_query=data_query, 

194 skip_existing_in=skip_existing_in, 

195 skip_existing=skip_existing, 

196 enableLsstDebug=debug, 

197 fail_fast=fail_fast, 

198 clobber_outputs=clobber_outputs, 

199 summary=summary, 

200 # Mock options only used by qgraph. 

201 enable_implicit_threading=enable_implicit_threading, 

202 cores_per_quantum=cores_per_quantum, 

203 memory_per_quantum=memory_per_quantum, 

204 ) 

205 

206 f = CmdLineFwk() 

207 taskFactory = TaskFactory() 

208 

209 # If we have no output run specified, use the one from the graph rather 

210 # than letting a new timestamped run be created. 

211 if not args.output_run and qgraphObj.metadata and (output_run := qgraphObj.metadata.get("output_run")): 

212 args.output_run = output_run 

213 

214 f.runPipeline(qgraphObj, taskFactory, args)