Coverage for python/lsst/pipe/base/config.py: 48%

83 statements  

« prev     ^ index     » next       coverage.py v7.2.5, created at 2023-05-19 04:46 -0700

1# This file is part of pipe_base. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22"""Module defining config classes for PipelineTask. 

23""" 

24 

25from __future__ import annotations 

26 

27__all__ = ["ResourceConfig", "PipelineTaskConfig"] 

28 

29# ------------------------------- 

30# Imports of standard modules -- 

31# ------------------------------- 

32import os 

33from collections.abc import Iterable 

34from numbers import Number 

35from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple, Type, TypeVar 

36 

37# ----------------------------- 

38# Imports for other modules -- 

39# ----------------------------- 

40import lsst.pex.config as pexConfig 

41 

42from ._instrument import Instrument 

43from .configOverrides import ConfigOverrides 

44from .connections import PipelineTaskConnections 

45from .pipelineIR import ConfigIR, ParametersIR 

46 

47if TYPE_CHECKING: 47 ↛ 48line 47 didn't jump to line 48, because the condition on line 47 was never true

48 from lsst.pex.config.callStack import StackFrame 

49 

50# ---------------------------------- 

51# Local non-exported definitions -- 

52# ---------------------------------- 

53 

54_S = TypeVar("_S", bound="PipelineTaskConfigMeta") 

55 

56# ------------------------ 

57# Exported definitions -- 

58# ------------------------ 

59 

60 

61class TemplateField(pexConfig.Field): 

62 """This Field is specialized for use with connection templates. 

63 Specifically it treats strings or numbers as valid input, as occasionally 

64 numbers are used as a cycle counter in templates. 

65 

66 The reason for the specialized field, is that when numbers are involved 

67 with the config override system through pipelines or from the command line, 

68 sometimes the quoting to get appropriate values as strings gets 

69 complicated. This will simplify the process greatly. 

70 """ 

71 

72 def _validateValue(self, value: Any) -> None: 

73 if value is None: 

74 return 

75 

76 if not (isinstance(value, str) or isinstance(value, Number)): 

77 raise TypeError( 

78 f"Value {value} is of incorrect type {pexConfig.config._typeStr(value)}." 

79 " Expected type str or a number" 

80 ) 

81 if self.check is not None and not self.check(value): 

82 ValueError("Value {value} is not a valid value") 

83 

84 def __set__( 

85 self, 

86 instance: pexConfig.Config, 

87 value: Any, 

88 at: Optional[StackFrame] = None, 

89 label: str = "assignment", 

90 ) -> None: 

91 # validate first, even though validate will be called in super 

92 self._validateValue(value) 

93 # now, explicitly make it into a string 

94 value = str(value) 

95 super().__set__(instance, value, at, label) 

96 

97 

98class PipelineTaskConfigMeta(pexConfig.ConfigMeta): 

99 """Metaclass used in the creation of PipelineTaskConfig classes 

100 

101 This metaclass ensures a `PipelineTaskConnections` class is specified in 

102 the class construction parameters with a parameter name of 

103 pipelineConnections. Using the supplied connection class, this metaclass 

104 constructs a `lsst.pex.config.Config` instance which can be used to 

105 configure the connections class. This config is added to the config class 

106 under declaration with the name "connections" used as an identifier. The 

107 connections config also has a reference to the connections class used in 

108 its construction associated with an atttribute named `ConnectionsClass`. 

109 Finally the newly constructed config class (not an instance of it) is 

110 assigned to the Config class under construction with the attribute name 

111 `ConnectionsConfigClass`. 

112 """ 

113 

114 def __new__( 

115 cls: Type[_S], 

116 name: str, 

117 bases: Tuple[type[PipelineTaskConfig], ...], 

118 dct: Dict[str, Any], 

119 **kwargs: Any, 

120 ) -> _S: 

121 if name != "PipelineTaskConfig": 

122 # Verify that a connection class was specified and the argument is 

123 # an instance of PipelineTaskConfig 

124 if "pipelineConnections" not in kwargs: 124 ↛ 125line 124 didn't jump to line 125, because the condition on line 124 was never true

125 for base in bases: 

126 if hasattr(base, "connections"): 

127 kwargs["pipelineConnections"] = base.connections.dtype.ConnectionsClass 

128 break 

129 if "pipelineConnections" not in kwargs: 129 ↛ 130line 129 didn't jump to line 130, because the condition on line 129 was never true

130 raise NameError("PipelineTaskConfig or a base class must be defined with connections class") 

131 connectionsClass = kwargs["pipelineConnections"] 

132 if not issubclass(connectionsClass, PipelineTaskConnections): 132 ↛ 133line 132 didn't jump to line 133, because the condition on line 132 was never true

133 raise ValueError("Can only assign a PipelineTaskConnections Class to pipelineConnections") 

134 

135 # Create all the fields that will be used in the newly created sub 

136 # config (under the attribute name "connections") 

137 configConnectionsNamespace: dict[str, pexConfig.Field] = {} 

138 for fieldName, obj in connectionsClass.allConnections.items(): 

139 configConnectionsNamespace[fieldName] = pexConfig.Field[str]( 

140 doc=f"name for connection {fieldName}", default=obj.name 

141 ) 

142 # If there are default templates also add them as fields to 

143 # configure the template values 

144 if hasattr(connectionsClass, "defaultTemplates"): 144 ↛ 152line 144 didn't jump to line 152, because the condition on line 144 was never false

145 docString = "Template parameter used to format corresponding field template parameter" 

146 for templateName, default in connectionsClass.defaultTemplates.items(): 

147 configConnectionsNamespace[templateName] = TemplateField( 

148 dtype=str, doc=docString, default=default 

149 ) 

150 # add a reference to the connection class used to create this sub 

151 # config 

152 configConnectionsNamespace["ConnectionsClass"] = connectionsClass 

153 

154 # Create a new config class with the fields defined above 

155 Connections = type(f"{name}Connections", (pexConfig.Config,), configConnectionsNamespace) 

156 # add it to the Config class that is currently being declared 

157 dct["connections"] = pexConfig.ConfigField( 

158 dtype=Connections, 

159 doc="Configurations describing the connections of the PipelineTask to datatypes", 

160 ) 

161 dct["ConnectionsConfigClass"] = Connections 

162 dct["ConnectionsClass"] = connectionsClass 

163 inst = super().__new__(cls, name, bases, dct) 

164 return inst 

165 

166 def __init__( 

167 self, name: str, bases: Tuple[Type[PipelineTaskConfig], ...], dct: Dict[str, Any], **kwargs: Any 

168 ): 

169 # This overrides the default init to drop the kwargs argument. Python 

170 # metaclasses will have this argument set if any kwargs are passes at 

171 # class construction time, but should be consumed before calling 

172 # __init__ on the type metaclass. This is in accordance with python 

173 # documentation on metaclasses 

174 super().__init__(name, bases, dct) 

175 

176 ConnectionsClass: type[PipelineTaskConnections] 

177 ConnectionsConfigClass: type[pexConfig.Config] 

178 

179 

180class PipelineTaskConfig(pexConfig.Config, metaclass=PipelineTaskConfigMeta): 

181 """Configuration class for `PipelineTask` 

182 

183 This Configuration class functions in largely the same manner as any other 

184 derived from `lsst.pex.config.Config`. The only difference is in how it is 

185 declared. `PipelineTaskConfig` children need to be declared with a 

186 pipelineConnections argument. This argument should specify a child class of 

187 `PipelineTaskConnections`. During the declaration of a `PipelineTaskConfig` 

188 a config class is created with information from the supplied connections 

189 class to allow configuration of the connections class. This dynamically 

190 created config class is then attached to the `PipelineTaskConfig` via a 

191 `~lsst.pex.config.ConfigField` with the attribute name `connections`. 

192 """ 

193 

194 connections: pexConfig.ConfigField 

195 """Field which refers to a dynamically added configuration class which is 

196 based on a PipelineTaskConnections class. 

197 """ 

198 

199 saveMetadata = pexConfig.Field[bool]( 

200 default=True, 

201 optional=False, 

202 doc="Flag to enable/disable metadata saving for a task, enabled by default.", 

203 ) 

204 saveLogOutput = pexConfig.Field[bool]( 

205 default=True, 

206 optional=False, 

207 doc="Flag to enable/disable saving of log output for a task, enabled by default.", 

208 ) 

209 

210 def applyConfigOverrides( 

211 self, 

212 instrument: Instrument | None, 

213 taskDefaultName: str, 

214 pipelineConfigs: Iterable[ConfigIR] | None, 

215 parameters: ParametersIR, 

216 label: str, 

217 ) -> None: 

218 r"""Apply config overrides to this config instance. 

219 

220 Parameters 

221 ---------- 

222 instrument : `Instrument` or `None` 

223 An instance of the `Instrument` specified in a pipeline. 

224 If `None` then the pipeline did not specify and instrument. 

225 taskDefaultName : `str` 

226 The default name associated with the `Task` class. This 

227 may be used with instrumental overrides. 

228 pipelineConfigs : `Iterable` of `ConfigIR` 

229 An iterable of `ConfigIR` objects that contain overrides 

230 to apply to this config instance. 

231 parameters : `ParametersIR` 

232 Parameters defined in a Pipeline which are used in formatting 

233 of config values across multiple `Task`\ s in a pipeline. 

234 label : `str` 

235 The label associated with this class's Task in a pipeline. 

236 """ 

237 overrides = ConfigOverrides() 

238 if instrument is not None: 

239 overrides.addInstrumentOverride(instrument, taskDefaultName) 

240 if pipelineConfigs is not None: 

241 for subConfig in (configIr.formatted(parameters) for configIr in pipelineConfigs): 

242 if subConfig.dataId is not None: 

243 raise NotImplementedError( 

244 "Specializing a config on a partial data id is not yet " 

245 "supported in Pipeline definition" 

246 ) 

247 # only apply override if it applies to everything 

248 if subConfig.dataId is None: 

249 if subConfig.file: 

250 for configFile in subConfig.file: 

251 overrides.addFileOverride(os.path.expandvars(configFile)) 

252 if subConfig.python is not None: 

253 overrides.addPythonOverride(subConfig.python) 

254 for key, value in subConfig.rest.items(): 

255 overrides.addValueOverride(key, value) 

256 overrides.applyTo(self) 

257 

258 

259class ResourceConfig(pexConfig.Config): 

260 """Configuration for resource requirements. 

261 

262 This configuration class will be used by some activators to estimate 

263 resource use by pipeline. Additionally some tasks could use it to adjust 

264 their resource use (e.g. reduce the number of threads). 

265 

266 For some resources their limit can be estimated by corresponding task, 

267 in that case task could set the field value. For many fields defined in 

268 this class their associated resource used by a task will depend on the 

269 size of the data and is not known in advance. For these resources their 

270 value will be configured through overrides based on some external 

271 estimates. 

272 """ 

273 

274 minMemoryMB = pexConfig.Field[int]( 

275 default=None, 

276 optional=True, 

277 doc="Minimal memory needed by task, can be None if estimate is unknown.", 

278 ) 

279 minNumCores = pexConfig.Field[int](default=1, doc="Minimal number of cores needed by task.")