Coverage for python/lsst/pipe/base/config.py: 48%

81 statements  

« prev     ^ index     » next       coverage.py v7.2.5, created at 2023-05-11 03:12 -0700

1# This file is part of pipe_base. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22"""Module defining config classes for PipelineTask. 

23""" 

24 

25from __future__ import annotations 

26 

27__all__ = ["ResourceConfig", "PipelineTaskConfig"] 

28 

29# ------------------------------- 

30# Imports of standard modules -- 

31# ------------------------------- 

32import os 

33from collections.abc import Iterable 

34from numbers import Number 

35from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple, Type, TypeVar 

36 

37# ----------------------------- 

38# Imports for other modules -- 

39# ----------------------------- 

40import lsst.pex.config as pexConfig 

41 

42from ._instrument import Instrument 

43from .configOverrides import ConfigOverrides 

44from .connections import PipelineTaskConnections 

45from .pipelineIR import ConfigIR, ParametersIR 

46 

47if TYPE_CHECKING: 47 ↛ 48line 47 didn't jump to line 48, because the condition on line 47 was never true

48 from lsst.pex.config.callStack import StackFrame 

49 

50# ---------------------------------- 

51# Local non-exported definitions -- 

52# ---------------------------------- 

53 

54_S = TypeVar("_S", bound="PipelineTaskConfigMeta") 

55 

56# ------------------------ 

57# Exported definitions -- 

58# ------------------------ 

59 

60 

61class TemplateField(pexConfig.Field): 

62 """This Field is specialized for use with connection templates. 

63 Specifically it treats strings or numbers as valid input, as occasionally 

64 numbers are used as a cycle counter in templates. 

65 

66 The reason for the specialized field, is that when numbers are involved 

67 with the config override system through pipelines or from the command line, 

68 sometimes the quoting to get appropriate values as strings gets 

69 complicated. This will simplify the process greatly. 

70 """ 

71 

72 def _validateValue(self, value: Any) -> None: 

73 if value is None: 

74 return 

75 

76 if not (isinstance(value, str) or isinstance(value, Number)): 

77 raise TypeError( 

78 f"Value {value} is of incorrect type {pexConfig.config._typeStr(value)}." 

79 " Expected type str or a number" 

80 ) 

81 if self.check is not None and not self.check(value): 

82 ValueError("Value {value} is not a valid value") 

83 

84 def __set__( 

85 self, 

86 instance: pexConfig.Config, 

87 value: Any, 

88 at: Optional[StackFrame] = None, 

89 label: str = "assignment", 

90 ) -> None: 

91 # validate first, even though validate will be called in super 

92 self._validateValue(value) 

93 # now, explicitly make it into a string 

94 value = str(value) 

95 super().__set__(instance, value, at, label) 

96 

97 

98class PipelineTaskConfigMeta(pexConfig.ConfigMeta): 

99 """Metaclass used in the creation of PipelineTaskConfig classes 

100 

101 This metaclass ensures a `PipelineTaskConnections` class is specified in 

102 the class construction parameters with a parameter name of 

103 pipelineConnections. Using the supplied connection class, this metaclass 

104 constructs a `lsst.pex.config.Config` instance which can be used to 

105 configure the connections class. This config is added to the config class 

106 under declaration with the name "connections" used as an identifier. The 

107 connections config also has a reference to the connections class used in 

108 its construction associated with an atttribute named `ConnectionsClass`. 

109 Finally the newly constructed config class (not an instance of it) is 

110 assigned to the Config class under construction with the attribute name 

111 `ConnectionsConfigClass`. 

112 """ 

113 

114 def __new__( 

115 cls: Type[_S], 

116 name: str, 

117 bases: Tuple[type[PipelineTaskConfig], ...], 

118 dct: Dict[str, Any], 

119 **kwargs: Any, 

120 ) -> _S: 

121 if name != "PipelineTaskConfig": 

122 # Verify that a connection class was specified and the argument is 

123 # an instance of PipelineTaskConfig 

124 if "pipelineConnections" not in kwargs: 124 ↛ 125line 124 didn't jump to line 125, because the condition on line 124 was never true

125 for base in bases: 

126 if hasattr(base, "connections"): 

127 kwargs["pipelineConnections"] = base.connections.dtype.ConnectionsClass 

128 break 

129 if "pipelineConnections" not in kwargs: 129 ↛ 130line 129 didn't jump to line 130, because the condition on line 129 was never true

130 raise NameError("PipelineTaskConfig or a base class must be defined with connections class") 

131 connectionsClass = kwargs["pipelineConnections"] 

132 if not issubclass(connectionsClass, PipelineTaskConnections): 132 ↛ 133line 132 didn't jump to line 133, because the condition on line 132 was never true

133 raise ValueError("Can only assign a PipelineTaskConnections Class to pipelineConnections") 

134 

135 # Create all the fields that will be used in the newly created sub 

136 # config (under the attribute name "connections") 

137 configConnectionsNamespace: dict[str, pexConfig.Field] = {} 

138 for fieldName, obj in connectionsClass.allConnections.items(): 

139 configConnectionsNamespace[fieldName] = pexConfig.Field[str]( 

140 doc=f"name for connection {fieldName}", default=obj.name 

141 ) 

142 # If there are default templates also add them as fields to 

143 # configure the template values 

144 if hasattr(connectionsClass, "defaultTemplates"): 144 ↛ 152line 144 didn't jump to line 152, because the condition on line 144 was never false

145 docString = "Template parameter used to format corresponding field template parameter" 

146 for templateName, default in connectionsClass.defaultTemplates.items(): 

147 configConnectionsNamespace[templateName] = TemplateField( 

148 dtype=str, doc=docString, default=default 

149 ) 

150 # add a reference to the connection class used to create this sub 

151 # config 

152 configConnectionsNamespace["ConnectionsClass"] = connectionsClass 

153 

154 # Create a new config class with the fields defined above 

155 Connections = type("Connections", (pexConfig.Config,), configConnectionsNamespace) 

156 # add it to the Config class that is currently being declared 

157 dct["connections"] = pexConfig.ConfigField( 

158 dtype=Connections, 

159 doc="Configurations describing the connections of the PipelineTask to datatypes", 

160 ) 

161 dct["ConnectionsConfigClass"] = Connections 

162 dct["ConnectionsClass"] = connectionsClass 

163 inst = super().__new__(cls, name, bases, dct) 

164 return inst 

165 

166 def __init__( 

167 self, name: str, bases: Tuple[Type[PipelineTaskConfig], ...], dct: Dict[str, Any], **kwargs: Any 

168 ): 

169 # This overrides the default init to drop the kwargs argument. Python 

170 # metaclasses will have this argument set if any kwargs are passes at 

171 # class construction time, but should be consumed before calling 

172 # __init__ on the type metaclass. This is in accordance with python 

173 # documentation on metaclasses 

174 super().__init__(name, bases, dct) 

175 

176 

177class PipelineTaskConfig(pexConfig.Config, metaclass=PipelineTaskConfigMeta): 

178 """Configuration class for `PipelineTask` 

179 

180 This Configuration class functions in largely the same manner as any other 

181 derived from `lsst.pex.config.Config`. The only difference is in how it is 

182 declared. `PipelineTaskConfig` children need to be declared with a 

183 pipelineConnections argument. This argument should specify a child class of 

184 `PipelineTaskConnections`. During the declaration of a `PipelineTaskConfig` 

185 a config class is created with information from the supplied connections 

186 class to allow configuration of the connections class. This dynamically 

187 created config class is then attached to the `PipelineTaskConfig` via a 

188 `~lsst.pex.config.ConfigField` with the attribute name `connections`. 

189 """ 

190 

191 connections: pexConfig.ConfigField 

192 """Field which refers to a dynamically added configuration class which is 

193 based on a PipelineTaskConnections class. 

194 """ 

195 

196 saveMetadata = pexConfig.Field[bool]( 

197 default=True, 

198 optional=False, 

199 doc="Flag to enable/disable metadata saving for a task, enabled by default.", 

200 ) 

201 saveLogOutput = pexConfig.Field[bool]( 

202 default=True, 

203 optional=False, 

204 doc="Flag to enable/disable saving of log output for a task, enabled by default.", 

205 ) 

206 

207 def applyConfigOverrides( 

208 self, 

209 instrument: Instrument | None, 

210 taskDefaultName: str, 

211 pipelineConfigs: Iterable[ConfigIR] | None, 

212 parameters: ParametersIR, 

213 label: str, 

214 ) -> None: 

215 r"""Apply config overrides to this config instance. 

216 

217 Parameters 

218 --------- 

219 instrument : `Instrument` or `None` 

220 An instance of the `Instrument` specified in a pipeline. 

221 If `None` then the pipeline did not specify and instrument. 

222 taskDefaultName : `str` 

223 The default name associated with the `Task` class. This 

224 may be used with instrumental overrides. 

225 pipelineConfigs : `Iterable` of `ConfigIR` 

226 An iterable of `ConfigIR` objects that contain overrides 

227 to apply to this config instance. 

228 parameters : `ParametersIR` 

229 Parameters defined in a Pipeline which are used in formatting 

230 of config values across multiple `Task`\ s in a pipeline. 

231 label : `str` 

232 The label associated with this class's Task in a pipeline. 

233 """ 

234 overrides = ConfigOverrides() 

235 if instrument is not None: 

236 overrides.addInstrumentOverride(instrument, taskDefaultName) 

237 if pipelineConfigs is not None: 

238 for subConfig in (configIr.formatted(parameters) for configIr in pipelineConfigs): 

239 if subConfig.dataId is not None: 

240 raise NotImplementedError( 

241 "Specializing a config on a partial data id is not yet " 

242 "supported in Pipeline definition" 

243 ) 

244 # only apply override if it applies to everything 

245 if subConfig.dataId is None: 

246 if subConfig.file: 

247 for configFile in subConfig.file: 

248 overrides.addFileOverride(os.path.expandvars(configFile)) 

249 if subConfig.python is not None: 

250 overrides.addPythonOverride(subConfig.python) 

251 for key, value in subConfig.rest.items(): 

252 overrides.addValueOverride(key, value) 

253 overrides.applyTo(self) 

254 

255 

256class ResourceConfig(pexConfig.Config): 

257 """Configuration for resource requirements. 

258 

259 This configuration class will be used by some activators to estimate 

260 resource use by pipeline. Additionally some tasks could use it to adjust 

261 their resource use (e.g. reduce the number of threads). 

262 

263 For some resources their limit can be estimated by corresponding task, 

264 in that case task could set the field value. For many fields defined in 

265 this class their associated resource used by a task will depend on the 

266 size of the data and is not known in advance. For these resources their 

267 value will be configured through overrides based on some external 

268 estimates. 

269 """ 

270 

271 minMemoryMB = pexConfig.Field[int]( 

272 default=None, 

273 optional=True, 

274 doc="Minimal memory needed by task, can be None if estimate is unknown.", 

275 ) 

276 minNumCores = pexConfig.Field[int](default=1, doc="Minimal number of cores needed by task.")