Coverage for python/lsst/pipe/base/config.py: 47%

77 statements  

« prev     ^ index     » next       coverage.py v7.5.0, created at 2024-04-26 02:50 -0700

1# This file is part of pipe_base. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This software is dual licensed under the GNU General Public License and also 

10# under a 3-clause BSD license. Recipients may choose which of these licenses 

11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, 

12# respectively. If you choose the GPL option then the following text applies 

13# (but note that there is still no warranty even if you opt for BSD instead): 

14# 

15# This program is free software: you can redistribute it and/or modify 

16# it under the terms of the GNU General Public License as published by 

17# the Free Software Foundation, either version 3 of the License, or 

18# (at your option) any later version. 

19# 

20# This program is distributed in the hope that it will be useful, 

21# but WITHOUT ANY WARRANTY; without even the implied warranty of 

22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

23# GNU General Public License for more details. 

24# 

25# You should have received a copy of the GNU General Public License 

26# along with this program. If not, see <http://www.gnu.org/licenses/>. 

27 

28"""Module defining config classes for PipelineTask. 

29""" 

30 

31from __future__ import annotations 

32 

33__all__ = ["PipelineTaskConfig"] 

34 

35# ------------------------------- 

36# Imports of standard modules -- 

37# ------------------------------- 

38import os 

39from collections.abc import Iterable 

40from numbers import Number 

41from typing import TYPE_CHECKING, Any, TypeVar 

42 

43# ----------------------------- 

44# Imports for other modules -- 

45# ----------------------------- 

46import lsst.pex.config as pexConfig 

47 

48from ._instrument import Instrument 

49from .configOverrides import ConfigOverrides 

50from .connections import PipelineTaskConnections 

51from .pipelineIR import ConfigIR, ParametersIR 

52 

53if TYPE_CHECKING: 

54 from lsst.pex.config.callStack import StackFrame 

55 

56# ---------------------------------- 

57# Local non-exported definitions -- 

58# ---------------------------------- 

59 

60_S = TypeVar("_S", bound="PipelineTaskConfigMeta") 

61 

62# ------------------------ 

63# Exported definitions -- 

64# ------------------------ 

65 

66 

67class TemplateField(pexConfig.Field): 

68 """Field specialized for use with connection templates. 

69 

70 Specifically it treats strings or numbers as valid input, as occasionally 

71 numbers are used as a cycle counter in templates. 

72 

73 The reason for the specialized field, is that when numbers are involved 

74 with the config override system through pipelines or from the command line, 

75 sometimes the quoting to get appropriate values as strings gets 

76 complicated. This will simplify the process greatly. 

77 """ 

78 

79 def _validateValue(self, value: Any) -> None: 

80 if value is None: 

81 return 

82 

83 if not (isinstance(value, str | Number)): 

84 raise TypeError( 

85 f"Value {value} is of incorrect type {pexConfig.config._typeStr(value)}." 

86 " Expected type str or a number" 

87 ) 

88 if self.check is not None and not self.check(value): 

89 ValueError("Value {value} is not a valid value") 

90 

91 def __set__( 

92 self, 

93 instance: pexConfig.Config, 

94 value: Any, 

95 at: StackFrame | None = None, 

96 label: str = "assignment", 

97 ) -> None: 

98 # validate first, even though validate will be called in super 

99 self._validateValue(value) 

100 # now, explicitly make it into a string 

101 value = str(value) 

102 super().__set__(instance, value, at, label) 

103 

104 

105class PipelineTaskConfigMeta(pexConfig.ConfigMeta): 

106 """Metaclass used in the creation of PipelineTaskConfig classes. 

107 

108 This metaclass ensures a `PipelineTaskConnections` class is specified in 

109 the class construction parameters with a parameter name of 

110 pipelineConnections. Using the supplied connection class, this metaclass 

111 constructs a `lsst.pex.config.Config` instance which can be used to 

112 configure the connections class. This config is added to the config class 

113 under declaration with the name "connections" used as an identifier. The 

114 connections config also has a reference to the connections class used in 

115 its construction associated with an atttribute named ``ConnectionsClass``. 

116 Finally the newly constructed config class (not an instance of it) is 

117 assigned to the Config class under construction with the attribute name 

118 ``ConnectionsConfigClass``. 

119 

120 Parameters 

121 ---------- 

122 name : `str` 

123 Name of config. 

124 bases : `~collections.abc.Collection` 

125 Base classes. 

126 dct : `~collections.abc.Mapping` 

127 Parameter dict. 

128 **kwargs : `~typing.Any` 

129 Additional parameters. 

130 """ 

131 

132 def __new__( 

133 cls: type[_S], 

134 name: str, 

135 bases: tuple[type[PipelineTaskConfig], ...], 

136 dct: dict[str, Any], 

137 **kwargs: Any, 

138 ) -> _S: 

139 if name != "PipelineTaskConfig": 

140 # Verify that a connection class was specified and the argument is 

141 # an instance of PipelineTaskConfig 

142 if "pipelineConnections" not in kwargs: 142 ↛ 143line 142 didn't jump to line 143, because the condition on line 142 was never true

143 for base in bases: 

144 if hasattr(base, "connections"): 

145 kwargs["pipelineConnections"] = base.connections.dtype.ConnectionsClass 

146 break 

147 if "pipelineConnections" not in kwargs: 147 ↛ 148line 147 didn't jump to line 148, because the condition on line 147 was never true

148 raise NameError("PipelineTaskConfig or a base class must be defined with connections class") 

149 connectionsClass = kwargs["pipelineConnections"] 

150 if not issubclass(connectionsClass, PipelineTaskConnections): 150 ↛ 151line 150 didn't jump to line 151, because the condition on line 150 was never true

151 raise ValueError("Can only assign a PipelineTaskConnections Class to pipelineConnections") 

152 

153 # Create all the fields that will be used in the newly created sub 

154 # config (under the attribute name "connections") 

155 configConnectionsNamespace: dict[str, pexConfig.Field] = {} 

156 for fieldName, obj in connectionsClass.allConnections.items(): 

157 configConnectionsNamespace[fieldName] = pexConfig.Field[str]( 

158 doc=f"name for connection {fieldName}", default=obj.name, deprecated=obj.deprecated 

159 ) 

160 # If there are default templates also add them as fields to 

161 # configure the template values 

162 if hasattr(connectionsClass, "defaultTemplates"): 162 ↛ 173line 162 didn't jump to line 173, because the condition on line 162 was never false

163 docString = "Template parameter used to format corresponding field template parameter" 

164 for templateName, default in connectionsClass.defaultTemplates.items(): 

165 configConnectionsNamespace[templateName] = TemplateField( 

166 dtype=str, 

167 doc=docString, 

168 default=default, 

169 deprecated=connectionsClass.deprecatedTemplates.get(templateName), 

170 ) 

171 # add a reference to the connection class used to create this sub 

172 # config 

173 configConnectionsNamespace["ConnectionsClass"] = connectionsClass 

174 

175 # Create a new config class with the fields defined above 

176 Connections = type(f"{name}Connections", (pexConfig.Config,), configConnectionsNamespace) 

177 # add it to the Config class that is currently being declared 

178 dct["connections"] = pexConfig.ConfigField( 

179 dtype=Connections, 

180 doc="Configurations describing the connections of the PipelineTask to datatypes", 

181 ) 

182 dct["ConnectionsConfigClass"] = Connections 

183 dct["ConnectionsClass"] = connectionsClass 

184 inst = super().__new__(cls, name, bases, dct) 

185 return inst 

186 

187 def __init__( 

188 self, name: str, bases: tuple[type[PipelineTaskConfig], ...], dct: dict[str, Any], **kwargs: Any 

189 ): 

190 # This overrides the default init to drop the kwargs argument. Python 

191 # metaclasses will have this argument set if any kwargs are passes at 

192 # class construction time, but should be consumed before calling 

193 # __init__ on the type metaclass. This is in accordance with python 

194 # documentation on metaclasses 

195 super().__init__(name, bases, dct) 

196 

197 ConnectionsClass: type[PipelineTaskConnections] 

198 ConnectionsConfigClass: type[pexConfig.Config] 

199 

200 

201class PipelineTaskConfig(pexConfig.Config, metaclass=PipelineTaskConfigMeta): 

202 """Configuration class for `PipelineTask`. 

203 

204 This Configuration class functions in largely the same manner as any other 

205 derived from `lsst.pex.config.Config`. The only difference is in how it is 

206 declared. `PipelineTaskConfig` children need to be declared with a 

207 pipelineConnections argument. This argument should specify a child class of 

208 `PipelineTaskConnections`. During the declaration of a `PipelineTaskConfig` 

209 a config class is created with information from the supplied connections 

210 class to allow configuration of the connections class. This dynamically 

211 created config class is then attached to the `PipelineTaskConfig` via a 

212 `~lsst.pex.config.ConfigField` with the attribute name ``connections``. 

213 """ 

214 

215 connections: pexConfig.ConfigField 

216 """Field which refers to a dynamically added configuration class which is 

217 based on a PipelineTaskConnections class. 

218 """ 

219 

220 saveLogOutput = pexConfig.Field[bool]( 

221 default=True, 

222 optional=False, 

223 doc="Flag to enable/disable saving of log output for a task, enabled by default.", 

224 ) 

225 

226 def applyConfigOverrides( 

227 self, 

228 instrument: Instrument | None, 

229 taskDefaultName: str, 

230 pipelineConfigs: Iterable[ConfigIR] | None, 

231 parameters: ParametersIR, 

232 label: str, 

233 ) -> None: 

234 """Apply config overrides to this config instance. 

235 

236 Parameters 

237 ---------- 

238 instrument : `Instrument` or `None` 

239 An instance of the `Instrument` specified in a pipeline. 

240 If `None` then the pipeline did not specify and instrument. 

241 taskDefaultName : `str` 

242 The default name associated with the `Task` class. This 

243 may be used with instrumental overrides. 

244 pipelineConfigs : `~collections.abc.Iterable` \ 

245 of `~.pipelineIR.ConfigIR` 

246 An iterable of `~.pipelineIR.ConfigIR` objects that contain 

247 overrides to apply to this config instance. 

248 parameters : `~.pipelineIR.ParametersIR` 

249 Parameters defined in a Pipeline which are used in formatting 

250 of config values across multiple `Task` in a pipeline. 

251 label : `str` 

252 The label associated with this class's Task in a pipeline. 

253 """ 

254 overrides = ConfigOverrides() 

255 overrides.addParameters(parameters) 

256 if instrument is not None: 

257 overrides.addInstrumentOverride(instrument, taskDefaultName) 

258 if pipelineConfigs is not None: 

259 for subConfig in (configIr.formatted(parameters) for configIr in pipelineConfigs): 

260 if subConfig.dataId is not None: 

261 raise NotImplementedError( 

262 "Specializing a config on a partial data id is not yet " 

263 "supported in Pipeline definition" 

264 ) 

265 # only apply override if it applies to everything 

266 if subConfig.dataId is None: 

267 if subConfig.file: 

268 for configFile in subConfig.file: 

269 overrides.addFileOverride(os.path.expandvars(configFile)) 

270 if subConfig.python is not None: 

271 overrides.addPythonOverride(subConfig.python) 

272 for key, value in subConfig.rest.items(): 

273 overrides.addValueOverride(key, value) 

274 overrides.applyTo(self)