Coverage for python/lsst/pipe/base/config.py: 48%

78 statements  

« prev     ^ index     » next       coverage.py v7.3.0, created at 2023-08-23 10:31 +0000

1# This file is part of pipe_base. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22"""Module defining config classes for PipelineTask. 

23""" 

24 

25from __future__ import annotations 

26 

27__all__ = ["PipelineTaskConfig"] 

28 

29# ------------------------------- 

30# Imports of standard modules -- 

31# ------------------------------- 

32import os 

33from collections.abc import Iterable 

34from numbers import Number 

35from typing import TYPE_CHECKING, Any, TypeVar 

36 

37# ----------------------------- 

38# Imports for other modules -- 

39# ----------------------------- 

40import lsst.pex.config as pexConfig 

41 

42from ._instrument import Instrument 

43from .configOverrides import ConfigOverrides 

44from .connections import PipelineTaskConnections 

45from .pipelineIR import ConfigIR, ParametersIR 

46 

47if TYPE_CHECKING: 

48 from lsst.pex.config.callStack import StackFrame 

49 

50# ---------------------------------- 

51# Local non-exported definitions -- 

52# ---------------------------------- 

53 

54_S = TypeVar("_S", bound="PipelineTaskConfigMeta") 

55 

56# ------------------------ 

57# Exported definitions -- 

58# ------------------------ 

59 

60 

61class TemplateField(pexConfig.Field): 

62 """Field specialized for use with connection templates. 

63 

64 Specifically it treats strings or numbers as valid input, as occasionally 

65 numbers are used as a cycle counter in templates. 

66 

67 The reason for the specialized field, is that when numbers are involved 

68 with the config override system through pipelines or from the command line, 

69 sometimes the quoting to get appropriate values as strings gets 

70 complicated. This will simplify the process greatly. 

71 """ 

72 

73 def _validateValue(self, value: Any) -> None: 

74 if value is None: 

75 return 

76 

77 if not (isinstance(value, str | Number)): 

78 raise TypeError( 

79 f"Value {value} is of incorrect type {pexConfig.config._typeStr(value)}." 

80 " Expected type str or a number" 

81 ) 

82 if self.check is not None and not self.check(value): 

83 ValueError("Value {value} is not a valid value") 

84 

85 def __set__( 

86 self, 

87 instance: pexConfig.Config, 

88 value: Any, 

89 at: StackFrame | None = None, 

90 label: str = "assignment", 

91 ) -> None: 

92 # validate first, even though validate will be called in super 

93 self._validateValue(value) 

94 # now, explicitly make it into a string 

95 value = str(value) 

96 super().__set__(instance, value, at, label) 

97 

98 

99class PipelineTaskConfigMeta(pexConfig.ConfigMeta): 

100 """Metaclass used in the creation of PipelineTaskConfig classes 

101 

102 This metaclass ensures a `PipelineTaskConnections` class is specified in 

103 the class construction parameters with a parameter name of 

104 pipelineConnections. Using the supplied connection class, this metaclass 

105 constructs a `lsst.pex.config.Config` instance which can be used to 

106 configure the connections class. This config is added to the config class 

107 under declaration with the name "connections" used as an identifier. The 

108 connections config also has a reference to the connections class used in 

109 its construction associated with an atttribute named ``ConnectionsClass``. 

110 Finally the newly constructed config class (not an instance of it) is 

111 assigned to the Config class under construction with the attribute name 

112 ``ConnectionsConfigClass``. 

113 """ 

114 

115 def __new__( 

116 cls: type[_S], 

117 name: str, 

118 bases: tuple[type[PipelineTaskConfig], ...], 

119 dct: dict[str, Any], 

120 **kwargs: Any, 

121 ) -> _S: 

122 if name != "PipelineTaskConfig": 

123 # Verify that a connection class was specified and the argument is 

124 # an instance of PipelineTaskConfig 

125 if "pipelineConnections" not in kwargs: 125 ↛ 126line 125 didn't jump to line 126, because the condition on line 125 was never true

126 for base in bases: 

127 if hasattr(base, "connections"): 

128 kwargs["pipelineConnections"] = base.connections.dtype.ConnectionsClass 

129 break 

130 if "pipelineConnections" not in kwargs: 130 ↛ 131line 130 didn't jump to line 131, because the condition on line 130 was never true

131 raise NameError("PipelineTaskConfig or a base class must be defined with connections class") 

132 connectionsClass = kwargs["pipelineConnections"] 

133 if not issubclass(connectionsClass, PipelineTaskConnections): 133 ↛ 134line 133 didn't jump to line 134, because the condition on line 133 was never true

134 raise ValueError("Can only assign a PipelineTaskConnections Class to pipelineConnections") 

135 

136 # Create all the fields that will be used in the newly created sub 

137 # config (under the attribute name "connections") 

138 configConnectionsNamespace: dict[str, pexConfig.Field] = {} 

139 for fieldName, obj in connectionsClass.allConnections.items(): 

140 configConnectionsNamespace[fieldName] = pexConfig.Field[str]( 

141 doc=f"name for connection {fieldName}", default=obj.name, deprecated=obj.deprecated 

142 ) 

143 # If there are default templates also add them as fields to 

144 # configure the template values 

145 if hasattr(connectionsClass, "defaultTemplates"): 145 ↛ 156line 145 didn't jump to line 156, because the condition on line 145 was never false

146 docString = "Template parameter used to format corresponding field template parameter" 

147 for templateName, default in connectionsClass.defaultTemplates.items(): 

148 configConnectionsNamespace[templateName] = TemplateField( 

149 dtype=str, 

150 doc=docString, 

151 default=default, 

152 deprecated=connectionsClass.deprecatedTemplates.get(templateName), 

153 ) 

154 # add a reference to the connection class used to create this sub 

155 # config 

156 configConnectionsNamespace["ConnectionsClass"] = connectionsClass 

157 

158 # Create a new config class with the fields defined above 

159 Connections = type(f"{name}Connections", (pexConfig.Config,), configConnectionsNamespace) 

160 # add it to the Config class that is currently being declared 

161 dct["connections"] = pexConfig.ConfigField( 

162 dtype=Connections, 

163 doc="Configurations describing the connections of the PipelineTask to datatypes", 

164 ) 

165 dct["ConnectionsConfigClass"] = Connections 

166 dct["ConnectionsClass"] = connectionsClass 

167 inst = super().__new__(cls, name, bases, dct) 

168 return inst 

169 

170 def __init__( 

171 self, name: str, bases: tuple[type[PipelineTaskConfig], ...], dct: dict[str, Any], **kwargs: Any 

172 ): 

173 # This overrides the default init to drop the kwargs argument. Python 

174 # metaclasses will have this argument set if any kwargs are passes at 

175 # class construction time, but should be consumed before calling 

176 # __init__ on the type metaclass. This is in accordance with python 

177 # documentation on metaclasses 

178 super().__init__(name, bases, dct) 

179 

180 ConnectionsClass: type[PipelineTaskConnections] 

181 ConnectionsConfigClass: type[pexConfig.Config] 

182 

183 

184class PipelineTaskConfig(pexConfig.Config, metaclass=PipelineTaskConfigMeta): 

185 """Configuration class for `PipelineTask` 

186 

187 This Configuration class functions in largely the same manner as any other 

188 derived from `lsst.pex.config.Config`. The only difference is in how it is 

189 declared. `PipelineTaskConfig` children need to be declared with a 

190 pipelineConnections argument. This argument should specify a child class of 

191 `PipelineTaskConnections`. During the declaration of a `PipelineTaskConfig` 

192 a config class is created with information from the supplied connections 

193 class to allow configuration of the connections class. This dynamically 

194 created config class is then attached to the `PipelineTaskConfig` via a 

195 `~lsst.pex.config.ConfigField` with the attribute name ``connections``. 

196 """ 

197 

198 connections: pexConfig.ConfigField 

199 """Field which refers to a dynamically added configuration class which is 

200 based on a PipelineTaskConnections class. 

201 """ 

202 

203 saveMetadata = pexConfig.Field[bool]( 

204 default=True, 

205 optional=False, 

206 doc="Flag to enable/disable metadata saving for a task, enabled by default.", 

207 deprecated="This field is deprecated and will be removed after v26.", 

208 ) 

209 saveLogOutput = pexConfig.Field[bool]( 

210 default=True, 

211 optional=False, 

212 doc="Flag to enable/disable saving of log output for a task, enabled by default.", 

213 ) 

214 

215 def applyConfigOverrides( 

216 self, 

217 instrument: Instrument | None, 

218 taskDefaultName: str, 

219 pipelineConfigs: Iterable[ConfigIR] | None, 

220 parameters: ParametersIR, 

221 label: str, 

222 ) -> None: 

223 """Apply config overrides to this config instance. 

224 

225 Parameters 

226 ---------- 

227 instrument : `Instrument` or `None` 

228 An instance of the `Instrument` specified in a pipeline. 

229 If `None` then the pipeline did not specify and instrument. 

230 taskDefaultName : `str` 

231 The default name associated with the `Task` class. This 

232 may be used with instrumental overrides. 

233 pipelineConfigs : `~collections.abc.Iterable` \ 

234 of `~.pipelineIR.ConfigIR` 

235 An iterable of `~.pipelineIR.ConfigIR` objects that contain 

236 overrides to apply to this config instance. 

237 parameters : `~.pipelineIR.ParametersIR` 

238 Parameters defined in a Pipeline which are used in formatting 

239 of config values across multiple `Task` in a pipeline. 

240 label : `str` 

241 The label associated with this class's Task in a pipeline. 

242 """ 

243 overrides = ConfigOverrides() 

244 overrides.addParameters(parameters) 

245 if instrument is not None: 

246 overrides.addInstrumentOverride(instrument, taskDefaultName) 

247 if pipelineConfigs is not None: 

248 for subConfig in (configIr.formatted(parameters) for configIr in pipelineConfigs): 

249 if subConfig.dataId is not None: 

250 raise NotImplementedError( 

251 "Specializing a config on a partial data id is not yet " 

252 "supported in Pipeline definition" 

253 ) 

254 # only apply override if it applies to everything 

255 if subConfig.dataId is None: 

256 if subConfig.file: 

257 for configFile in subConfig.file: 

258 overrides.addFileOverride(os.path.expandvars(configFile)) 

259 if subConfig.python is not None: 

260 overrides.addPythonOverride(subConfig.python) 

261 for key, value in subConfig.rest.items(): 

262 overrides.addValueOverride(key, value) 

263 overrides.applyTo(self)