Coverage for python/lsst/pipe/base/config.py: 64%

59 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2023-04-12 09:13 +0000

1# This file is part of pipe_base. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22"""Module defining config classes for PipelineTask. 

23""" 

24 

25from __future__ import annotations 

26 

27__all__ = ["ResourceConfig", "PipelineTaskConfig"] 

28 

29# ------------------------------- 

30# Imports of standard modules -- 

31# ------------------------------- 

32from numbers import Number 

33from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple, Type, TypeVar 

34 

35# ----------------------------- 

36# Imports for other modules -- 

37# ----------------------------- 

38import lsst.pex.config as pexConfig 

39 

40from .connections import PipelineTaskConnections 

41 

42if TYPE_CHECKING: 42 ↛ 43line 42 didn't jump to line 43, because the condition on line 42 was never true

43 from lsst.pex.config.callStack import StackFrame 

44 

45# ---------------------------------- 

46# Local non-exported definitions -- 

47# ---------------------------------- 

48 

49_S = TypeVar("_S", bound="PipelineTaskConfigMeta") 

50 

51# ------------------------ 

52# Exported definitions -- 

53# ------------------------ 

54 

55 

56class TemplateField(pexConfig.Field): 

57 """This Field is specialized for use with connection templates. 

58 Specifically it treats strings or numbers as valid input, as occasionally 

59 numbers are used as a cycle counter in templates. 

60 

61 The reason for the specialized field, is that when numbers are involved 

62 with the config override system through pipelines or from the command line, 

63 sometimes the quoting to get appropriate values as strings gets 

64 complicated. This will simplify the process greatly. 

65 """ 

66 

67 def _validateValue(self, value: Any) -> None: 

68 if value is None: 

69 return 

70 

71 if not (isinstance(value, str) or isinstance(value, Number)): 

72 raise TypeError( 

73 f"Value {value} is of incorrect type {pexConfig.config._typeStr(value)}." 

74 " Expected type str or a number" 

75 ) 

76 if self.check is not None and not self.check(value): 

77 ValueError("Value {value} is not a valid value") 

78 

79 def __set__( 

80 self, 

81 instance: pexConfig.Config, 

82 value: Any, 

83 at: Optional[StackFrame] = None, 

84 label: str = "assignment", 

85 ) -> None: 

86 # validate first, even though validate will be called in super 

87 self._validateValue(value) 

88 # now, explicitly make it into a string 

89 value = str(value) 

90 super().__set__(instance, value, at, label) 

91 

92 

93class PipelineTaskConfigMeta(pexConfig.ConfigMeta): 

94 """Metaclass used in the creation of PipelineTaskConfig classes 

95 

96 This metaclass ensures a `PipelineTaskConnections` class is specified in 

97 the class construction parameters with a parameter name of 

98 pipelineConnections. Using the supplied connection class, this metaclass 

99 constructs a `lsst.pex.config.Config` instance which can be used to 

100 configure the connections class. This config is added to the config class 

101 under declaration with the name "connections" used as an identifier. The 

102 connections config also has a reference to the connections class used in 

103 its construction associated with an atttribute named `ConnectionsClass`. 

104 Finally the newly constructed config class (not an instance of it) is 

105 assigned to the Config class under construction with the attribute name 

106 `ConnectionsConfigClass`. 

107 """ 

108 

109 def __new__( 

110 cls: Type[_S], 

111 name: str, 

112 bases: Tuple[type[PipelineTaskConfig], ...], 

113 dct: Dict[str, Any], 

114 **kwargs: Any, 

115 ) -> _S: 

116 if name != "PipelineTaskConfig": 

117 # Verify that a connection class was specified and the argument is 

118 # an instance of PipelineTaskConfig 

119 if "pipelineConnections" not in kwargs: 119 ↛ 120line 119 didn't jump to line 120, because the condition on line 119 was never true

120 for base in bases: 

121 if hasattr(base, "connections"): 

122 kwargs["pipelineConnections"] = base.connections.dtype.ConnectionsClass 

123 break 

124 if "pipelineConnections" not in kwargs: 124 ↛ 125line 124 didn't jump to line 125, because the condition on line 124 was never true

125 raise NameError("PipelineTaskConfig or a base class must be defined with connections class") 

126 connectionsClass = kwargs["pipelineConnections"] 

127 if not issubclass(connectionsClass, PipelineTaskConnections): 127 ↛ 128line 127 didn't jump to line 128, because the condition on line 127 was never true

128 raise ValueError("Can only assign a PipelineTaskConnections Class to pipelineConnections") 

129 

130 # Create all the fields that will be used in the newly created sub 

131 # config (under the attribute name "connections") 

132 configConnectionsNamespace: dict[str, pexConfig.Field] = {} 

133 for fieldName, obj in connectionsClass.allConnections.items(): 

134 configConnectionsNamespace[fieldName] = pexConfig.Field[str]( 

135 doc=f"name for connection {fieldName}", default=obj.name 

136 ) 

137 # If there are default templates also add them as fields to 

138 # configure the template values 

139 if hasattr(connectionsClass, "defaultTemplates"): 139 ↛ 147line 139 didn't jump to line 147, because the condition on line 139 was never false

140 docString = "Template parameter used to format corresponding field template parameter" 

141 for templateName, default in connectionsClass.defaultTemplates.items(): 

142 configConnectionsNamespace[templateName] = TemplateField( 

143 dtype=str, doc=docString, default=default 

144 ) 

145 # add a reference to the connection class used to create this sub 

146 # config 

147 configConnectionsNamespace["ConnectionsClass"] = connectionsClass 

148 

149 # Create a new config class with the fields defined above 

150 Connections = type("Connections", (pexConfig.Config,), configConnectionsNamespace) 

151 # add it to the Config class that is currently being declared 

152 dct["connections"] = pexConfig.ConfigField( 

153 dtype=Connections, 

154 doc="Configurations describing the connections of the PipelineTask to datatypes", 

155 ) 

156 dct["ConnectionsConfigClass"] = Connections 

157 dct["ConnectionsClass"] = connectionsClass 

158 inst = super().__new__(cls, name, bases, dct) 

159 return inst 

160 

161 def __init__( 

162 self, name: str, bases: Tuple[Type[PipelineTaskConfig], ...], dct: Dict[str, Any], **kwargs: Any 

163 ): 

164 # This overrides the default init to drop the kwargs argument. Python 

165 # metaclasses will have this argument set if any kwargs are passes at 

166 # class construction time, but should be consumed before calling 

167 # __init__ on the type metaclass. This is in accordance with python 

168 # documentation on metaclasses 

169 super().__init__(name, bases, dct) 

170 

171 

172class PipelineTaskConfig(pexConfig.Config, metaclass=PipelineTaskConfigMeta): 

173 """Configuration class for `PipelineTask` 

174 

175 This Configuration class functions in largely the same manner as any other 

176 derived from `lsst.pex.config.Config`. The only difference is in how it is 

177 declared. `PipelineTaskConfig` children need to be declared with a 

178 pipelineConnections argument. This argument should specify a child class of 

179 `PipelineTaskConnections`. During the declaration of a `PipelineTaskConfig` 

180 a config class is created with information from the supplied connections 

181 class to allow configuration of the connections class. This dynamically 

182 created config class is then attached to the `PipelineTaskConfig` via a 

183 `~lsst.pex.config.ConfigField` with the attribute name `connections`. 

184 """ 

185 

186 connections: pexConfig.ConfigField 

187 """Field which refers to a dynamically added configuration class which is 

188 based on a PipelineTaskConnections class. 

189 """ 

190 

191 saveMetadata = pexConfig.Field[bool]( 

192 default=True, 

193 optional=False, 

194 doc="Flag to enable/disable metadata saving for a task, enabled by default.", 

195 ) 

196 saveLogOutput = pexConfig.Field[bool]( 

197 default=True, 

198 optional=False, 

199 doc="Flag to enable/disable saving of log output for a task, enabled by default.", 

200 ) 

201 

202 

203class ResourceConfig(pexConfig.Config): 

204 """Configuration for resource requirements. 

205 

206 This configuration class will be used by some activators to estimate 

207 resource use by pipeline. Additionally some tasks could use it to adjust 

208 their resource use (e.g. reduce the number of threads). 

209 

210 For some resources their limit can be estimated by corresponding task, 

211 in that case task could set the field value. For many fields defined in 

212 this class their associated resource used by a task will depend on the 

213 size of the data and is not known in advance. For these resources their 

214 value will be configured through overrides based on some external 

215 estimates. 

216 """ 

217 

218 minMemoryMB = pexConfig.Field[int]( 

219 default=None, 

220 optional=True, 

221 doc="Minimal memory needed by task, can be None if estimate is unknown.", 

222 ) 

223 minNumCores = pexConfig.Field[int](default=1, doc="Minimal number of cores needed by task.")