Coverage for python/lsst/pipe/base/config.py: 65%

Shortcuts on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

52 statements  

1# This file is part of pipe_base. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22"""Module defining config classes for PipelineTask. 

23""" 

24__all__ = ["ResourceConfig", "PipelineTaskConfig"] 

25 

26# ------------------------------- 

27# Imports of standard modules -- 

28# ------------------------------- 

29from numbers import Number 

30 

31# ----------------------------- 

32# Imports for other modules -- 

33# ----------------------------- 

34import lsst.pex.config as pexConfig 

35 

36from .connections import PipelineTaskConnections 

37 

38# ---------------------------------- 

39# Local non-exported definitions -- 

40# ---------------------------------- 

41 

42# ------------------------ 

43# Exported definitions -- 

44# ------------------------ 

45 

46 

47class TemplateField(pexConfig.Field): 

48 """This Field is specialized for use with connection templates. 

49 Specifically it treats strings or numbers as valid input, as occasionally 

50 numbers are used as a cycle counter in templates. 

51 

52 The reason for the specialized field, is that when numbers are involved 

53 with the config override system through pipelines or from the command line, 

54 sometimes the quoting to get appropriate values as strings gets 

55 complicated. This will simplify the process greatly. 

56 """ 

57 

58 def _validateValue(self, value): 

59 if value is None: 

60 return 

61 

62 if not (isinstance(value, str) or isinstance(value, Number)): 

63 raise TypeError( 

64 f"Value {value} is of incorrect type {pexConfig.config._typeStr(value)}." 

65 f" Expected type str or a number" 

66 ) 

67 if self.check is not None and not self.check(value): 

68 ValueError("Value {value} is not a valid value") 

69 

70 def __set__(self, instance, value, at=None, label="assignment"): 

71 # validate first, even though validate will be called in super 

72 self._validateValue(value) 

73 # now, explicitly make it into a string 

74 value = str(value) 

75 super().__set__(instance, value, at, label) 

76 

77 

78class PipelineTaskConfigMeta(pexConfig.ConfigMeta): 

79 """Metaclass used in the creation of PipelineTaskConfig classes 

80 

81 This metaclass ensures a `PipelineTaskConnections` class is specified in 

82 the class construction parameters with a parameter name of 

83 pipelineConnections. Using the supplied connection class, this metaclass 

84 constructs a `lsst.pex.config.Config` instance which can be used to 

85 configure the connections class. This config is added to the config class 

86 under declaration with the name "connections" used as an identifier. The 

87 connections config also has a reference to the connections class used in 

88 its construction associated with an atttribute named `ConnectionsClass`. 

89 Finally the newly constructed config class (not an instance of it) is 

90 assigned to the Config class under construction with the attribute name 

91 `ConnectionsConfigClass`. 

92 """ 

93 

94 def __new__(cls, name, bases, dct, **kwargs): 

95 if name != "PipelineTaskConfig": 

96 # Verify that a connection class was specified and the argument is 

97 # an instance of PipelineTaskConfig 

98 if "pipelineConnections" not in kwargs: 98 ↛ 99line 98 didn't jump to line 99, because the condition on line 98 was never true

99 for base in bases: 

100 if hasattr(base, "connections"): 

101 kwargs["pipelineConnections"] = base.connections.dtype.ConnectionsClass 

102 break 

103 if "pipelineConnections" not in kwargs: 103 ↛ 104line 103 didn't jump to line 104, because the condition on line 103 was never true

104 raise NameError("PipelineTaskConfig or a base class must be defined with connections class") 

105 connectionsClass = kwargs["pipelineConnections"] 

106 if not issubclass(connectionsClass, PipelineTaskConnections): 106 ↛ 107line 106 didn't jump to line 107, because the condition on line 106 was never true

107 raise ValueError("Can only assign a PipelineTaskConnections Class to pipelineConnections") 

108 

109 # Create all the fields that will be used in the newly created sub 

110 # config (under the attribute name "connections") 

111 configConnectionsNamespace = {} 

112 for fieldName, obj in connectionsClass.allConnections.items(): 

113 configConnectionsNamespace[fieldName] = pexConfig.Field( 

114 dtype=str, doc=f"name for connection {fieldName}", default=obj.name 

115 ) 

116 # If there are default templates also add them as fields to 

117 # configure the template values 

118 if hasattr(connectionsClass, "defaultTemplates"): 118 ↛ 126line 118 didn't jump to line 126, because the condition on line 118 was never false

119 docString = "Template parameter used to format corresponding field template parameter" 

120 for templateName, default in connectionsClass.defaultTemplates.items(): 

121 configConnectionsNamespace[templateName] = TemplateField( 

122 dtype=str, doc=docString, default=default 

123 ) 

124 # add a reference to the connection class used to create this sub 

125 # config 

126 configConnectionsNamespace["ConnectionsClass"] = connectionsClass 

127 

128 # Create a new config class with the fields defined above 

129 Connections = type("Connections", (pexConfig.Config,), configConnectionsNamespace) 

130 # add it to the Config class that is currently being declared 

131 dct["connections"] = pexConfig.ConfigField( 

132 dtype=Connections, 

133 doc="Configurations describing the connections of the PipelineTask to datatypes", 

134 ) 

135 dct["ConnectionsConfigClass"] = Connections 

136 dct["ConnectionsClass"] = connectionsClass 

137 inst = super().__new__(cls, name, bases, dct) 

138 return inst 

139 

140 def __init__(self, name, bases, dct, **kwargs): 

141 # This overrides the default init to drop the kwargs argument. Python 

142 # metaclasses will have this argument set if any kwargs are passes at 

143 # class construction time, but should be consumed before calling 

144 # __init__ on the type metaclass. This is in accordance with python 

145 # documentation on metaclasses 

146 super().__init__(name, bases, dct) 

147 

148 

149class PipelineTaskConfig(pexConfig.Config, metaclass=PipelineTaskConfigMeta): 

150 """Configuration class for `PipelineTask` 

151 

152 This Configuration class functions in largely the same manner as any other 

153 derived from `lsst.pex.config.Config`. The only difference is in how it is 

154 declared. `PipelineTaskConfig` children need to be declared with a 

155 pipelineConnections argument. This argument should specify a child class of 

156 `PipelineTaskConnections`. During the declaration of a `PipelineTaskConfig` 

157 a config class is created with information from the supplied connections 

158 class to allow configuration of the connections class. This dynamically 

159 created config class is then attached to the `PipelineTaskConfig` via a 

160 `~lsst.pex.config.ConfigField` with the attribute name `connections`. 

161 """ 

162 

163 saveMetadata = pexConfig.Field( 

164 dtype=bool, 

165 default=True, 

166 optional=False, 

167 doc="Flag to enable/disable metadata saving for a task, enabled by default.", 

168 ) 

169 saveLogOutput = pexConfig.Field( 

170 dtype=bool, 

171 default=True, 

172 optional=False, 

173 doc="Flag to enable/disable saving of log output for a task, enabled by default.", 

174 ) 

175 

176 

177class ResourceConfig(pexConfig.Config): 

178 """Configuration for resource requirements. 

179 

180 This configuration class will be used by some activators to estimate 

181 resource use by pipeline. Additionally some tasks could use it to adjust 

182 their resource use (e.g. reduce the number of threads). 

183 

184 For some resources their limit can be estimated by corresponding task, 

185 in that case task could set the field value. For many fields defined in 

186 this class their associated resource used by a task will depend on the 

187 size of the data and is not known in advance. For these resources their 

188 value will be configured through overrides based on some external 

189 estimates. 

190 """ 

191 

192 minMemoryMB = pexConfig.Field( 

193 dtype=int, 

194 default=None, 

195 optional=True, 

196 doc="Minimal memory needed by task, can be None if estimate is unknown.", 

197 ) 

198 minNumCores = pexConfig.Field(dtype=int, default=1, doc="Minimal number of cores needed by task.")