Coverage for python/lsst/pipe/base/config.py: 65%

Shortcuts on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

52 statements  

1# This file is part of pipe_base. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22"""Module defining config classes for PipelineTask. 

23""" 

24__all__ = ["ResourceConfig", "PipelineTaskConfig"] 

25 

26# ------------------------------- 

27# Imports of standard modules -- 

28# ------------------------------- 

29from numbers import Number 

30 

31# ----------------------------- 

32# Imports for other modules -- 

33# ----------------------------- 

34import lsst.pex.config as pexConfig 

35from .connections import PipelineTaskConnections 

36 

37# ---------------------------------- 

38# Local non-exported definitions -- 

39# ---------------------------------- 

40 

41# ------------------------ 

42# Exported definitions -- 

43# ------------------------ 

44 

45 

46class TemplateField(pexConfig.Field): 

47 """This Field is specialized for use with connection templates. 

48 Specifically it treats strings or numbers as valid input, as occasionally 

49 numbers are used as a cycle counter in templates. 

50 

51 The reason for the specialized field, is that when numbers are involved 

52 with the config override system through pipelines or from the command line, 

53 sometimes the quoting to get appropriate values as strings gets 

54 complicated. This will simplify the process greatly. 

55 """ 

56 def _validateValue(self, value): 

57 if value is None: 

58 return 

59 

60 if not (isinstance(value, str) or isinstance(value, Number)): 

61 raise TypeError(f"Value {value} is of incorrect type {pexConfig.config._typeStr(value)}." 

62 f" Expected type str or a number") 

63 if self.check is not None and not self.check(value): 

64 ValueError("Value {value} is not a valid value") 

65 

66 def __set__(self, instance, value, at=None, label='assignment'): 

67 # validate first, even though validate will be called in super 

68 self._validateValue(value) 

69 # now, explicitly make it into a string 

70 value = str(value) 

71 super().__set__(instance, value, at, label) 

72 

73 

74class PipelineTaskConfigMeta(pexConfig.ConfigMeta): 

75 """Metaclass used in the creation of PipelineTaskConfig classes 

76 

77 This metaclass ensures a `PipelineTaskConnections` class is specified in 

78 the class construction parameters with a parameter name of 

79 pipelineConnections. Using the supplied connection class, this metaclass 

80 constructs a `lsst.pex.config.Config` instance which can be used to 

81 configure the connections class. This config is added to the config class 

82 under declaration with the name "connections" used as an identifier. The 

83 connections config also has a reference to the connections class used in 

84 its construction associated with an atttribute named `ConnectionsClass`. 

85 Finally the newly constructed config class (not an instance of it) is 

86 assigned to the Config class under construction with the attribute name 

87 `ConnectionsConfigClass`. 

88 """ 

89 def __new__(cls, name, bases, dct, **kwargs): 

90 if name != "PipelineTaskConfig": 

91 # Verify that a connection class was specified and the argument is 

92 # an instance of PipelineTaskConfig 

93 if 'pipelineConnections' not in kwargs: 93 ↛ 94line 93 didn't jump to line 94, because the condition on line 93 was never true

94 for base in bases: 

95 if hasattr(base, "connections"): 

96 kwargs['pipelineConnections'] = base.connections.dtype.ConnectionsClass 

97 break 

98 if 'pipelineConnections' not in kwargs: 98 ↛ 99line 98 didn't jump to line 99, because the condition on line 98 was never true

99 raise NameError("PipelineTaskConfig or a base class must be defined with connections class") 

100 connectionsClass = kwargs['pipelineConnections'] 

101 if not issubclass(connectionsClass, PipelineTaskConnections): 101 ↛ 102line 101 didn't jump to line 102, because the condition on line 101 was never true

102 raise ValueError("Can only assign a PipelineTaskConnections Class to pipelineConnections") 

103 

104 # Create all the fields that will be used in the newly created sub 

105 # config (under the attribute name "connections") 

106 configConnectionsNamespace = {} 

107 for fieldName, obj in connectionsClass.allConnections.items(): 

108 configConnectionsNamespace[fieldName] = pexConfig.Field(dtype=str, 

109 doc=f"name for " 

110 f"connection {fieldName}", 

111 default=obj.name) 

112 # If there are default templates also add them as fields to 

113 # configure the template values 

114 if hasattr(connectionsClass, 'defaultTemplates'): 114 ↛ 122line 114 didn't jump to line 122, because the condition on line 114 was never false

115 docString = "Template parameter used to format corresponding field template parameter" 

116 for templateName, default in connectionsClass.defaultTemplates.items(): 

117 configConnectionsNamespace[templateName] = TemplateField(dtype=str, 

118 doc=docString, 

119 default=default) 

120 # add a reference to the connection class used to create this sub 

121 # config 

122 configConnectionsNamespace['ConnectionsClass'] = connectionsClass 

123 

124 # Create a new config class with the fields defined above 

125 Connections = type("Connections", (pexConfig.Config,), configConnectionsNamespace) 

126 # add it to the Config class that is currently being declared 

127 dct['connections'] = pexConfig.ConfigField(dtype=Connections, 

128 doc='Configurations describing the' 

129 ' connections of the PipelineTask to datatypes') 

130 dct['ConnectionsConfigClass'] = Connections 

131 dct['ConnectionsClass'] = connectionsClass 

132 inst = super().__new__(cls, name, bases, dct) 

133 return inst 

134 

135 def __init__(self, name, bases, dct, **kwargs): 

136 # This overrides the default init to drop the kwargs argument. Python 

137 # metaclasses will have this argument set if any kwargs are passes at 

138 # class construction time, but should be consumed before calling 

139 # __init__ on the type metaclass. This is in accordance with python 

140 # documentation on metaclasses 

141 super().__init__(name, bases, dct) 

142 

143 

144class PipelineTaskConfig(pexConfig.Config, metaclass=PipelineTaskConfigMeta): 

145 """Configuration class for `PipelineTask` 

146 

147 This Configuration class functions in largely the same manner as any other 

148 derived from `lsst.pex.config.Config`. The only difference is in how it is 

149 declared. `PipelineTaskConfig` children need to be declared with a 

150 pipelineConnections argument. This argument should specify a child class of 

151 `PipelineTaskConnections`. During the declaration of a `PipelineTaskConfig` 

152 a config class is created with information from the supplied connections 

153 class to allow configuration of the connections class. This dynamically 

154 created config class is then attached to the `PipelineTaskConfig` via a 

155 `~lsst.pex.config.ConfigField` with the attribute name `connections`. 

156 """ 

157 saveMetadata = pexConfig.Field( 

158 dtype=bool, default=True, optional=False, 

159 doc="Flag to enable/disable metadata saving for a task, enabled by default.") 

160 saveLogOutput = pexConfig.Field( 

161 dtype=bool, default=True, optional=False, 

162 doc="Flag to enable/disable saving of log output for a task, enabled by default.") 

163 

164 

165class ResourceConfig(pexConfig.Config): 

166 """Configuration for resource requirements. 

167 

168 This configuration class will be used by some activators to estimate 

169 resource use by pipeline. Additionally some tasks could use it to adjust 

170 their resource use (e.g. reduce the number of threads). 

171 

172 For some resources their limit can be estimated by corresponding task, 

173 in that case task could set the field value. For many fields defined in 

174 this class their associated resource used by a task will depend on the 

175 size of the data and is not known in advance. For these resources their 

176 value will be configured through overrides based on some external 

177 estimates. 

178 """ 

179 minMemoryMB = pexConfig.Field(dtype=int, default=None, optional=True, 

180 doc="Minimal memory needed by task, can be None if estimate is unknown.") 

181 minNumCores = pexConfig.Field(dtype=int, default=1, 

182 doc="Minimal number of cores needed by task.")