Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of pipe_base. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22"""Module defining config classes for PipelineTask. 

23""" 

24__all__ = ["ResourceConfig", "PipelineTaskConfig"] 

25 

26# ------------------------------- 

27# Imports of standard modules -- 

28# ------------------------------- 

29 

30# ----------------------------- 

31# Imports for other modules -- 

32# ----------------------------- 

33import lsst.pex.config as pexConfig 

34from .connections import PipelineTaskConnections 

35 

36# ---------------------------------- 

37# Local non-exported definitions -- 

38# ---------------------------------- 

39 

40# ------------------------ 

41# Exported definitions -- 

42# ------------------------ 

43 

44 

45class PipelineTaskConfigMeta(pexConfig.ConfigMeta): 

46 """Metaclass used in the creation of PipelineTaskConfig classes 

47 

48 This metaclass ensures a `PipelineTaskConnections` class is specified in 

49 the class construction parameters with a parameter name of 

50 pipelineConnections. Using the supplied connection class, this metaclass 

51 constructs a `lsst.pex.config.Config` instance which can be used to 

52 configure the connections class. This config is added to the config class 

53 under declaration with the name "connections" used as an identifier. The 

54 connections config also has a reference to the connections class used in 

55 its construction associated with an atttribute named `ConnectionsClass`. 

56 Finally the newly constructed config class (not an instance of it) is 

57 assigned to the Config class under construction with the attribute name 

58 `ConnectionsConfigClass`. 

59 """ 

60 def __new__(cls, name, bases, dct, **kwargs): 

61 if name != "PipelineTaskConfig": 

62 # Verify that a connection class was specified and the argument is an instance of 

63 # PipelineTaskConfig 

64 if 'pipelineConnections' not in kwargs: 64 ↛ 65line 64 didn't jump to line 65, because the condition on line 64 was never true

65 for base in bases: 

66 if hasattr(base, "connections"): 

67 kwargs['pipelineConnections'] = base.connections.dtype.ConnectionsClass 

68 break 

69 if 'pipelineConnections' not in kwargs: 69 ↛ 70line 69 didn't jump to line 70, because the condition on line 69 was never true

70 raise NameError("PipelineTaskConfig or a base class must be defined with connections class") 

71 connectionsClass = kwargs['pipelineConnections'] 

72 if not issubclass(connectionsClass, PipelineTaskConnections): 72 ↛ 73line 72 didn't jump to line 73, because the condition on line 72 was never true

73 raise ValueError("Can only assign a PipelineTaskConnections Class to pipelineConnections") 

74 

75 # Create all the fields that will be used in the newly created sub config 

76 # (under the attribute name "connections") 

77 configConnectionsNamespace = {} 

78 for fieldName, obj in connectionsClass.allConnections.items(): 

79 configConnectionsNamespace[fieldName] = pexConfig.Field(dtype=str, 

80 doc=f"name for " 

81 f"connection {fieldName}", 

82 default=obj.name) 

83 # If there are default templates also add them as fields to configure the template values 

84 if hasattr(connectionsClass, 'defaultTemplates'): 84 ↛ 91line 84 didn't jump to line 91, because the condition on line 84 was never false

85 docString = "Template parameter used to format corresponding field template parameter" 

86 for templateName, default in connectionsClass.defaultTemplates.items(): 86 ↛ 87line 86 didn't jump to line 87, because the loop on line 86 never started

87 configConnectionsNamespace[templateName] = pexConfig.Field(dtype=str, 

88 doc=docString, 

89 default=default) 

90 # add a reference to the connection class used to create this sub config 

91 configConnectionsNamespace['ConnectionsClass'] = connectionsClass 

92 

93 # Create a new config class with the fields defined above 

94 Connections = type("Connections", (pexConfig.Config,), configConnectionsNamespace) 

95 # add it to the Config class that is currently being declared 

96 dct['connections'] = pexConfig.ConfigField(dtype=Connections, 

97 doc='Configurations describing the' 

98 ' connections of the PipelineTask to datatypes') 

99 dct['ConnectionsConfigClass'] = Connections 

100 dct['ConnectionsClass'] = connectionsClass 

101 inst = super().__new__(cls, name, bases, dct) 

102 return inst 

103 

104 def __init__(self, name, bases, dct, **kwargs): 

105 # This overrides the default init to drop the kwargs argument. Python 

106 # metaclasses will have this argument set if any kwargs are passes at 

107 # class construction time, but should be consumed before calling 

108 # __init__ on the type metaclass. This is in accordance with python 

109 # documentation on metaclasses 

110 super().__init__(name, bases, dct) 

111 

112 

113class PipelineTaskConfig(pexConfig.Config, metaclass=PipelineTaskConfigMeta): 

114 """Configuration class for `PipelineTask` 

115 

116 This Configuration class functions in largely the same manner as any other 

117 derived from `lsst.pex.config.Config`. The only difference is in how it is 

118 declared. `PipelineTaskConfig` children need to be declared with a 

119 pipelineConnections argument. This argument should specify a child class of 

120 `PipelineTaskConnections`. During the declaration of a `PipelineTaskConfig` 

121 a config class is created with information from the supplied connections 

122 class to allow configuration of the connections class. This dynamically 

123 created config class is then attached to the `PipelineTaskConfig` via a 

124 `~lsst.pex.config.ConfigField` with the attribute name `connections`. 

125 """ 

126 saveMetadata = pexConfig.Field( 

127 dtype=bool, default=True, optional=False, 

128 doc="Flag to enable/disable metadata saving for a task, enabled by default.") 

129 

130 

131class ResourceConfig(pexConfig.Config): 

132 """Configuration for resource requirements. 

133 

134 This configuration class will be used by some activators to estimate 

135 resource use by pipeline. Additionally some tasks could use it to adjust 

136 their resource use (e.g. reduce the number of threads). 

137 

138 For some resources their limit can be estimated by corresponding task, 

139 in that case task could set the field value. For many fields defined in 

140 this class their associated resource used by a task will depend on the 

141 size of the data and is not known in advance. For these resources their 

142 value will be configured through overrides based on some external 

143 estimates. 

144 """ 

145 minMemoryMB = pexConfig.Field(dtype=int, default=None, optional=True, 

146 doc="Minimal memory needed by task, can be None if estimate is unknown.") 

147 minNumCores = pexConfig.Field(dtype=int, default=1, 

148 doc="Minimal number of cores needed by task.")