Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

23

24

25

26

27

28

29

30

31

32

33

34

35

36

37

38

39

40

41

42

43

44

45

46

47

48

49

50

51

52

53

54

55

56

57

58

59

60

61

62

63

64

65

66

67

68

69

70

71

72

73

74

75

76

77

78

79

80

81

82

83

84

85

86

87

88

89

90

91

92

93

94

95

96

97

98

99

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

# This file is part of pipe_base. 

# 

# Developed for the LSST Data Management System. 

# This product includes software developed by the LSST Project 

# (http://www.lsst.org). 

# See the COPYRIGHT file at the top-level directory of this distribution 

# for details of code ownership. 

# 

# This program is free software: you can redistribute it and/or modify 

# it under the terms of the GNU General Public License as published by 

# the Free Software Foundation, either version 3 of the License, or 

# (at your option) any later version. 

# 

# This program is distributed in the hope that it will be useful, 

# but WITHOUT ANY WARRANTY; without even the implied warranty of 

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

# GNU General Public License for more details. 

# 

# You should have received a copy of the GNU General Public License 

# along with this program. If not, see <http://www.gnu.org/licenses/>. 

 

"""Module defining config classes for PipelineTask. 

""" 

__all__ = ["ResourceConfig", "PipelineTaskConfig"] 

 

# ------------------------------- 

# Imports of standard modules -- 

# ------------------------------- 

 

# ----------------------------- 

# Imports for other modules -- 

# ----------------------------- 

import lsst.pex.config as pexConfig 

from .connections import PipelineTaskConnections 

 

# ---------------------------------- 

# Local non-exported definitions -- 

# ---------------------------------- 

 

# ------------------------ 

# Exported definitions -- 

# ------------------------ 

 

 

class PipelineTaskConfigMeta(pexConfig.ConfigMeta): 

"""Metaclass used in the creation of PipelineTaskConfig classes 

 

This metaclass ensures a `PipelineTaskConnections` class is specified in 

the class construction parameters with a parameter name of 

pipelineConnections. Using the supplied connection class, this metaclass 

constructs a `lsst.pex.config.Config` instance which can be used to 

configure the connections class. This config is added to the config class 

under declaration with the name "connections" used as an identifier. The 

connections config also has a reference to the connections class used in 

its construction associated with an atttribute named `ConnectionsClass`. 

Finally the newly constructed config class (not an instance of it) is 

assigned to the Config class under construction with the attribute name 

`ConnectionsConfigClass`. 

""" 

def __new__(cls, name, bases, dct, **kwargs): 

if name != "PipelineTaskConfig": 

# Verify that a connection class was specified and the argument is an instance of 

# PipelineTaskConfig 

64 ↛ 65line 64 didn't jump to line 65, because the condition on line 64 was never true if 'pipelineConnections' not in kwargs: 

for base in bases: 

if hasattr(base, "connections"): 

kwargs['pipelineConnections'] = base.connections.dtype.ConnectionsClass 

break 

69 ↛ 70line 69 didn't jump to line 70, because the condition on line 69 was never true if 'pipelineConnections' not in kwargs: 

raise NameError("PipelineTaskConfig or a base class must be defined with connections class") 

connectionsClass = kwargs['pipelineConnections'] 

72 ↛ 73line 72 didn't jump to line 73, because the condition on line 72 was never true if not issubclass(connectionsClass, PipelineTaskConnections): 

raise ValueError("Can only assign a PipelineTaskConnections Class to pipelineConnections") 

 

# Create all the fields that will be used in the newly created sub config 

# (under the attribute name "connections") 

configConnectionsNamespace = {} 

for fieldName, obj in connectionsClass.allConnections.items(): 

configConnectionsNamespace[fieldName] = pexConfig.Field(dtype=str, 

doc=f"name for " 

f"connection {fieldName}", 

default=obj.name) 

# If there are default templates also add them as fields to configure the template values 

84 ↛ 91line 84 didn't jump to line 91, because the condition on line 84 was never false if hasattr(connectionsClass, 'defaultTemplates'): 

docString = "Template parameter used to format corresponding field template parameter" 

86 ↛ 87line 86 didn't jump to line 87, because the loop on line 86 never started for templateName, default in connectionsClass.defaultTemplates.items(): 

configConnectionsNamespace[templateName] = pexConfig.Field(dtype=str, 

doc=docString, 

default=default) 

# add a reference to the connection class used to create this sub config 

configConnectionsNamespace['ConnectionsClass'] = connectionsClass 

 

# Create a new config class with the fields defined above 

Connections = type("Connections", (pexConfig.Config,), configConnectionsNamespace) 

# add it to the Config class that is currently being declared 

dct['connections'] = pexConfig.ConfigField(dtype=Connections, 

doc='Configurations describing the' 

' connections of the PipelineTask to datatypes') 

dct['ConnectionsConfigClass'] = Connections 

dct['ConnectionsClass'] = connectionsClass 

inst = super().__new__(cls, name, bases, dct) 

return inst 

 

def __init__(self, name, bases, dct, **kwargs): 

# This overrides the default init to drop the kwargs argument. Python 

# metaclasses will have this argument set if any kwargs are passes at 

# class construction time, but should be consumed before calling 

# __init__ on the type metaclass. This is in accordance with python 

# documentation on metaclasses 

super().__init__(name, bases, dct) 

 

 

class PipelineTaskConfig(pexConfig.Config, metaclass=PipelineTaskConfigMeta): 

"""Configuration class for `PipelineTask`s 

 

This Configuration class functions in largely the same manner as any other 

derived from `lsst.pex.config.Config`. The only difference is in how it is 

declared. `PipelineTaskConfig` children need to be declared with a 

pipelineConnections argument. This argument should specify a child class of 

`PipelineTaskConnections`. During the declaration of a `PipelineTaskConfig` 

a config class is created with information from the supplied connections 

class to allow configuration of the connections class. This dynamically 

created config class is then attached to the `PipelineTaskConfig` via a 

`~lsst.pex.config.ConfigField` with the attribute name `connections`. 

""" 

saveMetadata = pexConfig.Field( 

dtype=bool, default=True, optional=False, 

doc="Flag to enable/disable metadata saving for a task, enabled by default.") 

 

 

class ResourceConfig(pexConfig.Config): 

"""Configuration for resource requirements. 

 

This configuration class will be used by some activators to estimate 

resource use by pipeline. Additionally some tasks could use it to adjust 

their resource use (e.g. reduce the number of threads). 

 

For some resources their limit can be estimated by corresponding task, 

in that case task could set the field value. For many fields defined in 

this class their associated resource used by a task will depend on the 

size of the data and is not known in advance. For these resources their 

value will be configured through overrides based on some external 

estimates. 

""" 

minMemoryMB = pexConfig.Field(dtype=int, default=None, optional=True, 

doc="Minimal memory needed by task, can be None if estimate is unknown.") 

minNumCores = pexConfig.Field(dtype=int, default=1, 

doc="Minimal number of cores needed by task.")