Coverage for python/lsst/ctrl/bps/parsl/site.py: 54%

47 statements  

« prev     ^ index     » next       coverage.py v7.3.0, created at 2023-08-31 09:55 +0000

1from abc import ABC, abstractmethod 

2from types import ModuleType 

3from typing import TYPE_CHECKING 

4 

5from lsst.ctrl.bps import BpsConfig 

6from lsst.utils import doImport 

7from parsl.addresses import address_by_hostname 

8from parsl.executors.base import ParslExecutor 

9from parsl.monitoring import MonitoringHub 

10 

11from .configuration import get_bps_config_value, get_workflow_name 

12from .environment import export_environment 

13 

14if TYPE_CHECKING: 14 ↛ 15line 14 didn't jump to line 15, because the condition on line 14 was never true

15 from .job import ParslJob 

16 

17__all__ = ("SiteConfig",) 

18 

19 

20class SiteConfig(ABC): 

21 """Base class for site configuration 

22 

23 Subclasses need to override at least the ``get_executors`` and 

24 ``select_executor`` methods. 

25 

26 Parameters 

27 ---------- 

28 config : `BpsConfig` 

29 BPS configuration. 

30 add_resources : `bool` 

31 Add resource specification when submitting the job? This is only 

32 appropriate for the ``WorkQueue`` executor; other executors will 

33 raise an exception. 

34 """ 

35 

36 def __init__(self, config: BpsConfig, add_resources: bool = False): 

37 self.config = config 

38 self.site = self.get_site_subconfig(config) 

39 self.add_resources = add_resources 

40 

41 @staticmethod 

42 def get_site_subconfig(config: BpsConfig) -> BpsConfig: 

43 """Get BPS configuration for the site of interest 

44 

45 We return the BPS sub-configuration for the site indicated by the 

46 ``computeSite`` value, which is ``site.<computeSite>``. 

47 

48 Parameters 

49 ---------- 

50 config : `BpsConfig` 

51 BPS configuration. 

52 

53 Returns 

54 ------- 

55 site : `BpsConfig` 

56 Site sub-configuration. 

57 """ 

58 computeSite = get_bps_config_value(config, "computeSite", str, required=True) 

59 return get_bps_config_value(config, f".site.{computeSite}", BpsConfig, required=True) 

60 

61 @classmethod 

62 def from_config(cls, config: BpsConfig) -> "SiteConfig": 

63 """Get the site configuration nominated in the BPS config 

64 

65 The ``computeSite`` (`str`) value in the BPS configuration is used to 

66 select a site configuration. The site configuration class to use is 

67 specified by the BPS configuration as ``site.<computeSite>.class`` 

68 (`str`), which should be the fully-qualified name of a python class 

69 that inherits from `SiteConfig`. 

70 

71 Parameters 

72 ---------- 

73 config : `BpsConfig` 

74 BPS configuration. 

75 

76 Returns 

77 ------- 

78 site_config : subclass of `SiteConfig` 

79 Site configuration. 

80 """ 

81 site = cls.get_site_subconfig(config) 

82 name = get_bps_config_value(site, "class", str, required=True) 

83 site_config = doImport(name) 

84 if isinstance(site_config, ModuleType) or not issubclass(site_config, SiteConfig): 

85 raise RuntimeError(f"Site class={name} is not a SiteConfig subclass") 

86 return site_config(config) 

87 

88 @abstractmethod 

89 def get_executors(self) -> list[ParslExecutor]: 

90 """Get a list of executors to be used in processing 

91 

92 Each executor should have a unique ``label``. 

93 """ 

94 raise NotImplementedError("Subclasses must define") 

95 

96 @abstractmethod 

97 def select_executor(self, job: "ParslJob") -> str: 

98 """Get the ``label`` of the executor to use to execute a job 

99 

100 Parameters 

101 ---------- 

102 job : `ParslJob` 

103 Job to be executed. 

104 

105 Returns 

106 ------- 

107 label : `str` 

108 Label of executor to use to execute ``job``. 

109 """ 

110 raise NotImplementedError("Subclasses must define") 

111 

112 def get_address(self) -> str: 

113 """Return the IP address of the machine hosting the driver/submission 

114 

115 This address should be accessible from the workers. This should 

116 generally by the return value of one of the functions in 

117 ``parsl.addresses``. 

118 

119 This is used by the default implementation of ``get_monitor``, but will 

120 generally be used by ``get_executors`` too. 

121 

122 This default implementation gets the address from the hostname, but 

123 that will not work if the workers don't access the driver/submission 

124 node by that address. 

125 """ 

126 return address_by_hostname() 

127 

128 def get_command_prefix(self) -> str: 

129 """Return command(s) to add before each job command 

130 

131 These may be used to configure the environment for the job. 

132 

133 This default implementation respects the BPS configuration elements: 

134 

135 - ``site.<computeSite>.commandPrefix`` (`str`): command(s) to use as a 

136 prefix to executing a job command on a worker. 

137 - ``site.<computeSite>.environment`` (`bool`): add bash commands that 

138 replicate the environment on the driver/submit machine? 

139 """ 

140 prefix = get_bps_config_value(self.site, "commandPrefix", str, "") 

141 if get_bps_config_value(self.site, "environment", bool, False): 

142 prefix += "\n" + export_environment() 

143 return prefix 

144 

145 def get_monitor(self) -> MonitoringHub | None: 

146 """Get parsl monitor 

147 

148 The parsl monitor provides a database that tracks the progress of the 

149 workflow and the use of resources on the workers. 

150 

151 This implementation respects the BPS configuration elements: 

152 

153 - ``site.<computeSite>.monitorEnable`` (`bool`): enable monitor? 

154 - ``site.<computeSite>.monitorInterval`` (`float`): time interval (sec) 

155 between logging of resource usage. 

156 - ``site.<computeSite>.monitorFilename`` (`str`): name of file to use 

157 for the monitor sqlite database. 

158 

159 Returns 

160 ------- 

161 monitor : `MonitoringHub` or `None` 

162 Parsl monitor, or `None` for no monitor. 

163 """ 

164 if not get_bps_config_value(self.site, "monitorEnable", bool, False): 

165 return None 

166 return MonitoringHub( 

167 workflow_name=get_workflow_name(self.config), 

168 hub_address=self.get_address(), 

169 resource_monitoring_interval=get_bps_config_value(self.site, "monitorInterval", float, 30.0), 

170 logging_endpoint="sqlite:///" 

171 + get_bps_config_value(self.site, "monitorFilename", str, "monitor.sqlite"), 

172 )