Coverage for python/lsst/ctrl/bps/parsl/site.py: 54%
47 statements
« prev ^ index » next coverage.py v7.2.7, created at 2023-08-10 07:57 +0000
« prev ^ index » next coverage.py v7.2.7, created at 2023-08-10 07:57 +0000
1from abc import ABC, abstractmethod
2from types import ModuleType
3from typing import TYPE_CHECKING
5from lsst.ctrl.bps import BpsConfig
6from lsst.utils import doImport
7from parsl.addresses import address_by_hostname
8from parsl.executors.base import ParslExecutor
9from parsl.monitoring import MonitoringHub
11from .configuration import get_bps_config_value, get_workflow_name
12from .environment import export_environment
14if TYPE_CHECKING: 14 ↛ 15line 14 didn't jump to line 15, because the condition on line 14 was never true
15 from .job import ParslJob
17__all__ = ("SiteConfig",)
20class SiteConfig(ABC):
21 """Base class for site configuration
23 Subclasses need to override at least the ``get_executors`` and
24 ``select_executor`` methods.
26 Parameters
27 ----------
28 config : `BpsConfig`
29 BPS configuration.
30 add_resources : `bool`
31 Add resource specification when submitting the job? This is only
32 appropriate for the ``WorkQueue`` executor; other executors will
33 raise an exception.
34 """
36 def __init__(self, config: BpsConfig, add_resources: bool = False):
37 self.config = config
38 self.site = self.get_site_subconfig(config)
39 self.add_resources = add_resources
41 @staticmethod
42 def get_site_subconfig(config: BpsConfig) -> BpsConfig:
43 """Get BPS configuration for the site of interest
45 We return the BPS sub-configuration for the site indicated by the
46 ``computeSite`` value, which is ``site.<computeSite>``.
48 Parameters
49 ----------
50 config : `BpsConfig`
51 BPS configuration.
53 Returns
54 -------
55 site : `BpsConfig`
56 Site sub-configuration.
57 """
58 computeSite = get_bps_config_value(config, "computeSite", str, required=True)
59 return get_bps_config_value(config, f".site.{computeSite}", BpsConfig, required=True)
61 @classmethod
62 def from_config(cls, config: BpsConfig) -> "SiteConfig":
63 """Get the site configuration nominated in the BPS config
65 The ``computeSite`` (`str`) value in the BPS configuration is used to
66 select a site configuration. The site configuration class to use is
67 specified by the BPS configuration as ``site.<computeSite>.class``
68 (`str`), which should be the fully-qualified name of a python class
69 that inherits from `SiteConfig`.
71 Parameters
72 ----------
73 config : `BpsConfig`
74 BPS configuration.
76 Returns
77 -------
78 site_config : subclass of `SiteConfig`
79 Site configuration.
80 """
81 site = cls.get_site_subconfig(config)
82 name = get_bps_config_value(site, "class", str, required=True)
83 site_config = doImport(name)
84 if isinstance(site_config, ModuleType) or not issubclass(site_config, SiteConfig):
85 raise RuntimeError(f"Site class={name} is not a SiteConfig subclass")
86 return site_config(config)
88 @abstractmethod
89 def get_executors(self) -> list[ParslExecutor]:
90 """Get a list of executors to be used in processing
92 Each executor should have a unique ``label``.
93 """
94 raise NotImplementedError("Subclasses must define")
96 @abstractmethod
97 def select_executor(self, job: "ParslJob") -> str:
98 """Get the ``label`` of the executor to use to execute a job
100 Parameters
101 ----------
102 job : `ParslJob`
103 Job to be executed.
105 Returns
106 -------
107 label : `str`
108 Label of executor to use to execute ``job``.
109 """
110 raise NotImplementedError("Subclasses must define")
112 def get_address(self) -> str:
113 """Return the IP address of the machine hosting the driver/submission
115 This address should be accessible from the workers. This should
116 generally by the return value of one of the functions in
117 ``parsl.addresses``.
119 This is used by the default implementation of ``get_monitor``, but will
120 generally be used by ``get_executors`` too.
122 This default implementation gets the address from the hostname, but
123 that will not work if the workers don't access the driver/submission
124 node by that address.
125 """
126 return address_by_hostname()
128 def get_command_prefix(self) -> str:
129 """Return command(s) to add before each job command
131 These may be used to configure the environment for the job.
133 This default implementation respects the BPS configuration elements:
135 - ``site.<computeSite>.commandPrefix`` (`str`): command(s) to use as a
136 prefix to executing a job command on a worker.
137 - ``site.<computeSite>.environment`` (`bool`): add bash commands that
138 replicate the environment on the driver/submit machine?
139 """
140 prefix = get_bps_config_value(self.site, "commandPrefix", str, "")
141 if get_bps_config_value(self.site, "environment", bool, False):
142 prefix += "\n" + export_environment()
143 return prefix
145 def get_monitor(self) -> MonitoringHub | None:
146 """Get parsl monitor
148 The parsl monitor provides a database that tracks the progress of the
149 workflow and the use of resources on the workers.
151 This implementation respects the BPS configuration elements:
153 - ``site.<computeSite>.monitorEnable`` (`bool`): enable monitor?
154 - ``site.<computeSite>.monitorInterval`` (`float`): time interval (sec)
155 between logging of resource usage.
156 - ``site.<computeSite>.monitorFilename`` (`str`): name of file to use
157 for the monitor sqlite database.
159 Returns
160 -------
161 monitor : `MonitoringHub` or `None`
162 Parsl monitor, or `None` for no monitor.
163 """
164 if not get_bps_config_value(self.site, "monitorEnable", bool, False):
165 return None
166 return MonitoringHub(
167 workflow_name=get_workflow_name(self.config),
168 hub_address=self.get_address(),
169 resource_monitoring_interval=get_bps_config_value(self.site, "monitorInterval", float, 30.0),
170 logging_endpoint="sqlite:///"
171 + get_bps_config_value(self.site, "monitorFilename", str, "monitor.sqlite"),
172 )