Coverage for tests / test_sites.py: 20%

1# This file is part of ctrl_bps_parsl. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org) and the LSST DESC (https://www.lsstdesc.org/). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This software is dual licensed under the GNU General Public License and also 

10# under a 3-clause BSD license. Recipients may choose which of these licenses 

11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, 

12# respectively. If you choose the GPL option then the following text applies 

13# (but note that there is still no warranty even if you opt for BSD instead): 

14# 

15# This program is free software: you can redistribute it and/or modify 

16# it under the terms of the GNU General Public License as published by 

17# the Free Software Foundation, either version 3 of the License, or 

18# (at your option) any later version. 

19# 

20# This program is distributed in the hope that it will be useful, 

21# but WITHOUT ANY WARRANTY; without even the implied warranty of 

22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

23# GNU General Public License for more details. 

24# 

25# You should have received a copy of the GNU General Public License 

26# along with this program. If not, see <https://www.gnu.org/licenses/>. 

27 

28 

29import platform 

30 

31import parsl.config 

32from parsl.dataflow.memoization import BasicMemoizer 

33from parsl.executors import HighThroughputExecutor, WorkQueueExecutor 

34from parsl.launchers import SingleNodeLauncher, SrunLauncher 

35 

36from lsst.ctrl.bps import BpsConfig 

37from lsst.ctrl.bps.parsl.sites import ( 

38 Ccin2p3, 

39 Local, 

40 LocalSrunWorkQueue, 

41 Slurm, 

42 Torque, 

43 TripleSlurm, 

44) 

45 

46 

47def get_bps_config(nodes=1, site_class="lsst.ctrl.bps.parsl.sites.Slurm"): 

48 """Provide a minimal config that allows compute site classes to be 

49 created, allowing for the site class type and the number of nodes 

50 to be set. 

51 """ 

52 return BpsConfig( 

53 { 

54 "submitPath": ".", 

55 "operator": "operator", 

56 "computeSite": "slurm", 

57 "uniqProcName": "test_run", 

58 "site": { 

59 "slurm": { 

60 "class": site_class, 

61 "cores": 1, 

62 "nodes": nodes, 

63 "walltime": "00:01:00", 

64 } 

65 }, 

66 } 

67 ) 

68 

69 

70def testSiteResourceLists(): 

71 """Test compute site resource lists.""" 

72 config = get_bps_config() 

73 expected_resources = { 

74 HighThroughputExecutor: {"priority"}, 

75 WorkQueueExecutor: {"memory", "cores", "disk", "running_time_min", "priority"}, 

76 } 

77 

78 site_classes = [Local, Slurm, Torque] 

79 if platform.machine() in ("aarch64", "x86_64"): 

80 # If running on a supported architecture, then add Ccin2p3 site class. 

81 site_classes.append(Ccin2p3) 

82 

83 try: 

84 # If the ndcctools module is available, then add WorkQueue 

85 # subclasses to the site_classes list. 

86 import ndcctools # noqa F401 

87 

88 site_classes.append(LocalSrunWorkQueue) 

89 except ImportError: 

90 pass 

91 

92 for site_class in site_classes: 

93 compute_site = site_class(config) 

94 executor_type = type(compute_site.get_executors()[0]) 

95 assert expected_resources[executor_type] == set(compute_site.resource_list) 

96 

97 

98def testGetParslConfig(): 

99 """ 

100 Test that get_parsl_config returns a properly configured parsl Config. 

101 """ 

102 config = get_bps_config() 

103 site_config = Slurm(config) 

104 parsl_config = site_config.get_parsl_config() 

105 

106 assert isinstance(parsl_config, parsl.config.Config) 

107 assert isinstance(parsl_config.memoizer, BasicMemoizer) 

108 assert parsl_config.monitoring is None 

109 assert parsl_config.retries == 1 

110 assert parsl_config.run_dir == "runinfo" 

111 assert parsl_config.memoizer.checkpoint_mode == "task_exit" 

112 

113 

114def testSlurmProviderLauncher(): 

115 """Test that the correct parsl launcher is selected for the 

116 SlurmProvider given the number of nodes. 

117 """ 

118 site_classes = [ 

119 ("lsst.ctrl.bps.parsl.Slurm", Slurm), 

120 ("lsst.ctrl.bps.parsl.TripleSlurm", TripleSlurm), 

121 ] 

122 for nodes in (1, 2, 3): 

123 for site_class_name, site_class in site_classes: 

124 config = get_bps_config(nodes=nodes, site_class=site_class_name) 

125 site_config = site_class(config) 

126 executor = site_config.get_executors()[0] 

127 if nodes > 1: 

128 assert isinstance(executor.provider.launcher, SrunLauncher) 

129 else: 

130 assert isinstance(executor.provider.launcher, SingleNodeLauncher)