Coverage for python/lsst/daf/butler/_butlerRepoIndex.py: 40%

47 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2023-04-14 09:22 +0000

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from __future__ import annotations 

23 

24__all__ = ("ButlerRepoIndex",) 

25 

26import os 

27from typing import ClassVar, Dict, Set 

28 

29from lsst.resources import ResourcePath, ResourcePathExpression 

30 

31from .core import Config 

32 

33 

34class ButlerRepoIndex: 

35 """Index of all known butler repositories. 

36 

37 The index of butler repositories is found by looking for a 

38 configuration file at the URI pointed at by the environment 

39 variable ``$DAF_BUTLER_REPOSITORY_INDEX``. The configuration file 

40 is a simple dictionary lookup of the form: 

41 

42 .. code-block:: yaml 

43 

44 label1: uri1 

45 label2: uri2 

46 

47 and can be in YAML or JSON format. The content of the file will be 

48 cached. 

49 """ 

50 

51 index_env_var: ClassVar[str] = "DAF_BUTLER_REPOSITORY_INDEX" 

52 """The name of the environment variable to read to locate the index.""" 

53 

54 _cache: ClassVar[Dict[ResourcePath, Config]] = {} 

55 """Cache of indexes. In most scenarios only one index will be found 

56 and the environment will not change. In tests this may not be true.""" 

57 

58 @classmethod 

59 def _read_repository_index(cls, index_uri: ResourcePathExpression) -> Config: 

60 """Read the repository index from the supplied URI. 

61 

62 Parameters 

63 ---------- 

64 index_uri : `lsst.resources.ResourcePathExpression` 

65 URI of the repository index. 

66 

67 Returns 

68 ------- 

69 repo_index : `Config` 

70 The index found at this URI. 

71 

72 Raises 

73 ------ 

74 FileNotFoundError 

75 Raised if the URI does not exist. 

76 

77 Notes 

78 ----- 

79 Does check the cache before reading the file. 

80 """ 

81 # Force the given value to a ResourcePath so that it can be used 

82 # as an index into the cache consistently. 

83 uri = ResourcePath(index_uri) 

84 

85 if index_uri in cls._cache: 

86 return cls._cache[uri] 

87 

88 repo_index = Config(uri) 

89 cls._cache[uri] = repo_index 

90 

91 return repo_index 

92 

93 @classmethod 

94 def _get_index_uri(cls) -> ResourcePath: 

95 """Find the URI to the repository index. 

96 

97 Returns 

98 ------- 

99 index_uri : `lsst.resources.ResourcePath` 

100 URI to the repository index. 

101 

102 Raises 

103 ------ 

104 KeyError 

105 Raised if the location of the index could not be determined. 

106 """ 

107 index_uri = os.environ.get(cls.index_env_var) 

108 if index_uri is None: 

109 raise KeyError(f"No repository index defined in enviroment variable {cls.index_env_var}") 

110 return ResourcePath(index_uri) 

111 

112 @classmethod 

113 def _read_repository_index_from_environment(cls) -> Config: 

114 """Look in environment for index location and read it. 

115 

116 Returns 

117 ------- 

118 repo_index : `Config` 

119 The index found in the environment. 

120 """ 

121 index_uri = cls._get_index_uri() 

122 return cls._read_repository_index(index_uri) 

123 

124 @classmethod 

125 def get_known_repos(cls) -> Set[str]: 

126 """Retrieve the list of known repository labels. 

127 

128 Returns 

129 ------- 

130 repos : `set` of `str` 

131 All the known labels. Can be empty if no index can be found. 

132 """ 

133 try: 

134 repo_index = cls._read_repository_index_from_environment() 

135 except (FileNotFoundError, KeyError): 

136 return set() 

137 return set(repo_index) 

138 

139 @classmethod 

140 def get_repo_uri(cls, label: str) -> ResourcePath: 

141 """Look up the label in a butler repository index. 

142 

143 Parameters 

144 ---------- 

145 label : `str` 

146 Label of the Butler repository to look up. 

147 

148 Returns 

149 ------- 

150 uri : `lsst.resources.ResourcePath` 

151 URI to the Butler repository associated with the given label. 

152 

153 Raises 

154 ------ 

155 KeyError 

156 Raised if the label is not found in the index, or if an index 

157 can not be found at all. 

158 FileNotFoundError 

159 Raised if an index is defined in the environment but it 

160 can not be found. 

161 """ 

162 repo_index = cls._read_repository_index_from_environment() 

163 repo_uri = repo_index.get(label) 

164 if repo_uri is None: 

165 # This should not raise since it worked earlier. 

166 try: 

167 index_uri = str(cls._get_index_uri()) 

168 except KeyError: 

169 index_uri = "<environment variable not defined>" 

170 raise KeyError(f"Label '{label}' not known to repository index at {index_uri}") 

171 return ResourcePath(repo_uri)