Coverage for python/lsst/daf/butler/_butlerRepoIndex.py: 33%

54 statements  

« prev     ^ index     » next       coverage.py v7.2.7, created at 2023-06-08 05:05 -0700

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from __future__ import annotations 

23 

24__all__ = ("ButlerRepoIndex",) 

25 

26import os 

27from typing import ClassVar, Dict, Set 

28 

29from lsst.resources import ResourcePath, ResourcePathExpression 

30 

31from .core import Config 

32 

33 

34class ButlerRepoIndex: 

35 """Index of all known butler repositories. 

36 

37 The index of butler repositories is found by looking for a 

38 configuration file at the URI pointed at by the environment 

39 variable ``$DAF_BUTLER_REPOSITORY_INDEX``. The configuration file 

40 is a simple dictionary lookup of the form: 

41 

42 .. code-block:: yaml 

43 

44 label1: uri1 

45 label2: uri2 

46 

47 and can be in YAML or JSON format. The content of the file will be 

48 cached. 

49 """ 

50 

51 index_env_var: ClassVar[str] = "DAF_BUTLER_REPOSITORY_INDEX" 

52 """The name of the environment variable to read to locate the index.""" 

53 

54 _cache: ClassVar[Dict[ResourcePath, Config]] = {} 

55 """Cache of indexes. In most scenarios only one index will be found 

56 and the environment will not change. In tests this may not be true.""" 

57 

58 @classmethod 

59 def _read_repository_index(cls, index_uri: ResourcePathExpression) -> Config: 

60 """Read the repository index from the supplied URI. 

61 

62 Parameters 

63 ---------- 

64 index_uri : `lsst.resources.ResourcePathExpression` 

65 URI of the repository index. 

66 

67 Returns 

68 ------- 

69 repo_index : `Config` 

70 The index found at this URI. 

71 

72 Raises 

73 ------ 

74 FileNotFoundError 

75 Raised if the URI does not exist. 

76 

77 Notes 

78 ----- 

79 Does check the cache before reading the file. 

80 """ 

81 # Force the given value to a ResourcePath so that it can be used 

82 # as an index into the cache consistently. 

83 uri = ResourcePath(index_uri) 

84 

85 if index_uri in cls._cache: 

86 return cls._cache[uri] 

87 

88 repo_index = Config(uri) 

89 cls._cache[uri] = repo_index 

90 

91 return repo_index 

92 

93 @classmethod 

94 def _get_index_uri(cls) -> ResourcePath: 

95 """Find the URI to the repository index. 

96 

97 Returns 

98 ------- 

99 index_uri : `lsst.resources.ResourcePath` 

100 URI to the repository index. 

101 

102 Raises 

103 ------ 

104 KeyError 

105 Raised if the location of the index could not be determined. 

106 """ 

107 index_uri = os.environ.get(cls.index_env_var) 

108 if index_uri is None: 

109 raise KeyError(f"No repository index defined in environment variable {cls.index_env_var}") 

110 return ResourcePath(index_uri) 

111 

112 @classmethod 

113 def _read_repository_index_from_environment(cls) -> Config: 

114 """Look in environment for index location and read it. 

115 

116 Returns 

117 ------- 

118 repo_index : `Config` 

119 The index found in the environment. 

120 """ 

121 index_uri = cls._get_index_uri() 

122 return cls._read_repository_index(index_uri) 

123 

124 @classmethod 

125 def get_known_repos(cls) -> Set[str]: 

126 """Retrieve the list of known repository labels. 

127 

128 Returns 

129 ------- 

130 repos : `set` of `str` 

131 All the known labels. Can be empty if no index can be found. 

132 """ 

133 try: 

134 repo_index = cls._read_repository_index_from_environment() 

135 except (FileNotFoundError, KeyError): 

136 return set() 

137 return set(repo_index) 

138 

139 @classmethod 

140 def get_repo_uri(cls, label: str, return_label: bool = False) -> ResourcePath: 

141 """Look up the label in a butler repository index. 

142 

143 Parameters 

144 ---------- 

145 label : `str` 

146 Label of the Butler repository to look up. 

147 return_label : `bool`, optional 

148 If ``label`` cannot be found in the repository index (either 

149 because index is not defined or ``label`` is not in the index) and 

150 ``return_label`` is `True` then return ``ResourcePath(label)``. 

151 If ``return_label`` is `False` (default) then an exception will be 

152 raised instead. 

153 

154 Returns 

155 ------- 

156 uri : `lsst.resources.ResourcePath` 

157 URI to the Butler repository associated with the given label or 

158 default value if it is provided. 

159 

160 Raises 

161 ------ 

162 KeyError 

163 Raised if the label is not found in the index, or if an index 

164 is not defined, and ``return_label`` is `False`. 

165 FileNotFoundError 

166 Raised if an index is defined in the environment but it 

167 can not be found. 

168 """ 

169 try: 

170 repo_index = cls._read_repository_index_from_environment() 

171 except KeyError: 

172 if return_label: 

173 return ResourcePath(label) 

174 raise 

175 

176 repo_uri = repo_index.get(label) 

177 if repo_uri is None: 

178 if return_label: 

179 return ResourcePath(label) 

180 # This should not raise since it worked earlier. 

181 try: 

182 index_uri = str(cls._get_index_uri()) 

183 except KeyError: 

184 index_uri = "<environment variable not defined>" 

185 raise KeyError(f"Label '{label}' not known to repository index at {index_uri}") 

186 return ResourcePath(repo_uri)