Coverage for python/lsst/dax/apdb/apdbIndex.py: 28%

55 statements  

« prev     ^ index     » next       coverage.py v7.4.4, created at 2024-04-16 03:10 -0700

1# This file is part of dax_apdb. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from __future__ import annotations 

23 

24__all__ = ["ApdbIndex"] 

25 

26import io 

27import logging 

28import os 

29from collections.abc import Mapping 

30from typing import ClassVar 

31 

32import yaml 

33from lsst.resources import ResourcePath 

34from pydantic import TypeAdapter, ValidationError 

35 

36_LOG = logging.getLogger(__name__) 

37 

38 

39class ApdbIndex: 

40 """Index of well-known Apdb instances. 

41 

42 Parameters 

43 ---------- 

44 index_path : `str`, optional 

45 Path to the index configuration file, if not provided then the value 

46 of ``DAX_APDB_INDEX_URI`` environment variable is used to locate 

47 configuration file. 

48 

49 The index is configured from a simple YAML file whose location is 

50 determined from ``DAX_APDB_INDEX_URI`` environment variable. The content 

51 of the index file is a mapping of the labels to URIs in YAML format, e.g.: 

52 

53 .. code-block:: yaml 

54 

55 dev: "/path/to/config-file.yaml" 

56 "prod/pex_config": "s3://bucket/apdb-prod.py" 

57 "prod/yaml": "s3://bucket/apdb-prod.yaml" 

58 

59 The labels in the index file consists of the label name and an optional 

60 format name separated from label by slash. `get_apdb_uri` method can 

61 use its ``format`` argument to return either a format-specific 

62 configuration or a label-only configuration if format-specific is not 

63 defined. 

64 """ 

65 

66 index_env_var: ClassVar[str] = "DAX_APDB_INDEX_URI" 

67 """The name of the environment variable containing the URI of the index 

68 configuration file. 

69 """ 

70 

71 _cache: Mapping[str, str] | None = None 

72 """Contents of the index file cached in memory.""" 

73 

74 def __init__(self, index_path: str | None = None): 

75 self._index_path = index_path 

76 

77 def _read_index(self, index_path: str | None = None) -> Mapping[str, str]: 

78 """Return contents of the index file. 

79 

80 Parameters 

81 ---------- 

82 index_path : `str`, optional 

83 Location of the index file, if not provided then default location 

84 is used. 

85 

86 Returns 

87 ------- 

88 entries : `~collections.abc.Mapping` [`str`, `str`] 

89 All known entries. Can be empty if no index can be found. 

90 

91 Raises 

92 ------ 

93 RuntimeError 

94 Raised if ``index_path`` is not provided and environment variable 

95 is not set. 

96 TypeError 

97 Raised if content of the configuration file is incorrect. 

98 """ 

99 if self._cache is not None: 

100 return self._cache 

101 if index_path is None: 

102 index_path = os.getenv(self.index_env_var) 

103 if not index_path: 

104 raise RuntimeError( 

105 f"No repository index defined, environment variable {self.index_env_var} is not set." 

106 ) 

107 index_uri = ResourcePath(index_path) 

108 _LOG.debug("Opening YAML index file: %s", index_uri.geturl()) 

109 try: 

110 content = index_uri.read() 

111 except IsADirectoryError as exc: 

112 raise FileNotFoundError(f"Index file {index_uri.geturl()} is a directory") from exc 

113 stream = io.BytesIO(content) 

114 if index_data := yaml.load(stream, Loader=yaml.SafeLoader): 

115 try: 

116 self._cache = TypeAdapter(dict[str, str]).validate_python(index_data) 

117 except ValidationError as e: 

118 raise TypeError(f"Repository index {index_uri.geturl()} not in expected format") from e 

119 else: 

120 self._cache = {} 

121 return self._cache 

122 

123 def get_apdb_uri(self, label: str, format: str | None = None) -> ResourcePath: 

124 """Return URI for APDB configuration file given its label. 

125 

126 Parameters 

127 ---------- 

128 label : `str` 

129 Label of APDB instance. 

130 format : `str` 

131 Format of the APDB configuration file, arbitrary string. This can 

132 be used to support an expected migration from pex_config to YAML 

133 configuration for APDB, code that uses pex_config could provide 

134 "pex_config" for ``format``. The actual key in the index is 

135 either a slash-separated label and format, or, if that is missing, 

136 just a label. 

137 

138 Returns 

139 ------- 

140 uri : `~lsst.resources.ResourcePath` 

141 URI for the configuration file for APDB instance. 

142 

143 Raises 

144 ------ 

145 FileNotFoundError 

146 Raised if an index is defined in the environment but it 

147 can not be found. 

148 ValueError 

149 Raised if the label is not found in the index. 

150 RuntimeError 

151 Raised if ``index_path`` is not provided and environment variable 

152 is not set. 

153 TypeError 

154 Raised if the format of the index file is incorrect. 

155 """ 

156 index = self._read_index(self._index_path) 

157 labels: list[str] = [label] 

158 if format: 

159 labels.insert(0, f"{label}/{format}") 

160 for label in labels: 

161 if (uri_str := index.get(label)) is not None: 

162 return ResourcePath(uri_str) 

163 if len(labels) == 1: 

164 message = f"Label {labels[0]} is not defined in index file" 

165 else: 

166 labels_str = ", ".join(labels) 

167 message = f"None of labels {labels_str} is defined in index file" 

168 all_labels = set(index) 

169 raise ValueError(f"{message}, labels known to index: {all_labels}") 

170 

171 def get_entries(self) -> Mapping[str, str]: 

172 """Retrieve all entries defined in index. 

173 

174 Returns 

175 ------- 

176 entries : `~collections.abc.Mapping` [`str`, `str`] 

177 All known index entries. 

178 

179 Raises 

180 ------ 

181 RuntimeError 

182 Raised if ``index_path`` is not provided and environment variable 

183 is not set. 

184 TypeError 

185 Raised if content of the configuration file is incorrect. 

186 """ 

187 return self._read_index(self._index_path)