Coverage for python/lsst/analysis/tools/interfaces/datastore/_sasquatchDatastore.py: 54%

102 statements  

« prev     ^ index     » next       coverage.py v7.5.1, created at 2024-05-15 10:01 +0000

1# This file is part of analysis_tools. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from __future__ import annotations 

23 

24__all__ = ("SasquatchDatastore",) 

25 

26"""Sasquatch datastore""" 

27import logging 

28import os 

29from collections.abc import Iterable, Mapping, Sequence 

30from typing import TYPE_CHECKING, Any, ClassVar 

31 

32from lsst.daf.butler import DatasetRef, DatasetTypeNotSupportedError, StorageClass 

33from lsst.daf.butler.datastore import DatasetRefURIs, DatastoreConfig, DatastoreOpaqueTable 

34from lsst.daf.butler.datastore.generic_base import GenericBaseDatastore 

35from lsst.daf.butler.datastore.record_data import DatastoreRecordData 

36from lsst.daf.butler.registry.interfaces import DatastoreRegistryBridge 

37from lsst.resources import ResourcePath, ResourcePathExpression 

38 

39from . import SasquatchDispatcher 

40 

41if TYPE_CHECKING: 41 ↛ 42line 41 didn't jump to line 42, because the condition on line 41 was never true

42 from lsst.daf.butler import Config, DatasetType, LookupKey 

43 from lsst.daf.butler.registry.interfaces import DatasetIdRef, DatastoreRegistryBridgeManager 

44 

45 

46log = logging.getLogger(__name__) 

47 

48 

49class SasquatchDatastore(GenericBaseDatastore): 

50 """Basic Datastore for writing to an in Sasquatch instance. 

51 

52 This Datastore is currently write only, meaning that it can dispatch data 

53 to a Sasquatch instance, but at the present can not be used to retrieve 

54 values. 

55 

56 

57 Parameters 

58 ---------- 

59 config : `DatastoreConfig` or `str` 

60 Configuration. 

61 bridgeManager : `DatastoreRegistryBridgeManager` 

62 Object that manages the interface between `Registry` and datastores. 

63 butlerRoot : `str`, optional 

64 Unused parameter. 

65 """ 

66 

67 defaultConfigFile: ClassVar[str | None] = "sasquatchDatastore.yaml" 

68 """Path to configuration defaults. Accessed within the ``configs`` resource 

69 or relative to a search path. Can be None if no defaults specified. 

70 """ 

71 

72 restProxyUrl: str 

73 """Url which points to the http rest proxy. This is where datasets will be 

74 dispatched to. 

75 """ 

76 

77 accessToken: str 

78 """Access token which is used to authenticate to the restProxy. 

79 """ 

80 

81 namespace: str 

82 """The namespace in Sasquatch where the uploaded metrics will be 

83 dispatched. 

84 

85 The namespace can be read from the environment using the 

86 ``$DAF_BUTLER_SASQUATCH_NAMESPACE`` environment variable. If that is not 

87 set the datastore config ``"namespace"`` field will be checked. A default 

88 value of "lsst.dm" is used if no other value can be obtained. 

89 """ 

90 

91 extra_fields: dict[str, str | int | float] | None 

92 """Extra key/value pairs that should be passed along with the metric 

93 when storing in Sasquatch. 

94 

95 Extra fields can be obtained both from the ``$SASQUATCH_EXTRAS`` 

96 environment variable and the `"extra_fields"` entry in the datastore 

97 config. The two sources of information are merged with the environment 

98 variable taking priority. The environment variable must have the form of 

99 "k1=v1;k2=v2". 

100 """ 

101 

102 def __init__( 

103 self, 

104 config: DatastoreConfig, 

105 bridgeManager: DatastoreRegistryBridgeManager, 

106 butlerRoot: str | None = None, 

107 ): 

108 super().__init__(config, bridgeManager) 

109 

110 # Name ourselves either using an explicit name or a name 

111 # derived from the (unexpanded) root. 

112 self.name = self.config.get("name", "{}@{}".format(type(self).__name__, self.config["restProxyUrl"])) 

113 log.debug("Creating datastore %s", self.name) 

114 

115 self._bridge = bridgeManager.register(self.name, ephemeral=False) 

116 

117 self.restProxyUrl = self.config["restProxyUrl"] 

118 

119 self.accessToken = self.config.get("accessToken", "na") 

120 

121 self.namespace = os.environ.get( 

122 "DAF_BUTLER_SASQUATCH_NAMESPACE", # Prioritize the environment 

123 self.config.get( 

124 "namespace", # Fallback to datastore config 

125 "lsst.dm", 

126 ), 

127 ) 

128 

129 extra_fields: dict[str, str | int | float] | None = self.config.get("extra_fields", {}) 

130 if extras_str := os.environ.get("SASQUATCH_EXTRAS"): 

131 for item in extras_str.split(";"): 

132 k, v = item.split("=") 

133 extra_fields[k] = v 

134 self.extra_fields = extra_fields if extra_fields else None 

135 

136 self._dispatcher = SasquatchDispatcher(self.restProxyUrl, self.accessToken, self.namespace) 

137 

138 @classmethod 

139 def _create_from_config( 

140 cls, 

141 config: DatastoreConfig, 

142 bridgeManager: DatastoreRegistryBridgeManager, 

143 butlerRoot: ResourcePathExpression | None, 

144 ) -> SasquatchDatastore: 

145 return SasquatchDatastore(config, bridgeManager) 

146 

147 def clone(self, bridgeManager: DatastoreRegistryBridgeManager) -> SasquatchDatastore: 

148 return SasquatchDatastore(self.config, bridgeManager) 

149 

150 @property 

151 def bridge(self) -> DatastoreRegistryBridge: 

152 return self._bridge 

153 

154 def put(self, inMemoryDataset: Any, ref: DatasetRef) -> None: 

155 if self.constraints.isAcceptable(ref): 

156 self._dispatcher.dispatchRef(inMemoryDataset, ref, extraFields=self.extra_fields) 

157 else: 

158 log.debug("Could not put dataset type %s with Sasquatch datastore", ref.datasetType) 

159 raise DatasetTypeNotSupportedError( 

160 f"Could not put dataset type {ref.datasetType} with Sasquatch datastore" 

161 ) 

162 

163 def put_new(self, in_memory_dataset: Any, dataset_ref: DatasetRef) -> Mapping[str, DatasetRef]: 

164 # Docstring inherited from the base class. 

165 self.put(in_memory_dataset, dataset_ref) 

166 # Sasquatch is a sort of ephemeral, because we do not store its 

167 # datastore records in registry, so return empty dict. 

168 return {} 

169 

170 def addStoredItemInfo(self, refs: Iterable[DatasetRef], infos: Iterable[Any]) -> None: 

171 raise NotImplementedError() 

172 

173 def getStoredItemsInfo(self, ref: DatasetRef) -> Sequence[Any]: 

174 raise NotImplementedError() 

175 

176 def removeStoredItemInfo(self, ref: DatasetRef) -> None: 

177 raise NotImplementedError() 

178 

179 def trash(self, ref: DatasetRef | Iterable[DatasetRef], ignore_errors: bool = True) -> None: 

180 log.debug("Sasquatch datastore does not support trashing skipping %s", ref) 

181 raise FileNotFoundError() 

182 

183 def emptyTrash(self, ignore_errors: bool = True) -> None: 

184 log.debug("Sasquatch datastore does not support trash, nothing to empty") 

185 

186 def forget(self, ref: Iterable[DatasetRef]) -> None: 

187 pass 

188 

189 def exists(self, datasetRef: DatasetRef) -> bool: 

190 # sasquatch is not currently searchable 

191 return False 

192 

193 def knows(self, ref: DatasetRef) -> bool: 

194 return False 

195 

196 def get( 

197 self, 

198 datasetRef: DatasetRef, 

199 parameters: Mapping[str, Any] | None = None, 

200 storageClass: StorageClass | str | None = None, 

201 ) -> Any: 

202 raise FileNotFoundError() 

203 

204 def validateConfiguration( 

205 self, entities: Iterable[DatasetRef | DatasetType | StorageClass], logFailures: bool = False 

206 ) -> None: 

207 """Validate some of the configuration for this datastore. 

208 

209 Parameters 

210 ---------- 

211 entities : iterable of `DatasetRef`, `DatasetType`, or `StorageClass` 

212 Entities to test against this configuration. Can be differing 

213 types. 

214 logFailures : `bool`, optional 

215 If `True`, output a log message for every validation error 

216 detected. 

217 

218 Raises 

219 ------ 

220 DatastoreValidationError 

221 Raised if there is a validation problem with a configuration. 

222 All the problems are reported in a single exception. 

223 

224 Notes 

225 ----- 

226 This method is a no-op. 

227 """ 

228 return 

229 

230 def validateKey(self, lookupKey: LookupKey, entity: DatasetRef | DatasetType | StorageClass) -> None: 

231 # Docstring is inherited from base class. 

232 return 

233 

234 def getLookupKeys(self) -> set[LookupKey]: 

235 # Docstring is inherited from base class. 

236 return self.constraints.getLookupKeys() 

237 

238 def needs_expanded_data_ids( 

239 self, 

240 transfer: str | None, 

241 entity: DatasetRef | DatasetType | StorageClass | None = None, 

242 ) -> bool: 

243 # Docstring inherited. 

244 return False 

245 

246 def import_records(self, data: Mapping[str, DatastoreRecordData]) -> None: 

247 # Docstring inherited from the base class. 

248 return 

249 

250 def export_records(self, refs: Iterable[DatasetIdRef]) -> Mapping[str, DatastoreRecordData]: 

251 # Docstring inherited from the base class. 

252 

253 # Sasquatch Datastore records cannot be exported or imported. 

254 return {} 

255 

256 def getURI(self, datasetRef: DatasetRef, predict: bool = False) -> ResourcePath: 

257 raise NotImplementedError() 

258 

259 def getURIs(self, datasetRef: DatasetRef, predict: bool = False) -> DatasetRefURIs: 

260 raise NotImplementedError() 

261 

262 def retrieveArtifacts( 

263 self, 

264 refs: Iterable[DatasetRef], 

265 destination: ResourcePath, 

266 transfer: str = "auto", 

267 preserve_path: bool = True, 

268 overwrite: bool = False, 

269 ) -> list[ResourcePath]: 

270 raise NotImplementedError() 

271 

272 @classmethod 

273 def setConfigRoot(cls, root: str, config: Config, full: Config, overwrite: bool = True) -> None: 

274 pass 

275 

276 def get_opaque_table_definitions(self) -> Mapping[str, DatastoreOpaqueTable]: 

277 # Docstring inherited from the base class. 

278 return {}