Coverage for python/lsst/analysis/tools/interfaces/datastore/_sasquatchDatastore.py: 54%
102 statements
« prev ^ index » next coverage.py v7.4.4, created at 2024-03-28 04:18 -0700
« prev ^ index » next coverage.py v7.4.4, created at 2024-03-28 04:18 -0700
1# This file is part of analysis_tools.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22from __future__ import annotations
24__all__ = ("SasquatchDatastore",)
26"""Sasquatch datastore"""
27import logging
28import os
29from collections.abc import Iterable, Mapping, Sequence
30from typing import TYPE_CHECKING, Any, ClassVar
32from lsst.daf.butler import DatasetRef, DatasetTypeNotSupportedError, StorageClass
33from lsst.daf.butler.datastore import DatasetRefURIs, DatastoreConfig, DatastoreOpaqueTable
34from lsst.daf.butler.datastore.generic_base import GenericBaseDatastore
35from lsst.daf.butler.datastore.record_data import DatastoreRecordData
36from lsst.daf.butler.registry.interfaces import DatastoreRegistryBridge
37from lsst.resources import ResourcePath, ResourcePathExpression
39from . import SasquatchDispatcher
41if TYPE_CHECKING: 41 ↛ 42line 41 didn't jump to line 42, because the condition on line 41 was never true
42 from lsst.daf.butler import Config, DatasetType, LookupKey
43 from lsst.daf.butler.registry.interfaces import DatasetIdRef, DatastoreRegistryBridgeManager
46log = logging.getLogger(__name__)
49class SasquatchDatastore(GenericBaseDatastore):
50 """Basic Datastore for writing to an in Sasquatch instance.
52 This Datastore is currently write only, meaning that it can dispatch data
53 to a Sasquatch instance, but at the present can not be used to retrieve
54 values.
57 Parameters
58 ----------
59 config : `DatastoreConfig` or `str`
60 Configuration.
61 bridgeManager : `DatastoreRegistryBridgeManager`
62 Object that manages the interface between `Registry` and datastores.
63 butlerRoot : `str`, optional
64 Unused parameter.
65 """
67 defaultConfigFile: ClassVar[str | None] = "sasquatchDatastore.yaml"
68 """Path to configuration defaults. Accessed within the ``configs`` resource
69 or relative to a search path. Can be None if no defaults specified.
70 """
72 restProxyUrl: str
73 """Url which points to the http rest proxy. This is where datasets will be
74 dispatched to.
75 """
77 accessToken: str
78 """Access token which is used to authenticate to the restProxy.
79 """
81 namespace: str
82 """The namespace in Sasquatch where the uploaded metrics will be
83 dispatched.
85 The namespace can be read from the environment using the
86 ``$DAF_BUTLER_SASQUATCH_NAMESPACE`` environment variable. If that is not
87 set the datastore config ``"namespace"`` field will be checked. A default
88 value of "lsst.dm" is used if no other value can be obtained.
89 """
91 extra_fields: dict[str, str | int | float] | None
92 """Extra key/value pairs that should be passed along with the metric
93 when storing in Sasquatch.
95 Extra fields can be obtained both from the ``$SASQUATCH_EXTRAS``
96 environment variable and the `"extra_fields"` entry in the datastore
97 config. The two sources of information are merged with the environment
98 variable taking priority. The environment variable must have the form of
99 "k1=v1;k2=v2".
100 """
102 def __init__(
103 self,
104 config: DatastoreConfig,
105 bridgeManager: DatastoreRegistryBridgeManager,
106 butlerRoot: str | None = None,
107 ):
108 super().__init__(config, bridgeManager)
110 # Name ourselves either using an explicit name or a name
111 # derived from the (unexpanded) root.
112 self.name = self.config.get("name", "{}@{}".format(type(self).__name__, self.config["restProxyUrl"]))
113 log.debug("Creating datastore %s", self.name)
115 self._bridge = bridgeManager.register(self.name, ephemeral=False)
117 self.restProxyUrl = self.config["restProxyUrl"]
119 self.accessToken = self.config.get("accessToken", "na")
121 self.namespace = os.environ.get(
122 "DAF_BUTLER_SASQUATCH_NAMESPACE", # Prioritize the environment
123 self.config.get(
124 "namespace", # Fallback to datastore config
125 "lsst.dm",
126 ),
127 )
129 extra_fields: dict[str, str | int | float] | None = self.config.get("extra_fields", {})
130 if extras_str := os.environ.get("SASQUATCH_EXTRAS"):
131 for item in extras_str.split(";"):
132 k, v = item.split("=")
133 extra_fields[k] = v
134 self.extra_fields = extra_fields if extra_fields else None
136 self._dispatcher = SasquatchDispatcher(self.restProxyUrl, self.accessToken, self.namespace)
138 @classmethod
139 def _create_from_config(
140 cls,
141 config: DatastoreConfig,
142 bridgeManager: DatastoreRegistryBridgeManager,
143 butlerRoot: ResourcePathExpression | None,
144 ) -> SasquatchDatastore:
145 return SasquatchDatastore(config, bridgeManager)
147 def clone(self, bridgeManager: DatastoreRegistryBridgeManager) -> SasquatchDatastore:
148 return SasquatchDatastore(self.config, bridgeManager)
150 @property
151 def bridge(self) -> DatastoreRegistryBridge:
152 return self._bridge
154 def put(self, inMemoryDataset: Any, ref: DatasetRef) -> None:
155 if self.constraints.isAcceptable(ref):
156 self._dispatcher.dispatchRef(inMemoryDataset, ref, extraFields=self.extra_fields)
157 else:
158 log.debug("Could not put dataset type %s with Sasquatch datastore", ref.datasetType)
159 raise DatasetTypeNotSupportedError(
160 f"Could not put dataset type {ref.datasetType} with Sasquatch datastore"
161 )
163 def put_new(self, in_memory_dataset: Any, dataset_ref: DatasetRef) -> Mapping[str, DatasetRef]:
164 # Docstring inherited from the base class.
165 self.put(in_memory_dataset, dataset_ref)
166 # Sasquatch is a sort of ephemeral, because we do not store its
167 # datastore records in registry, so return empty dict.
168 return {}
170 def addStoredItemInfo(self, refs: Iterable[DatasetRef], infos: Iterable[Any]) -> None:
171 raise NotImplementedError()
173 def getStoredItemsInfo(self, ref: DatasetRef) -> Sequence[Any]:
174 raise NotImplementedError()
176 def removeStoredItemInfo(self, ref: DatasetRef) -> None:
177 raise NotImplementedError()
179 def trash(self, ref: DatasetRef | Iterable[DatasetRef], ignore_errors: bool = True) -> None:
180 log.debug("Sasquatch datastore does not support trashing skipping %s", ref)
181 raise FileNotFoundError()
183 def emptyTrash(self, ignore_errors: bool = True) -> None:
184 log.debug("Sasquatch datastore does not support trash, nothing to empty")
186 def forget(self, ref: Iterable[DatasetRef]) -> None:
187 pass
189 def exists(self, datasetRef: DatasetRef) -> bool:
190 # sasquatch is not currently searchable
191 return False
193 def knows(self, ref: DatasetRef) -> bool:
194 return False
196 def get(
197 self,
198 datasetRef: DatasetRef,
199 parameters: Mapping[str, Any] | None = None,
200 storageClass: StorageClass | str | None = None,
201 ) -> Any:
202 raise FileNotFoundError()
204 def validateConfiguration(
205 self, entities: Iterable[DatasetRef | DatasetType | StorageClass], logFailures: bool = False
206 ) -> None:
207 """Validate some of the configuration for this datastore.
209 Parameters
210 ----------
211 entities : iterable of `DatasetRef`, `DatasetType`, or `StorageClass`
212 Entities to test against this configuration. Can be differing
213 types.
214 logFailures : `bool`, optional
215 If `True`, output a log message for every validation error
216 detected.
218 Raises
219 ------
220 DatastoreValidationError
221 Raised if there is a validation problem with a configuration.
222 All the problems are reported in a single exception.
224 Notes
225 -----
226 This method is a no-op.
227 """
228 return
230 def validateKey(self, lookupKey: LookupKey, entity: DatasetRef | DatasetType | StorageClass) -> None:
231 # Docstring is inherited from base class.
232 return
234 def getLookupKeys(self) -> set[LookupKey]:
235 # Docstring is inherited from base class.
236 return self.constraints.getLookupKeys()
238 def needs_expanded_data_ids(
239 self,
240 transfer: str | None,
241 entity: DatasetRef | DatasetType | StorageClass | None = None,
242 ) -> bool:
243 # Docstring inherited.
244 return False
246 def import_records(self, data: Mapping[str, DatastoreRecordData]) -> None:
247 # Docstring inherited from the base class.
248 return
250 def export_records(self, refs: Iterable[DatasetIdRef]) -> Mapping[str, DatastoreRecordData]:
251 # Docstring inherited from the base class.
253 # Sasquatch Datastore records cannot be exported or imported.
254 return {}
256 def getURI(self, datasetRef: DatasetRef, predict: bool = False) -> ResourcePath:
257 raise NotImplementedError()
259 def getURIs(self, datasetRef: DatasetRef, predict: bool = False) -> DatasetRefURIs:
260 raise NotImplementedError()
262 def retrieveArtifacts(
263 self,
264 refs: Iterable[DatasetRef],
265 destination: ResourcePath,
266 transfer: str = "auto",
267 preserve_path: bool = True,
268 overwrite: bool = False,
269 ) -> list[ResourcePath]:
270 raise NotImplementedError()
272 @classmethod
273 def setConfigRoot(cls, root: str, config: Config, full: Config, overwrite: bool = True) -> None:
274 pass
276 def get_opaque_table_definitions(self) -> Mapping[str, DatastoreOpaqueTable]:
277 # Docstring inherited from the base class.
278 return {}