Coverage for python/lsst/daf/butler/core/storedFileInfo.py: 48%
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22from __future__ import annotations
24__all__ = ("StoredFileInfo", "StoredDatastoreItemInfo")
26import inspect
27from dataclasses import dataclass
28from typing import TYPE_CHECKING, Any, Dict, Optional, Type
30from lsst.resources import ResourcePath
32from .formatter import Formatter, FormatterParameter
33from .location import Location, LocationFactory
34from .storageClass import StorageClass, StorageClassFactory
36if TYPE_CHECKING: 36 ↛ 37line 36 didn't jump to line 37, because the condition on line 36 was never true
37 from .datasets import DatasetId, DatasetRef
39# String to use when a Python None is encountered
40NULLSTR = "__NULL_STRING__"
43class StoredDatastoreItemInfo:
44 """Internal information associated with a stored dataset in a `Datastore`.
46 This is an empty base class. Datastore implementations are expected to
47 write their own subclasses.
48 """
50 __slots__ = ()
52 def file_location(self, factory: LocationFactory) -> Location:
53 """Return the location of artifact.
55 Parameters
56 ----------
57 factory : `LocationFactory`
58 Factory relevant to the datastore represented by this item.
60 Returns
61 -------
62 location : `Location`
63 The location of the item within this datastore.
64 """
65 raise NotImplementedError("The base class does not know how to locate an item in a datastore.")
67 @classmethod
68 def from_record(cls: Type[StoredDatastoreItemInfo], record: Dict[str, Any]) -> StoredDatastoreItemInfo:
69 """Create instance from database record.
71 Parameters
72 ----------
73 record : `dict`
74 The record associated with this item.
76 Returns
77 -------
78 info : instance of the relevant type.
79 The newly-constructed item corresponding to the record.
80 """
81 raise NotImplementedError()
83 @property
84 def dataset_id(self) -> DatasetId:
85 """Dataset ID associated with this record (`DatasetId`)"""
86 raise NotImplementedError()
89@dataclass(frozen=True)
90class StoredFileInfo(StoredDatastoreItemInfo):
91 """Datastore-private metadata associated with a Datastore file."""
93 __slots__ = {"formatter", "path", "storageClass", "component", "checksum", "file_size", "dataset_id"}
95 storageClassFactory = StorageClassFactory()
97 def __init__(
98 self,
99 formatter: FormatterParameter,
100 path: str,
101 storageClass: StorageClass,
102 component: Optional[str],
103 checksum: Optional[str],
104 file_size: int,
105 dataset_id: DatasetId,
106 ):
108 # Use these shenanigans to allow us to use a frozen dataclass
109 object.__setattr__(self, "path", path)
110 object.__setattr__(self, "storageClass", storageClass)
111 object.__setattr__(self, "component", component)
112 object.__setattr__(self, "checksum", checksum)
113 object.__setattr__(self, "file_size", file_size)
114 object.__setattr__(self, "dataset_id", dataset_id)
116 if isinstance(formatter, str):
117 # We trust that this string refers to a Formatter
118 formatterStr = formatter
119 elif isinstance(formatter, Formatter) or (
120 inspect.isclass(formatter) and issubclass(formatter, Formatter)
121 ):
122 formatterStr = formatter.name()
123 else:
124 raise TypeError(f"Supplied formatter '{formatter}' is not a Formatter")
125 object.__setattr__(self, "formatter", formatterStr)
127 formatter: str
128 """Fully-qualified name of Formatter. If a Formatter class or instance
129 is given the name will be extracted."""
131 path: str
132 """Path to dataset within Datastore."""
134 storageClass: StorageClass
135 """StorageClass associated with Dataset."""
137 component: Optional[str]
138 """Component associated with this file. Can be None if the file does
139 not refer to a component of a composite."""
141 checksum: Optional[str]
142 """Checksum of the serialized dataset."""
144 file_size: int
145 """Size of the serialized dataset in bytes."""
147 dataset_id: DatasetId
148 """DatasetId associated with this record."""
150 def to_record(self, ref: Optional[DatasetRef] = None) -> Dict[str, Any]:
151 """Convert the supplied ref to a database record."""
152 component = ref.datasetType.component() if ref is not None else None
153 if component is None and self.component is not None:
154 component = self.component
155 if component is None:
156 # Use empty string since we want this to be part of the
157 # primary key.
158 component = NULLSTR
159 dataset_id = ref.id if ref is not None else self.dataset_id
161 return dict(
162 dataset_id=dataset_id,
163 formatter=self.formatter,
164 path=self.path,
165 storage_class=self.storageClass.name,
166 component=component,
167 checksum=self.checksum,
168 file_size=self.file_size,
169 )
171 def file_location(self, factory: LocationFactory) -> Location:
172 """Return the location of artifact.
174 Parameters
175 ----------
176 factory : `LocationFactory`
177 Factory relevant to the datastore represented by this item.
179 Returns
180 -------
181 location : `Location`
182 The location of the item within this datastore.
183 """
184 uriInStore = ResourcePath(self.path, forceAbsolute=False)
185 if uriInStore.isabs():
186 location = Location(None, uriInStore)
187 else:
188 location = factory.fromPath(uriInStore)
189 return location
191 @classmethod
192 def from_record(cls: Type[StoredFileInfo], record: Dict[str, Any]) -> StoredFileInfo:
193 """Create instance from database record.
195 Parameters
196 ----------
197 record : `dict`
198 The record associated with this item.
200 Returns
201 -------
202 info : `StoredFileInfo`
203 The newly-constructed item corresponding to the record.
204 """
205 # Convert name of StorageClass to instance
206 storageClass = cls.storageClassFactory.getStorageClass(record["storage_class"])
207 component = record["component"] if (record["component"] and record["component"] != NULLSTR) else None
209 info = StoredFileInfo(
210 formatter=record["formatter"],
211 path=record["path"],
212 storageClass=storageClass,
213 component=component,
214 checksum=record["checksum"],
215 file_size=record["file_size"],
216 dataset_id=record["dataset_id"],
217 )
218 return info