Coverage for python/lsst/daf/butler/registry/opaque.py: 32%

79 statements  

« prev     ^ index     » next       coverage.py v7.4.4, created at 2024-04-04 02:55 -0700

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This software is dual licensed under the GNU General Public License and also 

10# under a 3-clause BSD license. Recipients may choose which of these licenses 

11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, 

12# respectively. If you choose the GPL option then the following text applies 

13# (but note that there is still no warranty even if you opt for BSD instead): 

14# 

15# This program is free software: you can redistribute it and/or modify 

16# it under the terms of the GNU General Public License as published by 

17# the Free Software Foundation, either version 3 of the License, or 

18# (at your option) any later version. 

19# 

20# This program is distributed in the hope that it will be useful, 

21# but WITHOUT ANY WARRANTY; without even the implied warranty of 

22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

23# GNU General Public License for more details. 

24# 

25# You should have received a copy of the GNU General Public License 

26# along with this program. If not, see <http://www.gnu.org/licenses/>. 

27 

28"""The default concrete implementations of the classes that manage 

29opaque tables for `Registry`. 

30""" 

31 

32from __future__ import annotations 

33 

34__all__ = ["ByNameOpaqueTableStorage", "ByNameOpaqueTableStorageManager"] 

35 

36import itertools 

37from collections.abc import Iterable, Iterator 

38from typing import TYPE_CHECKING, Any, ClassVar 

39 

40import sqlalchemy 

41 

42from .._utilities.thread_safe_cache import ThreadSafeCache 

43from ..ddl import FieldSpec, TableSpec 

44from .interfaces import ( 

45 Database, 

46 OpaqueTableStorage, 

47 OpaqueTableStorageManager, 

48 StaticTablesContext, 

49 VersionTuple, 

50) 

51 

52if TYPE_CHECKING: 

53 from ..datastore import DatastoreTransaction 

54 

55# This has to be updated on every schema change 

56_VERSION = VersionTuple(0, 2, 0) 

57 

58 

59class ByNameOpaqueTableStorage(OpaqueTableStorage): 

60 """An implementation of `OpaqueTableStorage` that simply creates a true 

61 table for each different named opaque logical table. 

62 

63 A `ByNameOpaqueTableStorageManager` instance should always be used to 

64 construct and manage instances of this class. 

65 

66 Parameters 

67 ---------- 

68 db : `Database` 

69 Database engine interface for the namespace in which this table lives. 

70 name : `str` 

71 Name of the logical table (also used as the name of the actual table). 

72 table : `sqlalchemy.schema.Table` 

73 SQLAlchemy representation of the table, which must have already been 

74 created in the namespace managed by ``db`` (this is the responsibility 

75 of `ByNameOpaqueTableStorageManager`). 

76 """ 

77 

78 def __init__(self, *, db: Database, name: str, table: sqlalchemy.schema.Table): 

79 super().__init__(name=name) 

80 self._db = db 

81 self._table = table 

82 

83 def insert(self, *data: dict, transaction: DatastoreTransaction | None = None) -> None: 

84 # Docstring inherited from OpaqueTableStorage. 

85 # The provided transaction object can be ignored since we rely on 

86 # the database itself providing any rollback functionality. 

87 self._db.insert(self._table, *data) 

88 

89 def ensure(self, *data: dict, transaction: DatastoreTransaction | None = None) -> None: 

90 # Docstring inherited from OpaqueTableStorage. 

91 # The provided transaction object can be ignored since we rely on 

92 # the database itself providing any rollback functionality. 

93 self._db.ensure(self._table, *data) 

94 

95 def replace(self, *data: dict, transaction: DatastoreTransaction | None = None) -> None: 

96 # Docstring inherited from OpaqueTableStorage. 

97 # The provided transaction object can be ignored since we rely on 

98 # the database itself providing any rollback functionality. 

99 self._db.replace(self._table, *data) 

100 

101 def fetch(self, **where: Any) -> Iterator[sqlalchemy.RowMapping]: 

102 # Docstring inherited from OpaqueTableStorage. 

103 

104 def _batch_in_clause( 

105 column: sqlalchemy.schema.Column, values: Iterable[Any] 

106 ) -> Iterator[sqlalchemy.sql.expression.ClauseElement]: 

107 """Split one long IN clause into a series of shorter ones.""" 

108 in_limit = 1000 

109 # We have to remove possible duplicates from values; and in many 

110 # cases it should be helpful to order the items in the clause. 

111 values = sorted(set(values)) 

112 for iposn in range(0, len(values), in_limit): 

113 in_clause = column.in_(values[iposn : iposn + in_limit]) 

114 yield in_clause 

115 

116 def _batch_in_clauses(**where: Any) -> Iterator[sqlalchemy.sql.expression.ColumnElement]: 

117 """Generate a sequence of WHERE clauses with a limited number of 

118 items in IN clauses. 

119 """ 

120 batches: list[Iterable[Any]] = [] 

121 for k, v in where.items(): 

122 column = self._table.columns[k] 

123 if isinstance(v, list | tuple | set): 

124 batches.append(_batch_in_clause(column, v)) 

125 else: 

126 # single "batch" for a regular eq operator 

127 batches.append([column == v]) 

128 

129 for clauses in itertools.product(*batches): 

130 yield sqlalchemy.sql.and_(*clauses) 

131 

132 sql = self._table.select() 

133 if where: 

134 # Split long IN clauses into shorter batches 

135 batched_sql = [sql.where(clause) for clause in _batch_in_clauses(**where)] 

136 else: 

137 batched_sql = [sql] 

138 for sql_batch in batched_sql: 

139 with self._db.query(sql_batch) as sql_result: 

140 sql_mappings = sql_result.mappings().fetchall() 

141 yield from sql_mappings 

142 

143 def delete(self, columns: Iterable[str], *rows: dict) -> None: 

144 # Docstring inherited from OpaqueTableStorage. 

145 self._db.delete(self._table, columns, *rows) 

146 

147 

148class ByNameOpaqueTableStorageManager(OpaqueTableStorageManager): 

149 """An implementation of `OpaqueTableStorageManager` that simply creates a 

150 true table for each different named opaque logical table. 

151 

152 Instances of this class should generally be constructed via the 

153 `initialize` class method instead of invoking ``__init__`` directly. 

154 

155 Parameters 

156 ---------- 

157 db : `Database` 

158 Database engine interface for the namespace in which this table lives. 

159 metaTable : `sqlalchemy.schema.Table` 

160 SQLAlchemy representation of the table that records which opaque 

161 logical tables exist. 

162 tables : `ThreadSafeCache` [`str`, `~sqlalchemy.schema.Table`] 

163 Mapping from string to table, to track which tables have already been 

164 created. This mapping is shared between cloned instances of this 

165 manager. 

166 registry_schema_version : `VersionTuple` or `None`, optional 

167 Version of registry schema. 

168 """ 

169 

170 def __init__( 

171 self, 

172 db: Database, 

173 metaTable: sqlalchemy.schema.Table, 

174 tables: ThreadSafeCache[str, sqlalchemy.schema.Table], 

175 registry_schema_version: VersionTuple | None = None, 

176 ): 

177 super().__init__(registry_schema_version=registry_schema_version) 

178 self._db = db 

179 self._metaTable = metaTable 

180 self._tables = tables 

181 

182 def clone(self, db: Database) -> ByNameOpaqueTableStorageManager: 

183 return ByNameOpaqueTableStorageManager( 

184 db, self._metaTable, self._tables, self._registry_schema_version 

185 ) 

186 

187 _META_TABLE_NAME: ClassVar[str] = "opaque_meta" 

188 

189 _META_TABLE_SPEC: ClassVar[TableSpec] = TableSpec( 

190 fields=[ 

191 FieldSpec("table_name", dtype=sqlalchemy.String, length=128, primaryKey=True), 

192 ], 

193 ) 

194 

195 @classmethod 

196 def initialize( 

197 cls, db: Database, context: StaticTablesContext, registry_schema_version: VersionTuple | None = None 

198 ) -> OpaqueTableStorageManager: 

199 # Docstring inherited from OpaqueTableStorageManager. 

200 metaTable = context.addTable(cls._META_TABLE_NAME, cls._META_TABLE_SPEC) 

201 return cls( 

202 db=db, 

203 metaTable=metaTable, 

204 tables=ThreadSafeCache(), 

205 registry_schema_version=registry_schema_version, 

206 ) 

207 

208 def get(self, name: str) -> OpaqueTableStorage | None: 

209 # Docstring inherited from OpaqueTableStorageManager. 

210 table = self._tables.get(name) 

211 if table is None: 

212 return None 

213 return ByNameOpaqueTableStorage(name=name, table=table, db=self._db) 

214 

215 def register(self, name: str, spec: TableSpec) -> OpaqueTableStorage: 

216 # Docstring inherited from OpaqueTableStorageManager. 

217 result = self.get(name) 

218 if result is None: 

219 # Create the table itself. If it already exists but wasn't in 

220 # the dict because it was added by another client since this one 

221 # was initialized, that's fine. 

222 table = self._db.ensureTableExists(name, spec) 

223 # Add a row to the meta table so we can find this table in the 

224 # future. Also okay if that already exists, so we use sync. 

225 self._db.sync(self._metaTable, keys={"table_name": name}) 

226 self._tables.set_or_get(name, table) 

227 result = self.get(name) 

228 assert result is not None 

229 return result 

230 

231 @classmethod 

232 def currentVersions(cls) -> list[VersionTuple]: 

233 # Docstring inherited from VersionedExtension. 

234 return [_VERSION]