Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21from __future__ import annotations 

22 

23__all__ = ["BasicGovernorDimensionRecordStorage"] 

24 

25from typing import AbstractSet, Any, Callable, Dict, Iterable, List, Mapping, Optional 

26 

27import sqlalchemy 

28 

29from ...core import ( 

30 DataCoordinateIterable, 

31 DimensionElement, 

32 DimensionRecord, 

33 GovernorDimension, 

34 NamedKeyDict, 

35 TimespanDatabaseRepresentation, 

36) 

37from ..interfaces import ( 

38 Database, 

39 GovernorDimensionRecordStorage, 

40 StaticTablesContext, 

41) 

42from ..queries import QueryBuilder 

43 

44 

45class BasicGovernorDimensionRecordStorage(GovernorDimensionRecordStorage): 

46 """A record storage implementation for `GovernorDimension` that 

47 aggressively fetches and caches all values from the database. 

48 

49 Parameters 

50 ---------- 

51 db : `Database` 

52 Interface to the database engine and namespace that will hold these 

53 dimension records. 

54 dimension : `GovernorDimension` 

55 The dimension whose records this storage will manage. 

56 table : `sqlalchemy.schema.Table` 

57 The logical table for the dimension. 

58 """ 

59 def __init__(self, db: Database, dimension: GovernorDimension, table: sqlalchemy.schema.Table): 

60 self._db = db 

61 self._dimension = dimension 

62 self._table = table 

63 self._cache: Dict[str, DimensionRecord] = {} 

64 self._callbacks: List[Callable[[DimensionRecord], None]] = [] 

65 

66 @classmethod 

67 def initialize(cls, db: Database, element: GovernorDimension, *, 

68 context: Optional[StaticTablesContext] = None, 

69 config: Mapping[str, Any]) -> GovernorDimensionRecordStorage: 

70 # Docstring inherited from GovernorDimensionRecordStorage. 

71 spec = element.RecordClass.fields.makeTableSpec( 

72 RegionReprClass=db.getSpatialRegionRepresentation(), 

73 TimespanReprClass=db.getTimespanRepresentation(), 

74 ) 

75 if context is not None: 75 ↛ 78line 75 didn't jump to line 78, because the condition on line 75 was never false

76 table = context.addTable(element.name, spec) 

77 else: 

78 table = db.ensureTableExists(element.name, spec) 

79 return cls(db, element, table) 

80 

81 @property 

82 def element(self) -> GovernorDimension: 

83 # Docstring inherited from DimensionRecordStorage.element. 

84 return self._dimension 

85 

86 def refresh(self) -> None: 

87 # Docstring inherited from GovernorDimensionRecordStorage. 

88 RecordClass = self._dimension.RecordClass 

89 sql = sqlalchemy.sql.select( 

90 [self._table.columns[name] for name in RecordClass.fields.standard.names] 

91 ).select_from(self._table) 

92 cache: Dict[str, DimensionRecord] = {} 

93 for row in self._db.query(sql): 

94 record = RecordClass(**dict(row)) 

95 cache[getattr(record, self._dimension.primaryKey.name)] = record 

96 self._cache = cache 

97 

98 @property 

99 def values(self) -> AbstractSet[str]: 

100 # Docstring inherited from GovernorDimensionRecordStorage. 

101 return self._cache.keys() 

102 

103 @property 

104 def table(self) -> sqlalchemy.schema.Table: 

105 return self._table 

106 

107 def registerInsertionListener(self, callback: Callable[[DimensionRecord], None]) -> None: 

108 # Docstring inherited from GovernorDimensionRecordStorage. 

109 self._callbacks.append(callback) 

110 

111 def clearCaches(self) -> None: 

112 # Docstring inherited from DimensionRecordStorage.clearCaches. 

113 self._cache.clear() 

114 

115 def join( 

116 self, 

117 builder: QueryBuilder, *, 

118 regions: Optional[NamedKeyDict[DimensionElement, sqlalchemy.sql.ColumnElement]] = None, 

119 timespans: Optional[NamedKeyDict[DimensionElement, TimespanDatabaseRepresentation]] = None, 

120 ) -> None: 

121 # Docstring inherited from DimensionRecordStorage. 

122 joinOn = builder.startJoin(self._table, self.element.dimensions, 

123 self.element.RecordClass.fields.dimensions.names) 

124 builder.finishJoin(self._table, joinOn) 

125 return self._table 

126 

127 def insert(self, *records: DimensionRecord) -> None: 

128 # Docstring inherited from DimensionRecordStorage.insert. 

129 elementRows = [record.toDict() for record in records] 

130 with self._db.transaction(): 

131 self._db.insert(self._table, *elementRows) 

132 for record in records: 

133 self._cache[getattr(record, self.element.primaryKey.name)] = record 

134 for callback in self._callbacks: 

135 callback(record) 

136 

137 def sync(self, record: DimensionRecord) -> bool: 

138 # Docstring inherited from DimensionRecordStorage.sync. 

139 compared = record.toDict() 

140 keys = {} 

141 for name in record.fields.required.names: 

142 keys[name] = compared.pop(name) 

143 with self._db.transaction(): 

144 _, inserted = self._db.sync( 

145 self._table, 

146 keys=keys, 

147 compared=compared, 

148 ) 

149 if inserted: 

150 self._cache[getattr(record, self.element.primaryKey.name)] = record 

151 for callback in self._callbacks: 

152 callback(record) 

153 return inserted 

154 

155 def fetch(self, dataIds: DataCoordinateIterable) -> Iterable[DimensionRecord]: 

156 # Docstring inherited from DimensionRecordStorage.fetch. 

157 try: 

158 return [self._cache[dataId[self.element]] for dataId in dataIds] # type: ignore 

159 except KeyError: 

160 pass 

161 # If at first we don't succeed, refresh and try again. But this time 

162 # we use dict.get to return None if we don't find something. 

163 self.refresh() 

164 return [self._cache.get(dataId[self.element]) for dataId in dataIds] # type: ignore 

165 

166 def digestTables(self) -> Iterable[sqlalchemy.schema.Table]: 

167 # Docstring inherited from DimensionRecordStorage.digestTables. 

168 return [self._table]