Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21from __future__ import annotations 

22 

23__all__ = ["CachingDimensionRecordStorage"] 

24 

25from typing import Dict, Iterable, Optional, Set 

26 

27import sqlalchemy 

28 

29from ...core import ( 

30 DataCoordinate, 

31 DataCoordinateIterable, 

32 DataCoordinateSet, 

33 DimensionElement, 

34 DimensionRecord, 

35 NamedKeyDict, 

36 Timespan 

37) 

38from ..interfaces import Database, DimensionRecordStorage, StaticTablesContext 

39from ..queries import QueryBuilder 

40 

41 

42class CachingDimensionRecordStorage(DimensionRecordStorage): 

43 """A record storage implementation that adds caching to some other nested 

44 storage implementation. 

45 

46 Parameters 

47 ---------- 

48 nested : `DimensionRecordStorage` 

49 The other storage to cache fetches from and to delegate all other 

50 operations to. 

51 """ 

52 def __init__(self, nested: DimensionRecordStorage): 

53 self._nested = nested 

54 self._cache: Dict[DataCoordinate, Optional[DimensionRecord]] = {} 

55 

56 @classmethod 

57 def initialize(cls, db: Database, element: DimensionElement, *, 

58 context: Optional[StaticTablesContext] = None) -> DimensionRecordStorage: 

59 # Docstring inherited from DimensionRecordStorage. 

60 NestedClass = DimensionRecordStorage.getDefaultImplementation(element, ignoreCached=True) 

61 nested = NestedClass.initialize(db, element, context=context) 

62 return cls(nested) 

63 

64 @property 

65 def element(self) -> DimensionElement: 

66 # Docstring inherited from DimensionRecordStorage.element. 

67 return self._nested.element 

68 

69 def clearCaches(self) -> None: 

70 # Docstring inherited from DimensionRecordStorage.clearCaches. 

71 self._cache.clear() 

72 self._nested.clearCaches() 

73 

74 def join( 

75 self, 

76 builder: QueryBuilder, *, 

77 regions: Optional[NamedKeyDict[DimensionElement, sqlalchemy.sql.ColumnElement]] = None, 

78 timespans: Optional[NamedKeyDict[DimensionElement, Timespan[sqlalchemy.sql.ColumnElement]]] = None, 

79 ) -> None: 

80 # Docstring inherited from DimensionRecordStorage. 

81 return self._nested.join(builder, regions=regions, timespans=timespans) 

82 

83 def insert(self, *records: DimensionRecord) -> None: 

84 # Docstring inherited from DimensionRecordStorage.insert. 

85 self._nested.insert(*records) 

86 for record in records: 

87 self._cache[record.dataId] = record 

88 

89 def sync(self, record: DimensionRecord) -> bool: 

90 # Docstring inherited from DimensionRecordStorage.sync. 

91 inserted = self._nested.sync(record) 

92 if inserted: 

93 self._cache[record.dataId] = record 

94 return inserted 

95 

96 def fetch(self, dataIds: DataCoordinateIterable) -> Iterable[DimensionRecord]: 

97 # Docstring inherited from DimensionRecordStorage.fetch. 

98 missing: Set[DataCoordinate] = set() 

99 for dataId in dataIds: 

100 # Use ... as sentinal value so we can also cache None == "no such 

101 # record exists". 

102 record = self._cache.get(dataId, ...) 

103 if record is ...: 

104 missing.add(dataId) 

105 elif record is not None: 105 ↛ 99line 105 didn't jump to line 99, because the condition on line 105 was never false

106 # Unclear why MyPy can't tell that this isn't ..., but it 

107 # thinks it's still a possibility. 

108 yield record # type: ignore 

109 if missing: 

110 toFetch = DataCoordinateSet(missing, graph=self.element.graph) 

111 for record in self._nested.fetch(toFetch): 

112 self._cache[record.dataId] = record 

113 yield record 

114 missing -= self._cache.keys() 

115 for dataId in missing: 

116 self._cache[dataId] = None 

117 

118 def digestTables(self) -> Iterable[sqlalchemy.schema.Table]: 

119 # Docstring inherited from DimensionRecordStorage.digestTables. 

120 return self._nested.digestTables()