Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21from __future__ import annotations 

22 

23__all__ = ["CachingDimensionRecordStorage"] 

24 

25from typing import Any, Dict, Iterable, Mapping, Optional, Set 

26 

27import sqlalchemy 

28 

29from lsst.utils import doImport 

30 

31from ...core import ( 

32 DatabaseDimensionElement, 

33 DataCoordinate, 

34 DataCoordinateIterable, 

35 DataCoordinateSet, 

36 DimensionElement, 

37 DimensionRecord, 

38 GovernorDimension, 

39 NamedKeyDict, 

40 NamedKeyMapping, 

41 SpatialRegionDatabaseRepresentation, 

42 TimespanDatabaseRepresentation, 

43) 

44from ..interfaces import ( 

45 Database, 

46 DatabaseDimensionRecordStorage, 

47 GovernorDimensionRecordStorage, 

48 StaticTablesContext, 

49) 

50from ..queries import QueryBuilder 

51 

52 

53class CachingDimensionRecordStorage(DatabaseDimensionRecordStorage): 

54 """A record storage implementation that adds caching to some other nested 

55 storage implementation. 

56 

57 Parameters 

58 ---------- 

59 nested : `DatabaseDimensionRecordStorage` 

60 The other storage to cache fetches from and to delegate all other 

61 operations to. 

62 """ 

63 def __init__(self, nested: DatabaseDimensionRecordStorage): 

64 self._nested = nested 

65 self._cache: Dict[DataCoordinate, Optional[DimensionRecord]] = {} 

66 

67 @classmethod 

68 def initialize( 

69 cls, 

70 db: Database, 

71 element: DatabaseDimensionElement, *, 

72 context: Optional[StaticTablesContext] = None, 

73 config: Mapping[str, Any], 

74 governors: NamedKeyMapping[GovernorDimension, GovernorDimensionRecordStorage], 

75 ) -> DatabaseDimensionRecordStorage: 

76 # Docstring inherited from DatabaseDimensionRecordStorage. 

77 config = config["nested"] 

78 NestedClass = doImport(config["cls"]) 

79 nested = NestedClass.initialize(db, element, context=context, config=config, governors=governors) 

80 return cls(nested) 

81 

82 @property 

83 def element(self) -> DatabaseDimensionElement: 

84 # Docstring inherited from DimensionRecordStorage.element. 

85 return self._nested.element 

86 

87 def clearCaches(self) -> None: 

88 # Docstring inherited from DimensionRecordStorage.clearCaches. 

89 self._cache.clear() 

90 self._nested.clearCaches() 

91 

92 def join( 

93 self, 

94 builder: QueryBuilder, *, 

95 regions: Optional[NamedKeyDict[DimensionElement, SpatialRegionDatabaseRepresentation]] = None, 

96 timespans: Optional[NamedKeyDict[DimensionElement, TimespanDatabaseRepresentation]] = None, 

97 ) -> None: 

98 # Docstring inherited from DimensionRecordStorage. 

99 return self._nested.join(builder, regions=regions, timespans=timespans) 

100 

101 def insert(self, *records: DimensionRecord) -> None: 

102 # Docstring inherited from DimensionRecordStorage.insert. 

103 self._nested.insert(*records) 

104 for record in records: 

105 self._cache[record.dataId] = record 

106 

107 def sync(self, record: DimensionRecord) -> bool: 

108 # Docstring inherited from DimensionRecordStorage.sync. 

109 inserted = self._nested.sync(record) 

110 if inserted: 

111 self._cache[record.dataId] = record 

112 return inserted 

113 

114 def fetch(self, dataIds: DataCoordinateIterable) -> Iterable[DimensionRecord]: 

115 # Docstring inherited from DimensionRecordStorage.fetch. 

116 missing: Set[DataCoordinate] = set() 

117 for dataId in dataIds: 

118 # Use ... as sentinal value so we can also cache None == "no such 

119 # record exists". 

120 record = self._cache.get(dataId, ...) 

121 if record is ...: 

122 missing.add(dataId) 

123 elif record is not None: 123 ↛ 117line 123 didn't jump to line 117, because the condition on line 123 was never false

124 # Unclear why MyPy can't tell that this isn't ..., but it 

125 # thinks it's still a possibility. 

126 yield record # type: ignore 

127 if missing: 

128 toFetch = DataCoordinateSet(missing, graph=self.element.graph) 

129 for record in self._nested.fetch(toFetch): 

130 self._cache[record.dataId] = record 

131 yield record 

132 missing -= self._cache.keys() 

133 for dataId in missing: 133 ↛ 134line 133 didn't jump to line 134, because the loop on line 133 never started

134 self._cache[dataId] = None 

135 

136 def digestTables(self) -> Iterable[sqlalchemy.schema.Table]: 

137 # Docstring inherited from DimensionRecordStorage.digestTables. 

138 return self._nested.digestTables()