Coverage for python/lsst/daf/butler/registry/interfaces/_obscore.py: 96%

28 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2023-04-14 09:22 +0000

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21from __future__ import annotations 

22 

23"""Interfaces for classes that manage obscore table(s) in a `Registry`. 

24""" 

25 

26__all__ = ["ObsCoreTableManager"] 

27 

28from abc import abstractmethod 

29from collections.abc import Iterable, Iterator, Mapping 

30from contextlib import contextmanager 

31from typing import TYPE_CHECKING, Any, Type 

32 

33import sqlalchemy 

34 

35from ._versioning import VersionedExtension, VersionTuple 

36 

37if TYPE_CHECKING: 

38 from lsst.sphgeom import Region 

39 

40 from ...core import DatasetRef, DimensionUniverse 

41 from ..queries import SqlQueryContext 

42 from ._collections import CollectionRecord 

43 from ._database import Database, StaticTablesContext 

44 from ._datasets import DatasetRecordStorageManager 

45 from ._dimensions import DimensionRecordStorageManager 

46 

47 

48class ObsCoreTableManager(VersionedExtension): 

49 """An interface for populating ObsCore tables(s).""" 

50 

51 def __init__(self, *, registry_schema_version: VersionTuple | None = None): 

52 super().__init__(registry_schema_version=registry_schema_version) 

53 

54 @classmethod 

55 @abstractmethod 

56 def initialize( 

57 cls, 

58 db: Database, 

59 context: StaticTablesContext, 

60 *, 

61 universe: DimensionUniverse, 

62 config: Mapping, 

63 datasets: Type[DatasetRecordStorageManager], 

64 dimensions: DimensionRecordStorageManager, 

65 registry_schema_version: VersionTuple | None = None, 

66 ) -> ObsCoreTableManager: 

67 """Construct an instance of the manager. 

68 

69 Parameters 

70 ---------- 

71 db : `Database` 

72 Interface to the underlying database engine and namespace. 

73 context : `StaticTablesContext` 

74 Context object obtained from `Database.declareStaticTables`; used 

75 to declare any tables that should always be present in a layer 

76 implemented with this manager. 

77 universe : `DimensionUniverse` 

78 All dimensions known to the registry. 

79 config : `dict` [ `str`, `Any` ] 

80 Configuration of the obscore manager. 

81 datasets : `type` 

82 Type of dataset manager. 

83 dimensions: `DimensionRecordStorageManager` 

84 Manager for Registry dimensions. 

85 registry_schema_version : `VersionTuple` or `None` 

86 Schema version of this extension as defined in registry. 

87 

88 Returns 

89 ------- 

90 manager : `ObsCoreTableManager` 

91 An instance of a concrete `ObsCoreTableManager` subclass. 

92 """ 

93 raise NotImplementedError() 

94 

95 @abstractmethod 

96 def config_json(self) -> str: 

97 """Dump configuration in JSON format. 

98 

99 Returns 

100 ------- 

101 json : `str` 

102 Configuration serialized in JSON format. 

103 """ 

104 raise NotImplementedError() 

105 

106 @abstractmethod 

107 def add_datasets(self, refs: Iterable[DatasetRef], context: SqlQueryContext) -> int: 

108 """Possibly add datasets to the obscore table. 

109 

110 This method should be called when new datasets are added to a RUN 

111 collection. 

112 

113 Parameters 

114 ---------- 

115 refs : `iterable` [ `DatasetRef` ] 

116 Dataset refs to add. Dataset refs have to be completely expanded. 

117 If a record with the same dataset ID is already in obscore table, 

118 the dataset is ignored. 

119 context : `SqlQueryContext` 

120 Context used to execute queries for additional dimension metadata. 

121 

122 Returns 

123 ------- 

124 count : `int` 

125 Actual number of records inserted into obscore table. 

126 

127 Notes 

128 ----- 

129 Dataset data types and collection names are checked against configured 

130 list of collections and dataset types, non-matching datasets are 

131 ignored and not added to the obscore table. 

132 

133 When configuration parameter ``collection_type`` is not "RUN", this 

134 method should return immediately. 

135 

136 Note that there is no matching method to remove datasets from obscore 

137 table, we assume that removal happens via foreign key constraint to 

138 dataset table with "ON DELETE CASCADE" option. 

139 """ 

140 raise NotImplementedError() 

141 

142 @abstractmethod 

143 def associate( 

144 self, refs: Iterable[DatasetRef], collection: CollectionRecord, context: SqlQueryContext 

145 ) -> int: 

146 """Possibly add datasets to the obscore table. 

147 

148 This method should be called when existing datasets are associated with 

149 a TAGGED collection. 

150 

151 Parameters 

152 ---------- 

153 refs : `iterable` [ `DatasetRef` ] 

154 Dataset refs to add. Dataset refs have to be completely expanded. 

155 If a record with the same dataset ID is already in obscore table, 

156 the dataset is ignored. 

157 collection : `CollectionRecord` 

158 Collection record for a TAGGED collection. 

159 context : `SqlQueryContext` 

160 Context used to execute queries for additional dimension metadata. 

161 

162 Returns 

163 ------- 

164 count : `int` 

165 Actual number of records inserted into obscore table. 

166 

167 Notes 

168 ----- 

169 Dataset data types and collection names are checked against configured 

170 list of collections and dataset types, non-matching datasets are 

171 ignored and not added to the obscore table. 

172 

173 When configuration parameter ``collection_type`` is not "TAGGED", this 

174 method should return immediately. 

175 """ 

176 raise NotImplementedError() 

177 

178 @abstractmethod 

179 def disassociate(self, refs: Iterable[DatasetRef], collection: CollectionRecord) -> int: 

180 """Possibly remove datasets from the obscore table. 

181 

182 This method should be called when datasets are disassociated from a 

183 TAGGED collection. 

184 

185 Parameters 

186 ---------- 

187 refs : `iterable` [ `DatasetRef` ] 

188 Dataset refs to remove. Dataset refs have to be resolved. 

189 collection : `CollectionRecord` 

190 Collection record for a TAGGED collection. 

191 

192 Returns 

193 ------- 

194 count : `int` 

195 Actual number of records removed from obscore table. 

196 

197 Notes 

198 ----- 

199 Dataset data types and collection names are checked against configured 

200 list of collections and dataset types, non-matching datasets are 

201 ignored and not added to the obscore table. 

202 

203 When configuration parameter ``collection_type`` is not "TAGGED", this 

204 method should return immediately. 

205 """ 

206 raise NotImplementedError() 

207 

208 @abstractmethod 

209 def update_exposure_regions(self, instrument: str, region_data: Iterable[tuple[int, int, Region]]) -> int: 

210 """Update existing exposure records with spatial region data. 

211 

212 Parameters 

213 ---------- 

214 instrument : `str` 

215 Instrument name. 

216 region_data : `Iterable`[`tuple`[`int`, `int`, `~lsst.sphgeom.Region`]] 

217 Sequence of tuples, each tuple contains three values - exposure ID, 

218 detector ID, and corresponding region. 

219 

220 Returns 

221 ------- 

222 count : `int` 

223 Actual number of records updated. 

224 

225 Notes 

226 ----- 

227 This method is needed to update obscore records for raw exposures which 

228 are ingested before their corresponding visits are defined. Exposure 

229 records added when visit is already defined will get their regions 

230 from their matching visits automatically. 

231 """ 

232 raise NotImplementedError() 

233 

234 @abstractmethod 

235 @contextmanager 

236 def query( 

237 self, columns: Iterable[str | sqlalchemy.sql.expression.ColumnElement] | None = None, /, **kwargs: Any 

238 ) -> Iterator[sqlalchemy.engine.CursorResult]: 

239 """Run a SELECT query against obscore table and return result rows. 

240 

241 Parameters 

242 ---------- 

243 columns : `~collections.abc.Iterable` [`str`] 

244 Columns to return from query. It is a sequence which can include 

245 column names or any other column elements (e.g. 

246 `sqlalchemy.sql.functions.count` function). 

247 **kwargs 

248 Restriction on values of individual obscore columns. Key is the 

249 column name, value is the required value of the column. Multiple 

250 restrictions are ANDed together. 

251 

252 Returns 

253 ------- 

254 result_context : `sqlalchemy.engine.CursorResult` 

255 Context manager that returns the query result object when entered. 

256 These results are invalidated when the context is exited. 

257 """ 

258 raise NotImplementedError()