Coverage for python/lsst/daf/butler/registry/interfaces/_obscore.py: 98%

28 statements  

« prev     ^ index     » next       coverage.py v7.3.2, created at 2023-12-05 11:07 +0000

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This software is dual licensed under the GNU General Public License and also 

10# under a 3-clause BSD license. Recipients may choose which of these licenses 

11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, 

12# respectively. If you choose the GPL option then the following text applies 

13# (but note that there is still no warranty even if you opt for BSD instead): 

14# 

15# This program is free software: you can redistribute it and/or modify 

16# it under the terms of the GNU General Public License as published by 

17# the Free Software Foundation, either version 3 of the License, or 

18# (at your option) any later version. 

19# 

20# This program is distributed in the hope that it will be useful, 

21# but WITHOUT ANY WARRANTY; without even the implied warranty of 

22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

23# GNU General Public License for more details. 

24# 

25# You should have received a copy of the GNU General Public License 

26# along with this program. If not, see <http://www.gnu.org/licenses/>. 

27 

28"""Interfaces for classes that manage obscore table(s) in a `Registry`. 

29""" 

30 

31from __future__ import annotations 

32 

33__all__ = ["ObsCoreTableManager"] 

34 

35from abc import abstractmethod 

36from collections.abc import Iterable, Iterator, Mapping 

37from contextlib import contextmanager 

38from typing import TYPE_CHECKING, Any 

39 

40import sqlalchemy 

41 

42from ._versioning import VersionedExtension, VersionTuple 

43 

44if TYPE_CHECKING: 

45 from lsst.sphgeom import Region 

46 

47 from ..._dataset_ref import DatasetRef 

48 from ...dimensions import DimensionUniverse 

49 from ..queries import SqlQueryContext 

50 from ._collections import CollectionRecord 

51 from ._database import Database, StaticTablesContext 

52 from ._datasets import DatasetRecordStorageManager 

53 from ._dimensions import DimensionRecordStorageManager 

54 

55 

56class ObsCoreTableManager(VersionedExtension): 

57 """An interface for populating ObsCore tables(s).""" 

58 

59 def __init__(self, *, registry_schema_version: VersionTuple | None = None): 

60 super().__init__(registry_schema_version=registry_schema_version) 

61 

62 @classmethod 

63 @abstractmethod 

64 def initialize( 

65 cls, 

66 db: Database, 

67 context: StaticTablesContext, 

68 *, 

69 universe: DimensionUniverse, 

70 config: Mapping, 

71 datasets: type[DatasetRecordStorageManager], 

72 dimensions: DimensionRecordStorageManager, 

73 registry_schema_version: VersionTuple | None = None, 

74 ) -> ObsCoreTableManager: 

75 """Construct an instance of the manager. 

76 

77 Parameters 

78 ---------- 

79 db : `Database` 

80 Interface to the underlying database engine and namespace. 

81 context : `StaticTablesContext` 

82 Context object obtained from `Database.declareStaticTables`; used 

83 to declare any tables that should always be present in a layer 

84 implemented with this manager. 

85 universe : `DimensionUniverse` 

86 All dimensions known to the registry. 

87 config : `dict` [ `str`, `Any` ] 

88 Configuration of the obscore manager. 

89 datasets : `type` 

90 Type of dataset manager. 

91 dimensions: `DimensionRecordStorageManager` 

92 Manager for Registry dimensions. 

93 registry_schema_version : `VersionTuple` or `None` 

94 Schema version of this extension as defined in registry. 

95 

96 Returns 

97 ------- 

98 manager : `ObsCoreTableManager` 

99 An instance of a concrete `ObsCoreTableManager` subclass. 

100 """ 

101 raise NotImplementedError() 

102 

103 @abstractmethod 

104 def config_json(self) -> str: 

105 """Dump configuration in JSON format. 

106 

107 Returns 

108 ------- 

109 json : `str` 

110 Configuration serialized in JSON format. 

111 """ 

112 raise NotImplementedError() 

113 

114 @abstractmethod 

115 def add_datasets(self, refs: Iterable[DatasetRef], context: SqlQueryContext) -> int: 

116 """Possibly add datasets to the obscore table. 

117 

118 This method should be called when new datasets are added to a RUN 

119 collection. 

120 

121 Parameters 

122 ---------- 

123 refs : `iterable` [ `DatasetRef` ] 

124 Dataset refs to add. Dataset refs have to be completely expanded. 

125 If a record with the same dataset ID is already in obscore table, 

126 the dataset is ignored. 

127 context : `SqlQueryContext` 

128 Context used to execute queries for additional dimension metadata. 

129 

130 Returns 

131 ------- 

132 count : `int` 

133 Actual number of records inserted into obscore table. 

134 

135 Notes 

136 ----- 

137 Dataset data types and collection names are checked against configured 

138 list of collections and dataset types, non-matching datasets are 

139 ignored and not added to the obscore table. 

140 

141 When configuration parameter ``collection_type`` is not "RUN", this 

142 method should return immediately. 

143 

144 Note that there is no matching method to remove datasets from obscore 

145 table, we assume that removal happens via foreign key constraint to 

146 dataset table with "ON DELETE CASCADE" option. 

147 """ 

148 raise NotImplementedError() 

149 

150 @abstractmethod 

151 def associate( 

152 self, refs: Iterable[DatasetRef], collection: CollectionRecord, context: SqlQueryContext 

153 ) -> int: 

154 """Possibly add datasets to the obscore table. 

155 

156 This method should be called when existing datasets are associated with 

157 a TAGGED collection. 

158 

159 Parameters 

160 ---------- 

161 refs : `iterable` [ `DatasetRef` ] 

162 Dataset refs to add. Dataset refs have to be completely expanded. 

163 If a record with the same dataset ID is already in obscore table, 

164 the dataset is ignored. 

165 collection : `CollectionRecord` 

166 Collection record for a TAGGED collection. 

167 context : `SqlQueryContext` 

168 Context used to execute queries for additional dimension metadata. 

169 

170 Returns 

171 ------- 

172 count : `int` 

173 Actual number of records inserted into obscore table. 

174 

175 Notes 

176 ----- 

177 Dataset data types and collection names are checked against configured 

178 list of collections and dataset types, non-matching datasets are 

179 ignored and not added to the obscore table. 

180 

181 When configuration parameter ``collection_type`` is not "TAGGED", this 

182 method should return immediately. 

183 """ 

184 raise NotImplementedError() 

185 

186 @abstractmethod 

187 def disassociate(self, refs: Iterable[DatasetRef], collection: CollectionRecord) -> int: 

188 """Possibly remove datasets from the obscore table. 

189 

190 This method should be called when datasets are disassociated from a 

191 TAGGED collection. 

192 

193 Parameters 

194 ---------- 

195 refs : `iterable` [ `DatasetRef` ] 

196 Dataset refs to remove. Dataset refs have to be resolved. 

197 collection : `CollectionRecord` 

198 Collection record for a TAGGED collection. 

199 

200 Returns 

201 ------- 

202 count : `int` 

203 Actual number of records removed from obscore table. 

204 

205 Notes 

206 ----- 

207 Dataset data types and collection names are checked against configured 

208 list of collections and dataset types, non-matching datasets are 

209 ignored and not added to the obscore table. 

210 

211 When configuration parameter ``collection_type`` is not "TAGGED", this 

212 method should return immediately. 

213 """ 

214 raise NotImplementedError() 

215 

216 @abstractmethod 

217 def update_exposure_regions(self, instrument: str, region_data: Iterable[tuple[int, int, Region]]) -> int: 

218 """Update existing exposure records with spatial region data. 

219 

220 Parameters 

221 ---------- 

222 instrument : `str` 

223 Instrument name. 

224 region_data : `~collections.abc.Iterable` [`tuple` [`int`, `int`, \ 

225 `~lsst.sphgeom.Region` ]] 

226 Sequence of tuples, each tuple contains three values - exposure ID, 

227 detector ID, and corresponding region. 

228 

229 Returns 

230 ------- 

231 count : `int` 

232 Actual number of records updated. 

233 

234 Notes 

235 ----- 

236 This method is needed to update obscore records for raw exposures which 

237 are ingested before their corresponding visits are defined. Exposure 

238 records added when visit is already defined will get their regions 

239 from their matching visits automatically. 

240 """ 

241 raise NotImplementedError() 

242 

243 @abstractmethod 

244 @contextmanager 

245 def query( 

246 self, columns: Iterable[str | sqlalchemy.sql.expression.ColumnElement] | None = None, /, **kwargs: Any 

247 ) -> Iterator[sqlalchemy.engine.CursorResult]: 

248 """Run a SELECT query against obscore table and return result rows. 

249 

250 Parameters 

251 ---------- 

252 columns : `~collections.abc.Iterable` [`str`] 

253 Columns to return from query. It is a sequence which can include 

254 column names or any other column elements (e.g. 

255 `sqlalchemy.sql.functions.count` function). 

256 **kwargs 

257 Restriction on values of individual obscore columns. Key is the 

258 column name, value is the required value of the column. Multiple 

259 restrictions are ANDed together. 

260 

261 Returns 

262 ------- 

263 result_context : `sqlalchemy.engine.CursorResult` 

264 Context manager that returns the query result object when entered. 

265 These results are invalidated when the context is exited. 

266 """ 

267 raise NotImplementedError()