Coverage for python/lsst/daf/butler/registry/interfaces/_obscore.py: 98%

28 statements  

« prev     ^ index     » next       coverage.py v7.4.0, created at 2024-01-16 10:44 +0000

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This software is dual licensed under the GNU General Public License and also 

10# under a 3-clause BSD license. Recipients may choose which of these licenses 

11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, 

12# respectively. If you choose the GPL option then the following text applies 

13# (but note that there is still no warranty even if you opt for BSD instead): 

14# 

15# This program is free software: you can redistribute it and/or modify 

16# it under the terms of the GNU General Public License as published by 

17# the Free Software Foundation, either version 3 of the License, or 

18# (at your option) any later version. 

19# 

20# This program is distributed in the hope that it will be useful, 

21# but WITHOUT ANY WARRANTY; without even the implied warranty of 

22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

23# GNU General Public License for more details. 

24# 

25# You should have received a copy of the GNU General Public License 

26# along with this program. If not, see <http://www.gnu.org/licenses/>. 

27 

28"""Interfaces for classes that manage obscore table(s) in a `Registry`. 

29""" 

30 

31from __future__ import annotations 

32 

33__all__ = ["ObsCoreTableManager"] 

34 

35from abc import abstractmethod 

36from collections.abc import Iterable, Iterator, Mapping 

37from contextlib import contextmanager 

38from typing import TYPE_CHECKING, Any 

39 

40import sqlalchemy 

41 

42from ._versioning import VersionedExtension, VersionTuple 

43 

44if TYPE_CHECKING: 

45 from lsst.sphgeom import Region 

46 

47 from ..._dataset_ref import DatasetRef 

48 from ...dimensions import DimensionUniverse 

49 from ..queries import SqlQueryContext 

50 from ._collections import CollectionRecord 

51 from ._database import Database, StaticTablesContext 

52 from ._datasets import DatasetRecordStorageManager 

53 from ._dimensions import DimensionRecordStorageManager 

54 

55 

56class ObsCoreTableManager(VersionedExtension): 

57 """An interface for populating ObsCore tables(s). 

58 

59 Parameters 

60 ---------- 

61 registry_schema_version : `VersionTuple` or `None`, optional 

62 Version of registry schema. 

63 """ 

64 

65 def __init__(self, *, registry_schema_version: VersionTuple | None = None): 

66 super().__init__(registry_schema_version=registry_schema_version) 

67 

68 @classmethod 

69 @abstractmethod 

70 def initialize( 

71 cls, 

72 db: Database, 

73 context: StaticTablesContext, 

74 *, 

75 universe: DimensionUniverse, 

76 config: Mapping, 

77 datasets: type[DatasetRecordStorageManager], 

78 dimensions: DimensionRecordStorageManager, 

79 registry_schema_version: VersionTuple | None = None, 

80 ) -> ObsCoreTableManager: 

81 """Construct an instance of the manager. 

82 

83 Parameters 

84 ---------- 

85 db : `Database` 

86 Interface to the underlying database engine and namespace. 

87 context : `StaticTablesContext` 

88 Context object obtained from `Database.declareStaticTables`; used 

89 to declare any tables that should always be present in a layer 

90 implemented with this manager. 

91 universe : `DimensionUniverse` 

92 All dimensions known to the registry. 

93 config : `dict` [ `str`, `Any` ] 

94 Configuration of the obscore manager. 

95 datasets : `type` 

96 Type of dataset manager. 

97 dimensions : `DimensionRecordStorageManager` 

98 Manager for Registry dimensions. 

99 registry_schema_version : `VersionTuple` or `None` 

100 Schema version of this extension as defined in registry. 

101 

102 Returns 

103 ------- 

104 manager : `ObsCoreTableManager` 

105 An instance of a concrete `ObsCoreTableManager` subclass. 

106 """ 

107 raise NotImplementedError() 

108 

109 @abstractmethod 

110 def config_json(self) -> str: 

111 """Dump configuration in JSON format. 

112 

113 Returns 

114 ------- 

115 json : `str` 

116 Configuration serialized in JSON format. 

117 """ 

118 raise NotImplementedError() 

119 

120 @abstractmethod 

121 def add_datasets(self, refs: Iterable[DatasetRef], context: SqlQueryContext) -> int: 

122 """Possibly add datasets to the obscore table. 

123 

124 This method should be called when new datasets are added to a RUN 

125 collection. 

126 

127 Parameters 

128 ---------- 

129 refs : `iterable` [ `DatasetRef` ] 

130 Dataset refs to add. Dataset refs have to be completely expanded. 

131 If a record with the same dataset ID is already in obscore table, 

132 the dataset is ignored. 

133 context : `SqlQueryContext` 

134 Context used to execute queries for additional dimension metadata. 

135 

136 Returns 

137 ------- 

138 count : `int` 

139 Actual number of records inserted into obscore table. 

140 

141 Notes 

142 ----- 

143 Dataset data types and collection names are checked against configured 

144 list of collections and dataset types, non-matching datasets are 

145 ignored and not added to the obscore table. 

146 

147 When configuration parameter ``collection_type`` is not "RUN", this 

148 method should return immediately. 

149 

150 Note that there is no matching method to remove datasets from obscore 

151 table, we assume that removal happens via foreign key constraint to 

152 dataset table with "ON DELETE CASCADE" option. 

153 """ 

154 raise NotImplementedError() 

155 

156 @abstractmethod 

157 def associate( 

158 self, refs: Iterable[DatasetRef], collection: CollectionRecord, context: SqlQueryContext 

159 ) -> int: 

160 """Possibly add datasets to the obscore table. 

161 

162 This method should be called when existing datasets are associated with 

163 a TAGGED collection. 

164 

165 Parameters 

166 ---------- 

167 refs : `iterable` [ `DatasetRef` ] 

168 Dataset refs to add. Dataset refs have to be completely expanded. 

169 If a record with the same dataset ID is already in obscore table, 

170 the dataset is ignored. 

171 collection : `CollectionRecord` 

172 Collection record for a TAGGED collection. 

173 context : `SqlQueryContext` 

174 Context used to execute queries for additional dimension metadata. 

175 

176 Returns 

177 ------- 

178 count : `int` 

179 Actual number of records inserted into obscore table. 

180 

181 Notes 

182 ----- 

183 Dataset data types and collection names are checked against configured 

184 list of collections and dataset types, non-matching datasets are 

185 ignored and not added to the obscore table. 

186 

187 When configuration parameter ``collection_type`` is not "TAGGED", this 

188 method should return immediately. 

189 """ 

190 raise NotImplementedError() 

191 

192 @abstractmethod 

193 def disassociate(self, refs: Iterable[DatasetRef], collection: CollectionRecord) -> int: 

194 """Possibly remove datasets from the obscore table. 

195 

196 This method should be called when datasets are disassociated from a 

197 TAGGED collection. 

198 

199 Parameters 

200 ---------- 

201 refs : `iterable` [ `DatasetRef` ] 

202 Dataset refs to remove. Dataset refs have to be resolved. 

203 collection : `CollectionRecord` 

204 Collection record for a TAGGED collection. 

205 

206 Returns 

207 ------- 

208 count : `int` 

209 Actual number of records removed from obscore table. 

210 

211 Notes 

212 ----- 

213 Dataset data types and collection names are checked against configured 

214 list of collections and dataset types, non-matching datasets are 

215 ignored and not added to the obscore table. 

216 

217 When configuration parameter ``collection_type`` is not "TAGGED", this 

218 method should return immediately. 

219 """ 

220 raise NotImplementedError() 

221 

222 @abstractmethod 

223 def update_exposure_regions(self, instrument: str, region_data: Iterable[tuple[int, int, Region]]) -> int: 

224 """Update existing exposure records with spatial region data. 

225 

226 Parameters 

227 ---------- 

228 instrument : `str` 

229 Instrument name. 

230 region_data : `~collections.abc.Iterable` [`tuple` [`int`, `int`, \ 

231 `~lsst.sphgeom.Region` ]] 

232 Sequence of tuples, each tuple contains three values - exposure ID, 

233 detector ID, and corresponding region. 

234 

235 Returns 

236 ------- 

237 count : `int` 

238 Actual number of records updated. 

239 

240 Notes 

241 ----- 

242 This method is needed to update obscore records for raw exposures which 

243 are ingested before their corresponding visits are defined. Exposure 

244 records added when visit is already defined will get their regions 

245 from their matching visits automatically. 

246 """ 

247 raise NotImplementedError() 

248 

249 @abstractmethod 

250 @contextmanager 

251 def query( 

252 self, columns: Iterable[str | sqlalchemy.sql.expression.ColumnElement] | None = None, /, **kwargs: Any 

253 ) -> Iterator[sqlalchemy.engine.CursorResult]: 

254 """Run a SELECT query against obscore table and return result rows. 

255 

256 Parameters 

257 ---------- 

258 columns : `~collections.abc.Iterable` [`str`] 

259 Columns to return from query. It is a sequence which can include 

260 column names or any other column elements (e.g. 

261 `sqlalchemy.sql.functions.count` function). 

262 **kwargs 

263 Restriction on values of individual obscore columns. Key is the 

264 column name, value is the required value of the column. Multiple 

265 restrictions are ANDed together. 

266 

267 Returns 

268 ------- 

269 result_context : `sqlalchemy.engine.CursorResult` 

270 Context manager that returns the query result object when entered. 

271 These results are invalidated when the context is exited. 

272 """ 

273 raise NotImplementedError()