Coverage for python/lsst/daf/butler/registry/interfaces/_obscore.py: 98%

28 statements  

« prev     ^ index     » next       coverage.py v7.3.1, created at 2023-10-02 08:00 +0000

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This software is dual licensed under the GNU General Public License and also 

10# under a 3-clause BSD license. Recipients may choose which of these licenses 

11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, 

12# respectively. If you choose the GPL option then the following text applies 

13# (but note that there is still no warranty even if you opt for BSD instead): 

14# 

15# This program is free software: you can redistribute it and/or modify 

16# it under the terms of the GNU General Public License as published by 

17# the Free Software Foundation, either version 3 of the License, or 

18# (at your option) any later version. 

19# 

20# This program is distributed in the hope that it will be useful, 

21# but WITHOUT ANY WARRANTY; without even the implied warranty of 

22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

23# GNU General Public License for more details. 

24# 

25# You should have received a copy of the GNU General Public License 

26# along with this program. If not, see <http://www.gnu.org/licenses/>. 

27 

28"""Interfaces for classes that manage obscore table(s) in a `Registry`. 

29""" 

30 

31from __future__ import annotations 

32 

33__all__ = ["ObsCoreTableManager"] 

34 

35from abc import abstractmethod 

36from collections.abc import Iterable, Iterator, Mapping 

37from contextlib import contextmanager 

38from typing import TYPE_CHECKING, Any 

39 

40import sqlalchemy 

41 

42from ._versioning import VersionedExtension, VersionTuple 

43 

44if TYPE_CHECKING: 

45 from lsst.sphgeom import Region 

46 

47 from ...core import DatasetRef, DimensionUniverse 

48 from ..queries import SqlQueryContext 

49 from ._collections import CollectionRecord 

50 from ._database import Database, StaticTablesContext 

51 from ._datasets import DatasetRecordStorageManager 

52 from ._dimensions import DimensionRecordStorageManager 

53 

54 

55class ObsCoreTableManager(VersionedExtension): 

56 """An interface for populating ObsCore tables(s).""" 

57 

58 def __init__(self, *, registry_schema_version: VersionTuple | None = None): 

59 super().__init__(registry_schema_version=registry_schema_version) 

60 

61 @classmethod 

62 @abstractmethod 

63 def initialize( 

64 cls, 

65 db: Database, 

66 context: StaticTablesContext, 

67 *, 

68 universe: DimensionUniverse, 

69 config: Mapping, 

70 datasets: type[DatasetRecordStorageManager], 

71 dimensions: DimensionRecordStorageManager, 

72 registry_schema_version: VersionTuple | None = None, 

73 ) -> ObsCoreTableManager: 

74 """Construct an instance of the manager. 

75 

76 Parameters 

77 ---------- 

78 db : `Database` 

79 Interface to the underlying database engine and namespace. 

80 context : `StaticTablesContext` 

81 Context object obtained from `Database.declareStaticTables`; used 

82 to declare any tables that should always be present in a layer 

83 implemented with this manager. 

84 universe : `DimensionUniverse` 

85 All dimensions known to the registry. 

86 config : `dict` [ `str`, `Any` ] 

87 Configuration of the obscore manager. 

88 datasets : `type` 

89 Type of dataset manager. 

90 dimensions: `DimensionRecordStorageManager` 

91 Manager for Registry dimensions. 

92 registry_schema_version : `VersionTuple` or `None` 

93 Schema version of this extension as defined in registry. 

94 

95 Returns 

96 ------- 

97 manager : `ObsCoreTableManager` 

98 An instance of a concrete `ObsCoreTableManager` subclass. 

99 """ 

100 raise NotImplementedError() 

101 

102 @abstractmethod 

103 def config_json(self) -> str: 

104 """Dump configuration in JSON format. 

105 

106 Returns 

107 ------- 

108 json : `str` 

109 Configuration serialized in JSON format. 

110 """ 

111 raise NotImplementedError() 

112 

113 @abstractmethod 

114 def add_datasets(self, refs: Iterable[DatasetRef], context: SqlQueryContext) -> int: 

115 """Possibly add datasets to the obscore table. 

116 

117 This method should be called when new datasets are added to a RUN 

118 collection. 

119 

120 Parameters 

121 ---------- 

122 refs : `iterable` [ `DatasetRef` ] 

123 Dataset refs to add. Dataset refs have to be completely expanded. 

124 If a record with the same dataset ID is already in obscore table, 

125 the dataset is ignored. 

126 context : `SqlQueryContext` 

127 Context used to execute queries for additional dimension metadata. 

128 

129 Returns 

130 ------- 

131 count : `int` 

132 Actual number of records inserted into obscore table. 

133 

134 Notes 

135 ----- 

136 Dataset data types and collection names are checked against configured 

137 list of collections and dataset types, non-matching datasets are 

138 ignored and not added to the obscore table. 

139 

140 When configuration parameter ``collection_type`` is not "RUN", this 

141 method should return immediately. 

142 

143 Note that there is no matching method to remove datasets from obscore 

144 table, we assume that removal happens via foreign key constraint to 

145 dataset table with "ON DELETE CASCADE" option. 

146 """ 

147 raise NotImplementedError() 

148 

149 @abstractmethod 

150 def associate( 

151 self, refs: Iterable[DatasetRef], collection: CollectionRecord, context: SqlQueryContext 

152 ) -> int: 

153 """Possibly add datasets to the obscore table. 

154 

155 This method should be called when existing datasets are associated with 

156 a TAGGED collection. 

157 

158 Parameters 

159 ---------- 

160 refs : `iterable` [ `DatasetRef` ] 

161 Dataset refs to add. Dataset refs have to be completely expanded. 

162 If a record with the same dataset ID is already in obscore table, 

163 the dataset is ignored. 

164 collection : `CollectionRecord` 

165 Collection record for a TAGGED collection. 

166 context : `SqlQueryContext` 

167 Context used to execute queries for additional dimension metadata. 

168 

169 Returns 

170 ------- 

171 count : `int` 

172 Actual number of records inserted into obscore table. 

173 

174 Notes 

175 ----- 

176 Dataset data types and collection names are checked against configured 

177 list of collections and dataset types, non-matching datasets are 

178 ignored and not added to the obscore table. 

179 

180 When configuration parameter ``collection_type`` is not "TAGGED", this 

181 method should return immediately. 

182 """ 

183 raise NotImplementedError() 

184 

185 @abstractmethod 

186 def disassociate(self, refs: Iterable[DatasetRef], collection: CollectionRecord) -> int: 

187 """Possibly remove datasets from the obscore table. 

188 

189 This method should be called when datasets are disassociated from a 

190 TAGGED collection. 

191 

192 Parameters 

193 ---------- 

194 refs : `iterable` [ `DatasetRef` ] 

195 Dataset refs to remove. Dataset refs have to be resolved. 

196 collection : `CollectionRecord` 

197 Collection record for a TAGGED collection. 

198 

199 Returns 

200 ------- 

201 count : `int` 

202 Actual number of records removed from obscore table. 

203 

204 Notes 

205 ----- 

206 Dataset data types and collection names are checked against configured 

207 list of collections and dataset types, non-matching datasets are 

208 ignored and not added to the obscore table. 

209 

210 When configuration parameter ``collection_type`` is not "TAGGED", this 

211 method should return immediately. 

212 """ 

213 raise NotImplementedError() 

214 

215 @abstractmethod 

216 def update_exposure_regions(self, instrument: str, region_data: Iterable[tuple[int, int, Region]]) -> int: 

217 """Update existing exposure records with spatial region data. 

218 

219 Parameters 

220 ---------- 

221 instrument : `str` 

222 Instrument name. 

223 region_data : `~collections.abc.Iterable` [`tuple` [`int`, `int`, \ 

224 `~lsst.sphgeom.Region` ]] 

225 Sequence of tuples, each tuple contains three values - exposure ID, 

226 detector ID, and corresponding region. 

227 

228 Returns 

229 ------- 

230 count : `int` 

231 Actual number of records updated. 

232 

233 Notes 

234 ----- 

235 This method is needed to update obscore records for raw exposures which 

236 are ingested before their corresponding visits are defined. Exposure 

237 records added when visit is already defined will get their regions 

238 from their matching visits automatically. 

239 """ 

240 raise NotImplementedError() 

241 

242 @abstractmethod 

243 @contextmanager 

244 def query( 

245 self, columns: Iterable[str | sqlalchemy.sql.expression.ColumnElement] | None = None, /, **kwargs: Any 

246 ) -> Iterator[sqlalchemy.engine.CursorResult]: 

247 """Run a SELECT query against obscore table and return result rows. 

248 

249 Parameters 

250 ---------- 

251 columns : `~collections.abc.Iterable` [`str`] 

252 Columns to return from query. It is a sequence which can include 

253 column names or any other column elements (e.g. 

254 `sqlalchemy.sql.functions.count` function). 

255 **kwargs 

256 Restriction on values of individual obscore columns. Key is the 

257 column name, value is the required value of the column. Multiple 

258 restrictions are ANDed together. 

259 

260 Returns 

261 ------- 

262 result_context : `sqlalchemy.engine.CursorResult` 

263 Context manager that returns the query result object when entered. 

264 These results are invalidated when the context is exited. 

265 """ 

266 raise NotImplementedError()